def test_reply_keypair_creation_and_expiration_dates(source_app): with source_app.app_context(): codename = source_app.crypto_util.genrandomid() filesystem_id = source_app.crypto_util.hash_codename(codename) journalist_filename = source_app.crypto_util.display_id() source = models.Source(filesystem_id, journalist_filename) db.session.add(source) db.session.commit() source_app.crypto_util.genkeypair(source.filesystem_id, codename) # crypto_util.get_fingerprint only returns the fingerprint of the key. We need # the full output of gpg.list_keys() to check the creation and # expire dates. # # TODO: it might be generally useful to refactor crypto_util.get_fingerprint so # it always returns the entire key dictionary instead of just the # fingerprint (which is always easily extracted from the entire key # dictionary). new_key_fingerprint = source_app.crypto_util.get_fingerprint( filesystem_id) new_key = [ key for key in source_app.crypto_util.gpg.list_keys() if new_key_fingerprint == key['fingerprint'] ][0] # All keys should share the same creation date to avoid leaking # information about when sources first created accounts. creation_date = parse_gpg_date_string(new_key['date']) assert (creation_date.date() == CryptoUtil.DEFAULT_KEY_CREATION_DATE) # Reply keypairs should not expire expire_date = new_key['expires'] assert expire_date == ''
def _parse_response(response, category=None): """Parse the response from the news api.""" response_dict = response if isinstance(response, dict) else response.json() if response_dict.get("status") != "ok": raise NewsApiError(response_dict.get("message")) if response_dict.get("articles"): return {article["url"]: models.Article.create_from_dict(article, category=category) for article in response_dict.get("articles", [])}.values() return [models.Source(source) for source in response_dict.get("sources", [])]
def create_source_user( db_session: Session, source_passphrase: "DicewarePassphrase", source_app_storage: "Storage", ) -> SourceUser: # Derive the source's info from their passphrase scrypt_manager = _SourceScryptManager.get_default() filesystem_id = scrypt_manager.derive_source_filesystem_id(source_passphrase) gpg_secret = scrypt_manager.derive_source_gpg_secret(source_passphrase) # Create a unique journalist designation for the source # TODO: Add unique=True to models.Source.journalist_designation to enforce uniqueness # as the logic below has a race condition (time we check VS time when we add to the DB) designation_generation_attempts = 0 valid_designation = None designation_generator = _DesignationGenerator.get_default() while designation_generation_attempts < 50: # Generate a designation designation_generation_attempts += 1 new_designation = designation_generator.generate_journalist_designation() # Check to see if it's already used by an existing source existing_source_with_same_designation = ( db_session.query(models.Source) .filter_by(journalist_designation=new_designation) .one_or_none() ) if not existing_source_with_same_designation: # The designation is not already used - good to go valid_designation = new_designation break if not valid_designation: # Could not generate a designation that is not already used raise SourceDesignationCollisionError() # Store the source in the DB source_db_record = models.Source( filesystem_id=filesystem_id, journalist_designation=valid_designation ) db_session.add(source_db_record) try: db_session.commit() except IntegrityError: db_session.rollback() raise SourcePassphraseCollisionError( "Passphrase already used by another Source (filesystem_id {})".format(filesystem_id) ) # Create the source's folder os.mkdir(source_app_storage.path(filesystem_id)) # All done return SourceUser(source_db_record, filesystem_id, gpg_secret)
def test_genkeypair(source_app): with source_app.app_context(): codename = source_app.crypto_util.genrandomid() filesystem_id = source_app.crypto_util.hash_codename(codename) journalist_filename = source_app.crypto_util.display_id() source = models.Source(filesystem_id, journalist_filename) db.session.add(source) db.session.commit() source_app.crypto_util.genkeypair(source.filesystem_id, codename) assert source_app.crypto_util.getkey(filesystem_id) is not None
def test_genkeypair(source_app): with source_app.app_context(): codename = PassphraseGenerator.get_default().generate_passphrase() filesystem_id = source_app.crypto_util.hash_codename(codename) journalist_filename = source_app.crypto_util.display_id() source = models.Source(filesystem_id, journalist_filename) db.session.add(source) db.session.commit() source_app.crypto_util.genkeypair(source.filesystem_id, codename) assert source_app.crypto_util.get_fingerprint( filesystem_id) is not None
def addSource(srcname, srcreliability): """ @TODO: add error checking and stricter code -- USE TO TEST / PROTOTYPE """ print("ADD SOURCE CALLED") try: print("in try") sobj = models.Source(name="azerty") print("object created") sobj.save() print("object saved") except: e = sys.exc_info()[0] tb = sys.exc_info()[2] print("DEBUG: IN ADD SOURCE EXCEPTION!! %s" % (e)) traceback.print_tb(tb) return
def populate_sources(self): logger.info('Populating sources.') session = self._db_client.get_session() for row in self._read_workbook('Sources'): r_source_id, r_jurisdiction_id, r_name, r_code, r_rank = row session.add( models.Source(sourceId=r_source_id, jurisdictionId=utils.format_string( r_jurisdiction_id, utils.NullBehaviour.NULL_TO_NULL), name=utils.format_string(r_name), code=utils.format_string(r_code), rank=utils.string_to_int(r_rank))) session.commit()
def init_source_without_keypair(): """Initialize a source: create their database record and the filesystem directory that stores their submissions & replies. Return a source object and their codename string. :returns: A 2-tuple. The first entry, the :class:`models.Source` initialized. The second, their codename string. """ # Create source identity and database record codename = current_app.crypto_util.genrandomid() filesystem_id = current_app.crypto_util.hash_codename(codename) journalist_filename = current_app.crypto_util.display_id() source = models.Source(filesystem_id, journalist_filename) db.session.add(source) db.session.commit() # Create the directory to store their submissions and replies os.mkdir(current_app.storage.path(source.filesystem_id)) return source, codename
def createSource(uid, payload): """ if id_source in payload, use that id, provided no record already exists, if not use new id returns the id """ source = None url = payload["url"] page = parsePage(url) if "id_source" in payload: id = payload["id_source"] source = M.Source.objects.get(pk=id) if source is not None and source.numpages!=0: assert False, "already a source w/ id=%s !!!" % (id,) return None else: source.submittedby_id = uid else: source = M.Source(submittedby_id=uid) source.title = page["path"][1:] id = source.save() # Django's default behavior is auto-commit return id
def main(): global src_count, number_of_sources, total_pkt_in_queue, sources validate() for i in range(number_of_sources): start_time = uniform(0., 10.) new_src = models.Source(i, start_time) sources.append(new_src) pkt_id, gen_time = new_src.generatePacket() # print "Source ", i, "at ", start_time, pkt_id, gen_time ev = models.Event(i, pkt_id, gen_time, 0) pushEvent(ev) for i in range(1000): global events keys = events.keys() keys.sort() key = keys[0] values = events[key] del events[key] values.sort() for evv in values: processEvent(evv) global delay for src in sources: packets = src.packets for pkt_id in packets.keys(): if packets[pkt_id].status < 2: continue pkt = packets[pkt_id] delay += pkt.switchLeavingTime - pkt.switchReachingTime print delay
import models from dispatcher import Dispatcher types = ['woonhuis', 'appartement'] funda = ScrapeModel( name='funda.nl', domain='http://funda.nl', num_sources=2, phases=[ Phase( sources=[ models.Source(url='http://funda.nl/koop/amsterdam/' + t + '/') for t in types ], templates=[ Template( name='house', selector='.search-result', db_type='MongoDB', db='funda', table='on_sale1', attrs=[ Attr(name='price', selector='.search-result-price', func='sel_text', kws={'numbers': True}), Attr(name='street', selector='.search-result-title', func='sel_text'), Attr(name='realtor',
def createSourceID(): o = M.Source() o.save() return o.id