def test_reply_keypair_creation_and_expiration_dates(source_app):
with source_app.app_context():
codename = source_app.crypto_util.genrandomid()
filesystem_id = source_app.crypto_util.hash_codename(codename)
journalist_filename = source_app.crypto_util.display_id()
source = models.Source(filesystem_id, journalist_filename)
db.session.add(source)
db.session.commit()
source_app.crypto_util.genkeypair(source.filesystem_id, codename)
# crypto_util.get_fingerprint only returns the fingerprint of the key. We need
# the full output of gpg.list_keys() to check the creation and
# expire dates.
#
# TODO: it might be generally useful to refactor crypto_util.get_fingerprint so
# it always returns the entire key dictionary instead of just the
# fingerprint (which is always easily extracted from the entire key
# dictionary).
new_key_fingerprint = source_app.crypto_util.get_fingerprint(
filesystem_id)
new_key = [
key for key in source_app.crypto_util.gpg.list_keys()
if new_key_fingerprint == key['fingerprint']
][0]
# All keys should share the same creation date to avoid leaking
# information about when sources first created accounts.
creation_date = parse_gpg_date_string(new_key['date'])
assert (creation_date.date() == CryptoUtil.DEFAULT_KEY_CREATION_DATE)
# Reply keypairs should not expire
expire_date = new_key['expires']
assert expire_date == ''
def _parse_response(response, category=None):
"""Parse the response from the news api."""
response_dict = response if isinstance(response, dict) else response.json()
if response_dict.get("status") != "ok":
raise NewsApiError(response_dict.get("message"))
if response_dict.get("articles"):
return {article["url"]: models.Article.create_from_dict(article, category=category)
for article in response_dict.get("articles", [])}.values()
return [models.Source(source) for source in response_dict.get("sources", [])]
def create_source_user(
db_session: Session,
source_passphrase: "DicewarePassphrase",
source_app_storage: "Storage",
) -> SourceUser:
# Derive the source's info from their passphrase
scrypt_manager = _SourceScryptManager.get_default()
filesystem_id = scrypt_manager.derive_source_filesystem_id(source_passphrase)
gpg_secret = scrypt_manager.derive_source_gpg_secret(source_passphrase)
# Create a unique journalist designation for the source
# TODO: Add unique=True to models.Source.journalist_designation to enforce uniqueness
# as the logic below has a race condition (time we check VS time when we add to the DB)
designation_generation_attempts = 0
valid_designation = None
designation_generator = _DesignationGenerator.get_default()
while designation_generation_attempts < 50:
# Generate a designation
designation_generation_attempts += 1
new_designation = designation_generator.generate_journalist_designation()
# Check to see if it's already used by an existing source
existing_source_with_same_designation = (
db_session.query(models.Source)
.filter_by(journalist_designation=new_designation)
.one_or_none()
)
if not existing_source_with_same_designation:
# The designation is not already used - good to go
valid_designation = new_designation
break
if not valid_designation:
# Could not generate a designation that is not already used
raise SourceDesignationCollisionError()
# Store the source in the DB
source_db_record = models.Source(
filesystem_id=filesystem_id, journalist_designation=valid_designation
)
db_session.add(source_db_record)
try:
db_session.commit()
except IntegrityError:
db_session.rollback()
raise SourcePassphraseCollisionError(
"Passphrase already used by another Source (filesystem_id {})".format(filesystem_id)
)
# Create the source's folder
os.mkdir(source_app_storage.path(filesystem_id))
# All done
return SourceUser(source_db_record, filesystem_id, gpg_secret)
def test_genkeypair(source_app):
with source_app.app_context():
codename = source_app.crypto_util.genrandomid()
filesystem_id = source_app.crypto_util.hash_codename(codename)
journalist_filename = source_app.crypto_util.display_id()
source = models.Source(filesystem_id, journalist_filename)
db.session.add(source)
db.session.commit()
source_app.crypto_util.genkeypair(source.filesystem_id, codename)
assert source_app.crypto_util.getkey(filesystem_id) is not None
def test_genkeypair(source_app):
with source_app.app_context():
codename = PassphraseGenerator.get_default().generate_passphrase()
filesystem_id = source_app.crypto_util.hash_codename(codename)
journalist_filename = source_app.crypto_util.display_id()
source = models.Source(filesystem_id, journalist_filename)
db.session.add(source)
db.session.commit()
source_app.crypto_util.genkeypair(source.filesystem_id, codename)
assert source_app.crypto_util.get_fingerprint(
filesystem_id) is not None
def addSource(srcname, srcreliability):
"""
@TODO: add error checking and stricter code
-- USE TO TEST / PROTOTYPE
"""
print("ADD SOURCE CALLED")
try:
print("in try")
sobj = models.Source(name="azerty")
print("object created")
sobj.save()
print("object saved")
except:
e = sys.exc_info()[0]
tb = sys.exc_info()[2]
print("DEBUG: IN ADD SOURCE EXCEPTION!! %s" % (e))
traceback.print_tb(tb)
return
def populate_sources(self):
logger.info('Populating sources.')
session = self._db_client.get_session()
for row in self._read_workbook('Sources'):
r_source_id, r_jurisdiction_id, r_name, r_code, r_rank = row
session.add(
models.Source(sourceId=r_source_id,
jurisdictionId=utils.format_string(
r_jurisdiction_id,
utils.NullBehaviour.NULL_TO_NULL),
name=utils.format_string(r_name),
code=utils.format_string(r_code),
rank=utils.string_to_int(r_rank)))
session.commit()
def init_source_without_keypair():
"""Initialize a source: create their database record and the
filesystem directory that stores their submissions & replies.
Return a source object and their codename string.
:returns: A 2-tuple. The first entry, the :class:`models.Source`
initialized. The second, their codename string.
"""
# Create source identity and database record
codename = current_app.crypto_util.genrandomid()
filesystem_id = current_app.crypto_util.hash_codename(codename)
journalist_filename = current_app.crypto_util.display_id()
source = models.Source(filesystem_id, journalist_filename)
db.session.add(source)
db.session.commit()
# Create the directory to store their submissions and replies
os.mkdir(current_app.storage.path(source.filesystem_id))
return source, codename
def createSource(uid, payload):
"""
if id_source in payload, use that id, provided no record already exists, if not use new id
returns the id
"""
source = None
url = payload["url"]
page = parsePage(url)
if "id_source" in payload:
id = payload["id_source"]
source = M.Source.objects.get(pk=id)
if source is not None and source.numpages!=0:
assert False, "already a source w/ id=%s !!!" % (id,)
return None
else:
source.submittedby_id = uid
else:
source = M.Source(submittedby_id=uid)
source.title = page["path"][1:]
id = source.save() # Django's default behavior is auto-commit
return id
def main(): global src_count, number_of_sources, total_pkt_in_queue, sources validate() for i in range(number_of_sources): start_time = uniform(0., 10.) new_src = models.Source(i, start_time) sources.append(new_src) pkt_id, gen_time = new_src.generatePacket() # print "Source ", i, "at ", start_time, pkt_id, gen_time ev = models.Event(i, pkt_id, gen_time, 0) pushEvent(ev) for i in range(1000): global events keys = events.keys() keys.sort() key = keys[0] values = events[key] del events[key] values.sort() for evv in values: processEvent(evv) global delay for src in sources: packets = src.packets for pkt_id in packets.keys(): if packets[pkt_id].status < 2: continue pkt = packets[pkt_id] delay += pkt.switchLeavingTime - pkt.switchReachingTime print delay
import models
from dispatcher import Dispatcher
types = ['woonhuis', 'appartement']
funda = ScrapeModel(
name='funda.nl',
domain='http://funda.nl',
num_sources=2,
phases=[
Phase(
sources=[
models.Source(url='http://funda.nl/koop/amsterdam/' + t + '/')
for t in types
],
templates=[
Template(
name='house',
selector='.search-result',
db_type='MongoDB',
db='funda',
table='on_sale1',
attrs=[
Attr(name='price',
selector='.search-result-price',
func='sel_text',
kws={'numbers': True}),
Attr(name='street',
selector='.search-result-title',
func='sel_text'),
Attr(name='realtor',
def createSourceID():
o = M.Source()
o.save()
return o.id
