def check_pending_upload(session, pu, _test_shim=lambda: None):
# we can check the upload any time between the expiration of the URL
# (after which the user can't make any more changes, but the upload
# may yet be incomplete) and 1 day afterward (ample time for the upload
# to complete)
sha512 = pu.file.sha512
size = pu.file.size
log = logger.bind(tooltool_sha512=sha512)
if time.now() < pu.expires:
# URL is not expired yet
return
elif time.now() > pu.expires + timedelta(days=1):
# Upload will probably never complete
log.info(
"Deleting abandoned pending upload for {}".format(sha512))
session.delete(pu)
return
# connect and see if the file exists..
s3 = current_app.aws.connect_to('s3', pu.region)
cfg = current_app.config.get('TOOLTOOL_REGIONS')
if not cfg or pu.region not in cfg:
log.warning("Pending upload for {} was to an un-configured "
"region".format(sha512))
session.delete(pu)
return
bucket = s3.get_bucket(cfg[pu.region], validate=False)
key = bucket.get_key(util.keyname(sha512))
if not key:
# not uploaded yet
return
# commit the session before verifying the file instance, since the
# DB connection may otherwise go away while we're distracted.
session.commit()
_test_shim()
if not verify_file_instance(sha512, size, key):
log.warning(
"Upload of {} was invalid; deleting key".format(sha512))
key.delete()
session.delete(pu)
session.commit()
return
log.info("Upload of {} considered valid".format(sha512))
# add a file instance, but it's OK if it already exists
try:
tables.FileInstance(file=pu.file, region=pu.region)
session.commit()
except sa.exc.IntegrityError:
session.rollback()
# and delete the pending upload
session.delete(pu)
session.commit()
def add_file_row(size, sha512, instances=[]):
session = current_app.db.session('relengapi')
file_row = tables.File(size=size, visibility='public', sha512=sha512)
session.add(file_row)
for region in instances:
session.add(tables.FileInstance(file=file_row, region=region))
session.commit()
return file_row
def add_file_to_db(app, content, regions=['us-east-1'],
pending_regions=[], visibility='public'):
with app.app_context():
session = app.db.session('relengapi')
file_row = tables.File(size=len(content),
visibility=visibility,
sha512=hashlib.sha512(content).hexdigest())
session.add(file_row)
session.commit()
for region in regions:
session.add(tables.FileInstance(
file_id=file_row.id, region=region))
for region in pending_regions:
session.add(tables.PendingUpload(
file=file_row, region=region,
expires=relengapi_time.now() + datetime.timedelta(seconds=60)))
session.commit()
return file_row
def replicate_file(session, file, _test_shim=lambda: None):
log = logger.bind(tooltool_sha512=file.sha512, mozdef=True)
config = current_app.config['TOOLTOOL_REGIONS']
regions = set(config)
file_regions = set([i.region for i in file.instances])
# only use configured source regions; if a region is removed
# from the configuration, we can't copy from it.
source_regions = file_regions & regions
if not source_regions:
# this should only happen when the only region containing a
# file is removed from the configuration
log.warning("no source regions for {}".format(file.sha512))
return
source_region = source_regions.pop()
source_bucket = config[source_region]
target_regions = regions - file_regions
log.info("replicating {} from {} to {}".format(file.sha512, source_region,
', '.join(target_regions)))
key_name = util.keyname(file.sha512)
for target_region in target_regions:
target_bucket = config[target_region]
conn = current_app.aws.connect_to('s3', target_region)
bucket = conn.get_bucket(target_bucket)
# commit the session before replicating, since the DB connection may
# otherwise go away while we're distracted.
session.commit()
_test_shim()
bucket.copy_key(new_key_name=key_name,
src_key_name=key_name,
src_bucket_name=source_bucket,
storage_class='STANDARD',
preserve_acl=False)
try:
session.add(tables.FileInstance(file=file, region=target_region))
session.commit()
except sa.exc.IntegrityError:
session.rollback()
def test_shim():
session = app.db.session('relengapi')
session.add(tables.FileInstance(file=file, region='us-west-2'))
session.commit()
def test_shim():
session.add(tables.FileInstance(file=file_row, region='us-west-2'))
session.commit()
def test_shim():
session = app.db.session(tables.DB_DECLARATIVE_BASE)
session.add(tables.FileInstance(file=file, region='us-west-2'))
session.commit()