def copy_hdf5_files(args):
s3_helper = S3Helper()
bucket_aws_pogs = s3_helper.get_bucket(BUCKET_NAME)
bucket_pogs = s3_helper.get_bucket(BUCKET_NAME_OLD_POGS)
engine = create_engine(DB_LOGIN)
connection = engine.connect()
subquery = select([STEP_DONE.c.galaxy_id]).distinct()
for galaxy in connection.execute(select([GALAXY]).where(~GALAXY.c.galaxy_id.in_(subquery)).order_by(GALAXY.c.galaxy_id)):
if args.verbose:
LOG.info('Checking galaxy_id: {0}, name: {1}'.format(galaxy[GALAXY.c.galaxy_id], galaxy[GALAXY.c.name]))
# noinspection PyBroadException
try:
block_dir = get_block_dir(galaxy[GALAXY.c.galaxy_id] / 1000)
name_aws_pogs = get_galaxy_file_name(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy[GALAXY.c.galaxy_id])
key_aws_pogs = Key(bucket_aws_pogs)
key_aws_pogs.key = '{0}/{1}.hdf5'.format(block_dir, name_aws_pogs)
if not key_aws_pogs.exists():
# Does it exist in POGS?
name_pogs = get_galaxy_file_name_pogs(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy[GALAXY.c.galaxy_id])
key_pogs = Key(bucket_pogs)
key_pogs.key = '{0}/{0}.hdf5'.format(name_pogs)
if key_pogs.exists():
check_and_copy_or_restore(galaxy[GALAXY.c.name], galaxy[GALAXY.c.run_id], galaxy[GALAXY.c.galaxy_id], args.verbose)
else:
add_step_done_id(connection, galaxy[GALAXY.c.galaxy_id], STEP_DONE_ID_NO_HDF5_FILE)
except BaseException:
LOG.exception('error during fetch, quitting')
break
if args.verbose:
LOG.info('Done')
def check_and_copy_or_restore(galaxy_name, run_id, galaxy_id, verbose):
# noinspection PyBroadException
try:
s3_helper = S3Helper(PROFILE_NAME)
bucket_aws_pogs = s3_helper.get_bucket(BUCKET_NAME)
bucket_pogs = s3_helper.get_bucket(BUCKET_NAME_OLD_POGS)
name_aws_pogs = get_galaxy_file_name(galaxy_name, run_id, galaxy_id)
name_pogs = get_galaxy_file_name_pogs(galaxy_name, run_id, galaxy_id)
# Get the new key from the old key
block_dir = get_block_dir(galaxy_id / 1000)
key_aws_pogs = Key(bucket_aws_pogs)
key_aws_pogs.key = '{0}/{1}.hdf5'.format(block_dir, name_aws_pogs)
keyname_pogs = '{0}/{0}.hdf5'.format(name_pogs)
# Work around to silly bug in boto
key_pogs = bucket_pogs.get_key(keyname_pogs)
if key_pogs.storage_class == 'GLACIER' and key_pogs.expiry_date is None:
if key_pogs.ongoing_restore is None:
if verbose:
LOG.info('Retrieving from glacier: {0}'.format(key_pogs.key))
key_pogs.restore(days=10)
else:
if verbose:
LOG.info('The file is being retrieved from glacier: {0}'.format(key_pogs.key))
else:
if verbose:
LOG.info('Copy: {0} to {1}'.format(key_pogs.key, key_aws_pogs.key))
copy_files_from_bucket_to_bucket(s3_helper, key_pogs.key, key_aws_pogs.key, galaxy_name, run_id, galaxy_id)
except BaseException:
LOG.exception('Error during copy')