def copy_files(date, vis_file):
s3_helper = S3Helper()
# Look in the output directory
for root, dir_names, filenames in os.walk(CHILES_CVEL_OUTPUT):
LOGGER.info('root: {0}, dir_names: {1}, filenames: {2}'.format(
root, dir_names, filenames))
for match in fnmatch.filter(dir_names, vis_file):
result_dir = join(root, match)
LOGGER.info('Working on: {0}'.format(result_dir))
if can_be_multipart_tar(result_dir):
LOGGER.info('Using add_tar_to_bucket_multipart')
s3_helper.add_tar_to_bucket_multipart(
CHILES_BUCKET_NAME,
'CVEL/{0}/{1}/data.tar'.format(vis_file, date), result_dir)
else:
LOGGER.info('Using make_tarfile, then adding file to bucket')
output_tar_filename = join(root, match + '.tar')
make_tarfile(output_tar_filename, result_dir)
s3_helper.add_file_to_bucket(
CHILES_BUCKET_NAME,
'CVEL/{0}/{1}/data.tar'.format(vis_file,
date), output_tar_filename)
# Clean up
os.remove(output_tar_filename)
shutil.rmtree(result_dir, ignore_errors=True)
def copy_files(frequency_id):
s3_helper = S3Helper()
# Look in the output directory
LOGGER.info('directory_data: {0}'.format(CHILES_CLEAN_OUTPUT))
for dir_name in os.listdir(CHILES_CLEAN_OUTPUT):
LOGGER.info('dir_name: {0}'.format(dir_name))
result_dir = join(CHILES_CLEAN_OUTPUT, dir_name)
if isdir(result_dir) and dir_name.startswith('cube_') and dir_name.endswith('.image'):
LOGGER.info('dir_name: {0}'.format(dir_name))
output_tar_filename = join(CHILES_CLEAN_OUTPUT, dir_name + '.tar')
if can_be_multipart_tar(result_dir):
LOGGER.info('Using add_tar_to_bucket_multipart')
s3_helper.add_tar_to_bucket_multipart(
CHILES_BUCKET_NAME,
'/CLEAN/{0}/{1}'.format(frequency_id, basename(output_tar_filename)),
result_dir)
else:
LOGGER.info('Using make_tarfile, then adding file to bucket')
make_tarfile(output_tar_filename, result_dir)
s3_helper.add_file_to_bucket(
CHILES_BUCKET_NAME,
'CVEL/{0}/{1}/data.tar'.format(frequency_id, basename(output_tar_filename)),
output_tar_filename)
# Clean up
os.remove(output_tar_filename)
def copy_files(date, vis_file):
s3_helper = S3Helper()
# Look in the output directory
for root, dir_names, filenames in os.walk(CHILES_CVEL_OUTPUT):
LOGGER.info('root: {0}, dir_names: {1}, filenames: {2}'.format(root, dir_names, filenames))
for match in fnmatch.filter(dir_names, vis_file):
result_dir = join(root, match)
LOGGER.info('Working on: {0}'.format(result_dir))
if can_be_multipart_tar(result_dir):
LOGGER.info('Using add_tar_to_bucket_multipart')
s3_helper.add_tar_to_bucket_multipart(
CHILES_BUCKET_NAME,
'CVEL/{0}/{1}/data.tar'.format(vis_file, date),
result_dir)
else:
LOGGER.info('Using make_tarfile, then adding file to bucket')
output_tar_filename = join(root, match + '.tar')
make_tarfile(output_tar_filename, result_dir)
s3_helper.add_file_to_bucket(
CHILES_BUCKET_NAME,
'CVEL/{0}/{1}/data.tar'.format(vis_file, date),
output_tar_filename)
# Clean up
os.remove(output_tar_filename)
shutil.rmtree(result_dir, ignore_errors=True)
def copy_files(cube):
s3_helper = S3Helper()
# Look in the output directory
directory_to_save = join(CHILES_IMGCONCAT_OUTPUT, cube) + '.cube'
if isdir(directory_to_save):
LOGGER.info('dir_name: {0}'.format(directory_to_save))
output_tar_filename = directory_to_save + '.tar'
if can_be_multipart_tar(directory_to_save):
LOGGER.info('Using add_tar_to_bucket_multipart')
s3_helper.add_tar_to_bucket_multipart(
CHILES_BUCKET_NAME,
'IMGCONCAT/{0}'.format(basename(output_tar_filename)),
directory_to_save,
bufsize=20 * 1024 * 1024)
else:
LOGGER.info('Using make_tarfile, then adding file to bucket')
make_tarfile(output_tar_filename, directory_to_save)
s3_helper.add_file_to_bucket(
CHILES_BUCKET_NAME,
'IMGCONCAT/{0}'.format(basename(output_tar_filename)),
output_tar_filename)
# Clean up
os.remove(output_tar_filename)
def copy_files(cube):
s3_helper = S3Helper()
# Look in the output directory
directory_to_save = join(CHILES_IMGCONCAT_OUTPUT, cube) + '.cube'
if isdir(directory_to_save):
LOGGER.info('dir_name: {0}'.format(directory_to_save))
output_tar_filename = directory_to_save + '.tar'
if can_be_multipart_tar(directory_to_save):
LOGGER.info('Using add_tar_to_bucket_multipart')
s3_helper.add_tar_to_bucket_multipart(
CHILES_BUCKET_NAME,
'IMGCONCAT/{0}'.format(basename(output_tar_filename)),
directory_to_save,
bufsize=20 * 1024 * 1024)
else:
LOGGER.info('Using make_tarfile, then adding file to bucket')
make_tarfile(output_tar_filename, directory_to_save)
s3_helper.add_file_to_bucket(
CHILES_BUCKET_NAME,
'IMGCONCAT/{0}'.format(basename(output_tar_filename)),
output_tar_filename)
# Clean up
os.remove(output_tar_filename)
def copy_files(frequency_id):
s3_helper = S3Helper()
# Look in the output directory
LOGGER.info('directory_data: {0}'.format(CHILES_CLEAN_OUTPUT))
for dir_name in os.listdir(CHILES_CLEAN_OUTPUT):
LOGGER.info('dir_name: {0}'.format(dir_name))
result_dir = join(CHILES_CLEAN_OUTPUT, dir_name)
if isdir(result_dir) and dir_name.startswith(
'cube_') and dir_name.endswith('.image'):
LOGGER.info('dir_name: {0}'.format(dir_name))
output_tar_filename = join(CHILES_CLEAN_OUTPUT, dir_name + '.tar')
if can_be_multipart_tar(result_dir):
LOGGER.info('Using add_tar_to_bucket_multipart')
s3_helper.add_tar_to_bucket_multipart(
CHILES_BUCKET_NAME,
'/CLEAN/{0}/{1}'.format(frequency_id,
basename(output_tar_filename)),
result_dir)
else:
LOGGER.info('Using make_tarfile, then adding file to bucket')
make_tarfile(output_tar_filename, result_dir)
s3_helper.add_file_to_bucket(
CHILES_BUCKET_NAME, 'CVEL/{0}/{1}/data.tar'.format(
frequency_id, basename(output_tar_filename)),
output_tar_filename)
# Clean up
os.remove(output_tar_filename)