def copy_files(processes, bottom_frequency, frequency_range):
# Create the directory
if not exists(DIRECTORY):
os.makedirs(DIRECTORY)
# Scan the bucket
s3_helper = S3Helper()
bucket = s3_helper.get_bucket(CHILES_BUCKET_NAME)
LOGGER.info('Scanning bucket: {0}/CLEAN'.format(bucket))
# Create the queue
queue = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(queue)
consumer.start()
for key in bucket.list(prefix='CLEAN/'):
LOGGER.info('Checking {0}'.format(key.key))
# Ignore the key
if key.key.endswith('.image.tar.gz') or key.key.endswith('.image.tar'):
# Do we need this file?
basename_key = basename(key.key)
if in_frequency_range(basename_key, bottom_frequency, frequency_range):
# Queue the copy of the file
temp_file = os.path.join(DIRECTORY, basename_key)
queue.put(Task(key, temp_file, DIRECTORY))
# Add a poison pill to shut things down
for x in range(processes):
queue.put(None)
# Wait for the queue to terminate
queue.join()
def copy_files(frequency_id, processes, days):
s3_helper = S3Helper()
bucket = s3_helper.get_bucket(CHILES_BUCKET_NAME)
LOGGER.info('Scanning bucket: {0}, frequency_id: {1}'.format(bucket, frequency_id))
# Create the queue
queue = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(queue)
consumer.start()
for key in bucket.list(prefix='CVEL/{0}'.format(frequency_id)):
LOGGER.info('Checking {0}'.format(key.key))
# Ignore the key
if key.key.endswith('/data.tar.gz') or key.key.endswith('/data.tar'):
elements = key.key.split('/')
if elements[2] in days:
directory = '/mnt/output/Chiles/split_vis/{0}/'.format(elements[2])
# Queue the copy of the file
temp_file = os.path.join(directory, 'data.tar.gz' if key.key.endswith('/data.tar.gz') else 'data.tar')
queue.put(Task(key, temp_file, directory, frequency_id))
# Add a poison pill to shut things down
for x in range(processes):
queue.put(None)
# Wait for the queue to terminate
queue.join()
def copy_files(frequency_id, processes, days):
s3_helper = S3Helper()
bucket = s3_helper.get_bucket(CHILES_BUCKET_NAME)
LOGGER.info('Scanning bucket: {0}, frequency_id: {1}'.format(
bucket, frequency_id))
# Create the queue
queue = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(queue)
consumer.start()
for key in bucket.list(prefix='CVEL/{0}'.format(frequency_id)):
LOGGER.info('Checking {0}'.format(key.key))
# Ignore the key
if key.key.endswith('/data.tar.gz') or key.key.endswith('/data.tar'):
elements = key.key.split('/')
if elements[2] in days:
directory = '/mnt/output/Chiles/split_vis/{0}/'.format(
elements[2])
# Queue the copy of the file
temp_file = os.path.join(
directory, 'data.tar.gz'
if key.key.endswith('/data.tar.gz') else 'data.tar')
queue.put(Task(key, temp_file, directory, frequency_id))
# Add a poison pill to shut things down
for x in range(processes):
queue.put(None)
# Wait for the queue to terminate
queue.join()
def copy_files(args):
# Create the queue
queue = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(PROCESSES):
consumer = Consumer(queue)
consumer.start()
# Look in the output directory
for root, dir_names, filenames in os.walk(args.product_dir):
LOGGER.debug('root: {0}, dir_names: {1}, filenames: {2}'.format(
root, dir_names, filenames))
for match in fnmatch.filter(dir_names,
'13B-266*calibrated_deepfield.ms'):
result_dir = join(root, match)
LOGGER.info('Queuing result_dir: {0}'.format(result_dir))
queue.put(
CopyTask(args.bucket, match, result_dir,
args.aws_access_key_id, args.aws_secret_access_key))
# Add a poison pill to shut things down
for x in range(PROCESSES):
queue.put(None)
# Wait for the queue to terminate
queue.join()
def copy_files(s3_tag, processes):
# Create the queue
queue = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(queue)
consumer.start()
# Look in the output directory
today = datetime.date.today()
for root, dir_names, filenames in os.walk(CHILES_LOGS):
for match in fnmatch.filter(filenames, '*.log'):
LOGGER.info('Looking at: {0}'.format(join(root, match)))
queue.put(CopyTask(join(root, match), '{0}/{1}{2:02d}{3:02d}/{4}'.format(s3_tag, today.year, today.month, today.day, match)))
for root, dir_names, filenames in os.walk(BENCHMARKING_LOGS):
for match in fnmatch.filter(filenames, '*.csv'):
LOGGER.info('Looking at: {0}'.format(join(root, match)))
queue.put(CopyTask(join(root, match), '{0}/{1}{2:02d}{3:02d}/{4}'.format(s3_tag, today.year, today.month, today.day, match)))
for match in fnmatch.filter(filenames, '*.log'):
LOGGER.info('Looking at: {0}'.format(join(root, match)))
queue.put(CopyTask(join(root, match), '{0}/{1}{2:02d}{3:02d}/{4}'.format(s3_tag, today.year, today.month, today.day, match)))
queue.put(CopyTask('/var/log/chiles-output.log', '{0}/{1}{2:02d}{3:02d}/chiles-output.log'.format(s3_tag, today.year, today.month, today.day)))
# Add a poison pill to shut things down
for x in range(processes):
queue.put(None)
# Wait for the queue to terminate
queue.join()
def copy_files(args):
# Create the queue
queue = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(PROCESSES):
consumer = Consumer(queue)
consumer.start()
# Look in the output directory
for root, dir_names, filenames in os.walk(args.product_dir):
LOGGER.debug('root: {0}, dir_names: {1}, filenames: {2}'.format(root, dir_names, filenames))
for match in fnmatch.filter(dir_names, '13B-266*calibrated_deepfield.ms'):
result_dir = join(root, match)
LOGGER.info('Queuing result_dir: {0}'.format(result_dir))
queue.put(
CopyTask(
args.bucket,
match,
result_dir,
args.aws_access_key_id,
args.aws_secret_access_key
)
)
# Add a poison pill to shut things down
for x in range(PROCESSES):
queue.put(None)
# Wait for the queue to terminate
queue.join()
def start_servers(processes, ami_id, user_data, setup_disks, instance_type,
obs_ids, created_by, name, instance_details, spot_price,
frequency_channels, force):
cvel_data = get_cvel()
# Create the queue
tasks = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(tasks)
consumer.start()
counter = 1
for obs_id in obs_ids:
snapshot_id = OBS_IDS.get(obs_id)
if snapshot_id is None:
LOGGER.warning(
'The obs-id: {0} does not exist in the settings file')
else:
obs_id_dashes = obs_id.replace('_', '-')
for frequency_groups in get_frequency_groups(
frequency_channels, obs_id_dashes, cvel_data, force):
tasks.put(
Task(ami_id, user_data, setup_disks, instance_type, obs_id,
snapshot_id, created_by, name, spot_price,
instance_details, frequency_groups, counter))
counter += 1
# Add a poison pill to shut things down
for x in range(processes):
tasks.put(None)
# Wait for the queue to terminate
tasks.join()
def start_servers(
processes,
ami_id,
user_data,
setup_disks,
instance_type,
obs_ids,
created_by,
name,
instance_details,
spot_price,
frequency_channels,
force):
cvel_data = get_cvel()
# Create the queue
tasks = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(tasks)
consumer.start()
counter = 1
for obs_id in obs_ids:
snapshot_id = OBS_IDS.get(obs_id)
if snapshot_id is None:
LOGGER.warning('The obs-id: {0} does not exist in the settings file')
else:
obs_id_dashes = obs_id.replace('_', '-')
for frequency_groups in get_frequency_groups(frequency_channels, obs_id_dashes, cvel_data, force):
tasks.put(
Task(
ami_id,
user_data,
setup_disks,
instance_type,
obs_id,
snapshot_id,
created_by,
name,
spot_price,
instance_details,
frequency_groups,
counter
)
)
counter += 1
# Add a poison pill to shut things down
for x in range(processes):
tasks.put(None)
# Wait for the queue to terminate
tasks.join()
def copy_files(s3_tag, processes):
# Create the queue
queue = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(queue)
consumer.start()
# Look in the output directory
today = datetime.date.today()
for root, dir_names, filenames in os.walk(CHILES_LOGS):
for match in fnmatch.filter(filenames, '*.log'):
LOGGER.info('Looking at: {0}'.format(join(root, match)))
queue.put(
CopyTask(
join(root, match),
'{0}/{1}{2:02d}{3:02d}/{4}'.format(s3_tag, today.year,
today.month, today.day,
match)))
for root, dir_names, filenames in os.walk(BENCHMARKING_LOGS):
for match in fnmatch.filter(filenames, '*.csv'):
LOGGER.info('Looking at: {0}'.format(join(root, match)))
queue.put(
CopyTask(
join(root, match),
'{0}/{1}{2:02d}{3:02d}/{4}'.format(s3_tag, today.year,
today.month, today.day,
match)))
for match in fnmatch.filter(filenames, '*.log'):
LOGGER.info('Looking at: {0}'.format(join(root, match)))
queue.put(
CopyTask(
join(root, match),
'{0}/{1}{2:02d}{3:02d}/{4}'.format(s3_tag, today.year,
today.month, today.day,
match)))
queue.put(
CopyTask(
'/var/log/chiles-output.log',
'{0}/{1}{2:02d}{3:02d}/chiles-output.log'.format(
s3_tag, today.year, today.month, today.day)))
# Add a poison pill to shut things down
for x in range(processes):
queue.put(None)
# Wait for the queue to terminate
queue.join()
def start_servers(processes, ami_id, user_data, setup_disks, instance_type,
frequency_ids, created_by, name, instance_details,
spot_price):
# Create the queue
tasks = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(tasks)
consumer.start()
for frequency_id in frequency_ids:
tasks.put(
Task(ami_id, user_data, setup_disks, instance_type, frequency_id,
created_by, name, spot_price, instance_details))
# Add a poison pill to shut things down
for x in range(processes):
tasks.put(None)
# Wait for the queue to terminate
tasks.join()
def start_servers(
processes,
ami_id,
user_data,
setup_disks,
instance_type,
frequency_ids,
created_by,
name,
instance_details,
spot_price):
# Create the queue
tasks = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(tasks)
consumer.start()
for frequency_id in frequency_ids:
tasks.put(
Task(
ami_id,
user_data,
setup_disks,
instance_type,
frequency_id,
created_by,
name,
spot_price,
instance_details))
# Add a poison pill to shut things down
for x in range(processes):
tasks.put(None)
# Wait for the queue to terminate
tasks.join()
def copy_files(processes, bottom_frequency, frequency_range):
# Create the directory
if not exists(DIRECTORY):
os.makedirs(DIRECTORY)
# Scan the bucket
s3_helper = S3Helper()
bucket = s3_helper.get_bucket(CHILES_BUCKET_NAME)
LOGGER.info('Scanning bucket: {0}/CLEAN'.format(bucket))
# Create the queue
queue = multiprocessing.JoinableQueue()
# Start the consumers
for x in range(processes):
consumer = Consumer(queue)
consumer.start()
for key in bucket.list(prefix='CLEAN/'):
LOGGER.info('Checking {0}'.format(key.key))
# Ignore the key
if key.key.endswith('.image.tar.gz') or key.key.endswith('.image.tar'):
# Do we need this file?
basename_key = basename(key.key)
if in_frequency_range(basename_key, bottom_frequency,
frequency_range):
# Queue the copy of the file
temp_file = os.path.join(DIRECTORY, basename_key)
queue.put(Task(key, temp_file, DIRECTORY))
# Add a poison pill to shut things down
for x in range(processes):
queue.put(None)
# Wait for the queue to terminate
queue.join()
