def run_map_jobs_singlethreaded(json_files, json_db, args):
results = []
for i, j in enumerate(json_files):
logger.debug(j)
results.append(clonify_map(j, json_db, args))
progbar.progress_bar(i + 1, len(json_files))
return results
def copy_from_basemount(basemount_directory, destination_directory):
make_dir(os.path.abspath(destination_directory))
fastqs = []
for (path, dirs, files) in os.walk(basemount_directory):
for f in files:
if f.endswith('.fastq.gz'):
fastqs.append(os.path.join(path, f))
print('')
print('')
print('========================================')
print('Copying files from BaseMount')
print('========================================')
print('')
print('Found {0} FASTQ files.'.format(len(fastqs)))
print('')
print('Copying to {}:'.format(destination_directory))
start = datetime.now()
progress_bar(0,
len(fastqs),
start_time=start,
completion_string='ITS DONE! good job')
for i, fastq in enumerate(fastqs):
dest = os.path.join(destination_directory, os.path.basename(fastq))
copyfile(fastq, dest)
progress_bar(i + 1, len(fastqs), start_time=start)
print('\n')
def wait_for_instance_state(ec2c, instance_ids, state):
print('')
print("Waiting for instance{} to be in a '{}' state".format(
's' if len(instance_ids) > 1 else '', state))
start = datetime.now()
response = ec2c.describe_instances(InstanceIds=instance_ids)
instances = [i for r in response['Reservations'] for i in r['Instances']]
states = [i['State']['Name'] for i in instances]
pending = [s != state for s in states]
while any(pending):
# total = len(instances)
# finished = total - sum(pending)
finished = len(instances) - sum(pending)
progbar.progress_bar(finished, len(instances), start)
time.sleep(15)
response = ec2c.describe_instances(InstanceIds=instance_ids)
instances = [
i for r in response['Reservations'] for i in r['Instances']
]
states = [i['State']['Name'] for i in instances]
pending = [s != state for s in states]
progbar.progress_bar(len(instances),
len(instances),
start,
completion_string='')
def monitor_mp_jobs(results):
finished = 0
jobs = len(results)
while finished < jobs:
time.sleep(1)
ready = [ar for ar in results if ar.ready()]
finished = len(ready)
progbar.progress_bar(finished, jobs)
sys.stdout.write('\n')
def monitor_results(results):
finished = 0
jobs = len(results)
while finished < jobs:
time.sleep(1)
ready = [ar for ar in results if ar.ready()]
finished = len(ready)
progbar.progress_bar(finished, jobs)
progbar.progress_bar(finished, jobs)
def multiprocess_mongoimport(jsons, db, coll, args):
progbar.progress_bar(0, len(jsons))
async_results = []
p = mp.Pool()
for j in jsons:
async_results.append(p.apply_async(do_mongoimport, args=(j, args.ip, args.port, db, coll, args.user, args.password)))
monitor_results(async_results)
remove_temp_files(args)
print('')
def monitor_celery_jobs(results):
finished = 0
jobs = len(results)
while finished < jobs:
time.sleep(1)
succeeded = [ar for ar in results if ar.successful()]
failed = [ar for ar in results if ar.failed()]
finished = len(succeeded) + len(failed)
progbar.progress_bar(finished, jobs)
sys.stdout.write('\n')
return succeeded, failed