def delete_nodes(client, kind, count):
print('Deleting %d %s server node(s) to cluster...' % (count, kind))
prev_count = util.get_previous_count(client, kind)
if prev_count < count:
print('There are only %d %s server node(s)' % (prev_count, kind))
return
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' + kind).items
pods_to_delete = pods[:count]
deleted_pod_ips = []
for pod in pods_to_delete:
deleted_pod_ips.append(pod.status.pod_ip)
podname = pod.metadata.name
client.delete_namespaced_pod(name=podname,
namespace=util.NAMESPACE,
body=k8s.client.V1DeleteOptions())
client.delete_node(name=hostname, body=k8s.client.V1DeleteOptions())
util.run_process(['./modify_ig.sh', ntype,
str(prev_count - count)], 'kops')
# Notify routers about deleted key nodes
if kind == 'keynode':
deregister(client, deleted_pod_ips)
def build_kraken_bracken_db(subgroup_dict, kraken_file_path,
kraken_fasta_file_path, quiet):
for index, subgroups in subgroup_dict.items():
for subgroup in subgroups:
proc = run_process(
"kraken-build --db {0} --add-to-library {1}/{2}.fasta;\n".
format(kraken_file_path, kraken_fasta_file_path, subgroup[0]),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process("kraken-build --db {0} --build --threads 8;\n".format(
kraken_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"bracken-build -k 31 -d {0} -x /jc/bin/kraken/ -t 8;\n".format(
kraken_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
def make_gene_profile(df_sample_info, gene_profile_file_path, tmp_file_path,
quiet):
for sample in df_sample_info.index.tolist():
if not os.path.isfile("{0}/{1}.txt".format(gene_profile_file_path,
sample)):
proc = run_process("mmseqs createdb {0} {1}/{2} --dbtype 2".format(
df_sample_info.loc[sample, 'cds_file'], gene_profile_file_path,
sample),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs search {0}/{1} {0}/Lactobacillus_reuteri_specific {0}/{1}_aln {2} --min-seq-id 0.9 -c 0.9 --threads 8 --search-type 3"
.format(gene_profile_file_path, sample, tmp_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs createtsv {0}/{1} {0}/Lactobacillus_reuteri_specific {0}/{1}_aln {0}/{1}.txt --threads 8"
.format(gene_profile_file_path, sample),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
def debug():
config = updated_config('physics_settings/configd.sh')
rescaled_outfile = config['rescaled_outfiled']
print("Debugging:")
print(config['catalog_file']) # jedisim_out/out0/scaled_disk/trial1_catalog.txt
print(config['dislist_file']) # jedisim_out/out0/scaled_disk/trial1_dislist.txt
run_process("jeditransform", ['./executables/jeditransform',
config['catalog_file'],
config['dislist_file']
])
def lsst_monochromatic(config):
"""Convolve lsst chromatic with psf_mono.fits and get monochromatic image.
:Output: jedisim_out/out0/lsst_mono.fits
"""
# Add noise to monochromatic image and choose this as lsst_mono final output.
run_process("jedinoise", [
'./executables/jedinoise', config['90_rescaled_outfilem'],
config['exp_time'], config['noise_mean'], config["90_lsst_mono"]
])
def remove_node(ip, ntype):
client, _ = util.init_k8s()
pod = util.get_pod_from_ip(client, ip)
hostname = 'ip-%s.ec2.internal' % (ip.replace('.', '-'))
podname = pod.metadata.name
client.delete_namespaced_pod(name=podname, namespace=util.NAMESPACE,
body=k8s.client.V1DeleteOptions())
client.delete_node(name=hostname, body=k8s.client.V1DeleteOptions())
prev_count = util.get_previous_count(client, ntype)
util.run_process(['./modify_ig.sh', ntype, str(prev_count - 1)])
def jedimaster():
# Using run_process
run_process("Replace output dirs.", ['python', "a1_create_odirs.py"])
run_process("Create 3 catalogs.", ['python', "a2_create_3catalogs.py"])
run_process("Run the simulation for normal case.",
['python', "a3_jedisimulate.py"])
run_process("Run the simulation for rotated case.",
['python', "a4_jedisimulate90.py"])
def clang(self, req, filename, call_id):
cmd = None
sys.stdout.flush()
if util.is_unit(filename):
cmd = self.clang_base_cmd() + [filename]
util.send_sexp(
req, [key(":clear-file-notes"), [filename]])
elif util.is_header(filename):
self.__nuke_all_precompiled_headers()
util.send_sexp(
req,
[key(":clear-all-notes"), True])
dg = depgraph.DepGraph(self.source_roots, self.compile_include_dirs)
invalidated = dg.files_including(filename)
units = [f for f in invalidated if util.is_unit(f)]
# print "Clearing errors in " + str(len(units)) + " files."
# util.send_sexp(
# req, [key(":clear-file-notes"), list(invalidated)])
#
print "Recompiling " + str(len(units)) + " dependent units."
cmd = self.clang_base_cmd() + units
sys.stdout.flush()
else:
assert False, "WTF. Not header OR source unit?"
clang_output = util.run_process(" ".join(cmd))
self.receive_syntax_checker_output(req, clang_output)
util.send_sexp(req, util.return_ok(True, call_id))
def make_GST_profile(df_sample_info, gst_profile_file_path, kraken_database_path, quiet):
for sample in df_sample_info.index.tolist():
if not os.path.isfile("{0}/{1}.txt".format(gst_profile_file_path, sample)):
proc = run_process("kraken --db {0} --threads 8 --preload --paired {1} {2} > {3}/{4}.kraken".format(kraken_database_path, df_sample_info.loc[sample, 'read_file1'], df_sample_info.loc[sample, 'read_file2'], gst_profile_file_path, sample), print_only = False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process("kraken-report --db {0} {1}/{2}.kraken > {1}/{2}.kreport".format(kraken_database_path, gst_profile_file_path, sample), print_only = False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process("bracken -d {0} -i {1}/{2}.kreport -o {1}/{2}.bracken".format(kraken_database_path, gst_profile_file_path, sample), print_only = False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
def jedimaster():
# Using run_process
run_process("Replace output dirs for both 0 & 90 deg",
['python', "a7_jedisim_odirs.py"])
run_process("Create 3 catalogs for both 0 & 90 deg",
['python', "a8_jedisim_3cats.py"])
run_process("Run the simulation for normal case.",
['python', "a9_jedisimulate.py"])
run_process("Run the simulation for rotated case.",
['python', "a10_jedisimulate90.py"])
def run_jedimaster(start, end):
# Jedisim Outfolders
config = config_dict(config_path)
z = config['fixed_redshift']
outfolder = 'jedisim_output/jout_z{}_2017'.format(z) + time.strftime(
"_%b_%d_%H_%M/")
if not os.path.exists(outfolder):
os.makedirs(outfolder)
# For output names in Dropbox
computer = 'pisces'
odir = '/Users/poudel/Dropbox/jedisim_output_texts'
tm = time.strftime("_%b_%d_%H_%M")
if not os.path.isdir(odir):
os.makedirs(odir)
otxt = '{}/{}_jout_z{}_2017{}.txt'.format(odir, computer, z, tm)
# Run jedimaster in a loop
for i in range(start, end + 1):
print('{} {} {}'.format('Running jedimaster loop :', i, ''))
run_process("jedimaster.py", ['python', "jedimaster.py"])
# Copy final output files
infile1 = r'jedisim_out/out0/trial1_LSST_averaged_noised.fits'
outfile1 = outfolder + 'lsst_{:d}_z{}.fits'.format(i, z)
infile2 = r'jedisim_out/out90/90_trial1_LSST_averaged_noised.fits'
outfile2 = outfolder + 'lsst_90_{:d}_z{}.fits'.format(i, z)
infile3 = r'jedisim_out/rescaled_lsst/rescaled_noised_lsst_10.fits'
outfile3 = outfolder + 'monochromatic_{:d}z_{}.fits'.format(i, z)
infile4 = r'jedisim_out/rescaled_lsst90/rescaled_noised_lsst90_10.fits'
outfile4 = outfolder + 'monochromatic_90_{:d}z_{}.fits'.format(i, z)
shutil.copyfile(infile1, outfile1)
shutil.copyfile(infile2, outfile2)
shutil.copyfile(infile3, outfile3)
shutil.copyfile(infile4, outfile4)
# Append output names in Dropbox
with open(otxt, 'a') as fo:
fo.write(outfile1 + '\n')
def prepare_reuteri_specific_db(pangenome_file_path, gene_profile_file_path,
quiet):
proc = run_process(
"mmseqs createdb {0}/Lactobacillus_reuteri_specific.fasta {1}/Lactobacillus_reuteri_specific --dbtype 2"
.format(pangenome_file_path, gene_profile_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
def lsst_chromatic(configb, configd, configm):
"""Combine lsst_bulge and lsst_disk and take this as chromatic.
:Output: jedisim_out/out0/lsst.fits
"""
# Combine lsst_bulge with lsst_disk and call it lsst_unnoised.fits
dat = fits.getdata(configb['90_rescaled_outfileb']) + fits.getdata(
configd['90_rescaled_outfiled'])
fits.writeto(configm["90_lsst_unnoised"],
dat,
header=fits.getheader(configb['90_rescaled_outfileb']),
overwrite=True)
# Add noise to chromatic image and choose this as lsst final output.
run_process("jedinoise", [
'./executables/jedinoise', configm['90_lsst_unnoised'],
configm['exp_time'], configm['noise_mean'], configm["90_lsst"]
])
def clang_analyze_file(self, req, filename, call_id):
cmd = self.analyzer_base_cmd() + [filename]
clang_output = util.run_process(" ".join(cmd))
util.send_sexp(
req,
[key(":clear-all-notes"), True])
self.receive_syntax_checker_output(req, clang_output)
util.send_sexp(req, util.return_ok(True, call_id))
def generate_ml_tree(suffix, core_gene_file_path, quiet):
if not os.path.isfile("{1}/reuteri_core{0}.nwk".format(
"" if suffix is None else ("_" + suffix), core_gene_file_path)):
proc = run_process(
"raxmlHPC-PTHREADS -s {0}/all_align_core.fasta -n reuteri_core{1} -T 8 -m GTRCAT -p 123"
.format(core_gene_file_path, "" if suffix is None else
("_" + suffix)),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if line.strip() and not quiet:
print(line.decode('utf-8').rstrip())
run_process(
"mv RAxML_bestTree.reuteri_core{0} {1}/reuteri_core{0}.nwk".format(
"" if suffix is None else ("_" + suffix), core_gene_file_path))
run_process(
"rm RAxML_*.reuteri_core{0}*".format("" if suffix is None else (
"_" + suffix)))
return Tree("{1}/reuteri_core{0}.nwk".format(
"" if suffix is None else ("_" + suffix), core_gene_file_path))
def extract_core_gene(df_genome_info, output_dir, core_gene_file_path, quiet):
df_complete = df_genome_info[df_genome_info['type'] == 'complete']
for i in range(df_complete.index.size):
genome_id = df_complete.index.tolist()[i]
proc = run_process(
"prokka {0} --outdir {1}/gff --prefix {2} --force".format(
df_complete.loc[genome_id, 'genome_file'], output_dir, i + 1),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"roary {0}/gff/*.gff -f {0}/roary_output -p 8 -e".format(output_dir),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
core_gene_name_list = write_core_gene_file(
"{0}/roary_output".format(output_dir), core_gene_file_path,
df_complete)
return core_gene_name_list
def clang_analyze_all(self, req, call_id):
cmd = self.analyzer_base_cmd()
for s in self.all_units():
cmd.append(s)
clang_output = util.run_process(" ".join(cmd))
util.send_sexp(
req,
[key(":clear-all-notes"), True])
self.receive_syntax_checker_output(req, clang_output)
util.send_sexp(req, util.return_ok(True, call_id))
util.send_sexp(
req,
[key(":full-check-finished"), True])
def clang_completions(self, req, filename, line, col, prefix, call_id):
cmd = self.clang_completions_base_cmd(filename, line, col)
print cmd
sys.stdout.flush()
clang_output = util.run_process(" ".join(cmd))
candidates = []
for line in clang_output:
print line
m = self.RE_COMPLETION.match(line)
if m:
name = m.group(1)
tpe = m.group(2)
if name.find(prefix) == 0:
candidates.append(
[key(":name"),m.group(1),
key(":type-sig"),m.group(2),
key(":is-callable"),False,
])
util.send_sexp(req, util.return_ok(candidates, call_id))
def main():
"""Run main function."""
# Before running jedimaster, create config files
z = 0.7
run_process(
"Create configs",
['python', "a1_jedisim_config.py -z ",
str(z)]) # physics_settings/configb.sh, configd.sh, configm.sh
run_process("Interpolate sed",
['python', "a2_interpolate_sed.py -z ",
str(z)])
run_process("Create jedicolor_args.txt",
['python', "a3_jcol_args.py -z ",
str(z)])
# Now run the loop
# start, end inclusive
# no. of loop = end - start + 1
start, end = 0, 1
run_jedimaster(start, end)
def run_jedimaster(start, end):
# create outfolders and get their names.
jouts = jedisim_outfolders()
# Write output names in dropbox textfile.
computer = 'simplici'
odir = '/Users/poudel/Dropbox/jout'
tm = time.strftime("_%b_%d_%H_%M")
if not os.path.isdir(odir):
os.makedirs(odir)
otxt = '{}/jout_{}_z{}_2017{}.txt'.format(odir, computer, z, tm)
# Create empty textfile in dropbox to be added later.
print('Creating: {}'.format(otxt))
with open(otxt, 'w') as fo:
fo.write("")
# Run jedimaster in a loop
for i in range(start, end + 1):
# Indivisual times
loop_start_time = time.time()
print('{} {} {}'.format('Running jedimaster loop :', i, ''))
run_process("jedimaster.py", ['python', "jedimaster.py"])
# Copy lsst file
infile = r'jedisim_out/out0/scaled_bulge_disk/trial1_lsst.fits'
outfile = jouts['lsst'] + 'lsst_z{}_{}.fits'.format(z, i)
shutil.copyfile(infile, outfile)
print('From and To for Loop {}: \n{}'.format(i, infile))
print('{}\n'.format(outfile))
# Copy lsst_mono file
infile = r'jedisim_out/out0/scaled_bulge_disk/trial1_lsst_mono.fits'
outfile = jouts['lsst_mono'] + 'lsst_mono_z{}_{}.fits'.format(z, i)
shutil.copyfile(infile, outfile)
print('From and To for Loop {}: \n{}'.format(i, infile))
print('{}\n'.format(outfile))
# Copy gcsb
infile = r'jedisim_out/out0/scaled_bulge/lsst_bulge.fits'
outfile = jouts['gcsb'] + 'gcsb_z{}_{}.fits'.format(z, i)
shutil.copyfile(infile, outfile)
print('From and To for Loop {}: \n{}'.format(i, infile))
print('{}\n'.format(outfile))
# Copy gcsd
infile = r'jedisim_out/out0/scaled_disk/lsst_disk.fits'
outfile = jouts['gcsd'] + 'gscd_z{}_{}.fits'.format(z, i)
shutil.copyfile(infile, outfile)
print('From and To for Loop {}: \n{}'.format(i, infile))
print('{}\n'.format(outfile))
# Copy gcsm
infile = r'jedisim_out/out0/scaled_bulge_disk/lsst_bulge_disk.fits'
outfile = jouts['gcsm'] + 'gscm_z{}_{}.fits'.format(z, i)
shutil.copyfile(infile, outfile)
print('From and To for Loop {}: \n{}'.format(i, infile))
print('{}\n'.format(outfile))
# Copy psfb
infile = r'psf/psfb.fits'
outfile = jouts['psfb'] + 'psfb_z{}_{}.fits'.format(z, i)
shutil.copyfile(infile, outfile)
print('From and To for Loop {}: \n{}'.format(i, infile))
print('{}\n'.format(outfile))
# Copy psfd
infile = r'psfd.fits'
outfile = jouts['psfd'] + 'psfd_z{}_{}.fits'.format(z, i)
shutil.copyfile(infile, outfile)
print('From and To for Loop {}: \n{}'.format(i, infile))
print('{}\n'.format(outfile))
# Copy psfm
infile = r'psfm.fits'
outfile = jouts['psfm'] + 'psfm_z{}_{}.fits'.format(z, i)
shutil.copyfile(infile, outfile)
print('From and To for Loop {}: \n{}'.format(i, infile))
print('{}\n'.format(outfile))
# Append output names in Dropbox
hostname = 'simplici'
loop_end_time = time.time()
loop_time = loop_end_time - loop_start_time
loop_mins = loop_time / 60
date = time.ctime()
with open(otxt, 'a') as fo:
fo.write(
'{} {}: Runtime {:.0f} mins and EndDate: {}\n{}\n\n'.format(
hostname, i, loop_mins, date, outfile1))
def add_nodes(client,
apps_client,
cfile,
kinds,
counts,
management_ip,
aws_key_id=None,
aws_key=None,
create=False,
prefix=None):
for i in range(len(kinds)):
print('Adding %d %s server node(s) to cluster...' %
(counts[i], kinds[i]))
prev_count = util.get_previous_count(client, kinds[i])
util.run_process(
['./modify_ig.sh', kinds[i],
str(counts[i] + prev_count)])
util.run_process(['./validate_cluster.sh'])
replica_str = ' '.join(util.get_node_ips(client, 'role=aft'))
# Create should only be true when the DaemonSet is being created for the
# first time -- i.e., when this is called from create_cluster. After that,
# we can basically ignore this because the DaemonSet will take care of
# adding pods to created nodes.
if create:
for i in range(len(kinds)):
kind = kinds[i]
fname = 'yaml/ds/%s-ds.yml' % kind
yml = util.load_yaml(fname, prefix)
for container in yml['spec']['template']['spec']['containers']:
env = container['env']
util.replace_yaml_val(env, 'REPLICA_IPS', replica_str)
util.replace_yaml_val(env, 'MANAGER', management_ip)
util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE,
body=yml)
# Wait until all pods of this kind are running
res = []
while len(res) != counts[i]:
res = util.get_pod_ips(client, 'role=' + kind, is_running=True)
created_pods = []
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' +
kind).items
# Generate list of all recently created pods.
for pod in pods:
pname = pod.metadata.name
for container in pod.spec.containers:
cname = container.name
created_pods.append((pname, cname))
pod.metadata.labels['aftReady'] = 'isready'
client.patch_namespaced_pod(pod.metadata.name,
util.NAMESPACE, pod)
# Copy the KVS config into all recently created pods.
os.system('cp %s ./aft-config.yml' % cfile)
for pname, cname in created_pods:
util.copy_file_to_pod(
client, 'aft-config.yml', pname,
'/go/src/github.com/Alchem-Lab/aft/config', cname)
os.system('rm ./aft-config.yml')
def add_nodes(client, apps_client, cfile, kinds, counts, create=False):
for i in range(len(kinds)):
print('Adding %d %s server node(s) to cluster...' %
(counts[i], kinds[i]))
# get the previous number of nodes of type kind that are running
prev_count = util.get_previous_count(client, kinds[i])
# we only add new nodes if we didn't pass in a node IP
util.run_process(['./modify_ig.sh', kinds[i],
str(counts[i] + prev_count)])
util.run_process(['./validate_cluster.sh'])
kops_ip = util.get_pod_ips(client, 'role=kops')[0]
route_ips = util.get_pod_ips(client, 'role=routing')
if len(route_ips) > 0:
seed_ip = random.choice(route_ips)
else:
seed_ip = ''
mon_str = ' '.join(util.get_pod_ips(client, 'role=monitoring'))
route_str = ' '.join(route_ips)
sched_str = ' '.join(util.get_pod_ips(client, 'role=scheduler'))
route_addr = util.get_service_address(client, 'routing-service')
function_addr = util.get_service_address(client, 'function-service')
# create should only be true when the DaemonSet is being created for the
# first time -- i.e., when this is called from create_cluster
if create:
for i in range(len(kinds)):
kind = kinds[i]
fname = 'yaml/ds/%s-ds.yml' % kind
yml = util.load_yaml(fname)
for container in yml['spec']['template']['spec']['containers']:
env = container['env']
util.replace_yaml_val(env, 'ROUTING_IPS', route_str)
util.replace_yaml_val(env, 'ROUTE_ADDR', route_addr)
util.replace_yaml_val(env, 'SCHED_IPS', sched_str)
util.replace_yaml_val(env, 'FUNCTION_ADDR', function_addr)
util.replace_yaml_val(env, 'MON_IPS', mon_str)
util.replace_yaml_val(env, 'MGMT_IP', kops_ip)
util.replace_yaml_val(env, 'SEED_IP', seed_ip)
apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE,
body=yml)
# wait until all pods of this kind are running
res = []
while len(res) != counts[i]:
res = util.get_pod_ips(client, 'role='+kind, is_running=True)
created_pods = []
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' +
kind).items
for pod in pods:
pname = pod.metadata.name
for container in pod.spec.containers:
cname = container.name
created_pods.append((pname, cname))
os.system('cp %s ./kvs-config.yml' % cfile)
for pname, cname in created_pods:
util.copy_file_to_pod(client, 'kvs-config.yml', pname,
'/fluent/conf/', cname)
os.system('rm ./kvs-config.yml')
def create_cluster(replica_count, gc_count, lb_count, bench_count, cfile,
ssh_key, cluster_name, kops_bucket, aws_key_id, aws_key):
prefix = './'
util.run_process(['./create_cluster_object.sh', kops_bucket, ssh_key])
client, apps_client = util.init_k8s()
print('Creating management pod')
# management_spec = util.load_yaml('yaml/pods/management-pod.yml')
# env = management_spec['spec']['containers'][0]['env']
# util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
# util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
#
# client.create_namespaced_pod(namespace=util.NAMESPACE,
# body=management_spec)
# management_ip = util.get_pod_ips(client, 'role=management',
# is_running=True)[0]
management_ip = ""
print('Creating standby replicas...')
util.run_process(['./modify_ig.sh', 'standby', '1'])
util.run_process(['./validate_cluster.sh'])
print('Creating %d load balancer, %d GC replicas...' %
(lb_count, gc_count))
add_nodes(client, apps_client, cfile, ['lb', 'gc'], [lb_count, gc_count],
management_ip, aws_key_id, aws_key, True, prefix)
lb_pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector="role=lb").items
kubecfg = os.path.join(os.environ['HOME'], '.kube/config')
for pod in lb_pods:
util.copy_file_to_pod(client, kubecfg, pod.metadata.name,
'/root/.kube', 'lb-container')
replica_ips = util.get_node_ips(client, 'role=gc', 'ExternalIP')
with open('gcs.txt', 'w') as f:
for ip in replica_ips:
f.write(ip + '\n')
# Wait until the monitoring pod is finished creating to get its IP address
# and then copy KVS config into the monitoring pod.
print('Creating %d Aft replicas...' % (replica_count))
add_nodes(client, apps_client, cfile, ['aft'], [replica_count],
management_ip, aws_key_id, aws_key, True, prefix)
util.get_pod_ips(client, 'role=aft')
replica_ips = util.get_node_ips(client, 'role=aft', 'ExternalIP')
with open('replicas.txt', 'w') as f:
for ip in replica_ips:
f.write(ip + '\n')
os.system('cp %s aft-config.yml' % cfile)
management_pname = management_spec['metadata']['name']
management_cname = management_spec['spec']['containers'][0]['name']
util.copy_file_to_pod(client, 'aft-config.yml', management_pname,
'/go/src/github.com/tajshaik24/aft/config',
management_cname)
util.copy_file_to_pod(client, 'replicas.txt', management_pname,
'/go/src/github.com/tajshaik24/aft',
management_cname)
util.copy_file_to_pod(client, 'gcs.txt', management_pname,
'/go/src/github.com/tajshaik24/aft',
management_cname)
util.copy_file_to_pod(client, kubecfg, management_pname, '/root/.kube/',
management_cname)
os.system('rm aft-config.yml')
os.system('rm gcs.txt')
# Copy replicas.txt to all Aft pods.
aft_pod_list = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector="role=aft").items
aft_pod_list = list(map(lambda pod: pod.metadata.name, aft_pod_list))
for pname in aft_pod_list:
util.copy_file_to_pod(client, 'replicas.txt', pname,
'/go/src/github.com/tajshaik24/aft',
'aft-container')
gc_pod_list = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector="role=gc").items
gc_pod_list = list(map(lambda pod: pod.metadata.name, gc_pod_list))
for pname in gc_pod_list:
util.copy_file_to_pod(client, 'replicas.txt', pname,
'/go/src/github.com/tajshaik24/aft',
'gc-container')
os.system('rm replicas.txt')
print('Adding %d benchmark nodes...' % (bench_count))
add_nodes(client, apps_client, cfile, ['benchmark'], [bench_count],
management_ip, aws_key_id, aws_key, True, prefix)
print('Finished creating all pods...')
print('Creating Aft service...')
service_spec = util.load_yaml('yaml/services/aft.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
print('Authorizing ports for Aft replicas...')
permission = [{
'FromPort': 7654,
'IpProtocol': 'tcp',
'ToPort': 7656,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}, {
'FromPort': 7777,
'IpProtocol': 'tcp',
'ToPort': 7782,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}, {
'FromPort': 8000,
'IpProtocol': 'tcp',
'ToPort': 8003,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}]
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permission)
print('Finished!')
def run_jedicatalog(config_path):
# Make the catalog of galaxies (three text files)
run_process("jedicatalog", ["./executables/jedicatalog", config_path])
def search_core_gene(df_genome_info, genome_id, core_gene_file_path,
delta_output_path, snp_output_path, quiet):
if not os.path.isfile("{0}/{1}.delta".format(delta_output_path,
genome_id)):
proc = run_process(
"nucmer {0}/core_gene_ref.fasta {1} -p {2}/{3}".format(
core_gene_file_path, df_genome_info.loc[genome_id,
'genome_file'],
delta_output_path, genome_id),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
if not os.path.isfile("{0}/{1}_filter.delta".format(
delta_output_path, genome_id)):
proc = run_process(
"delta-filter -r -q {0}/{1}.delta > {0}/{1}_filter.delta".format(
delta_output_path, genome_id),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
core_block_dict = dict()
core_len_dict = dict()
with open("{0}/{1}_filter.delta".format(delta_output_path,
genome_id)) as delta_file:
line = delta_file.readline()
while line != "":
line = line.rstrip()
if line[0] == '>':
info = line[1:].split(' ')
core, ref_len = info[0], int(info[2])
core_len_dict[core] = ref_len
if core not in core_block_dict.keys():
core_block_dict[core] = []
elif len(line.split(' ')) == 7:
core_block_dict[core].append(
(int(line.split(' ')[0]), int(line.split(' ')[1])))
line = delta_file.readline()
core_gene_list = sorted(core_len_dict.keys())
core_empty_block_dict = dict()
for core in core_block_dict.keys():
block_list = sorted(core_block_dict[core], key=lambda block: block[0])
length = core_len_dict[core]
curr_start = 1
for start, end in block_list:
if start - 1 >= curr_start:
if core not in core_empty_block_dict.keys():
core_empty_block_dict[core] = []
core_empty_block_dict[core].append((curr_start, start - 1))
if curr_start <= end:
curr_start = end + 1
if length >= curr_start:
if core not in core_empty_block_dict.keys():
core_empty_block_dict[core] = []
core_empty_block_dict[core].append((curr_start, length))
core_gap_dict = dict()
core_var_dict = dict()
if not os.path.isfile("{0}/{1}.snp".format(snp_output_path, genome_id)):
with open("{0}/{1}.snp".format(snp_output_path, genome_id),
'w') as snp_file:
proc = run_process("show-snps -H -T {0}/{1}_filter.delta".format(
delta_output_path, genome_id),
print_only=False)
for line in iter(proc.stdout.readline, b''):
line = line.decode('utf-8').rstrip()
if not quiet:
print(line)
snp_file.write(line + '\n')
with open("{0}/{1}.snp".format(snp_output_path, genome_id)) as snp_file:
line = snp_file.readline()
while line != "":
line = line.rstrip()
info = line.split('\t')
pos = int(info[0])
ref_char = '-' if info[1] == '.' else info[1]
query_char = '-' if info[2] == '.' else info[2]
core = info[10]
if ref_char == '-':
if pos in core_gap_dict.keys():
if core not in core_gap_dict.keys():
core_gap_dict[core] = dict()
core_gap_dict[core][pos].append(query_char)
else:
if core not in core_gap_dict.keys():
core_gap_dict[core] = dict()
core_gap_dict[core][pos] = [query_char]
else:
if core not in core_var_dict.keys():
core_var_dict[core] = dict()
core_var_dict[core][pos] = (ref_char, query_char)
line = snp_file.readline()
return core_gene_list, core_empty_block_dict, core_gap_dict, core_var_dict
def create_cluster(txn_count, keynode_count, rtr_count, worker_count, lb_count,
benchmark_count, config_file, branch_name, ssh_key,
cluster_name, kops_bucket, aws_key_id, aws_key,
anna_config_file):
prefix = './'
util.run_process(['./create_cluster_object.sh', kops_bucket, ssh_key],
'kops')
client, apps_client = util.init_k8s()
print('Creating Monitor Node...')
add_nodes(client, apps_client, config_file, "monitor", 1, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating %d Anna Routing Nodes...' % (rtr_count))
add_nodes(client, apps_client, anna_config_file, "routing", rtr_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating routing service...')
service_spec = util.load_yaml('yaml/services/routing.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
util.get_service_address(client, 'routing-service')
print('Creating %d Key Nodes...' % (keynode_count))
add_nodes(client, apps_client, config_file, "keynode", keynode_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating %d Worker Nodes...' % (worker_count))
add_nodes(client, apps_client, config_file, "worker", worker_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating Worker Service...')
service_spec = util.load_yaml('yaml/services/worker.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
util.get_service_address(client, 'worker-service')
print('Creating %d TASC nodes...' % (txn_count))
add_nodes(client, apps_client, config_file, 'tasc', txn_count, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating %d Load Balancers...' % (lb_count))
add_nodes(client, apps_client, config_file, 'lb', lb_count, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating TASC Load Balancing service...')
service_spec = util.load_yaml('yaml/services/tasc.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Creating %d Benchmark nodes...' % (benchmark_count))
add_nodes(client, apps_client, config_file, 'benchmark', benchmark_count,
aws_key_id, aws_key, True, prefix, branch_name)
benchmark_ips = util.get_node_ips(client, 'role=benchmark', 'ExternalIP')
with open('../cmd/benchmark/benchmarks.txt', 'w+') as f:
for ip in benchmark_ips:
f.write(ip + '\n')
print('Finished creating all pods...')
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
print("Authorizing Ports for TASC...")
permission = [{
'FromPort': 0,
'IpProtocol': 'tcp',
'ToPort': 65535,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}]
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permission)
print('Registering Key Nodes...')
keynode_pod_ips = util.get_pod_ips(client, 'role=keynode', is_running=True)
register(client, keynode_pod_ips)
print("\nThe TASC ELB Endpoint: " +
util.get_service_address(client, "tasc-service") + "\n")
print('Finished!')
def create_cluster(mem_count, ebs_count, func_count, sched_count, route_count,
bench_count, cfile, ssh_key, cluster_name, kops_bucket,
aws_key_id, aws_key):
# create the cluster object with kops
util.run_process(
['./create_cluster_object.sh', cluster_name, kops_bucket, ssh_key])
client, apps_client = util.init_k8s()
# create the kops pod
print('Creating management pods...')
kops_spec = util.load_yaml('yaml/pods/kops-pod.yml')
env = kops_spec['spec']['containers'][0]['env']
util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
util.replace_yaml_val(env, 'KOPS_STATE_STORE', kops_bucket)
util.replace_yaml_val(env, 'FLUENT_CLUSTER_NAME', cluster_name)
client.create_namespaced_pod(namespace=util.NAMESPACE, body=kops_spec)
# wait for the kops pod to start
kops_ip = util.get_pod_ips(client, 'role=kops', is_running=True)[0]
# copy kube config file to kops pod, so it can execute kubectl commands
kops_podname = kops_spec['metadata']['name']
kcname = kops_spec['spec']['containers'][0]['name']
os.system('cp %s kvs-config.yml' % cfile)
util.copy_file_to_pod(client, '/home/ubuntu/.kube/config', kops_podname,
'/root/.kube/', kcname)
util.copy_file_to_pod(client, ssh_key, kops_podname, '/root/.ssh/', kcname)
util.copy_file_to_pod(client, ssh_key + '.pub', kops_podname,
'/root/.ssh/', kcname)
util.copy_file_to_pod(client, 'kvs-config.yml', kops_podname,
'/fluent/conf/', kcname)
# start the monitoring pod
mon_spec = util.load_yaml('yaml/pods/monitoring-pod.yml')
util.replace_yaml_val(mon_spec['spec']['containers'][0]['env'], 'MGMT_IP',
kops_ip)
client.create_namespaced_pod(namespace=util.NAMESPACE, body=mon_spec)
util.get_pod_ips(client, 'role=monitoring')
# copy config file into monitoring pod -- wait till we create routing pods,
# so we're sure that the monitoring nodes are up and running
util.copy_file_to_pod(client, 'kvs-config.yml',
mon_spec['metadata']['name'], '/fluent/conf/',
mon_spec['spec']['containers'][0]['name'])
os.system('rm kvs-config.yml')
print('Creating %d routing nodes...' % (route_count))
add_nodes(client, apps_client, cfile, ['routing'], [route_count], True)
util.get_pod_ips(client, 'role=routing')
print('Creating %d memory, %d ebs node(s)...' % (mem_count, ebs_count))
add_nodes(client, apps_client, cfile, ['memory', 'ebs'],
[mem_count, ebs_count], True)
print('Creating routing service...')
service_spec = util.load_yaml('yaml/services/routing.yml')
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Adding %d scheduler nodes...' % (sched_count))
add_nodes(client, apps_client, cfile, ['scheduler'], [sched_count], True)
util.get_pod_ips(client, 'role=scheduler')
print('Adding %d function serving nodes...' % (func_count))
add_nodes(client, apps_client, cfile, ['function'], [func_count], True)
print('Creating function service...')
service_spec = util.load_yaml('yaml/services/function.yml')
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Adding %d benchmark nodes...' % (bench_count))
add_nodes(client, apps_client, cfile, ['benchmark'], [bench_count], True)
print('Finished creating all pods...')
os.system('touch setup_complete')
util.copy_file_to_pod(client, 'setup_complete', kops_podname, '/fluent',
kcname)
os.system('rm setup_complete')
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
print('Authorizing ports for routing service...')
permission = [{
'FromPort': 6200,
'IpProtocol': 'tcp',
'ToPort': 6203,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}]
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permission)
routing_svc_addr = util.get_service_address(client, 'routing-service')
function_svc_addr = util.get_service_address(client, 'function-service')
print('The routing service can be accessed here: \n\t%s' %
(routing_svc_addr))
print('The function service can be accessed here: \n\t%s' %
(function_svc_addr))
def lsst_TDCR(config, psf_name, rescaled_outfile90):
"""Create a single lsst_bulge or lsst_disk or lsst_bulge_disk image
images after running 6 programs. We will add noise to this later.
Args:
config_path (str): input physics config file.
e.g. physics_settings/configb.sh
e.g. physics_settings/configd.sh
psf_names (str): input psf file for scaled_bulge or unscaled_disk
e.g. config['psfb'] = psf/psfb.fits
e.g. config['psfd'] = psf/psfd.fits
rescaled_outfile (str): output after running 6 programs.
This is the final output of program jedirescale.
e.g. config['rescaled_outfileb'] = lsst_bulge.fits
e.g. config['rescaled_outfiled'] = lsst_disk.fits
e.g. config['rescaled_outfilebd'] = lsst_bulge_disk.fits
:Outputs: The outputs are following:
jedisim_out/out0/lsst_bulge.fits
jedisim_out/out0/lsst_disk.fits
jedisim_out/out0/lsst_bulge_disk.fits
Note that we combine these two files and call it chromatic lsst.fits.
We convolve this image with monochromatic psf and call that lsst_mono.fits.
The keys that has 90 prefix on them are following ::
keys = ['catalog_file', 'dislist_file',
'distortedlist_file', 'convolvedlist_file',
'HST_image', 'HST_convolved_image',
'rescaled_outfileb', 'rescaled_outfiled',
'rescaled_outfilem', 'lsst_unnoised',
'lsst', 'lsst_mono' ]
:Runtime: 5 min 44 sec for 201 scaled bulge galaxies.
"""
# Transform scaled_bulge or scaled_disk fitsfiles according to:
# jedisim_out/out0/scaled_bulge/trial1_catalog.txt.
#
# jeditransform will create 12420 .gz fitsfiles. For example:
# jedisim_out/out0/scaled_bulge/stamp_0/stamp_0_to_999.fits.gz (for stamps 0 to 12 )
#
# It also creates dislist for the jedidistort,viz.,
# jedisim_out/out0/scaled_bulge/trial1_dislist.txt
run_process("jeditransform", [
'./executables/jeditransform', config['90_catalog_file'],
config['90_dislist_file']
])
# This program distorts the 12420 galaxies from jedisim_out/out0/stamp_/
# according to dislist.txt and lens.txt and write distorted
# galaxies inside 13 folders jedisim_out/out0/distorted_/
#
# In the end we get unzipped distorted images in 13 folders.
# jedisim_out/out0/distorted_0/distorted_0.fits 1000*12+ 420 fitsfiles.
run_process("jedidistort", [
'./executables/jedidistort', config['nx'], config['ny'],
config['90_dislist_file'], config['lens_file'], config['pix_scale'],
config['lens_z']
])
# This program combines 12,420 distorted fits files inside the
# jedisim_out/out0/distorted_/distorted_fits/
# into a single large embedded image: jedisim_out/out0/HST.fits.
run_process("jedipaste", [
'./executables/jedipaste', config['nx'], config['ny'],
config['90_distortedlist_file'], config['90_HST_image']
])
# Convolve the given single fitsfile with given PSF and write 6 bands
# of convolved images.
# E.g. convolve HST.fits with psfb.fits to get 6 bands of convolved images.
run_process("jediconvolve", [
'./executables/jediconvolve', config['90_HST_image'], psf_name,
config['90_output_folder'] + 'convolved/'
])
# Combine 6 convolved bands into one HST_convolved image.
run_process("jedipaste", [
'./executables/jedipaste', config['nx'], config['ny'],
config['90_convolvedlist_file'], config['90_HST_convolved_image']
])
# Change the PIXSCALE of the input fitsfile into another one.
# e.g. HST has pixscale 0.06 and we change it to LSST pixscale = 0.2
run_process("jedirescale", [
'./executables/jedirescale', config['90_HST_convolved_image'],
config['pix_scale'], config['final_pix_scale'], config['x_trim'],
config['y_trim'], rescaled_outfile90
])
def generate_reuteri_pangenome(df_genome_info, pangenome_file_path,
tmp_file_path, quiet):
records = []
for genome_id in [
gn for gn in df_genome_info.index.tolist()
if df_genome_info.loc[gn, 'type'] not in 'genus'
]:
records = records + list(
SeqIO.to_dict(
SeqIO.parse(
"{0}".format(df_genome_info.loc[genome_id, 'cds_file']),
'fasta')).values())
SeqIO.write(
records,
"{0}/Lactobacillus_reuteri_raw.fasta".format(pangenome_file_path),
"fasta")
proc = run_process(
"mmseqs createdb {0}/Lactobacillus_reuteri_raw.fasta {0}/Lactobacillus_reuteri_raw --dbtype 2;\n"
.format(pangenome_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs linclust {0}/Lactobacillus_reuteri_raw {0}/Lactobacillus_reuteri_clu {1} --min-seq-id 0.9 -c 0.9 --kmer-per-seq 200 --adjust-kmer-len;\n"
.format(pangenome_file_path, tmp_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs result2repseq {0}/Lactobacillus_reuteri_raw {0}/Lactobacillus_reuteri_clu {0}/Lactobacillus_reuteri_rep;\n"
.format(pangenome_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs createsubdb {0}/Lactobacillus_reuteri_rep {0}/Lactobacillus_reuteri_raw_h {0}/Lactobacillus_reuteri_rep_h;\n"
.format(pangenome_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs result2flat {0}/Lactobacillus_reuteri_raw {0}/Lactobacillus_reuteri_raw {0}/Lactobacillus_reuteri_rep {0}/Lactobacillus_reuteri.fasta;\n"
.format(pangenome_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs createtsv {0}/Lactobacillus_reuteri_raw {0}/Lactobacillus_reuteri_raw {0}/Lactobacillus_reuteri_clu {0}/Lactobacillus_reuteri.tsv;\n"
.format(pangenome_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
def add_nodes(client, apps_client, cfile, kind, count, aws_key_id=None,
aws_key=None, create=False, prefix=None, branch="master"):
print('Adding %d %s server node(s) to cluster...' % (count, kind))
prev_count = util.get_previous_count(client, kind)
util.run_process(['./modify_ig.sh', kind, str(count + prev_count)], 'kops')
util.run_process(['./validate_cluster.sh'], 'kops')
if create:
fname = 'yaml/ds/%s-ds.yml' % kind
yml = util.load_yaml(fname, prefix)
for container in yml['spec']['template']['spec']['containers']:
env = container['env']
util.replace_yaml_val(env, 'BRANCH', branch)
util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
if kind == "tasc":
routing_svc = util.get_service_address(client, 'routing-service')
util.replace_yaml_val(env, 'ROUTING_ILB', routing_svc)
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
worker_svc = util.get_service_address(client, 'worker-service')
util.replace_yaml_val(env, 'WORKER_ILB', worker_svc)
if kind == "keynode":
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
if kind == 'worker':
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
routing_svc = util.get_service_address(client, 'routing-service')
util.replace_yaml_val(env, 'ROUTING_ILB', routing_svc)
apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE,
body=yml)
# Wait until all pods of this kind are running
res = []
while len(res) != count:
res = util.get_pod_ips(client, 'role=' + kind, is_running=True)
created_pods = []
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' +
kind).items
# Send kube config to lb
if kind == 'lb':
kubecfg = os.path.join(os.environ['HOME'], '.kube/config')
for pod in pods:
cname = pod.spec.containers[0].name
util.copy_file_to_pod(client, kubecfg, pod.metadata.name,
'/root/.kube', cname)
# Generate list of all recently created pods.
created_pod_ips = []
for pod in pods:
created_pod_ips.append(pod.status.pod_ip)
pname = pod.metadata.name
for container in pod.spec.containers:
cname = container.name
created_pods.append((pname, cname))
# Copy the KVS config into all recently created pods.
cfile_name = './tasc-config.yml' if kind != 'routing' else './anna-config.yml'
cfile_dir = '/go/src/github.com/saurav-c/tasc/config' if kind != 'routing' else 'hydro/anna/conf'
os.system(str('cp %s ' + cfile_name) % cfile)
for pname, cname in created_pods:
util.copy_file_to_pod(client, cfile_name[2:], pname,
cfile_dir, cname)
os.system('rm ' + cfile_name)
def generate_lactobacillus_pangenome(df_genome_info, pangenome_file_path,
tmp_file_path, quiet):
records = list(
SeqIO.to_dict(
SeqIO.parse(
"{0}/Lactobacillus_reuteri.fasta".format(pangenome_file_path),
'fasta')).values())
for genome_id in df_genome_info[df_genome_info['type'] ==
'genus'].index.tolist():
count = 0
for seq_id, rec in SeqIO.to_dict(
SeqIO.parse(
"{0}".format(df_genome_info.loc[genome_id, 'cds_file']),
'fasta')).items():
count += 1
rec.id = rec.id + "|{0}".format(count)
records.append(rec)
SeqIO.write(records,
"{0}/Lactobacillus_raw.fasta".format(pangenome_file_path),
"fasta")
proc = run_process(
"mmseqs createdb {0}/Lactobacillus_raw.fasta {0}/Lactobacillus_raw --dbtype 2;\n"
.format(pangenome_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs linclust {0}/Lactobacillus_raw {0}/Lactobacillus_clu {1} --min-seq-id 0.9 -c 0.9 --kmer-per-seq 200 --adjust-kmer-len;\n"
.format(pangenome_file_path, tmp_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs result2repseq {0}/Lactobacillus_raw {0}/Lactobacillus_clu {0}/Lactobacillus_rep;\n"
.format(pangenome_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs createsubdb {0}/Lactobacillus_rep {0}/Lactobacillus_raw_h {0}/Lactobacillus_rep_h;\n"
.format(pangenome_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs result2flat {0}/Lactobacillus_raw {0}/Lactobacillus_raw {0}/Lactobacillus_rep {0}/Lactobacillus.fasta;\n"
.format(pangenome_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
proc = run_process(
"mmseqs createtsv {0}/Lactobacillus_raw {0}/Lactobacillus_raw {0}/Lactobacillus_clu {0}/Lactobacillus.tsv;\n"
.format(pangenome_file_path),
print_only=False)
for line in iter(proc.stdout.readline, b''):
if not quiet:
print(line.decode('utf-8').rstrip())
