def analysis_step(self, iteration, instance):
k1 = Kernel(name="misc.randval")
k1.arguments = ["--upperlimit=3", "--filename=iters.dat"]
k1.download_output_data = "iters.dat"
k2 = Kernel(name="misc.randval")
k2.arguments = ["--upperlimit=16", "--filename=sims.dat"]
k2.download_output_data = "sims.dat"
return [k1, k2]
def analysis_step(self, iteration, instance):
k1 = Kernel(name="misc.randval")
k1.arguments = ["--upperlimit=3", "--filename=iters.dat"]
k1.download_output_data = "iters.dat"
k2 = Kernel(name="misc.randval")
k2.arguments = ["--upperlimit=16","--filename=sims.dat"]
k2.download_output_data = "sims.dat"
return [k1,k2]
def simulation_step(self, iteration, instance):
"""In the simulation step we
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=asciifile-{0}.dat".format(instance)]
k.download_output_data = ['asciifile-{0}.dat'.format(instance)]
return [k]
def element_comparison(self, elements1, elements2):
"""In the comparison, we take the previously generated modified trajectory
and perform a Hausdorff distance calculation between all the unique pairs
of trajectories
"""
input_filenames1 = ["traj_flat%d.npz.npy"%(el1) for el1 in elements1]
input_filenames2 = ["traj_flat%d.npz.npy"%(el2) for el2 in elements2]
output_filename = "comparison-%03d-%03d.dat"%(elements1[0],elements2[0])
print "Element Comparison {0} - {1}".format(elements1,elements2)
k = Kernel(name="my.hausdorff")
k.arguments = ["--dist_file=hausdorff_kernel.py",
"--inputfile1={0}".format(input_filenames1),
"--inputfile2={0}".format(input_filenames2),
"--outputfile={0}".format(output_filename)]
k.upload_input_data = ["hausdorff_kernel.py"]
# If the input data are in in a web server use the following
# k.download_input_data = ["/<PATH>/<TO>/<WEB>?<SERVER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"]
# If the input data are in a folder to the target machine use the following
# k.link_input_data = ["/<PATH>/<TO>/<FOLDER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"]
# The result files comparison-x-y.dat are downloaded.
k.download_output_data = output_filename
print "Element Comparison Finished {0} - {1}".format(elements1,elements2)
return k
def analysis_step(self, iteration, instance):
"""In the analysis step, we take the previously generated simulation
output and perform a Levenshtein distance calculation between it
and the 'reference' file.
..note:: The placeholder ``$PRE_LOOP`` used in ``link_input_data`` is
a reference to the working directory of pre_loop.
The placeholder ``$PREV_SIMULATION`` used in ``link_input_data``
is a reference to the working directory of the previous
simulation step.
It is also possible to reference a specific
simulation step using ``$SIMULATION_N`` or all simulations
via ``$SIMULATIONS``. Analogous placeholders exist for
``ANALYSIS``.
"""
input_filename = "simulation-{0}-{1}.dat".format(iteration, instance)
output_filename = "analysis-{0}-{1}.dat".format(iteration, instance)
k = Kernel(name="misc.levenshtein")
k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"]
k.arguments = ["--inputfile1=reference.dat",
"--inputfile2={0}".format(input_filename),
"--outputfile={0}".format(output_filename)]
k.download_output_data = output_filename
return k
def prepare_replica_for_exchange(self, replica):
"""Prepares md.re_exchange kernel to launch namd_matrix_calculator.py
script on target resource in order to populate columns of swap matrix.
Arguments:
replica - object representing a given replica and it's attributes
Returns:
k - an instance of Kernel class
"""
basename = self.inp_basename[:-5]
matrix_col = "matrix_column_{cycle}_{replica}.dat"\
.format(cycle=replica.cycle-1, replica=replica.id )
k = Kernel(name="md.re_exchange")
k.arguments = [
"--calculator=namd_matrix_calculator.py",
"--replica_id=" + str(replica.id),
"--replica_cycle=" + str(replica.cycle - 1),
"--replicas=" + str(self.replicas),
"--replica_basename=" + str(basename)
]
k.upload_input_data = "namd_matrix_calculator.py"
k.download_output_data = matrix_col
return k
def element_comparison(self, elements1, elements2):
"""In the comparison, we take the previously generated modified trajectory
and perform a Hausdorff distance calculation between all the unique pairs
of trajectories
"""
input_filenames1 = ["traj_flat%d.npz.npy" % (el1) for el1 in elements1]
input_filenames2 = ["traj_flat%d.npz.npy" % (el2) for el2 in elements2]
output_filename = "comparison-%03d-%03d.dat" % (elements1[0],
elements2[0])
print "Element Comparison {0} - {1}".format(elements1, elements2)
k = Kernel(name="my.hausdorff")
k.arguments = [
"--dist_file=hausdorff_kernel.py",
"--inputfile1={0}".format(input_filenames1),
"--inputfile2={0}".format(input_filenames2),
"--outputfile={0}".format(output_filename)
]
k.upload_input_data = ["hausdorff_kernel.py"]
# If the input data are in in a web server use the following
# k.download_input_data = ["/<PATH>/<TO>/<WEB>?<SERVER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"]
# If the input data are in a folder to the target machine use the following
# k.link_input_data = ["/<PATH>/<TO>/<FOLDER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"]
# The result files comparison-x-y.dat are downloaded.
k.download_output_data = output_filename
print "Element Comparison Finished {0} - {1}".format(
elements1, elements2)
return k
def analysis_stage(self, iteration, instance):
"""In the analysis stage, we take the previously generated simulation
output and perform a Levenshtein distance calculation between it
and the 'reference' file.
..note:: The placeholder ``$PRE_LOOP`` used in ``link_input_data`` is
a reference to the working directory of pre_loop.
The placeholder ``$PREV_SIMULATION`` used in ``link_input_data``
is a reference to the working directory of the previous
simulation stage.
It is also possible to reference a specific
simulation stage using ``$SIMULATION_N`` or all simulations
via ``$SIMULATIONS``. Analogous placeholders exist for
``ANALYSIS``.
"""
input_filename = "simulation-{0}-{1}.dat".format(iteration, instance)
output_filename = "analysis-{0}-{1}.dat".format(iteration, instance)
k = Kernel(name="misc.levenshtein")
k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"]
k.arguments = ["--inputfile1=reference.dat",
"--inputfile2={0}".format(input_filename),
"--outputfile={0}".format(output_filename)]
k.download_output_data = output_filename
return k
def simulation_stage(self, iteration, instance):
"""In the simulation stage we simply create files with 1000 characters.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=asciifile-{0}.dat".format(instance)]
k.download_output_data = ['asciifile-{0}.dat > iter{1}/asciifile-{0}.dat'.format(instance,iteration)]
return [k]
def prepare_replica_for_md(self, replica):
"""Specifies input and output files and passes them to kernel
Arguments:
replica - object representing a given replica and it's associated
parameters
"""
input_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".md"
output_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".out"
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=" + \
input_name + " " + \
self.sh_file, "--outputfile=" + \
output_name]
# no need to specify shared data here
# everything in shared_files list will be staged in
k.upload_input_data = [input_name]
k.download_output_data = output_name
replica.cycle = replica.cycle + 1
return k
def prepare_replica_for_exchange(self, replica):
"""Prepares md.re_exchange kernel to launch namd_matrix_calculator.py
script on target resource in order to populate columns of swap matrix.
Arguments:
replica - object representing a given replica and it's attributes
Returns:
k - an instance of Kernel class
"""
basename = self.inp_basename[:-5]
matrix_col = "matrix_column_{cycle}_{replica}.dat"\
.format(cycle=replica.cycle-1, replica=replica.id )
k = Kernel(name="md.re_exchange")
k.arguments = ["--calculator=namd_matrix_calculator.py",
"--replica_id=" + str(replica.id),
"--replica_cycle=" + str(replica.cycle-1),
"--replicas=" + str(self.replicas),
"--replica_basename=" + str(basename)]
k.upload_input_data = "namd_matrix_calculator.py"
k.download_output_data = matrix_col
return k
def simulation_step(self, iteration, instance):
"""In the simulation step we simply create files with 1000 characters.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=asciifile-{0}.dat".format(instance)]
k.download_output_data = ['asciifile-{0}.dat > iter{1}/asciifile-{0}.dat'.format(instance,iteration)]
return [k]
def prepare_replica_for_md(self, replica):
"""Specifies input and output files and passes them to kernel
Arguments:
replica - object representing a given replica and it's associated
parameters
"""
input_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".md"
output_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".out"
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=" + \
input_name + " " + \
self.sh_file, "--outputfile=" + \
output_name]
# no need to specify shared data here
# everything in shared_files list will be staged in
k.upload_input_data = [input_name]
k.download_output_data = output_name
replica.cycle = replica.cycle + 1
return k
def simulation_step(self, iteration, instance):
'''
function : if iteration = 1, use .crd file from pre_loop, else use .crd output from analysis generated
in the previous iteration. Perform amber on the .crd files to generate a set of .ncdf files.
amber :-
Purpose : Run amber on each of the coordinate files. Currently, a non-MPI version of Amber is used.
Generates a .ncdf file in each instance.
Arguments : --mininfile = minimization filename
--mdinfile = MD input filename
--topfile = Topology filename
--cycle = current iteration number
'''
k = Kernel(name="md.amber")
k.arguments = ["--mininfile={0}".format(os.path.basename(Kconfig.minimization_input_file)),
"--mdinfile={0}".format(os.path.basename(Kconfig.md_input_file)),
"--topfile={0}".format(os.path.basename(Kconfig.top_file)),
"--cycle=%s"%(iteration)]
k.link_input_data = ['$PRE_LOOP/{0}'.format(os.path.basename(Kconfig.minimization_input_file)),
'$PRE_LOOP/{0}'.format(os.path.basename(Kconfig.top_file)),
'$PRE_LOOP/{0}'.format(os.path.basename(Kconfig.md_input_file))]
if((iteration-1)==0):
k.link_input_data = k.link_input_data + ['$PRE_LOOP/{0} > min1.crd'.format(os.path.basename(Kconfig.initial_crd_file))]
else:
k.link_input_data = k.link_input_data + ['$PREV_ANALYSIS_INSTANCE_1/min{0}{1}.crd > min{2}.crd'.format(iteration-1,instance-1,iteration)]
if(iteration%Kconfig.nsave==0):
k.download_output_data = ['md{0}.ncdf > backup/iter{0}/md_{0}_{1}.ncdf'.format(iteration,instance)]
return k
def step_1(self, instance):
k = Kernel(name="misc.chksum")
k.arguments = ["--inputfile=UTF-8-demo.txt", "--outputfile=checksum{0}.sha1".format(instance)]
k.download_input_data = "htpttpt://malformed.url"
k.download_output_data = "checksum{0}.sha1".format(instance)
return k
def simulation_step(self, iteration, instance):
'''
function : In iter=1, use the input files from pre_loop, else use the outputs of the analysis stage in the
previous iteration. Run gromacs in each instance using these files.
gromacs :-
Purpose : Run the gromacs simulation on each of the smaller files. Parameter files and executables are input
from pre_loop. There are 'numCUs' number of instances of gromacs per iteration.
Arguments : --grompp = gromacs parameters filename
--topol = topology filename
'''
gromacs = Kernel(name="md.gromacs")
gromacs.arguments = ["--grompp={0}".format(os.path.basename(Kconfig.mdp_file)),
"--topol={0}".format(os.path.basename(Kconfig.top_file))]
gromacs.link_input_data = ['$PRE_LOOP/{0} > {0}'.format(os.path.basename(Kconfig.mdp_file)),
'$PRE_LOOP/{0} > {0}'.format(os.path.basename(Kconfig.top_file)),
'$PRE_LOOP/run.py > run.py']
#gromacs.copy_output_data = ['out{0}.gro > $PRE_LOOP/out{0}.gro'.format(instance-1)]
gromacs.download_output_data = ['md.log > md_{0}.log'.format(instance)]
if (iteration-1==0):
gromacs.link_input_data.append('$PRE_LOOP/temp/start{0}.gro > start.gro'.format(instance-1))
else:
gromacs.link_input_data.append('$ANALYSIS_ITERATION_{0}_INSTANCE_1/temp/start{1}.gro > start.gro'.format(iteration-1,instance-1))
return gromacs
def step_1(self, instance):
"""The first step of the pipeline creates a 1 MB ASCI file.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000000", "--filename=asciifile-{0}.dat".format(instance)]
k.download_output_data = ['asciifile-{0}.dat'.format(instance)]
return k
def pre_loop(self):
k = Kernel(name="misc.chksum")
k.arguments = ["--inputfile=UTF-8-demo.txt", "--outputfile=checksum.sha1"]
k.download_input_data = "htpttpt://malformed.url"
k.download_output_data = "checksum.sha1"
return k
def element_comparison(self, element1, element2):
"""In the comparison, we take the previously generated files
and perform a difference between those files. Each file coresponds to
an elements of the set.
"""
input_filename1 = "asciifile-{0}.dat".format(element1)
input_filename2 = "asciifile-{0}.dat".format(element2)
output_filename = "comparison-{0}-{1}.log".format(element1, element2)
print "Comparing {0} with {1}. Saving result in {2}".format(
input_filename1, input_filename2, output_filename)
# Compare the previously generated files with the misc.diff kernel and
# write the result of each comparison to a specific output file.
k = Kernel(name="misc.diff")
k.arguments = [
"--inputfile1={0}".format(input_filename1),
"--inputfile2={0}".format(input_filename2),
"--outputfile={0}".format(output_filename)
]
# Download the result files.
k.download_output_data = output_filename
return k
def stage_2(self, instances): k = Kernel(name="misc.cat") k.upload_input_data = ['./output_file_2.txt > file2.txt'] k.copy_input_data = ['$STAGE_1/temp.txt > file1.txt'] k.arguments = ["--file1=file1.txt","--file2=file2.txt"] k.download_output_data = ['./file1.txt > output_file.txt'] return k
def step_1(self, instance):
k = Kernel(name="misc.hello")
k.upload_input_data = ['./input_file.txt > temp.txt']
k.arguments = ["--file=temp.txt"]
k.download_output_data = [
'./temp.txt > output_file_{0}.txt'.format(instance)
]
return k
def step_2(self, instances):
k = Kernel(name="misc.cat")
k.upload_input_data = ['./output_file_2.txt > file2.txt']
k.copy_input_data = ['$STEP_1/temp.txt > file1.txt']
k.arguments = ["--file1=file1.txt", "--file2=file2.txt"]
k.download_output_data = ['./file1.txt > output_file.txt']
return k
def step_1(self, instance):
k = Kernel(name="misc.hello")
k.upload_input_data = ['./input_file.txt > temp.txt']
k.arguments = ["--file=temp.txt"]
k.download_output_data = [
'./temp.txt > /var/lib/jenkins/workspace/EnsembleMDTesting/temp_results/remote_file.txt'
]
return k
def step_1(self, instance):
k = Kernel(name="misc.chksum")
k.arguments = [
"--inputfile=UTF-8-demo.txt",
"--outputfile=checksum{0}.sha1".format(instance)
]
k.download_input_data = "http://testing.saga-project.org/cybertools/UTF-8-demo.txt"
k.download_output_data = "checksum{0}.sha1".format(instance)
def stage_2(self, instance):
k = Kernel(name="misc.ccount")
k.arguments = [
"--inputfile=asciifile-{0}.dat".format(instance),
"--outputfile=cfreqs-{0}.dat".format(instance)
]
k.copy_input_data = "$STAGE_1/asciifile-{0}.dat".format(instance)
k.download_output_data = "cfreqs-{0}.dat".format(instance)
return k
def step_1(self, instance):
k = Kernel(name="misc.chksum")
k.arguments = [
"--inputfile={0}".format(self._checksum_inputfile),
"--outputfile={0}".format(self._download_output)
]
k.upload_input_data = self._upload_directives
k.download_output_data = self._download_output
return k
def simulation_step(self, iteration, instance):
"""In the simulation step we
"""
k = Kernel(name="misc.mkfile")
k.arguments = [
"--size=1000", "--filename=asciifile-{0}.dat".format(instance)
]
k.download_output_data = ['asciifile-{0}.dat'.format(instance)]
return [k]
def step_1(self, instance):
k = Kernel(name="misc.chksum")
k.arguments = [
"--inputfile=UTF-8-demo.txt",
"--outputfile=checksum{0}.sha1".format(instance)
]
k.download_input_data = "htpttpt://malformed.url"
k.download_output_data = "checksum{0}.sha1".format(instance)
return k
def step_3(self, instance):
"""The third step of the pipeline creates a checksum of the output file
of the second step. The result is transferred back to the host
running this script.
"""
k = Kernel(name="misc.chksum")
k.arguments = ["--inputfile=cfreqs-{0}.dat".format(instance), "--outputfile=cfreqs-{0}.sha1".format(instance)]
k.link_input_data = "$STEP_2/cfreqs-{0}.dat".format(instance)
k.download_output_data = "cfreqs-{0}.sha1".format(instance)
return k
def stage_1(self, instance):
k = Kernel(name="qbird")
k.upload_input_data = ['SeaIceUnSuperGMM.py']
k.arguments = ["--script=SeaIceUnSuperGMM.py",
"--imagepath=/home/vivek91/xsede-hack/IMG/XinImages/TEST/072610_00332.jpg",
"--trainingpath=/home/vivek91/xsede-hack/IMG/XinImages/TRAINING/",
"--num_train=3",
"--icetypes=4"]
k.download_output_data = ["labelledImage3trainingImages.png > image_{0}.png".format(instance)]
return k
def step_3(self, instance):
"""The third step of the pipeline creates a checksum of the output file
of the second step. The result is transferred back to the host
running this script.
"""
k = Kernel(name="misc.chksum")
k.arguments = ["--inputfile=cfreqs-{0}.dat".format(instance), "--outputfile=cfreqs-{0}.sha1".format(instance)]
k.link_input_data = "$STEP_2/cfreqs-{0}.dat".format(instance)
k.download_output_data = "cfreqs-{0}.sha1".format(instance)
return k
def analysis_step(self, iteration, instance):
'''
function : Perform CoCo Analysis on the output of the simulation from the current iteration. Using the .ncdf
files generated in all the instance, generate the .crd file to be used in the next simulation.
coco :-
Purpose : Runs CoCo analysis on a set of .ncdf files and generates a coordinate file.
Arguments : --grid = Number of points along each dimension of the CoCo histogram
--dims = The number of projections to consider from the input pcz file
--frontpoints = Number of CUs
--topfile = Topology filename
--mdfile = MD Input filename
--output = Output filename
--cycle = Current iteration number
'''
k1 = Kernel(name="custom.coco")
k1.arguments = ["--grid={0}".format(Kconfig.grid),
"--dims={0}".format(Kconfig.dims),
"--frontpoints=64".format(Kconfig.num_CUs),
"--topfile={0}".format(os.path.basename(Kconfig.top_file)),
"--mdfile=*.ncdf",
"--output=pdbs",
"--atom_selection={0}".format(Kconfig.atom_selection)]
#k1.cores = min(Kconfig.num_CUs,RPconfig.PILOTSIZE)
k1.cores = 64
k1.uses_mpi = True
k1.link_input_data = ['$SHARED/{0}'.format(os.path.basename(Kconfig.top_file))]
for iter in range(1,iteration+1):
# for i in range(1,Kconfig.num_CUs+1):
for i in range(1+(instance-1)*64, instance*64 + 1):
k1.link_input_data = k1.link_input_data + ['$SIMULATION_ITERATION_{0}_INSTANCE_{1}/md{0}.ncdf > md_{0}_{1}.ncdf'.format(iter,i)]
k1.copy_output_data = list()
for i in range(0,64):
k1.copy_output_data = k1.copy_output_data + ['pdbs{1}.pdb > $SHARED/pentaopt{0}{2}.pdb'.format(iteration,i,(instance-1)*64+i)]
if(iteration%Kconfig.nsave==0):
k1.download_output_data = ['coco.log > output/iter{0}/coco.log'.format(iteration,instance)]
k2 = Kernel(name="custom.tleap",instance_type='single')
k2.arguments = ["--numofsims={0}".format(Kconfig.num_CUs),
"--cycle={0}".format(iteration)]
k2.link_input_data = ['$SHARED/postexec.py > postexec.py']
for i in range(0,Kconfig.num_CUs):
k2.link_input_data = k2.link_input_data + ['$SHARED/pentaopt{0}{1}.pdb > pentaopt{0}{1}.pdb'.format(iteration,i)]
return [k1,k2]
def analysis_step(self, iteration, instance):
input_filename = "simulation-{0}-{1}.dat".format(iteration, instance)
output_filename = "analysis-{0}-{1}.dat".format(iteration, instance)
k = Kernel(name="misc.levenshtein")
k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"]
k.arguments = ["--inputfile1=reference.dat",
"--inputfile2={0}".format(input_filename),
"--outputfile={0}".format(output_filename)]
k.download_output_data = output_filename
return k
def analysis_step(self, iteration, instance):
"""In the analysis step we use the ``$PREV_SIMULATION`` data reference
to refer to the previous simulation. The same
instance is picked implicitly, i.e., if this is instance 5, the
previous simulation with instance 5 is referenced.
"""
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=asciifile.dat", "--outputfile=cfreqs.dat"]
k.link_input_data = "$PREV_SIMULATION/asciifile.dat".format(instance=instance)
k.download_output_data = "cfreqs.dat > cfreqs-{iteration}-{instance}.dat".format(instance=instance, iteration=iteration)
return k
def analysis_step(self, iteration, instance):
"""In the analysis step we use the ``$PREV_SIMULATION`` data reference
to refer to the previous simulation. The same
instance is picked implicitly, i.e., if this is instance 5, the
previous simulation with instance 5 is referenced.
"""
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=asciifile.dat", "--outputfile=cfreqs.dat"]
k.link_input_data = "$PREV_SIMULATION/asciifile.dat".format(instance=instance)
k.download_output_data = "cfreqs.dat > cfreqs-{iteration}-{instance}.dat".format(instance=instance, iteration=iteration)
return k
def analysis_step(self, iteration, instance):
input_filename = "simulation-{0}-{1}.dat".format(iteration, instance)
output_filename = "analysis-{0}-{1}.dat".format(iteration, instance)
k = Kernel(name="misc.levenshtein")
k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"]
k.arguments = ["--inputfile1=reference.dat",
"--inputfile2={0}".format(input_filename),
"--outputfile={0}".format(output_filename)]
k.download_output_data = output_filename
return k
def step_1(self, instance):
"""Run AMBER MD Simulations"""
k = Kernel(name="amber")
k.arguments = [ "--minfile=min.in",
"--topfile=penta.top",
"--crdfile=penta.crd",
"--output=md.crd"]
k.upload_input_data = ['amber_input/min.in','amber_input/penta.top','amber_input/penta.crd']
k.download_output_data = ['md.crd > amber_output/md_{0}.crd'.format(instance)]
return k
def step_1(self, instance):
"""This step downloads a sample UTF-8 file from a remote websever and
calculates the SHA1 checksum of that file. The checksum is written
to an output file and tranferred back to the host running this
script.
"""
k = Kernel(name="misc.chksum")
k.arguments = ["--inputfile=UTF-8-demo.txt", "--outputfile=checksum{0}.sha1".format(instance)]
k.download_input_data = "http://gist.githubusercontent.com/oleweidner/6084b9d56b04389717b9/raw/611dd0c184be5f35d75f876b13604c86c470872f/gistfile1.txt > UTF-8-demo.txt"
k.download_output_data = "checksum{0}.sha1".format(instance)
return k
def step_1(self, instance):
"""This step downloads a sample UTF-8 file from a remote websever and
calculates the SHA1 checksum of that file. The checksum is written
to an output file and tranferred back to the host running this
script.
"""
k = Kernel(name="misc.chksum")
k.arguments = ["--inputfile=UTF-8-demo.txt", "--outputfile=checksum{0}.sha1".format(instance)]
k.upload_input_data = "UTF-8-demo.txt"
k.download_output_data = "checksum{0}.sha1".format(instance)
return k
def analysis_step(self, iteration, instance):
'''
function : Perform CoCo Analysis on the output of the simulation from the current iteration. Using the .xtc
files generated in all instances, generate .gro files (as many as the num_CUs) to be used in the next simulations.
coco :-
Purpose : Runs CoCo analysis on a set of MD trajectory files in this case xtc files and generates several coordinates file to be
Arguments : --grid = Number of points along each dimension of the CoCo histogram
--dims = The number of projections to consider from the input pcz file
--frontpoints = Number of CUs
--topfile = Topology filename
--mdfile = MD Input filename
--output = Output filename
--cycle = Current iteration number
--atom_selection = Selection of the biological part of the system we want to consider for analysis
'''
k1_ana_kernel = Kernel(name="md.coco")
k1_ana_kernel.link_input_data = ['$PRE_LOOP/{0}'.format(os.path.basename(Kconfig.top_file)),
'$SIMULATION_ITERATION_{0}_INSTANCE_1/md-{1}_0.gro > md-{1}_0.gro'.format(iteration,iteration-1)]
for iter in range(1,iteration+1):
for i in range(1,Kconfig.num_CUs+1):
k1_ana_kernel.link_input_data = k1_ana_kernel.link_input_data + ['$SIMULATION_ITERATION_{0}_INSTANCE_{1}/md-{2}_{3}.xtc > md-{2}_{3}.xtc'.format(iter,i,iter-1,i-1)]
k1_ana_kernel.cores = 1
k1_ana_kernel.uses_mpi = False
outbase, ext = os.path.basename(Kconfig.output).split('.')
#Not sure why this if condition is required
#if ext == '':
# ext = '.pdb'
k1_ana_kernel.arguments = ["--grid={0}".format(Kconfig.grid),
"--dims={0}".format(Kconfig.dims),
"--frontpoints={0}".format(Kconfig.num_CUs),
"--topfile=md-{0}_0.gro".format(iteration-1),
"--mdfile=*.xtc",
"--output={0}_{1}.{2}".format(outbase,iteration-1,ext),
"--atom_selection={0}".format(Kconfig.sel)]
k1_ana_kernel.copy_output_data = []
for i in range(0,Kconfig.num_CUs):
k1_ana_kernel.copy_output_data += ["{0}_{1}{2}.gro > $PRE_LOOP/{0}_{1}{2}.gro".format(outbase,iteration-1,i,ext)]
k1_ana_kernel.download_output_data = ["coco.log > output/coco-iter{0}.log".format(iteration-1)]
return [k1_ana_kernel]
def analysis_step(self, iteration, instance):
link_input_data = []
for i in range(1, self.simulation_instances+1):
link_input_data.append("$PREV_SIMULATION_INSTANCE_{instance}/asciifile.dat > asciifile-{instance}.dat".format(instance=i))
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=asciifile-*.dat", "--outputfile=cfreqs.dat"]
k.link_input_data = link_input_data
k.download_output_data = "cfreqs.dat > cfreqs-{iteration}.dat".format(iteration=iteration)
return [k]
def stage_1(self, instance):
k = Kernel(name="qbird")
k.upload_input_data = ['SeaIceUnSuperGMM.py']
k.arguments = [
"--script=SeaIceUnSuperGMM.py",
"--imagepath=/home/vivek91/xsede-hack/IMG/XinImages/TEST/072610_00332.jpg",
"--trainingpath=/home/vivek91/xsede-hack/IMG/XinImages/TRAINING/",
"--num_train=3", "--icetypes=4"
]
k.download_output_data = [
"labelledImage3trainingImages.png > image_{0}.png".format(instance)
]
return k
def step_1(self, instance):
"""This step downloads a sample UTF-8 file from a remote websever and
calculates the SHA1 checksum of that file. The checksum is written
to an output file and tranferred back to the host running this
script.
"""
k = Kernel(name="misc.chksum")
k.arguments = [
"--inputfile=UTF-8-demo.txt",
"--outputfile=checksum{0}.sha1".format(instance)
]
k.download_input_data = "http://gist.githubusercontent.com/oleweidner/6084b9d56b04389717b9/raw/611dd0c184be5f35d75f876b13604c86c470872f/gistfile1.txt > UTF-8-demo.txt"
k.download_output_data = "checksum{0}.sha1".format(instance)
return k
def step_1(self, instance):
"""This step downloads a sample UTF-8 file from a remote websever and
calculates the SHA1 checksum of that file. The checksum is written
to an output file and tranferred back to the host running this
script.
"""
k = Kernel(name="misc.chksum")
k.arguments = [
"--inputfile=UTF-8-demo.txt",
"--outputfile=checksum{0}.sha1".format(instance)
]
k.upload_input_data = "UTF-8-demo.txt"
k.download_output_data = "checksum{0}.sha1".format(instance)
return k
def analysis_step(self, iteration, instance):
k1 = Kernel(name="custom.concat")
k1.arguments = ["--format=dat", "--filename=output.dat"]
k1.download_output_data = "output.dat"
link_input_data = []
for i in range(1, self.simulation_instances + 1):
link_input_data.append(
"$PREV_SIMULATION_INSTANCE_{instance}/asciifile.dat > asciifile-{instance}.dat".format(instance=i)
)
k1.link_input_data = link_input_data
return [k1]
def step_2(self, instance):
"""The second step of the pipeline does a character frequency analysis
on the file generated the first step. The result is transferred back
to the host running this script.
..note:: The placeholder ``$STEP_1`` used in ``link_input_data`` is
a reference to the working directory of step 1. ``$STEP_``
can be used analogous to refernce other steps.
"""
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=asciifile-{0}.dat".format(instance), "--outputfile=cfreqs-{0}.dat".format(instance)]
k.link_input_data = "$STEP_1/asciifile-{0}.dat".format(instance)
k.download_output_data = "cfreqs-{0}.dat".format(instance)
return k
def step_2(self, instance):
"""The second step of the pipeline does a character frequency analysis
on the file generated the first step. The result is transferred back
to the host running this script.
..note:: The placeholder ``$STEP_1`` used in ``link_input_data`` is
a reference to the working directory of step 1. ``$STEP_``
can be used analogous to refernce other steps.
"""
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=asciifile-{0}.dat".format(instance), "--outputfile=cfreqs-{0}.dat".format(instance)]
k.link_input_data = "$STEP_1/asciifile-{0}.dat".format(instance)
k.download_output_data = "cfreqs-{0}.dat".format(instance)
return k
def analysis_step(self, iteration, instance):
link_input_data = []
for i in range(1, self.simulation_instances + 1):
link_input_data.append(
"$PREV_SIMULATION_INSTANCE_{instance}/asciifile.dat > asciifile-{instance}.dat"
.format(instance=i))
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=asciifile.dat", "--outputfile=cfreqs.dat"]
k.link_input_data = link_input_data
k.download_output_data = "cfreqs.dat"
k.cores = 1
return k
def simulation_step(self, iteration, instance):
'''
function : if iteration = 1, use .crd file from pre_loop, else use .crd output from analysis generated
in the previous iteration. Perform amber on the .crd files to generate a set of .ncdf files.
amber :-
Purpose : Run amber on each of the coordinate files. Currently, a non-MPI version of Amber is used.
Generates a .ncdf file in each instance.
Arguments : --mininfile = minimization filename
--mdinfile = MD input filename
--topfile = Topology filename and/or reference coordinates file filename
--cycle = current iteration number
'''
k1 = Kernel(name="custom.amber")
k1.arguments = ["--mininfile={0}".format(os.path.basename(Kconfig.minimization_input_file)),
"--topfile={0}".format(os.path.basename(Kconfig.top_file)),
"--crdfile={0}".format(os.path.basename(Kconfig.initial_crd_file)),
"--cycle=%s"%(iteration)]
k1.link_input_data = ['$SHARED/{0}'.format(os.path.basename(Kconfig.minimization_input_file)),
'$SHARED/{0}'.format(os.path.basename(Kconfig.top_file)),
'$SHARED/{0}'.format(os.path.basename(Kconfig.initial_crd_file))]
k1.cores = Kconfig.num_cores_per_sim_cu
if((iteration-1)==0):
k1.link_input_data = k1.link_input_data + ['$SHARED/{0} > min1.rst7'.format(os.path.basename(Kconfig.initial_crd_file))]
k1.copy_output_data = ['min1.rst7 > $SHARED/md_{0}_{1}.rst'.format(iteration,instance)]
else:
k1.link_input_data = k1.link_input_data + ['$SHARED/min_{0}_{1}.rst7 > min{2}.rst7'.format(iteration-1,instance-1,iteration)]
k1.copy_output_data = ['md{0}.rst > $SHARED/md_{0}_{1}.rst'.format(iteration,instance)]
k2 = Kernel(name="custom.amber")
k2.arguments = [
"--mdinfile={0}".format(os.path.basename(Kconfig.md_input_file)),
"--topfile={0}".format(os.path.basename(Kconfig.top_file)),
"--cycle=%s"%(iteration)
]
k2.link_input_data = [
"$SHARED/{0}".format(os.path.basename(Kconfig.md_input_file)),
"$SHARED/{0}".format(os.path.basename(Kconfig.top_file)),
"$SHARED/md_{0}_{1}.rst > md{0}.rst".format(iteration,instance),
]
k2.cores = Kconfig.num_cores_per_sim_cu
if(iteration%Kconfig.nsave==0):
k2.download_output_data = ['md{0}.nc > output/iter{0}/md_{0}_{1}.nc'.format(iteration,instance)]
return [k1,k2]
def analysis_step(self, iteration, instance):
"""In the analysis step we use the ``$PREV_SIMULATION`` data reference
to refer to the previous simulation. The same
instance is picked implicitly, i.e., if this is instance 5, the
previous simulation with instance 5 is referenced.
"""
#upload_input_data = []
#for i in range(1, self.simlation_instances+1):
# upload_input_data.append("asciifile-{0}.dat".format(i))
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=asciifile-*.dat", "--outputfile=cfreqs.dat"]
k.upload_input_data = ['asciifile-{0}.dat'.format(instance)]
k.download_output_data = ["cfreqs.dat > cfreqs-{0}.dat".format(instance)]
return [k]
def step_01(self, instance):
"""This steps does the trajectory simulation. First, the input files
are downloaded from a remote HTTP server. Next, NAMD reads the 'inp'
("input") file and simulates the trajectory. Finally, the output is
transferred back to the machine on which this script is executed.
"""
namd = Kernel(name="md.namd")
namd.core = 4
namd.arguments = ["eq{0}.inp".format(instance)]
namd.download_input_data = ["http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/complex.pdb > complex.pdb",
"http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/complex.top > complex.top",
"http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/cons.pdb > cons.pdb",
"http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/eq0.inp > eq{0}.inp".format(instance)]
namd.download_output_data = "STDOUT > eq{0}.out".format(instance)
return namd
def analysis_step(self, iteration, instance):
"""In the analysis step we use the ``$PREV_SIMULATION`` data reference
to refer to the previous simulation. The same
instance is picked implicitly, i.e., if this is instance 5, the
previous simulation with instance 5 is referenced.
"""
link_input_data = []
for i in range(1, self.simlation_instances+1):
link_input_data.append("$PREV_SIMULATION_INSTANCE_{instance}/asciifile.dat > asciifile-{instance}.dat".format(instance=i))
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=asciifile-*.dat", "--outputfile=cfreqs.dat"]
k.link_input_data = link_input_data
k.download_output_data = "cfreqs.dat > cfreqs-{iteration}.dat".format(iteration=iteration)
return [k]
def step_1(self, instance):
"""Run AMBER MD Simulations"""
k = Kernel(name="amber")
k.arguments = [
"--minfile=min.in", "--topfile=penta.top", "--crdfile=penta.crd",
"--output=md.crd"
]
k.upload_input_data = [
'amber_input/min.in', 'amber_input/penta.top',
'amber_input/penta.crd'
]
k.download_output_data = [
'md.crd > amber_output/md_{0}.crd'.format(instance)
]
return k
def step_1(self, instance):
input_filename = "simulation-1-{0}.dat".format(instance)
output_filename = "analysis-1-{0}.dat".format(instance)
k = Kernel(name="misc.levenshtein")
k.link_input_data = [
"/home/suvigya/radical.ensemblemd-master/tests/tests/E2E_test/input_files/reference.dat",
"/home/suvigya/radical.ensemblemd-master/tests/tests/E2E_result/input_files/simulation-1-1.dat",
"/home/suvigya/radical.ensemblemd-master/tests/tests/E2E_test/levenshtein.py"
]
k.arguments = [
"--inputfile1=reference.dat",
"--inputfile2={0}".format(input_filename),
"--outputfile={0}".format(output_filename)
]
k.download_output_data = output_filename
return k
def element_comparison(self, elements1, elements2):
input_filename1 = "asciifile_{0}.dat".format(elements1[0])
input_filename2 = "newfile_{0}.dat".format(elements2[0])
output_filename = "comparison_{0}_{1}.log".format(elements1[0], elements2[0])
print "\nComparing {0} with {1}. Saving result in {2}".format(input_filename1,input_filename2,output_filename)
# Compare the previously generated files with the misc.diff kernel and
# write the result of each comparison to a specific output file.
k = Kernel(name="misc.diff")
k.arguments = ["--inputfile1={0}".format(input_filename1),
"--inputfile2={0}".format(input_filename2),
"--outputfile={0}".format(output_filename)]
# Download the result files.
k.download_output_data = output_filename
return k
def prepare_replica_for_md(self, replica):
input_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".md"
output_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".out"
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=" + input_name,
"--outputfile=" + output_name]
k.upload_input_data = input_name
k.download_output_data = output_name
k.cores = 1
replica.cycle = replica.cycle + 1
return k
def prepare_replica_for_md(self, replica):
input_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".md"
output_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".out"
k = Kernel(name="misc.ccount")
k.arguments = [
"--inputfile=" + input_name, "--outputfile=" + output_name
]
k.upload_input_data = input_name
k.download_output_data = output_name
k.cores = 1
replica.cycle = replica.cycle + 1
return k
def step_01(self, instance):
"""This steps does the trajectory simulation. First, the input files
are downloaded from a remote HTTP server. Next, NAMD reads the 'inp'
("input") file and simulates the trajectory. Finally, the output is
transferred back to the machine on which this script is executed.
"""
namd = Kernel(name="md.namd")
namd.core = 4
namd.arguments = ["eq{0}.inp".format(instance)]
namd.download_input_data = [
"http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/complex.pdb > complex.pdb",
"http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/complex.top > complex.top",
"http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/cons.pdb > cons.pdb",
"http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/eq0.inp > eq{0}.inp"
.format(instance)
]
namd.download_output_data = "STDOUT > eq{0}.out".format(instance)
return namd
def analysis_step(self, iteration, instance):
"""In the analysis step we use the ``$PREV_SIMULATION`` data reference
to refer to the previous simulation. The same
instance is picked implicitly, i.e., if this is instance 5, the
previous simulation with instance 5 is referenced.
"""
link_input_data = []
for i in range(1, self.simlation_instances + 1):
link_input_data.append(
"$PREV_SIMULATION_INSTANCE_{instance}/asciifile.dat > asciifile-{instance}.dat"
.format(instance=i))
k = Kernel(name="misc.ccount")
k.arguments = [
"--inputfile=asciifile-*.dat", "--outputfile=cfreqs.dat"
]
k.link_input_data = link_input_data
k.download_output_data = "cfreqs.dat > cfreqs-{iteration}.dat".format(
iteration=iteration)
return [k]
