def simulation_step(self, iteration, instance):
"""In the simulation step we
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=asciifile.dat"]
k.exists_remote = ['asciifile.dat']
return [k]
def prepare_replica_for_md(self, replica):
"""Specifies input and output files and passes them to kernel
Arguments:
replica - object representing a given replica and it's associated
parameters
"""
input_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".md"
output_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".out"
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=" + \
input_name + " " + \
self.sh_file, "--outputfile=" + \
output_name]
# no need to specify shared data here
# everything in shared_files list will be staged in
k.upload_input_data = [input_name]
k.download_output_data = output_name
replica.cycle = replica.cycle + 1
return k
def step_1(self, instance):
"""The first step of the pipeline creates a 1 MB ASCI file.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000000", "--filename=asciifile-{0}.dat".format(instance)]
k.download_output_data = ['asciifile-{0}.dat'.format(instance)]
return k
def simulation_step(self, iteration, instance):
'''
function : In iter=1, use the input files from pre_loop, else use the outputs of the analysis stage in the
previous iteration. Run gromacs in each instance using these files.
gromacs :-
Purpose : Run the gromacs simulation on each of the smaller files. Parameter files and executables are input
from pre_loop. There are 'numCUs' number of instances of gromacs per iteration.
Arguments : --grompp = gromacs parameters filename
--topol = topology filename
'''
gromacs = Kernel(name="md.gromacs")
gromacs.arguments = [
"--grompp={0}".format(os.path.basename(Kconfig.mdp_file)),
"--topol={0}".format(os.path.basename(Kconfig.top_file))
]
gromacs.link_input_data = [
'$PRE_LOOP/{0} > {0}'.format(os.path.basename(Kconfig.mdp_file)),
'$PRE_LOOP/{0} > {0}'.format(os.path.basename(Kconfig.top_file)),
'$PRE_LOOP/run.py > run.py'
]
if (iteration - 1 == 0):
gromacs.link_input_data.append(
'$PRE_LOOP/temp/start{0}.gro > start.gro'.format(instance - 1))
else:
gromacs.link_input_data.append(
'$ANALYSIS_ITERATION_{0}_INSTANCE_1/temp/start{1}.gro > start.gro'
.format(iteration - 1, instance - 1))
return gromacs
def element_comparison(self, elements1, elements2):
"""In the comparison, we take the previously generated modified trajectory
and perform a Hausdorff distance calculation between all the unique pairs
of trajectories
"""
input_filenames1 = ["traj_flat%d.npz.npy" % (el1) for el1 in elements1]
input_filenames2 = ["traj_flat%d.npz.npy" % (el2) for el2 in elements2]
output_filename = "comparison-%03d-%03d.dat" % (elements1[0],
elements2[0])
print "Element Comparison {0} - {1}".format(elements1, elements2)
k = Kernel(name="my.hausdorff")
k.arguments = [
"--dist_file=hausdorff_kernel.py",
"--inputfile1={0}".format(input_filenames1),
"--inputfile2={0}".format(input_filenames2),
"--outputfile={0}".format(output_filename)
]
k.upload_input_data = ["hausdorff_kernel.py"]
# If the input data are in in a web server use the following
# k.download_input_data = ["/<PATH>/<TO>/<WEB>?<SERVER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"]
# If the input data are in a folder to the target machine use the following
# k.link_input_data = ["/<PATH>/<TO>/<FOLDER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"]
# The result files comparison-x-y.dat are downloaded.
k.download_output_data = output_filename
print "Element Comparison Finished {0} - {1}".format(
elements1, elements2)
return k
def simulation_step(self, iteration, instance):
"""In the simulation step we simply create files with 1000 characters.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=asciifile-{0}.dat".format(instance)]
k.download_output_data = ['asciifile-{0}.dat > iter{1}/asciifile-{0}.dat'.format(instance,iteration)]
return [k]
def pre_loop(self):
'''
function : transfers input files, intermediate executables
pre_coam_loop :-
Purpose : Transfers files that will be used for this workflow
in the remote machine. This is a kernel that can be used from
all workflows for transfering data files.
Arguments : None
'''
k = Kernel(name="md.pre_coam_loop")
k.upload_input_data = [Kconfig.initial_crd_file,
Kconfig.md_input_file,
Kconfig.top_file,
Kconfig.itp_file,
Kconfig.restr_file,
Kconfig.eminrestr_md,
Kconfig.eeqrestr_md,
'{0}/postprocessing.py'.format(Kconfig.misc_loc)]
outbase, ext = os.path.basename(Kconfig.output).split('.')
return k
def pre_loop(self):
'''
function : transfers input files and intermediate executables
pre_grlsd_loop :-
Purpose : Transfers files, Split the input file into smaller files to be used by each of the
gromacs instances in the first iteration.
Arguments : --inputfile = file to be split
--numCUs = number of simulation instances/ number of smaller files
'''
k = Kernel(name="md.pre_grlsd_loop")
k.upload_input_data = [Kconfig.md_input_file,
Kconfig.lsdm_config_file,
Kconfig.top_file,
Kconfig.mdp_file,
'{0}/spliter.py'.format(Kconfig.misc_loc),
'{0}/gro.py'.format(Kconfig.misc_loc),
'{0}/run.py'.format(Kconfig.misc_loc),
'{0}/pre_analyze.py'.format(Kconfig.misc_loc),
'{0}/post_analyze.py'.format(Kconfig.misc_loc),
'{0}/selection.py'.format(Kconfig.misc_loc),
'{0}/reweighting.py'.format(Kconfig.misc_loc)]
k.arguments = ["--inputfile={0}".format(os.path.basename(Kconfig.md_input_file)),"--numCUs={0}".format(Kconfig.num_CUs)]
return k
def analysis_step(self, iteration, instance):
'''
function : Perform CoCo Analysis on the output of the simulation from the current iteration. Using the .ncdf
files generated in all the instance, generate the .crd file to be used in the next simulation.
coco :-
Purpose : Runs CoCo analysis on a set of .ncdf files and generates a coordinate file.
Arguments : --grid = Number of points along each dimension of the CoCo histogram
--dims = The number of projections to consider from the input pcz file
--frontpoints = Number of CUs
--topfile = Topology filename
--mdfile = MD Input filename
--output = Output filename
--cycle = Current iteration number
'''
k = Kernel(name="md.coco")
k.arguments = ["--grid={0}".format(Kconfig.grid),
"--dims={0}".format(Kconfig.dims),
"--frontpoints={0}".format(Kconfig.num_CUs),
"--topfile={0}".format(os.path.basename(Kconfig.top_file)),
"--mdfile=*.ncdf",
"--output=pentaopt%s"%(iteration),
"--cycle=%s"%(iteration)]
k.link_input_data = ['$PRE_LOOP/{0}'.format(os.path.basename(Kconfig.top_file)),
'$PRE_LOOP/postexec.py']
k.cores = RPconfig.PILOTSIZE
for iter in range(1,iteration+1):
for i in range(1,Kconfig.num_CUs+1):
k.link_input_data = k.link_input_data + ['$SIMULATION_ITERATION_{0}_INSTANCE_{1}/md{0}.ncdf > md_{0}_{1}.ncdf'.format(iter,i)]
return k
def simulation_step(self, iteration, instance):
"""
function : In iter=1, use the input files from pre_loop, else use the outputs of the analysis stage in the
previous iteration. Run gromacs in each instance using these files.
gromacs :-
Purpose : Run the gromacs simulation on each of the smaller files. Parameter files and executables are input
from pre_loop. There are 'numCUs' number of instances of gromacs per iteration.
Arguments : --grompp = gromacs parameters filename
--topol = topology filename
"""
gromacs = Kernel(name="md.gromacs")
gromacs.arguments = [
"--grompp={0}".format(os.path.basename(Kconfig.mdp_file)),
"--topol={0}".format(os.path.basename(Kconfig.top_file)),
]
gromacs.link_input_data = [
"$PRE_LOOP/{0} > {0}".format(os.path.basename(Kconfig.mdp_file)),
"$PRE_LOOP/{0} > {0}".format(os.path.basename(Kconfig.top_file)),
"$PRE_LOOP/run.py > run.py",
]
if iteration - 1 == 0:
gromacs.link_input_data.append("$PRE_LOOP/temp/start{0}.gro > start.gro".format(instance - 1))
else:
gromacs.link_input_data.append(
"$ANALYSIS_ITERATION_{0}_INSTANCE_1/temp/start{1}.gro > start.gro".format(iteration - 1, instance - 1)
)
return gromacs
def simulation_stage(self, iteration, instance):
"""In the simulation stage we simply create files with 1000 characters.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=asciifile-{0}.dat".format(instance)]
k.download_output_data = ['asciifile-{0}.dat > iter{1}/asciifile-{0}.dat'.format(instance,iteration)]
return [k]
def prepare_replica_for_md(self, replica):
"""Specifies input and output files and passes them to kernel
Arguments:
replica - object representing a given replica and it's associated
parameters
"""
input_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".md"
output_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".out"
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=" + \
input_name + " " + \
self.sh_file, "--outputfile=" + \
output_name]
# no need to specify shared data here
# everything in shared_files list will be staged in
k.upload_input_data = [input_name]
k.download_output_data = output_name
replica.cycle = replica.cycle + 1
return k
def set2element_initialization(self,element):
# Creating an ASCII file by using the misc.mkfile kernel. Each file represents
# a element of the set.
print "Creating Element {0}".format(element)
k = Kernel(name = "misc.mkfile")
k.arguments = ["--size=10000", "--filename=newfile_{0}.dat".format(element)]
return k
def step_1(self, instance):
k = Kernel(name="misc.chksum")
k.arguments = ["--inputfile=UTF-8-demo.txt", "--outputfile=checksum{0}.sha1".format(instance)]
k.download_input_data = "htpttpt://malformed.url"
k.download_output_data = "checksum{0}.sha1".format(instance)
return k
def simulation_step(self, iteration, instance):
"""In the simulation step we
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=asciifile-{0}.dat".format(instance)]
k.download_output_data = ['asciifile-{0}.dat'.format(instance)]
return [k]
def analysis_stage(self, iteration, instance):
"""In the analysis stage, we take the previously generated simulation
output and perform a Levenshtein distance calculation between it
and the 'reference' file.
..note:: The placeholder ``$PRE_LOOP`` used in ``link_input_data`` is
a reference to the working directory of pre_loop.
The placeholder ``$PREV_SIMULATION`` used in ``link_input_data``
is a reference to the working directory of the previous
simulation stage.
It is also possible to reference a specific
simulation stage using ``$SIMULATION_N`` or all simulations
via ``$SIMULATIONS``. Analogous placeholders exist for
``ANALYSIS``.
"""
input_filename = "simulation-{0}-{1}.dat".format(iteration, instance)
output_filename = "analysis-{0}-{1}.dat".format(iteration, instance)
k = Kernel(name="misc.levenshtein")
k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"]
k.arguments = ["--inputfile1=reference.dat",
"--inputfile2={0}".format(input_filename),
"--outputfile={0}".format(output_filename)]
k.download_output_data = output_filename
return k
def stage_1(self, instance):
k = Kernel(name="synapse.sample")
k.arguments = [
"--path=$HOME/ves/synapse_local", "--mode=sample", "--flops=1000",
"--samples=1"
]
return k
def element_comparison(self, element1, element2):
"""In the comparison, we take the previously generated files
and perform a difference between those files. Each file coresponds to
an elements of the set.
"""
input_filename1 = "asciifile-{0}.dat".format(element1)
input_filename2 = "asciifile-{0}.dat".format(element2)
output_filename = "comparison-{0}-{1}.log".format(element1, element2)
print "Comparing {0} with {1}. Saving result in {2}".format(
input_filename1, input_filename2, output_filename)
# Compare the previously generated files with the misc.diff kernel and
# write the result of each comparison to a specific output file.
k = Kernel(name="misc.diff")
k.arguments = [
"--inputfile1={0}".format(input_filename1),
"--inputfile2={0}".format(input_filename2),
"--outputfile={0}".format(output_filename)
]
# Download the result files.
k.download_output_data = output_filename
return k
def pre_loop(self):
'''
function : transfers input files and intermediate executables
pre_grlsd_loop :-
Purpose : Transfers files, Split the input file into smaller files to be used by each of the
gromacs instances in the first iteration.
Arguments : --inputfile = file to be split
--numCUs = number of simulation instances/ number of smaller files
'''
k = Kernel(name="md.pre_grlsd_loop")
k.upload_input_data = [
Kconfig.md_input_file, Kconfig.lsdm_config_file, Kconfig.top_file,
Kconfig.mdp_file, '{0}/spliter.py'.format(Kconfig.misc_loc),
'{0}/gro.py'.format(Kconfig.misc_loc),
'{0}/run.py'.format(Kconfig.misc_loc),
'{0}/pre_analyze.py'.format(Kconfig.misc_loc),
'{0}/post_analyze.py'.format(Kconfig.misc_loc),
'{0}/selection.py'.format(Kconfig.misc_loc),
'{0}/reweighting.py'.format(Kconfig.misc_loc)
]
k.arguments = [
"--inputfile={0}".format(os.path.basename(Kconfig.md_input_file)),
"--numCUs={0}".format(Kconfig.num_CUs)
]
return k
def simulation_step(self, iteration, instance):
k = Kernel(name="misc.mkfile")
k.arguments = [
"--size=1000",
"--filename=simulation-{0}-{1}.dat".format(iteration, instance)
]
return k
def prepare_replica_for_exchange(self, replica):
"""Prepares md.re_exchange kernel to launch namd_matrix_calculator.py
script on target resource in order to populate columns of swap matrix.
Arguments:
replica - object representing a given replica and it's attributes
Returns:
k - an instance of Kernel class
"""
basename = self.inp_basename[:-5]
matrix_col = "matrix_column_{cycle}_{replica}.dat"\
.format(cycle=replica.cycle-1, replica=replica.id )
k = Kernel(name="md.re_exchange")
k.arguments = ["--calculator=namd_matrix_calculator.py",
"--replica_id=" + str(replica.id),
"--replica_cycle=" + str(replica.cycle-1),
"--replicas=" + str(self.replicas),
"--replica_basename=" + str(basename)]
k.upload_input_data = "namd_matrix_calculator.py"
k.download_output_data = matrix_col
return k
def element_comparison(self, elements1, elements2):
"""In the comparison, we take the previously generated modified trajectory
and perform a Hausdorff distance calculation between all the unique pairs
of trajectories
"""
input_filenames1 = ["traj_flat%d.npz.npy"%(el1) for el1 in elements1]
input_filenames2 = ["traj_flat%d.npz.npy"%(el2) for el2 in elements2]
output_filename = "comparison-%03d-%03d.dat"%(elements1[0],elements2[0])
print "Element Comparison {0} - {1}".format(elements1,elements2)
k = Kernel(name="my.hausdorff")
k.arguments = ["--dist_file=hausdorff_kernel.py",
"--inputfile1={0}".format(input_filenames1),
"--inputfile2={0}".format(input_filenames2),
"--outputfile={0}".format(output_filename)]
k.upload_input_data = ["hausdorff_kernel.py"]
# If the input data are in in a web server use the following
# k.download_input_data = ["/<PATH>/<TO>/<WEB>?<SERVER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"]
# If the input data are in a folder to the target machine use the following
# k.link_input_data = ["/<PATH>/<TO>/<FOLDER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"]
# The result files comparison-x-y.dat are downloaded.
k.download_output_data = output_filename
print "Element Comparison Finished {0} - {1}".format(elements1,elements2)
return k
def pre_loop(self):
"""
function : transfers input files and intermediate executables
pre_grlsd_loop :-
Purpose : Transfers files, Split the input file into smaller files to be used by each of the
gromacs instances in the first iteration.
Arguments : --inputfile = file to be split
--numCUs = number of simulation instances/ number of smaller files
"""
k = Kernel(name="md.pre_grlsd_loop")
k.upload_input_data = [
Kconfig.md_input_file,
Kconfig.lsdm_config_file,
Kconfig.top_file,
Kconfig.mdp_file,
"{0}/spliter.py".format(Kconfig.misc_loc),
"{0}/gro.py".format(Kconfig.misc_loc),
"{0}/run.py".format(Kconfig.misc_loc),
"{0}/pre_analyze.py".format(Kconfig.misc_loc),
"{0}/post_analyze.py".format(Kconfig.misc_loc),
"{0}/select.py".format(Kconfig.misc_loc),
"{0}/reweighting.py".format(Kconfig.misc_loc),
]
k.download_input_data = [
"http://sourceforge.net/p/lsdmap/git/ci/extasy-0.1-rc2/tree/lsdmap/lsdm.py?format=raw > lsdm.py"
]
k.arguments = [
"--inputfile={0}".format(os.path.basename(Kconfig.md_input_file)),
"--numCUs={0}".format(Kconfig.num_CUs),
]
return k
def simulation_step(self, iteration, instance):
"""In the simulation step we
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=asciifile.dat"]
k.exists_remote = ["asciifile.dat"]
return [k]
def pre_loop(self):
k = Kernel(name="misc.chksum")
k.arguments = ["--inputfile=UTF-8-demo.txt", "--outputfile=checksum.sha1"]
k.download_input_data = "htpttpt://malformed.url"
k.download_output_data = "checksum.sha1"
return k
def prepare_replica_for_exchange(self, replica):
"""Prepares md.re_exchange kernel to launch namd_matrix_calculator.py
script on target resource in order to populate columns of swap matrix.
Arguments:
replica - object representing a given replica and it's attributes
Returns:
k - an instance of Kernel class
"""
basename = self.inp_basename[:-5]
matrix_col = "matrix_column_{cycle}_{replica}.dat"\
.format(cycle=replica.cycle-1, replica=replica.id )
k = Kernel(name="md.re_exchange")
k.arguments = [
"--calculator=namd_matrix_calculator.py",
"--replica_id=" + str(replica.id),
"--replica_cycle=" + str(replica.cycle - 1),
"--replicas=" + str(self.replicas),
"--replica_basename=" + str(basename)
]
k.upload_input_data = "namd_matrix_calculator.py"
k.download_output_data = matrix_col
return k
def analysis_step(self, iteration, instance):
"""In the analysis step, we take the previously generated simulation
output and perform a Levenshtein distance calculation between it
and the 'reference' file.
..note:: The placeholder ``$PRE_LOOP`` used in ``link_input_data`` is
a reference to the working directory of pre_loop.
The placeholder ``$PREV_SIMULATION`` used in ``link_input_data``
is a reference to the working directory of the previous
simulation step.
It is also possible to reference a specific
simulation step using ``$SIMULATION_N`` or all simulations
via ``$SIMULATIONS``. Analogous placeholders exist for
``ANALYSIS``.
"""
input_filename = "simulation-{0}-{1}.dat".format(iteration, instance)
output_filename = "analysis-{0}-{1}.dat".format(iteration, instance)
k = Kernel(name="misc.levenshtein")
k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"]
k.arguments = ["--inputfile1=reference.dat",
"--inputfile2={0}".format(input_filename),
"--outputfile={0}".format(output_filename)]
k.download_output_data = output_filename
return k
def stage_2(self, instance):
k2 = Kernel(name="md.gromacs")
k2.link_input_data = ['$STAGE_1/in.tpr > in.tpr']
k2.executable = ['path/to/gromacs/gmx']
k2.arguments = ['mdrun', '-s', 'in.tpr', '-deffnm', 'out']
k2.cores = 1
return k2
def simulation_step(self, iteration, instance):
"""The simulation step generates a 1 kB file containing random ASCII
characters that is compared against the 'reference' file in the
subsequent analysis step.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=simulation-{0}-{1}.dat".format(iteration, instance)]
return k
def step_1(self, instance):
k = Kernel(name="misc.chksum")
k.arguments = [
"--inputfile=UTF-8-demo.txt",
"--outputfile=checksum{0}.sha1".format(instance)
]
k.download_input_data = "http://testing.saga-project.org/cybertools/UTF-8-demo.txt"
k.download_output_data = "checksum{0}.sha1".format(instance)
def step_1(self, instance):
k = Kernel(name="misc.hello")
k.upload_input_data = ['./input_file.txt > temp.txt']
k.arguments = ["--file=temp.txt"]
k.download_output_data = [
'./temp.txt > /var/lib/jenkins/workspace/EnsembleMDTesting/temp_results/remote_file.txt'
]
return k
def pre_loop(self):
"""pre_loop is executed before the main simulation-analysis loop is
started. In this example we create an initial 1 kB random ASCII file
that we use as the reference for all analysis steps.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=reference.dat"]
return k
def step_1(self, instance):
k = Kernel(name="misc.hello")
k.upload_input_data = ['./input_file.txt > temp.txt']
k.arguments = ["--file=temp.txt"]
k.download_output_data = [
'./temp.txt > output_file_{0}.txt'.format(instance)
]
return k
def stage_1(self, instance):
k = Kernel(name="synapse.sample")
k.arguments = [ "--path=$HOME/ves/synapse_local",
"--mode=sample",
"--flops=1000",
"--samples=1"
]
return k
def stage_2(self, instances): k = Kernel(name="misc.cat") k.upload_input_data = ['./output_file_2.txt > file2.txt'] k.copy_input_data = ['$STAGE_1/temp.txt > file1.txt'] k.arguments = ["--file1=file1.txt","--file2=file2.txt"] k.download_output_data = ['./file1.txt > output_file.txt'] return k
def pre_loop(self):
"""pre_loop is executed before the main simulation-analysis loop is
started. In this example we create an initial 1 kB random ASCII file
that we use as the reference for all analysis steps.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=reference.dat"]
return k
def stage_1(self, instance):
k1 = Kernel(name="md.gromacs")
k1.upload_input_data = ['in.gro', 'in.top', '*.itp', 'in.mdp']
k1.executable = ['path/to/gromacs/gmx']
k1.arguments = ['grompp', '-f', 'in.mdp', '-c', 'in.gro', '-o', 'in.tpr', '-p', 'in.top']
k1.cores = 1
return k1
def step_1(self, instance):
"""The first step of the pipeline creates a 1 MB ASCI file.
"""
k = Kernel(name="misc.mkfile")
k.arguments = [
"--size=1000000", "--filename=asciifile-{0}.dat".format(instance)
]
return k
def stage_1(self, instances):
"""This step sleeps for 60 seconds."""
k = Kernel(name="spark")
k.upload_input_data = ['leafletfinder.py','traj_positions.npy']
k.arguments = ["--exec-mem=60g","--driver-mem=30g",
"--max-result-size=25g","--spark-script=leafletfinder.py","--input-file=traj_positions.npy","--partitions=378"]
return k
def simulation_stage(self, iteration, instance):
"""The simulation stage generates a 1 kB file containing random ASCII
characters that is compared against the 'reference' file in the
subsequent analysis stage.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=simulation-{0}-{1}.dat".format(iteration, instance)]
return k
def analysis_step(self, iteration, instance):
""" In the analysis step, we use the 'randval' kernel to output a random number within
the upperlimit. The output is simply a number (and no other messages). Hence, we do not mention
and extraction scripts. The pattern automatically picks up the number.
"""
k = Kernel(name="misc.randval")
k.arguments = ["--upperlimit=16"]
return [k]
def analysis_stage(self, iteration, instance): """ In the analysis stage, we use the 'randval' kernel to output a random number within the upperlimit. The output is simply a number (and no other messages). Hence, we do not mention and extraction scripts. The pattern automatically picks up the number. """ k = Kernel(name="misc.randval_2") k.arguments = ["--upperlimit=16"] return [k]
def simulation_step(self, iteration, instance):
"""In the simulation step we
"""
k = Kernel(name="misc.mkfile")
k.arguments = [
"--size=1000", "--filename=asciifile-{0}.dat".format(instance)
]
k.download_output_data = ['asciifile-{0}.dat'.format(instance)]
return [k]
def step_2(self, instance):
"""The second step of the pipeline does a character frequency analysis
on the file generated the first step.
"""
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=asciifile-{0}.dat".format(instance), "--outputfile=cfreqs-{0}.dat".format(instance)]
k.link_input_data = "$STEP_1/asciifile-{0}.dat".format(instance)
return k
def step_1(self, instance):
k = Kernel(name="misc.chksum")
k.arguments = [
"--inputfile={0}".format(self._checksum_inputfile),
"--outputfile={0}".format(self._download_output)
]
k.upload_input_data = self._upload_directives
k.download_output_data = self._download_output
return k
def stage_2(self, instance):
k = Kernel(name="misc.ccount")
k.arguments = [
"--inputfile=asciifile-{0}.dat".format(instance),
"--outputfile=cfreqs-{0}.dat".format(instance)
]
k.copy_input_data = "$STAGE_1/asciifile-{0}.dat".format(instance)
k.download_output_data = "cfreqs-{0}.dat".format(instance)
return k
def step_3(self, instance):
"""The third step of the pipeline creates a checksum of the output file
of the second step. The result is transferred back to the host
running this script.
"""
k = Kernel(name="misc.chksum")
k.arguments = ["--inputfile=cfreqs-{0}.dat".format(instance), "--outputfile=cfreqs-{0}.sha1".format(instance)]
k.link_input_data = "$STEP_2/cfreqs-{0}.dat".format(instance)
k.download_output_data = "cfreqs-{0}.sha1".format(instance)
return k
def stage_1(self, instance):
k = Kernel(name="qbird")
k.upload_input_data = ['SeaIceUnSuperGMM.py']
k.arguments = ["--script=SeaIceUnSuperGMM.py",
"--imagepath=/home/vivek91/xsede-hack/IMG/XinImages/TEST/072610_00332.jpg",
"--trainingpath=/home/vivek91/xsede-hack/IMG/XinImages/TRAINING/",
"--num_train=3",
"--icetypes=4"]
k.download_output_data = ["labelledImage3trainingImages.png > image_{0}.png".format(instance)]
return k
def test_ArgumentError(self):
from radical.ensemblemd import Kernel
from radical.ensemblemd.exceptions import ArgumentError
try:
k = Kernel(name="misc.hello")
k.arguments = ["a"]
except ArgumentError, er:
print 'ArgumentError: Passed: ', er
assert "Invalid argument(s) for kernel 'misc.hello': Unknown / malformed argument 'a'. Valid arguments are {'--file=': {'_value': None, 'mandatory': True, 'description': 'The input file.', '_is_set': False}}." \
in er
def test_ArgumentError(self):
from radical.ensemblemd import Kernel
from radical.ensemblemd.exceptions import ArgumentError
try:
k = Kernel(name="misc.hello")
k.arguments = ["a"]
except ArgumentError, er:
print 'ArgumentError: Passed: ',er
assert "Invalid argument(s) for kernel 'misc.hello': Unknown / malformed argument 'a'. Valid arguments are {'--file=': {'_value': None, 'mandatory': True, 'description': 'The input file.', '_is_set': False}}." \
in er
def step_3(self, instance):
"""The third step of the pipeline creates a checksum of the output file
of the second step. The result is transferred back to the host
running this script.
"""
k = Kernel(name="misc.chksum")
k.arguments = ["--inputfile=cfreqs-{0}.dat".format(instance), "--outputfile=cfreqs-{0}.sha1".format(instance)]
k.link_input_data = "$STEP_2/cfreqs-{0}.dat".format(instance)
k.download_output_data = "cfreqs-{0}.sha1".format(instance)
return k
def step_1(self, instance):
k = Kernel(name="misc.chksum")
k.arguments = [
"--inputfile=UTF-8-demo.txt",
"--outputfile=checksum{0}.sha1".format(instance)
]
k.download_input_data = "htpttpt://malformed.url"
k.download_output_data = "checksum{0}.sha1".format(instance)
return k
def analysis_step(self, iteration, instance):
input_filename = "simulation-{0}-{1}.dat".format(iteration, instance)
output_filename = "analysis-{0}-{1}.dat".format(iteration, instance)
k = Kernel(name="misc.levenshtein")
k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"]
k.arguments = ["--inputfile1=reference.dat",
"--inputfile2={0}".format(input_filename),
"--outputfile={0}".format(output_filename)]
k.download_output_data = output_filename
return k
def analysis_step(self, iteration, instance):
"""In the analysis step we use the ``$PREV_SIMULATION`` data reference
to refer to the previous simulation. The same
instance is picked implicitly, i.e., if this is instance 5, the
previous simulation with instance 5 is referenced.
"""
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=asciifile.dat", "--outputfile=cfreqs.dat"]
k.link_input_data = "$PREV_SIMULATION/asciifile.dat".format(instance=instance)
k.download_output_data = "cfreqs.dat > cfreqs-{iteration}-{instance}.dat".format(instance=instance, iteration=iteration)
return k
def analysis_step(self, iteration, instance):
input_filename = "simulation-{0}-{1}.dat".format(iteration, instance)
output_filename = "analysis-{0}-{1}.dat".format(iteration, instance)
k = Kernel(name="misc.levenshtein")
k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"]
k.arguments = ["--inputfile1=reference.dat",
"--inputfile2={0}".format(input_filename),
"--outputfile={0}".format(output_filename)]
k.download_output_data = output_filename
return k
def set1element_initialization(self,element):
"""The initialization step creates the necessary files that will be
needed for the comparison over the elements of the set.
"""
# Creating an ASCII file by using the misc.mkfile kernel. Each file represents
# a element of the set.
print "Creating Element {0}".format(element)
k = Kernel(name = "misc.mkfile")
k.arguments = ["--size=10000", "--filename=asciifile_{0}.dat".format(element)]
return k
def analysis_step(self, iteration, instance):
"""In the analysis step we use the ``$PREV_SIMULATION`` data reference
to refer to the previous simulation. The same
instance is picked implicitly, i.e., if this is instance 5, the
previous simulation with instance 5 is referenced.
"""
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=asciifile.dat", "--outputfile=cfreqs.dat"]
k.link_input_data = "$PREV_SIMULATION/asciifile.dat".format(instance=instance)
k.download_output_data = "cfreqs.dat > cfreqs-{iteration}-{instance}.dat".format(instance=instance, iteration=iteration)
return k