def simulation_step(self, iteration, instance):
'''
function : In iter=1, use the input files from pre_loop, else use the outputs of the analysis stage in the
previous iteration. Run gromacs in each instance using these files.
gromacs :-
Purpose : Run the gromacs simulation on each of the smaller files. Parameter files and executables are input
from pre_loop. There are 'numCUs' number of instances of gromacs per iteration.
Arguments : --grompp = gromacs parameters filename
--topol = topology filename
'''
#----------------------------------------------------------------------------------------------------------
# GROMPP kernel
k1 = Kernel(name="custom.grompp")
k1.arguments = [
"--mdp={0}".format(os.path.basename(Kconfig.mdp_file)),
"--gro=start.gro",
"--top={0}".format(os.path.basename(Kconfig.top_file)),
"--tpr=topol.tpr"
]
k1.link_input_data = [
'$SHARED/{0} > {0}'.format(os.path.basename(Kconfig.mdp_file)),
'$SHARED/{0} > {0}'.format(os.path.basename(Kconfig.top_file))
]
k1.copy_output_data = [
'topol.tpr > $SHARED/iter_{1}/topol_{0}.tpr'.format(
instance - 1, iteration - 1)
]
if (iteration - 1 == 0):
k1.link_input_data.append(
'$PRE_LOOP/temp/start{0}.gro > start.gro'.format(instance - 1))
else:
k1.link_input_data.append(
'$ANALYSIS_ITERATION_{0}_INSTANCE_1/temp/start{1}.gro > start.gro'
.format(iteration - 1, instance - 1))
#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------
# MDRUN kernel
k2 = Kernel(name="custom.mdrun")
k2.arguments = ["--size=1", "--tpr=topol.tpr", "--out=out.gro"]
k2.link_input_data = [
'$SHARED/iter_{1}/topol_{0}.tpr > topol.tpr'.format(
instance - 1, iteration - 1)
]
#----------------------------------------------------------------------------------------------------------
return [k1, k2]
def analysis_step(self, iteration, instance):
k1 = Kernel(name="misc.randval")
k1.arguments = ["--upperlimit=3", "--filename=iters.dat"]
k1.download_output_data = "iters.dat"
k2 = Kernel(name="misc.randval")
k2.arguments = ["--upperlimit=16", "--filename=sims.dat"]
k2.download_output_data = "sims.dat"
return [k1, k2]
def prepare_replica_for_exchange(self, replica):
"""Prepares md.re_exchange kernel to launch namd_matrix_calculator.py
script on target resource in order to populate columns of swap matrix.
Arguments:
replica - object representing a given replica and it's attributes
Returns:
k - an instance of Kernel class
"""
basename = self.inp_basename[:-5]
matrix_col = "matrix_column_{cycle}_{replica}.dat"\
.format(cycle=replica.cycle-1, replica=replica.id )
k = Kernel(name="md.re_exchange")
k.arguments = [
"--calculator=namd_matrix_calculator.py",
"--replica_id=" + str(replica.id),
"--replica_cycle=" + str(replica.cycle - 1),
"--replicas=" + str(self.replicas),
"--replica_basename=" + str(basename)
]
k.upload_input_data = "namd_matrix_calculator.py"
k.download_output_data = matrix_col
return k
def simulation_step(self, iteration, instance):
k = Kernel(name="misc.mkfile")
k.arguments = [
"--size=1000",
"--filename=simulation-{0}-{1}.dat".format(iteration, instance)
]
return k
def simulation_step(self, iteration, instance):
'''
function : In iter=1, use the input files from pre_loop, else use the outputs of the analysis stage in the
previous iteration. Run gromacs in each instance using these files.
gromacs :-
Purpose : Run the gromacs simulation on each of the smaller files. Parameter files and executables are input
from pre_loop. There are 'numCUs' number of instances of gromacs per iteration.
Arguments : --grompp = gromacs parameters filename
--topol = topology filename
'''
gromacs = Kernel(name="md.gromacs")
gromacs.arguments = [
"--grompp={0}".format(os.path.basename(Kconfig.mdp_file)),
"--topol={0}".format(os.path.basename(Kconfig.top_file))
]
gromacs.link_input_data = [
'$PRE_LOOP/{0} > {0}'.format(os.path.basename(Kconfig.mdp_file)),
'$PRE_LOOP/{0} > {0}'.format(os.path.basename(Kconfig.top_file)),
'$PRE_LOOP/run.py > run.py'
]
if (iteration - 1 == 0):
gromacs.link_input_data.append(
'$PRE_LOOP/temp/start{0}.gro > start.gro'.format(instance - 1))
else:
gromacs.link_input_data.append(
'$ANALYSIS_ITERATION_{0}_INSTANCE_1/temp/start{1}.gro > start.gro'
.format(iteration - 1, instance - 1))
return gromacs
def analysis_step(self, iteration, instance):
"""In the analysis step, we take the previously generated simulation
output and perform a Levenshtein distance calculation between it
and the 'reference' file.
..note:: The placeholder ``$PRE_LOOP`` used in ``link_input_data`` is
a reference to the working directory of pre_loop.
The placeholder ``$PREV_SIMULATION`` used in ``link_input_data``
is a reference to the working directory of the previous
simulation step.
It is also possible to reference a specific
simulation step using ``$SIMULATION_N`` or all simulations
via ``$SIMULATIONS``. Analogous placeholders exist for
``ANALYSIS``.
"""
input_filename = "simulation-{0}-{1}.dat".format(iteration, instance)
output_filename = "analysis-{0}-{1}.dat".format(iteration, instance)
k = Kernel(name="misc.levenshtein")
k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"]
k.arguments = ["--inputfile1=reference.dat",
"--inputfile2={0}".format(input_filename),
"--outputfile={0}".format(output_filename)]
k.download_output_data = output_filename
return k
def pre_loop(self):
k = Kernel(name="misc.chksum")
k.arguments = ["--inputfile=UTF-8-demo.txt", "--outputfile=checksum.sha1"]
k.download_input_data = "htpttpt://malformed.url"
k.download_output_data = "checksum.sha1"
return k
def set2element_initialization(self,element):
# Creating an ASCII file by using the misc.mkfile kernel. Each file represents
# a element of the set.
print "Creating Element {0}".format(element)
k = Kernel(name = "misc.mkfile")
k.arguments = ["--size=10000", "--filename=newfile_{0}.dat".format(element)]
return k
def element_comparison(self, element1, element2):
"""In the comparison, we take the previously generated files
and perform a difference between those files. Each file coresponds to
an elements of the set.
"""
input_filename1 = "asciifile-{0}.dat".format(element1)
input_filename2 = "asciifile-{0}.dat".format(element2)
output_filename = "comparison-{0}-{1}.log".format(element1, element2)
print "Comparing {0} with {1}. Saving result in {2}".format(
input_filename1, input_filename2, output_filename)
# Compare the previously generated files with the misc.diff kernel and
# write the result of each comparison to a specific output file.
k = Kernel(name="misc.diff")
k.arguments = [
"--inputfile1={0}".format(input_filename1),
"--inputfile2={0}".format(input_filename2),
"--outputfile={0}".format(output_filename)
]
# Download the result files.
k.download_output_data = output_filename
return k
def simulation_step(self, iteration, instance):
"""In the simulation step we
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=asciifile.dat"]
k.exists_remote = ['asciifile.dat']
return [k]
def prepare_replica_for_md(self, replica):
"""Specifies input and output files and passes them to kernel
Arguments:
replica - object representing a given replica and it's associated
parameters
"""
input_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".md"
output_name = self.inp_basename + "_" + \
str(replica.id) + "_" + \
str(replica.cycle) + ".out"
k = Kernel(name="misc.ccount")
k.arguments = ["--inputfile=" + \
input_name + " " + \
self.sh_file, "--outputfile=" + \
output_name]
# no need to specify shared data here
# everything in shared_files list will be staged in
k.upload_input_data = [input_name]
k.download_output_data = output_name
replica.cycle = replica.cycle + 1
return k
def pre_loop(self):
'''
function : transfers input files and intermediate executables
pre_grlsd_loop :-
Purpose : Transfers files, Split the input file into smaller files to be used by each of the
gromacs instances in the first iteration.
Arguments : --inputfile = file to be split
--numCUs = number of simulation instances/ number of smaller files
'''
k = Kernel(name="md.pre_grlsd_loop")
k.upload_input_data = [
Kconfig.md_input_file, Kconfig.lsdm_config_file, Kconfig.top_file,
Kconfig.mdp_file, '{0}/spliter.py'.format(Kconfig.misc_loc),
'{0}/gro.py'.format(Kconfig.misc_loc),
'{0}/run.py'.format(Kconfig.misc_loc),
'{0}/pre_analyze.py'.format(Kconfig.misc_loc),
'{0}/post_analyze.py'.format(Kconfig.misc_loc),
'{0}/selection.py'.format(Kconfig.misc_loc),
'{0}/reweighting.py'.format(Kconfig.misc_loc)
]
k.arguments = [
"--inputfile={0}".format(os.path.basename(Kconfig.md_input_file)),
"--numCUs={0}".format(Kconfig.num_CUs)
]
return k
def simulation_step(self, iteration, instance):
"""In the simulation step we simply create files with 1000 characters.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=asciifile-{0}.dat".format(instance)]
k.download_output_data = ['asciifile-{0}.dat > iter{1}/asciifile-{0}.dat'.format(instance,iteration)]
return [k]
def element_comparison(self, elements1, elements2):
"""In the comparison, we take the previously generated modified trajectory
and perform a Hausdorff distance calculation between all the unique pairs
of trajectories
"""
input_filenames1 = ["traj_flat%d.npz.npy" % (el1) for el1 in elements1]
input_filenames2 = ["traj_flat%d.npz.npy" % (el2) for el2 in elements2]
output_filename = "comparison-%03d-%03d.dat" % (elements1[0],
elements2[0])
print "Element Comparison {0} - {1}".format(elements1, elements2)
k = Kernel(name="my.hausdorff")
k.arguments = [
"--dist_file=hausdorff_kernel.py",
"--inputfile1={0}".format(input_filenames1),
"--inputfile2={0}".format(input_filenames2),
"--outputfile={0}".format(output_filename)
]
k.upload_input_data = ["hausdorff_kernel.py"]
# If the input data are in in a web server use the following
# k.download_input_data = ["/<PATH>/<TO>/<WEB>?<SERVER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"]
# If the input data are in a folder to the target machine use the following
# k.link_input_data = ["/<PATH>/<TO>/<FOLDER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"]
# The result files comparison-x-y.dat are downloaded.
k.download_output_data = output_filename
print "Element Comparison Finished {0} - {1}".format(
elements1, elements2)
return k
def stage_1(self, instance):
k = Kernel(name="synapse.sample")
k.arguments = [
"--path=$HOME/ves/synapse_local", "--mode=sample", "--flops=1000",
"--samples=1"
]
return k
def stage_2(self, instance):
k2 = Kernel(name="md.gromacs")
k2.link_input_data = ['$STAGE_1/in.tpr > in.tpr']
k2.executable = ['path/to/gromacs/gmx']
k2.arguments = ['mdrun', '-s', 'in.tpr', '-deffnm', 'out']
k2.cores = 1
return k2
def step_2(self, instances):
k = Kernel(name="misc.cat")
k.upload_input_data = ['./output_file_2.txt > file2.txt']
k.copy_input_data = ['$STEP_1/temp.txt > file1.txt']
k.arguments = ["--file1=file1.txt", "--file2=file2.txt"]
k.download_output_data = ['./file1.txt > output_file.txt']
return k
def step_1(self, instance):
k = Kernel(name="misc.hello")
k.upload_input_data = ['./input_file.txt > temp.txt']
k.arguments = ["--file=temp.txt"]
k.download_output_data = [
'./temp.txt > output_file_{0}.txt'.format(instance)
]
return k
def step_1(self, instance):
k = Kernel(name="misc.hello")
k.upload_input_data = ['./input_file.txt > temp.txt']
k.arguments = ["--file=temp.txt"]
k.download_output_data = [
'./temp.txt > /var/lib/jenkins/workspace/EnsembleMDTesting/temp_results/remote_file.txt'
]
return k
def stage_1(self, instances):
"""This step sleeps for 60 seconds."""
k = Kernel(name="spark")
k.upload_input_data = ['leafletfinder.py','traj_positions.npy']
k.arguments = ["--exec-mem=60g","--driver-mem=30g",
"--max-result-size=25g","--spark-script=leafletfinder.py","--input-file=traj_positions.npy","--partitions=378"]
return k
def step_1(self, instance):
"""The first step of the pipeline creates a 1 MB ASCI file.
"""
k = Kernel(name="misc.mkfile")
k.arguments = [
"--size=1000000", "--filename=asciifile-{0}.dat".format(instance)
]
return k
def pre_loop(self):
"""pre_loop is executed before the main simulation-analysis loop is
started. In this example we create an initial 1 kB random ASCII file
that we use as the reference for all analysis steps.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=reference.dat"]
return k
def simulation_step(self, iteration, instance):
"""The simulation step generates a 1 kB file containing random ASCII
characters that is compared against the 'reference' file in the
subsequent analysis step.
"""
k = Kernel(name="misc.mkfile")
k.arguments = ["--size=1000", "--filename=simulation-{0}-{1}.dat".format(iteration, instance)]
return k
def analysis_step(self, iteration, instance):
""" In the analysis step, we use the 'randval' kernel to output a random number within
the upperlimit. The output is simply a number (and no other messages). Hence, we do not mention
and extraction scripts. The pattern automatically picks up the number.
"""
k = Kernel(name="misc.randval")
k.arguments = ["--upperlimit=16"]
return [k]
def step_1(self, instance):
k = Kernel(name="misc.chksum")
k.arguments = [
"--inputfile=UTF-8-demo.txt",
"--outputfile=checksum{0}.sha1".format(instance)
]
k.download_input_data = "http://testing.saga-project.org/cybertools/UTF-8-demo.txt"
k.download_output_data = "checksum{0}.sha1".format(instance)
def test_NoKernelPluginError(self):
from radical.ensemblemd import Kernel
from radical.ensemblemd.exceptions import NoKernelPluginError
try:
Kernel(name="random")
except NoKernelPluginError, er:
print 'NoKernelPluginError: Passed: ', er
assert "Couldn't find a kernel plug-in named 'random'" in er
def test_TypeError(self):
from radical.ensemblemd import Kernel
from radical.ensemblemd.exceptions import TypeError
try:
Kernel(name=1)
except TypeError, er:
print 'TypeError: Passed: ', er
assert "Expected (base) type <type 'str'>, but got <type 'int'>." in er
def step_1(self, instance):
k = Kernel(name="misc.chksum")
k.arguments = [
"--inputfile={0}".format(self._checksum_inputfile),
"--outputfile={0}".format(self._download_output)
]
k.upload_input_data = self._upload_directives
k.download_output_data = self._download_output
return k
def simulation_step(self, iteration, instance):
"""In the simulation step we
"""
k = Kernel(name="misc.mkfile")
k.arguments = [
"--size=1000", "--filename=asciifile-{0}.dat".format(instance)
]
k.download_output_data = ['asciifile-{0}.dat'.format(instance)]
return [k]
def stage_2(self, instance):
k = Kernel(name="misc.ccount")
k.arguments = [
"--inputfile=asciifile-{0}.dat".format(instance),
"--outputfile=cfreqs-{0}.dat".format(instance)
]
k.copy_input_data = "$STAGE_1/asciifile-{0}.dat".format(instance)
k.download_output_data = "cfreqs-{0}.dat".format(instance)
return k