def analysis_step(self, iteration, instance): k1 = Kernel(name="misc.randval") k1.arguments = ["--upperlimit=3", "--filename=iters.dat"] k1.download_output_data = "iters.dat" k2 = Kernel(name="misc.randval") k2.arguments = ["--upperlimit=16", "--filename=sims.dat"] k2.download_output_data = "sims.dat" return [k1, k2]
def analysis_step(self, iteration, instance): k1 = Kernel(name="misc.randval") k1.arguments = ["--upperlimit=3", "--filename=iters.dat"] k1.download_output_data = "iters.dat" k2 = Kernel(name="misc.randval") k2.arguments = ["--upperlimit=16","--filename=sims.dat"] k2.download_output_data = "sims.dat" return [k1,k2]
def simulation_step(self, iteration, instance): """In the simulation step we """ k = Kernel(name="misc.mkfile") k.arguments = ["--size=1000", "--filename=asciifile-{0}.dat".format(instance)] k.download_output_data = ['asciifile-{0}.dat'.format(instance)] return [k]
def element_comparison(self, elements1, elements2): """In the comparison, we take the previously generated modified trajectory and perform a Hausdorff distance calculation between all the unique pairs of trajectories """ input_filenames1 = ["traj_flat%d.npz.npy"%(el1) for el1 in elements1] input_filenames2 = ["traj_flat%d.npz.npy"%(el2) for el2 in elements2] output_filename = "comparison-%03d-%03d.dat"%(elements1[0],elements2[0]) print "Element Comparison {0} - {1}".format(elements1,elements2) k = Kernel(name="my.hausdorff") k.arguments = ["--dist_file=hausdorff_kernel.py", "--inputfile1={0}".format(input_filenames1), "--inputfile2={0}".format(input_filenames2), "--outputfile={0}".format(output_filename)] k.upload_input_data = ["hausdorff_kernel.py"] # If the input data are in in a web server use the following # k.download_input_data = ["/<PATH>/<TO>/<WEB>?<SERVER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"] # If the input data are in a folder to the target machine use the following # k.link_input_data = ["/<PATH>/<TO>/<FOLDER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"] # The result files comparison-x-y.dat are downloaded. k.download_output_data = output_filename print "Element Comparison Finished {0} - {1}".format(elements1,elements2) return k
def analysis_step(self, iteration, instance): """In the analysis step, we take the previously generated simulation output and perform a Levenshtein distance calculation between it and the 'reference' file. ..note:: The placeholder ``$PRE_LOOP`` used in ``link_input_data`` is a reference to the working directory of pre_loop. The placeholder ``$PREV_SIMULATION`` used in ``link_input_data`` is a reference to the working directory of the previous simulation step. It is also possible to reference a specific simulation step using ``$SIMULATION_N`` or all simulations via ``$SIMULATIONS``. Analogous placeholders exist for ``ANALYSIS``. """ input_filename = "simulation-{0}-{1}.dat".format(iteration, instance) output_filename = "analysis-{0}-{1}.dat".format(iteration, instance) k = Kernel(name="misc.levenshtein") k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"] k.arguments = ["--inputfile1=reference.dat", "--inputfile2={0}".format(input_filename), "--outputfile={0}".format(output_filename)] k.download_output_data = output_filename return k
def prepare_replica_for_exchange(self, replica): """Prepares md.re_exchange kernel to launch namd_matrix_calculator.py script on target resource in order to populate columns of swap matrix. Arguments: replica - object representing a given replica and it's attributes Returns: k - an instance of Kernel class """ basename = self.inp_basename[:-5] matrix_col = "matrix_column_{cycle}_{replica}.dat"\ .format(cycle=replica.cycle-1, replica=replica.id ) k = Kernel(name="md.re_exchange") k.arguments = [ "--calculator=namd_matrix_calculator.py", "--replica_id=" + str(replica.id), "--replica_cycle=" + str(replica.cycle - 1), "--replicas=" + str(self.replicas), "--replica_basename=" + str(basename) ] k.upload_input_data = "namd_matrix_calculator.py" k.download_output_data = matrix_col return k
def element_comparison(self, elements1, elements2): """In the comparison, we take the previously generated modified trajectory and perform a Hausdorff distance calculation between all the unique pairs of trajectories """ input_filenames1 = ["traj_flat%d.npz.npy" % (el1) for el1 in elements1] input_filenames2 = ["traj_flat%d.npz.npy" % (el2) for el2 in elements2] output_filename = "comparison-%03d-%03d.dat" % (elements1[0], elements2[0]) print "Element Comparison {0} - {1}".format(elements1, elements2) k = Kernel(name="my.hausdorff") k.arguments = [ "--dist_file=hausdorff_kernel.py", "--inputfile1={0}".format(input_filenames1), "--inputfile2={0}".format(input_filenames2), "--outputfile={0}".format(output_filename) ] k.upload_input_data = ["hausdorff_kernel.py"] # If the input data are in in a web server use the following # k.download_input_data = ["/<PATH>/<TO>/<WEB>?<SERVER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"] # If the input data are in a folder to the target machine use the following # k.link_input_data = ["/<PATH>/<TO>/<FOLDER>/<WITH>/hausdorff_kernel.py > hausdorff_kernel.py"] # The result files comparison-x-y.dat are downloaded. k.download_output_data = output_filename print "Element Comparison Finished {0} - {1}".format( elements1, elements2) return k
def analysis_stage(self, iteration, instance): """In the analysis stage, we take the previously generated simulation output and perform a Levenshtein distance calculation between it and the 'reference' file. ..note:: The placeholder ``$PRE_LOOP`` used in ``link_input_data`` is a reference to the working directory of pre_loop. The placeholder ``$PREV_SIMULATION`` used in ``link_input_data`` is a reference to the working directory of the previous simulation stage. It is also possible to reference a specific simulation stage using ``$SIMULATION_N`` or all simulations via ``$SIMULATIONS``. Analogous placeholders exist for ``ANALYSIS``. """ input_filename = "simulation-{0}-{1}.dat".format(iteration, instance) output_filename = "analysis-{0}-{1}.dat".format(iteration, instance) k = Kernel(name="misc.levenshtein") k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"] k.arguments = ["--inputfile1=reference.dat", "--inputfile2={0}".format(input_filename), "--outputfile={0}".format(output_filename)] k.download_output_data = output_filename return k
def simulation_stage(self, iteration, instance): """In the simulation stage we simply create files with 1000 characters. """ k = Kernel(name="misc.mkfile") k.arguments = ["--size=1000", "--filename=asciifile-{0}.dat".format(instance)] k.download_output_data = ['asciifile-{0}.dat > iter{1}/asciifile-{0}.dat'.format(instance,iteration)] return [k]
def prepare_replica_for_md(self, replica): """Specifies input and output files and passes them to kernel Arguments: replica - object representing a given replica and it's associated parameters """ input_name = self.inp_basename + "_" + \ str(replica.id) + "_" + \ str(replica.cycle) + ".md" output_name = self.inp_basename + "_" + \ str(replica.id) + "_" + \ str(replica.cycle) + ".out" k = Kernel(name="misc.ccount") k.arguments = ["--inputfile=" + \ input_name + " " + \ self.sh_file, "--outputfile=" + \ output_name] # no need to specify shared data here # everything in shared_files list will be staged in k.upload_input_data = [input_name] k.download_output_data = output_name replica.cycle = replica.cycle + 1 return k
def prepare_replica_for_exchange(self, replica): """Prepares md.re_exchange kernel to launch namd_matrix_calculator.py script on target resource in order to populate columns of swap matrix. Arguments: replica - object representing a given replica and it's attributes Returns: k - an instance of Kernel class """ basename = self.inp_basename[:-5] matrix_col = "matrix_column_{cycle}_{replica}.dat"\ .format(cycle=replica.cycle-1, replica=replica.id ) k = Kernel(name="md.re_exchange") k.arguments = ["--calculator=namd_matrix_calculator.py", "--replica_id=" + str(replica.id), "--replica_cycle=" + str(replica.cycle-1), "--replicas=" + str(self.replicas), "--replica_basename=" + str(basename)] k.upload_input_data = "namd_matrix_calculator.py" k.download_output_data = matrix_col return k
def simulation_step(self, iteration, instance): """In the simulation step we simply create files with 1000 characters. """ k = Kernel(name="misc.mkfile") k.arguments = ["--size=1000", "--filename=asciifile-{0}.dat".format(instance)] k.download_output_data = ['asciifile-{0}.dat > iter{1}/asciifile-{0}.dat'.format(instance,iteration)] return [k]
def simulation_step(self, iteration, instance): ''' function : if iteration = 1, use .crd file from pre_loop, else use .crd output from analysis generated in the previous iteration. Perform amber on the .crd files to generate a set of .ncdf files. amber :- Purpose : Run amber on each of the coordinate files. Currently, a non-MPI version of Amber is used. Generates a .ncdf file in each instance. Arguments : --mininfile = minimization filename --mdinfile = MD input filename --topfile = Topology filename --cycle = current iteration number ''' k = Kernel(name="md.amber") k.arguments = ["--mininfile={0}".format(os.path.basename(Kconfig.minimization_input_file)), "--mdinfile={0}".format(os.path.basename(Kconfig.md_input_file)), "--topfile={0}".format(os.path.basename(Kconfig.top_file)), "--cycle=%s"%(iteration)] k.link_input_data = ['$PRE_LOOP/{0}'.format(os.path.basename(Kconfig.minimization_input_file)), '$PRE_LOOP/{0}'.format(os.path.basename(Kconfig.top_file)), '$PRE_LOOP/{0}'.format(os.path.basename(Kconfig.md_input_file))] if((iteration-1)==0): k.link_input_data = k.link_input_data + ['$PRE_LOOP/{0} > min1.crd'.format(os.path.basename(Kconfig.initial_crd_file))] else: k.link_input_data = k.link_input_data + ['$PREV_ANALYSIS_INSTANCE_1/min{0}{1}.crd > min{2}.crd'.format(iteration-1,instance-1,iteration)] if(iteration%Kconfig.nsave==0): k.download_output_data = ['md{0}.ncdf > backup/iter{0}/md_{0}_{1}.ncdf'.format(iteration,instance)] return k
def step_1(self, instance): k = Kernel(name="misc.chksum") k.arguments = ["--inputfile=UTF-8-demo.txt", "--outputfile=checksum{0}.sha1".format(instance)] k.download_input_data = "htpttpt://malformed.url" k.download_output_data = "checksum{0}.sha1".format(instance) return k
def simulation_step(self, iteration, instance): ''' function : In iter=1, use the input files from pre_loop, else use the outputs of the analysis stage in the previous iteration. Run gromacs in each instance using these files. gromacs :- Purpose : Run the gromacs simulation on each of the smaller files. Parameter files and executables are input from pre_loop. There are 'numCUs' number of instances of gromacs per iteration. Arguments : --grompp = gromacs parameters filename --topol = topology filename ''' gromacs = Kernel(name="md.gromacs") gromacs.arguments = ["--grompp={0}".format(os.path.basename(Kconfig.mdp_file)), "--topol={0}".format(os.path.basename(Kconfig.top_file))] gromacs.link_input_data = ['$PRE_LOOP/{0} > {0}'.format(os.path.basename(Kconfig.mdp_file)), '$PRE_LOOP/{0} > {0}'.format(os.path.basename(Kconfig.top_file)), '$PRE_LOOP/run.py > run.py'] #gromacs.copy_output_data = ['out{0}.gro > $PRE_LOOP/out{0}.gro'.format(instance-1)] gromacs.download_output_data = ['md.log > md_{0}.log'.format(instance)] if (iteration-1==0): gromacs.link_input_data.append('$PRE_LOOP/temp/start{0}.gro > start.gro'.format(instance-1)) else: gromacs.link_input_data.append('$ANALYSIS_ITERATION_{0}_INSTANCE_1/temp/start{1}.gro > start.gro'.format(iteration-1,instance-1)) return gromacs
def step_1(self, instance): """The first step of the pipeline creates a 1 MB ASCI file. """ k = Kernel(name="misc.mkfile") k.arguments = ["--size=1000000", "--filename=asciifile-{0}.dat".format(instance)] k.download_output_data = ['asciifile-{0}.dat'.format(instance)] return k
def pre_loop(self): k = Kernel(name="misc.chksum") k.arguments = ["--inputfile=UTF-8-demo.txt", "--outputfile=checksum.sha1"] k.download_input_data = "htpttpt://malformed.url" k.download_output_data = "checksum.sha1" return k
def element_comparison(self, element1, element2): """In the comparison, we take the previously generated files and perform a difference between those files. Each file coresponds to an elements of the set. """ input_filename1 = "asciifile-{0}.dat".format(element1) input_filename2 = "asciifile-{0}.dat".format(element2) output_filename = "comparison-{0}-{1}.log".format(element1, element2) print "Comparing {0} with {1}. Saving result in {2}".format( input_filename1, input_filename2, output_filename) # Compare the previously generated files with the misc.diff kernel and # write the result of each comparison to a specific output file. k = Kernel(name="misc.diff") k.arguments = [ "--inputfile1={0}".format(input_filename1), "--inputfile2={0}".format(input_filename2), "--outputfile={0}".format(output_filename) ] # Download the result files. k.download_output_data = output_filename return k
def stage_2(self, instances): k = Kernel(name="misc.cat") k.upload_input_data = ['./output_file_2.txt > file2.txt'] k.copy_input_data = ['$STAGE_1/temp.txt > file1.txt'] k.arguments = ["--file1=file1.txt","--file2=file2.txt"] k.download_output_data = ['./file1.txt > output_file.txt'] return k
def step_1(self, instance): k = Kernel(name="misc.hello") k.upload_input_data = ['./input_file.txt > temp.txt'] k.arguments = ["--file=temp.txt"] k.download_output_data = [ './temp.txt > output_file_{0}.txt'.format(instance) ] return k
def step_2(self, instances): k = Kernel(name="misc.cat") k.upload_input_data = ['./output_file_2.txt > file2.txt'] k.copy_input_data = ['$STEP_1/temp.txt > file1.txt'] k.arguments = ["--file1=file1.txt", "--file2=file2.txt"] k.download_output_data = ['./file1.txt > output_file.txt'] return k
def step_1(self, instance): k = Kernel(name="misc.hello") k.upload_input_data = ['./input_file.txt > temp.txt'] k.arguments = ["--file=temp.txt"] k.download_output_data = [ './temp.txt > /var/lib/jenkins/workspace/EnsembleMDTesting/temp_results/remote_file.txt' ] return k
def step_1(self, instance): k = Kernel(name="misc.chksum") k.arguments = [ "--inputfile=UTF-8-demo.txt", "--outputfile=checksum{0}.sha1".format(instance) ] k.download_input_data = "http://testing.saga-project.org/cybertools/UTF-8-demo.txt" k.download_output_data = "checksum{0}.sha1".format(instance)
def stage_2(self, instance): k = Kernel(name="misc.ccount") k.arguments = [ "--inputfile=asciifile-{0}.dat".format(instance), "--outputfile=cfreqs-{0}.dat".format(instance) ] k.copy_input_data = "$STAGE_1/asciifile-{0}.dat".format(instance) k.download_output_data = "cfreqs-{0}.dat".format(instance) return k
def step_1(self, instance): k = Kernel(name="misc.chksum") k.arguments = [ "--inputfile={0}".format(self._checksum_inputfile), "--outputfile={0}".format(self._download_output) ] k.upload_input_data = self._upload_directives k.download_output_data = self._download_output return k
def simulation_step(self, iteration, instance): """In the simulation step we """ k = Kernel(name="misc.mkfile") k.arguments = [ "--size=1000", "--filename=asciifile-{0}.dat".format(instance) ] k.download_output_data = ['asciifile-{0}.dat'.format(instance)] return [k]
def step_1(self, instance): k = Kernel(name="misc.chksum") k.arguments = [ "--inputfile=UTF-8-demo.txt", "--outputfile=checksum{0}.sha1".format(instance) ] k.download_input_data = "htpttpt://malformed.url" k.download_output_data = "checksum{0}.sha1".format(instance) return k
def step_3(self, instance): """The third step of the pipeline creates a checksum of the output file of the second step. The result is transferred back to the host running this script. """ k = Kernel(name="misc.chksum") k.arguments = ["--inputfile=cfreqs-{0}.dat".format(instance), "--outputfile=cfreqs-{0}.sha1".format(instance)] k.link_input_data = "$STEP_2/cfreqs-{0}.dat".format(instance) k.download_output_data = "cfreqs-{0}.sha1".format(instance) return k
def stage_1(self, instance): k = Kernel(name="qbird") k.upload_input_data = ['SeaIceUnSuperGMM.py'] k.arguments = ["--script=SeaIceUnSuperGMM.py", "--imagepath=/home/vivek91/xsede-hack/IMG/XinImages/TEST/072610_00332.jpg", "--trainingpath=/home/vivek91/xsede-hack/IMG/XinImages/TRAINING/", "--num_train=3", "--icetypes=4"] k.download_output_data = ["labelledImage3trainingImages.png > image_{0}.png".format(instance)] return k
def analysis_step(self, iteration, instance): ''' function : Perform CoCo Analysis on the output of the simulation from the current iteration. Using the .ncdf files generated in all the instance, generate the .crd file to be used in the next simulation. coco :- Purpose : Runs CoCo analysis on a set of .ncdf files and generates a coordinate file. Arguments : --grid = Number of points along each dimension of the CoCo histogram --dims = The number of projections to consider from the input pcz file --frontpoints = Number of CUs --topfile = Topology filename --mdfile = MD Input filename --output = Output filename --cycle = Current iteration number ''' k1 = Kernel(name="custom.coco") k1.arguments = ["--grid={0}".format(Kconfig.grid), "--dims={0}".format(Kconfig.dims), "--frontpoints=64".format(Kconfig.num_CUs), "--topfile={0}".format(os.path.basename(Kconfig.top_file)), "--mdfile=*.ncdf", "--output=pdbs", "--atom_selection={0}".format(Kconfig.atom_selection)] #k1.cores = min(Kconfig.num_CUs,RPconfig.PILOTSIZE) k1.cores = 64 k1.uses_mpi = True k1.link_input_data = ['$SHARED/{0}'.format(os.path.basename(Kconfig.top_file))] for iter in range(1,iteration+1): # for i in range(1,Kconfig.num_CUs+1): for i in range(1+(instance-1)*64, instance*64 + 1): k1.link_input_data = k1.link_input_data + ['$SIMULATION_ITERATION_{0}_INSTANCE_{1}/md{0}.ncdf > md_{0}_{1}.ncdf'.format(iter,i)] k1.copy_output_data = list() for i in range(0,64): k1.copy_output_data = k1.copy_output_data + ['pdbs{1}.pdb > $SHARED/pentaopt{0}{2}.pdb'.format(iteration,i,(instance-1)*64+i)] if(iteration%Kconfig.nsave==0): k1.download_output_data = ['coco.log > output/iter{0}/coco.log'.format(iteration,instance)] k2 = Kernel(name="custom.tleap",instance_type='single') k2.arguments = ["--numofsims={0}".format(Kconfig.num_CUs), "--cycle={0}".format(iteration)] k2.link_input_data = ['$SHARED/postexec.py > postexec.py'] for i in range(0,Kconfig.num_CUs): k2.link_input_data = k2.link_input_data + ['$SHARED/pentaopt{0}{1}.pdb > pentaopt{0}{1}.pdb'.format(iteration,i)] return [k1,k2]
def analysis_step(self, iteration, instance): input_filename = "simulation-{0}-{1}.dat".format(iteration, instance) output_filename = "analysis-{0}-{1}.dat".format(iteration, instance) k = Kernel(name="misc.levenshtein") k.link_input_data = ["$PRE_LOOP/reference.dat", "$SIMULATION_ITERATION_{1}_INSTANCE_{2}/{0}".format(input_filename,iteration,instance),"$PRE_LOOP/levenshtein.py"] k.arguments = ["--inputfile1=reference.dat", "--inputfile2={0}".format(input_filename), "--outputfile={0}".format(output_filename)] k.download_output_data = output_filename return k
def analysis_step(self, iteration, instance): """In the analysis step we use the ``$PREV_SIMULATION`` data reference to refer to the previous simulation. The same instance is picked implicitly, i.e., if this is instance 5, the previous simulation with instance 5 is referenced. """ k = Kernel(name="misc.ccount") k.arguments = ["--inputfile=asciifile.dat", "--outputfile=cfreqs.dat"] k.link_input_data = "$PREV_SIMULATION/asciifile.dat".format(instance=instance) k.download_output_data = "cfreqs.dat > cfreqs-{iteration}-{instance}.dat".format(instance=instance, iteration=iteration) return k
def step_1(self, instance): """Run AMBER MD Simulations""" k = Kernel(name="amber") k.arguments = [ "--minfile=min.in", "--topfile=penta.top", "--crdfile=penta.crd", "--output=md.crd"] k.upload_input_data = ['amber_input/min.in','amber_input/penta.top','amber_input/penta.crd'] k.download_output_data = ['md.crd > amber_output/md_{0}.crd'.format(instance)] return k
def step_1(self, instance): """This step downloads a sample UTF-8 file from a remote websever and calculates the SHA1 checksum of that file. The checksum is written to an output file and tranferred back to the host running this script. """ k = Kernel(name="misc.chksum") k.arguments = ["--inputfile=UTF-8-demo.txt", "--outputfile=checksum{0}.sha1".format(instance)] k.download_input_data = "http://gist.githubusercontent.com/oleweidner/6084b9d56b04389717b9/raw/611dd0c184be5f35d75f876b13604c86c470872f/gistfile1.txt > UTF-8-demo.txt" k.download_output_data = "checksum{0}.sha1".format(instance) return k
def step_1(self, instance): """This step downloads a sample UTF-8 file from a remote websever and calculates the SHA1 checksum of that file. The checksum is written to an output file and tranferred back to the host running this script. """ k = Kernel(name="misc.chksum") k.arguments = ["--inputfile=UTF-8-demo.txt", "--outputfile=checksum{0}.sha1".format(instance)] k.upload_input_data = "UTF-8-demo.txt" k.download_output_data = "checksum{0}.sha1".format(instance) return k
def analysis_step(self, iteration, instance): ''' function : Perform CoCo Analysis on the output of the simulation from the current iteration. Using the .xtc files generated in all instances, generate .gro files (as many as the num_CUs) to be used in the next simulations. coco :- Purpose : Runs CoCo analysis on a set of MD trajectory files in this case xtc files and generates several coordinates file to be Arguments : --grid = Number of points along each dimension of the CoCo histogram --dims = The number of projections to consider from the input pcz file --frontpoints = Number of CUs --topfile = Topology filename --mdfile = MD Input filename --output = Output filename --cycle = Current iteration number --atom_selection = Selection of the biological part of the system we want to consider for analysis ''' k1_ana_kernel = Kernel(name="md.coco") k1_ana_kernel.link_input_data = ['$PRE_LOOP/{0}'.format(os.path.basename(Kconfig.top_file)), '$SIMULATION_ITERATION_{0}_INSTANCE_1/md-{1}_0.gro > md-{1}_0.gro'.format(iteration,iteration-1)] for iter in range(1,iteration+1): for i in range(1,Kconfig.num_CUs+1): k1_ana_kernel.link_input_data = k1_ana_kernel.link_input_data + ['$SIMULATION_ITERATION_{0}_INSTANCE_{1}/md-{2}_{3}.xtc > md-{2}_{3}.xtc'.format(iter,i,iter-1,i-1)] k1_ana_kernel.cores = 1 k1_ana_kernel.uses_mpi = False outbase, ext = os.path.basename(Kconfig.output).split('.') #Not sure why this if condition is required #if ext == '': # ext = '.pdb' k1_ana_kernel.arguments = ["--grid={0}".format(Kconfig.grid), "--dims={0}".format(Kconfig.dims), "--frontpoints={0}".format(Kconfig.num_CUs), "--topfile=md-{0}_0.gro".format(iteration-1), "--mdfile=*.xtc", "--output={0}_{1}.{2}".format(outbase,iteration-1,ext), "--atom_selection={0}".format(Kconfig.sel)] k1_ana_kernel.copy_output_data = [] for i in range(0,Kconfig.num_CUs): k1_ana_kernel.copy_output_data += ["{0}_{1}{2}.gro > $PRE_LOOP/{0}_{1}{2}.gro".format(outbase,iteration-1,i,ext)] k1_ana_kernel.download_output_data = ["coco.log > output/coco-iter{0}.log".format(iteration-1)] return [k1_ana_kernel]
def analysis_step(self, iteration, instance): link_input_data = [] for i in range(1, self.simulation_instances+1): link_input_data.append("$PREV_SIMULATION_INSTANCE_{instance}/asciifile.dat > asciifile-{instance}.dat".format(instance=i)) k = Kernel(name="misc.ccount") k.arguments = ["--inputfile=asciifile-*.dat", "--outputfile=cfreqs.dat"] k.link_input_data = link_input_data k.download_output_data = "cfreqs.dat > cfreqs-{iteration}.dat".format(iteration=iteration) return [k]
def stage_1(self, instance): k = Kernel(name="qbird") k.upload_input_data = ['SeaIceUnSuperGMM.py'] k.arguments = [ "--script=SeaIceUnSuperGMM.py", "--imagepath=/home/vivek91/xsede-hack/IMG/XinImages/TEST/072610_00332.jpg", "--trainingpath=/home/vivek91/xsede-hack/IMG/XinImages/TRAINING/", "--num_train=3", "--icetypes=4" ] k.download_output_data = [ "labelledImage3trainingImages.png > image_{0}.png".format(instance) ] return k
def step_1(self, instance): """This step downloads a sample UTF-8 file from a remote websever and calculates the SHA1 checksum of that file. The checksum is written to an output file and tranferred back to the host running this script. """ k = Kernel(name="misc.chksum") k.arguments = [ "--inputfile=UTF-8-demo.txt", "--outputfile=checksum{0}.sha1".format(instance) ] k.download_input_data = "http://gist.githubusercontent.com/oleweidner/6084b9d56b04389717b9/raw/611dd0c184be5f35d75f876b13604c86c470872f/gistfile1.txt > UTF-8-demo.txt" k.download_output_data = "checksum{0}.sha1".format(instance) return k
def step_1(self, instance): """This step downloads a sample UTF-8 file from a remote websever and calculates the SHA1 checksum of that file. The checksum is written to an output file and tranferred back to the host running this script. """ k = Kernel(name="misc.chksum") k.arguments = [ "--inputfile=UTF-8-demo.txt", "--outputfile=checksum{0}.sha1".format(instance) ] k.upload_input_data = "UTF-8-demo.txt" k.download_output_data = "checksum{0}.sha1".format(instance) return k
def analysis_step(self, iteration, instance): k1 = Kernel(name="custom.concat") k1.arguments = ["--format=dat", "--filename=output.dat"] k1.download_output_data = "output.dat" link_input_data = [] for i in range(1, self.simulation_instances + 1): link_input_data.append( "$PREV_SIMULATION_INSTANCE_{instance}/asciifile.dat > asciifile-{instance}.dat".format(instance=i) ) k1.link_input_data = link_input_data return [k1]
def step_2(self, instance): """The second step of the pipeline does a character frequency analysis on the file generated the first step. The result is transferred back to the host running this script. ..note:: The placeholder ``$STEP_1`` used in ``link_input_data`` is a reference to the working directory of step 1. ``$STEP_`` can be used analogous to refernce other steps. """ k = Kernel(name="misc.ccount") k.arguments = ["--inputfile=asciifile-{0}.dat".format(instance), "--outputfile=cfreqs-{0}.dat".format(instance)] k.link_input_data = "$STEP_1/asciifile-{0}.dat".format(instance) k.download_output_data = "cfreqs-{0}.dat".format(instance) return k
def analysis_step(self, iteration, instance): link_input_data = [] for i in range(1, self.simulation_instances + 1): link_input_data.append( "$PREV_SIMULATION_INSTANCE_{instance}/asciifile.dat > asciifile-{instance}.dat" .format(instance=i)) k = Kernel(name="misc.ccount") k.arguments = ["--inputfile=asciifile.dat", "--outputfile=cfreqs.dat"] k.link_input_data = link_input_data k.download_output_data = "cfreqs.dat" k.cores = 1 return k
def simulation_step(self, iteration, instance): ''' function : if iteration = 1, use .crd file from pre_loop, else use .crd output from analysis generated in the previous iteration. Perform amber on the .crd files to generate a set of .ncdf files. amber :- Purpose : Run amber on each of the coordinate files. Currently, a non-MPI version of Amber is used. Generates a .ncdf file in each instance. Arguments : --mininfile = minimization filename --mdinfile = MD input filename --topfile = Topology filename and/or reference coordinates file filename --cycle = current iteration number ''' k1 = Kernel(name="custom.amber") k1.arguments = ["--mininfile={0}".format(os.path.basename(Kconfig.minimization_input_file)), "--topfile={0}".format(os.path.basename(Kconfig.top_file)), "--crdfile={0}".format(os.path.basename(Kconfig.initial_crd_file)), "--cycle=%s"%(iteration)] k1.link_input_data = ['$SHARED/{0}'.format(os.path.basename(Kconfig.minimization_input_file)), '$SHARED/{0}'.format(os.path.basename(Kconfig.top_file)), '$SHARED/{0}'.format(os.path.basename(Kconfig.initial_crd_file))] k1.cores = Kconfig.num_cores_per_sim_cu if((iteration-1)==0): k1.link_input_data = k1.link_input_data + ['$SHARED/{0} > min1.rst7'.format(os.path.basename(Kconfig.initial_crd_file))] k1.copy_output_data = ['min1.rst7 > $SHARED/md_{0}_{1}.rst'.format(iteration,instance)] else: k1.link_input_data = k1.link_input_data + ['$SHARED/min_{0}_{1}.rst7 > min{2}.rst7'.format(iteration-1,instance-1,iteration)] k1.copy_output_data = ['md{0}.rst > $SHARED/md_{0}_{1}.rst'.format(iteration,instance)] k2 = Kernel(name="custom.amber") k2.arguments = [ "--mdinfile={0}".format(os.path.basename(Kconfig.md_input_file)), "--topfile={0}".format(os.path.basename(Kconfig.top_file)), "--cycle=%s"%(iteration) ] k2.link_input_data = [ "$SHARED/{0}".format(os.path.basename(Kconfig.md_input_file)), "$SHARED/{0}".format(os.path.basename(Kconfig.top_file)), "$SHARED/md_{0}_{1}.rst > md{0}.rst".format(iteration,instance), ] k2.cores = Kconfig.num_cores_per_sim_cu if(iteration%Kconfig.nsave==0): k2.download_output_data = ['md{0}.nc > output/iter{0}/md_{0}_{1}.nc'.format(iteration,instance)] return [k1,k2]
def analysis_step(self, iteration, instance): """In the analysis step we use the ``$PREV_SIMULATION`` data reference to refer to the previous simulation. The same instance is picked implicitly, i.e., if this is instance 5, the previous simulation with instance 5 is referenced. """ #upload_input_data = [] #for i in range(1, self.simlation_instances+1): # upload_input_data.append("asciifile-{0}.dat".format(i)) k = Kernel(name="misc.ccount") k.arguments = ["--inputfile=asciifile-*.dat", "--outputfile=cfreqs.dat"] k.upload_input_data = ['asciifile-{0}.dat'.format(instance)] k.download_output_data = ["cfreqs.dat > cfreqs-{0}.dat".format(instance)] return [k]
def step_01(self, instance): """This steps does the trajectory simulation. First, the input files are downloaded from a remote HTTP server. Next, NAMD reads the 'inp' ("input") file and simulates the trajectory. Finally, the output is transferred back to the machine on which this script is executed. """ namd = Kernel(name="md.namd") namd.core = 4 namd.arguments = ["eq{0}.inp".format(instance)] namd.download_input_data = ["http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/complex.pdb > complex.pdb", "http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/complex.top > complex.top", "http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/cons.pdb > cons.pdb", "http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/eq0.inp > eq{0}.inp".format(instance)] namd.download_output_data = "STDOUT > eq{0}.out".format(instance) return namd
def analysis_step(self, iteration, instance): """In the analysis step we use the ``$PREV_SIMULATION`` data reference to refer to the previous simulation. The same instance is picked implicitly, i.e., if this is instance 5, the previous simulation with instance 5 is referenced. """ link_input_data = [] for i in range(1, self.simlation_instances+1): link_input_data.append("$PREV_SIMULATION_INSTANCE_{instance}/asciifile.dat > asciifile-{instance}.dat".format(instance=i)) k = Kernel(name="misc.ccount") k.arguments = ["--inputfile=asciifile-*.dat", "--outputfile=cfreqs.dat"] k.link_input_data = link_input_data k.download_output_data = "cfreqs.dat > cfreqs-{iteration}.dat".format(iteration=iteration) return [k]
def step_1(self, instance): """Run AMBER MD Simulations""" k = Kernel(name="amber") k.arguments = [ "--minfile=min.in", "--topfile=penta.top", "--crdfile=penta.crd", "--output=md.crd" ] k.upload_input_data = [ 'amber_input/min.in', 'amber_input/penta.top', 'amber_input/penta.crd' ] k.download_output_data = [ 'md.crd > amber_output/md_{0}.crd'.format(instance) ] return k
def step_1(self, instance): input_filename = "simulation-1-{0}.dat".format(instance) output_filename = "analysis-1-{0}.dat".format(instance) k = Kernel(name="misc.levenshtein") k.link_input_data = [ "/home/suvigya/radical.ensemblemd-master/tests/tests/E2E_test/input_files/reference.dat", "/home/suvigya/radical.ensemblemd-master/tests/tests/E2E_result/input_files/simulation-1-1.dat", "/home/suvigya/radical.ensemblemd-master/tests/tests/E2E_test/levenshtein.py" ] k.arguments = [ "--inputfile1=reference.dat", "--inputfile2={0}".format(input_filename), "--outputfile={0}".format(output_filename) ] k.download_output_data = output_filename return k
def element_comparison(self, elements1, elements2): input_filename1 = "asciifile_{0}.dat".format(elements1[0]) input_filename2 = "newfile_{0}.dat".format(elements2[0]) output_filename = "comparison_{0}_{1}.log".format(elements1[0], elements2[0]) print "\nComparing {0} with {1}. Saving result in {2}".format(input_filename1,input_filename2,output_filename) # Compare the previously generated files with the misc.diff kernel and # write the result of each comparison to a specific output file. k = Kernel(name="misc.diff") k.arguments = ["--inputfile1={0}".format(input_filename1), "--inputfile2={0}".format(input_filename2), "--outputfile={0}".format(output_filename)] # Download the result files. k.download_output_data = output_filename return k
def prepare_replica_for_md(self, replica): input_name = self.inp_basename + "_" + \ str(replica.id) + "_" + \ str(replica.cycle) + ".md" output_name = self.inp_basename + "_" + \ str(replica.id) + "_" + \ str(replica.cycle) + ".out" k = Kernel(name="misc.ccount") k.arguments = ["--inputfile=" + input_name, "--outputfile=" + output_name] k.upload_input_data = input_name k.download_output_data = output_name k.cores = 1 replica.cycle = replica.cycle + 1 return k
def prepare_replica_for_md(self, replica): input_name = self.inp_basename + "_" + \ str(replica.id) + "_" + \ str(replica.cycle) + ".md" output_name = self.inp_basename + "_" + \ str(replica.id) + "_" + \ str(replica.cycle) + ".out" k = Kernel(name="misc.ccount") k.arguments = [ "--inputfile=" + input_name, "--outputfile=" + output_name ] k.upload_input_data = input_name k.download_output_data = output_name k.cores = 1 replica.cycle = replica.cycle + 1 return k
def step_01(self, instance): """This steps does the trajectory simulation. First, the input files are downloaded from a remote HTTP server. Next, NAMD reads the 'inp' ("input") file and simulates the trajectory. Finally, the output is transferred back to the machine on which this script is executed. """ namd = Kernel(name="md.namd") namd.core = 4 namd.arguments = ["eq{0}.inp".format(instance)] namd.download_input_data = [ "http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/complex.pdb > complex.pdb", "http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/complex.top > complex.top", "http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/cons.pdb > cons.pdb", "http://testing.saga-project.org/cybertools/sampledata/BAC-SIMCHAIN/simchain-sample-data/eq0.inp > eq{0}.inp" .format(instance) ] namd.download_output_data = "STDOUT > eq{0}.out".format(instance) return namd
def analysis_step(self, iteration, instance): """In the analysis step we use the ``$PREV_SIMULATION`` data reference to refer to the previous simulation. The same instance is picked implicitly, i.e., if this is instance 5, the previous simulation with instance 5 is referenced. """ link_input_data = [] for i in range(1, self.simlation_instances + 1): link_input_data.append( "$PREV_SIMULATION_INSTANCE_{instance}/asciifile.dat > asciifile-{instance}.dat" .format(instance=i)) k = Kernel(name="misc.ccount") k.arguments = [ "--inputfile=asciifile-*.dat", "--outputfile=cfreqs.dat" ] k.link_input_data = link_input_data k.download_output_data = "cfreqs.dat > cfreqs-{iteration}.dat".format( iteration=iteration) return [k]