def generate():
#data = shelve.open(os.path.join(outputFolder,'shelvedata'))
#temp = data[session['uid']]
pipeline_name = session['pipeline_name']
#filterd_executables = pickle.loads(temp)
execs = pickle.loads(session['execs'])
# create output dir
pipeline_output = os.path.join(outputFolder, pipeline_name) + '/'
mkdir_p(pipeline_output)
for stubinfo in execs:
stubinfo.cores = int(request.form[stubinfo.command + '_cores'])
stubinfo.ram = int(request.form[stubinfo.command + '_ram'])
stubinfo.walltime = controller.parseWallTime(request.form[
stubinfo.command +
'_walltime']) #Parsing the walltime to ensure right format
if stubinfo.isParallelSplit:
stubinfo.split_parts = int(request.form[stubinfo.command +
'_splits'])
tempTupleList = list()
for outputfile in stubinfo.outputfiles:
tempTupleList.append(
(outputfile[0],
int(request.form['%s_%s_size' %
(stubinfo.command, outputfile[0])])))
stubinfo.outputfiles = tempTupleList
# Set Pipeline Input Size
files = session['files']
""":type files: dict"""
for (key, value) in files.items():
files[key] = int(request.form[key])
on = 'pipelineInputCheckBox' in request.form
StubsGenerator(os.path.join(pipeline_output, "bin")).generate_stubs(execs)
MockGenerator(pipeline_output).generate_script(files)
if on:
MockGenerator(pipeline_output).generate_mocks(files)
controller.writeComputingResources(execs, pipeline_output)
memory_file = controller.createZip(pipeline_output)
# remove output folder
shutil.rmtree(pipeline_output)
return send_file(memory_file,
attachment_filename='%s.zip' % pipeline_name,
as_attachment=True)
def setUp(self):
self.input_folder = os.path.abspath(
'../../euclidwf_examples/packages/pkgdefs')
self.output_folder = os.path.abspath('../test_data/output')
self.mock_output = os.path.join(self.output_folder, 'mock')
self.workdir = os.path.join(self.mock_output, 'workdir')
self.test_xml_exec = 'vis_correct_dark'
self.test_list_exec = 'vis_split_quadrants'
print(self.input_folder)
print(self.output_folder)
self.generator = StubsGenerator(self.output_folder)
self.mocker = MockGenerator(self.mock_output)
def main():
args = parse_cmd_args()
generator = StubsGenerator(args.destdir)
executables = exec_loader.get_all_executables(args.pkgdefs)
test_pipeline_name = 'vis_split_quadrants'
executables = dict({(k, v) for k, v in executables.items() if
k == test_pipeline_name})
generator.generate_stubs(executables,
{test_pipeline_name: {'quadrants_list': 20}})
mock_output = os.path.join(args.destdir, 'mock')
mocker = MockGenerator(mock_output)
mocker.generate_mocks({'vis_split': 50, 'vis_combine': 100})
def main():
args = parse_cmd_args()
generator = StubsGenerator(args.destdir)
executables = exec_loader.get_all_executables(args.pkgdefs)
test_pipeline_name = 'vis_split_quadrants'
executables = dict({(k, v)
for k, v in executables.items()
if k == test_pipeline_name})
generator.generate_stubs(executables,
{test_pipeline_name: {
'quadrants_list': 20
}})
mock_output = os.path.join(args.destdir, 'mock')
mocker = MockGenerator(mock_output)
mocker.generate_mocks({'vis_split': 50, 'vis_combine': 100})
class BasicTest(unittest.TestCase):
def setUp(self):
self.input_folder = os.path.abspath(
'../../euclidwf_examples/packages/pkgdefs')
self.output_folder = os.path.abspath('../test_data/output')
self.mock_output = os.path.join(self.output_folder, 'mock')
self.workdir = os.path.join(self.mock_output, 'workdir')
self.test_xml_exec = 'vis_correct_dark'
self.test_list_exec = 'vis_split_quadrants'
print(self.input_folder)
print(self.output_folder)
self.generator = StubsGenerator(self.output_folder)
self.mocker = MockGenerator(self.mock_output)
def tearDown(self):
# cleanup
pass
def test_a_generate_xml_stub(self):
info = StubInfo(self.test_xml_exec)
info.outputfiles = [('quadrant', 2), ('master_dark', 2), ('control_params', 2)]
info.inputfiles = ['corrected_frame', 'hot_pixels_map']
info.ram = 10
info.walltime = 2
info.cores = 1
self.generator.generate_stubs([info])
def test_b_mock_script_generator(self):
self.mocker.generate_script({'corrected_frame': 3, 'hot_pixels_map': 3})
def test_c_run_mock_script(self):
mock_script = os.path.join(self.workdir, "mock_script.py")
result = os.system("python %s --destdir %s" % (mock_script, self.workdir))
assert result == 0
def test_d_run_xml_exec(self):
test_script = os.path.join(self.output_folder,
self.test_xml_exec)
result = os.system(
"python %s --workdir %s"
" --corrected_frame %s --hot_pixels_map %s --quadrant %s --master_dark %s --control_params %s" %
(test_script, self.workdir, 'corrected_frame.dat', 'hot_pixels_map.dat', 'quadrant.dat',
'master_dark.dat', 'control_params.dat'))
assert result == 0
def test_e_generate_list_stub(self):
info = StubInfo(self.test_list_exec, nodeType=NodeType.split)
info.outputfiles = [('quadrants_list', 5)]
info.inputfiles = ['exposures']
info.isParallelSplit = True
info.ram = 10
info.walltime = 2
info.cores = 1
self.generator.generate_stubs([info])
def test_f_mock_generator(self):
self.mocker.generate_mocks({'exposures': 9})
def test_g_run_list_exec(self):
test_script = os.path.join(self.output_folder,
self.test_list_exec)
result = os.system(
"python %s --workdir %s --exposures %s --quadrants_list %s" %
(test_script, self.workdir, 'exposures.dat',
'quadrants_list.dat'))
assert result == 0
