Example #1
0
    def setUp(self):
        self.input_folder = os.path.abspath(
                '../../euclidwf_examples/packages/pkgdefs')
        self.output_folder = os.path.abspath('../test_data/output')
        self.mock_output = os.path.join(self.output_folder, 'mock')

        self.workdir = os.path.join(self.mock_output, 'workdir')

        self.test_xml_exec = 'vis_correct_dark'
        self.test_list_exec = 'vis_split_quadrants'

        print(self.input_folder)
        print(self.output_folder)

        self.generator = StubsGenerator(self.output_folder)
        self.mocker = MockGenerator(self.mock_output)
Example #2
0
def main():
    args = parse_cmd_args()

    generator = StubsGenerator(args.destdir)

    executables = exec_loader.get_all_executables(args.pkgdefs)

    test_pipeline_name = 'vis_split_quadrants'

    executables = dict({(k, v) for k, v in executables.items() if
                        k == test_pipeline_name})

    generator.generate_stubs(executables,
                             {test_pipeline_name: {'quadrants_list': 20}})

    mock_output = os.path.join(args.destdir, 'mock')
    mocker = MockGenerator(mock_output)
    mocker.generate_mocks({'vis_split': 50, 'vis_combine': 100})
Example #3
0
def generate():

    #data = shelve.open(os.path.join(outputFolder,'shelvedata'))
    #temp = data[session['uid']]
    pipeline_name = session['pipeline_name']

    #filterd_executables = pickle.loads(temp)

    execs = pickle.loads(session['execs'])

    # create output dir
    pipeline_output = os.path.join(outputFolder, pipeline_name) + '/'
    mkdir_p(pipeline_output)

    for stubinfo in execs:
        stubinfo.cores = int(request.form[stubinfo.command + '_cores'])
        stubinfo.ram = int(request.form[stubinfo.command + '_ram'])
        stubinfo.walltime = controller.parseWallTime(request.form[
            stubinfo.command +
            '_walltime'])  #Parsing the walltime to ensure right format
        if stubinfo.isParallelSplit:
            stubinfo.split_parts = int(request.form[stubinfo.command +
                                                    '_splits'])

        tempTupleList = list()
        for outputfile in stubinfo.outputfiles:
            tempTupleList.append(
                (outputfile[0],
                 int(request.form['%s_%s_size' %
                                  (stubinfo.command, outputfile[0])])))
        stubinfo.outputfiles = tempTupleList

    # Set Pipeline Input Size
    files = session['files']
    """:type files: dict"""
    for (key, value) in files.items():
        files[key] = int(request.form[key])

    on = 'pipelineInputCheckBox' in request.form

    StubsGenerator(os.path.join(pipeline_output, "bin")).generate_stubs(execs)
    MockGenerator(pipeline_output).generate_script(files)

    if on:
        MockGenerator(pipeline_output).generate_mocks(files)

    controller.writeComputingResources(execs, pipeline_output)

    memory_file = controller.createZip(pipeline_output)

    # remove output folder
    shutil.rmtree(pipeline_output)

    return send_file(memory_file,
                     attachment_filename='%s.zip' % pipeline_name,
                     as_attachment=True)
Example #4
0
def main():
    args = parse_cmd_args()

    generator = StubsGenerator(args.destdir)

    executables = exec_loader.get_all_executables(args.pkgdefs)

    test_pipeline_name = 'vis_split_quadrants'

    executables = dict({(k, v)
                        for k, v in executables.items()
                        if k == test_pipeline_name})

    generator.generate_stubs(executables,
                             {test_pipeline_name: {
                                 'quadrants_list': 20
                             }})

    mock_output = os.path.join(args.destdir, 'mock')
    mocker = MockGenerator(mock_output)
    mocker.generate_mocks({'vis_split': 50, 'vis_combine': 100})
Example #5
0
class BasicTest(unittest.TestCase):
    def setUp(self):
        self.input_folder = os.path.abspath(
                '../../euclidwf_examples/packages/pkgdefs')
        self.output_folder = os.path.abspath('../test_data/output')
        self.mock_output = os.path.join(self.output_folder, 'mock')

        self.workdir = os.path.join(self.mock_output, 'workdir')

        self.test_xml_exec = 'vis_correct_dark'
        self.test_list_exec = 'vis_split_quadrants'

        print(self.input_folder)
        print(self.output_folder)

        self.generator = StubsGenerator(self.output_folder)
        self.mocker = MockGenerator(self.mock_output)

    def tearDown(self):
        # cleanup
        pass

    def test_a_generate_xml_stub(self):
        info = StubInfo(self.test_xml_exec)
        info.outputfiles = [('quadrant', 2), ('master_dark', 2), ('control_params', 2)]
        info.inputfiles = ['corrected_frame', 'hot_pixels_map']

        info.ram = 10
        info.walltime = 2
        info.cores = 1

        self.generator.generate_stubs([info])

    def test_b_mock_script_generator(self):
        self.mocker.generate_script({'corrected_frame': 3, 'hot_pixels_map': 3})

    def test_c_run_mock_script(self):
        mock_script = os.path.join(self.workdir, "mock_script.py")

        result = os.system("python %s --destdir %s" % (mock_script, self.workdir))

        assert result == 0

    def test_d_run_xml_exec(self):
        test_script = os.path.join(self.output_folder,
                                   self.test_xml_exec)

        result = os.system(
                "python %s --workdir %s"
                " --corrected_frame %s --hot_pixels_map %s --quadrant %s --master_dark %s --control_params %s" %
                (test_script, self.workdir, 'corrected_frame.dat', 'hot_pixels_map.dat', 'quadrant.dat',
                 'master_dark.dat', 'control_params.dat'))

        assert result == 0

    def test_e_generate_list_stub(self):
        info = StubInfo(self.test_list_exec, nodeType=NodeType.split)
        info.outputfiles = [('quadrants_list', 5)]
        info.inputfiles = ['exposures']
        info.isParallelSplit = True

        info.ram = 10
        info.walltime = 2
        info.cores = 1

        self.generator.generate_stubs([info])

    def test_f_mock_generator(self):
        self.mocker.generate_mocks({'exposures': 9})

    def test_g_run_list_exec(self):
        test_script = os.path.join(self.output_folder,
                                   self.test_list_exec)

        result = os.system(
                "python %s --workdir %s --exposures %s --quadrants_list %s" %
                (test_script, self.workdir, 'exposures.dat',
                 'quadrants_list.dat'))

        assert result == 0