def run(schema, table): """run autopopulate on a table """ table_class = getattr(config['schemata'][schema], table) form = dynamic_runform(table_class)() table_name = '.'.join([schema, table]) # load existing process process = config['_autopopulate'].get(table_name, Run()) # guidance for running new populates dynamicform, _form = config.get_dynamicform(table_name, table_class, DynamicForm) filename = dynamicform.draw_relations() # load/notload table data = dynamicform.get_jsontable() toggle_off_keys = [0] if request.method == 'POST': submit = request.form.get('submit', None) if (not process.running and submit == 'Run' and form.validate_on_submit()): formatted_dict = form.get_formatted() formatted_dict['reserve_jobs'] = True kwargs = json.dumps(formatted_dict) # TODO currently hard-coded filepath = os.path.join( os.path.dirname(os.path.dirname(__file__)), "analysis", "run_populate.py", ) command = [ "python", "-u", filepath, "--schema", schema, "--table", table, "--kwargs", kwargs, "--user", dj.config['database.user'], "--password", dj.config['database.password'] ] config['_autopopulate'][table_name] = process process.start(command, config['tmp_folder']) elif submit == 'Abort': process.abort() stdout, stderr = process.check() return render_template('pages/run.html', form=form, schema=schema, table=table, stdout=stdout, stderr=stderr, table_name=table_name, data=data, toggle_off_keys=toggle_off_keys, filename=filename)
def monitorexperiment(): return render_template('pages/monitorexperiment.html', ) # return Response(generate_stdout()) stdout, stderr = config.get('subprocess', Run()).check() return Response( stream_template('pages/monitorexperiment.html', stdout=stdout, stderr=stderr))
def experiment(table_name, autoscript_folder): _id = request.args.get('_id', None) folder = config['autoscript_folder'] if autoscript_folder is not None and not autoscript_folder == 'None': autoscript_filepath = os.path.join(folder, autoscript_folder) else: autoscript_filepath = None reader = ConfigReader( autoscript_filepath, table_name, ) if reader.initialized: enter_show = ['show', 'true'] else: enter_show = ['', 'false'] form = dynamic_autoscriptform()(prefix='load') submit = request.args.get('submit', None) process = config.get('subprocess', Run()) if request.method == 'POST': reader.rm_hidden_entries() submit = request.form.get('submit', None) print(submit) if submit == 'Load' and form.validate_on_submit(): autoscript_filepath = form.autoscript.data table_name = form.experiment.data return redirect( url_for( 'experiment', autoscript_folder=os.path.basename(autoscript_filepath), table_name=table_name)) elif submit == 'Abort': process.abort() elif reader.initialized: if ((submit in reader.buttons) and reader.validate_on_submit( submit, flash_message=('Unable to run process; ' 'check all fields in the forms'))): reader.run(submit) elif ((submit == 'Save') and reader.validate_on_submit( check_settings_name=True, flash_message=('Unable to save settings; ' 'check all fields in the forms'))): reader.save_settings() # append hidden entries reader.append_hidden_entries() else: reader.populate_form(_id) stdout, stderr = process.check() return render_template( f'pages/experiment.html', form=form, data=reader.get_jsontable_settings(), data2=reader.get_jsontable_form(), toggle_off_keys2=[0], toggle_off_keys=reader.toggle_off_keys, ultra_form=reader.ultra_form, buttons=reader.buttons, enter_show=enter_show, stdout=stdout, stderr=stderr, url_experiment=(None if table_name is None else url_for( 'table', **{ key: value for key, value in zip(('schema', 'table', 'subtable'), table_name.split('.')) })))
def experimentprogress(): """get lastline of ouput """ return config.get('subprocess', Run()).lastline
jobs = config['schemata'][schema].schema.jobs jobs.reserve(table_class.full_table_name, primary_dict) try: with table_class.connection.transaction: dynamicform.insert(data['experiment_form'], check_reserved=False) command = [ "python", "-u", f"{args.script}", "--location", f"{args.location}", ] process = Run() cwd = os.path.dirname(args.script) process.start(command, cwd) lnumbers = 0 while True: length = len(process.lines) if length > lnumbers: print(''.join(process.lines[lnumbers:length])) lnumbers = length if process.p is not None and process.p.poll() is not None: break if process.thread.is_alive(): process.wait()
def run(self, button): """run subprocess given the button key """ # assert that script exists script = self.buttons[button].get('script') script_file = os.path.join(self.autoscript_filepath, script) assert os.path.exists(script_file), f'script {script} does not exist.' # get formatted form data and post_process keys = self.buttons[button].get('validate', []) data = {} for key in keys: idata = getattr(self.ultra_form, key).form.get_formatted() if idata is not None: idata = self.post_process_dict[key](idata) data[key] = idata # add experiment_form data['experiment_form'] = ( self.ultra_form.experiment_form.form.get_formatted()) # save configurations to pickle file with open(self.current_config_file, 'wb') as f: pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL) process = config.get('subprocess', Run()) if not process.running: if self.buttons[button].get('insert', False): # running the insert script command = [ "python", "-u", f"{INSERT_SCRIPT}", "--tablename", f"{self.table_name}", "--script", f"{script_file}", "--location", f"{self.current_config_file}", "--outputfile", f"{self.buttons.get('outputfile', 'null')}", "--configattr", f"{self.buttons.get('configattr', 'null')}", "--outputattr", f"{self.buttons.get('outputattr', 'null')}", ] else: # just run the python script command = [ "python", "-u", f"{script_file}", "--location", f"{self.current_config_file}", ] process.start(command, os.path.dirname(script_file)) config['subprocess'] = process flash(f'running script {script}') else: flash('Abort running subprocess before running a new process', 'warning')
def generate_stdout(): return Response(gen_stdout(config.get('subprocess', Run())), mimetype='multipart/x-mixed-replace')