def inputs(): user = root.authorized() app = request.query.app cid = request.query.cid if re.search("/", cid): (owner, c) = cid.split("/") else: owner = user c = cid shared = jobs(cid=c).shared # only allow admin to see other user's cases that have not been shared if owner != user and shared != "True" and user != "admin": return template('error', err="access forbidden") run_dir = os.path.join(user_dir, owner, root.myapps[app].appname, c) fn = os.path.join(run_dir, root.myapps[app].simfn) inputs = slurp_file(fn) # the following line will convert HTML chars like > to entities > # this is needed so that XML input files will show paramters labels inputs = cgi.escape(inputs) desc = jobs(cid=c).description params = { 'cid': cid, 'contents': inputs, 'app': app, 'user': owner, 'fn': fn, 'description': desc } return template('more', params)
def diff_jobs(): user = root.authorized() app = root.active_app() selected_cases = request.query.selected_diff_cases cases = selected_cases.rstrip(':').split(':') cids = list() contents = list() for jid in cases: cid = jobs(jid).cid cids.append(cid) app = jobs(jid).app base_dir = os.path.join(user_dir, user, root.myapps[app].appname) fn = os.path.join(base_dir, cid, root.myapps[app].simfn) content = slurp_file(fn).splitlines(1) contents.append(content) import difflib d = difflib.Differ() result = list(d.compare(contents[0], contents[1])) title = "diff " + cids[0] + " " + cids[1] params = { 'cid': cid, 'contents': ' '.join(result), 'app': app, 'user': user, 'fn': title } return template('more', params)
def annotate_job(): root.authorized() cid = request.forms.cid # jid = request.forms.jid desc = request.forms.description desc = desc.replace(',', ', ') jobs(cid=cid).update_record(description=desc) db.commit() redirect('/jobs')
def output(): user = root.authorized() app = request.query.app cid = request.query.cid jid = request.query.jid if re.search("/", cid): (owner, c) = cid.split("/") else: owner = user c = cid shared = jobs(cid=c).shared # only allow admin to see other user's cases that have not been shared if owner != user and shared != "True" and user != "admin": return template('error', err="access forbidden") run_dir = os.path.join(user_dir, owner, root.myapps[app].appname, c) fn = os.path.join(run_dir, root.myapps[app].outfn) # prevent 500 error in case config.worker not defined in config.py try: config.worker except: config.worker = 'local' if config.worker == 'remote': params = {'user': user, 'app': app, 'cid': cid} resp = requests.get(config.remote_worker_url + '/output', params=params) output = resp.text else: output = slurp_file(fn) # the following line will convert HTML chars like > to entities > # this is needed so that XML input files will show paramters labels output = cgi.escape(output) desc = jobs(cid=c).description params = { 'cid': cid, 'contents': output, 'app': app, 'user': owner, 'owner': owner, 'fn': fn, 'description': desc } if jid: params['jid'] = jid return template('more', params)
def share_case(): root.authorized() jid = request.forms.jid jobs(id=jid).update_record(shared="True") db.commit() # increase count in database for every user for u in db().select(users.ALL): nmsg = users(user=u.user).new_shared_jobs or 0 users(user=u.user).update_record(new_shared_jobs=nmsg+1) db.commit() redirect('/jobs')
def stop_job(): root.authorized() app = request.forms.app cid = request.forms.cid jid = request.forms.jid if re.search("/", cid): return template("error", err="only possible to stop cases that you own") root.sched.stop(jid) time.sleep(0.1) jobs(jid).update_record(state="X") db.commit() redirect("/case?app="+app+"&cid="+cid+"&jid="+jid)
def plot_flot_3d(plot, cid, app, sim_dir, owner, user, plot_title, pltid): # to handle data in user/cid format when looking at shared cases if re.search("/", cid): (owner, c) = cid.split("/") else: owner = user c = cid desc = jobs(cid=c).description list_of_plots = db((apps.id == plots.appid) & (apps.name == app)).select() options = json.loads(plot['options']) plot_data = [] z_data = [] data_dir = os.path.join(sim_dir, options['directory']) z_property = options['z_property'] file_names = sorted(os.listdir(data_dir)) for file_name in file_names: file_path = os.path.join(data_dir, file_name) if os.path.isfile(file_path) and not file_name.startswith( '.') and file_name.endswith('.json'): with open(file_path) as file_: file_data = json.load(file_) all_series = [] for source in options['datasources']: series = { 'data': zip(file_data[source['x_property']], file_data[source['y_property']]), } series.update(source['data_def']) all_series.append(series) plot_data.append(all_series) z_data.append(file_data[z_property]) params = { 'app': app, 'cid': cid, 'description': desc, 'owner': owner, 'plot_title': plot_title, 'pltid': pltid, 'rows': list_of_plots, 'stats': '', 'user': user, 'options_json': json.dumps(options['flot_options']), 'data_json': json.dumps(plot_data), 'z_data_json': json.dumps(z_data), 'z_label_json': json.dumps(options['z_label']), } return template('plots/flot-3d', params)
def more(): """given a form with the attribute plotpath, output the file to the browser""" user = root.authorized() app = request.query.app cid = request.query.cid filepath = request.query.filepath # get the owner from the filepath # e.g. "user_data/wes/mendel/y23022/file.dat" path_list = filepath.split("/") owner = path_list[1] if re.search("/", cid): (_, c) = cid.split("/") else: c = cid shared = jobs(cid=c).shared # only allow admin to see other user's cases that have not been shared if owner != user and shared != "True" and user != "admin": return template('error', err="access forbidden") contents = slurp_file(filepath) # convert html tags to entities (e.g. < to <) contents = cgi.escape(contents) params = { 'cid': c, 'contents': contents, 'app': app, 'user': user, 'fn': filepath } return template('more', params)
def delete_jobs(): user = root.authorized() selected_cases = request.forms.selected_cases cases = selected_cases.rstrip(':').split(':') # in case someone selected elements twice, get unique cases cases = list(set(cases)) for jid in cases: cid = jobs(id=jid).cid app = jobs(id=jid).app path = os.path.join(user_dir, user, app, cid) if cid is not None: print "removing path:", path if os.path.isdir(path): shutil.rmtree(path) root.sched.stop(jid) root.sched.qdel(jid) else: print "ERROR: not removing path:", path, "because cid missing" redirect("/jobs")
def list_files(): user = root.authorized() cid = request.query.cid app = request.query.app path = request.query.path params = dict() params['cid'] = cid params['app'] = app params['user'] = user q = request.query.q if "." not in q or q == "*.*": q = "" if re.search("/", cid): owner, cid = cid.split("/") else: owner = user shared = jobs(cid=cid).shared # only allow admin to see other user's cases that have not been shared if owner != user and shared != "True" and user != "admin": return template('error', err="access forbidden") if not path: path = os.path.join(user_dir, owner, app, cid) params['path'] = path if q: _, ext = q.split('.') params['files'] = sorted( [fn for fn in os.listdir(path) if fn.endswith(ext)]) else: q = "" params['files'] = sorted(os.listdir(path)) params['q'] = q num_files = len(params['files']) params['status'] = "listing " + str(num_files) + " files" params['description'] = jobs(cid=cid).description return template('files', params)
def get_user_data(filepath): user = root.authorized() # filepath = request.query.filepath # get the owner from the filepath # e.g. "user_data/wes/mendel/y23022/file.dat" path_list = filepath.split("/") owner = path_list[0] cid = path_list[2] shared = jobs(cid=cid).shared # print filepath, path_list, shared # only allow admin to see other user's cases that have not been shared if owner != user and shared != "True" and user != "admin": return template('error', err="access forbidden") return static_file(filepath, root=user_dir)
def delete_job(jid): user = root.authorized() app = request.forms.app cid = request.forms.cid state = jobs(jid).state if re.search("/", cid): return template("error", err="only possible to delete cases that you own") if state != "R": path = os.path.join(user_dir, user, app, cid) if os.path.isdir(path): shutil.rmtree(path) root.sched.stop(jid) root.sched.qdel(jid) else: return template("error", err="cannot delete while job is still running") redirect("/jobs")
def plot_interface(pltid): user = root.authorized() app = request.query.app cid = request.query.cid jid = request.query.jid params = dict() if not cid: params[ 'err'] = "No case id specified. First select a case id from the list of jobs." return template('error', params) if re.search("/", cid): (owner, c) = cid.split("/") else: owner = user c = cid shared = jobs(cid=c).shared # only allow admin to see other user's cases that have not been shared if owner != user and shared != "True" and user != "admin": return template('error', err="access forbidden") inputs, _, _ = root.myapps[app].read_params(owner, c) sim_dir = os.path.join(user_dir, owner, app, c) # use pltid of 0 to trigger finding the first pltid for the current app if int(pltid) == 0: query = (apps.id == plots.appid) & (apps.name == app) result = db(query).select().first() if result: pltid = result['plots']['id'] p = Plot() # get the data for the pltid given try: result = db(plots.id == pltid).select().first() plottype = result['ptype'] plot_title = result['title'] except: exc_type, exc_value, exc_traceback = sys.exc_info() print traceback.print_exception(exc_type, exc_value, exc_traceback) redirect('/plots/edit?app=' + app + '&cid=' + cid) # if plot not in DB return error if plottype is None: params = {'cid': cid, 'app': app, 'user': user} params['err'] = "Sorry! This app does not support plotting capability" return template('error', params) # determine which view template to use if plottype == 'flot-cat': tfn = 'plots/flot-cat' elif plottype == 'flot-scatter': tfn = 'plots/flot-scatter' elif plottype == 'flot-scatter-animated': tfn = 'plots/flot-scatter-animated' # for backwards compatability elif plottype == 'flot-line': tfn = 'plots/flot-scatter' elif plottype == 'plotly-hist': tfn = 'plots/plotly-hist' elif plottype == 'mpl-line' or plottype == 'mpl-bar': redirect('/mpl/' + pltid + '?app=' + app + '&cid=' + cid) elif plottype == 'handson': tfn = 'plots/handson' elif plottype == 'flot-3d': return plot_flot_3d(result, cid, app, sim_dir, owner, user, plot_title, pltid) else: return template("error", err="plot type not supported: " + plottype) if result['options']: options = replace_tags(result['options'], inputs) else: options = '' # get list of all plots for this app query = (apps.id == plots.appid) & (apps.name == app) list_of_plots = db(query).select() # extract data from files data = [] ticks = [] plotpath = '' result = db(datasource.pltid == pltid).select() datadef = "" for r in result: plotfn = r['filename'] # in addition to supporting input params, also support case id if "cid" not in inputs: inputs["cid"] = c # replace <cid>.dat with xyz123.dat plotfn = replace_tags(plotfn, inputs) plotpath = os.path.join(sim_dir, plotfn) # handle CSV data _, file_extension = os.path.splitext(plotfn) if file_extension == '.csv': data = p.get_csv_data(plotpath) stats = '' # handle X, Y columnar data else: cols = r['cols'] line_range = r['line_range'] try: datadef += r['data_def'] + ", " except: exc_type, exc_value, exc_traceback = sys.exc_info() print traceback.print_exception(exc_type, exc_value, exc_traceback) datadef = "" if cols.find(":") > 0: # two columns num_fields = 2 (col1str, col2str) = cols.split(":") col1 = int(col1str) col2 = int(col2str) else: # single column num_fields = 1 col1 = int(cols) # do some postprocessing if line_range is not None: # to prevent breaking current spc apps, still support # expressions like 1:1000, but in the future this should # be changed to a range 1-1000. Therefore, using : is deprecated # and will be removed in the future. (line1str, line2str) = re.split("[-:]", line_range) line1 = int(line1str) ## there is a problem with the following statement ## shows up in mendel app # if root.myapps[app].postprocess > 0: # dat = process.postprocess(plotpath, line1, line2) # else: try: # if line2 is specified line2 = int(line2str) dat = p.get_data(plotpath, col1, col2, line1, line2) except: # if line2 not specified exc_type, exc_value, exc_traceback = sys.exc_info() print traceback.print_exception(exc_type, exc_value, exc_traceback) if num_fields == 2: dat = p.get_data(plotpath, col1, col2, line1) else: # single column of data dat = p.get_data(plotpath, col1) # remove this app-specific code in future if app == "fpg": import process dat = process.postprocess(plotpath, line1, line2) else: dat = p.get_data(plotpath, col1, col2) if dat == -1: stats = "ERROR: Could not read data file" elif dat == -2: stats = "ERROR: file exists, but problem parsing data. Are column and line ranges setup properly? Is all the data there?" else: stats = compute_stats(plotpath) # [[1,2,3]] >>> [1,2,3] # clean data #dat = [d.replace('?', '0') for d in dat] data.append(dat) if num_fields == 1: data = data[0] if plottype == 'flot-cat': ticks = p.get_ticks(plotpath, col1, col2) desc = jobs(cid=c).description params = { 'cid': cid, 'pltid': pltid, 'data': data, 'app': app, 'user': user, 'owner': owner, 'ticks': ticks, 'plot_title': plot_title, 'plotpath': plotpath, 'rows': list_of_plots, 'options': options, 'datadef': datadef, 'stats': stats, 'description': desc } if jid: params['jid'] = jid return template(tfn, params)
def merge(rtype): user = root.authorized() selected_cases = request.forms.selected_merge_cases jids = selected_cases.rstrip(':').split(':') cases = list() output = dict() for jid in jids: app = jobs(id=jid).app cid = jobs(id=jid).cid cases.append(cid) fn = replace_tags(request.forms.file_pattern, {'cid': cid}) path = os.path.join(user_dir, user, app, cid, fn) with open(path, "r") as infile: # Loop over lines in each file for line in infile: line = str(line) # Skip comment lines if not re.search('^#', line): items = line.split() # If a line matching this one has been encountered in a previous # file, add the column values if len(items) > 0: currkey = int(items[0]) if currkey in output.keys(): for ii in range(len(output[currkey])): output[currkey][ii] += float(items[ii+1]) # Otherwise, add a new key to the output and create the columns else: output[currkey] = list(map(float, items[1:])) # Get total number of files for calculating average if rtype == "sum": nfile = 1 elif rtype == "avg": nfile = len(cases) else: raise ValueError(rtype + " operation no supported") print "nfile:", nfile, selected_cases # generate new case_id for outputtinging merged files while True: ocid = rand_cid() run_dir = os.path.join(user_dir, user, app, ocid) # check if this case exists or not, if it exists generate a new case id if not os.path.exists(run_dir): os.makedirs(run_dir) break # write a default input file b/c SPC requires a file in each job dir root.myapps[app].params['case_id'] = ocid root.myapps[app].write_params(root.myapps[app].params, user) # Sort the output keys skey = sorted(output.keys()) lines = list() # Loop through sorted keys and print each averaged column to stdout for key in skey: outline = str(int(key)) for item in output[key]: outline += ' ' + str("{0:.3e}".format(item/nfile,3)) lines.append(outline) ofn = replace_tags(request.forms.file_pattern, {'cid': ocid}) with open(os.path.join(run_dir, ofn), 'w') as f: f.writelines("%s\n" % l for l in lines) # save case to DB uid = users(user=user).id desc = "merge " + rtype + " cases " + str(cases) db.jobs.insert(uid=uid, app=app, cid=ocid, state='C', description=desc, time_submit=time.asctime(), np=config.np, priority=1) db.commit() return "merged file written to " + run_dir + "<meta http-equiv='refresh' content='2; url=/jobs'>"
def unshare_case(): root.authorized() jid = request.forms.jid jobs(id=jid).update_record(shared="False") db.commit() redirect('/jobs')
def star_case(): root.authorized() jid = request.forms.jid jobs(id=jid).update_record(starred="True") db.commit() redirect('/jobs')
def case(): user = root.authorized() app = request.query.app root.set_active(app) cid = request.query.cid jid = request.query.jid if re.search("/", cid): (owner, c) = cid.split("/") shared = jobs(cid=c).shared # only allow admin to see other user's cases that have not been shared if owner != user and shared != "True" and user != "admin": return template('error', err="access forbidden") else: owner, c = user, cid # if case does not exist return error if jobs(cid=c) is None: return template('error', err="case does not exist") state = jobs(cid=c).state run_dir = os.path.join(user_dir, owner, root.myapps[app].appname, c) fn = os.path.join(run_dir, root.myapps[app].outfn) # note: eventually need to merge the following two into one if re.search("/", cid): sid = request.query.sid # id of item in shared output = slurp_file(fn) params = { 'cid': cid, 'app': app, 'contents': output, 'sid': sid, 'user': user, 'fn': fn, 'state': state, 'owner': owner } if jid: params['jid'] = jid return template('case_public', params) else: result = db(jobs.cid == cid).select().first() desc = result['description'] shared = result['shared'] params = { 'cid': cid, 'app': app, 'jid': jid, 'user': user, 'fn': fn, 'description': desc, 'shared': shared, 'state': state, 'owner': owner } if jid: params['jid'] = jid #return template('case', params) return template('case_tail-f', params)