def inputs():
user = root.authorized()
app = request.query.app
cid = request.query.cid
if re.search("/", cid):
(owner, c) = cid.split("/")
else:
owner = user
c = cid
shared = jobs(cid=c).shared
# only allow admin to see other user's cases that have not been shared
if owner != user and shared != "True" and user != "admin":
return template('error', err="access forbidden")
run_dir = os.path.join(user_dir, owner, root.myapps[app].appname, c)
fn = os.path.join(run_dir, root.myapps[app].simfn)
inputs = slurp_file(fn)
# the following line will convert HTML chars like > to entities >
# this is needed so that XML input files will show paramters labels
inputs = cgi.escape(inputs)
desc = jobs(cid=c).description
params = {
'cid': cid,
'contents': inputs,
'app': app,
'user': owner,
'fn': fn,
'description': desc
}
return template('more', params)
def diff_jobs():
user = root.authorized()
app = root.active_app()
selected_cases = request.query.selected_diff_cases
cases = selected_cases.rstrip(':').split(':')
cids = list()
contents = list()
for jid in cases:
cid = jobs(jid).cid
cids.append(cid)
app = jobs(jid).app
base_dir = os.path.join(user_dir, user, root.myapps[app].appname)
fn = os.path.join(base_dir, cid, root.myapps[app].simfn)
content = slurp_file(fn).splitlines(1)
contents.append(content)
import difflib
d = difflib.Differ()
result = list(d.compare(contents[0], contents[1]))
title = "diff " + cids[0] + " " + cids[1]
params = { 'cid': cid, 'contents': ' '.join(result), 'app': app, 'user': user, 'fn': title }
return template('more', params)
def annotate_job():
root.authorized()
cid = request.forms.cid
# jid = request.forms.jid
desc = request.forms.description
desc = desc.replace(',', ', ')
jobs(cid=cid).update_record(description=desc)
db.commit()
redirect('/jobs')
def output():
user = root.authorized()
app = request.query.app
cid = request.query.cid
jid = request.query.jid
if re.search("/", cid):
(owner, c) = cid.split("/")
else:
owner = user
c = cid
shared = jobs(cid=c).shared
# only allow admin to see other user's cases that have not been shared
if owner != user and shared != "True" and user != "admin":
return template('error', err="access forbidden")
run_dir = os.path.join(user_dir, owner, root.myapps[app].appname, c)
fn = os.path.join(run_dir, root.myapps[app].outfn)
# prevent 500 error in case config.worker not defined in config.py
try:
config.worker
except:
config.worker = 'local'
if config.worker == 'remote':
params = {'user': user, 'app': app, 'cid': cid}
resp = requests.get(config.remote_worker_url + '/output',
params=params)
output = resp.text
else:
output = slurp_file(fn)
# the following line will convert HTML chars like > to entities >
# this is needed so that XML input files will show paramters labels
output = cgi.escape(output)
desc = jobs(cid=c).description
params = {
'cid': cid,
'contents': output,
'app': app,
'user': owner,
'owner': owner,
'fn': fn,
'description': desc
}
if jid: params['jid'] = jid
return template('more', params)
def share_case():
root.authorized()
jid = request.forms.jid
jobs(id=jid).update_record(shared="True")
db.commit()
# increase count in database for every user
for u in db().select(users.ALL):
nmsg = users(user=u.user).new_shared_jobs or 0
users(user=u.user).update_record(new_shared_jobs=nmsg+1)
db.commit()
redirect('/jobs')
def stop_job():
root.authorized()
app = request.forms.app
cid = request.forms.cid
jid = request.forms.jid
if re.search("/", cid):
return template("error", err="only possible to stop cases that you own")
root.sched.stop(jid)
time.sleep(0.1)
jobs(jid).update_record(state="X")
db.commit()
redirect("/case?app="+app+"&cid="+cid+"&jid="+jid)
def plot_flot_3d(plot, cid, app, sim_dir, owner, user, plot_title, pltid):
# to handle data in user/cid format when looking at shared cases
if re.search("/", cid):
(owner, c) = cid.split("/")
else:
owner = user
c = cid
desc = jobs(cid=c).description
list_of_plots = db((apps.id == plots.appid) & (apps.name == app)).select()
options = json.loads(plot['options'])
plot_data = []
z_data = []
data_dir = os.path.join(sim_dir, options['directory'])
z_property = options['z_property']
file_names = sorted(os.listdir(data_dir))
for file_name in file_names:
file_path = os.path.join(data_dir, file_name)
if os.path.isfile(file_path) and not file_name.startswith(
'.') and file_name.endswith('.json'):
with open(file_path) as file_:
file_data = json.load(file_)
all_series = []
for source in options['datasources']:
series = {
'data':
zip(file_data[source['x_property']],
file_data[source['y_property']]),
}
series.update(source['data_def'])
all_series.append(series)
plot_data.append(all_series)
z_data.append(file_data[z_property])
params = {
'app': app,
'cid': cid,
'description': desc,
'owner': owner,
'plot_title': plot_title,
'pltid': pltid,
'rows': list_of_plots,
'stats': '',
'user': user,
'options_json': json.dumps(options['flot_options']),
'data_json': json.dumps(plot_data),
'z_data_json': json.dumps(z_data),
'z_label_json': json.dumps(options['z_label']),
}
return template('plots/flot-3d', params)
def more():
"""given a form with the attribute plotpath,
output the file to the browser"""
user = root.authorized()
app = request.query.app
cid = request.query.cid
filepath = request.query.filepath
# get the owner from the filepath
# e.g. "user_data/wes/mendel/y23022/file.dat"
path_list = filepath.split("/")
owner = path_list[1]
if re.search("/", cid):
(_, c) = cid.split("/")
else:
c = cid
shared = jobs(cid=c).shared
# only allow admin to see other user's cases that have not been shared
if owner != user and shared != "True" and user != "admin":
return template('error', err="access forbidden")
contents = slurp_file(filepath)
# convert html tags to entities (e.g. < to <)
contents = cgi.escape(contents)
params = {
'cid': c,
'contents': contents,
'app': app,
'user': user,
'fn': filepath
}
return template('more', params)
def delete_jobs():
user = root.authorized()
selected_cases = request.forms.selected_cases
cases = selected_cases.rstrip(':').split(':')
# in case someone selected elements twice, get unique cases
cases = list(set(cases))
for jid in cases:
cid = jobs(id=jid).cid
app = jobs(id=jid).app
path = os.path.join(user_dir, user, app, cid)
if cid is not None:
print "removing path:", path
if os.path.isdir(path): shutil.rmtree(path)
root.sched.stop(jid)
root.sched.qdel(jid)
else:
print "ERROR: not removing path:", path, "because cid missing"
redirect("/jobs")
def list_files():
user = root.authorized()
cid = request.query.cid
app = request.query.app
path = request.query.path
params = dict()
params['cid'] = cid
params['app'] = app
params['user'] = user
q = request.query.q
if "." not in q or q == "*.*": q = ""
if re.search("/", cid):
owner, cid = cid.split("/")
else:
owner = user
shared = jobs(cid=cid).shared
# only allow admin to see other user's cases that have not been shared
if owner != user and shared != "True" and user != "admin":
return template('error', err="access forbidden")
if not path:
path = os.path.join(user_dir, owner, app, cid)
params['path'] = path
if q:
_, ext = q.split('.')
params['files'] = sorted(
[fn for fn in os.listdir(path) if fn.endswith(ext)])
else:
q = ""
params['files'] = sorted(os.listdir(path))
params['q'] = q
num_files = len(params['files'])
params['status'] = "listing " + str(num_files) + " files"
params['description'] = jobs(cid=cid).description
return template('files', params)
def get_user_data(filepath):
user = root.authorized()
# filepath = request.query.filepath
# get the owner from the filepath
# e.g. "user_data/wes/mendel/y23022/file.dat"
path_list = filepath.split("/")
owner = path_list[0]
cid = path_list[2]
shared = jobs(cid=cid).shared
# print filepath, path_list, shared
# only allow admin to see other user's cases that have not been shared
if owner != user and shared != "True" and user != "admin":
return template('error', err="access forbidden")
return static_file(filepath, root=user_dir)
def delete_job(jid):
user = root.authorized()
app = request.forms.app
cid = request.forms.cid
state = jobs(jid).state
if re.search("/", cid):
return template("error", err="only possible to delete cases that you own")
if state != "R":
path = os.path.join(user_dir, user, app, cid)
if os.path.isdir(path): shutil.rmtree(path)
root.sched.stop(jid)
root.sched.qdel(jid)
else:
return template("error", err="cannot delete while job is still running")
redirect("/jobs")
def plot_interface(pltid):
user = root.authorized()
app = request.query.app
cid = request.query.cid
jid = request.query.jid
params = dict()
if not cid:
params[
'err'] = "No case id specified. First select a case id from the list of jobs."
return template('error', params)
if re.search("/", cid):
(owner, c) = cid.split("/")
else:
owner = user
c = cid
shared = jobs(cid=c).shared
# only allow admin to see other user's cases that have not been shared
if owner != user and shared != "True" and user != "admin":
return template('error', err="access forbidden")
inputs, _, _ = root.myapps[app].read_params(owner, c)
sim_dir = os.path.join(user_dir, owner, app, c)
# use pltid of 0 to trigger finding the first pltid for the current app
if int(pltid) == 0:
query = (apps.id == plots.appid) & (apps.name == app)
result = db(query).select().first()
if result: pltid = result['plots']['id']
p = Plot()
# get the data for the pltid given
try:
result = db(plots.id == pltid).select().first()
plottype = result['ptype']
plot_title = result['title']
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
print traceback.print_exception(exc_type, exc_value, exc_traceback)
redirect('/plots/edit?app=' + app + '&cid=' + cid)
# if plot not in DB return error
if plottype is None:
params = {'cid': cid, 'app': app, 'user': user}
params['err'] = "Sorry! This app does not support plotting capability"
return template('error', params)
# determine which view template to use
if plottype == 'flot-cat':
tfn = 'plots/flot-cat'
elif plottype == 'flot-scatter':
tfn = 'plots/flot-scatter'
elif plottype == 'flot-scatter-animated':
tfn = 'plots/flot-scatter-animated' # for backwards compatability
elif plottype == 'flot-line':
tfn = 'plots/flot-scatter'
elif plottype == 'plotly-hist':
tfn = 'plots/plotly-hist'
elif plottype == 'mpl-line' or plottype == 'mpl-bar':
redirect('/mpl/' + pltid + '?app=' + app + '&cid=' + cid)
elif plottype == 'handson':
tfn = 'plots/handson'
elif plottype == 'flot-3d':
return plot_flot_3d(result, cid, app, sim_dir, owner, user, plot_title,
pltid)
else:
return template("error", err="plot type not supported: " + plottype)
if result['options']:
options = replace_tags(result['options'], inputs)
else:
options = ''
# get list of all plots for this app
query = (apps.id == plots.appid) & (apps.name == app)
list_of_plots = db(query).select()
# extract data from files
data = []
ticks = []
plotpath = ''
result = db(datasource.pltid == pltid).select()
datadef = ""
for r in result:
plotfn = r['filename']
# in addition to supporting input params, also support case id
if "cid" not in inputs: inputs["cid"] = c
# replace <cid>.dat with xyz123.dat
plotfn = replace_tags(plotfn, inputs)
plotpath = os.path.join(sim_dir, plotfn)
# handle CSV data
_, file_extension = os.path.splitext(plotfn)
if file_extension == '.csv':
data = p.get_csv_data(plotpath)
stats = ''
# handle X, Y columnar data
else:
cols = r['cols']
line_range = r['line_range']
try:
datadef += r['data_def'] + ", "
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
print traceback.print_exception(exc_type, exc_value,
exc_traceback)
datadef = ""
if cols.find(":") > 0: # two columns
num_fields = 2
(col1str, col2str) = cols.split(":")
col1 = int(col1str)
col2 = int(col2str)
else: # single column
num_fields = 1
col1 = int(cols)
# do some postprocessing
if line_range is not None:
# to prevent breaking current spc apps, still support
# expressions like 1:1000, but in the future this should
# be changed to a range 1-1000. Therefore, using : is deprecated
# and will be removed in the future.
(line1str, line2str) = re.split("[-:]", line_range)
line1 = int(line1str)
## there is a problem with the following statement
## shows up in mendel app
# if root.myapps[app].postprocess > 0:
# dat = process.postprocess(plotpath, line1, line2)
# else:
try: # if line2 is specified
line2 = int(line2str)
dat = p.get_data(plotpath, col1, col2, line1, line2)
except: # if line2 not specified
exc_type, exc_value, exc_traceback = sys.exc_info()
print traceback.print_exception(exc_type, exc_value,
exc_traceback)
if num_fields == 2:
dat = p.get_data(plotpath, col1, col2, line1)
else: # single column of data
dat = p.get_data(plotpath, col1)
# remove this app-specific code in future
if app == "fpg":
import process
dat = process.postprocess(plotpath, line1, line2)
else:
dat = p.get_data(plotpath, col1, col2)
if dat == -1:
stats = "ERROR: Could not read data file"
elif dat == -2:
stats = "ERROR: file exists, but problem parsing data. Are column and line ranges setup properly? Is all the data there?"
else:
stats = compute_stats(plotpath)
# [[1,2,3]] >>> [1,2,3]
# clean data
#dat = [d.replace('?', '0') for d in dat]
data.append(dat)
if num_fields == 1: data = data[0]
if plottype == 'flot-cat':
ticks = p.get_ticks(plotpath, col1, col2)
desc = jobs(cid=c).description
params = {
'cid': cid,
'pltid': pltid,
'data': data,
'app': app,
'user': user,
'owner': owner,
'ticks': ticks,
'plot_title': plot_title,
'plotpath': plotpath,
'rows': list_of_plots,
'options': options,
'datadef': datadef,
'stats': stats,
'description': desc
}
if jid: params['jid'] = jid
return template(tfn, params)
def merge(rtype):
user = root.authorized()
selected_cases = request.forms.selected_merge_cases
jids = selected_cases.rstrip(':').split(':')
cases = list()
output = dict()
for jid in jids:
app = jobs(id=jid).app
cid = jobs(id=jid).cid
cases.append(cid)
fn = replace_tags(request.forms.file_pattern, {'cid': cid})
path = os.path.join(user_dir, user, app, cid, fn)
with open(path, "r") as infile:
# Loop over lines in each file
for line in infile:
line = str(line)
# Skip comment lines
if not re.search('^#', line):
items = line.split()
# If a line matching this one has been encountered in a previous
# file, add the column values
if len(items) > 0:
currkey = int(items[0])
if currkey in output.keys():
for ii in range(len(output[currkey])):
output[currkey][ii] += float(items[ii+1])
# Otherwise, add a new key to the output and create the columns
else:
output[currkey] = list(map(float, items[1:]))
# Get total number of files for calculating average
if rtype == "sum":
nfile = 1
elif rtype == "avg":
nfile = len(cases)
else:
raise ValueError(rtype + " operation no supported")
print "nfile:", nfile, selected_cases
# generate new case_id for outputtinging merged files
while True:
ocid = rand_cid()
run_dir = os.path.join(user_dir, user, app, ocid)
# check if this case exists or not, if it exists generate a new case id
if not os.path.exists(run_dir):
os.makedirs(run_dir)
break
# write a default input file b/c SPC requires a file in each job dir
root.myapps[app].params['case_id'] = ocid
root.myapps[app].write_params(root.myapps[app].params, user)
# Sort the output keys
skey = sorted(output.keys())
lines = list()
# Loop through sorted keys and print each averaged column to stdout
for key in skey:
outline = str(int(key))
for item in output[key]:
outline += ' ' + str("{0:.3e}".format(item/nfile,3))
lines.append(outline)
ofn = replace_tags(request.forms.file_pattern, {'cid': ocid})
with open(os.path.join(run_dir, ofn), 'w') as f:
f.writelines("%s\n" % l for l in lines)
# save case to DB
uid = users(user=user).id
desc = "merge " + rtype + " cases " + str(cases)
db.jobs.insert(uid=uid, app=app, cid=ocid, state='C', description=desc,
time_submit=time.asctime(), np=config.np, priority=1)
db.commit()
return "merged file written to " + run_dir + "<meta http-equiv='refresh' content='2; url=/jobs'>"
def unshare_case():
root.authorized()
jid = request.forms.jid
jobs(id=jid).update_record(shared="False")
db.commit()
redirect('/jobs')
def star_case():
root.authorized()
jid = request.forms.jid
jobs(id=jid).update_record(starred="True")
db.commit()
redirect('/jobs')
def case():
user = root.authorized()
app = request.query.app
root.set_active(app)
cid = request.query.cid
jid = request.query.jid
if re.search("/", cid):
(owner, c) = cid.split("/")
shared = jobs(cid=c).shared
# only allow admin to see other user's cases that have not been shared
if owner != user and shared != "True" and user != "admin":
return template('error', err="access forbidden")
else:
owner, c = user, cid
# if case does not exist return error
if jobs(cid=c) is None:
return template('error', err="case does not exist")
state = jobs(cid=c).state
run_dir = os.path.join(user_dir, owner, root.myapps[app].appname, c)
fn = os.path.join(run_dir, root.myapps[app].outfn)
# note: eventually need to merge the following two into one
if re.search("/", cid):
sid = request.query.sid # id of item in shared
output = slurp_file(fn)
params = {
'cid': cid,
'app': app,
'contents': output,
'sid': sid,
'user': user,
'fn': fn,
'state': state,
'owner': owner
}
if jid: params['jid'] = jid
return template('case_public', params)
else:
result = db(jobs.cid == cid).select().first()
desc = result['description']
shared = result['shared']
params = {
'cid': cid,
'app': app,
'jid': jid,
'user': user,
'fn': fn,
'description': desc,
'shared': shared,
'state': state,
'owner': owner
}
if jid: params['jid'] = jid
#return template('case', params)
return template('case_tail-f', params)
