def get_mjd_for_tiling(pname, filt_name):
'''
Return a list of MJD and Exposure time of image used to produce a tile.
INPUT: PNAme
Fileter
'''
tile_info = get_tile(pname, filt_name)
query = db(db.t80tileImgs.Tile_ID == tile_info.id)
rcid = query.select(db.t80tileImgs.RC_ID)
proc_images = [
db(db.rc.id == rcidi.RC_ID).select(db.rc.ori_id).first()
for rcidi in rcid
]
images = [
db(db.t80oa.id == primg.ori_id).select(
db.t80oa.Date,
db.t80oa.Time,
db.t80oa.ExpTime,
).first() for primg in proc_images
]
image_date_time = [
"{0}T{1}".format(img.Date.strftime("%Y-%m-%d"),
img.Time.strftime("%H:%M:%S")) for img in images
]
mjd = Time(image_date_time).mjd
return [[mjd[i], float(images[i].ExpTime)] for i in range(len(images))]
def showapps():
user = root.authorized()
q = request.query.q
if not q:
result = db().select(apps.ALL, orderby=apps.name)
else:
result = db(
db.apps.name.contains(q, case_sensitive=False)
| db.apps.category.contains(q, case_sensitive=False)
| db.apps.description.contains(q, case_sensitive=False)).select()
# find out what apps have already been activated so that a user can't activate twice
uid = users(user=user).id
activated = db(app_user.uid == uid).select()
activated_apps = []
for row in activated:
activated_apps.append(row.appid)
if user == "admin":
configurable = True
else:
configurable = False
params = {'configurable': configurable, 'user': user}
return template('apps', params, rows=result, activated=activated_apps)
def export():
user = root.authorized()
if user != 'admin':
return template('error', err="must be admin to use export function")
app = request.forms.app
result = db(apps.name == app).select().first()
data = {}
data['name'] = result.name
data['description'] = result.description
data['category'] = result.category
data['language'] = result.language
data['input_format'] = result.input_format
data['command'] = result.command
data['preprocess'] = result.preprocess
data['postprocess'] = result.postprocess
assets = list()
if result.assets is not None:
for asset in result.assets.split(","):
assets.append(asset.strip())
data['assets'] = assets
appid = apps(name=app).id
myplots = db(plots.appid == appid).select()
data['plots'] = list()
for p in myplots:
thisplot = {}
thisplot['ptype'] = p.ptype
thisplot['title'] = p.title
thisplot['options'] = p.options
thisplot['datasource'] = list()
myds = db(datasource.pltid == p.id).select()
for ds in myds:
thisds = {}
thisds['label'] = ds.label
thisds['filename'] = ds.filename
thisds['cols'] = ds.cols
thisds['line_range'] = ds.line_range
thisds['data_def'] = ds.data_def
thisplot['datasource'].append(thisds)
data['plots'].append(thisplot)
path = os.path.join(apprw.apps_dir, app, 'spc.json')
with open(path, 'w') as outfile:
json.dump(data, outfile, indent=3)
return "spc.json file written to " + path + "<meta http-equiv='refresh' content='2; url=/app/" + app + "'>"
def view_app(app):
user = root.authorized()
if app: root.set_active(app)
else: redirect('/myapps')
if user != 'admin':
return template('error', err="must be admin to edit app")
cid = request.query.cid
result = db(apps.name == app).select().first()
params = {}
params['app'] = app
params['user'] = user
params['cid'] = cid
#if request.query.edit:
# return template('appedit', params, rows=result)
#else:
try:
return template('app', params, rows=result)
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
print traceback.print_exception(exc_type, exc_value, exc_traceback)
return template(
'error',
err="there was a problem showing the app template. Check traceback."
)
def admin_show_users():
user = root.authorized()
if not user == "admin":
return template("error", err="must be admin to delete")
result = db().select(users.ALL)
params = { 'user': user, 'app': root.active_app() }
return template('admin/users', params, rows=result)
def execute():
app = request.forms['app']
user = request.forms['user']
cid = request.forms['cid']
desc = request.forms['desc']
np = request.forms['np']
appmod = pickle.loads(request.forms['appmod'])
# remove the appmod key
del request.forms['appmod']
appmod.write_params(request.forms, user)
# if preprocess is set run the preprocessor
try:
if appmod.preprocess:
run_params, _, _ = appmod.read_params(user, cid)
base_dir = os.path.join(user_dir, user, app)
process.preprocess(run_params, appmod.preprocess, base_dir)
if appmod.preprocess == "terra.in":
appmod.outfn = "out" + run_params['casenum'] + ".00"
except:
return template('error',
err="There was an error with the preprocessor")
# submit job to queue
try:
priority = db(users.user == user).select(
users.priority).first().priority
uid = users(user=user).id
jid = sched.qsub(app, cid, uid, np, priority, desc)
return str(jid)
#redirect("http://localhost:"+str(config.port)+"/case?app="+str(app)+"&cid="+str(cid)+"&jid="+str(jid))
except OSError:
return "ERROR: a problem occurred"
def get_all_jobs():
user = root.authorized()
if not user == "admin":
return template("error", err="must be admin to use this feature")
cid = request.query.cid
app = request.query.app or root.active_app()
n = request.query.n
if not n:
n = config.jobs_num_rows
else:
n = int(n)
# sort by descending order of jobs.id
result = db((db.jobs.uid==users.id)).select(orderby=~jobs.id)[:n]
# clear notifications
users(user=user).update_record(new_shared_jobs=0)
db.commit()
params = {}
params['cid'] = cid
params['app'] = app
params['user'] = user
params['n'] = n
params['num_rows'] = config.jobs_num_rows
return template('shared', params, rows=result)
def get_stats():
root.authorized()
params = {}
# number of jobs in queued, running, and completed states
params['nq'] = db(jobs.state=='Q').count()
params['nr'] = db(jobs.state=='R').count()
params['nc'] = db(jobs.state=='C').count()
params['cpu'] = psutil.cpu_percent()
params['vm'] = psutil.virtual_memory().percent
params['disk'] = psutil.disk_usage('/').percent
params['cid'] = request.query.cid
params['app'] = request.query.app
return template("stats", params)
def plot_flot_3d(plot, cid, app, sim_dir, owner, user, plot_title, pltid):
# to handle data in user/cid format when looking at shared cases
if re.search("/", cid):
(owner, c) = cid.split("/")
else:
owner = user
c = cid
desc = jobs(cid=c).description
list_of_plots = db((apps.id == plots.appid) & (apps.name == app)).select()
options = json.loads(plot['options'])
plot_data = []
z_data = []
data_dir = os.path.join(sim_dir, options['directory'])
z_property = options['z_property']
file_names = sorted(os.listdir(data_dir))
for file_name in file_names:
file_path = os.path.join(data_dir, file_name)
if os.path.isfile(file_path) and not file_name.startswith(
'.') and file_name.endswith('.json'):
with open(file_path) as file_:
file_data = json.load(file_)
all_series = []
for source in options['datasources']:
series = {
'data':
zip(file_data[source['x_property']],
file_data[source['y_property']]),
}
series.update(source['data_def'])
all_series.append(series)
plot_data.append(all_series)
z_data.append(file_data[z_property])
params = {
'app': app,
'cid': cid,
'description': desc,
'owner': owner,
'plot_title': plot_title,
'pltid': pltid,
'rows': list_of_plots,
'stats': '',
'user': user,
'options_json': json.dumps(options['flot_options']),
'data_json': json.dumps(plot_data),
'z_data_json': json.dumps(z_data),
'z_label_json': json.dumps(options['z_label']),
}
return template('plots/flot-3d', params)
def get_shared():
"""Return the records from the shared table."""
user = root.authorized()
cid = request.query.cid
app = request.query.app or root.active_app()
n = request.query.n
if not n:
n = config.jobs_num_rows
else:
n = int(n)
# sort by descending order of jobs.id
result = db((db.jobs.shared=="True") & (db.jobs.uid==users.id)).select(orderby=~jobs.id)[:n]
# result = db((db.jobs.shared=="True") &
# (jobs.gid == users.gid)).select(orderby=~jobs.id)[:n]
# clear notifications
users(user=user).update_record(new_shared_jobs=0)
db.commit()
params = {}
params['cid'] = cid
params['app'] = app
params['user'] = user
params['n'] = n
params['num_rows'] = config.jobs_num_rows
return template('shared', params, rows=result)
def get_pnames():
'''
Return all PName from t80tiles
'''
pname = db().select(db.t80tiles.PName)
pname = [pn.PName for pn in pname]
return pname
def aws_conn(id):
"""create a connection to the EC2 machine and return the handle"""
user = root.authorized()
uid = users(user=user).id
try:
creds = db(db.aws_creds.uid == uid).select().first()
account_id = creds['account_id']
secret = creds['secret']
key = creds['key']
instances = db(db.aws_instances.id == id).select().first()
instance = instances['instance']
region = instances['region']
rate = instances['rate'] or 0.
return EC2(key, secret, account_id, instance, region, rate)
except:
return template('error', err="problem validating AWS credentials")
def editplotdef(pltid):
user = root.authorized()
if user != 'admin':
return template('error', err="must be admin to edit plots")
app = request.forms.app
result = db(plots.id == pltid).select().first()
params = {'app': app, 'user': user}
return template('plots/edit_plot', params, row=result)
def searchImages(startDate, endDate, frametype, filt):
"""
Return the Number of images in the Pipeline Data Base.
"""
typeID = db(db.frametype.Name == frametype).select(db.frametype.id).first()
nimages = 0
if (frametype == "BIAS"):
nimages = db((db.t80oa.ImageType_ID == typeID)
& (db.t80oa.Date >= startDate)
& (db.t80oa.Date <= endDate)).count()
else:
filterID = db(db.filter.Name == filt).select(db.filter.id).first()
nimages = db((db.t80oa.ImageType_ID == typeID)
& (db.t80oa.Filter_ID == filterID)
& (db.t80oa.Date >= startDate)
& (db.t80oa.Date <= endDate)).count()
return nimages
def get_aws():
user = root.authorized()
cid = request.query.cid
app = request.query.app or root.active_app()
uid = db(users.user == user).select(users.id).first()
#creds = db().select(db.aws_creds.ALL)
creds = db(aws_creds.uid == uid).select()
# look for aws instances registered by the current user
# which means first need to get the uid
instances = db(aws_instances.uid == uid).select()
params = {}
params['cid'] = cid
params['app'] = app
params['user'] = user
if request.query.status:
params['status'] = request.query.status
return template('aws', params, creds=creds, instances=instances)
def app_edit(appid):
user = root.authorized()
if user != 'admin':
return template('error', err="must be admin to edit app")
cid = request.forms.cid
app = request.forms.app
result = db(apps.name == app).select().first()
params = {'app': app, 'cid': cid}
return template('app_edit', params, rows=result)
def edit_datasource(pltid, dsid):
"""create a new datasource for given plot"""
user = root.authorized()
if user != 'admin':
return template('error', err="must be admin to edit plots")
app = request.query.app
query = (datasource.id == dsid)
result = db(query).select().first()
params = {'app': app, 'pltid': pltid, 'dsid': dsid}
return template('plots/edit_datasource', params, row=result)
def get_status(jid):
# following headers are needed because of CORS
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers[
'Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS'
response.headers[
'Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token'
resp = db(jobs.id == jid).select(jobs.state).first()
if resp is None: return 'X'
else: return resp.state
def share_case():
root.authorized()
jid = request.forms.jid
jobs(id=jid).update_record(shared="True")
db.commit()
# increase count in database for every user
for u in db().select(users.ALL):
nmsg = users(user=u.user).new_shared_jobs or 0
users(user=u.user).update_record(new_shared_jobs=nmsg+1)
db.commit()
redirect('/jobs')
def editplotdefs():
user = root.authorized()
if user != 'admin':
return template('error', err="must be admin to edit plots")
app = request.query.app
if config.auth and not root.authorized(): redirect('/login')
if app not in root.myapps: redirect('/apps')
query = (root.apps.id == plots.appid) & (root.apps.name == app)
result = db(query).select()
params = {'app': app, 'user': user}
return template('plots/plotdefs', params, rows=result)
def execute():
user = root.authorized()
app = request.forms.app
cid = request.forms.cid
np = int(request.forms.np) or 1
walltime = request.forms.walltime
desc = request.forms.desc
#priority = request.forms.priority
params = {}
# base_dir = os.path.join(user_dir, user, app, cid)
inputs, _, _ = root.myapps[app].read_params(user, cid)
# in addition to supporting input params, also support case id
if "cid" not in inputs: inputs["cid"] = cid
# if preprocess is set run the preprocessor
# try:
# if root.myapps[app].preprocess:
# processed_inputs = process.preprocess(inputs,
# root.myapps[app].preprocess,base_dir)
# except:
# exc_type, exc_value, exc_traceback = sys.exc_info()
# print traceback.print_exception(exc_type, exc_value, exc_traceback)
# return template('error', err="There was an error with the preprocessor")
cmd = apps(name=app).command
# for parallel runs
if np > 1: cmd = config.mpirun + " -np " + str(np) + " " + cmd
# this is the relative path to the executable from the case directory where
# the simulation files are stored
inputs['rel_apps_path'] = (os.pardir + os.sep)*4 + apprw.apps_dir
# replace placeholder tags in the command line, e.g. <cid> with appropriate params
cmd = replace_tags(cmd, inputs)
outfn = app + ".out"
# submit job to queue
try:
params['cid'] = cid
params['app'] = app
params['user'] = user
priority = db(users.user==user).select(users.priority).first().priority
uid = users(user=user).id
jid = root.sched.qsub(app, cid, uid, cmd, np, priority, walltime, desc)
redirect("/case?app="+app+"&cid="+cid+"&jid="+jid)
except OSError, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
print traceback.print_exception(exc_type, exc_value, exc_traceback)
print >> sys.stderr, "Execution failed:", e
params = { 'cid': cid, 'app': app, 'user': user, 'err': e }
return template('error', params)
def get_tile(pname, filt_name):
"""
Return data form tile table.
INPUT: pname, filt_name
"""
query = db((db.t80tiles.PName == pname) & (db.filter.Name == filt_name))
tile_info = query.select(db.t80tiles.id, db.t80tiles.RA, db.t80tiles.DEC,
db.t80tiles.PIXEL_SCALE,
db.t80tiles.IMAGE_SIZE).first()
return tile_info
def count_images(start_date, end_date, frametype, filt=None):
"""
Return the Number of images in the Pipeline Data Base.
"""
type_id = db(db.frametype.Name == frametype).select(
db.frametype.id).first()
nimages = 0
if frametype == "BIAS":
nimages = db((db.t80oa.ImageType_ID == type_id)
& (db.t80oa.Date >= start_date)
& (db.t80oa.Date <= end_date)).count()
else:
if filt is not None:
filter_id = db(db.filter.Name == filt).select(db.filter.id).first()
nimages = db((db.t80oa.ImageType_ID == type_id)
& (db.t80oa.Filter_ID == filter_id)
& (db.t80oa.Date >= start_date)
& (db.t80oa.Date <= end_date)).count()
else:
raise NameError("No Filter Passed")
return nimages
def showmyapps():
user = root.authorized()
uid = users(user=user).id
app = root.active_app()
result = db((apps.id == app_user.appid)
& (uid == app_user.uid)).select(orderby=apps.name)
if user == "admin":
configurable = True
else:
configurable = False
params = {'configurable': configurable, 'user': user, 'app': app}
return template('myapps', params, rows=result)
def get_zp(id_tilesinfo):
'''
Return zp info:
Input: ID of tilesinfo
'''
query = db(db.calib_zp_tiles.id_tilesinfo == id_tilesinfo)
zp_info = query.select(
db.calib_zp_tiles.zp,
db.calib_zp_tiles.err_zp,
db.calib_zp_tiles.calib_procedure,
).first()
if zp_info is None:
return [None, None, None]
return [zp_info.zp, zp_info.err_zp, zp_info.calib_procedure]
def appconfig_status():
root.authorized()
status = dict()
app = request.query.app
# check db file
appname = apps(name=app).name
if appname:
status['command'] = 1
else:
status['command'] = 0
# check template file
if os.path.exists("views/apps/" + app + ".tpl"):
status['template'] = 1
else:
status['template'] = 0
# check inputs file
extension = {
'namelist': '.in',
'ini': '.ini',
'xml': '.xml',
'json': '.json',
'yaml': '.yaml',
'toml': '.toml'
}
if os.path.exists(
os.path.join(apprw.apps_dir, app,
app + extension[root.myapps[app].input_format])):
status['inputs'] = 1
else:
status['inputs'] = 0
# check app binary
if os.path.exists(os.path.join(apprw.apps_dir, app, app)):
status['binary'] = 1
else:
status['binary'] = 0
# check plots
appid = apps(name=app).id
result = db(plots.appid == appid).select().first()
if result:
status['plots'] = 1
else:
status['plots'] = 0
return json.dumps(status)
def search_images(start_date, end_date, frametype, filt=None):
"""
Return the available images in a given day.
"""
type_id = db(db.frametype.Name == frametype).select(
db.frametype.id).first()
nimages = 0
if frametype == "BIAS":
nimages = db((db.t80oa.ImageType_ID == type_id)
& (db.t80oa.Date >= start_date)
& (db.t80oa.Date <= end_date)).select(db.t80oa.Name)
nimages = [img.Name for img in nimages]
else:
if filt is not None:
filter_id = db(db.filter.Name == filt).select(db.filter.id).first()
nimages = db((db.t80oa.ImageType_ID == type_id)
& (db.t80oa.Filter_ID == filter_id)
& (db.t80oa.Date >= start_date)
& (db.t80oa.Date <= end_date)).select(db.t80oa.Name)
nimages = [img.Name for img in nimages]
else:
raise NameError("No Filter Passed")
return nimages
def post_register():
valid = True
user = request.forms.user
if check_user(user) == 'true': valid = False
email = request.forms.email
if re.match(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)",
email) is None:
valid = False
pw1 = request.forms.password1
if (any(x.isupper() for x in pw1) and any(x.isdigit() for x in pw1)
and len(pw1) >= 7):
pass
else:
valid = False
pw2 = request.forms.password2
if pw1 != pw2: valid = False
if valid:
hashpw = _hash_pass(pw1)
try:
config.default_priority
except:
config.default_priority = 3
# insert into database
users.insert(user=user.lower(),
passwd=hashpw,
email=email,
priority=config.default_priority,
new_shared_jobs=0)
db.commit()
# email admin user
try:
server = smtplib.SMTP('localhost')
message = user + " just registered " + email
admin_email = db(users.user == "admin").select(users.email).first()
server.sendmail('*****@*****.**', [admin_email], message)
server.quit()
redirect('/login')
except:
redirect('/login')
else:
return "ERROR: there was a problem registering. Please try again...<p>" \
+ "<a href='/register'>Return to registration</a>"
def login(request):
db.user.login.requires = IS_NOT_EMPTY()
form = SQLFORM(db.user, fields=['login', 'password'])
if FORM.accepts(form, request.vars):
query = db((db.user.login == form.vars.login) &
(db.user.password == form.vars.password))
if query.count() == 1:
user = query.select()[0]
request.session['user'] = Storage(login=user.login, id=user.id,
email=user.email)
request.session['flash'] = 'You\'ve been logged in'
return redirect('/')
else:
request.session['flash'] = 'User not found'
return redirect('/login')
return render('form.html', form=form)
def tile_info(pname, filt_name):
'''
Return Tile Info.
Input: PNAME
filt_name
'''
query = db((db.t80tiles.PName == pname) & (db.filter.Name == filt_name))
tile_info = query.select(db.t80tilesinfo.id, db.t80tilesinfo.RefImage_ID,
db.t80tilesinfo.FWHM_Min,
db.t80tilesinfo.FWHM_Max,
db.t80tilesinfo.Filter_ID,
db.t80tilesinfo.MoffatBeta_Mean,
db.t80tilesinfo.Noise).first()
return [
tile_info.id, tile_info.RefImage_ID, tile_info.FWHM_Min,
tile_info.FWHM_Max, tile_info.Filter_ID, tile_info.MoffatBeta_Mean,
tile_info.Noise
]
def get_datasource(pltid):
"""get list of datasources for given plot"""
user = root.authorized()
if user != 'admin':
return template('error', err="must be admin to edit plots")
app = request.query.app
cid = request.query.cid
if root.myapps[app].appname not in root.myapps: redirect('/apps')
if config.auth and not root.authorized(): redirect('/login')
result = db(datasource.pltid == pltid).select()
title = plots(pltid).title
params = {
'app': app,
'cid': cid,
'user': user,
'pltid': pltid,
'rows': result,
'title': title
}
return template('plots/datasources', params, rows=result)
