def get_mjd_for_tiling(pname, filt_name): ''' Return a list of MJD and Exposure time of image used to produce a tile. INPUT: PNAme Fileter ''' tile_info = get_tile(pname, filt_name) query = db(db.t80tileImgs.Tile_ID == tile_info.id) rcid = query.select(db.t80tileImgs.RC_ID) proc_images = [ db(db.rc.id == rcidi.RC_ID).select(db.rc.ori_id).first() for rcidi in rcid ] images = [ db(db.t80oa.id == primg.ori_id).select( db.t80oa.Date, db.t80oa.Time, db.t80oa.ExpTime, ).first() for primg in proc_images ] image_date_time = [ "{0}T{1}".format(img.Date.strftime("%Y-%m-%d"), img.Time.strftime("%H:%M:%S")) for img in images ] mjd = Time(image_date_time).mjd return [[mjd[i], float(images[i].ExpTime)] for i in range(len(images))]
def showapps(): user = root.authorized() q = request.query.q if not q: result = db().select(apps.ALL, orderby=apps.name) else: result = db( db.apps.name.contains(q, case_sensitive=False) | db.apps.category.contains(q, case_sensitive=False) | db.apps.description.contains(q, case_sensitive=False)).select() # find out what apps have already been activated so that a user can't activate twice uid = users(user=user).id activated = db(app_user.uid == uid).select() activated_apps = [] for row in activated: activated_apps.append(row.appid) if user == "admin": configurable = True else: configurable = False params = {'configurable': configurable, 'user': user} return template('apps', params, rows=result, activated=activated_apps)
def export(): user = root.authorized() if user != 'admin': return template('error', err="must be admin to use export function") app = request.forms.app result = db(apps.name == app).select().first() data = {} data['name'] = result.name data['description'] = result.description data['category'] = result.category data['language'] = result.language data['input_format'] = result.input_format data['command'] = result.command data['preprocess'] = result.preprocess data['postprocess'] = result.postprocess assets = list() if result.assets is not None: for asset in result.assets.split(","): assets.append(asset.strip()) data['assets'] = assets appid = apps(name=app).id myplots = db(plots.appid == appid).select() data['plots'] = list() for p in myplots: thisplot = {} thisplot['ptype'] = p.ptype thisplot['title'] = p.title thisplot['options'] = p.options thisplot['datasource'] = list() myds = db(datasource.pltid == p.id).select() for ds in myds: thisds = {} thisds['label'] = ds.label thisds['filename'] = ds.filename thisds['cols'] = ds.cols thisds['line_range'] = ds.line_range thisds['data_def'] = ds.data_def thisplot['datasource'].append(thisds) data['plots'].append(thisplot) path = os.path.join(apprw.apps_dir, app, 'spc.json') with open(path, 'w') as outfile: json.dump(data, outfile, indent=3) return "spc.json file written to " + path + "<meta http-equiv='refresh' content='2; url=/app/" + app + "'>"
def view_app(app): user = root.authorized() if app: root.set_active(app) else: redirect('/myapps') if user != 'admin': return template('error', err="must be admin to edit app") cid = request.query.cid result = db(apps.name == app).select().first() params = {} params['app'] = app params['user'] = user params['cid'] = cid #if request.query.edit: # return template('appedit', params, rows=result) #else: try: return template('app', params, rows=result) except: exc_type, exc_value, exc_traceback = sys.exc_info() print traceback.print_exception(exc_type, exc_value, exc_traceback) return template( 'error', err="there was a problem showing the app template. Check traceback." )
def admin_show_users(): user = root.authorized() if not user == "admin": return template("error", err="must be admin to delete") result = db().select(users.ALL) params = { 'user': user, 'app': root.active_app() } return template('admin/users', params, rows=result)
def execute(): app = request.forms['app'] user = request.forms['user'] cid = request.forms['cid'] desc = request.forms['desc'] np = request.forms['np'] appmod = pickle.loads(request.forms['appmod']) # remove the appmod key del request.forms['appmod'] appmod.write_params(request.forms, user) # if preprocess is set run the preprocessor try: if appmod.preprocess: run_params, _, _ = appmod.read_params(user, cid) base_dir = os.path.join(user_dir, user, app) process.preprocess(run_params, appmod.preprocess, base_dir) if appmod.preprocess == "terra.in": appmod.outfn = "out" + run_params['casenum'] + ".00" except: return template('error', err="There was an error with the preprocessor") # submit job to queue try: priority = db(users.user == user).select( users.priority).first().priority uid = users(user=user).id jid = sched.qsub(app, cid, uid, np, priority, desc) return str(jid) #redirect("http://localhost:"+str(config.port)+"/case?app="+str(app)+"&cid="+str(cid)+"&jid="+str(jid)) except OSError: return "ERROR: a problem occurred"
def get_all_jobs(): user = root.authorized() if not user == "admin": return template("error", err="must be admin to use this feature") cid = request.query.cid app = request.query.app or root.active_app() n = request.query.n if not n: n = config.jobs_num_rows else: n = int(n) # sort by descending order of jobs.id result = db((db.jobs.uid==users.id)).select(orderby=~jobs.id)[:n] # clear notifications users(user=user).update_record(new_shared_jobs=0) db.commit() params = {} params['cid'] = cid params['app'] = app params['user'] = user params['n'] = n params['num_rows'] = config.jobs_num_rows return template('shared', params, rows=result)
def get_stats(): root.authorized() params = {} # number of jobs in queued, running, and completed states params['nq'] = db(jobs.state=='Q').count() params['nr'] = db(jobs.state=='R').count() params['nc'] = db(jobs.state=='C').count() params['cpu'] = psutil.cpu_percent() params['vm'] = psutil.virtual_memory().percent params['disk'] = psutil.disk_usage('/').percent params['cid'] = request.query.cid params['app'] = request.query.app return template("stats", params)
def plot_flot_3d(plot, cid, app, sim_dir, owner, user, plot_title, pltid): # to handle data in user/cid format when looking at shared cases if re.search("/", cid): (owner, c) = cid.split("/") else: owner = user c = cid desc = jobs(cid=c).description list_of_plots = db((apps.id == plots.appid) & (apps.name == app)).select() options = json.loads(plot['options']) plot_data = [] z_data = [] data_dir = os.path.join(sim_dir, options['directory']) z_property = options['z_property'] file_names = sorted(os.listdir(data_dir)) for file_name in file_names: file_path = os.path.join(data_dir, file_name) if os.path.isfile(file_path) and not file_name.startswith( '.') and file_name.endswith('.json'): with open(file_path) as file_: file_data = json.load(file_) all_series = [] for source in options['datasources']: series = { 'data': zip(file_data[source['x_property']], file_data[source['y_property']]), } series.update(source['data_def']) all_series.append(series) plot_data.append(all_series) z_data.append(file_data[z_property]) params = { 'app': app, 'cid': cid, 'description': desc, 'owner': owner, 'plot_title': plot_title, 'pltid': pltid, 'rows': list_of_plots, 'stats': '', 'user': user, 'options_json': json.dumps(options['flot_options']), 'data_json': json.dumps(plot_data), 'z_data_json': json.dumps(z_data), 'z_label_json': json.dumps(options['z_label']), } return template('plots/flot-3d', params)
def get_shared(): """Return the records from the shared table.""" user = root.authorized() cid = request.query.cid app = request.query.app or root.active_app() n = request.query.n if not n: n = config.jobs_num_rows else: n = int(n) # sort by descending order of jobs.id result = db((db.jobs.shared=="True") & (db.jobs.uid==users.id)).select(orderby=~jobs.id)[:n] # result = db((db.jobs.shared=="True") & # (jobs.gid == users.gid)).select(orderby=~jobs.id)[:n] # clear notifications users(user=user).update_record(new_shared_jobs=0) db.commit() params = {} params['cid'] = cid params['app'] = app params['user'] = user params['n'] = n params['num_rows'] = config.jobs_num_rows return template('shared', params, rows=result)
def get_pnames(): ''' Return all PName from t80tiles ''' pname = db().select(db.t80tiles.PName) pname = [pn.PName for pn in pname] return pname
def aws_conn(id): """create a connection to the EC2 machine and return the handle""" user = root.authorized() uid = users(user=user).id try: creds = db(db.aws_creds.uid == uid).select().first() account_id = creds['account_id'] secret = creds['secret'] key = creds['key'] instances = db(db.aws_instances.id == id).select().first() instance = instances['instance'] region = instances['region'] rate = instances['rate'] or 0. return EC2(key, secret, account_id, instance, region, rate) except: return template('error', err="problem validating AWS credentials")
def editplotdef(pltid): user = root.authorized() if user != 'admin': return template('error', err="must be admin to edit plots") app = request.forms.app result = db(plots.id == pltid).select().first() params = {'app': app, 'user': user} return template('plots/edit_plot', params, row=result)
def searchImages(startDate, endDate, frametype, filt): """ Return the Number of images in the Pipeline Data Base. """ typeID = db(db.frametype.Name == frametype).select(db.frametype.id).first() nimages = 0 if (frametype == "BIAS"): nimages = db((db.t80oa.ImageType_ID == typeID) & (db.t80oa.Date >= startDate) & (db.t80oa.Date <= endDate)).count() else: filterID = db(db.filter.Name == filt).select(db.filter.id).first() nimages = db((db.t80oa.ImageType_ID == typeID) & (db.t80oa.Filter_ID == filterID) & (db.t80oa.Date >= startDate) & (db.t80oa.Date <= endDate)).count() return nimages
def get_aws(): user = root.authorized() cid = request.query.cid app = request.query.app or root.active_app() uid = db(users.user == user).select(users.id).first() #creds = db().select(db.aws_creds.ALL) creds = db(aws_creds.uid == uid).select() # look for aws instances registered by the current user # which means first need to get the uid instances = db(aws_instances.uid == uid).select() params = {} params['cid'] = cid params['app'] = app params['user'] = user if request.query.status: params['status'] = request.query.status return template('aws', params, creds=creds, instances=instances)
def app_edit(appid): user = root.authorized() if user != 'admin': return template('error', err="must be admin to edit app") cid = request.forms.cid app = request.forms.app result = db(apps.name == app).select().first() params = {'app': app, 'cid': cid} return template('app_edit', params, rows=result)
def edit_datasource(pltid, dsid): """create a new datasource for given plot""" user = root.authorized() if user != 'admin': return template('error', err="must be admin to edit plots") app = request.query.app query = (datasource.id == dsid) result = db(query).select().first() params = {'app': app, 'pltid': pltid, 'dsid': dsid} return template('plots/edit_datasource', params, row=result)
def get_status(jid): # following headers are needed because of CORS response.headers['Access-Control-Allow-Origin'] = '*' response.headers[ 'Access-Control-Allow-Methods'] = 'GET, POST, PUT, OPTIONS' response.headers[ 'Access-Control-Allow-Headers'] = 'Origin, Accept, Content-Type, X-Requested-With, X-CSRF-Token' resp = db(jobs.id == jid).select(jobs.state).first() if resp is None: return 'X' else: return resp.state
def share_case(): root.authorized() jid = request.forms.jid jobs(id=jid).update_record(shared="True") db.commit() # increase count in database for every user for u in db().select(users.ALL): nmsg = users(user=u.user).new_shared_jobs or 0 users(user=u.user).update_record(new_shared_jobs=nmsg+1) db.commit() redirect('/jobs')
def editplotdefs(): user = root.authorized() if user != 'admin': return template('error', err="must be admin to edit plots") app = request.query.app if config.auth and not root.authorized(): redirect('/login') if app not in root.myapps: redirect('/apps') query = (root.apps.id == plots.appid) & (root.apps.name == app) result = db(query).select() params = {'app': app, 'user': user} return template('plots/plotdefs', params, rows=result)
def execute(): user = root.authorized() app = request.forms.app cid = request.forms.cid np = int(request.forms.np) or 1 walltime = request.forms.walltime desc = request.forms.desc #priority = request.forms.priority params = {} # base_dir = os.path.join(user_dir, user, app, cid) inputs, _, _ = root.myapps[app].read_params(user, cid) # in addition to supporting input params, also support case id if "cid" not in inputs: inputs["cid"] = cid # if preprocess is set run the preprocessor # try: # if root.myapps[app].preprocess: # processed_inputs = process.preprocess(inputs, # root.myapps[app].preprocess,base_dir) # except: # exc_type, exc_value, exc_traceback = sys.exc_info() # print traceback.print_exception(exc_type, exc_value, exc_traceback) # return template('error', err="There was an error with the preprocessor") cmd = apps(name=app).command # for parallel runs if np > 1: cmd = config.mpirun + " -np " + str(np) + " " + cmd # this is the relative path to the executable from the case directory where # the simulation files are stored inputs['rel_apps_path'] = (os.pardir + os.sep)*4 + apprw.apps_dir # replace placeholder tags in the command line, e.g. <cid> with appropriate params cmd = replace_tags(cmd, inputs) outfn = app + ".out" # submit job to queue try: params['cid'] = cid params['app'] = app params['user'] = user priority = db(users.user==user).select(users.priority).first().priority uid = users(user=user).id jid = root.sched.qsub(app, cid, uid, cmd, np, priority, walltime, desc) redirect("/case?app="+app+"&cid="+cid+"&jid="+jid) except OSError, e: exc_type, exc_value, exc_traceback = sys.exc_info() print traceback.print_exception(exc_type, exc_value, exc_traceback) print >> sys.stderr, "Execution failed:", e params = { 'cid': cid, 'app': app, 'user': user, 'err': e } return template('error', params)
def get_tile(pname, filt_name): """ Return data form tile table. INPUT: pname, filt_name """ query = db((db.t80tiles.PName == pname) & (db.filter.Name == filt_name)) tile_info = query.select(db.t80tiles.id, db.t80tiles.RA, db.t80tiles.DEC, db.t80tiles.PIXEL_SCALE, db.t80tiles.IMAGE_SIZE).first() return tile_info
def count_images(start_date, end_date, frametype, filt=None): """ Return the Number of images in the Pipeline Data Base. """ type_id = db(db.frametype.Name == frametype).select( db.frametype.id).first() nimages = 0 if frametype == "BIAS": nimages = db((db.t80oa.ImageType_ID == type_id) & (db.t80oa.Date >= start_date) & (db.t80oa.Date <= end_date)).count() else: if filt is not None: filter_id = db(db.filter.Name == filt).select(db.filter.id).first() nimages = db((db.t80oa.ImageType_ID == type_id) & (db.t80oa.Filter_ID == filter_id) & (db.t80oa.Date >= start_date) & (db.t80oa.Date <= end_date)).count() else: raise NameError("No Filter Passed") return nimages
def showmyapps(): user = root.authorized() uid = users(user=user).id app = root.active_app() result = db((apps.id == app_user.appid) & (uid == app_user.uid)).select(orderby=apps.name) if user == "admin": configurable = True else: configurable = False params = {'configurable': configurable, 'user': user, 'app': app} return template('myapps', params, rows=result)
def get_zp(id_tilesinfo): ''' Return zp info: Input: ID of tilesinfo ''' query = db(db.calib_zp_tiles.id_tilesinfo == id_tilesinfo) zp_info = query.select( db.calib_zp_tiles.zp, db.calib_zp_tiles.err_zp, db.calib_zp_tiles.calib_procedure, ).first() if zp_info is None: return [None, None, None] return [zp_info.zp, zp_info.err_zp, zp_info.calib_procedure]
def appconfig_status(): root.authorized() status = dict() app = request.query.app # check db file appname = apps(name=app).name if appname: status['command'] = 1 else: status['command'] = 0 # check template file if os.path.exists("views/apps/" + app + ".tpl"): status['template'] = 1 else: status['template'] = 0 # check inputs file extension = { 'namelist': '.in', 'ini': '.ini', 'xml': '.xml', 'json': '.json', 'yaml': '.yaml', 'toml': '.toml' } if os.path.exists( os.path.join(apprw.apps_dir, app, app + extension[root.myapps[app].input_format])): status['inputs'] = 1 else: status['inputs'] = 0 # check app binary if os.path.exists(os.path.join(apprw.apps_dir, app, app)): status['binary'] = 1 else: status['binary'] = 0 # check plots appid = apps(name=app).id result = db(plots.appid == appid).select().first() if result: status['plots'] = 1 else: status['plots'] = 0 return json.dumps(status)
def search_images(start_date, end_date, frametype, filt=None): """ Return the available images in a given day. """ type_id = db(db.frametype.Name == frametype).select( db.frametype.id).first() nimages = 0 if frametype == "BIAS": nimages = db((db.t80oa.ImageType_ID == type_id) & (db.t80oa.Date >= start_date) & (db.t80oa.Date <= end_date)).select(db.t80oa.Name) nimages = [img.Name for img in nimages] else: if filt is not None: filter_id = db(db.filter.Name == filt).select(db.filter.id).first() nimages = db((db.t80oa.ImageType_ID == type_id) & (db.t80oa.Filter_ID == filter_id) & (db.t80oa.Date >= start_date) & (db.t80oa.Date <= end_date)).select(db.t80oa.Name) nimages = [img.Name for img in nimages] else: raise NameError("No Filter Passed") return nimages
def post_register(): valid = True user = request.forms.user if check_user(user) == 'true': valid = False email = request.forms.email if re.match(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)", email) is None: valid = False pw1 = request.forms.password1 if (any(x.isupper() for x in pw1) and any(x.isdigit() for x in pw1) and len(pw1) >= 7): pass else: valid = False pw2 = request.forms.password2 if pw1 != pw2: valid = False if valid: hashpw = _hash_pass(pw1) try: config.default_priority except: config.default_priority = 3 # insert into database users.insert(user=user.lower(), passwd=hashpw, email=email, priority=config.default_priority, new_shared_jobs=0) db.commit() # email admin user try: server = smtplib.SMTP('localhost') message = user + " just registered " + email admin_email = db(users.user == "admin").select(users.email).first() server.sendmail('*****@*****.**', [admin_email], message) server.quit() redirect('/login') except: redirect('/login') else: return "ERROR: there was a problem registering. Please try again...<p>" \ + "<a href='/register'>Return to registration</a>"
def login(request): db.user.login.requires = IS_NOT_EMPTY() form = SQLFORM(db.user, fields=['login', 'password']) if FORM.accepts(form, request.vars): query = db((db.user.login == form.vars.login) & (db.user.password == form.vars.password)) if query.count() == 1: user = query.select()[0] request.session['user'] = Storage(login=user.login, id=user.id, email=user.email) request.session['flash'] = 'You\'ve been logged in' return redirect('/') else: request.session['flash'] = 'User not found' return redirect('/login') return render('form.html', form=form)
def tile_info(pname, filt_name): ''' Return Tile Info. Input: PNAME filt_name ''' query = db((db.t80tiles.PName == pname) & (db.filter.Name == filt_name)) tile_info = query.select(db.t80tilesinfo.id, db.t80tilesinfo.RefImage_ID, db.t80tilesinfo.FWHM_Min, db.t80tilesinfo.FWHM_Max, db.t80tilesinfo.Filter_ID, db.t80tilesinfo.MoffatBeta_Mean, db.t80tilesinfo.Noise).first() return [ tile_info.id, tile_info.RefImage_ID, tile_info.FWHM_Min, tile_info.FWHM_Max, tile_info.Filter_ID, tile_info.MoffatBeta_Mean, tile_info.Noise ]
def get_datasource(pltid): """get list of datasources for given plot""" user = root.authorized() if user != 'admin': return template('error', err="must be admin to edit plots") app = request.query.app cid = request.query.cid if root.myapps[app].appname not in root.myapps: redirect('/apps') if config.auth and not root.authorized(): redirect('/login') result = db(datasource.pltid == pltid).select() title = plots(pltid).title params = { 'app': app, 'cid': cid, 'user': user, 'pltid': pltid, 'rows': result, 'title': title } return template('plots/datasources', params, rows=result)