def isMatchingIP(self, ip): for permitted in PERMITTED_IPS: match = fnmatch.filter([ip], permitted) if match: logging.info("WebHook: isMatchingIP() [%s, %s, %s]" % (ip, permitted, match)) return True return False
def get(self, identity): logging.info("uri=%s [%s] [%s]" % (self.request.uri, identity, self.request.arguments)) args = self.request.arguments query = {} for key in args.keys(): if key != "cancer": continue iargs = args[key] if len(iargs) == 1: query[key] = args[key][0].lower() else: query[key] = {"$in": map(lambda x: x.lower(), args[key])} if "max_rank" not in args: query["rank"] = {"$lt": 21} else: query["rank"] = {"$lt": int(args["max_rank"][0]) + 1} collection = self.open_collection("qed_lookups", "mutsig_rankings") items = [] if "cancer" in query: items = collection.find(query) json_items = map(self.jsonable_item, items) if self.get_argument("output", "json") == "tsv": WriteTsv(self, json_items) self.set_status(200) return self.write(json.dumps({ "items": json_items })) self.set_status(200)
def get(self, identity): logging.info("uri=%s [%s] [%s]" % (self.request.uri, identity, self.request.arguments)) args = self.request.arguments ids = identity.split("/") query = { "chr": str(args["chr"][0]), "start": {"$gt": int(args["start"][0])}, "end": {"$lt": int(args["end"][0])}, "cancer": {"$in": map(lambda x: x.lower(), args["cancer"])}, "source": {"$in": map(lambda x: x.lower(), args["source"])} } logging.info("query=%s" % str(query)) query_limit = options.mongo_lookup_query_limit collection = self.open_collection(ids[1], ids[2]) items = [] for idx, item in enumerate(collection.find(query, {'values':0})): if idx > query_limit: break items.append(item) self.write(json.dumps({ "items": map(self.jsonable_item, items) })) self.set_status(200)
def get(self, vcf_id, chromosome, coordinate): global VCF_MAP if vcf_id not in DATA_MAP: logging.info("Unknown VCF id \'" + vcf_id + "\'") raise tornado.web.HTTPError(404) file_info = DATA_MAP[vcf_id] file_path = file_info['path'] lookup_fn = None if file_info['type'] == 'vcf': lookup_fn = vcf_lookup elif file_info['type'] == 'trio': lookup_fn = triotype_lookup else: logging.error("Unknown type for file " + file_path) raise tornado.web.HTTPError(404) try: result = lookup_fn(options.tabix_executable, file_path, chromosome, coordinate, coordinate) response = { "chr": result.chromosome, "coordinate": result.coordinate, "values": result.values } self.write(json.dumps(response, sort_keys=True)) self.set_status(200) except Exception as e: logging.error("Running tabix failed:") logging.error(e) raise tornado.web.HTTPError(404)
def main(): define("port", default=8321, help="Server port", type=int) define("verbose", default=False, help="Prints debugging statements") define("tabix_executable", default=".", help="Path to tabix executable") define("vcf_config", default=".", help="Path to VCF configuration JSON") tornado.options.parse_command_line() logging.info("Starting Tornado web server on http://localhost:%s" % options.port) logging.info("--tabix_executable=%s" % options.tabix_executable) logging.info("--vcf_config=%s" % options.vcf_config) logging.info("--verbose=%s" % options.verbose) # Try loading the VCF mapping logging.info("Loading VCF configuration file \'" + options.vcf_config + "\'...") try: global DATA_MAP DATA_MAP = parse_vcf_config(options.vcf_config) except Exception as e: logging.error("Failed to load VCF configuration file:") logging.error(e) sys.exit(1) application = tornado.web.Application([ (r"/(\w+)/(X|Y|M|\d{1,2})/(\d+)", TabixLookup) ], **settings) application.listen(options.port, **server_settings) tornado.ioloop.IOLoop.instance().start()
def open_collection(db_name, collection_name): #if options.verbose: logging.info("open_collection(%s)" % collection_name) connection = pymongo.Connection(options.mongo_lookup_uri) database = connection[db_name] return database[collection_name]
def get(self, identity): logging.info("uri=%s [%s] [%s]" % (self.request.uri, identity, self.request.arguments)) # TODO : Improve this logic to correctly parse arguments and convert to a proper mongo DB query args = self.request.arguments query = {} for key in args.keys(): query[key] = args[key][0] ids = identity.split("/") db_name = ids[1] collection = open_collection(db_name, ids[2]) query_limit = options.mongo_lookup_query_limit json_items = [] for idx, item in enumerate(collection.find(query)): if idx > query_limit: break json_item = self.jsonable_item(item) json_item["uri"] = self.request.uri + "/" + json_item["id"] json_items.append(json_item) self.write({ "items": json_items }) self.set_status(200) return
def open_collection(self, db_name, collection_name): #if options.verbose: logging.info("open_collection(%s)" % collection_name) connection = pymongo.Connection(options.mongo_lookup_uri) database = connection[db_name] return database[collection_name]
def get(self, vcf_id, chromosome, start_coordinate, end_coordinate): global VCF_MAP if vcf_id not in DATA_MAP: logging.info("Unknown VCF id \'" + vcf_id + "\'") raise tornado.web.HTTPError(404) file_info = DATA_MAP[vcf_id] file_path = file_info['path'] lookup_fn = None if file_info['type'] == 'tsv': lookup_fn = tsv_region_lookup else: logging.error("Unknown type for file " + file_path) raise tornado.web.HTTPError(404) try: # The result.values array contains dictiories, for which the keys are read from the header line # of the TSV file. For the header line to be included in the output of the tabix command, the "-h" # flag has to be included in the command line. result = lookup_fn(options.tabix_executable + " -h", file_path, chromosome, start_coordinate, end_coordinate) response = { "chr": result.chromosome, "start": result.start, "end": result.end, "values": result.values } self.write(json.dumps(response, sort_keys=True)) self.set_status(200) except Exception as e: logging.error("Running tabix failed:") logging.error(e) raise tornado.web.HTTPError(404)
def post(self, *args, **kwargs): logging.info("WebHook: post()") if not self.isPermittedIP(): self.set_status(401) return http_client = tornado.httpclient.HTTPClient() response = http_client.fetch(options.github_repo_api_url) repository = json.loads(response.body) clone_url = repository["clone_url"].replace("https", "http") self.pull(clone_url, options.github_project_root, "master") branches_url = repository["branches_url"].replace("{/branch}", "") response = http_client.fetch(branches_url) branches = json.loads(response.body) write_branches = [] for i, branch in enumerate(branches): branch_name = branch["name"] if not branch_name == "master": deploy_path = os.path.join(options.github_branches_root, branch_name) logging.info("WebHook: deploying branch [%s] to [%s]" % (branch_name, deploy_path)) self.pull(clone_url, deploy_path, branch_name) write_branches.append({ "name": branch_name, "label": branch_name }) if not options.github_branches_json_path is None: json.dump(write_branches, open(os.path.join(options.github_branches_json_path, "branches.json"), "w"))
def main(): tornado.options.parse_command_line() logging.info("Starting Tornado web server on http://localhost:%s" % options.port) application = tornado.web.Application([ (r"/", MainHandler), ], **settings) application.listen(options.port, **server_settings) tornado.ioloop.IOLoop.instance().start()
def get(self): logging.info("AuthWhoAmiHandler.get") whoami = { "user": "******", "label": "Hector Rovira" } self.write(whoami) self.set_status(200)
def SaveUserinfo(whoami, userinfo): logging.info("SaveUserinfo(%s)" % whoami) existing_user = GetUserinfo(whoami) userinfo["whoami"] = whoami collection = open_collection("private_userinfo") if existing_user is None: collection.insert(userinfo) else: collection.update(existing_user, userinfo)
def on_message(self, message): logging.info("New Note %s" % message) newnote = message newnote_json = json.loads(newnote) newnote_json["timestamp"] = datetime.now() notes = get_notes_collection() notes.insert(newnote_json) for connected_user in connected_users: connected_user.write_message(json.dumps([newnote_json], default=json_util.default))
def redis_listener(): try: ps = r.pubsub() ps.subscribe(settings.REDIS['channel']) io_loop = tornado.ioloop.IOLoop.instance() for message in ps.listen(): for socket in LISTENERS['ws']: io_loop.add_callback(partial(socket.on_message,message)) # Note: Not sure if this block will be reached at all except: logging.info('redis_listener() aborted. Stop subscribing.') ps.close()
def main(): tornado.options.parse_command_line() tornado.web.RequestHandler.instanceNum = options.instn port = 9900 + options.instn logging.info("Starting Tornado web server on http://localhost:%s" % port) application = tornado.web.Application([ (r"/",MainHandler), (r"/greet", Greeter), (r"/fingreet",MongoGreeter) ], **settings) application.listen(port, **server_settings) tornado.ioloop.IOLoop.instance().start()
def open(self): logging.info("Connection opened") notes = get_notes_collection() most_recent_notes = list( notes.find({"loc": {"$exists": "true"}, "timestamp": {"$exists": "true"}}, limit=25).sort( "timestamp", DESCENDING ) ) connected_users.append(self) self.write_message(json.dumps(most_recent_notes, default=json_util.default))
def post(self, identity): logging.info("MongoDbHandler.post(%s)" % identity) ids = identity.split("/") if len(ids) <= 0: raise HTTPError(401) stored_item = json.loads(self.request.body) # Figure out issue where label is getting set as an array labels = stored_item["label"] if not labels is None and type(labels) is list: stored_item["label"] = labels[0] collection = open_collection(ids[0]) insert_id = str(collection.insert(stored_item)) self.write({ "id": insert_id, "uri": self.request.uri + "/" + insert_id }) self.set_status(200)
def plot(timeseries,fig_name): matplotlib.rc('font', **settings.PLOT['font']) plt.figure(figsize=(14,2.5), dpi=72) ax = plt.subplot(1,1,1) rts = pd.rolling_mean(timeseries, settings.PLOT['rolling_mean_min']) plt.plot(rts.index, rts.values, settings.PLOT['fill']) plt.ylabel('Tweets per minute (TPM)') ax.fill_between(rts.index, rts.values, 0, color=settings.PLOT['fill']) ax.xaxis.set_major_locator(md.HourLocator(interval=settings.PLOT['xaxis_interval_hr'])) ax.xaxis.set_major_formatter(md.DateFormatter(settings.PLOT['datetime_format'])) ax.autoscale_view() plt.grid(True) plt.xlabel('Time (SGT)') fig = os.path.join(settings.SERVER['static_files_path'],'images',fig_name) plt.savefig(fig,bbox_inches='tight') logging.info('Saving %s' % fig)
def get(self, identity): logging.info("MongoDbHandler.get(%s)" % identity) ids = identity.split("/") if len(ids) == 1: args = self.request.arguments query = {} normalize_fn = lambda x: x.lower() for key in args.keys(): iargs = args[key] if len(iargs) == 1: query[key] = normalize_fn(args[key][0]) else: query[key] = {"$in": map(normalize_fn, args[key])} collection = open_collection(ids[0]) json_items = [] for idx, item in enumerate(collection.find(query)): json_item = jsonable_item(item) json_item["uri"] = self.request.uri + "/" + json_item["id"] json_items.append(json_item) json_items.append({ "First Name": "Hector", "Last Name": "Rovira", "Birth Date": "03/14/13", "status": "incomplete" }) self.write({ "items": json_items }) self.set_status(200) return if len(ids) == 2: collection = open_collection(ids[0]) item = collection.find_one({"_id": objectid.ObjectId(ids[1]) }) if not item is None: json_item = jsonable_item(item) json_item["uri"] = self.request.uri self.write(json_item) self.set_status(200) return self.set_status(404)
def get(self, tabix_id, chromosome, start_coordinate, end_coordinate=None): if end_coordinate is None: end_coordinate = start_coordinate if tabix_id not in self._config_map.keys(): if options.verbose: logging.info("Unknown tabix loookup ID [%s]" % tabix_id) raise tornado.web.HTTPError(404) file_info = self._config_map[tabix_id] file_path = file_info['path'] lookup_fn = None tabix_exe = None if file_info['type'] == 'vcf': lookup_fn = vcf_singleline_lookup tabix_exe = options.tabix_executable elif file_info['type'] == 'trio': lookup_fn = triotype_singleline_lookup tabix_exe = options.tabix_executable elif file_info['type'] == 'tsv': lookup_fn = tsv_region_lookup # For parsing the tabix output for TSV files, the header has to be included. Therefore, the "-h" # flag has to be included in the command line. tabix_exe = options.tabix_executable + " -h" else: logging.error("Unknown type for file " + file_path) raise tornado.web.HTTPError(404) try: result = lookup_fn(tabix_exe, file_path, chromosome, start_coordinate, end_coordinate) response = { "chr": result.chromosome, "start": result.start, "end": result.end, "values": result.values, "snpid": result.snpid, "info": result.info } self.write(json.dumps(response, sort_keys=True)) self.set_status(200) except Exception as e: logging.error("Running tabix failed:") logging.error(e) raise tornado.web.HTTPError(404)
def get(self, identity): logging.info("uri=%s [%s] [%s]" % (self.request.uri, identity, self.request.arguments)) args = self.request.arguments ids = identity.split("/") feature_matrix_name = ids[1] gene_label_1 = args['gene1'][0] gene_label_2 = args['gene2'][0] cancer_label = args['cancer'][0].lower() # Get feature IDs fmx_collection = self.open_feature_matrix_collection("qed_lookups", "fmx_" + feature_matrix_name) pairwise_collection = self.open_pairwise_collection("qed_lookups", "pw_" + feature_matrix_name + "_" + cancer_label) features_1 = filter(self.feature_filter_fn, fmx_collection.find({"cancer": cancer_label, "gene": gene_label_1})) features_2 = filter(self.feature_filter_fn, fmx_collection.find({"cancer": cancer_label, "gene": gene_label_2})) feature_ids_1 = map(lambda f: f['id'], features_1) feature_ids_2 = map(lambda f: f['id'], features_2) # Get pairwise values pairwise_results = [] for id1, id2 in product(feature_ids_1, feature_ids_2): pw = self.get_pairwise_result(pairwise_collection, id1, id2) if pw is not None: pairwise_results.append(pw) result = { "features": { gene_label_1: map(self.jsonable_item, features_1), gene_label_2: map(self.jsonable_item, features_2) }, "pairwise_results": map(self.jsonable_item, pairwise_results) } log_msg = "Features found: " log_msg += gene_label_1 + ": " + str(len(feature_ids_1)) log_msg += "\t" + gene_label_2 + ": " + str(len(feature_ids_2)) log_msg += "\tPairwise results: " + str(len(pairwise_results)) logging.info(log_msg) self.write(json.dumps(result)) self.set_status(200)
def get(self, identity): logging.info("uri=%s [%s] [%s]" % (self.request.uri, identity, self.request.arguments)) ids = identity.split("/") db_name = ids[1] collection = self.open_collection(db_name, ids[2]) # TODO : Improve this logic to correctly parse arguments and convert to a proper mongo DB query args = self.request.arguments query = {} case_sensitive_lookups = frozenset(options.case_sensitive_lookups) normalize_fn = None if db_name in case_sensitive_lookups: normalize_fn = lambda x: x else: normalize_fn = lambda x: x.lower() for key in args.keys(): if key != "output": iargs = args[key] if len(iargs) == 1: query[key] = normalize_fn(args[key][0]) else: query[key] = {"$in": map(normalize_fn, args[key])} query_limit = options.mongo_lookup_query_limit json_items = [] for idx, item in enumerate(collection.find(query)): if idx > query_limit: break json_item = self.jsonable_item(item) json_item["uri"] = self.request.uri + "/" + json_item["id"] json_items.append(json_item) if self.get_argument("output", "json") == "tsv": WriteTsv(self, json_items) self.set_status(200) return self.write({"items": json_items}) self.set_status(200) return
def get(self, identity): logging.info("uri=%s [%s] [%s]" % (self.request.uri, identity, self.request.arguments)) args = self.request.arguments cancer_mappings= { "brca": "brca_pw_manuscript" } gene = args["gene"][0] source = args["source"][0] dataset = cancer_mappings[args["cancer"][0]] logging.info("query=[%s][%s][%s]" % (gene, source, dataset)) cli = HTTPClient() response = cli.fetch("http://explorer.cancerregulome.org/data/distributed_select/?q=%2Bf1source%3A%22" + source + "%22%2Bf1label%3A(%22" + gene + "%22)%20%2Blogged_pvalue%3A%5B6%20TO%20*%5D&sort=logged_pvalue%20desc&rows=200&fl=alias1%2Calias2%2Cf1qtinfo%2Cf2qtinfo%2Clink_distance%2Clogged_pvalue%2Ccorrelation%2Cnum_nonna&wt=json&fq=%2Bdataset%3A" + dataset) cli.close() self.write(response.body) self.set_status(200)
def main(): tornado.options.parse_command_line() logging.info("Starting Tornado web server on http://localhost:%s" % options.port) logging.info("--data_path=%s" % options.data_path) logging.info("--executable=%s" % options.executable) logging.info("--verbose=%s" % options.verbose) application = tornado.web.Application([ (r"/", RealtimePairwise), (r"/svc/rt_pairwise", RealtimePairwise), ], **settings) application.listen(options.port, **server_settings) tornado.ioloop.IOLoop.instance().start()
def main(): parse_command_line() if not options.config_file is None: parse_config_file(options.config_file) parse_command_line() if not options.client_secret is None: settings["cookie_secret"] = options.client_secret logging.info("Starting tornadoweb on http://localhost:%s" % options.port) if not options.config_file is None: logging.info("--config_file=%s" % options.config_file) application = Application([ (r"/auth/whoami", AuthWhoAmiHandler), (r"/auth/signout", AuthSignoutHandler), (r"/dao/?(.*)", MongoDbHandler), (r"/", RootHandler), (r"/(.*)", StaticFileHandler, {"path": "_public"}) ], **settings) application.listen(options.port, **server_settings) IOLoop.instance().start()
def GetUserinfo(whoami): logging.info("GetUserinfo(%s)" % whoami) collection = open_collection("private_userinfo") return collection.find_one({ "whoami": whoami })
from tornado.options import logging from tornado import httpserver, ioloop def handle_request(request): message = """ <!DOCTYPE html> <html lang="en"> <head> <title>Good to see it. - test designed By Richard, </title> <meta charset="UTF-8" /> </head> <body> It's good to see it here. Server has been started in master branch. </body> </html> """ request.write("HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n%s" % (len(message), message)) request.finish() if __name__ == "__main__": logging.info("Now I am at master branch! In the first commit") http_server = httpserver.HTTPServer(handle_request) http_server.listen(8888) ioloop.IOLoop.instance().start() # master.py ended here
def main(): tornado.options.parse_command_line() if not options.config_file is None: tornado.options.parse_config_file(options.config_file) tornado.options.parse_command_line() settings["cookie_secret"] = options.client_secret logging.info("Starting Tornado web server on http://localhost:%s" % options.port) logging.info("--data_path=%s" % options.data_path) logging.info("--client_host=%s" % options.client_host) logging.info("--authorized_users=%s" % options.authorized_users) logging.info("--mongo_uri=%s" % options.mongo_uri) logging.info("--mongo_lookup_uri=%s" % options.mongo_lookup_uri) logging.info("--mongo_lookup_query_limit=%s" % options.mongo_lookup_query_limit) if not options.config_file is None: logging.info("--config_file=%s" % options.config_file) application = tornado.web.Application([ (r"/", MainHandler), (r"/data?(.*)", LocalFileHandler), (r"/auth/signin/google", GoogleOAuth2Handler), (r"/auth/signin/google/oauth2_callback", GoogleOAuth2Handler), (r"/auth/signout/google", GoogleSignoutHandler), (r"/auth/whoami", WhoamiHandler), (r"/auth/providers", AuthProvidersHandler), (r"/storage/(.*)", MongoDbStorageHandler), (r"/lookups?(.*)", MongoDbLookupHandler), (r"/mutsig_rankings?(.*)", MongoDbMutSigHandler), (r"/pw_lookups?(.*)", MongoDbMutSigHandler), (r"/features_by_location?(.*)", MongoDbFeaturesByLocationHandler), (r"/RE/query/?(.*)", RegulomeExplorerDbQueryHandler) ], **settings) application.listen(options.port, **server_settings) tornado.ioloop.IOLoop.instance().start()
def main(): tornado.options.parse_command_line() if not options.config_file is None: tornado.options.parse_config_file(options.config_file) tornado.options.parse_command_line() settings["cookie_secret"] = options.client_secret logging.info("Starting Tornado web server on http://localhost:%s" % options.port) logging.info("--data_path=%s" % options.data_path) logging.info("--client_host=%s" % options.client_host) logging.info("--authorized_users=%s" % options.authorized_users) logging.info("--mongo_uri=%s" % options.mongo_uri) logging.info("--mongo_lookup_uri=%s" % options.mongo_lookup_uri) logging.info("--mongo_lookup_query_limit=%s" % options.mongo_lookup_query_limit) if not options.config_file is None: logging.info("--config_file=%s" % options.config_file) application = tornado.web.Application( [(r"/", MainHandler), (r"/data?(.*)", LocalFileHandler), (r"/auth/signin/google", GoogleOAuth2Handler), (r"/auth/signin/google/oauth2_callback", GoogleOAuth2Handler), (r"/auth/signout/google", GoogleSignoutHandler), (r"/auth/whoami", WhoamiHandler), (r"/auth/providers", AuthProvidersHandler), (r"/storage/(.*)", MongoDbStorageHandler), (r"/lookups?(.*)", MongoDbLookupHandler)], **settings) application.listen(options.port, **server_settings) tornado.ioloop.IOLoop.instance().start()
import tornado.web from tornado.options import define, options, logging define('port', default=8888, help='run on the given port', type=int) # Set up logging location tornado.options.options['log_file_prefix'].set(settings.SERVER['log_file_prefix']) # Register handlers for each application handlers = [ (r'/stream', stream.WebHandler), # Register a handler for real-time TPM (r'/ws_stream', stream.WSHandler), # Register a handler for real-time TPM Websocket (r'/(.*)', tornado.web.StaticFileHandler, {'path': settings.SERVER['static_files_path']},), # Register a handler for a base directory for static files, e.g., images, scripts, etc. ] settings = dict( template_path=os.path.join(os.path.dirname(__file__),'static/templates'), debug='True', ) application = tornado.web.Application(handlers,**settings) # Start HTTP server if __name__ == '__main__': threading.Thread(target=stream.redis_listener).start() tornado.options.parse_command_line() # Enable pretty console logging logging.info('Starting Tornado server on port %s' % options.port) http_server = tornado.httpserver.HTTPServer(application) http_server.listen(options.port) tornado.ioloop.IOLoop.instance().start()
def open_collection(self, mongo_uri, db_name, collection_name): logging.info("open_collection(%s)" % collection_name) connection = pymongo.Connection(mongo_uri) database = connection[db_name] return database[collection_name]