def main(): parser = argparse.ArgumentParser(description="Utility to add custom labels to TCGA feature matrix in MongoDB") parser.add_argument("--host", required=True, help="MongoDB host name") parser.add_argument("--port", required=True, type=int, help="MongoDB port") parser.add_argument("--db", required=True, help="Database name") parser.add_argument("--tumor", required=True, help="Tumor type") parser.add_argument("--root", required=True, help="Root path to search for FFN custom files") parser.add_argument("--dir", default="aux", help="directory to look for FFN custom files") parser.add_argument("--loglevel", default="INFO", help="Logging Level") args = parser.parse_args() configure_logging(args.loglevel.upper()) logging.info('starting add custom labels to feature matrix:\n\t%s' % (args)) args.topfiles = []; args.tumorfiles = [] path.walk path.walk(args.root, findFNFfiles, args) logging.info('%s %s', args.topfiles, args.tumorfiles) conn = pymongo.Connection(args.host, args.port) collection = conn[args.db]["feature_matrix"] if 0 == len(args.topfiles): raise ValueError('did not find a general custom file') else: for topfile in args.topfiles: addLabels(collection, topfile); if 0 == len(args.tumorfiles): logging.info('did not find a tumor type custom file') else: for tumorfile in args.tumorfiles: addLabels(collection, tumorfile); conn.close() logging.info('finished add custom labels to feature matrix')
def main(): args = parse_args() configure_logging(args.log_config) config = load_config(args.config) stations_pf = load_stations(config) data_gridded, data_gridded_lt, timestamps = load_forecast( args.wrfout, args.wrfout_long_term, args.components, config.get('spinup', 0)) for location in stations_pf: try: pf = timeseries_for_location(location, args, data_gridded, data_gridded_lt) except TargetOutsideGridError: LOG.info( 'Location of station %s outside wrfout grid, ignoring the station.', location['ref']) continue filename = templated_filename(config, analysis_date=args.analysis, ref=location['ref'], create_dirs=True) metadata = OrderedDict([ ('ref', location['ref']), ('name', location['name']), ('analysis_date', args.analysis.strftime('%Y-%m-%dT%H:%M:%S')), ('longitude', '{:.4f}'.format(location['lon'])), ('latitude', '{:.4f}'.format(location['lat'])) ]) save_timeseries(timestamps, pf, filename, metadata)
def main(): parser = argparse.ArgumentParser(description="Utility to annotate features with genes (tags) in MongoDB") parser.add_argument("--host", required=True, help="MongoDB host name") parser.add_argument("--port", required=True, type=int, help="MongoDB port") parser.add_argument("--db", required=True, help="Database name") parser.add_argument("--f", required=True, help="Path to gene map file") parser.add_argument("--loglevel", default="INFO", help="Logging Level") args = parser.parse_args() configure_logging(args.loglevel.upper()) logging.info("import file: %s" % args.f) logging.info("uploading to %s:%s/%s" % (args.host, args.port, args.db)) conn = pymongo.Connection(args.host, args.port) db = conn[args.db] count = 0 batch_counter = 0 # Extraction Phase tags_by_id = extract_tags_by_id(args.f) # Database Load Phase find_and_modify(db["feature_matrix"], tags_by_id) conn.close() logging.info("COMPLETE")
def main(): parser = argparse.ArgumentParser(description="Utility to import TCGA feature matrix to MongoDB") parser.add_argument("--host", required=True, help="MongoDB host name") parser.add_argument("--port", required=True, type=int, help="MongoDB port") parser.add_argument("--db", required=True, help="Database name") parser.add_argument("--fmx", required=True, help="Path to feature matrix file") parser.add_argument("--loglevel", default="INFO", help="Logging Level") args = parser.parse_args() configure_logging(args.loglevel.upper()) logging.info("import file: %s" % args.fmx) logging.info("uploading to %s:%s/%s" % (args.host, args.port, args.db)) conn = pymongo.Connection(args.host, args.port) collection = conn[args.db]["feature_matrix"] count = 0 for feature_object in extract_features(args.fmx): collection.insert(feature_object) count += 1 logging.info("inserted count=%s" % count) conn.close()
def main(): parser = argparse.ArgumentParser(description="Utility to import TCGA mutation summaries to MongoDB") parser.add_argument("--host", required=True, help="MongoDB host name") parser.add_argument("--port", required=True, type=int, help="MongoDB port") parser.add_argument("--db", required=True, help="Database name") parser.add_argument("--f", required=True, help="Path to mutation summary file") parser.add_argument("--loglevel", default="INFO", help="Logging Level") args = parser.parse_args() configure_logging(args.loglevel.upper()) logging.info("import file: %s" % args.f) logging.info("uploading to %s:%s/%s" % (args.host, args.port, args.db)) conn = pymongo.Connection(args.host, args.port) collection = conn[args.db]["mutation_summary"] exception_filename = "%s_uniprot_fail.out" % args.db if os.path.exists(exception_filename): os.remove(exception_filename) count = 0 exception_uniprot_fail = open(exception_filename, "w") for mut_row in extract_mutations(args.f, exception_uniprot_fail): collection.insert(mut_row) count += 1 exception_uniprot_fail.close() line_count = sum(1 for line in open(exception_filename, "r")) logging.info("inserted count=%s" % count) logging.info("uniprot_fail exception file=%s:%s" % (exception_filename, line_count)) conn.close()
def main(): parser = argparse.ArgumentParser(description="Utility to produce a table of genes vs. tumor types w/ median values") parser.add_argument("--host", required=True, help="MongoDB host name") parser.add_argument("--port", required=True, type=int, help="MongoDB port") parser.add_argument("dbs", nargs='+', help="Database names") parser.add_argument("--out", required=True, help="where to output the table") parser.add_argument("--platform", default="GEXP", help="feature type to find the median from") parser.add_argument("--loglevel", default="INFO", help="Logging Level") args = parser.parse_args() configure_logging(args.loglevel.upper()) logging.info('starting extract medians:\n\t%s' % (args)) gene2tumor2median = processTumors(args); outputValues(args.out, args.dbs, gene2tumor2median) logging.info('finished extract medians')
def main(): parser = argparse.ArgumentParser(description="Utility to import data into the Cancer Regulome Data Warehouse") parser.add_argument("--config_file", required=True, help="Configuration File") parser.add_argument("--loglevel", default="INFO", help="Logging Level") args = parser.parse_args() configure_logging(args.loglevel.upper()) startAt = time() logging.info("\n------------------\nSTART\n------------------") logging.debug("config_file: %s" % args.config_file) process_import(json.load(open(args.config_file))) endAt = time() durAt = round(endAt - startAt, 3) logging.info("\n------------------\nCOMPLETED in %s sec(s)\n------------------" % str(durAt))
if data_source['snmp_version']: add_snmp_api_fn = getattr( data_source_api, data_source_api_name['snmp_config']) response = add_snmp_api_fn( id=response.entity_id, body=get_snmp_request_body(data_source)) logger.info( "Successfully added: {} {} snmp : Response : {}". format(data_source_type, data_source['IP'], response)) except ApiException as e: logger.exception( "Failed adding data source: {} : Error : {} ".format( data_source['IP'], json.loads(e.body))) def parse_arguments(): parser = init_api_client.parse_arguments() parser.add_argument("--data_sources_csv", action="store", default='data_sources.csv', help="csv file with your own data sources") args = parser.parse_args() return args if __name__ == '__main__': args = parse_arguments() utilities.configure_logging("/tmp") api_client = init_api_client.get_api_client(args) main(api_client, args)
default=None, required=False, help="Output folder.") utility_group.add_argument( "--dest-file", action="store", dest="dest_file", default=None, required=False, help= "Output file name. Optional. If not set, file will be named based on mako template name." ) args = parser.parse_args() if args.verbose > 1: utilities.configure_logging(logging.DEBUG, args.logfile) elif args.verbose == 1: utilities.configure_logging(logging.INFO, args.logfile) else: utilities.configure_logging(logging.WARNING, args.logfile) logging.info(pp.pformat(args)) for m in args.metadata: if not os.path.isabs(m): m = os.path.normpath(os.path.join(os.getcwd(), m)) metadata = utilities.load_build(m) if args.dest_dir is None: logging.error('--dest-dir is required when using --template') sys.exit(1)