def main():
parser = argparse.ArgumentParser(description="Utility to add custom labels to TCGA feature matrix in MongoDB")
parser.add_argument("--host", required=True, help="MongoDB host name")
parser.add_argument("--port", required=True, type=int, help="MongoDB port")
parser.add_argument("--db", required=True, help="Database name")
parser.add_argument("--tumor", required=True, help="Tumor type")
parser.add_argument("--root", required=True, help="Root path to search for FFN custom files")
parser.add_argument("--dir", default="aux", help="directory to look for FFN custom files")
parser.add_argument("--loglevel", default="INFO", help="Logging Level")
args = parser.parse_args()
configure_logging(args.loglevel.upper())
logging.info('starting add custom labels to feature matrix:\n\t%s' % (args))
args.topfiles = [];
args.tumorfiles = []
path.walk
path.walk(args.root, findFNFfiles, args)
logging.info('%s %s', args.topfiles, args.tumorfiles)
conn = pymongo.Connection(args.host, args.port)
collection = conn[args.db]["feature_matrix"]
if 0 == len(args.topfiles):
raise ValueError('did not find a general custom file')
else:
for topfile in args.topfiles:
addLabels(collection, topfile);
if 0 == len(args.tumorfiles):
logging.info('did not find a tumor type custom file')
else:
for tumorfile in args.tumorfiles:
addLabels(collection, tumorfile);
conn.close()
logging.info('finished add custom labels to feature matrix')
def main():
args = parse_args()
configure_logging(args.log_config)
config = load_config(args.config)
stations_pf = load_stations(config)
data_gridded, data_gridded_lt, timestamps = load_forecast(
args.wrfout, args.wrfout_long_term, args.components,
config.get('spinup', 0))
for location in stations_pf:
try:
pf = timeseries_for_location(location, args, data_gridded,
data_gridded_lt)
except TargetOutsideGridError:
LOG.info(
'Location of station %s outside wrfout grid, ignoring the station.',
location['ref'])
continue
filename = templated_filename(config,
analysis_date=args.analysis,
ref=location['ref'],
create_dirs=True)
metadata = OrderedDict([
('ref', location['ref']), ('name', location['name']),
('analysis_date', args.analysis.strftime('%Y-%m-%dT%H:%M:%S')),
('longitude', '{:.4f}'.format(location['lon'])),
('latitude', '{:.4f}'.format(location['lat']))
])
save_timeseries(timestamps, pf, filename, metadata)
def main():
parser = argparse.ArgumentParser(description="Utility to annotate features with genes (tags) in MongoDB")
parser.add_argument("--host", required=True, help="MongoDB host name")
parser.add_argument("--port", required=True, type=int, help="MongoDB port")
parser.add_argument("--db", required=True, help="Database name")
parser.add_argument("--f", required=True, help="Path to gene map file")
parser.add_argument("--loglevel", default="INFO", help="Logging Level")
args = parser.parse_args()
configure_logging(args.loglevel.upper())
logging.info("import file: %s" % args.f)
logging.info("uploading to %s:%s/%s" % (args.host, args.port, args.db))
conn = pymongo.Connection(args.host, args.port)
db = conn[args.db]
count = 0
batch_counter = 0
# Extraction Phase
tags_by_id = extract_tags_by_id(args.f)
# Database Load Phase
find_and_modify(db["feature_matrix"], tags_by_id)
conn.close()
logging.info("COMPLETE")
def main():
parser = argparse.ArgumentParser(description="Utility to import TCGA feature matrix to MongoDB")
parser.add_argument("--host", required=True, help="MongoDB host name")
parser.add_argument("--port", required=True, type=int, help="MongoDB port")
parser.add_argument("--db", required=True, help="Database name")
parser.add_argument("--fmx", required=True, help="Path to feature matrix file")
parser.add_argument("--loglevel", default="INFO", help="Logging Level")
args = parser.parse_args()
configure_logging(args.loglevel.upper())
logging.info("import file: %s" % args.fmx)
logging.info("uploading to %s:%s/%s" % (args.host, args.port, args.db))
conn = pymongo.Connection(args.host, args.port)
collection = conn[args.db]["feature_matrix"]
count = 0
for feature_object in extract_features(args.fmx):
collection.insert(feature_object)
count += 1
logging.info("inserted count=%s" % count)
conn.close()
def main():
parser = argparse.ArgumentParser(description="Utility to import TCGA mutation summaries to MongoDB")
parser.add_argument("--host", required=True, help="MongoDB host name")
parser.add_argument("--port", required=True, type=int, help="MongoDB port")
parser.add_argument("--db", required=True, help="Database name")
parser.add_argument("--f", required=True, help="Path to mutation summary file")
parser.add_argument("--loglevel", default="INFO", help="Logging Level")
args = parser.parse_args()
configure_logging(args.loglevel.upper())
logging.info("import file: %s" % args.f)
logging.info("uploading to %s:%s/%s" % (args.host, args.port, args.db))
conn = pymongo.Connection(args.host, args.port)
collection = conn[args.db]["mutation_summary"]
exception_filename = "%s_uniprot_fail.out" % args.db
if os.path.exists(exception_filename): os.remove(exception_filename)
count = 0
exception_uniprot_fail = open(exception_filename, "w")
for mut_row in extract_mutations(args.f, exception_uniprot_fail):
collection.insert(mut_row)
count += 1
exception_uniprot_fail.close()
line_count = sum(1 for line in open(exception_filename, "r"))
logging.info("inserted count=%s" % count)
logging.info("uniprot_fail exception file=%s:%s" % (exception_filename, line_count))
conn.close()
def main():
parser = argparse.ArgumentParser(description="Utility to produce a table of genes vs. tumor types w/ median values")
parser.add_argument("--host", required=True, help="MongoDB host name")
parser.add_argument("--port", required=True, type=int, help="MongoDB port")
parser.add_argument("dbs", nargs='+', help="Database names")
parser.add_argument("--out", required=True, help="where to output the table")
parser.add_argument("--platform", default="GEXP", help="feature type to find the median from")
parser.add_argument("--loglevel", default="INFO", help="Logging Level")
args = parser.parse_args()
configure_logging(args.loglevel.upper())
logging.info('starting extract medians:\n\t%s' % (args))
gene2tumor2median = processTumors(args);
outputValues(args.out, args.dbs, gene2tumor2median)
logging.info('finished extract medians')
def main():
parser = argparse.ArgumentParser(description="Utility to import data into the Cancer Regulome Data Warehouse")
parser.add_argument("--config_file", required=True, help="Configuration File")
parser.add_argument("--loglevel", default="INFO", help="Logging Level")
args = parser.parse_args()
configure_logging(args.loglevel.upper())
startAt = time()
logging.info("\n------------------\nSTART\n------------------")
logging.debug("config_file: %s" % args.config_file)
process_import(json.load(open(args.config_file)))
endAt = time()
durAt = round(endAt - startAt, 3)
logging.info("\n------------------\nCOMPLETED in %s sec(s)\n------------------" % str(durAt))
if data_source['snmp_version']:
add_snmp_api_fn = getattr(
data_source_api, data_source_api_name['snmp_config'])
response = add_snmp_api_fn(
id=response.entity_id,
body=get_snmp_request_body(data_source))
logger.info(
"Successfully added: {} {} snmp : Response : {}".
format(data_source_type, data_source['IP'], response))
except ApiException as e:
logger.exception(
"Failed adding data source: {} : Error : {} ".format(
data_source['IP'], json.loads(e.body)))
def parse_arguments():
parser = init_api_client.parse_arguments()
parser.add_argument("--data_sources_csv",
action="store",
default='data_sources.csv',
help="csv file with your own data sources")
args = parser.parse_args()
return args
if __name__ == '__main__':
args = parse_arguments()
utilities.configure_logging("/tmp")
api_client = init_api_client.get_api_client(args)
main(api_client, args)
default=None,
required=False,
help="Output folder.")
utility_group.add_argument(
"--dest-file",
action="store",
dest="dest_file",
default=None,
required=False,
help=
"Output file name. Optional. If not set, file will be named based on mako template name."
)
args = parser.parse_args()
if args.verbose > 1:
utilities.configure_logging(logging.DEBUG, args.logfile)
elif args.verbose == 1:
utilities.configure_logging(logging.INFO, args.logfile)
else:
utilities.configure_logging(logging.WARNING, args.logfile)
logging.info(pp.pformat(args))
for m in args.metadata:
if not os.path.isabs(m):
m = os.path.normpath(os.path.join(os.getcwd(), m))
metadata = utilities.load_build(m)
if args.dest_dir is None:
logging.error('--dest-dir is required when using --template')
sys.exit(1)
