def app_register_blueprints(app): # TODO: (jsm) deprecate the index endpoints on the root path, # these are currently duplicated under /index (the ultimate # path) for migration app.url_map.strict_slashes = False if "DICTIONARY_URL" in app.config: url = app.config["DICTIONARY_URL"] datadictionary = DataDictionary(url=url) elif "PATH_TO_SCHEMA_DIR" in app.config: datadictionary = DataDictionary( root_dir=app.config["PATH_TO_SCHEMA_DIR"]) else: import gdcdictionary datadictionary = gdcdictionary.gdcdictionary dictionary.init(datadictionary) from gdcdatamodel import models as md from gdcdatamodel import validators as vd models.init(md) validators.init(vd) sheepdog_blueprint = sheepdog.create_blueprint("submission") v0 = "/v0" app.register_blueprint(sheepdog_blueprint, url_prefix=v0 + "/submission") app.register_blueprint(sheepdog_blueprint, url_prefix="/submission") app.register_blueprint(oauth2_blueprint.blueprint, url_prefix=v0 + "/oauth2") app.register_blueprint(oauth2_blueprint.blueprint, url_prefix="/oauth2")
def dictionary_init(app): start = time.time() if "DICTIONARY_URL" in app.config: app.logger.info("Initializing dictionary from url") url = app.config["DICTIONARY_URL"] d = DataDictionary(url=url) dict_init.init(d) elif "PATH_TO_SCHEMA_DIR" in app.config: app.logger.info("Initializing dictionary from schema dir") d = DataDictionary(root_dir=app.config["PATH_TO_SCHEMA_DIR"]) dict_init.init(d) else: app.logger.info("Initializing dictionary from gdcdictionary") import gdcdictionary d = gdcdictionary.gdcdictionary dictionary.init(d) from gdcdatamodel import models as md from gdcdatamodel import validators as vd datamodelutils.validators.init(vd) datamodelutils.models.init(md) end = int(round(time.time() - start)) app.logger.info("Initialized dictionary in {} sec".format(end))
def _from_dict(writer, url_or_path): if writer.isatty: click.secho("Error: cannot output to TTY.", fg="red", bold=True, err=True) return click.secho("Loading dictionary: {}".format(url_or_path), fg="cyan", err=True) if url_or_path.lower().startswith("http"): d = DataDictionary(url=url_or_path) elif url_or_path.lower().endswith(".json"): d = DataDictionary(local_file=str(url_or_path)) else: d = DataDictionary(root_dir=url_or_path) dictionary.init(d) click.secho("Parsing dictionary...", fg="cyan", err=True) records, ontology_references, links = _parse_dictionary(d) metadata = _get_ontology_references(ontology_references, links) click.secho("Writing PFB...", fg="blue", err=True) writer.set_schema(records) writer.set_metadata(metadata) writer.write() click.secho( "Done, created PFB file at: ", fg="green", err=True, nl=False, bold=True ) click.secho(writer.name, fg="white", err=True, bold=True)
def init_dictionary(url): d = DataDictionary(url=url) dictionary.init(d) # the gdcdatamodel expects dictionary initiated on load, so this can't be # imported on module level from gdcdatamodel import models as md return d, md
def dictionary_setup(_app): url = 's3://testurl' session = requests.Session() adapter = requests_mock.Adapter() session.mount('s3', adapter) json_dict = json.load(open(PATH_TO_SCHEMA_DIR + '/dictionary.json')) adapter.register_uri('GET', url, json=json_dict, status_code=200) resp = session.get(url) with patch('requests.get') as get_mocked: get_mocked.return_value = resp datadictionary = DataDictionary(url=url) dictionary.init(datadictionary) from gdcdatamodel import models as md from gdcdatamodel import validators as vd models.init(md) validators.init(vd) sheepdog_blueprint = sheepdog.create_blueprint( 'submission' ) try: _app.register_blueprint(sheepdog_blueprint, url_prefix='/v0/submission') except AssertionError: _app.logger.info('Blueprint is already registered!!!')
def init_datamodel(args): """ register GDC models """ if args.dict_url: d = DataDictionary(url=args.dict_url) dictionary.init(d) from gdcdatamodel import models as md # noqa models.init(md)
def dictionary_init(app): start = time.time() if ('DICTIONARY_URL' in app.config): app.logger.info('Initializing dictionary from url') url = app.config['DICTIONARY_URL'] d = DataDictionary(url=url) dict_init.init(d) elif ('PATH_TO_SCHEMA_DIR' in app.config): app.logger.info('Initializing dictionary from schema dir') d = DataDictionary(root_dir=app.config['PATH_TO_SCHEMA_DIR']) dict_init.init(d) else: app.logger.info('Initializing dictionary from gdcdictionary') import gdcdictionary d = gdcdictionary.gdcdictionary dictionary.init(d) from gdcdatamodel import models as md from gdcdatamodel import validators as vd datamodelutils.validators.init(vd) datamodelutils.models.init(md) end = int(round(time.time() - start)) app.logger.info('Initialized dictionary in {} sec'.format(end))
def dictionary_init(app): dictionary_url = app.config.get('DICTIONARY_URL') if dictionary_url: app.logger.info('Initializing dictionary from url') d = DataDictionary(url=dictionary_url) dict_init.init(d) dictionary.init(d) else: app.logger.info('Initializing dictionary from gdcdictionary') from gdcdictionary import gdcdictionary dictionary.init(gdcdictionary) from gdcdatamodel import models as md from gdcdatamodel import validators as vd datamodelutils.validators.init(vd) datamodelutils.models.init(md)
def dictionary_setup(_app): print 'dictionary setup' url = 's3://testurl' session = requests.Session() adapter = requests_mock.Adapter() session.mount('s3', adapter) json_dict = json.load(open(PATH_TO_SCHEMA_DIR + '/dictionary.json')) adapter.register_uri('GET', url, json=json_dict, status_code=200) resp = session.get(url) with patch('requests.get') as get_mocked: get_mocked.return_value = resp datadictionary = DataDictionary(url=url) dictionary.init(datadictionary) from gdcdatamodel import models as md from gdcdatamodel import validators as vd models.init(md) validators.init(vd)
def dictionary_setup(_app): url = "s3://testurl" session = requests.Session() adapter = requests_mock.Adapter() session.mount("s3", adapter) json_dict = json.load(open(PATH_TO_SCHEMA_DIR + "/dictionary.json")) adapter.register_uri("GET", url, json=json_dict, status_code=200) resp = session.get(url) with patch("requests.get") as get_mocked: get_mocked.return_value = resp datadictionary = DataDictionary(url=url) dictionary.init(datadictionary) from gdcdatamodel import models as md from gdcdatamodel import validators as vd models.init(md) validators.init(vd)
def main(): import IPython import os db_host = os.environ["PG_HOST"] db_name = os.environ["PG_NAME"] db_user = os.environ["PG_USER"] db_pass = os.environ["PG_PASS"] dict_url = os.environ.get("DICTIONARY_URL") if dict_url: from dictionaryutils import DataDictionary, dictionary d = DataDictionary(url=dict_url) dictionary.init(d) from gdcdatamodel import models as md g = PsqlGraphDriver(host=db_host, user=db_user, password=db_pass, database=db_name) ss = g.session_scope with g.session_scope() as session: IPython.embed()
def main(): args = parse_arguments() if args.action == "submitting_data": logger.info("Submitting data...") submit_test_data( args.host, args.project, args.dir, args.access_token_file, int(args.chunk_size), ) logger.info("Done!") return logger.info("Data simulator initialization...") if args.url: logger.info("Loading dictionary from url {}".format(args.url)) dictionary.init(DataDictionary(url=args.url)) else: logger.info("Loading dictionary from installed dictionary") if args.action == "simulate": # Initialize graph logger.info("Initializing graph...") graph = Graph(dictionary, program=args.program, project=args.project) graph.generate_nodes_from_dictionary(args.consent_codes) graph.construct_graph_edges() max_samples = int(args.max_samples) # just print error messages graph.graph_validation(required_only=args.required_only) # simulate data no matter what the graph passes validation or not logger.info("Generating data...") graph.simulate_graph_data( path=args.path, n_samples=max_samples, node_num_instances_file=args.node_num_instances_file, random=args.random, required_only=args.required_only, skip=args.skip, ) elif args.action == "validate": # Initialize graph logger.info("Initializing graph...") graph = Graph(dictionary) graph.generate_nodes_from_dictionary() graph.construct_graph_edges() logger.info("Validating...") graph.graph_validation() elif args.action == "submission_order": # Initialize graph logger.info("Initializing graph...") graph = Graph(dictionary) graph.generate_nodes_from_dictionary() graph.construct_graph_edges() logger.info("Generating data submission order...") if args.node_name: node = graph.get_node_with_name(args.node_name) submission_order = graph.generate_submission_order_path_to_node( node) else: submission_order = graph.generate_submission_order() with open(os.path.join(args.path, "DataImportOrderPath.txt"), "w") as outfile: for node in submission_order: outfile.write(node.name + "\t" + node.category + "\n") logger.info("Done!")
def init_dictionary(): datadictionary = DataDictionary(root_dir=os.path.join(MOD_DIR, "schemas")) dictionary.init(datadictionary)
def initalize_dictionary(url): dictionary.init(DataDictionary(url=url))