def app_register_blueprints(app):
# TODO: (jsm) deprecate the index endpoints on the root path,
# these are currently duplicated under /index (the ultimate
# path) for migration
app.url_map.strict_slashes = False
if "DICTIONARY_URL" in app.config:
url = app.config["DICTIONARY_URL"]
datadictionary = DataDictionary(url=url)
elif "PATH_TO_SCHEMA_DIR" in app.config:
datadictionary = DataDictionary(
root_dir=app.config["PATH_TO_SCHEMA_DIR"])
else:
import gdcdictionary
datadictionary = gdcdictionary.gdcdictionary
dictionary.init(datadictionary)
from gdcdatamodel import models as md
from gdcdatamodel import validators as vd
models.init(md)
validators.init(vd)
sheepdog_blueprint = sheepdog.create_blueprint("submission")
v0 = "/v0"
app.register_blueprint(sheepdog_blueprint, url_prefix=v0 + "/submission")
app.register_blueprint(sheepdog_blueprint, url_prefix="/submission")
app.register_blueprint(oauth2_blueprint.blueprint,
url_prefix=v0 + "/oauth2")
app.register_blueprint(oauth2_blueprint.blueprint, url_prefix="/oauth2")
def dictionary_init(app):
start = time.time()
if "DICTIONARY_URL" in app.config:
app.logger.info("Initializing dictionary from url")
url = app.config["DICTIONARY_URL"]
d = DataDictionary(url=url)
dict_init.init(d)
elif "PATH_TO_SCHEMA_DIR" in app.config:
app.logger.info("Initializing dictionary from schema dir")
d = DataDictionary(root_dir=app.config["PATH_TO_SCHEMA_DIR"])
dict_init.init(d)
else:
app.logger.info("Initializing dictionary from gdcdictionary")
import gdcdictionary
d = gdcdictionary.gdcdictionary
dictionary.init(d)
from gdcdatamodel import models as md
from gdcdatamodel import validators as vd
datamodelutils.validators.init(vd)
datamodelutils.models.init(md)
end = int(round(time.time() - start))
app.logger.info("Initialized dictionary in {} sec".format(end))
def _from_dict(writer, url_or_path):
if writer.isatty:
click.secho("Error: cannot output to TTY.", fg="red", bold=True, err=True)
return
click.secho("Loading dictionary: {}".format(url_or_path), fg="cyan", err=True)
if url_or_path.lower().startswith("http"):
d = DataDictionary(url=url_or_path)
elif url_or_path.lower().endswith(".json"):
d = DataDictionary(local_file=str(url_or_path))
else:
d = DataDictionary(root_dir=url_or_path)
dictionary.init(d)
click.secho("Parsing dictionary...", fg="cyan", err=True)
records, ontology_references, links = _parse_dictionary(d)
metadata = _get_ontology_references(ontology_references, links)
click.secho("Writing PFB...", fg="blue", err=True)
writer.set_schema(records)
writer.set_metadata(metadata)
writer.write()
click.secho(
"Done, created PFB file at: ", fg="green", err=True, nl=False, bold=True
)
click.secho(writer.name, fg="white", err=True, bold=True)
def init_dictionary(url):
d = DataDictionary(url=url)
dictionary.init(d)
# the gdcdatamodel expects dictionary initiated on load, so this can't be
# imported on module level
from gdcdatamodel import models as md
return d, md
def dictionary_setup(_app):
url = 's3://testurl'
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('s3', adapter)
json_dict = json.load(open(PATH_TO_SCHEMA_DIR + '/dictionary.json'))
adapter.register_uri('GET', url, json=json_dict, status_code=200)
resp = session.get(url)
with patch('requests.get') as get_mocked:
get_mocked.return_value = resp
datadictionary = DataDictionary(url=url)
dictionary.init(datadictionary)
from gdcdatamodel import models as md
from gdcdatamodel import validators as vd
models.init(md)
validators.init(vd)
sheepdog_blueprint = sheepdog.create_blueprint(
'submission'
)
try:
_app.register_blueprint(sheepdog_blueprint, url_prefix='/v0/submission')
except AssertionError:
_app.logger.info('Blueprint is already registered!!!')
def init_datamodel(args):
"""
register GDC models
"""
if args.dict_url:
d = DataDictionary(url=args.dict_url)
dictionary.init(d)
from gdcdatamodel import models as md # noqa
models.init(md)
def dictionary_init(app):
dictionary_url = app.config.get('DICTIONARY_URL')
if dictionary_url:
app.logger.info('Initializing dictionary from url')
d = DataDictionary(url=dictionary_url)
dict_init.init(d)
dictionary.init(d)
else:
app.logger.info('Initializing dictionary from gdcdictionary')
from gdcdictionary import gdcdictionary
dictionary.init(gdcdictionary)
from gdcdatamodel import models as md
from gdcdatamodel import validators as vd
datamodelutils.validators.init(vd)
datamodelutils.models.init(md)
def dictionary_setup(_app):
print 'dictionary setup'
url = 's3://testurl'
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount('s3', adapter)
json_dict = json.load(open(PATH_TO_SCHEMA_DIR + '/dictionary.json'))
adapter.register_uri('GET', url, json=json_dict, status_code=200)
resp = session.get(url)
with patch('requests.get') as get_mocked:
get_mocked.return_value = resp
datadictionary = DataDictionary(url=url)
dictionary.init(datadictionary)
from gdcdatamodel import models as md
from gdcdatamodel import validators as vd
models.init(md)
validators.init(vd)
def dictionary_setup(_app):
url = "s3://testurl"
session = requests.Session()
adapter = requests_mock.Adapter()
session.mount("s3", adapter)
json_dict = json.load(open(PATH_TO_SCHEMA_DIR + "/dictionary.json"))
adapter.register_uri("GET", url, json=json_dict, status_code=200)
resp = session.get(url)
with patch("requests.get") as get_mocked:
get_mocked.return_value = resp
datadictionary = DataDictionary(url=url)
dictionary.init(datadictionary)
from gdcdatamodel import models as md
from gdcdatamodel import validators as vd
models.init(md)
validators.init(vd)
def main():
import IPython
import os
db_host = os.environ["PG_HOST"]
db_name = os.environ["PG_NAME"]
db_user = os.environ["PG_USER"]
db_pass = os.environ["PG_PASS"]
dict_url = os.environ.get("DICTIONARY_URL")
if dict_url:
from dictionaryutils import DataDictionary, dictionary
d = DataDictionary(url=dict_url)
dictionary.init(d)
from gdcdatamodel import models as md
g = PsqlGraphDriver(host=db_host,
user=db_user,
password=db_pass,
database=db_name)
ss = g.session_scope
with g.session_scope() as session:
IPython.embed()
def dictionary_init(app):
start = time.time()
if ('DICTIONARY_URL' in app.config):
app.logger.info('Initializing dictionary from url')
url = app.config['DICTIONARY_URL']
d = DataDictionary(url=url)
dict_init.init(d)
elif ('PATH_TO_SCHEMA_DIR' in app.config):
app.logger.info('Initializing dictionary from schema dir')
d = DataDictionary(root_dir=app.config['PATH_TO_SCHEMA_DIR'])
dict_init.init(d)
else:
app.logger.info('Initializing dictionary from gdcdictionary')
import gdcdictionary
d = gdcdictionary.gdcdictionary
dictionary.init(d)
from gdcdatamodel import models as md
from gdcdatamodel import validators as vd
datamodelutils.validators.init(vd)
datamodelutils.models.init(md)
end = int(round(time.time() - start))
app.logger.info('Initialized dictionary in {} sec'.format(end))
def main():
args = parse_arguments()
if args.action == "submitting_data":
logger.info("Submitting data...")
submit_test_data(
args.host,
args.project,
args.dir,
args.access_token_file,
int(args.chunk_size),
)
logger.info("Done!")
return
logger.info("Data simulator initialization...")
if args.url:
logger.info("Loading dictionary from url {}".format(args.url))
dictionary.init(DataDictionary(url=args.url))
else:
logger.info("Loading dictionary from installed dictionary")
if args.action == "simulate":
# Initialize graph
logger.info("Initializing graph...")
graph = Graph(dictionary, program=args.program, project=args.project)
graph.generate_nodes_from_dictionary(args.consent_codes)
graph.construct_graph_edges()
max_samples = int(args.max_samples)
# just print error messages
graph.graph_validation(required_only=args.required_only)
# simulate data no matter what the graph passes validation or not
logger.info("Generating data...")
graph.simulate_graph_data(
path=args.path,
n_samples=max_samples,
node_num_instances_file=args.node_num_instances_file,
random=args.random,
required_only=args.required_only,
skip=args.skip,
)
elif args.action == "validate":
# Initialize graph
logger.info("Initializing graph...")
graph = Graph(dictionary)
graph.generate_nodes_from_dictionary()
graph.construct_graph_edges()
logger.info("Validating...")
graph.graph_validation()
elif args.action == "submission_order":
# Initialize graph
logger.info("Initializing graph...")
graph = Graph(dictionary)
graph.generate_nodes_from_dictionary()
graph.construct_graph_edges()
logger.info("Generating data submission order...")
if args.node_name:
node = graph.get_node_with_name(args.node_name)
submission_order = graph.generate_submission_order_path_to_node(
node)
else:
submission_order = graph.generate_submission_order()
with open(os.path.join(args.path, "DataImportOrderPath.txt"),
"w") as outfile:
for node in submission_order:
outfile.write(node.name + "\t" + node.category + "\n")
logger.info("Done!")
def init_dictionary():
datadictionary = DataDictionary(root_dir=os.path.join(MOD_DIR, "schemas"))
dictionary.init(datadictionary)
def initalize_dictionary(url):
dictionary.init(DataDictionary(url=url))
