def __init__(
self,
schema_filename=None,
root_schema_dict=None,
main_sheet_name="main",
rollup=False,
root_id="ocid",
use_titles=False,
):
self.sub_sheets = {}
self.main_sheet = Sheet()
self.sub_sheet_mapping = {}
self.main_sheet_name = main_sheet_name
self.rollup = rollup
self.root_id = root_id
self.use_titles = use_titles
if root_schema_dict is None and schema_filename is None:
raise ValueError("One of schema_filename or root_schema_dict must be supplied")
if root_schema_dict is not None and schema_filename is not None:
raise ValueError("Only one of schema_filename or root_schema_dict should be supplied")
if schema_filename:
if schema_filename.startswith("http"):
import requests
r = requests.get(schema_filename)
self.root_schema_dict = jsonref.loads(r.text, object_pairs_hook=OrderedDict)
else:
with codecs.open(schema_filename, encoding="utf-8") as schema_file:
self.root_schema_dict = jsonref.load(schema_file, object_pairs_hook=OrderedDict)
else:
self.root_schema_dict = root_schema_dict
def generate_models(self, files):
loader = JmgLoader()
for fname in (f for fileGlob in files for f in glob.glob(fileGlob)):
if self.root_name:
scope = [self.root_name]
else:
base_name = os.path.basename(fname)
base_uri = os.path.splitext(base_name)[0]
base_uri = base_uri.replace('.schema', '')
scope = [base_uri]
with open(fname) as jsonFile:
print("%s" % fname)
# root_schema = json.load(jsonFile)
# base_uri = 'file://' + os.path.split(os.path.realpath(f))[0]
base_uri = 'file://' + os.path.realpath(fname)
root_schema = jsonref.load(jsonFile,
base_uri=base_uri,
jsonschema=False, # resolve references relative to local tree, not against "id" URIs
loader=loader,
object_pairs_hook=collections.OrderedDict
)
# import json
# print(json.dumps(root_schema, indent=4, separators=(',', ': ')))
if self.validate:
# TODO: Add exception handling
try:
Draft4Validator.check_schema(root_schema)
except SchemaError as e:
print(e)
sys.exit(-1)
assert isinstance(root_schema, dict)
if JsonSchema2Model.SCHEMA_URI not in root_schema:
root_schema[JsonSchema2Model.SCHEMA_URI] = fname
self.create_model(root_schema, scope)
self.render_models()
self.copy_static_files()
if self.include_dependencies:
self.copy_dependencies()
def __init__(self, schema_filename=None, root_schema_dict=None, rollup=False, root_id=None, use_titles=False,
disable_local_refs=False, truncation_length=3, exclude_deprecated_fields=False):
self.sub_sheets = {}
self.main_sheet = Sheet()
self.sub_sheet_mapping = {}
self.rollup = rollup
self.root_id = root_id
self.use_titles = use_titles
self.truncation_length = truncation_length
self.title_lookup = TitleLookup()
self.flattened = {}
self.exclude_deprecated_fields = exclude_deprecated_fields
if root_schema_dict is None and schema_filename is None:
raise ValueError('One of schema_filename or root_schema_dict must be supplied')
if root_schema_dict is not None and schema_filename is not None:
raise ValueError('Only one of schema_filename or root_schema_dict should be supplied')
if schema_filename:
if schema_filename.startswith('http'):
import requests
r = requests.get(schema_filename)
self.root_schema_dict = jsonref.loads(r.text, object_pairs_hook=OrderedDict)
else:
if disable_local_refs:
with codecs.open(schema_filename, encoding="utf-8") as schema_file:
self.root_schema_dict = jsonref.load(schema_file, object_pairs_hook=OrderedDict,
loader=JsonLoaderLocalRefsDisabled())
else:
if sys.version_info[:2] > (3, 0):
base_uri = pathlib.Path(os.path.realpath(schema_filename)).as_uri()
else:
base_uri = urlparse.urljoin('file:', urllib.pathname2url(os.path.abspath(schema_filename)))
with codecs.open(schema_filename, encoding="utf-8") as schema_file:
self.root_schema_dict = jsonref.load(schema_file, object_pairs_hook=OrderedDict,
base_uri=base_uri)
else:
self.root_schema_dict = root_schema_dict
def _parse_json(self, jsonfile, base_path):
"""
Parses JSON as well as resolves any `$ref`s, including references to
local files and remote (HTTP/S) files.
"""
base_path = os.path.abspath(base_path)
if not base_path.endswith("/"):
base_path = base_path + "/"
base_path = "file://" + base_path
with open(jsonfile, "r") as f:
schema = jsonref.load(f, base_uri=base_path, jsonschema=True)
return schema
def get_expanded_schema(self, schema_name):
"""
Return a schema file with all $ref properties expanded
"""
if schema_name not in self.expanded_schemas:
fn = self.get_schema_file(schema_name)
schemas_folder = self.get_schemas_folder()
base_uri = self.get_schema_path(schemas_folder)
with open(fn) as f:
jsn_schema = jsonref.load(f, base_uri=base_uri)
# cache the schema for future use
self.expanded_schemas[schema_name] = jsn_schema
else:
jsn_schema = self.expanded_schemas[schema_name]
return jsn_schema
def create_daijin_validator(simple=True):
cname = resource_filename("Mikado.configuration",
"configuration_blueprint.json")
# We have to repeate twice the ending configuration (bug in jsonref?)
baseuri = "file://" + os.path.join(
os.path.dirname(cname), os.path.basename(os.path.dirname(cname)))
with io.TextIOWrapper(
resource_stream("Mikado.configuration",
"daijin_schema.json")) as blue:
blue_print = jsonref.load(blue, jsonschema=True, base_uri=baseuri)
# _substitute_conf(blue_print)
validator = extend_with_default(jsonschema.Draft7Validator, simple=simple)
validator = validator(blue_print)
return validator
def init_app(
self,
app,
response_emit_error=False,
response_emit_warning=False,
response_emit_log=False,
):
self.app = app
default_file = os.path.join(app.root_path, "schemas", "oas.json")
schema_path = app.config.get("OAS_FILE", default_file)
with open(schema_path, "r") as schema_file:
schema = jsonref.load(schema_file)
app.extensions["flask_oasschema"] = {
"schema": schema,
"response_emit_error": response_emit_error,
"response_emit_warning": response_emit_warning,
"response_emit_log": response_emit_log,
}
return schema
def load_resolved_schema(spec_path,
file_name=None,
schema_obj=None,
path_prefix=True):
"""
Parses JSON as well as resolves any `$ref`s, including references to
local files and remote (HTTP/S) files.
"""
# Only one of file_name or schema_obj must be set
assert bool(file_name) != bool(schema_obj)
if path_prefix:
spec_path = os.path.join(spec_path, "APIs/schemas/")
base_path = os.path.abspath(spec_path)
if not base_path.endswith("/"):
base_path = base_path + "/"
if os.name == "nt":
base_uri_path = "file:///" + base_path.replace('\\', '/')
else:
base_uri_path = "file://" + base_path
loader = jsonref.JsonLoader(cache_results=False)
if file_name:
json_file = str(Path(base_path) / file_name)
with open(json_file, "r") as f:
schema = jsonref.load(f,
base_uri=base_uri_path,
loader=loader,
jsonschema=True)
elif schema_obj:
# Work around an exception when there's nothing to resolve using an object
if "$ref" in schema_obj:
schema = jsonref.JsonRef.replace_refs(schema_obj,
base_uri=base_uri_path,
loader=loader,
jsonschema=True)
else:
schema = schema_obj
return schema
def validate_pipeline():
jsonified_pipeline = soloSoft.pipelineToJSON()
current_directory = os.path.abspath(os.path.dirname(__file__))
schema_file_path = os.path.join(
current_directory, "../../src/json_schema/SoloSoft/pipeline.json"
)
json_schema_path = os.path.join(current_directory, "../../src/json_schema/SoloSoft")
if os.name == "nt":
base_uri = "file:///{}/".format(json_schema_path)
else:
base_uri = "file://{}/".format(json_schema_path)
with open(os.path.join(os.path.dirname(__file__), "test.json"), "w") as f:
json.dump(jsonified_pipeline, f, indent=4, sort_keys=True)
with open(schema_file_path) as schema_file:
my_schema = jsonref.load(schema_file, base_uri=base_uri, jsonschema=True)
validate(instance=jsonified_pipeline, schema=my_schema)
def _validate(self, data):
"""
This function validates the clinical file to make sure it adheres
to the clinical SOP.
Args:
data: Pandas data frame with individual metadata
Returns:
Error and warning messages
"""
total_error = ""
warning = ""
data_dicts = data.to_dict(orient="records")
clean_data_dicts = []
for row in data_dicts:
for k in row:
if row[k] in ['TRUE', 'FALSE']:
row[k] = True if row[k] == 'TRUE' else False
if row[k] in ['']:
row[k] = None
clean_row = {k: v for k, v in row.items() if v is not None}
clean_data_dicts.append(clean_row)
schema = jsonref.load(urllib.request.urlopen(self._schema_url))
val = jsonschema.Draft7Validator(schema=schema)
for n, record in enumerate(clean_data_dicts):
i = val.iter_errors(record)
for error in i:
total_error += f"Row {n} had the following error in column '{error.path}': {error.message}\n"
# Check if the count of the primary key is not distinct
res = data.groupby(self._primary_key_columns).size()
if (res > 1).any():
total_error += "Found duplicated {primaryKey}'s in the file.".format(
primaryKey=self._primary_key_columns)
return (total_error, warning)
def generate_models(self, files):
loader = JmgLoader()
for fname in (f for fileGlob in files for f in glob.glob(fileGlob)):
if self.root_name:
scope = [self.root_name]
else:
base_name = os.path.basename(fname)
base_uri = os.path.splitext(base_name)[0]
base_uri = base_uri.replace('.schema', '')
scope = [base_uri]
with open(fname) as jsonFile:
# root_schema = json.load(jsonFile)
# base_uri = 'file://' + os.path.split(os.path.realpath(f))[0]
base_uri = 'file://' + os.path.realpath(fname)
root_schema = jsonref.load(jsonFile, base_uri=base_uri, jsonschema=True, loader=loader)
if self.validate:
# TODO: Add exception handling
try:
Draft4Validator.check_schema(root_schema)
except SchemaError as e:
print(e)
assert isinstance(root_schema, dict)
if JsonSchema2Model.SCHEMA_URI not in root_schema:
root_schema[JsonSchema2Model.SCHEMA_URI] = fname
self.create_model(root_schema, scope)
self.render_models()
self.copy_static_files()
if self.include_dependencies:
self.copy_dependencies()
def cli(input_json, output_json):
"""Convert a openapi yaml specification to json without $ref references"""
# Due to https://github.com/OpenAPITools/openapi-generator/issues/2172
# we need to resolve references on the properties used for the pagination
# envelopes. However, we must not change all other references because it
# makes the generator create many inline_response objects.
# Here, we find all references and keep only those referring to pagination
# parameters
whitelist = (
'#/components/schemas/PaginationEnvelope/properties/page',
'#/components/schemas/PaginationEnvelope/properties/pages',
'#/components/schemas/PaginationEnvelope/properties/total',
)
api_noref = unref(jsonref.load(input_json), whitelist)
# Summarize the description so that docstrings are legible
api_noref['info'][
'description'] = 'Quetzal: an API to manage data files and their associated metadata.'
json.dump(api_noref, output_json, indent=4)
def load_scenario(path):
with open(path) as f, open("schema/scenario.schema.json") as schema_file:
schema = jsonref.load(schema_file)
contents = yaml.safe_load(f)
validator = jsonschema.Draft7Validator(schema)
errors = validator.iter_errors(contents)
tree = jsonschema.ErrorTree(errors)
if tree.total_errors > 0:
error: jsonschema.ValidationError = jsonschema.exceptions.best_match(
validator.iter_errors(contents))
out.println(f"error reading {f.name}: {error.message}")
if len(error.path) >= 3:
out.println(
f'The error is located in the #{error.path[2]} '
f'instruction of the "{contents[error.path[0]]["name"]}" scenario'
)
out.start_paragraph()
raise ValueError("error while parsing file")
return contents
def _parse_json(self, jsonfile, base_path):
"""
Parses JSON as well as resolves any `$ref`s, including references to
local files and remote (HTTP/S) files.
"""
base_path = os.path.abspath(base_path)
if not base_path.endswith("/"):
base_path = base_path + "/"
if os.name == "nt":
base_uri_path = "file:///" + base_path.replace('\\', '/')
else:
base_uri_path = "file://" + base_path
loader = jsonref.JsonLoader(cache_results=False)
with open(jsonfile, "r") as f:
schema = jsonref.load(f,
base_uri=base_uri_path,
loader=loader,
jsonschema=True)
return schema
def compile_schema(source, verbose):
"""Resolve $ref in a jsonschema file."""
import jsonref
from jsonref import JsonLoader
import jsonschema
class TestJSONResolver(JsonLoader):
def __init__(self, store=(), cache_results=True):
return super(TestJSONResolver, self).__init__(store, cache_results)
def __call__(self, uri, **kwargs):
from invenio_jsonschemas import current_jsonschemas
from invenio_jsonschemas.errors import JSONSchemaNotFound
try:
return current_jsonschemas.get_schema(uri)
except JSONSchemaNotFound:
return super(TestJSONResolver, self).__call__(uri, *kwargs)
resolved_schema = jsonref.load(source, loader=TestJSONResolver())
jsonschema.Draft4Validator.check_schema(resolved_schema)
click.echo(json.dumps(resolved_schema, indent=2))
def LoadData(logger, argParam):
'''download data
This function makes a connection, downloads the data from the database.
Parameters
----------
logger : {logging.Logger}
The logger used for logging error information
'''
print('We are in LoadData module.')
try:
print('hi')
jsonConfig = jsonref.load(open('../config/modules/loadData.json'))
print('hEre I am.')
schema = jsonConfig['saveData']['schema']
table = jsonConfig['saveData']['table']
saveFolder = jsonConfig['saveData']['saveFolder']
query = sql.SQL('''
SELECT *
FROM {schema}.{table}
''').format(schema=sql.Identifier(schema),
table=sql.Identifier(table))
data = pgIO.getAllData(query)
# Check that the data is properly loaded
print("-" * 10)
data = np.array(data)
# Save the data to the /data/raw folder
np.save(os.path.join(saveFolder, 'raw_data.npy'), data)
return data
except Exception as e:
logger.error(f'Unable to run LoadData \n {e}')
def doSomething(logger, argParam):
'''print a line
This function simply prints a single line
Parameters
----------
logger : {logging.Logger}
The logger used for logging error information
'''
print('We are in module 2')
configModule2 = jsonref.load(open('../config/module2.json'))
if 'value1' in argParam:
configModule2['value1'] = argParam['value1']
print("+" * 50)
print('#' * 100)
print(configModule2)
print('#' * 100)
return
def __init__(self,
schema_filename=None,
root_schema_dict=None,
main_sheet_name='main',
rollup=False,
root_id='ocid',
use_titles=False):
self.sub_sheets = {}
self.main_sheet = Sheet()
self.sub_sheet_mapping = {}
self.main_sheet_name = main_sheet_name
self.rollup = rollup
self.root_id = root_id
self.use_titles = use_titles
self.title_lookup = TitleLookup()
self.flattened = {}
if root_schema_dict is None and schema_filename is None:
raise ValueError(
'One of schema_filename or root_schema_dict must be supplied')
if root_schema_dict is not None and schema_filename is not None:
raise ValueError(
'Only one of schema_filename or root_schema_dict should be supplied'
)
if schema_filename:
if schema_filename.startswith('http'):
import requests
r = requests.get(schema_filename)
self.root_schema_dict = jsonref.loads(
r.text, object_pairs_hook=OrderedDict)
else:
with codecs.open(schema_filename,
encoding="utf-8") as schema_file:
self.root_schema_dict = jsonref.load(
schema_file, object_pairs_hook=OrderedDict)
else:
self.root_schema_dict = root_schema_dict
def readJson(filename='configuration.json', resolveReferences=True):
"""
Reads the input json file (defaults to configuration.json) and returns the data.
:param filename: Name of file to read.
:param resolveReferences: Resolve json refs, defaults to True.
:return: Return json data.
"""
if os.path.splitext(filename)[1] != '.json':
filename = filename + '.json'
file_path = fullJsonPath(filename)
# If it doesn't exist in the correct settings/json folder
# return without attempting to open a file that doesn't exist
if not os.path.isfile(file_path):
print 'JSON File [%s] does not exist.'
return 1
with open(file_path) as data_file:
if resolveReferences:
data = jsonref.load(data_file)
return data
data = json.load(data_file)
return data
import json
from collections import OrderedDict
import jsonref
import sys
import csv
with open("../schema/360-giving-schema.json") as schema_file:
data = jsonref.load(schema_file,object_pairs_hook=OrderedDict)
def get_field_info(field_def):
validation = ''
if 'string' in field_def['type']:
suffix = ''
field_format = ''
if 'date-time' in field_def.get('format',''):
field_type = 'datetime'
else:
field_type = 'string'
if field_def.get('enum',False):
validation = ",".join(field_def.get('enum'))
elif 'number' in field_def['type']:
suffix = ':number'
field_type = 'number'
field_format = ''
elif 'integer' in field_def['type']:
suffix = ':integer'
field_type = 'integer'
field_format = ''
elif 'boolean' in field_def['type']:
def validate(self):
"""validation of filepath"""
if self.filepath is None:
self.logger.warning('No filepath found for sub type: %s from data type: %s ',
self.sub_data_type,
self.data_type)
self.logger.warning('Skipping validation.')
return
# TODO -- The method below can be reworked once we switch to the submission system.
# Begin temporary validation skipping method.
# This method attempts to create a _temp_val_check file via "open"
# If the file exists, it means the validation has already run and we should skip it.
# If the file doesn't exist, we should create it and run the validation.
self.already_downloaded = False
try:
open(self.filepath + '_temp_val_check', 'x')
except FileExistsError:
self.already_downloaded = True
except TypeError: # if self.filepath is "None".
pass
# End of temporary validation method.
# The code below can run "as is" for validation skipping using the Download
# / S3 methods to check for existing files.
# The submission system needs to be in place (files are downloaded as .json)
# for this to work.
if self.already_downloaded is True:
self.logger.debug('Found temp validation file flag for %s. Skipping validation.',
self.filepath)
return
self.logger.debug("Attempting to validate: %s", self.filepath)
schema_lookup_dict = {
'Disease': 'schemas/disease/diseaseMetaDataDefinition.json',
'BGI': 'schemas/gene/geneMetaData.json',
'Orthology': 'schemas/orthology/orthologyMetaData.json',
'Allele': 'schemas/allele/alleleMetaData.json',
'Phenotype': 'schemas/phenotype/phenotypeMetaDataDefinition.json',
'Expression': 'schemas/expression/wildtypeExpressionMetaDataDefinition.json'
}
schema_file_name = schema_lookup_dict.get(self.data_type)
if schema_file_name is None:
self.logger.warning('No schema or method found. Skipping validation.')
return # Exit validation.
with open(self.filepath, encoding='utf-8') as data_file:
data = json.load(data_file)
# These variables are used to dynamically "fill out" all the references in the schema file.
base_dir_url = Path(os.path.realpath(os.getcwd())).as_uri() + '/'
base_file_url = urljoin(base_dir_url, schema_file_name)
with open(schema_file_name, encoding='utf-8') as schema_file:
# jsonref builds out our json #ref for the schema validation to work correctly.
expanded_schema_file = jsonref.load(schema_file, base_uri=base_file_url)
try:
jsonschema.validate(data, expanded_schema_file)
self.logger.debug("'%s' successfully validated against '%s'",
self.filepath,
schema_file_name)
except jsonschema.ValidationError as error:
self.logger.critical(error.message)
self.logger.critical(error)
raise SystemExit("FATAL ERROR in JSON validation.")
except jsonschema.SchemaError as error:
self.logger.critical(error.message)
self.logger.critical(error)
raise SystemExit("FATAL ERROR in JSON validation.")
def setModel(self):
with open(self.modelpath, "rw+") as json_model_data:
self.model = jsonref.load(json_model_data)
import jsonref
import os
from json import dump
import errno
from jsonschema import Draft7Validator
from pprint import pprint
json_schema_path = os.path.abspath(os.path.dirname(__file__))
file_path = os.path.join(json_schema_path, "job.json")
if os.name == "nt":
base_uri = "file:///{}/".format(json_schema_path)
else:
base_uri = "file://{}/".format(json_schema_path)
with open(file_path) as schema_file:
my_schema = jsonref.load(schema_file, base_uri=base_uri, jsonschema=True)
Draft7Validator.check_schema(my_schema)
schema_dir = os.path.join(json_schema_path, "../../schema")
if not os.path.exists(os.path.dirname(schema_dir)):
try:
os.makedirs(os.path.dirname(schema_dir))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
with open(os.path.join(schema_dir, "schema.json"), "w") as f:
dump(my_schema, f, indent=4, sort_keys=True)
def read_schema(schema_file, absolute=False):
schema_path = get_schema_path(schema_file, absolute)
schema_uri = 'file://{}'.format(schema_path)
with open(schema_path) as f:
return jsonref.load(f, base_uri=schema_uri)
def setSchema(self):
with open(self.schemapath, 'r') as schema_data:
self.schema = jsonref.load(schema_data)
def test_load(self, tmpdir):
json = """{"a": 1, "b": {"$ref": "#/a"}}"""
tmpdir.join("in.json").write(json)
assert load(tmpdir.join("in.json")) == {"a": 1, "b": 1}
def getAllData(logger, query, values=None, dbName=None):
'''query data from the database
Query the data over here. If there is a problem with the data, it is going
to return the value of None, and log the error. Your program needs to check
whether there was an error with the query by checking for a None return
value. Note that the location of the dataabses are assumed to be present
within the file ``../config/db.json``.
Parameters
----------
logger : {logging.logger}
logging element
query : {str}
The query to be made to the databse
values : {tuple or list-like}, optional
Additional values to be passed to the query (the default is None)
dbName : {str or None}, optional
The name of the database to use. If this is None, the function will
attempt to read the name from the ``defaultDB`` item within the
file ``../config/db.json``.
Returns
-------
list or None
A list of tuples containing the values is returned. In case
there is an error, the error will be logged, and a None will
be return
'''
vals = None
try:
db = jsonref.load(open('../config/db.json'))
# Check whether a dbName is available
if (dbName is None) and ('defaultDB' in db):
dbName = db['defaultDB']
# Check whether a dbName has been specified
if dbName is None:
logger.error('A database name has not been specified.')
return None
conn = psycopg2.connect(db[dbName]['connection'])
cur = conn.cursor()
except Exception as e:
logger.error('Unable to connect to the database')
logger.error(str(e))
return
try:
if values is None:
cur.execute(query)
else:
cur.execute(query, values)
# We assume that the data is small so we
# can download the entire thing here ...
# -------------------------------------------
vals = cur.fetchall()
except Exception as e:
logger.error(
'Unable to obtain data from the database for:\n query: {}\nvalues'.
format(query, values))
logger.error(str(e))
try:
cur.close()
conn.close()
except Exception as e:
logger.error('Unable to disconnect to the database')
logger.error(str(e))
return
return vals
def load_remote_schema(base_uri, schema):
"Load and compile a schema remotely"
schema_uri = f"{base_uri}/{schema}"
return json.load(schema_uri, jsonschema=True)
from logs import logDecorator as lD
import jsonref, pprint
from lib.databaseIO import pgIO
from lib.LaTeXreport import reportWriter as rw
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets, linear_model
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error, r2_score
import pickle
config = jsonref.load(open('../config/config.json'))
logBase = config['logging']['logBase'] + '.modules.module1.module1'
projConfig = jsonref.load(open('../config/modules/module1.json'))
@lD.log(logBase + '.getData')
def getData(logger):
'''get data and generate some tables and plots.
Parameters
----------
logger : {logging.Logger}
The logger used for logging error information
'''
dbName = projConfig['inputs']['dbName']
dbVersion = projConfig['inputs']['dbVersion']
cohortWindow = [0, 1000]
daysWindow = [0, 365]
import jsonref, pprint
import numpy as np
from tqdm import tqdm
import warnings
from sklearn.datasets import load_wine
from sklearn.model_selection import train_test_split
from sklearn.linear_model import SGDClassifier
from btb.tuning import Tunable
from btb.tuning.tuners import GPTuner
import matplotlib.pyplot as plt
config = jsonref.load(open('../config/config.json'))
logBase = config['logging']['logBase'] + '.modules.bayTune.utils.utils'
configM = jsonref.load(open('../config/modules/bayTune.json'))['params']
@lD.log(logBase + '.createDataSet')
def createDataSet(logger):
'''get the wind dataset
This functionis going to load the wine dataset and generate
outputs for training testing. If there is an error, a None
is returned
Parameters
----------
def read_schema(schema_file, absolute=False):
schema_path = get_schema_path(schema_file, absolute)
schema_uri = 'file://{}'.format(schema_path)
with open(schema_path) as f:
return jsonref.load(f, base_uri=schema_uri)
from logs import logDecorator as lD
import jsonref, pprint
from lib.celeryWorkerExample import worker_1
config = jsonref.load(open('../config/config.json'))
logBase = config['logging']['logBase'] + '.modules.celeryCheck.celeryCheck'
@lD.log(logBase + '.doSomething')
def doSomething(logger):
'''print a line
This function simply prints a single line
Parameters
----------
logger : {logging.Logger}
The logger used for logging error information
'''
try:
result = worker_1.add.delay(2, 2)
for i in range(100):
print(result.state)
if result.state == 'SUCCESS':
r = result.get()
print(f'The result of this calculation is: {r}')
except Exception as e:
logger.error(f'Unable to geenrate the celery module" {e}')
def main():
parser = argparse.ArgumentParser()
parser.add_argument("json_schema_file", type=argparse.FileType("r"),
help="Full pathname for the JSON schema file")
parser.add_argument("output_file", type=str,
help="Full pathname for the output Excel file")
parser.add_argument("--reference_path", type=str,
help="Full pathname location for references")
args = parser.parse_args()
bool_to_string = {True: "true", False: "false"}
template_workbook = Workbook()
# Check to see if a reference path has been passed in. If it has, use jsonref to load
# the validation schema. If not, it is assumed that all of the keys are defined within
# the schema. If they are not, it will not be possible to generate a definition for
# those keys.
#
# For local files, the path that is passed in must be preceded by "file://". For remote
# references, the full URL must be provided.
if args.reference_path is not None:
json_schema = jsonref.load(args.json_schema_file, base_uri=args.reference_path, jsonschema=True)
else:
json_schema = json.load(args.json_schema_file)
# Get the schema keys into a list and then write them to the template worksheet. The
# template worksheet will only have one row, but will have multiple columns.
template_ws = template_workbook.active
template_ws.title = "Template"
column_header_list = []
for column_header in json_schema["properties"].keys():
column_header_list.append(column_header)
column_number = 1
for header_value in column_header_list:
template_ws.cell(1, column_number).value = header_value
column_number += 1
# Write the dictionary to the dictionary worksheet. For keys that have a values list or
# a regex, write the possible values to the values worksheet.
dictionary_ws = template_workbook.create_sheet()
dictionary_ws.title = "Dictionary"
dictionary_ws.cell(1, 1).value = "key"
dictionary_ws.cell(1, 2).value = "description"
values_ws = template_workbook.create_sheet()
values_ws.title = "Values"
values_ws.cell(1, 1).value = "key"
values_ws.cell(1, 2).value = "description"
values_ws.cell(1, 3).value = "valueDescription"
values_ws.cell(1, 4).value = "source"
dictionary_row_number = 2
values_row_number = 2
for json_key in json_schema["properties"]:
# Dictionary worksheet
dictionary_ws.cell(dictionary_row_number, 1).value = json_key
if "description" in json_schema["properties"][json_key]:
dictionary_ws.cell(dictionary_row_number, 2).value = json_schema["properties"][json_key]["description"]
dictionary_row_number += 1
# Values worksheet
if "pattern" in json_schema["properties"][json_key]:
values_ws.cell(values_row_number, 1).value = json_key
values_ws.cell(values_row_number, 2).value = json_schema["properties"][json_key]["pattern"]
values_row_number += 1
elif "anyOf" in json_schema["properties"][json_key]:
for anyof_row in json_schema["properties"][json_key]["anyOf"]:
if "const" in anyof_row:
values_ws.cell(values_row_number, 1).value = json_key
# If the value is a Boolean, we have to convert it to a string; otherwise
# Excel will force it into all-caps, i.e. (TRUE, FALSE) and this is not
# what we want.
if isinstance(anyof_row["const"], bool):
converted_value = bool_to_string.get(anyof_row["const"], anyof_row["const"])
else:
converted_value = anyof_row["const"]
values_ws.cell(values_row_number, 2).value = converted_value
if "description" in anyof_row:
values_ws.cell(values_row_number, 3).value = anyof_row["description"]
if "source" in anyof_row:
values_ws.cell(values_row_number, 4).value = anyof_row["source"]
values_row_number += 1
template_workbook.save(args.output_file)
import jsonref, argparse
from importlib import util
from logs import logDecorator as lD
from lib.testLib import simpleLib as sL
from lib.argParsers import addAllParsers as aP
config = jsonref.load(open('../config/config.json'))
logBase = config['logging']['logBase']
logLevel = config['logging']['level']
logSpecs = config['logging']['specs']
@lD.log(logBase + '.importModules')
def importModules(logger, resultsDict):
'''import and execute required modules
This function is used for importing all the
modules as defined in the ../config/modules.json
file and executing the main function within it
if present. In error, it fails gracefully ...
Parameters
----------
logger : {logging.Logger}
logger module for logging information
'''
modules = jsonref.load(open('../config/modules.json'))
# update modules in the right order. Also get rid of the frivilous
# modules
def commitDataList(logger, query, values, dbName=None):
'''query data from the database
Query the data over here. If there is a problem with
the data, it is going to return the value of None, and
log the error. Your program needs to check whether
there was an error with the query by checking for a ``None``
return value
Parameters
----------
logger : {logging.logger}
logging element
query : {str}
The query to be made to the databse
values : {tuple or list-like}, optional
Additional values to be passed to the query (the default
is None)
dbName : {str or None}, optional
The name of the database to use. If this is None, the function will
attempt to read the name from the ``defaultDB`` item within the
file ``../config/db.json``.
Returns
-------
True or None
A successful completion of this function returns a ``True``.
In case there is an error, the error will be logged, and a ``None`` will
be returned
'''
val = True
try:
db = jsonref.load(open('../config/db.json'))
# Check whether a dbName is available
if (dbName is None) and ('defaultDB' in db):
dbName = db['defaultDB']
# Check whether a dbName has been specified
if dbName is None:
logger.error('A database name has not been specified.')
return None
conn = psycopg2.connect(db[dbName]['connection'])
cur = conn.cursor()
except Exception as e:
logger.error('Unable to connect to the database')
logger.error(str(e))
return None
try:
query = cur.mogrify(query)
execute_values(cur, query, values)
except Exception as e:
logger.error('Unable to execute query for:\n query: {}\nvalues'.format(
query, values))
logger.error(str(e))
val = None
try:
conn.commit()
cur.close()
conn.close()
except Exception as e:
logger.error('Unable to disconnect to the database')
logger.error(str(e))
return None
return val
#!/usr/bin/env python
import json
import jsonschema
res1 = json.load(open('./res/res1.json'))
res2 = json.load(open('./res/res2.json'))
schema = {'$ref': 'http://localhost:8000/schemas/schema.json'}
print jsonschema.validate(res1, schema)
print jsonschema.validate(res2, schema)
import jsonref
res = jsonref.load(open('./res/res1.json'))
print res
def getDataIterator(logger, query, values=None, chunks=100, dbName=None):
'''Create an iterator from a largish query
This is a generator that returns values in chunks of chunksize ``chunks``.
Parameters
----------
logger : {logging.logger}
logging element
query : {str}
The query to be made to the databse
values : {tuple or list-like}, optional
Additional values to be passed to the query (the default
is None)
chunks : {number}, optional
This is the number of rows that the data is going to return at every call
if __next__() to this function. (the default is 100)
dbName : {str or None}, optional
The name of the database to use. If this is None, the function will
attempt to read the name from the ``defaultDB`` item within the
file ``../config/db.json``.
Yields
------
list of tuples
A list of tuples from the query, with a maximum of ``chunks`` tuples returned
at one time.
'''
try:
db = jsonref.load(open('../config/db.json'))
# Check whether a dbName is available
if (dbName is None) and ('defaultDB' in db):
dbName = db['defaultDB']
# Check whether a dbName has been specified
if dbName is None:
logger.error('A database name has not been specified.')
return None
conn = psycopg2.connect(db[dbName]['connection'])
cur = conn.cursor('remote')
except Exception as e:
logger.error('Unable to connect to the database')
logger.error(str(e))
return
try:
if values is None:
cur.execute(query)
else:
cur.execute(query, values)
while True:
vals = cur.fetchmany(chunks)
if len(vals) == 0:
break
yield vals
except Exception as e:
logger.error(
'Unable to obtain data from the database for:\n query: {}\nvalues'.
format(query, values))
logger.error(str(e))
try:
conn.close()
except Exception as e:
logger.error('Unable to disconnect to the database')
logger.error(str(e))
return
return
def generate_models(self, files, generate_schema: bool = False, cmake: bool = False):
loader = JmgLoader()
for file_path in (f for fileGlob in files for f in glob.glob(fileGlob)):
if generate_schema:
#
# Generate the a JSON Schema file from the given JSON file
#
schema = genson.Schema()
with open(file_path) as f:
json_sample = json.load(f)
schema.add_object(json_sample)
schema_base_name = os.path.basename(file_path)
schema_root_name, schema_ext = os.path.splitext(schema_base_name)
schema_file_path = os.path.join(self.outdir, schema_root_name + '.schema' + schema_ext)
with open(schema_file_path, 'w') as s_f:
s_f.write(schema.to_json())
else:
schema_file_path = file_path
if self.root_name:
scope = [self.root_name]
else:
base_name = os.path.basename(schema_file_path)
base_uri = os.path.splitext(base_name)[0]
base_uri = base_uri.replace('.schema', '')
scope = [base_uri]
with open(schema_file_path) as jsonFile:
# root_schema = json.load(jsonFile)
# base_uri = 'file://' + os.path.split(os.path.realpath(f))[0]
base_uri = 'file://' + os.path.realpath(schema_file_path)
root_schema = jsonref.load(jsonFile, base_uri=base_uri, jsonschema=True, loader=loader)
if self.validate:
# TODO: Add exception handling
try:
Draft4Validator.check_schema(root_schema)
except SchemaError as e:
print(e)
assert isinstance(root_schema, dict)
if JsonSchema2Model.SCHEMA_URI not in root_schema:
root_schema[JsonSchema2Model.SCHEMA_URI] = schema_file_path
self.create_model(root_schema, scope)
if not cmake:
self.render_models()
self.copy_static_files()
if self.include_dependencies:
self.copy_dependencies()
else:
self.print_for_cmake()
from logs import logDecorator as lD
import jsonref, pprint
import matplotlib.pyplot as plt
from tqdm import tqdm
import operator
import csv
import json
from psycopg2.sql import SQL, Identifier, Literal
from lib.databaseIO import pgIO
from modules.table1 import comFunctions as cf
table1_config = jsonref.load(open('../config/modules/tejasT1.json'))
config = jsonref.load(open('../config/config.json'))
logBase = config['logging']['logBase'] + '.modules.table1.table1'
@lD.log(logBase + '.main')
def main(logger, resultsDict):
'''main function for module1
This function finishes all the tasks for the
main function. This is a way in which a
particular module is going to be executed.
Parameters
----------
logger : {logging.Logger}
The logger used for logging error information
import json
import jsonschema
import os
import jsonref
with open("parent.json") as f:
parent = jsonref.load(f)
with open("data.json") as f:
json_data = json.load(f)
# try:
jsonschema.validate(json_data, parent)
# except jsonschema.ValidationError as e:
# print(e)
# except jsonschema.SchemaError as e:
# print(e)
import json
from collections import OrderedDict
import jsonref
import sys
import csv
with open("../schema/360-giving-schema.json") as schema_file:
data = jsonref.load(schema_file, object_pairs_hook=OrderedDict)
def get_field_info(field_def):
validation = ''
if 'string' in field_def['type']:
suffix = ''
field_format = ''
if 'date-time' in field_def.get('format', ''):
field_type = 'datetime'
else:
field_type = 'string'
if field_def.get('enum', False):
validation = ",".join(field_def.get('enum'))
elif 'number' in field_def['type']:
suffix = ':number'
field_type = 'number'
field_format = ''
elif 'integer' in field_def['type']:
suffix = ':integer'
field_type = 'integer'
field_format = ''
def main(logger, resultDict):
'''
main function for module1
This function finishes all the tasks for the
main function. This is a way in which a
particular module is going to be executed.
Parameters
----------
logger : {logging.Logger}
The logger used for logging error information
resultsDict: {dict}
A dintionary containing information about the
command line arguments. These can be used for
overwriting command line arguments as needed.
'''
print('=' * 30)
print('Main function of reportMaker module')
print('=' * 30)
# Table 1 Report creation
writeT1.genIntro()
mainRace = jsonref.load(open("../data/final/sampleCount.json"))
writeT1.genRace(mainRace)
raceAgeDict = cfT1.countRaceAge()
writeT1.genRaceAge(raceAgeDict)
#race vs sex (M/F) splits
raceSexDict = cfT1.countRaceSex()
writeT1.genRaceSex(raceSexDict)
# race vs patient_type splits (in/out)
raceSettingDict = cfT1.countRaceSetting()
writeT1.genRaceSetting(raceSettingDict)
# Figure 1 Info
plotF1.genIntro()
rd = cfF1.genDiagCount("../data/final/t3PatientCount.json")
plotF1.genFig(rd)
# Table 2 Info
writeT2.genIntro()
countRaceSUD = cfT2.countRaceSUDppl()
allAgesGeneral = cfT2.allAgesGeneralSUD()
table2_dict2 = cfT2.allAgesCategorisedSUD()
table2_dict3 = cfT2.ageBinnedGeneralSUD()
writeT2.genAllAgesOverallSUD(allAgesGeneral)
writeT2.genAllAgesCategorySUD(table2_dict2, allAgesGeneral)
writeT2.genAllAgesBinnedSUD(table2_dict3, allAgesGeneral)
table2_dict4AA = cfT2.ageBinnedCategorisedSUD("AA")
writeT2.genBC(table2_dict4AA, "Asian American")
table2_dict4NHPI = cfT2.ageBinnedCategorisedSUD("NHPI")
writeT2.genBC(table2_dict4NHPI, "Native Hawaiian")
table2_dict4MR = cfT2.ageBinnedCategorisedSUD("MR")
writeT2.genBC(table2_dict4MR, "Multi-ethnic")
# Table 3 Report Creation
writeT3.genIntro()
t3Patients = jsonref.load(open("../data/final/t3PatientCount.json"))
table3_dict1 = jsonref.load(
open("../data/final/oddsratios_allRaces_anySUD.json"))
table3_dict2 = jsonref.load(
open("../data/final/oddsratios_allRaces_2SUDormore.json"))
writeT3.oddsRatiosAllRaces(table3_dict1, table3_dict2, t3Patients)
table3_dict3 = jsonref.load(open("../data/final/oddsratios_byRace.json"))
writeT3.oddsRatiosByRace(table3_dict3, t3Patients)
# Table 4 Report Creation
aa, nh, mr = cfT4.allTheOtherStuff()
writeT4.genIntro()
writeT4.oddsRatiosByRace(aa, nh, mr, t3Patients)
print('Getting out of reportMaker module')
print('-' * 30)
return
def importModules(logger, resultsDict):
'''import and execute required modules
This function is used for importing all the
modules as defined in the ../config/modules.json
file and executing the main function within it
if present. In error, it fails gracefully ...
Parameters
----------
logger : {logging.Logger}
logger module for logging information
'''
modules = jsonref.load(open('../config/modules.json'))
# update modules in the right order. Also get rid of the frivilous
# modules
if resultsDict['modules'] is not None:
tempModules = []
for m in resultsDict['modules']:
toAdd = [n for n in modules if n['moduleName'] == m][0]
tempModules.append(toAdd)
modules = tempModules
for m in modules:
if (resultsDict['modules'] is None):
# skip based upon modules.json
logger.info('Obtaining module information from modules.json')
try:
if not m['execute']:
logger.info('Module {} is being skipped'.format(
m['moduleName']))
continue
except Exception as e:
logger.error(
f'Unable to check whether module the module should be skipped: {e}'
)
logger.error(f'this module is being skipped')
continue
else:
# skip based upon CLI
try:
if m['moduleName'] not in resultsDict['modules']:
logger.info(
f'{m} not present within the list of CLI modules. Module is skipped'
)
continue
except Exception as e:
logger.error(
f'Unable to determine whether this module should be skipped: {e}.\n Module is being skipped.'
)
continue
try:
name, path = m['moduleName'], m['path']
logger.info('Module {} is being executed'.format(name))
module_spec = util.spec_from_file_location(name, path)
module = util.module_from_spec(module_spec)
module_spec.loader.exec_module(module)
module.main(resultsDict)
except Exception as e:
print('Unable to load module: {}->{}\n{}'.format(
name, path, str(e)))
return
"""
from pprint import pprint
import jsonref
from jsonref import JsonRef
# Sample JSON data, like from json.load
document = {
"data": ["a", "b", "c"],
"reference": {"$ref": "#/data/1"}
}
import os
d = r"D:\GitHub\mappyfile\mappyfile\schemas"
os.chdir(d)
fn = "map.json"
uri = "file:///D:/GitHub/mappyfile/mappyfile/schemas/"
with open(fn) as f:
j = jsonref.load(f, base_uri=uri)
jsn = jsonref.dumps(j, indent=4, sort_keys=False)
with open(r"D:\Temp\mapfile.json", "w") as f:
f.write(jsonref.dumps(j, indent=4, sort_keys=False))
## The JsonRef.replace_refs class method will return a copy of the document
## with refs replaced by :class:`JsonRef` objects
#pprint(JsonRef.replace_refs(document))
print("Done!")