def infer_schema(samples: List[Dict[str, Any]]) -> Schema:
builder = SchemaBuilder("http://json-schema.org/draft-07/schema#")
for sample in samples:
builder.add_object(sample)
builder.add_schema(extension)
return builder.to_schema()
def _make_schema(self, filename):
self.load(filename)
from genson import SchemaBuilder
sc = SchemaBuilder()
sc.add_object(self)
# filename = filename[:filename.rfind('.')]
return sc.to_json(indent=4) + '\n'
def to_json_schema(obj, mode: UpdateMode, schema: Optional[Schema] = None):
"""Create JSON schema based on an object.
Arguments:
obj {dict} -- Dictionary object
Keyword Arguments:
schema {dict} -- Existing schema if exists. (default: {None})
Returns:
[dict] -- New or updated schema.
"""
if schema is not None:
schema = convert_from_openapi(schema)
if mode == UpdateMode.GEN:
builder = SchemaBuilder()
if schema is not None:
builder.add_schema(schema)
builder.add_object(obj)
out = builder.to_schema()
return out
elif schema is None:
return {"oneOf": [to_const(obj)]}
else:
return {"oneOf": [to_const(obj), schema]}
def guessSchema(self, dba, kind, existingSchema=None):
'''
Guess a JSONSchema for a model kind from examples.
:param DatabaseAccess dba: the fasion database to search.
:param string kind: the model kind to guess.
:param JSONobject existingSchema: starting schema, if any.
:returns: True if the schema was guessed and created.
:rtype: boolean
'''
objs = dba.table(kind).all()
builder = SchemaBuilder()
if existingSchema is not None:
builder.add_schema(existingSchema)
elif len(objs) == 0:
logging.error(
"Can't guess with no schema and no examples of kind {0}".
format(kind))
return False
for o in objs:
builder.add_object(o)
schema = builder.to_schema()
localSeg = self.loadSegment("local", dba)
localSeg.createSchema(kind, schema)
return True
def generate_schema(items):
"""Creates json schema based on items."""
builder = SchemaBuilder()
builder.add_schema({"type": "object", "properties": {}})
for item in items:
builder.add_object(item)
return builder.to_schema()
def build_schema_by_json(json):
"""
Build Json Schema by json value
"""
sch_builder = SchemaBuilder()
sch_builder.add_object(json)
return sch_builder.to_schema()
def __create_shema(attnames, atttypes, catname):
b1 = SchemaBuilder()
shema_name = int(
hashlib.sha1(catname.encode('utf-8')).hexdigest(), 16) % (10**8)
for i, name in enumerate(attnames):
b1.add_schema({
"type": "object",
"properties": {
"{0}".format(name): {
"type": "{0}".format(atttypes[i])
}
}
})
b1.add_schema({"required": attnames})
shema = str(b1.to_json())
# with codecs.open('attSchemas/{0}.json'.format(shema_name),'w', 'utf-8') as fp:
# fp.write(json.dumps(shema, ensure_ascii=False))
with open('attSchemas/{0}.json'.format(shema_name),
'w',
encoding='utf8') as f:
f.write(shema)
# b1.to_json('attSchemas/{0}.json'.format(shema_name))
return '{0}.json'.format(shema_name)
def generate_json_schema(self, data):
'''
Uses the genson package to introspect json type data and generate the skeleton of a JSON Schema document
(Draft 6) for further documentation.
:param data: must be one of the following - python dictionary object, python list of dictionaries, json string
that can be loaded to a dictionary or list of dictionaries
:return: json string containing the generated json schema skeleton
'''
if isinstance(data, str):
data = json.loads(data)
if isinstance(data, dict):
data = [data]
if len(data) == 0:
return "Error: your list of objects (dictionaries) must contain at least one object to process"
if not isinstance(data[0], dict):
return "Error: your list must contain a dictionary type object"
try:
builder = SchemaBuilder()
builder.add_schema({"type": "object", "properties": {}})
for r in data:
for k, v in r.items():
builder.add_object({k: v})
except Exception as e:
return f"Error: {e}"
return builder.to_json()
def infer_schema(samples):
builder = SchemaBuilder("http://json-schema.org/draft-07/schema#")
for s in samples:
builder.add_object(s)
builder.add_schema(extension)
return builder.to_schema()
def main(indianApiUrlList, globalUrlList, deepsetUrlList):
for i in globalUrlList:
resp = requests.get(i[1])
data = resp.json()
builder = SchemaBuilder()
# builder.add_object(data)
# latestSchema = json.dumps(builder.to_json())
# apiStatus = statusCheck(resp,i)
# print(apiStatus)
builder.add_object(data)
latestSchema = builder.to_json()
schema_update = schemaUpdate(data, latestSchema, i)
print(schema_update)
# api_check = schemaUpdate(data,i)
# print(api_check)
print("\n\n")
for i in deepsetUrlList:
resp = requests.get(i[1])
data = resp.json()
builder = SchemaBuilder()
# builder.add_object(data)
# latestSchema = json.dumps(builder.to_json())
# apiStatus = statusCheck(resp,i)
# print(apiStatus)builder = SchemaBuilder()
builder.add_object(data)
latestSchema = builder.to_json()
schema_update = schemaUpdate(data, latestSchema, i)
print(schema_update)
# api_check = schemaUpdate(data,i)
# print(api_check)
print("\n\n")
for i in indianApiUrlList:
resp = requests.get(i[1])
data = resp.json()
builder = SchemaBuilder()
# builder.add_object(data)
# latestSchema = json.dumps(builder.to_json())
# apiStatus = statusCheck(resp,i)
# print(apiStatus)
builder.add_object(data)
latestSchema = builder.to_json()
schema_update = schemaUpdate(data, latestSchema, i)
print(schema_update)
# api_check = schemaUpdate(data,i)
# print(api_check)
print("\n\n")
def generate_schema(self) -> dict:
"""
:return: a dict json-schema based on the current data
"""
builder = SchemaBuilder()
builder.add_schema({"type": "object", "properties": {}})
builder.add_object(self.to_dict())
return builder.to_schema()
def get_json_schemas(json_data):
"""Return the standard json schema for given json"""
builder = SchemaBuilder()
builder.add_schema({"type": "object", "properties": {}})
builder.add_object(json_data)
api_schema = builder.to_schema()
return api_schema
def load_nested_json_schema(fp) -> dict:
# Use Genson Library to take JSON objects and generate schemas that describe them,
builder = SchemaBuilder()
builder.add_object(json.load(fp))
result = builder.to_schema()
if "items" in result and "properties" in result["items"]:
result = result["items"]["properties"]
return result
def OutputToJson():
# output to JSON
global tmp, dataofjsonschema, DisplayJSON
tmp = json.dumps(file_data, ensure_ascii=False, indent="\t")
builder = SchemaBuilder()
datastore = json.loads(tmp)
# print(datastore)
builder.add_object(datastore)
dataofjsonschema = builder.to_schema()
def test_add_other(self):
test_uri = 'TEST_URI'
other = SchemaBuilder(schema_uri=test_uri)
other.add_object(1)
self.add_object('one')
self.add_schema(other)
self.assertResult({
"$schema": test_uri,
"type": ["integer", "string"]})
def build_schemas(self):
"""Do a pass over the files and use GenSon to generate their schemas"""
# TODO add sampling so that we don't have to pass over every single record
LOGGER.info('Building schemas')
if not self.state.get('schemas'):
self.state['schemas'] = {}
for dirpath, d in self.directories.items():
dirname = d['dirname']
LOGGER.info('Building schema for `{}`'.format(dirname))
schema_builder = SchemaBuilder()
if not self.state['schemas'].get(dirname):
self.state['schemas'][dirname] = {
"type": "object",
"properties": {}
}
else:
LOGGER.info(
"Existing schema for `{}` will be used as seed schema".
format(dirname))
schema_builder.add_schema(self.state['schemas'][dirname])
for f in d['files']:
if self.file_format == 'jsonl':
for line in open(f['absolute_path'], 'r'):
parsed_line = json.loads(line)
parsed_line = self._add_key_to_rec(parsed_line, line)
schema_builder.add_object(parsed_line)
elif self.file_format == 'csv':
# Note: parsing dates is pointless until date formatting support in GenSon
for df in pd.read_csv(f['absolute_path'],
parse_dates=False,
chunksize=1):
rec = df.to_dict('records')[0]
rec = self._add_key_to_rec(rec)
schema_builder.add_object(rec)
elif self.file_format == 'log':
# TODO Use pattern per table and get it not from config
grok = Grok(CONFIG['grok_pattern'])
for line in open(f['absolute_path'], 'r'):
parsed_line = grok.match(line)
if not parsed_line:
parsed_line = {}
parsed_line['_sdc_raw_log_line'] = line
schema_builder.add_object(parsed_line)
self.directories[dirpath]['schema'] = schema_builder.to_schema()
self.state['schemas'][dirname] = self.directories[dirpath][
'schema']
LOGGER.info('Done building schemas')
def create_json_schema(method):
"""
Create new json-schema from method.
:param method: link for schema creation
:return: created json-schema
"""
response = requests.get(method)
builder = SchemaBuilder()
builder.add_object(response.json())
return builder.to_schema()
def gen_single_schema(path: Path) -> SchemaBuilder:
glob_exp = os.path.join(
*["*" if part == "$id" else part for part in path.parts]
)
file_names = list(glob.iglob(glob_exp, recursive=True))
schema = SchemaBuilder()
for file_name in tqdm(file_names, desc=str(path.parent)):
with open(file_name) as f:
schema.add_object(json.load(f))
return schema
def test_add_other_no_uri_overwrite(self):
test_uri = 'TEST_URI'
other = SchemaBuilder()
other.add_object(1)
self.add_object('one')
self.add_schema(other)
self.add_schema({'$schema': test_uri})
self.assertResult({
"$schema": test_uri,
"type": ["integer", "string"]})
def infer_schema(data_array, ):
builder = SchemaBuilder()
for data in data_array:
builder.add_schema({
"type": "object",
"properties": {},
})
builder.add_object(data)
schema = builder.to_schema()
return schema
def save_common_schemas():
for name, model in COMMON_MODELS.items():
schema_builder = SchemaBuilder()
schema_builder.add_schema(model)
schema = schema_builder.to_schema()
if name.endswith("resource_list.json"):
schema["properties"]["next"]["type"] = ["null", "string"]
schema["properties"]["previous"]["type"] = ["null", "string"]
with Path(name).relative_to(Path(name).root).open("w") as f:
f.write(json.dumps(schema, indent=4, sort_keys=True))
def generate(json_obj) -> dict:
"""
generate json schema from json obj
:param json_obj: json obj
"""
builder = SchemaBuilder(
schema_uri='http://json-schema.org/draft-07/schema#')
builder.add_object(json_obj)
return builder.to_schema()
def __init__(self):
self.schema_builder = SchemaBuilder()
self.types = {
'string': self.__primitive_type,
'bool': self.__primitive_type,
'array': self.__array_type,
'object': self.__object_type,
'anyOf': self.__get_any_of_type,
'null': self.__primitive_type,
'integer': self.__primitive_type
}
def print_schema(self, collection_str):
'''Print out schema of a collection
removed '_id' from collection due to its object type
and universality
'''
_, _, collection = self.con_db(collection_str)
doc = collection.find_one({})
builder = SchemaBuilder()
del doc['_id']
builder.add_object(doc)
return builder.to_schema()
def main():
print(str(sys.argv))
mesh = sys.argv[2]
dataLocation = os.path.join(sys.argv[1], sys.argv[3])
material = os.path.join(sys.argv[1], "materials/" + sys.argv[4])
builder = SchemaBuilder()
builder.add_schema({
"frame_time":
float(sys.argv[5]),
"frame_steps":
int(sys.argv[6]),
"end_time":
int(sys.argv[7]),
"cloths": [{
"mesh": mesh,
"materials": [{
"data": material,
"thicken": 2
}],
"remeshing": {
"refine_angle": 0.3,
"refine_compression": 0.005,
"refine_velocity": 0.5,
"size": [10e-3, 200e-3],
"aspect_min": 0.2
}
}],
"handles": [{
"end_time": 7
}],
"motions": [[{
"time": 0,
"transform": {
"scale": 1.0
},
}]],
"obstacles": [{
"mesh": dataLocation + "/body.obj",
"motion": 0
}],
"gravity": [0, 0, -9.8],
"disable": ["popfilter", "remeshing"],
"magic": {
"repulsion_thickness": 5e-3,
"collision_stiffness": 1e6
}
})
builder.to_schema()
file = open(os.path.join(str(dataLocation), 'conf.json'), 'w')
file.write(builder.to_json(indent=1))
file.close()
def generate_json_schema(self,
data,
return_type="json",
build_definitions=False):
'''
Uses the genson package to introspect json type data and generate the skeleton of a JSON Schema document
(Draft 6) for further documentation.
:param data: must be one of the following - python dictionary object, python list of dictionaries, json string
that can be loaded to a dictionary or list of dictionaries
:param return_type: JSON string or defaults to dictionary
:param build_definitions: Run a process to prompt for title and description on schema and properties
:return: json string containing the generated json schema skeleton
'''
if isinstance(data, str):
data = json.loads(data)
if isinstance(data, dict):
data = [data]
if len(data) == 0:
return "Error: your list of objects (dictionaries) must contain at least one object to process"
if not isinstance(data[0], dict):
return "Error: your list must contain a dictionary type object"
try:
builder = SchemaBuilder()
builder.add_schema({"type": "object", "properties": {}})
for r in data:
for k, v in r.items():
builder.add_object({k: v})
except Exception as e:
return f"Error: {e}"
schema = json.loads(builder.to_json())
if build_definitions:
schema["title"] = input("schema title: ")
schema["description"] = input("schema description: ")
new_props = dict()
for item in schema["properties"].items():
new_props[item[0]] = dict()
new_props[item[0]]["type"] = item[1]["type"]
new_props[item[0]]["title"] = input(f"title for {item[0]}: ")
new_props[item[0]]["description"] = input(
f"description for {item[0]}: ")
schema["properties"] = new_props
if return_type == "json":
schema = json.dumps(schema)
return schema
def load_nested_json_schema(self, fp) -> dict:
# Use Genson Library to take JSON objects and generate schemas that describe them,
builder = SchemaBuilder()
if self._reader_format == "jsonl":
for o in self.read():
builder.add_object(o)
else:
builder.add_object(json.load(fp))
result = builder.to_schema()
result["$schema"] = "http://json-schema.org/draft-07/schema#"
return result
def get_json_schema(json_data):
"""Get JSON schema fron JSON object
Args:
json_data (dict): JSON data to extract schema from
Returns:
dict: JSDON schema
"""
builder = SchemaBuilder()
builder.add_object(json_data)
json_schema = builder.to_schema()
return json_schema
def expect_request(self, schema, merge=False):
schema = self._input_object(schema)
if "properties" not in schema:
schema = { "properties": schema }
if self._input_boolean(merge):
new_schema = SchemaBuilder(schema_uri=False)
new_schema.add_schema(self.schema['properties']['request'])
new_schema.add_schema(schema)
self.schema['properties']['request'] = new_schema.to_schema()
else:
self.schema['properties']['request'] = schema
return self.schema['properties']['request']
def extract_schema(cls, jobj_or_str):
if type(jobj_or_str) is str:
jobj = cls.from_str(jobj_or_str.strip())
else:
jobj = jobj_or_str
from genson import SchemaBuilder
builder = SchemaBuilder()
if type(jobj) in {JsonDict, JsonList}:
builder.add_object(jobj.raw_object)
else:
builder.add_object(jobj)
return JsonSchema(builder.to_schema())