def build_schema_by_json(json):
"""
Build Json Schema by json value
"""
sch_builder = SchemaBuilder()
sch_builder.add_object(json)
return sch_builder.to_schema()
def _make_schema(self, filename):
self.load(filename)
from genson import SchemaBuilder
sc = SchemaBuilder()
sc.add_object(self)
# filename = filename[:filename.rfind('.')]
return sc.to_json(indent=4) + '\n'
def load_nested_json_schema(fp) -> dict:
# Use Genson Library to take JSON objects and generate schemas that describe them,
builder = SchemaBuilder()
builder.add_object(json.load(fp))
result = builder.to_schema()
if "items" in result and "properties" in result["items"]:
result = result["items"]["properties"]
return result
def test_add_other(self):
test_uri = 'TEST_URI'
other = SchemaBuilder(schema_uri=test_uri)
other.add_object(1)
self.add_object('one')
self.add_schema(other)
self.assertResult({
"$schema": test_uri,
"type": ["integer", "string"]})
def OutputToJson():
# output to JSON
global tmp, dataofjsonschema, DisplayJSON
tmp = json.dumps(file_data, ensure_ascii=False, indent="\t")
builder = SchemaBuilder()
datastore = json.loads(tmp)
# print(datastore)
builder.add_object(datastore)
dataofjsonschema = builder.to_schema()
def save_common_schemas():
for name, model in COMMON_MODELS.items():
schema_builder = SchemaBuilder()
schema_builder.add_schema(model)
schema = schema_builder.to_schema()
if name.endswith("resource_list.json"):
schema["properties"]["next"]["type"] = ["null", "string"]
schema["properties"]["previous"]["type"] = ["null", "string"]
with Path(name).relative_to(Path(name).root).open("w") as f:
f.write(json.dumps(schema, indent=4, sort_keys=True))
def test_add_other_no_uri_overwrite(self):
test_uri = 'TEST_URI'
other = SchemaBuilder()
other.add_object(1)
self.add_object('one')
self.add_schema(other)
self.add_schema({'$schema': test_uri})
self.assertResult({
"$schema": test_uri,
"type": ["integer", "string"]})
def generate(json_obj) -> dict:
"""
generate json schema from json obj
:param json_obj: json obj
"""
builder = SchemaBuilder(
schema_uri='http://json-schema.org/draft-07/schema#')
builder.add_object(json_obj)
return builder.to_schema()
def create_json_schema(method):
"""
Create new json-schema from method.
:param method: link for schema creation
:return: created json-schema
"""
response = requests.get(method)
builder = SchemaBuilder()
builder.add_object(response.json())
return builder.to_schema()
def gen_single_schema(path: Path) -> SchemaBuilder:
glob_exp = os.path.join(
*["*" if part == "$id" else part for part in path.parts]
)
file_names = list(glob.iglob(glob_exp, recursive=True))
schema = SchemaBuilder()
for file_name in tqdm(file_names, desc=str(path.parent)):
with open(file_name) as f:
schema.add_object(json.load(f))
return schema
def print_schema(self, collection_str):
'''Print out schema of a collection
removed '_id' from collection due to its object type
and universality
'''
_, _, collection = self.con_db(collection_str)
doc = collection.find_one({})
builder = SchemaBuilder()
del doc['_id']
builder.add_object(doc)
return builder.to_schema()
def __init__(self):
self.schema_builder = SchemaBuilder()
self.types = {
'string': self.__primitive_type,
'bool': self.__primitive_type,
'array': self.__array_type,
'object': self.__object_type,
'anyOf': self.__get_any_of_type,
'null': self.__primitive_type,
'integer': self.__primitive_type
}
def schema_builder(json_object):
"""
builds schema
"""
builder = SchemaBuilder()
builder.add_object(json_object)
schema = builder.to_schema()
root_keys = list(schema['properties'].keys())
for i in root_keys:
if schema['properties'][i]['type'] == 'array':
schema['properties'][i]['mergeStrategy'] = "append"
return schema
def get_json_schema(json_data):
"""Get JSON schema fron JSON object
Args:
json_data (dict): JSON data to extract schema from
Returns:
dict: JSDON schema
"""
builder = SchemaBuilder()
builder.add_object(json_data)
json_schema = builder.to_schema()
return json_schema
def __init__(self, tap, tap_schema, tap_name: str, tap_key: str):
if not tap_schema is None and not isinstance(tap, dict):
raise ValueError('tap_schema must be None or a dict')
self.tap = tap
self.tap_schema = tap_schema
self.tap_name = tap_name
self.tap_key = tap_key
if not self.tap_schema:
self.builder = SchemaBuilder()
self.builder.add_schema({'type': 'object', 'properties': {}})
def load_nested_json_schema(config, logger) -> dict:
url, file_to_close = SourceFile.open_file_url(config, logger)
try:
# Use Genson Library to take JSON objects and generate schemas that describe them,
builder = SchemaBuilder()
builder.add_object(json.load(url))
result = builder.to_schema()
if "items" in result and "properties" in result["items"]:
result = result["items"]["properties"]
finally:
if file_to_close:
file_to_close.close()
return result
class Tap:
@check_type
def __init__(self, tap, tap_schema, tap_name: str, tap_key: str):
if not tap_schema is None and not isinstance(tap, dict):
raise ValueError('tap_schema must be None or a dict')
self.tap = tap
self.tap_schema = tap_schema
self.tap_name = tap_name
self.tap_key = tap_key
if not self.tap_schema:
self.builder = SchemaBuilder()
self.builder.add_schema({'type': 'object', 'properties': {}})
def __iter__(self):
return self
def __next__(self):
row = self.tap.emit()
if row:
if not isinstance(row, dict):
raise ValueError('tap.emit() must returned a dict')
if self.tap_key not in row:
raise ValueError('tap key not exist in elements from tap')
if not self.tap_schema:
self.builder.add_object(row)
schema = self.builder.to_schema()
else:
schema = self.tap_schema
if isinstance(self.tap_key, (str, bytes)):
key_properties = [self.tap_key]
if not isinstance(key_properties, list):
raise Exception('tap key must be a string or list of strings')
r = SchemaMessage(
stream = self.tap_name,
schema = schema,
key_properties = key_properties,
bookmark_properties = None,
)
s = format_message(r)
r = RecordMessage(
stream = self.tap_name, record = row, time_extracted = None
)
r = format_message(r)
row = (s.encode(), r.encode())
return row
def json_to_schema(self, json_data):
json_builder = SchemaBuilder()
json_builder.add_object(json_data)
json_data = json_builder.to_schema()
json_data = fixSchema(json_data)
if json_data.get('anyOf'):
for x in json_data.get('anyOf'):
if 'items' in x:
schema_items = x
break
if schema_items:
json_data.pop('anyOf', None)
json_data.update(schema_items)
return json_data
def merge_json_schemas(
schemas: List[Schema]) -> Union[JsonSchema, InvalidSchema]:
try:
builder = SchemaBuilder()
for schema in schemas:
if isinstance(schema, JsonSchema):
builder.add_schema(schema.schema)
else:
return InvalidSchema(
"merge_json_schemas Only supports JsonSchema type")
merged = builder.to_schema()
return JsonSchema(merged, schemas[0].path) # path does not matter here
except Exception as e:
return InvalidSchema(f"Invalid Schema, builder error: {message(e)}")
def translate(input_text: Union[Json, Dict[str, Any]], all_optional: bool,
snake_case_field: bool) -> str:
builder = SchemaBuilder()
builder.add_object(input_text)
schema = builder.to_schema()
if all_optional:
schema["required"] = []
parser = JsonSchemaParser(
source=json.dumps(schema),
base_class="pydantic.BaseModel",
snake_case_field=snake_case_field,
)
return parser.parse()
def _infer_schema(records, field_names=None):
builder = SchemaBuilder()
count = 0
for record in records:
builder.add_object(record)
count += 1
if count == 0:
return {
"type": "record",
"name": "Root",
}
#print(builder.to_json(indent=2))
schema = _json_to_avro_schema(builder.to_schema(), field_names)
#print(json.dumps(schema, indent=True))
return schema
def main():
builder = SchemaBuilder()
print("Generating new Syft json schema...")
for filepath in glob.glob(os.path.join(EXAMPLES_DIR, '*.json')):
with open(filepath, 'r') as f:
print(f" adding {filepath}")
builder.add_object(json.loads(f.read()))
print("Building schema...")
new_schema = builder.to_schema()
with open(OUTPUT, 'w') as f:
f.write(json.dumps(new_schema, sort_keys=True, indent=4))
print(f"New schema written to '{OUTPUT}'")
def build_param(
self,
name: str,
value: Union[str, Sequence[str]],
required: bool,
mode: UpdateMode,
) -> Parameter:
"""Build a new OpenAPI compatible parameter definition from parameter
name and value.
Arguments:
name {str} -- Parameter name.
value {Any} -- Parameter value.
required {bool} -- Whether the parameter should be marked as required.
Returns:
Parameter -- [description]
"""
# TODO: this may not be accurate if there are duplicate parameters
# which is allowed in http
if isinstance(value, list) and len(value) == 1:
value = value[0]
else:
value = value
out: Schema
if mode == UpdateMode.GEN:
schema_builder = SchemaBuilder()
schema_builder.add_object(value)
out = convert_to_Schema(schema_builder.to_schema())
else:
out = Schema(oneOf=[convert_to_Schema(to_const(value))])
schema = out
return Parameter(
description=None,
deprecated=None,
allowEmptyValue=None,
style=None,
explode=None,
allowReserved=None,
content=None,
example=None,
examples=None,
_x=None,
name=name,
schema=schema,
required=required,
_in=self._in,
)
def generate_json_schema(self, data):
'''
Uses the genson package to introspect json type data and generate the skeleton of a JSON Schema document
(Draft 6) for further documentation.
:param data: must be one of the following - python dictionary object, python list of dictionaries, json string
that can be loaded to a dictionary or list of dictionaries
:return: json string containing the generated json schema skeleton
'''
if isinstance(data, str):
data = json.loads(data)
if isinstance(data, dict):
data = [data]
if len(data) == 0:
return "Error: your list of objects (dictionaries) must contain at least one object to process"
if not isinstance(data[0], dict):
return "Error: your list must contain a dictionary type object"
try:
builder = SchemaBuilder()
builder.add_schema({"type": "object", "properties": {}})
for r in data:
for k, v in r.items():
builder.add_object({k: v})
except Exception as e:
return f"Error: {e}"
return builder.to_json()
def to_json_schema(obj, mode: UpdateMode, schema: Optional[Schema] = None):
"""Create JSON schema based on an object.
Arguments:
obj {dict} -- Dictionary object
Keyword Arguments:
schema {dict} -- Existing schema if exists. (default: {None})
Returns:
[dict] -- New or updated schema.
"""
if schema is not None:
schema = convert_from_openapi(schema)
if mode == UpdateMode.GEN:
builder = SchemaBuilder()
if schema is not None:
builder.add_schema(schema)
builder.add_object(obj)
out = builder.to_schema()
return out
elif schema is None:
return {"oneOf": [to_const(obj)]}
else:
return {"oneOf": [to_const(obj), schema]}
def infer_schema(samples: List[Dict[str, Any]]) -> Schema:
builder = SchemaBuilder("http://json-schema.org/draft-07/schema#")
for sample in samples:
builder.add_object(sample)
builder.add_schema(extension)
return builder.to_schema()
def generate_schema(items):
"""Creates json schema based on items."""
builder = SchemaBuilder()
builder.add_schema({"type": "object", "properties": {}})
for item in items:
builder.add_object(item)
return builder.to_schema()
def __create_shema(attnames, atttypes, catname):
b1 = SchemaBuilder()
shema_name = int(
hashlib.sha1(catname.encode('utf-8')).hexdigest(), 16) % (10**8)
for i, name in enumerate(attnames):
b1.add_schema({
"type": "object",
"properties": {
"{0}".format(name): {
"type": "{0}".format(atttypes[i])
}
}
})
b1.add_schema({"required": attnames})
shema = str(b1.to_json())
# with codecs.open('attSchemas/{0}.json'.format(shema_name),'w', 'utf-8') as fp:
# fp.write(json.dumps(shema, ensure_ascii=False))
with open('attSchemas/{0}.json'.format(shema_name),
'w',
encoding='utf8') as f:
f.write(shema)
# b1.to_json('attSchemas/{0}.json'.format(shema_name))
return '{0}.json'.format(shema_name)
def guessSchema(self, dba, kind, existingSchema=None):
'''
Guess a JSONSchema for a model kind from examples.
:param DatabaseAccess dba: the fasion database to search.
:param string kind: the model kind to guess.
:param JSONobject existingSchema: starting schema, if any.
:returns: True if the schema was guessed and created.
:rtype: boolean
'''
objs = dba.table(kind).all()
builder = SchemaBuilder()
if existingSchema is not None:
builder.add_schema(existingSchema)
elif len(objs) == 0:
logging.error(
"Can't guess with no schema and no examples of kind {0}".
format(kind))
return False
for o in objs:
builder.add_object(o)
schema = builder.to_schema()
localSeg = self.loadSegment("local", dba)
localSeg.createSchema(kind, schema)
return True
def infer_schema(samples):
builder = SchemaBuilder("http://json-schema.org/draft-07/schema#")
for s in samples:
builder.add_object(s)
builder.add_schema(extension)
return builder.to_schema()
def main():
parser = ArgumentParser()
parser.add_argument('--csv_file', '-c', dest='csv_file', required=True, metavar='FILE',
help='CSV containing region metadata',
type=lambda x: valid_file(parser, x))
parser.add_argument('--api_version', '-a', dest='api_version', required=False,
default=strftime('%Y-%m-%d'),
help='Version of API')
parser.add_argument('--basedir', '-b', dest='base_dir', required=True,
help='Base directory to write feeds out to')
args = parser.parse_args()
csv = CSVHandler(args.csv_file)
csv.restructure()
version_directory = os.path.join(args.base_dir, args.api_version)
latest_directory = os.path.join(args.base_dir, 'latest')
schema = SchemaBuilder()
schema.add_object(csv.data)
schema.to_schema()
for directory in (version_directory, latest_directory):
if not os.path.exists(directory):
os.makedirs(directory)
with open(os.path.join(directory, 'regions.json'), 'w') as f:
f.write(csv.to_json())
with open(os.path.join(directory, 'regions.schema.json'), 'w') as f:
f.write(schema.to_json())
generate_version_list(args.base_dir)
def test_eq(self):
b1 = SchemaBuilder()
b1.add_object(1)
b2 = SchemaBuilder()
b2.add_object(1)
self.assertEqual(b1, b2)
