def main():
parser = ArgumentParser()
parser.add_argument('--csv_file', '-c', dest='csv_file', required=True, metavar='FILE',
help='CSV containing region metadata',
type=lambda x: valid_file(parser, x))
parser.add_argument('--api_version', '-a', dest='api_version', required=False,
default=strftime('%Y-%m-%d'),
help='Version of API')
parser.add_argument('--basedir', '-b', dest='base_dir', required=True,
help='Base directory to write feeds out to')
args = parser.parse_args()
csv = CSVHandler(args.csv_file)
csv.restructure()
version_directory = os.path.join(args.base_dir, args.api_version)
latest_directory = os.path.join(args.base_dir, 'latest')
schema = SchemaBuilder()
schema.add_object(csv.data)
schema.to_schema()
for directory in (version_directory, latest_directory):
if not os.path.exists(directory):
os.makedirs(directory)
with open(os.path.join(directory, 'regions.json'), 'w') as f:
f.write(csv.to_json())
with open(os.path.join(directory, 'regions.schema.json'), 'w') as f:
f.write(schema.to_json())
generate_version_list(args.base_dir)
def main():
print(str(sys.argv))
mesh = sys.argv[2]
dataLocation = os.path.join(sys.argv[1], sys.argv[3])
material = os.path.join(sys.argv[1], "materials/" + sys.argv[4])
builder = SchemaBuilder()
builder.add_schema({
"frame_time":
float(sys.argv[5]),
"frame_steps":
int(sys.argv[6]),
"end_time":
int(sys.argv[7]),
"cloths": [{
"mesh": mesh,
"materials": [{
"data": material,
"thicken": 2
}],
"remeshing": {
"refine_angle": 0.3,
"refine_compression": 0.005,
"refine_velocity": 0.5,
"size": [10e-3, 200e-3],
"aspect_min": 0.2
}
}],
"handles": [{
"end_time": 7
}],
"motions": [[{
"time": 0,
"transform": {
"scale": 1.0
},
}]],
"obstacles": [{
"mesh": dataLocation + "/body.obj",
"motion": 0
}],
"gravity": [0, 0, -9.8],
"disable": ["popfilter", "remeshing"],
"magic": {
"repulsion_thickness": 5e-3,
"collision_stiffness": 1e6
}
})
builder.to_schema()
file = open(os.path.join(str(dataLocation), 'conf.json'), 'w')
file.write(builder.to_json(indent=1))
file.close()
def generate_schema(items):
"""Creates json schema based on items."""
builder = SchemaBuilder()
builder.add_schema({"type": "object", "properties": {}})
for item in items:
builder.add_object(item)
return builder.to_schema()
def build_schema_by_json(json):
"""
Build Json Schema by json value
"""
sch_builder = SchemaBuilder()
sch_builder.add_object(json)
return sch_builder.to_schema()
def infer_schema(samples):
builder = SchemaBuilder("http://json-schema.org/draft-07/schema#")
for s in samples:
builder.add_object(s)
builder.add_schema(extension)
return builder.to_schema()
def infer_schema(samples: List[Dict[str, Any]]) -> Schema:
builder = SchemaBuilder("http://json-schema.org/draft-07/schema#")
for sample in samples:
builder.add_object(sample)
builder.add_schema(extension)
return builder.to_schema()
def guessSchema(self, dba, kind, existingSchema=None):
'''
Guess a JSONSchema for a model kind from examples.
:param DatabaseAccess dba: the fasion database to search.
:param string kind: the model kind to guess.
:param JSONobject existingSchema: starting schema, if any.
:returns: True if the schema was guessed and created.
:rtype: boolean
'''
objs = dba.table(kind).all()
builder = SchemaBuilder()
if existingSchema is not None:
builder.add_schema(existingSchema)
elif len(objs) == 0:
logging.error(
"Can't guess with no schema and no examples of kind {0}".
format(kind))
return False
for o in objs:
builder.add_object(o)
schema = builder.to_schema()
localSeg = self.loadSegment("local", dba)
localSeg.createSchema(kind, schema)
return True
def to_json_schema(obj, mode: UpdateMode, schema: Optional[Schema] = None):
"""Create JSON schema based on an object.
Arguments:
obj {dict} -- Dictionary object
Keyword Arguments:
schema {dict} -- Existing schema if exists. (default: {None})
Returns:
[dict] -- New or updated schema.
"""
if schema is not None:
schema = convert_from_openapi(schema)
if mode == UpdateMode.GEN:
builder = SchemaBuilder()
if schema is not None:
builder.add_schema(schema)
builder.add_object(obj)
out = builder.to_schema()
return out
elif schema is None:
return {"oneOf": [to_const(obj)]}
else:
return {"oneOf": [to_const(obj), schema]}
class JsonSchemaBuilder:
'''
JsonSchema builder class.
Keyword Arguments:
schema_uri: URI to be set for '$schema' key.
'''
def __init__(self, *, schema_uri=None):
if schema_uri:
self.__builder = SchemaBuilder(schema_uri=schema_uri)
else:
self.__builder = SchemaBuilder()
def schema(self, schema):
'''
Add schema to the overall schema.
Call multiple times to add more schema.
Arguments:
schema: Json schema dict
'''
self.__builder.add_schema(schema)
return self
def string(self, jstr):
'''
Add schema based on an object string to the overall schema.
Call multiple times to add more objects.
Arguments:
jstr: A string representing a Python object.
'''
jobj = Json.from_str(jstr.strip(), allow_any=True)
self.object(jobj)
return self
def object(self, jobj):
'''
Add schema based on an object to the overall schema.
Call multiple times to add more objects.
Arguments:
jobj_or_str: JsonDict/JsonList or a Python str/list/dict.
'''
if type(jobj) in {JsonDict, JsonList}:
self.__builder.add_object(jobj.raw_object)
else:
self.__builder.add_object(jobj)
return self
def build(self):
'''
Create `JsonSchema` based on the building constructs provided till this point.
'''
return JsonSchema(self.__builder.to_schema())
def generate_schema(self) -> dict:
"""
:return: a dict json-schema based on the current data
"""
builder = SchemaBuilder()
builder.add_schema({"type": "object", "properties": {}})
builder.add_object(self.to_dict())
return builder.to_schema()
def load_nested_json_schema(fp) -> dict:
# Use Genson Library to take JSON objects and generate schemas that describe them,
builder = SchemaBuilder()
builder.add_object(json.load(fp))
result = builder.to_schema()
if "items" in result and "properties" in result["items"]:
result = result["items"]["properties"]
return result
def get_json_schemas(json_data):
"""Return the standard json schema for given json"""
builder = SchemaBuilder()
builder.add_schema({"type": "object", "properties": {}})
builder.add_object(json_data)
api_schema = builder.to_schema()
return api_schema
def OutputToJson():
# output to JSON
global tmp, dataofjsonschema, DisplayJSON
tmp = json.dumps(file_data, ensure_ascii=False, indent="\t")
builder = SchemaBuilder()
datastore = json.loads(tmp)
# print(datastore)
builder.add_object(datastore)
dataofjsonschema = builder.to_schema()
def build_schemas(self):
"""Do a pass over the files and use GenSon to generate their schemas"""
# TODO add sampling so that we don't have to pass over every single record
LOGGER.info('Building schemas')
if not self.state.get('schemas'):
self.state['schemas'] = {}
for dirpath, d in self.directories.items():
dirname = d['dirname']
LOGGER.info('Building schema for `{}`'.format(dirname))
schema_builder = SchemaBuilder()
if not self.state['schemas'].get(dirname):
self.state['schemas'][dirname] = {
"type": "object",
"properties": {}
}
else:
LOGGER.info(
"Existing schema for `{}` will be used as seed schema".
format(dirname))
schema_builder.add_schema(self.state['schemas'][dirname])
for f in d['files']:
if self.file_format == 'jsonl':
for line in open(f['absolute_path'], 'r'):
parsed_line = json.loads(line)
parsed_line = self._add_key_to_rec(parsed_line, line)
schema_builder.add_object(parsed_line)
elif self.file_format == 'csv':
# Note: parsing dates is pointless until date formatting support in GenSon
for df in pd.read_csv(f['absolute_path'],
parse_dates=False,
chunksize=1):
rec = df.to_dict('records')[0]
rec = self._add_key_to_rec(rec)
schema_builder.add_object(rec)
elif self.file_format == 'log':
# TODO Use pattern per table and get it not from config
grok = Grok(CONFIG['grok_pattern'])
for line in open(f['absolute_path'], 'r'):
parsed_line = grok.match(line)
if not parsed_line:
parsed_line = {}
parsed_line['_sdc_raw_log_line'] = line
schema_builder.add_object(parsed_line)
self.directories[dirpath]['schema'] = schema_builder.to_schema()
self.state['schemas'][dirname] = self.directories[dirpath][
'schema']
LOGGER.info('Done building schemas')
def save_common_schemas():
for name, model in COMMON_MODELS.items():
schema_builder = SchemaBuilder()
schema_builder.add_schema(model)
schema = schema_builder.to_schema()
if name.endswith("resource_list.json"):
schema["properties"]["next"]["type"] = ["null", "string"]
schema["properties"]["previous"]["type"] = ["null", "string"]
with Path(name).relative_to(Path(name).root).open("w") as f:
f.write(json.dumps(schema, indent=4, sort_keys=True))
def generate(json_obj) -> dict:
"""
generate json schema from json obj
:param json_obj: json obj
"""
builder = SchemaBuilder(
schema_uri='http://json-schema.org/draft-07/schema#')
builder.add_object(json_obj)
return builder.to_schema()
def infer_schema(data_array, ):
builder = SchemaBuilder()
for data in data_array:
builder.add_schema({
"type": "object",
"properties": {},
})
builder.add_object(data)
schema = builder.to_schema()
return schema
def create_json_schema(method):
"""
Create new json-schema from method.
:param method: link for schema creation
:return: created json-schema
"""
response = requests.get(method)
builder = SchemaBuilder()
builder.add_object(response.json())
return builder.to_schema()
def print_schema(self, collection_str):
'''Print out schema of a collection
removed '_id' from collection due to its object type
and universality
'''
_, _, collection = self.con_db(collection_str)
doc = collection.find_one({})
builder = SchemaBuilder()
del doc['_id']
builder.add_object(doc)
return builder.to_schema()
def extract_schema(cls, jobj_or_str):
if type(jobj_or_str) is str:
jobj = cls.from_str(jobj_or_str.strip())
else:
jobj = jobj_or_str
from genson import SchemaBuilder
builder = SchemaBuilder()
if type(jobj) in {JsonDict, JsonList}:
builder.add_object(jobj.raw_object)
else:
builder.add_object(jobj)
return JsonSchema(builder.to_schema())
def load_nested_json_schema(self, fp) -> dict:
# Use Genson Library to take JSON objects and generate schemas that describe them,
builder = SchemaBuilder()
if self._reader_format == "jsonl":
for o in self.read():
builder.add_object(o)
else:
builder.add_object(json.load(fp))
result = builder.to_schema()
result["$schema"] = "http://json-schema.org/draft-07/schema#"
return result
def expect_request(self, schema, merge=False):
schema = self._input_object(schema)
if "properties" not in schema:
schema = { "properties": schema }
if self._input_boolean(merge):
new_schema = SchemaBuilder(schema_uri=False)
new_schema.add_schema(self.schema['properties']['request'])
new_schema.add_schema(schema)
self.schema['properties']['request'] = new_schema.to_schema()
else:
self.schema['properties']['request'] = schema
return self.schema['properties']['request']
def schema_builder(json_object):
"""
builds schema
"""
builder = SchemaBuilder()
builder.add_object(json_object)
schema = builder.to_schema()
root_keys = list(schema['properties'].keys())
for i in root_keys:
if schema['properties'][i]['type'] == 'array':
schema['properties'][i]['mergeStrategy'] = "append"
return schema
def get_json_schema(json_data):
"""Get JSON schema fron JSON object
Args:
json_data (dict): JSON data to extract schema from
Returns:
dict: JSDON schema
"""
builder = SchemaBuilder()
builder.add_object(json_data)
json_schema = builder.to_schema()
return json_schema
def load_nested_json_schema(config, logger) -> dict:
url, file_to_close = SourceFile.open_file_url(config, logger)
try:
# Use Genson Library to take JSON objects and generate schemas that describe them,
builder = SchemaBuilder()
builder.add_object(json.load(url))
result = builder.to_schema()
if "items" in result and "properties" in result["items"]:
result = result["items"]["properties"]
finally:
if file_to_close:
file_to_close.close()
return result
def load_nested_json_schema(self, fp) -> dict:
# Use Genson Library to take JSON objects and generate schemas that describe them,
builder = SchemaBuilder()
if self._reader_format == "jsonl":
for o in self.read():
builder.add_object(o)
else:
builder.add_object(json.load(fp))
result = builder.to_schema()
if "items" in result and "properties" in result["items"]:
result = result["items"]["properties"]
return result
class Tap:
@check_type
def __init__(self, tap, tap_schema, tap_name: str, tap_key: str):
if not tap_schema is None and not isinstance(tap, dict):
raise ValueError('tap_schema must be None or a dict')
self.tap = tap
self.tap_schema = tap_schema
self.tap_name = tap_name
self.tap_key = tap_key
if not self.tap_schema:
self.builder = SchemaBuilder()
self.builder.add_schema({'type': 'object', 'properties': {}})
def __iter__(self):
return self
def __next__(self):
row = self.tap.emit()
if row:
if not isinstance(row, dict):
raise ValueError('tap.emit() must returned a dict')
if self.tap_key not in row:
raise ValueError('tap key not exist in elements from tap')
if not self.tap_schema:
self.builder.add_object(row)
schema = self.builder.to_schema()
else:
schema = self.tap_schema
if isinstance(self.tap_key, (str, bytes)):
key_properties = [self.tap_key]
if not isinstance(key_properties, list):
raise Exception('tap key must be a string or list of strings')
r = SchemaMessage(
stream = self.tap_name,
schema = schema,
key_properties = key_properties,
bookmark_properties = None,
)
s = format_message(r)
r = RecordMessage(
stream = self.tap_name, record = row, time_extracted = None
)
r = format_message(r)
row = (s.encode(), r.encode())
return row
def json_to_schema(self, json_data):
json_builder = SchemaBuilder()
json_builder.add_object(json_data)
json_data = json_builder.to_schema()
json_data = fixSchema(json_data)
if json_data.get('anyOf'):
for x in json_data.get('anyOf'):
if 'items' in x:
schema_items = x
break
if schema_items:
json_data.pop('anyOf', None)
json_data.update(schema_items)
return json_data
def merge_json_schemas(
schemas: List[Schema]) -> Union[JsonSchema, InvalidSchema]:
try:
builder = SchemaBuilder()
for schema in schemas:
if isinstance(schema, JsonSchema):
builder.add_schema(schema.schema)
else:
return InvalidSchema(
"merge_json_schemas Only supports JsonSchema type")
merged = builder.to_schema()
return JsonSchema(merged, schemas[0].path) # path does not matter here
except Exception as e:
return InvalidSchema(f"Invalid Schema, builder error: {message(e)}")
def _infer_schema(records, field_names=None):
builder = SchemaBuilder()
count = 0
for record in records:
builder.add_object(record)
count += 1
if count == 0:
return {
"type": "record",
"name": "Root",
}
#print(builder.to_json(indent=2))
schema = _json_to_avro_schema(builder.to_schema(), field_names)
#print(json.dumps(schema, indent=True))
return schema
def main():
builder = SchemaBuilder()
print("Generating new Syft json schema...")
for filepath in glob.glob(os.path.join(EXAMPLES_DIR, '*.json')):
with open(filepath, 'r') as f:
print(f" adding {filepath}")
builder.add_object(json.loads(f.read()))
print("Building schema...")
new_schema = builder.to_schema()
with open(OUTPUT, 'w') as f:
f.write(json.dumps(new_schema, sort_keys=True, indent=4))
print(f"New schema written to '{OUTPUT}'")