def main(): parser = ArgumentParser() parser.add_argument('--csv_file', '-c', dest='csv_file', required=True, metavar='FILE', help='CSV containing region metadata', type=lambda x: valid_file(parser, x)) parser.add_argument('--api_version', '-a', dest='api_version', required=False, default=strftime('%Y-%m-%d'), help='Version of API') parser.add_argument('--basedir', '-b', dest='base_dir', required=True, help='Base directory to write feeds out to') args = parser.parse_args() csv = CSVHandler(args.csv_file) csv.restructure() version_directory = os.path.join(args.base_dir, args.api_version) latest_directory = os.path.join(args.base_dir, 'latest') schema = SchemaBuilder() schema.add_object(csv.data) schema.to_schema() for directory in (version_directory, latest_directory): if not os.path.exists(directory): os.makedirs(directory) with open(os.path.join(directory, 'regions.json'), 'w') as f: f.write(csv.to_json()) with open(os.path.join(directory, 'regions.schema.json'), 'w') as f: f.write(schema.to_json()) generate_version_list(args.base_dir)
def main(): print(str(sys.argv)) mesh = sys.argv[2] dataLocation = os.path.join(sys.argv[1], sys.argv[3]) material = os.path.join(sys.argv[1], "materials/" + sys.argv[4]) builder = SchemaBuilder() builder.add_schema({ "frame_time": float(sys.argv[5]), "frame_steps": int(sys.argv[6]), "end_time": int(sys.argv[7]), "cloths": [{ "mesh": mesh, "materials": [{ "data": material, "thicken": 2 }], "remeshing": { "refine_angle": 0.3, "refine_compression": 0.005, "refine_velocity": 0.5, "size": [10e-3, 200e-3], "aspect_min": 0.2 } }], "handles": [{ "end_time": 7 }], "motions": [[{ "time": 0, "transform": { "scale": 1.0 }, }]], "obstacles": [{ "mesh": dataLocation + "/body.obj", "motion": 0 }], "gravity": [0, 0, -9.8], "disable": ["popfilter", "remeshing"], "magic": { "repulsion_thickness": 5e-3, "collision_stiffness": 1e6 } }) builder.to_schema() file = open(os.path.join(str(dataLocation), 'conf.json'), 'w') file.write(builder.to_json(indent=1)) file.close()
def generate_schema(items): """Creates json schema based on items.""" builder = SchemaBuilder() builder.add_schema({"type": "object", "properties": {}}) for item in items: builder.add_object(item) return builder.to_schema()
def build_schema_by_json(json): """ Build Json Schema by json value """ sch_builder = SchemaBuilder() sch_builder.add_object(json) return sch_builder.to_schema()
def infer_schema(samples): builder = SchemaBuilder("http://json-schema.org/draft-07/schema#") for s in samples: builder.add_object(s) builder.add_schema(extension) return builder.to_schema()
def infer_schema(samples: List[Dict[str, Any]]) -> Schema: builder = SchemaBuilder("http://json-schema.org/draft-07/schema#") for sample in samples: builder.add_object(sample) builder.add_schema(extension) return builder.to_schema()
def guessSchema(self, dba, kind, existingSchema=None): ''' Guess a JSONSchema for a model kind from examples. :param DatabaseAccess dba: the fasion database to search. :param string kind: the model kind to guess. :param JSONobject existingSchema: starting schema, if any. :returns: True if the schema was guessed and created. :rtype: boolean ''' objs = dba.table(kind).all() builder = SchemaBuilder() if existingSchema is not None: builder.add_schema(existingSchema) elif len(objs) == 0: logging.error( "Can't guess with no schema and no examples of kind {0}". format(kind)) return False for o in objs: builder.add_object(o) schema = builder.to_schema() localSeg = self.loadSegment("local", dba) localSeg.createSchema(kind, schema) return True
def to_json_schema(obj, mode: UpdateMode, schema: Optional[Schema] = None): """Create JSON schema based on an object. Arguments: obj {dict} -- Dictionary object Keyword Arguments: schema {dict} -- Existing schema if exists. (default: {None}) Returns: [dict] -- New or updated schema. """ if schema is not None: schema = convert_from_openapi(schema) if mode == UpdateMode.GEN: builder = SchemaBuilder() if schema is not None: builder.add_schema(schema) builder.add_object(obj) out = builder.to_schema() return out elif schema is None: return {"oneOf": [to_const(obj)]} else: return {"oneOf": [to_const(obj), schema]}
class JsonSchemaBuilder: ''' JsonSchema builder class. Keyword Arguments: schema_uri: URI to be set for '$schema' key. ''' def __init__(self, *, schema_uri=None): if schema_uri: self.__builder = SchemaBuilder(schema_uri=schema_uri) else: self.__builder = SchemaBuilder() def schema(self, schema): ''' Add schema to the overall schema. Call multiple times to add more schema. Arguments: schema: Json schema dict ''' self.__builder.add_schema(schema) return self def string(self, jstr): ''' Add schema based on an object string to the overall schema. Call multiple times to add more objects. Arguments: jstr: A string representing a Python object. ''' jobj = Json.from_str(jstr.strip(), allow_any=True) self.object(jobj) return self def object(self, jobj): ''' Add schema based on an object to the overall schema. Call multiple times to add more objects. Arguments: jobj_or_str: JsonDict/JsonList or a Python str/list/dict. ''' if type(jobj) in {JsonDict, JsonList}: self.__builder.add_object(jobj.raw_object) else: self.__builder.add_object(jobj) return self def build(self): ''' Create `JsonSchema` based on the building constructs provided till this point. ''' return JsonSchema(self.__builder.to_schema())
def generate_schema(self) -> dict: """ :return: a dict json-schema based on the current data """ builder = SchemaBuilder() builder.add_schema({"type": "object", "properties": {}}) builder.add_object(self.to_dict()) return builder.to_schema()
def load_nested_json_schema(fp) -> dict: # Use Genson Library to take JSON objects and generate schemas that describe them, builder = SchemaBuilder() builder.add_object(json.load(fp)) result = builder.to_schema() if "items" in result and "properties" in result["items"]: result = result["items"]["properties"] return result
def get_json_schemas(json_data): """Return the standard json schema for given json""" builder = SchemaBuilder() builder.add_schema({"type": "object", "properties": {}}) builder.add_object(json_data) api_schema = builder.to_schema() return api_schema
def OutputToJson(): # output to JSON global tmp, dataofjsonschema, DisplayJSON tmp = json.dumps(file_data, ensure_ascii=False, indent="\t") builder = SchemaBuilder() datastore = json.loads(tmp) # print(datastore) builder.add_object(datastore) dataofjsonschema = builder.to_schema()
def build_schemas(self): """Do a pass over the files and use GenSon to generate their schemas""" # TODO add sampling so that we don't have to pass over every single record LOGGER.info('Building schemas') if not self.state.get('schemas'): self.state['schemas'] = {} for dirpath, d in self.directories.items(): dirname = d['dirname'] LOGGER.info('Building schema for `{}`'.format(dirname)) schema_builder = SchemaBuilder() if not self.state['schemas'].get(dirname): self.state['schemas'][dirname] = { "type": "object", "properties": {} } else: LOGGER.info( "Existing schema for `{}` will be used as seed schema". format(dirname)) schema_builder.add_schema(self.state['schemas'][dirname]) for f in d['files']: if self.file_format == 'jsonl': for line in open(f['absolute_path'], 'r'): parsed_line = json.loads(line) parsed_line = self._add_key_to_rec(parsed_line, line) schema_builder.add_object(parsed_line) elif self.file_format == 'csv': # Note: parsing dates is pointless until date formatting support in GenSon for df in pd.read_csv(f['absolute_path'], parse_dates=False, chunksize=1): rec = df.to_dict('records')[0] rec = self._add_key_to_rec(rec) schema_builder.add_object(rec) elif self.file_format == 'log': # TODO Use pattern per table and get it not from config grok = Grok(CONFIG['grok_pattern']) for line in open(f['absolute_path'], 'r'): parsed_line = grok.match(line) if not parsed_line: parsed_line = {} parsed_line['_sdc_raw_log_line'] = line schema_builder.add_object(parsed_line) self.directories[dirpath]['schema'] = schema_builder.to_schema() self.state['schemas'][dirname] = self.directories[dirpath][ 'schema'] LOGGER.info('Done building schemas')
def save_common_schemas(): for name, model in COMMON_MODELS.items(): schema_builder = SchemaBuilder() schema_builder.add_schema(model) schema = schema_builder.to_schema() if name.endswith("resource_list.json"): schema["properties"]["next"]["type"] = ["null", "string"] schema["properties"]["previous"]["type"] = ["null", "string"] with Path(name).relative_to(Path(name).root).open("w") as f: f.write(json.dumps(schema, indent=4, sort_keys=True))
def generate(json_obj) -> dict: """ generate json schema from json obj :param json_obj: json obj """ builder = SchemaBuilder( schema_uri='http://json-schema.org/draft-07/schema#') builder.add_object(json_obj) return builder.to_schema()
def infer_schema(data_array, ): builder = SchemaBuilder() for data in data_array: builder.add_schema({ "type": "object", "properties": {}, }) builder.add_object(data) schema = builder.to_schema() return schema
def create_json_schema(method): """ Create new json-schema from method. :param method: link for schema creation :return: created json-schema """ response = requests.get(method) builder = SchemaBuilder() builder.add_object(response.json()) return builder.to_schema()
def print_schema(self, collection_str): '''Print out schema of a collection removed '_id' from collection due to its object type and universality ''' _, _, collection = self.con_db(collection_str) doc = collection.find_one({}) builder = SchemaBuilder() del doc['_id'] builder.add_object(doc) return builder.to_schema()
def extract_schema(cls, jobj_or_str): if type(jobj_or_str) is str: jobj = cls.from_str(jobj_or_str.strip()) else: jobj = jobj_or_str from genson import SchemaBuilder builder = SchemaBuilder() if type(jobj) in {JsonDict, JsonList}: builder.add_object(jobj.raw_object) else: builder.add_object(jobj) return JsonSchema(builder.to_schema())
def load_nested_json_schema(self, fp) -> dict: # Use Genson Library to take JSON objects and generate schemas that describe them, builder = SchemaBuilder() if self._reader_format == "jsonl": for o in self.read(): builder.add_object(o) else: builder.add_object(json.load(fp)) result = builder.to_schema() result["$schema"] = "http://json-schema.org/draft-07/schema#" return result
def expect_request(self, schema, merge=False): schema = self._input_object(schema) if "properties" not in schema: schema = { "properties": schema } if self._input_boolean(merge): new_schema = SchemaBuilder(schema_uri=False) new_schema.add_schema(self.schema['properties']['request']) new_schema.add_schema(schema) self.schema['properties']['request'] = new_schema.to_schema() else: self.schema['properties']['request'] = schema return self.schema['properties']['request']
def schema_builder(json_object): """ builds schema """ builder = SchemaBuilder() builder.add_object(json_object) schema = builder.to_schema() root_keys = list(schema['properties'].keys()) for i in root_keys: if schema['properties'][i]['type'] == 'array': schema['properties'][i]['mergeStrategy'] = "append" return schema
def get_json_schema(json_data): """Get JSON schema fron JSON object Args: json_data (dict): JSON data to extract schema from Returns: dict: JSDON schema """ builder = SchemaBuilder() builder.add_object(json_data) json_schema = builder.to_schema() return json_schema
def load_nested_json_schema(config, logger) -> dict: url, file_to_close = SourceFile.open_file_url(config, logger) try: # Use Genson Library to take JSON objects and generate schemas that describe them, builder = SchemaBuilder() builder.add_object(json.load(url)) result = builder.to_schema() if "items" in result and "properties" in result["items"]: result = result["items"]["properties"] finally: if file_to_close: file_to_close.close() return result
def load_nested_json_schema(self, fp) -> dict: # Use Genson Library to take JSON objects and generate schemas that describe them, builder = SchemaBuilder() if self._reader_format == "jsonl": for o in self.read(): builder.add_object(o) else: builder.add_object(json.load(fp)) result = builder.to_schema() if "items" in result and "properties" in result["items"]: result = result["items"]["properties"] return result
class Tap: @check_type def __init__(self, tap, tap_schema, tap_name: str, tap_key: str): if not tap_schema is None and not isinstance(tap, dict): raise ValueError('tap_schema must be None or a dict') self.tap = tap self.tap_schema = tap_schema self.tap_name = tap_name self.tap_key = tap_key if not self.tap_schema: self.builder = SchemaBuilder() self.builder.add_schema({'type': 'object', 'properties': {}}) def __iter__(self): return self def __next__(self): row = self.tap.emit() if row: if not isinstance(row, dict): raise ValueError('tap.emit() must returned a dict') if self.tap_key not in row: raise ValueError('tap key not exist in elements from tap') if not self.tap_schema: self.builder.add_object(row) schema = self.builder.to_schema() else: schema = self.tap_schema if isinstance(self.tap_key, (str, bytes)): key_properties = [self.tap_key] if not isinstance(key_properties, list): raise Exception('tap key must be a string or list of strings') r = SchemaMessage( stream = self.tap_name, schema = schema, key_properties = key_properties, bookmark_properties = None, ) s = format_message(r) r = RecordMessage( stream = self.tap_name, record = row, time_extracted = None ) r = format_message(r) row = (s.encode(), r.encode()) return row
def json_to_schema(self, json_data): json_builder = SchemaBuilder() json_builder.add_object(json_data) json_data = json_builder.to_schema() json_data = fixSchema(json_data) if json_data.get('anyOf'): for x in json_data.get('anyOf'): if 'items' in x: schema_items = x break if schema_items: json_data.pop('anyOf', None) json_data.update(schema_items) return json_data
def merge_json_schemas( schemas: List[Schema]) -> Union[JsonSchema, InvalidSchema]: try: builder = SchemaBuilder() for schema in schemas: if isinstance(schema, JsonSchema): builder.add_schema(schema.schema) else: return InvalidSchema( "merge_json_schemas Only supports JsonSchema type") merged = builder.to_schema() return JsonSchema(merged, schemas[0].path) # path does not matter here except Exception as e: return InvalidSchema(f"Invalid Schema, builder error: {message(e)}")
def _infer_schema(records, field_names=None): builder = SchemaBuilder() count = 0 for record in records: builder.add_object(record) count += 1 if count == 0: return { "type": "record", "name": "Root", } #print(builder.to_json(indent=2)) schema = _json_to_avro_schema(builder.to_schema(), field_names) #print(json.dumps(schema, indent=True)) return schema
def main(): builder = SchemaBuilder() print("Generating new Syft json schema...") for filepath in glob.glob(os.path.join(EXAMPLES_DIR, '*.json')): with open(filepath, 'r') as f: print(f" adding {filepath}") builder.add_object(json.loads(f.read())) print("Building schema...") new_schema = builder.to_schema() with open(OUTPUT, 'w') as f: f.write(json.dumps(new_schema, sort_keys=True, indent=4)) print(f"New schema written to '{OUTPUT}'")