def _generate_md5(spec): asset = AssetResolver(None).resolve(spec) md5 = hashlib.md5() with asset.stream() as stream: for block in iter(lambda: stream.read(4096), b''): md5.update(block) return md5.hexdigest()
def _load_manifest(self): manifest = getattr(self, "_manifest", None) if manifest is None: manifest = AssetResolver().resolve(self.manifest_asset) with manifest.stream() as fp: manifest = json.loads(fp.read().decode("utf8")) if self.cache: self._manifest = manifest return manifest
def load_schema(filename): if isinstance(filename, dict): schema = filename resolver = NoRemoteResolver.from_schema(schema) else: utf8 = codecs.getreader("utf-8") asset = AssetResolver(caller_package()).resolve(filename) schema = json.load(utf8(asset.stream()), object_pairs_hook=collections.OrderedDict) resolver = RefResolver('file://' + asset.abspath(), schema) schema = mixinProperties(schema, resolver) # SchemaValidator is not thread safe for now SchemaValidator(schema, resolver=resolver, serialize=True) return schema
def load_schema(filename): if isinstance(filename, dict): schema = filename resolver = NoRemoteResolver.from_schema(schema) else: utf8 = codecs.getreader("utf-8") asset = AssetResolver(caller_package()).resolve(filename) schema = json.load(utf8(asset.stream()), object_pairs_hook=collections.OrderedDict) resolver = RefResolverOrdered('file://' + asset.abspath(), schema) # use mixinProperties, mixinFacets, mixinAggregations, and mixinColumns (if provided) schema = mixinSchemas( mixinSchemas( mixinSchemas( mixinSchemas(schema, resolver, 'properties'), resolver, 'facets' ), resolver, 'aggregations' ), resolver, 'columns' ) # SchemaValidator is not thread safe for now SchemaValidator(schema, resolver=resolver, serialize=True) return schema
def json_asset(spec, **kw): utf8 = codecs.getreader("utf-8") asset = AssetResolver(caller_package()).resolve(spec) return json.load(utf8(asset.stream()), **kw)