def _generate_md5(spec):
asset = AssetResolver(None).resolve(spec)
md5 = hashlib.md5()
with asset.stream() as stream:
for block in iter(lambda: stream.read(4096), b''):
md5.update(block)
return md5.hexdigest()
def _generate_md5(spec):
asset = AssetResolver(None).resolve(spec)
md5 = hashlib.md5()
with asset.stream() as stream:
for block in iter(lambda: stream.read(4096), b''):
md5.update(block)
return md5.hexdigest()
def _load_manifest(self):
manifest = getattr(self, "_manifest", None)
if manifest is None:
manifest = AssetResolver().resolve(self.manifest_asset)
with manifest.stream() as fp:
manifest = json.loads(fp.read().decode("utf8"))
if self.cache:
self._manifest = manifest
return manifest
def _load_manifest(self):
manifest = getattr(self, "_manifest", None)
if manifest is None:
manifest = AssetResolver().resolve(self.manifest_asset)
with manifest.stream() as fp:
manifest = json.loads(fp.read().decode("utf8"))
if self.cache:
self._manifest = manifest
return manifest
def load_schema(filename):
if isinstance(filename, dict):
schema = filename
resolver = NoRemoteResolver.from_schema(schema)
else:
utf8 = codecs.getreader("utf-8")
asset = AssetResolver(caller_package()).resolve(filename)
schema = json.load(utf8(asset.stream()),
object_pairs_hook=collections.OrderedDict)
resolver = RefResolver('file://' + asset.abspath(), schema)
schema = mixinProperties(schema, resolver)
# SchemaValidator is not thread safe for now
SchemaValidator(schema, resolver=resolver, serialize=True)
return schema
def load_schema(filename):
if isinstance(filename, dict):
schema = filename
resolver = NoRemoteResolver.from_schema(schema)
else:
utf8 = codecs.getreader("utf-8")
asset = AssetResolver(caller_package()).resolve(filename)
schema = json.load(utf8(asset.stream()),
object_pairs_hook=collections.OrderedDict)
resolver = RefResolverOrdered('file://' + asset.abspath(), schema)
# use mixinProperties, mixinFacets, mixinAggregations, and mixinColumns (if provided)
schema = mixinSchemas(
mixinSchemas(
mixinSchemas( mixinSchemas(schema, resolver, 'properties'), resolver, 'facets' ),
resolver,
'aggregations'
),
resolver, 'columns'
)
# SchemaValidator is not thread safe for now
SchemaValidator(schema, resolver=resolver, serialize=True)
return schema
def json_asset(spec, **kw):
utf8 = codecs.getreader("utf-8")
asset = AssetResolver(caller_package()).resolve(spec)
return json.load(utf8(asset.stream()), **kw)
def json_asset(spec, **kw):
utf8 = codecs.getreader("utf-8")
asset = AssetResolver(caller_package()).resolve(spec)
return json.load(utf8(asset.stream()), **kw)
