def __call__(self):
url = self.parameters['url']
dep_prefix = 'dependency://'
if url.startswith(dep_prefix):
dependency = url[len(dep_prefix):].strip()
url = get_dependency_datapackage_url(dependency)
assert url is not None, "Failed to fetch output datapackage for dependency '%s'" % dependency
resource = self.parameters['resource']
stream = self.parameters.get('stream', True)
name_matcher = ResourceMatcher(resource) if isinstance(resource,
str) else None
resource_index = resource if isinstance(resource, int) else None
selected_resources = []
found = False
dp = datapackage.DataPackage(url)
dp = self.process_datapackage(dp)
for i, orig_res in enumerate(dp.resources):
if resource_index == i or \
(name_matcher is not None and name_matcher.match(orig_res.descriptor.get('name'))):
found = True
orig_res.descriptor[PROP_STREAMED_FROM] = orig_res.source
self.dp['resources'].append(orig_res.descriptor)
if tabular(orig_res.descriptor) and stream:
orig_res.descriptor[PROP_STREAMING] = True
selected_resources.append(orig_res.iter(keyed=True))
else:
orig_res.descriptor[PROP_STREAMING] = False
assert found, "Failed to find resource with index or name matching %r" % resource
spew(self.dp, itertools.chain(self.res_iter, selected_resources))
def __call__(self):
url = self.parameters['url']
limit_rows = self.parameters.get('limit-rows')
dep_prefix = 'dependency://'
if url.startswith(dep_prefix):
dependency = url[len(dep_prefix):].strip()
url = get_dependency_datapackage_url(dependency)
assert url is not None, "Failed to fetch output datapackage for dependency '%s'" % dependency
resource = self.parameters['resource']
stream = self.parameters.get('stream', True)
name_matcher = ResourceMatcher(resource) if isinstance(resource,
str) else None
resource_index = resource if isinstance(resource, int) else None
selected_resources = []
found = False
dp = datapackage.DataPackage(url)
dp = self.process_datapackage(dp)
for i, orig_res in enumerate(dp.resources):
if resource_index == i or \
(name_matcher is not None and name_matcher.match(orig_res.descriptor.get('name'))):
found = True
desc = copy.deepcopy(orig_res.descriptor)
if 'primaryKey' in desc.get('schema', {}):
# Avoid duplication checks
del orig_res.descriptor['schema']['primaryKey']
orig_res.commit()
desc[PROP_STREAMED_FROM] = orig_res.source
self.dp['resources'].append(desc)
if tabular(desc) and stream:
desc[PROP_STREAMING] = True
orig_res_iter = orig_res.iter(keyed=True)
if limit_rows:
orig_res_iter = itertools.islice(
orig_res_iter, limit_rows)
selected_resources.append(orig_res_iter)
else:
desc[PROP_STREAMING] = False
assert found, "Failed to find resource with index or name matching %r" % resource
spew(self.dp, itertools.chain(self.res_iter, selected_resources))
def __call__(self):
self.parameters['resource'] = self.parameters['resource-name']
kv_cache = self.parameters.get('kv-cache', False)
kv_path = self.parameters['kv-path']
url = self.parameters['url']
limit_rows = self.parameters.get('limit-rows')
log_progress_rows = self.parameters.get('log-progress-rows')
dep_prefix = 'dependency://'
if url.startswith(dep_prefix):
dependency = url[len(dep_prefix):].strip()
url = get_dependency_datapackage_url(dependency)
assert url is not None, "Failed to fetch output datapackage for dependency '%s'" % dependency
stream = self.parameters.get('stream', True)
required = self.parameters.get('required', True)
resource = self.parameters.get('resource')
resources = self.parameters.get('resources')
if resource is not None:
assert not resources
resource_index = resource if isinstance(resource, int) else None
else:
assert resources
resource_index = None
resource = list(resources.keys())
name_matcher = ResourceMatcher(resource) if isinstance(resource, (str, list)) else None
selected_resources = []
found = False
try:
dp = datapackage.DataPackage(url)
except Exception:
if required:
raise
else:
dp = None
if dp:
dp = self.process_datapackage(dp)
for i, orig_res in enumerate(dp.resources):
if resource_index == i or \
(name_matcher is not None and name_matcher.match(orig_res.descriptor.get('name'))):
found = True
desc = copy.deepcopy(orig_res.descriptor)
if 'primaryKey' in desc.get('schema', {}):
# Avoid duplication checks
del orig_res.descriptor['schema']['primaryKey']
orig_res.commit()
desc[PROP_STREAMED_FROM] = orig_res.source
if resources:
desc.update(resources[desc['name']])
self.dp['resources'].append(desc)
if tabular(desc) and stream:
desc[PROP_STREAMING] = True
if kv_cache and os.path.exists(kv_path):
kv = PersistentKVFile(kv_path, concurrent=True)
orig_res_iter = kv_res_iter(kv, kv_key=self.parameters.get('kv-key'))
else:
kv = PersistentKVFile(kv_path, concurrent=True)
orig_res_iter = kv_res_iter(kv, orig_res.iter(keyed=True), kv_key=self.parameters.get('kv-key'))
if limit_rows:
orig_res_iter = itertools.islice(orig_res_iter, limit_rows)
if log_progress_rows:
orig_res_iter = progress_logger(orig_res_iter, log_progress_rows)
selected_resources.append(orig_res_iter)
else:
desc[PROP_STREAMING] = False
assert found or not required, "Failed to find resource with index or name matching %r" % resource
spew(self.dp, itertools.chain(self.res_iter, selected_resources))
import itertools
import datapackage
from datapackage_pipelines.wrapper import ingest, spew
from datapackage_pipelines.utilities.resource_matcher import ResourceMatcher
from datapackage_pipelines.utilities.resources import tabular, PROP_STREAMING
parameters, dp, res_iter = ingest()
url = parameters['url']
resource = parameters['resource']
name_matcher = ResourceMatcher(resource) if isinstance(resource, str) else None
resource_index = resource if isinstance(resource, int) else None
selected_resources = []
found = False
datapackage = datapackage.DataPackage(url)
for i, orig_res in enumerate(datapackage.resources):
if resource_index == i or \
(name_matcher is not None and name_matcher.match(orig_res.descriptor.get('name'))):
found = True
dp['resources'].append(orig_res.descriptor)
if tabular(orig_res.descriptor):
orig_res.descriptor[PROP_STREAMING] = True
selected_resources.append(orig_res.iter(keyed=True))
assert found, "Failed to find resource with index or name matching %r" % resource
spew(dp, itertools.chain(res_iter, selected_resources))