def _build_batch_kwargs_from_key(
self,
key,
asset_config=None,
reader_method=None,
reader_options=None,
limit=None,
):
batch_kwargs = {
"s3": "s3a://" + self.bucket + "/" + key,
"reader_options": self.reader_options,
}
if asset_config.get("reader_options"):
batch_kwargs["reader_options"].update(
asset_config.get("reader_options"))
if reader_options is not None:
batch_kwargs["reader_options"].update(reader_options)
if self._reader_method is not None:
batch_kwargs["reader_method"] = self._reader_method
if asset_config.get("reader_method"):
batch_kwargs["reader_method"] = asset_config.get("reader_method")
if reader_method is not None:
batch_kwargs["reader_method"] = reader_method
if limit:
batch_kwargs["limit"] = limit
return S3BatchKwargs(batch_kwargs)
def _build_batch_kwargs(self,
key,
asset_config=None,
reader_options=None,
limit=None):
batch_kwargs = {
"s3": "s3a://" + self.bucket + "/" + key,
'reader_options': self.reader_options
}
if asset_config.get("reader_options"):
batch_kwargs['reader_options'].update(
asset_config.get("reader_options"))
if reader_options is not None:
batch_kwargs['reader_options'].update(reader_options)
if self._reader_method is not None:
batch_kwargs["reader_method"] = ReaderMethods(self._reader_method)
if asset_config.get("reader_method"):
batch_kwargs['reader_method'] = ReaderMethods(
asset_config.get("reader_method"))
if limit:
batch_kwargs['limit'] = limit
return S3BatchKwargs(batch_kwargs)
def _build_batch_kwargs(self, key, asset_config=None):
batch_kwargs = {
"s3": "s3a://" + self.bucket + "/" + key,
}
batch_kwargs.update(self.reader_options)
if self._reader_method is not None:
batch_kwargs["reader_method"] = ReaderMethods(self._reader_method)
if asset_config.get("reader_options") is not None:
batch_kwargs.update(asset_config.get("reader_options"))
if asset_config.get("reader_method") is not None:
batch_kwargs.update(
ReaderMethods(asset_config.get("reader_method")))
return S3BatchKwargs(batch_kwargs)
def _build_batch_kwargs_from_path(
self, path, reader_method=None, reader_options=None, limit=None
):
batch_kwargs = self._datasource.process_batch_parameters(
reader_method=reader_method or self.reader_method,
reader_options=reader_options or self.reader_options,
limit=limit,
)
if "s3a://" not in path:
path = "s3a://" + path
batch_kwargs["s3"] = path
batch_kwargs["datasource"] = self._datasource.name
return S3BatchKwargs(batch_kwargs)