def make_target(file_path: str,
unique_id: Optional[str] = None,
processor: Optional[FileProcessor] = None) -> TargetOnKart:
file_path = _make_file_path(file_path, unique_id)
processor = processor or make_file_processor(file_path)
file_system_target = _make_file_system_target(file_path,
processor=processor)
return SingleFileTarget(target=file_system_target, processor=processor)
def _make_file_system_target(file_path: str) -> luigi.target.FileSystemTarget:
processor = make_file_processor(file_path)
if file_path.startswith('s3://'):
return luigi.contrib.s3.S3Target(
file_path,
client=WorkspaceConfig().get_s3_client(),
format=processor.format())
return luigi.LocalTarget(file_path, format=processor.format())
def _make_file_system_target(
file_path: str,
processor: Optional[FileProcessor] = None
) -> luigi.target.FileSystemTarget:
processor = processor or make_file_processor(file_path)
if ObjectStorage.if_object_storage_path(file_path):
return ObjectStorage.get_object_storage_target(file_path,
processor.format())
return luigi.LocalTarget(file_path, format=processor.format())
def _make_file_system_target(
file_path: str,
processor: Optional[FileProcessor] = None
) -> luigi.target.FileSystemTarget:
processor = processor or make_file_processor(file_path)
if file_path.startswith('s3://'):
return luigi.contrib.s3.S3Target(file_path,
client=S3Config().get_s3_client(),
format=processor.format())
return luigi.LocalTarget(file_path, format=processor.format())
def make_target(file_path: str,
unique_id: Optional[str] = None,
processor: Optional[FileProcessor] = None,
redis_params: RedisParams = None,
store_index_in_feather: bool = True) -> TargetOnKart:
_redis_params = redis_params if redis_params is not None else make_redis_params(file_path=file_path, unique_id=unique_id)
file_path = _make_file_path(file_path, unique_id)
processor = processor or make_file_processor(file_path, store_index_in_feather=store_index_in_feather)
file_system_target = _make_file_system_target(file_path, processor=processor, store_index_in_feather=store_index_in_feather)
return SingleFileTarget(target=file_system_target, processor=processor, redis_params=_redis_params)
