示例#1
0
def make_target(file_path: str,
                unique_id: Optional[str] = None,
                processor: Optional[FileProcessor] = None) -> TargetOnKart:
    file_path = _make_file_path(file_path, unique_id)
    processor = processor or make_file_processor(file_path)
    file_system_target = _make_file_system_target(file_path,
                                                  processor=processor)
    return SingleFileTarget(target=file_system_target, processor=processor)
示例#2
0
def _make_file_system_target(file_path: str) -> luigi.target.FileSystemTarget:
    processor = make_file_processor(file_path)
    if file_path.startswith('s3://'):
        return luigi.contrib.s3.S3Target(
            file_path,
            client=WorkspaceConfig().get_s3_client(),
            format=processor.format())
    return luigi.LocalTarget(file_path, format=processor.format())
示例#3
0
def _make_file_system_target(
    file_path: str,
    processor: Optional[FileProcessor] = None
) -> luigi.target.FileSystemTarget:
    processor = processor or make_file_processor(file_path)
    if ObjectStorage.if_object_storage_path(file_path):
        return ObjectStorage.get_object_storage_target(file_path,
                                                       processor.format())
    return luigi.LocalTarget(file_path, format=processor.format())
示例#4
0
def _make_file_system_target(
    file_path: str,
    processor: Optional[FileProcessor] = None
) -> luigi.target.FileSystemTarget:
    processor = processor or make_file_processor(file_path)
    if file_path.startswith('s3://'):
        return luigi.contrib.s3.S3Target(file_path,
                                         client=S3Config().get_s3_client(),
                                         format=processor.format())
    return luigi.LocalTarget(file_path, format=processor.format())
示例#5
0
文件: target.py 项目: Hi-king/gokart
def make_target(file_path: str,
                unique_id: Optional[str] = None,
                processor: Optional[FileProcessor] = None,
                redis_params: RedisParams = None,
                store_index_in_feather: bool = True) -> TargetOnKart:
    _redis_params = redis_params if redis_params is not None else make_redis_params(file_path=file_path, unique_id=unique_id)
    file_path = _make_file_path(file_path, unique_id)
    processor = processor or make_file_processor(file_path, store_index_in_feather=store_index_in_feather)
    file_system_target = _make_file_system_target(file_path, processor=processor, store_index_in_feather=store_index_in_feather)
    return SingleFileTarget(target=file_system_target, processor=processor, redis_params=_redis_params)