def _make_file_system_target(
file_path: str,
processor: Optional[FileProcessor] = None
) -> luigi.target.FileSystemTarget:
processor = processor or make_file_processor(file_path)
if ObjectStorage.if_object_storage_path(file_path):
return ObjectStorage.get_object_storage_target(file_path,
processor.format())
return luigi.LocalTarget(file_path, format=processor.format())
def _try_to_delete_unnecessary_output_file(cmdline_args: List[str]):
with CmdlineParser.global_instance(cmdline_args) as cp:
task = cp.get_task_obj() # type: gokart.TaskOnKart
if task.delete_unnecessary_output_files:
if ObjectStorage.if_object_storage_path(task.workspace_directory):
logger.info('delete-unnecessary-output-files is not support s3/gcs.')
else:
gokart.delete_local_unnecessary_outputs(task)
exit()
def make_zip_client(file_path: str, temporary_directory: str) -> ZipClient:
if ObjectStorage.if_object_storage_path(file_path):
return ObjectStorage.get_zip_client(
file_path=file_path, temporary_directory=temporary_directory)
return LocalZipClient(file_path=file_path,
temporary_directory=temporary_directory)
def _get_last_modification_time(path: str) -> datetime:
if ObjectStorage.if_object_storage_path(path):
if ObjectStorage.exists(path):
return ObjectStorage.get_timestamp(path)
raise FileNotFoundError(f'No such file or directory: {path}')
return datetime.fromtimestamp(os.path.getmtime(path))
def load(self, file):
if not ObjectStorage.is_buffered_reader(file):
return pickle.loads(file.read())
return pickle.load(_ChunkedLargeFileReader(file))
def load(self, file):
if ObjectStorage.is_readable_objectstorage_instance(file):
return pickle.loads(file.read())
return pickle.load(_LargeLocalFileReader(file))