async def upload_data(subpath: str, data): path_to = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) path_from = os.path.join(settings.AGENT_CONFIG.artifacts_root, subpath) check_or_create_path(path_from, is_dir=False) async with aiofiles.open(path_from, "w") as filepath_upload: await filepath_upload.write(data) manager.upload_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=True, )
async def upload_dir(subpath: str, path_from: str, workers: int = 0, last_time: datetime = None): path_to = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) manager.upload_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=False, workers=workers, last_time=last_time, )
def sync_artifacts(last_check: Optional[datetime], run_uuid: str): new_check = now() connection_type = get_artifacts_connection() path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid) # check if there's a path to sync if os.path.exists(path_from): path_to = os.path.join(connection_type.store_path, run_uuid) upload_file_or_dir( path_from=path_from, path_to=path_to, is_file=False, workers=5, last_time=last_check, connection_type=connection_type, ) return new_check
async def upload_file(subpath: str): path_from = get_path(settings.AGENT_CONFIG.artifacts_root, subpath) path_to = get_path(settings.AGENT_CONFIG.artifacts_store.store_path, subpath) try: return manager.upload_file_or_dir( connection_type=settings.AGENT_CONFIG.artifacts_store, path_from=path_from, path_to=path_to, is_file=True, ) except (OSError, PolyaxonException) as e: logger.warning("Could not upload %s. Error %s" % (path_from, e)) return None
def sync_artifacts(last_check: Optional[datetime], run_uuid: str): connection_type = get_artifacts_connection() path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid) new_check = path_last_modified(path_from) # check if there's a path to sync if os.path.exists(path_from): path_to = os.path.join(connection_type.store_path, run_uuid) upload_file_or_dir( path_from=path_from, path_to=path_to, is_file=False, workers=5, last_time=last_check, connection_type=connection_type, exclude=["plxlogs"], ) # Check if this run has trigger some related run paths if os.path.exists(CONTEXT_MOUNT_ARTIFACTS_RELATED): for sub_path in os.listdir(CONTEXT_MOUNT_ARTIFACTS_RELATED): # check if there's a path to sync path_from = CONTEXT_MOUNT_ARTIFACTS_RELATED_FORMAT.format(sub_path) if os.path.exists(path_from): path_to = os.path.join(connection_type.store_path, sub_path) upload_file_or_dir( path_from=path_from, path_to=path_to, is_file=False, workers=5, last_time=last_check, connection_type=connection_type, ) return new_check