async def upload_data(subpath: str, data):
path_to = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
subpath)
path_from = os.path.join(settings.AGENT_CONFIG.artifacts_root, subpath)
check_or_create_path(path_from, is_dir=False)
async with aiofiles.open(path_from, "w") as filepath_upload:
await filepath_upload.write(data)
manager.upload_file_or_dir(
connection_type=settings.AGENT_CONFIG.artifacts_store,
path_from=path_from,
path_to=path_to,
is_file=True,
)
async def upload_dir(subpath: str,
path_from: str,
workers: int = 0,
last_time: datetime = None):
path_to = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
subpath)
manager.upload_file_or_dir(
connection_type=settings.AGENT_CONFIG.artifacts_store,
path_from=path_from,
path_to=path_to,
is_file=False,
workers=workers,
last_time=last_time,
)
def sync_artifacts(last_check: Optional[datetime], run_uuid: str):
new_check = now()
connection_type = get_artifacts_connection()
path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid)
# check if there's a path to sync
if os.path.exists(path_from):
path_to = os.path.join(connection_type.store_path, run_uuid)
upload_file_or_dir(
path_from=path_from,
path_to=path_to,
is_file=False,
workers=5,
last_time=last_check,
connection_type=connection_type,
)
return new_check
async def upload_file(subpath: str):
path_from = get_path(settings.AGENT_CONFIG.artifacts_root, subpath)
path_to = get_path(settings.AGENT_CONFIG.artifacts_store.store_path,
subpath)
try:
return manager.upload_file_or_dir(
connection_type=settings.AGENT_CONFIG.artifacts_store,
path_from=path_from,
path_to=path_to,
is_file=True,
)
except (OSError, PolyaxonException) as e:
logger.warning("Could not upload %s. Error %s" % (path_from, e))
return None
def sync_artifacts(last_check: Optional[datetime], run_uuid: str):
connection_type = get_artifacts_connection()
path_from = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_uuid)
new_check = path_last_modified(path_from)
# check if there's a path to sync
if os.path.exists(path_from):
path_to = os.path.join(connection_type.store_path, run_uuid)
upload_file_or_dir(
path_from=path_from,
path_to=path_to,
is_file=False,
workers=5,
last_time=last_check,
connection_type=connection_type,
exclude=["plxlogs"],
)
# Check if this run has trigger some related run paths
if os.path.exists(CONTEXT_MOUNT_ARTIFACTS_RELATED):
for sub_path in os.listdir(CONTEXT_MOUNT_ARTIFACTS_RELATED):
# check if there's a path to sync
path_from = CONTEXT_MOUNT_ARTIFACTS_RELATED_FORMAT.format(sub_path)
if os.path.exists(path_from):
path_to = os.path.join(connection_type.store_path, sub_path)
upload_file_or_dir(
path_from=path_from,
path_to=path_to,
is_file=False,
workers=5,
last_time=last_check,
connection_type=connection_type,
)
return new_check
