async def execute_run(self, project: Project, flow_id: str, context: Context):
env = await context.get('env', YouwolEnvironment)
await context.info(text="create 'cdn.zip' in project")
files = await self.packaged_files(project, flow_id, context)
zip_path = project.path / 'cdn.zip'
await create_cdn_zip(zip_path=zip_path, project=project, flow_id=flow_id, files=files, context=context)
local_gtw = LocalClients.get_assets_gateway_client(env=env)
local_treedb = LocalClients.get_treedb_client(env=env)
asset_id = encode_id(project.id)
try:
item = await local_treedb.get_item(item_id=asset_id)
folder_id = item['folderId']
except HTTPException as e:
if e.status_code == 404:
await context.info("The package has not been published yet, start creation")
drive: DefaultDriveResponse = await env.get_default_drive(context=context)
folder_id = drive.downloadFolderId
else:
raise e
data = {'file': zip_path.read_bytes(), 'content_encoding': 'identity'}
resp = await local_gtw.put_asset_with_raw(kind='package', folder_id=folder_id, data=data, timeout=600)
await context.info(text="Asset posted in assets_gtw", data=resp)
local_cdn = LocalClients.get_cdn_client(env=env)
resp = await local_cdn.get_package(library_name=project.name, version=project.version, metadata=True)
await context.info(text="Package retrieved from local cdn", data=resp)
resp['src_files_fingerprint'] = files_check_sum(files)
base_path = env.pathsBook.artifacts_flow(project_name=project.name, flow_id=flow_id)
resp['src_base_path'] = str(base_path)
resp['src_files'] = [str(f.relative_to(base_path)) for f in files]
return resp
async def get_status(self, project: Project, flow_id: str, last_manifest: Optional[Manifest], context: Context) \
-> PipelineStepStatus:
if last_manifest and not last_manifest.succeeded:
return PipelineStepStatus.KO
env = await context.get('env', YouwolEnvironment)
local_cdn = LocalClients.get_cdn_client(env=env)
remote_gtw = await RemoteClients.get_assets_gateway_client(context=context)
local_info, remote_info = await asyncio.gather(
local_cdn.get_package(library_name=project.name, version=project.version, metadata=True),
remote_gtw.cdn_get_package(library_name=project.name, version=project.version, metadata=True),
return_exceptions=True
)
if isinstance(remote_info, HTTPException) and remote_info.status_code == 404:
return PipelineStepStatus.none
if isinstance(local_info, HTTPException) and local_info.status_code == 404:
return PipelineStepStatus.outdated
if isinstance(remote_info, Exception) or isinstance(local_info, Exception):
raise remote_info if isinstance(remote_info, Exception) else local_info
local_info = cast(Mapping, local_info)
remote_info = cast(Mapping, remote_info)
local_fp, remote_fp = local_info['fingerprint'], remote_info['fingerprint']
if local_fp == remote_fp:
return PipelineStepStatus.OK
return PipelineStepStatus.outdated
async def execute_run(self, project: Project, flow_id: str, context: Context):
env = await context.get('env', YouwolEnvironment)
await upload_asset(body={'assetId': encode_id(project.id)}, context=context)
# # No ideal solution to get back the fingerprint here:
# # (i) this one is brittle if the source code of the CDN is not the same between local vs remote
local_cdn = LocalClients.get_cdn_client(env=env)
resp = await local_cdn.get_package(library_name=project.name, version=project.version, metadata=True)
# # (ii) this one is brittle in terms of eventual consistency
# # resp = await remote_gtw.cdn_get_package(library_name=project.name, version=project.version, metadata=True)
return resp
async def is_local_up_to_date(self):
env = await self.context.get('env', YouwolEnvironment)
local_assets: AssetsClient = LocalClients.get_assets_client(env=env)
try:
await local_assets.get(asset_id=self.asset_id)
return True
except HTTPException as e:
if e.status_code == 404:
return False
else:
raise e
async def is_local_up_to_date(self):
env = await self.context.get('env', YouwolEnvironment)
local_cdn: CdnClient = LocalClients.get_cdn_client(env=env)
try:
await local_cdn.get_package(library_name=self.package_name,
version=self.version,
metadata=True)
return True
except HTTPException as e:
if e.status_code == 404:
return False
raise e
async def is_local_up_to_date(self):
env = await self.context.get('env', YouwolEnvironment)
local_flux: FluxClient = LocalClients.get_flux_client(env=env)
try:
await local_flux.get_project(project_id=self.raw_id)
return True
except HTTPException as e:
if e.status_code == 404:
return False
else:
raise e
async def get_status(self, project: Project, flow_id: str, last_manifest: Optional[Manifest], context: Context) \
-> PipelineStepStatus:
env = await context.get('env', YouwolEnvironment)
local_cdn = LocalClients.get_cdn_client(env=env)
if not last_manifest:
return PipelineStepStatus.none
try:
local_info = await local_cdn.get_package(library_name=project.name, version=project.version, metadata=True)
except HTTPException as e:
if e.status_code == 404:
return PipelineStepStatus.none
raise e
files = await self.packaged_files(project, flow_id, context)
src_files_fingerprint = files_check_sum(files)
if last_manifest.fingerprint == local_info['fingerprint'] and \
last_manifest.cmdOutputs['src_files_fingerprint'] == src_files_fingerprint:
return PipelineStepStatus.OK
if last_manifest.fingerprint != local_info['fingerprint']:
await context.info(
text="Mismatch between cdn-backend fingerprint and saved manifest's fingerprint",
data={
'cdn-backend fingerprint': local_info['fingerprint'],
"saved manifest's fingerprint": last_manifest.fingerprint
}
)
if last_manifest.cmdOutputs['src_files_fingerprint'] != src_files_fingerprint:
await context.info(
text="Mismatch between actual src files fingerprint and saved manifest's src_files_fingerprint",
data={
'actual src files fingerprint': src_files_fingerprint,
"saved manifest's src_files_fingerprint": last_manifest.cmdOutputs['src_files_fingerprint']
}
)
return PipelineStepStatus.outdated
async def create_asset_local(asset_id: str, kind: str,
default_owning_folder_id,
get_raw_data: Callable[[], Awaitable[T]],
to_post_raw_data: Callable[[T], any],
context: Context):
env = await context.get("env", YouwolEnvironment)
async with context.start(
action=f"Fetch asset {asset_id} of kind {kind}", ) as ctx:
local_treedb: TreeDbClient = LocalClients.get_treedb_client(env)
local_gtw: AssetsGatewayClient = LocalClients.get_assets_gateway_client(
env)
remote_gtw = await RemoteClients.get_assets_gateway_client(context)
remote_treedb = await RemoteClients.get_treedb_client(context)
raw_data, metadata, tree_items = await asyncio.gather(
get_raw_data(),
remote_gtw.get_asset_metadata(asset_id=asset_id),
remote_gtw.get_tree_items_by_related_id(related_id=asset_id),
return_exceptions=True)
if isinstance(raw_data, Exception):
await ctx.error(f"Can not fetch raw part of the asset")
raise raw_data
if isinstance(metadata, Exception):
await ctx.error(f"Can not fetch asset's metadata")
raise metadata
if isinstance(tree_items, Exception):
await ctx.error(f"Can not fetch tree-db items")
raise tree_items
raw_data = cast(Dict, raw_data)
metadata = cast(Dict, metadata)
tree_items = cast(Dict, tree_items)
await ctx.info(text="Raw & meta data retrieved",
data={
"metadata": metadata,
"tree_items": tree_items,
})
owning_location, borrowed_locations = await get_remote_paths(
remote_treedb=remote_treedb,
tree_items=ItemsResponse(**tree_items))
await ctx.info(text="Explorer paths retrieved",
data={
"owning_location":
owning_location.dict() if owning_location else
"No owning location in available groups",
"borrowed_locations":
[p.dict() for p in borrowed_locations]
})
owning_folder_id = await get_local_owning_folder_id(
owning_location=owning_location,
local_treedb=local_treedb,
default_folder_id=default_owning_folder_id)
await ctx.info(text="Owning folder retrieved",
data={"owning_folder_id": owning_folder_id})
await local_gtw.put_asset_with_raw(kind=kind,
folder_id=owning_folder_id,
data=to_post_raw_data(raw_data))
await ctx.info(text="Asset raw's data downloaded successfully")
await sync_borrowed_items(asset_id=asset_id,
borrowed_locations=borrowed_locations,
local_treedb=local_treedb,
local_gtw=local_gtw)
await ctx.info(text="Borrowed items created successfully")
# the next line is not fetching images
await local_gtw.update_asset(asset_id=asset_id, body=metadata)
await ctx.info(text="Asset metadata uploaded successfully")