def get_local_package(asset_id: str, config: YouwolEnvironment) -> Library:
"""
Not populated with tree items
"""
data_packages = parse_json(config.pathsBook.local_docdb / "cdn" /
"libraries" / "data.json")
raw_id = decode_id(asset_id)
library_name = decode_id(raw_id)
releases = [
d for d in data_packages['documents']
if d['library_name'] == library_name
]
if not releases:
raise HTTPException(status_code=404,
detail=f'Local package {library_name} not found')
return Library(
libraryName=library_name,
namespace=releases[0]["namespace"],
releases=[
Release(version=r['version'],
fingerprint=r['fingerprint'] if 'fingerprint' in r else '')
for r in releases
],
assetId=asset_id,
rawId=raw_id,
treeItems=[],
)
async def cdn_status(request: Request,
project_id: str,
config: YouwolEnvironment = Depends(yw_config)):
context = ContextFactory.get_instance(
request=request, web_socket=WebSocketsStore.userChannel)
async with context.start(action="Get local cdn status",
with_attributes={
'event': 'CdnResponsePending',
'projectId': project_id
}) as ctx:
data = parse_json(config.pathsBook.local_cdn_docdb)['documents']
data = [d for d in data if d["library_name"] == decode_id(project_id)]
def format_version(doc):
storage_cdn_path = config.pathsBook.local_cdn_storage
folder_path = storage_cdn_path / doc['path']
bundle_path = folder_path / doc['bundle']
files_count = sum(
[len(files) for r, d, files in os.walk(folder_path)])
bundle_size = bundle_path.stat().st_size
return CdnVersionResponse(name=doc['library_name'],
version=doc['version'],
versionNumber=doc['version_number'],
filesCount=files_count,
bundleSize=bundle_size,
path=folder_path,
namespace=doc['namespace'])
response = CdnResponse(name=decode_id(project_id),
versions=[format_version(d) for d in data])
await ctx.send(response)
return response
def get_zip_path(asset_id: str, version, env: YouwolEnvironment):
library_name = decode_id(decode_id(asset_id))
base_path = env.pathsBook.local_storage / "cdn" / "youwol-users" / "libraries"
namespace = None if '/' not in library_name else library_name.split(
'/')[0][1:]
library_name = library_name if '/' not in library_name else library_name.split(
'/')[1]
library_path = base_path / library_name / version \
if not namespace \
else base_path / namespace / library_name / version
return library_name, library_path / "__original.zip"
async def process_download_asset(queue: asyncio.Queue, factories: Dict[str,
Any],
downloaded_ids: set[str],
logger: DownloadLogger):
while True:
url, context, headers = await queue.get()
raw_id = url.split('/api/assets-gateway/raw/')[1].split('/')[1]
asset_id = encode_id(raw_id)
remote_gtw_client = await RemoteClients.get_assets_gateway_client(
context=context)
try:
asset = await remote_gtw_client.get_asset_metadata(
asset_id=asset_id, headers=headers)
raw_id = decode_id(asset_id)
process_id = str(uuid.uuid4())
await logger.info(
process_id=process_id,
title=
f"Lookup for eventual download of asset {asset['kind']} of id {raw_id}",
url=url,
raw_id=raw_id)
task = factories[asset['kind']](process_id=process_id,
raw_id=raw_id,
asset_id=asset_id,
url=url,
context=context)
download_id = task.download_id()
if download_id not in downloaded_ids and not await task.is_local_up_to_date(
):
downloaded_ids.add(download_id)
await task.create_local_asset()
except YouWolException as e:
print(e)
queue.task_done()
print("remaining asset to download in the queue:", queue.qsize())
if queue.qsize() == 0:
logger.dumps()
async def get_raw(self):
env = await self.context.get('env', YouwolEnvironment)
local_package = get_local_package(asset_id=self.asset_id, config=env)
assets_gateway_client = await RemoteClients.get_assets_gateway_client(
context=self.context)
to_sync_releases = [v.version for v in local_package.releases]
try:
raw_metadata = await assets_gateway_client.get_raw_metadata(
kind='package', raw_id=decode_id(self.asset_id))
except HTTPException as e:
if e.status_code == 404:
return to_sync_releases
raise e
remote_versions = {
release['version']: release['fingerprint']
for release in raw_metadata['releases']
}
local_versions = {
release.version: release.fingerprint
for release in local_package.releases
}
missing = [
v for v in local_versions.keys() if v not in remote_versions
]
mismatch = [
v for v, checksum in local_versions.items()
if v in remote_versions and checksum != remote_versions[v]
]
to_sync_releases = missing + mismatch
await self.context.info(text="package's versions to sync. resolved",
data={
"missing": missing,
"mismatch": mismatch
})
return to_sync_releases
async def upload_asset(body: JSON, context: Context):
upload_factories: Dict[str, any] = {
"data": UploadDataTask,
"flux-project": UploadFluxProjectTask,
"story": UploadStoryTask,
"package": UploadPackageTask
}
asset_id = body['assetId']
async with context.start(action="upload_asset",
with_attributes={'asset_id': asset_id}) as ctx:
env = await context.get('env', YouwolEnvironment)
local_treedb: TreeDbClient = LocalClients.get_treedb_client(env=env)
local_assets: AssetsClient = LocalClients.get_assets_client(env=env)
raw_id = decode_id(asset_id)
asset, tree_item = await asyncio.gather(
local_assets.get(asset_id=asset_id),
local_treedb.get_item(item_id=asset_id),
return_exceptions=True)
if isinstance(asset, HTTPException) and asset.status_code == 404:
await ctx.error(
text="Can not find the asset in the local assets store")
raise RuntimeError(
"Can not find the asset in the local assets store")
if isinstance(tree_item,
HTTPException) and tree_item.status_code == 404:
await ctx.error(
text="Can not find the tree item in the local treedb store")
raise RuntimeError(
"Can not find the tree item in the local treedb store")
if isinstance(asset, Exception) or isinstance(tree_item, Exception):
raise RuntimeError(
"A problem occurred while fetching the local asset/tree items")
asset = cast(Dict, asset)
tree_item = cast(Dict, tree_item)
factory: UploadTask = upload_factories[asset['kind']](
raw_id=raw_id, asset_id=asset_id, context=ctx)
local_data = await factory.get_raw()
try:
path_item = await local_treedb.get_path(item_id=tree_item['itemId']
)
except HTTPException as e:
if e.status_code == 404:
await ctx.error(
text=
f"Can not get path of item with id '{tree_item['itemId']}'",
data={
"tree_item": tree_item,
"error_detail": e.detail
})
raise e
await ctx.info(text="Data retrieved",
data={
"path_item": path_item,
"raw data": local_data
})
assets_gtw_client = await RemoteClients.get_assets_gateway_client(
context=ctx)
await ensure_path(path_item=PathResponse(**path_item),
assets_gateway_client=assets_gtw_client)
try:
await assets_gtw_client.get_asset_metadata(asset_id=asset_id)
await ctx.info(text="Asset already found in deployed environment")
await factory.update_raw(data=local_data,
folder_id=tree_item['folderId'])
except HTTPException as e:
if e.status_code != 404:
raise e
await ctx.info(labels=[Label.RUNNING],
text="Project not already found => start creation")
await factory.create_raw(data=local_data,
folder_id=tree_item['folderId'])
await synchronize_permissions_metadata_symlinks(
asset_id=asset_id,
tree_id=tree_item['itemId'],
assets_gtw_client=assets_gtw_client,
context=ctx)
return {}
def __post_init__(self):
super()
self.version = self.url.split('/api/assets-gateway/raw/')[1].split(
'/')[2]
self.package_name = decode_id(self.raw_id)