async def create_borrowed_items(asset_id: str, tree_id: str,
assets_gtw_client: AssetsGatewayClient,
context: Context):
env = await context.get('env', YouwolEnvironment)
async with context.start(action="create_borrowed_items",
with_attributes={
'assetId': asset_id,
'treeId': tree_id
}) as ctx:
items_treedb = parse_json(env.pathsBook.local_treedb_docdb)
tree_items = [
item for item in items_treedb['documents']
if item['related_id'] == asset_id
]
borrowed_items = [
item for item in tree_items
if json.loads(item['metadata'])['borrowed']
]
await asyncio.gather(*[
create_borrowed_item(item=item,
borrowed_tree_id=tree_id,
assets_gtw_client=assets_gtw_client,
context=ctx) for item in borrowed_items
])
async def get_status(
project: Project,
flow_id: str,
step: PipelineStep,
context: Context
) -> PipelineStepStatusResponse:
env = await context.get('env', YouwolEnvironment)
paths: PathsBook = env.pathsBook
async with context.start(
action="get status",
with_attributes={'projectId': project.id, 'flowId': flow_id, 'stepId': step.id}
) as _ctx:
path = paths.artifacts_step(project_name=project.name, flow_id=flow_id, step_id=step.id)
manifest = Manifest(**parse_json(path / 'manifest.json')) if (path / 'manifest.json').exists() else None
status = await step.get_status(project=project, flow_id=flow_id, last_manifest=manifest, context=context)
artifacts = [format_artifact_response(project=project, flow_id=flow_id, step=step, artifact=artifact,
env=env)
for artifact in step.artifacts]
return PipelineStepStatusResponse(
projectId=project.id,
flowId=flow_id,
stepId=step.id,
manifest=manifest,
artifactFolder=path,
artifacts=artifacts,
status=status
)
async def create_borrowed_item(borrowed_tree_id: str, item: Mapping[str, any],
assets_gtw_client: AssetsGatewayClient,
context: Context):
async with context.start(action="create_borrowed_items",
with_attributes={
'borrowed_tree_id': borrowed_tree_id,
'tree_id': item["item_id"]
}) as ctx:
tree_id = item["item_id"]
try:
await assets_gtw_client.get_tree_item(item_id=tree_id)
return
except HTTPException as e:
if e.status_code != 404:
raise e
path_item = await local_path({"treeId": tree_id}, context=ctx)
await ctx.info(labels=[Label.RUNNING],
text="Borrowed tree item not found, start creation",
data={"treeItemPath": to_json(path_item)})
await ensure_path(path_item, assets_gtw_client)
parent_id = path_item.drive.driveId
if len(path_item.folders) > 0:
parent_id = path_item.folders[0].folderId
await assets_gtw_client.borrow_tree_item(tree_id=borrowed_tree_id,
body={
"itemId":
tree_id,
"destinationFolderId":
parent_id
})
await ctx.info(text="Borrowed item created")
async def create_artifact(
project: Project,
flow_id: str,
step: PipelineStep,
artifact: Artifact,
fingerprint: str,
context: Context
):
env = await context.get('env', YouwolEnvironment)
async with context.start(
action=f"create artifact '{artifact.id}'",
with_attributes={'projectId': project.id, 'flowId': flow_id, 'stepId': step.id, 'artifactId': artifact.id,
'src fingerprint': fingerprint}
) as ctx:
files = matching_files(
folder=project.path,
patterns=artifact.files
)
paths: PathsBook = env.pathsBook
await ctx.info(text='got files listing', data=[str(f) for f in files])
destination_folder = paths.artifact(project_name=project.name, flow_id=flow_id, step_id=step.id,
artifact_id=artifact.id)
for f in files:
destination_path = destination_folder / f.relative_to(project.path)
os.makedirs(os.path.dirname(destination_path), exist_ok=True)
shutil.copy(src=f, dst=destination_path)
await context.info(text="Zip file created", data={'path': str(destination_path)})
async def download_package(package_name: str, version: str, context: Context):
async with context.start(
action=f"download package {package_name}#{version}",
with_labels=[str(Label.PACKAGE_DOWNLOADING)],
with_attributes={
'packageName': package_name,
'packageVersion': version,
}) as ctx:
env = await context.get('env', YouwolEnvironment)
remote_gtw = await RemoteClients.get_assets_gateway_client(context=ctx)
default_drive = await env.get_default_drive(context=ctx)
asset_id = encode_id(encode_id(package_name))
await ctx.info(text=f"asset_id: {asset_id} queued for download")
await create_asset_local(
asset_id=asset_id,
kind='package',
default_owning_folder_id=default_drive.systemPackagesFolderId,
get_raw_data=lambda: remote_gtw.cdn_get_package(
library_name=package_name, version=version),
to_post_raw_data=lambda pack: {'file': pack},
context=ctx)
db = parse_json(env.pathsBook.local_cdn_docdb)
record = [
d for d in db['documents']
if d['library_id'] == f"{package_name}#{version}"
]
response = DownloadedPackageResponse(
packageName=package_name,
version=version,
fingerprint=record[0]['fingerprint'])
await ctx.send(response)
async def synchronize_metadata(asset_id: str,
assets_gtw_client: AssetsGatewayClient,
context: Context):
env = await context.get('env', YouwolEnvironment)
async with context.start(action="synchronize_metadata",
with_attributes={'asset_id': asset_id}) as ctx:
local_assets_gtw: AssetsGatewayClient = LocalClients.get_assets_gateway_client(
env=env)
local_metadata, remote_metadata = await asyncio.gather(
local_assets_gtw.get_asset_metadata(asset_id=asset_id),
assets_gtw_client.get_asset_metadata(asset_id=asset_id))
missing_images_urls = [
p for p in local_metadata['images']
if p not in remote_metadata['images']
]
full_urls = [
f"http://localhost:{env.http_port}{url}"
for url in missing_images_urls
]
filenames = [url.split('/')[-1] for url in full_urls]
await ctx.info(labels=[str(Label.RUNNING)],
text="Synchronise metadata",
data={
'local_metadata': local_metadata,
'remote_metadata': remote_metadata,
'missing images': full_urls
})
async def download_img(session: ClientSession, url: str):
async with await session.get(url=url) as resp:
if resp.status == 200:
return await resp.read()
async with ClientSession() as http_session:
images_data = await asyncio.gather(
*[download_img(http_session, url) for url in full_urls])
forms = []
for filename, value in zip(filenames, images_data):
form_data = FormData()
form_data.add_field(name='file', value=value, filename=filename)
forms.append(form_data)
await asyncio.gather(
assets_gtw_client.update_asset(asset_id=asset_id,
body=local_metadata),
*[
assets_gtw_client.post_asset_image(asset_id=asset_id,
filename=name,
data=form)
for name, form in zip(filenames, forms)
])
async def execute_run(self, project: Project, flow_id: FlowId, context: Context):
outputs = []
await context.info(text="")
async with context.start(
action="deploy helm chart",
with_attributes={
"namespace": self.namespace
}) as ctx:
k8s_info = await get_cluster_info()
if not k8s_info:
outputs.append("Can not connect to k8s proxy")
raise CommandException(command="deploy helm chart", outputs=outputs)
with_values = self.overridingHelmValues(project, context)
chart_path = project.path / "chart"
helm_package = HelmPackage(
name=project.name,
namespace=self.namespace,
chart_folder=chart_path,
with_values=with_values,
values_filename='values.yaml',
secrets=self.secrets,
chart_explorer=get_chart_explorer(chart_path)
)
await ctx.send(data=helm_package)
installed = await helm_package.is_installed(context=ctx)
if installed and '-next' in project.version:
outputs.append("Mutable version used, proceed to chart uninstall")
await helm_package.uninstall(context=ctx)
installed = False
if installed:
outputs.append(f"Helm chart already installed, proceed to chart upgrade")
await ctx.info(text=f"Start helm chart install")
return_code, cmd, outputs_bash = await helm_package.upgrade(context=ctx)
outputs = outputs + [cmd] + outputs_bash
if return_code > 0:
raise CommandException(command="deploy helm chart", outputs=outputs)
if not installed:
outputs.append(f"Helm chart not already installed, start helm chart install")
await ctx.info(text=f"Start helm chart install")
return_code, cmd, outputs_bash = await helm_package.install(context=ctx)
outputs = outputs + [cmd] + outputs_bash
if return_code > 0:
raise CommandException(command="deploy helm chart", outputs=outputs)
return outputs
async def install(self, context: Context):
async with context.start(action='install helm package') as ctx:
await self.before_cmd(context=ctx)
keys = HelmPackage.flatten_schema_values(self.with_values)
args = functools.reduce(lambda acc, e: acc + f"--set {e[1:]} ", keys, "")
return await helm_install(
release_name=self.name,
namespace=self.namespace,
values_file=self.chart_folder / self.values_filename,
chart_folder=Path(self.chart_folder),
timeout=240,
args=args,
context=ctx)
async def check_update(local_package: TargetPackage, context: Context):
remote_gtw_client = await RemoteClients.get_assets_gateway_client(
context=context)
headers = {"authorization": context.request.headers.get("authorization")}
async with context.start(
action=f"Check update for {local_package.library_name}",
with_attributes={
'event': 'check_update_pending',
'packageName': local_package.library_name,
'packageVersion': local_package.version,
}) as ctx:
package_id = encode_id(local_package.library_name)
try:
remote_package = await remote_gtw_client.cdn_get_versions(
package_id=package_id, headers=headers)
except HTTPException as e:
if e.status_code == 404:
await ctx.info(
text=f"{local_package.library_name} does not exist in remote"
)
raise e
await ctx.info(text=f"Retrieved remote info",
data={'remote_package': remote_package})
latest = remote_package['releases'][0]
status = UpdateStatus.mismatch
if latest['fingerprint'] == local_package.fingerprint:
status = UpdateStatus.upToDate
elif latest['version_number'] > local_package.version_number:
status = UpdateStatus.remoteAhead
elif latest['version_number'] < local_package.version_number:
status = UpdateStatus.localAhead
await ctx.info(text=f"Status: {str(status)}")
response = CheckUpdateResponse(
status=status,
packageName=local_package.library_name,
localVersionInfo=PackageVersionInfo(
version=local_package.version,
fingerprint=local_package.fingerprint),
remoteVersionInfo=PackageVersionInfo(
version=latest['version'], fingerprint=latest['fingerprint']))
await ctx.send(response)
return response
async def execute_shell_cmd(cmd: str, context: Context, log_outputs=True):
async with context.start(action="execute shell command",
with_labels=["BASH"]) as ctx:
await ctx.info(text=cmd)
p = await asyncio.create_subprocess_shell(
cmd=cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
shell=True)
outputs = []
async with stream.merge(p.stdout,
p.stderr).stream() as messages_stream:
async for message in messages_stream:
outputs.append(message.decode('utf-8'))
log_outputs and await context.info(text=outputs[-1])
await p.communicate()
return p.returncode, outputs
async def create_artifacts(
project: Project,
flow_id: str,
step: PipelineStep,
fingerprint: str,
context: Context
):
async with context.start(
action="create artifacts",
with_attributes={'projectId': project.id, 'flowId': flow_id, 'stepId': step.id,
'src fingerprint': fingerprint}
) as ctx:
for artifact in step.artifacts:
await create_artifact(
project=project,
flow_id=flow_id,
step=step,
artifact=artifact,
fingerprint=fingerprint,
context=ctx)
async def get_status(self, project: Project, flow_id: str,
last_manifest: Optional[Manifest],
context: Context) -> PipelineStepStatus:
async with context.start(
action="get status of project's dependencies") as ctx:
if last_manifest is None:
return PipelineStepStatus.none
if not last_manifest.succeeded:
return PipelineStepStatus.KO
await ctx.info(text='previous manifest',
data=to_json(last_manifest))
data = await SyncFromDownstreamStep.get_input_data(project=project,
flow_id=flow_id,
context=context)
prev_checksums = last_manifest.cmdOutputs['checksums']
ok = len(data.keys()) == len(prev_checksums.keys())\
and all(k in prev_checksums and prev_checksums[k] == v.checksum for k, v in data.items())
if not ok:
return PipelineStepStatus.outdated
# Any of the inner dependencies code in node_modules should be checked to make sure
# no 'external' tool (e.g doing 'yarn') changed the node_module files
prev_node_module_checksums = last_manifest.cmdOutputs.get(
'nodeModuleChecksums', {})
node_module_checksums: Mapping[str, str] = {
name:
SyncFromDownstreamStep.node_module_checksum(project=project,
name=name)
for name in data.keys()
}
if not all(node_module_checksums[name] ==
prev_node_module_checksums.get(name, "")
for name in data.keys()):
return PipelineStepStatus.outdated
return PipelineStepStatus.OK
async def execute_run(self, project: Project, flow_id: FlowId,
context: Context):
async with context.start(
action="run synchronization of workspace dependencies") as ctx:
data = await SyncFromDownstreamStep.get_input_data(project=project,
flow_id=flow_id,
context=ctx)
destination_folders: Mapping[str, Path] = {
name: project.path / 'node_modules' / name
for name in data.keys()
}
for name, p in data.items():
if destination_folders[name].exists():
shutil.rmtree(destination_folders[name])
copy_tree(p.dist_folder, destination_folders[name])
for file in p.src_files:
destination = destination_folders[
name] / 'src' / file.relative_to(p.src_folder)
copy_file(source=file,
destination=destination,
create_folders=True)
all_files = functools.reduce(
lambda acc, e: acc + e.src_files + e.dist_files, data.values(),
[])
return {
'fingerprint': files_check_sum(all_files),
'checksums': {name: d.checksum
for name, d in data.items()},
'nodeModuleChecksums': {
name: SyncFromDownstreamStep.node_module_checksum(
project=project, name=name)
for name in data.keys()
}
}
async def synchronize_permissions(assets_gtw_client: AssetsGatewayClient,
asset_id: str, context: Context):
async with context.start(action="synchronize_permissions",
with_attributes={'assetId': asset_id}) as ctx:
env = await context.get('env', YouwolEnvironment)
local_assets_gtw = LocalClients.get_assets_gateway_client(env=env)
access_info = await local_assets_gtw.get_asset_access(asset_id=asset_id
)
await ctx.info(labels=[str(Label.RUNNING)],
text="Permissions retrieved",
data={"access_info": access_info})
default_permission = access_info["ownerInfo"]["defaultAccess"]
groups = access_info["ownerInfo"]["exposingGroups"]
await asyncio.gather(
assets_gtw_client.put_asset_access(asset_id=asset_id,
group_id='*',
body=default_permission),
*[
assets_gtw_client.put_asset_access(asset_id=asset_id,
group_id=g['groupId'],
body=g['access'])
for g in groups
])
async def create_asset_local(asset_id: str, kind: str,
default_owning_folder_id,
get_raw_data: Callable[[], Awaitable[T]],
to_post_raw_data: Callable[[T], any],
context: Context):
env = await context.get("env", YouwolEnvironment)
async with context.start(
action=f"Fetch asset {asset_id} of kind {kind}", ) as ctx:
local_treedb: TreeDbClient = LocalClients.get_treedb_client(env)
local_gtw: AssetsGatewayClient = LocalClients.get_assets_gateway_client(
env)
remote_gtw = await RemoteClients.get_assets_gateway_client(context)
remote_treedb = await RemoteClients.get_treedb_client(context)
raw_data, metadata, tree_items = await asyncio.gather(
get_raw_data(),
remote_gtw.get_asset_metadata(asset_id=asset_id),
remote_gtw.get_tree_items_by_related_id(related_id=asset_id),
return_exceptions=True)
if isinstance(raw_data, Exception):
await ctx.error(f"Can not fetch raw part of the asset")
raise raw_data
if isinstance(metadata, Exception):
await ctx.error(f"Can not fetch asset's metadata")
raise metadata
if isinstance(tree_items, Exception):
await ctx.error(f"Can not fetch tree-db items")
raise tree_items
raw_data = cast(Dict, raw_data)
metadata = cast(Dict, metadata)
tree_items = cast(Dict, tree_items)
await ctx.info(text="Raw & meta data retrieved",
data={
"metadata": metadata,
"tree_items": tree_items,
})
owning_location, borrowed_locations = await get_remote_paths(
remote_treedb=remote_treedb,
tree_items=ItemsResponse(**tree_items))
await ctx.info(text="Explorer paths retrieved",
data={
"owning_location":
owning_location.dict() if owning_location else
"No owning location in available groups",
"borrowed_locations":
[p.dict() for p in borrowed_locations]
})
owning_folder_id = await get_local_owning_folder_id(
owning_location=owning_location,
local_treedb=local_treedb,
default_folder_id=default_owning_folder_id)
await ctx.info(text="Owning folder retrieved",
data={"owning_folder_id": owning_folder_id})
await local_gtw.put_asset_with_raw(kind=kind,
folder_id=owning_folder_id,
data=to_post_raw_data(raw_data))
await ctx.info(text="Asset raw's data downloaded successfully")
await sync_borrowed_items(asset_id=asset_id,
borrowed_locations=borrowed_locations,
local_treedb=local_treedb,
local_gtw=local_gtw)
await ctx.info(text="Borrowed items created successfully")
# the next line is not fetching images
await local_gtw.update_asset(asset_id=asset_id, body=metadata)
await ctx.info(text="Asset metadata uploaded successfully")
async def upload_asset(body: JSON, context: Context):
upload_factories: Dict[str, any] = {
"data": UploadDataTask,
"flux-project": UploadFluxProjectTask,
"story": UploadStoryTask,
"package": UploadPackageTask
}
asset_id = body['assetId']
async with context.start(action="upload_asset",
with_attributes={'asset_id': asset_id}) as ctx:
env = await context.get('env', YouwolEnvironment)
local_treedb: TreeDbClient = LocalClients.get_treedb_client(env=env)
local_assets: AssetsClient = LocalClients.get_assets_client(env=env)
raw_id = decode_id(asset_id)
asset, tree_item = await asyncio.gather(
local_assets.get(asset_id=asset_id),
local_treedb.get_item(item_id=asset_id),
return_exceptions=True)
if isinstance(asset, HTTPException) and asset.status_code == 404:
await ctx.error(
text="Can not find the asset in the local assets store")
raise RuntimeError(
"Can not find the asset in the local assets store")
if isinstance(tree_item,
HTTPException) and tree_item.status_code == 404:
await ctx.error(
text="Can not find the tree item in the local treedb store")
raise RuntimeError(
"Can not find the tree item in the local treedb store")
if isinstance(asset, Exception) or isinstance(tree_item, Exception):
raise RuntimeError(
"A problem occurred while fetching the local asset/tree items")
asset = cast(Dict, asset)
tree_item = cast(Dict, tree_item)
factory: UploadTask = upload_factories[asset['kind']](
raw_id=raw_id, asset_id=asset_id, context=ctx)
local_data = await factory.get_raw()
try:
path_item = await local_treedb.get_path(item_id=tree_item['itemId']
)
except HTTPException as e:
if e.status_code == 404:
await ctx.error(
text=
f"Can not get path of item with id '{tree_item['itemId']}'",
data={
"tree_item": tree_item,
"error_detail": e.detail
})
raise e
await ctx.info(text="Data retrieved",
data={
"path_item": path_item,
"raw data": local_data
})
assets_gtw_client = await RemoteClients.get_assets_gateway_client(
context=ctx)
await ensure_path(path_item=PathResponse(**path_item),
assets_gateway_client=assets_gtw_client)
try:
await assets_gtw_client.get_asset_metadata(asset_id=asset_id)
await ctx.info(text="Asset already found in deployed environment")
await factory.update_raw(data=local_data,
folder_id=tree_item['folderId'])
except HTTPException as e:
if e.status_code != 404:
raise e
await ctx.info(labels=[Label.RUNNING],
text="Project not already found => start creation")
await factory.create_raw(data=local_data,
folder_id=tree_item['folderId'])
await synchronize_permissions_metadata_symlinks(
asset_id=asset_id,
tree_id=tree_item['itemId'],
assets_gtw_client=assets_gtw_client,
context=ctx)
return {}
async def uninstall(self, context: Context):
async with context.start(action='uninstall helm package') as ctx:
await self.before_cmd(context=ctx)
return await helm_uninstall(release_name=self.name, namespace=self.namespace, context=ctx)