async def execute_run(self, project: Project, flow_id: str, context: Context):
env = await context.get('env', YouwolEnvironment)
await context.info(text="create 'cdn.zip' in project")
files = await self.packaged_files(project, flow_id, context)
zip_path = project.path / 'cdn.zip'
await create_cdn_zip(zip_path=zip_path, project=project, flow_id=flow_id, files=files, context=context)
local_gtw = LocalClients.get_assets_gateway_client(env=env)
local_treedb = LocalClients.get_treedb_client(env=env)
asset_id = encode_id(project.id)
try:
item = await local_treedb.get_item(item_id=asset_id)
folder_id = item['folderId']
except HTTPException as e:
if e.status_code == 404:
await context.info("The package has not been published yet, start creation")
drive: DefaultDriveResponse = await env.get_default_drive(context=context)
folder_id = drive.downloadFolderId
else:
raise e
data = {'file': zip_path.read_bytes(), 'content_encoding': 'identity'}
resp = await local_gtw.put_asset_with_raw(kind='package', folder_id=folder_id, data=data, timeout=600)
await context.info(text="Asset posted in assets_gtw", data=resp)
local_cdn = LocalClients.get_cdn_client(env=env)
resp = await local_cdn.get_package(library_name=project.name, version=project.version, metadata=True)
await context.info(text="Package retrieved from local cdn", data=resp)
resp['src_files_fingerprint'] = files_check_sum(files)
base_path = env.pathsBook.artifacts_flow(project_name=project.name, flow_id=flow_id)
resp['src_base_path'] = str(base_path)
resp['src_files'] = [str(f.relative_to(base_path)) for f in files]
return resp
def md5_from_folder(dir_path: Path):
paths = []
for subdir, dirs, files in os.walk(dir_path):
for filename in files:
paths.append(Path(subdir) / filename)
md5_stamp = files_check_sum(paths)
return md5_stamp
async def get_fingerprint(self, project: 'Project', flow_id: FlowId, context: Context):
files = await self.get_sources(project=project, flow_id=flow_id, context=context)
if files is None:
return None, []
await context.info(text='got file listing', data={"files": [str(f) for f in files]})
checksum = files_check_sum(files)
return checksum, files
async def get_input_data(
project: Project, flow_id: str,
context: Context) -> Mapping[str, InputDataDependency]:
env = await context.get('env', YouwolEnvironment)
paths_book: PathsBook = env.pathsBook
project_step = [
(d,
next((s for s in d.get_flow_steps(flow_id=flow_id)
if isinstance(s, BuildStep)), None))
for d in await project.get_dependencies(
recursive=True,
projects=await ProjectLoader.get_projects(env, context),
context=context)
]
def is_succeeded(p: Project, s: BuildStep):
manifest = p.get_manifest(flow_id=flow_id, step=s, env=env)
return manifest.succeeded if manifest else False
dependencies = [(project, step) for project, step in project_step
if step is not None and is_succeeded(project, step)]
dist_folders = {
project.name: paths_book.artifact(project.name, flow_id, step.id,
step.artifacts[0].id)
for project, step in dependencies
}
dist_files = {
name: list_files(folder)
for name, folder in dist_folders.items()
}
src_files = {
p.name: list_files(p.path / 'src')
for p, s in dependencies
}
return {
project.name: InputDataDependency(
project=project,
dist_folder=dist_folders[project.name],
src_folder=project.path / 'src',
dist_files=dist_files[project.name],
src_files=src_files[project.name],
checksum=files_check_sum(dist_files[project.name] +
src_files[project.name]))
for project, step in dependencies
}
async def execute_run(self, project: Project, flow_id: FlowId,
context: Context):
async with context.start(
action="run synchronization of workspace dependencies") as ctx:
data = await SyncFromDownstreamStep.get_input_data(project=project,
flow_id=flow_id,
context=ctx)
destination_folders: Mapping[str, Path] = {
name: project.path / 'node_modules' / name
for name in data.keys()
}
for name, p in data.items():
if destination_folders[name].exists():
shutil.rmtree(destination_folders[name])
copy_tree(p.dist_folder, destination_folders[name])
for file in p.src_files:
destination = destination_folders[
name] / 'src' / file.relative_to(p.src_folder)
copy_file(source=file,
destination=destination,
create_folders=True)
all_files = functools.reduce(
lambda acc, e: acc + e.src_files + e.dist_files, data.values(),
[])
return {
'fingerprint': files_check_sum(all_files),
'checksums': {name: d.checksum
for name, d in data.items()},
'nodeModuleChecksums': {
name: SyncFromDownstreamStep.node_module_checksum(
project=project, name=name)
for name in data.keys()
}
}
async def get_status(self, project: Project, flow_id: str, last_manifest: Optional[Manifest], context: Context) \
-> PipelineStepStatus:
env = await context.get('env', YouwolEnvironment)
local_cdn = LocalClients.get_cdn_client(env=env)
if not last_manifest:
return PipelineStepStatus.none
try:
local_info = await local_cdn.get_package(library_name=project.name, version=project.version, metadata=True)
except HTTPException as e:
if e.status_code == 404:
return PipelineStepStatus.none
raise e
files = await self.packaged_files(project, flow_id, context)
src_files_fingerprint = files_check_sum(files)
if last_manifest.fingerprint == local_info['fingerprint'] and \
last_manifest.cmdOutputs['src_files_fingerprint'] == src_files_fingerprint:
return PipelineStepStatus.OK
if last_manifest.fingerprint != local_info['fingerprint']:
await context.info(
text="Mismatch between cdn-backend fingerprint and saved manifest's fingerprint",
data={
'cdn-backend fingerprint': local_info['fingerprint'],
"saved manifest's fingerprint": last_manifest.fingerprint
}
)
if last_manifest.cmdOutputs['src_files_fingerprint'] != src_files_fingerprint:
await context.info(
text="Mismatch between actual src files fingerprint and saved manifest's src_files_fingerprint",
data={
'actual src files fingerprint': src_files_fingerprint,
"saved manifest's src_files_fingerprint": last_manifest.cmdOutputs['src_files_fingerprint']
}
)
return PipelineStepStatus.outdated
def node_module_checksum(project: Project, name: str) -> Optional[str]:
node_module_folder = project.path / 'node_modules' / name
files = list_files(node_module_folder)
return files_check_sum(files)