def zip_local_story(raw_id: str, config: YouwolEnvironment) -> bytes:
stories = parse_json(config.pathsBook.local_stories_docdb)
documents = parse_json(config.pathsBook.local_stories_documents_docdb)
data = {
"story":
next(d for d in stories['documents'] if d['story_id'] == raw_id),
"documents":
[d for d in documents['documents'] if d['story_id'] == raw_id]
}
with tempfile.TemporaryDirectory() as tmp_folder:
base_path = Path(tmp_folder)
write_json(data=data, path=base_path / 'data.json')
storage_stories = config.pathsBook.local_stories_storage
for doc in data['documents']:
shutil.copy(storage_stories / doc['content_id'],
base_path / doc['content_id'])
zipper = zipfile.ZipFile(base_path / 'story.zip', 'w',
zipfile.ZIP_DEFLATED)
for filename in ['data.json'
] + [doc['content_id'] for doc in data['documents']]:
zipper.write(base_path / filename, arcname=filename)
zipper.close()
return (Path(tmp_folder) / "story.zip").read_bytes()
async def download_package(package_name: str, version: str, context: Context):
async with context.start(
action=f"download package {package_name}#{version}",
with_labels=[str(Label.PACKAGE_DOWNLOADING)],
with_attributes={
'packageName': package_name,
'packageVersion': version,
}) as ctx:
env = await context.get('env', YouwolEnvironment)
remote_gtw = await RemoteClients.get_assets_gateway_client(context=ctx)
default_drive = await env.get_default_drive(context=ctx)
asset_id = encode_id(encode_id(package_name))
await ctx.info(text=f"asset_id: {asset_id} queued for download")
await create_asset_local(
asset_id=asset_id,
kind='package',
default_owning_folder_id=default_drive.systemPackagesFolderId,
get_raw_data=lambda: remote_gtw.cdn_get_package(
library_name=package_name, version=version),
to_post_raw_data=lambda pack: {'file': pack},
context=ctx)
db = parse_json(env.pathsBook.local_cdn_docdb)
record = [
d for d in db['documents']
if d['library_id'] == f"{package_name}#{version}"
]
response = DownloadedPackageResponse(
packageName=package_name,
version=version,
fingerprint=record[0]['fingerprint'])
await ctx.send(response)
def get_local_package(asset_id: str, config: YouwolEnvironment) -> Library:
"""
Not populated with tree items
"""
data_packages = parse_json(config.pathsBook.local_docdb / "cdn" /
"libraries" / "data.json")
raw_id = decode_id(asset_id)
library_name = decode_id(raw_id)
releases = [
d for d in data_packages['documents']
if d['library_name'] == library_name
]
if not releases:
raise HTTPException(status_code=404,
detail=f'Local package {library_name} not found')
return Library(
libraryName=library_name,
namespace=releases[0]["namespace"],
releases=[
Release(version=r['version'],
fingerprint=r['fingerprint'] if 'fingerprint' in r else '')
for r in releases
],
assetId=asset_id,
rawId=raw_id,
treeItems=[],
)
async def create_borrowed_items(asset_id: str, tree_id: str,
assets_gtw_client: AssetsGatewayClient,
context: Context):
env = await context.get('env', YouwolEnvironment)
async with context.start(action="create_borrowed_items",
with_attributes={
'assetId': asset_id,
'treeId': tree_id
}) as ctx:
items_treedb = parse_json(env.pathsBook.local_treedb_docdb)
tree_items = [
item for item in items_treedb['documents']
if item['related_id'] == asset_id
]
borrowed_items = [
item for item in tree_items
if json.loads(item['metadata'])['borrowed']
]
await asyncio.gather(*[
create_borrowed_item(item=item,
borrowed_tree_id=tree_id,
assets_gtw_client=assets_gtw_client,
context=ctx) for item in borrowed_items
])
async def status(
request: Request,
config: YouwolEnvironment = Depends(yw_config)
):
context = ContextFactory.get_instance(
request=request,
web_socket=WebSocketsStore.userChannel
)
async with context.start(
action="Get environment status"
) as ctx:
connected = await connect_to_remote(config=config, context=context)
remote_gateway_info = config.get_remote_info()
if remote_gateway_info:
remote_gateway_info = RemoteGatewayInfo(name=remote_gateway_info.name,
host=remote_gateway_info.host,
connected=connected)
remotes_info = parse_json(config.pathsBook.remotesInfo)['remotes'].values()
response = EnvironmentStatusResponse(
users=config.get_users_list(),
userInfo=config.get_user_info(),
configuration=config,
remoteGatewayInfo=remote_gateway_info,
remotesInfo=list(remotes_info)
)
await ctx.send(response)
await ctx.send(ProjectsLoadingResults(results=await ProjectLoader.get_results(config, context)))
return response
async def get_status(
project: Project,
flow_id: str,
step: PipelineStep,
context: Context
) -> PipelineStepStatusResponse:
env = await context.get('env', YouwolEnvironment)
paths: PathsBook = env.pathsBook
async with context.start(
action="get status",
with_attributes={'projectId': project.id, 'flowId': flow_id, 'stepId': step.id}
) as _ctx:
path = paths.artifacts_step(project_name=project.name, flow_id=flow_id, step_id=step.id)
manifest = Manifest(**parse_json(path / 'manifest.json')) if (path / 'manifest.json').exists() else None
status = await step.get_status(project=project, flow_id=flow_id, last_manifest=manifest, context=context)
artifacts = [format_artifact_response(project=project, flow_id=flow_id, step=step, artifact=artifact,
env=env)
for artifact in step.artifacts]
return PipelineStepStatusResponse(
projectId=project.id,
flowId=flow_id,
stepId=step.id,
manifest=manifest,
artifactFolder=path,
artifacts=artifacts,
status=status
)
def get_manifest(self, flow_id: FlowId, step: PipelineStep, env: YouwolEnvironment):
paths_book: PathsBook = env.pathsBook
manifest_path = paths_book.artifacts_manifest(project_name=self.name, flow_id=flow_id, step_id=step.id)
if not manifest_path.exists():
return None
return Manifest(**parse_json(manifest_path))
async def cdn_status(request: Request,
project_id: str,
config: YouwolEnvironment = Depends(yw_config)):
context = ContextFactory.get_instance(
request=request, web_socket=WebSocketsStore.userChannel)
async with context.start(action="Get local cdn status",
with_attributes={
'event': 'CdnResponsePending',
'projectId': project_id
}) as ctx:
data = parse_json(config.pathsBook.local_cdn_docdb)['documents']
data = [d for d in data if d["library_name"] == decode_id(project_id)]
def format_version(doc):
storage_cdn_path = config.pathsBook.local_cdn_storage
folder_path = storage_cdn_path / doc['path']
bundle_path = folder_path / doc['bundle']
files_count = sum(
[len(files) for r, d, files in os.walk(folder_path)])
bundle_size = bundle_path.stat().st_size
return CdnVersionResponse(name=doc['library_name'],
version=doc['version'],
versionNumber=doc['version_number'],
filesCount=files_count,
bundleSize=bundle_size,
path=folder_path,
namespace=doc['namespace'])
response = CdnResponse(name=decode_id(project_id),
versions=[format_version(d) for d in data])
await ctx.send(response)
return response
def get_remote_info(self) -> Optional[RemoteGateway]:
info = parse_json(self.pathsBook.remotesInfo)['remotes']
if self.selectedRemote in info:
data = info[self.selectedRemote]
return RemoteGateway(**data)
return None
async def login(user_email: Union[str, None],
selected_remote: Union[str, None], users_info_path: Path,
remotes_info: Path, context: Union[Context,
None]) -> (str, str):
if user_email is None:
users_info = parse_json(users_info_path)
if 'default' in users_info['policies']:
user_email = users_info['policies']["default"]
if user_email is None:
raise HTTPException(
status_code=401,
detail=
f"User has not been identified, make sure it is defined in users info file ({users_info_path})"
)
if user_email not in parse_json(users_info_path)['users']:
context and await context.info(
text=
f"User {user_email} not registered in {users_info_path}: switch user",
data={
"user_email": user_email,
'usersInfo': parse_json(users_info_path)
})
return await login(user_email=None,
selected_remote=selected_remote,
users_info_path=users_info_path,
remotes_info=remotes_info,
context=context)
if remotes_info is None:
return user_email, None
remotes_info = parse_json(remotes_info)
remotes = remotes_info['remotes']
if selected_remote in remotes:
return user_email, selected_remote
if "policies" in remotes_info and "default" in remotes_info['policies']:
default = remotes_info['policies']['default']
if default in remotes:
return user_email, default
return user_email, None
async def sync_user(
request: Request,
body: SyncUserBody,
config: YouwolEnvironment = Depends(yw_config)
):
context = ContextFactory.get_instance(
request=request,
web_socket=WebSocketsStore.userChannel
)
async with context.start(f"Sync. user {body.email}") as ctx:
try:
auth_token = await get_public_user_auth_token(
username=body.email,
pwd=body.password,
client_id=config.get_remote_info().metadata['keycloakClientId'],
openid_host=config.openid_host
)
except Exception:
raise RuntimeError(f"Can not authorize from email/pwd @ {config.get_remote_info().host}")
await ctx.info(text="Login successful")
secrets = parse_json(config.pathsBook.secrets)
if body.email in secrets:
secrets[body.email] = {**secrets[body.email], **{"password": body.password}}
else:
secrets[body.email] = {"password": body.password}
write_json(secrets, config.pathsBook.secrets)
user_info = await retrieve_user_info(auth_token=auth_token, openid_host=config.openid_host)
users_info = parse_json(config.pathsBook.usersInfo)
users_info['users'][body.email] = {
"id": user_info['sub'],
"name": user_info['preferred_username'],
"memberOf": user_info['memberof'],
"email": user_info["email"]
}
write_json(users_info, config.pathsBook.usersInfo)
await login(request=request, body=LoginBody(email=body.email), config=config)
return users_info['users'][body.email]
def get_latest_local_cdn_version(
env: YouwolEnvironment) -> List[TargetPackage]:
db_path = parse_json(env.pathsBook.local_cdn_docdb)
data = sorted(db_path['documents'], key=lambda d: d['library_name'])
groups = [
list(g) for _, g in groupby(data, key=lambda d: d['library_name'])
]
targets = [max(g, key=lambda d: int(d['version_number'])) for g in groups]
return [TargetPackage.from_response(t) for t in targets]
def get_user_info(self) -> UserInfo:
users_info = parse_json(self.pathsBook.usersInfo)['users']
if self.userEmail in users_info:
data = users_info[self.userEmail]
return UserInfo(**data)
raise Exception(
f"User '{self.userEmail}' not reference in '{str(self.pathsBook.usersInfo)}"
)
async def publish_story(
request: Request,
file: UploadFile = File(...),
configuration: Configuration = Depends(get_configuration)):
headers = generate_headers_downstream(request.headers)
owner = Configuration.default_owner
doc_db_stories = configuration.doc_db_stories
doc_db_docs = configuration.doc_db_documents
storage = configuration.storage
with tempfile.TemporaryDirectory() as tmp_folder:
dir_path = Path(tmp_folder)
zip_path = (dir_path / file.filename).with_suffix('.zip')
extract_zip_file(file.file, zip_path=zip_path, dir_path=dir_path)
data = parse_json(dir_path / 'data.json')
story = data['story']
story_id = story['story_id']
documents = data['documents']
docs = await doc_db_stories.query(query_body=f"story_id={story_id}#1",
owner=Configuration.default_owner,
headers=headers)
if docs['documents']:
await delete_story(request,
story_id=story_id,
configuration=configuration)
await asyncio.gather(
doc_db_stories.create_document(doc=story,
owner=owner,
headers=headers),
*[
doc_db_docs.create_document(doc=doc,
owner=owner,
headers=headers)
for doc in documents
], *[
storage.post_object(
path=doc['content_id'],
content=(dir_path / doc['content_id']).read_text(),
content_type=Configuration.text_content_type,
owner=owner,
headers=headers) for doc in documents
])
return StoryResp(
storyId=story['story_id'],
title=next(
d for d in documents
if d['document_id'] == story['root_document_id'])['title'],
authors=story['authors'],
rootDocumentId=story['root_document_id'])
async def get_auth_token(self, context: Context, use_cache=True):
username = self.userEmail
remote = self.get_remote_info()
dependencies = {
"username": username,
"host": remote.host,
"type": "auth_token"
}
cached_token = next(
(c for c in self.tokensCache if c.is_valid(dependencies)), None)
if use_cache and cached_token:
return cached_token.value
secrets = parse_json(self.pathsBook.secrets)
if username not in secrets:
raise RuntimeError(
f"Can not find {username} in {str(self.pathsBook.secrets)}")
pwd = secrets[username]['password']
try:
access_token = await get_public_user_auth_token(
username=username,
pwd=pwd,
client_id=remote.metadata['keycloakClientId'],
openid_host=self.openid_host)
except Exception as e:
raise RuntimeError(
f"Can not authorize from email/pwd provided in " +
f"{str(self.pathsBook.secrets)} (error:{e})")
deadline = datetime.timestamp(datetime.now()) + 1 * 60 * 60 * 1000
self.tokensCache.append(
DeadlinedCache(value=access_token,
deadline=deadline,
dependencies=dependencies))
await context.info(text="Access token renewed",
data={
"host": remote.host,
"access_token": access_token
})
return access_token
def get_users_list(self) -> List[str]:
users = list(parse_json(self.pathsBook.usersInfo)['users'].keys())
return users
