def zip_local_story(raw_id: str, config: YouwolEnvironment) -> bytes: stories = parse_json(config.pathsBook.local_stories_docdb) documents = parse_json(config.pathsBook.local_stories_documents_docdb) data = { "story": next(d for d in stories['documents'] if d['story_id'] == raw_id), "documents": [d for d in documents['documents'] if d['story_id'] == raw_id] } with tempfile.TemporaryDirectory() as tmp_folder: base_path = Path(tmp_folder) write_json(data=data, path=base_path / 'data.json') storage_stories = config.pathsBook.local_stories_storage for doc in data['documents']: shutil.copy(storage_stories / doc['content_id'], base_path / doc['content_id']) zipper = zipfile.ZipFile(base_path / 'story.zip', 'w', zipfile.ZIP_DEFLATED) for filename in ['data.json' ] + [doc['content_id'] for doc in data['documents']]: zipper.write(base_path / filename, arcname=filename) zipper.close() return (Path(tmp_folder) / "story.zip").read_bytes()
async def download_package(package_name: str, version: str, context: Context): async with context.start( action=f"download package {package_name}#{version}", with_labels=[str(Label.PACKAGE_DOWNLOADING)], with_attributes={ 'packageName': package_name, 'packageVersion': version, }) as ctx: env = await context.get('env', YouwolEnvironment) remote_gtw = await RemoteClients.get_assets_gateway_client(context=ctx) default_drive = await env.get_default_drive(context=ctx) asset_id = encode_id(encode_id(package_name)) await ctx.info(text=f"asset_id: {asset_id} queued for download") await create_asset_local( asset_id=asset_id, kind='package', default_owning_folder_id=default_drive.systemPackagesFolderId, get_raw_data=lambda: remote_gtw.cdn_get_package( library_name=package_name, version=version), to_post_raw_data=lambda pack: {'file': pack}, context=ctx) db = parse_json(env.pathsBook.local_cdn_docdb) record = [ d for d in db['documents'] if d['library_id'] == f"{package_name}#{version}" ] response = DownloadedPackageResponse( packageName=package_name, version=version, fingerprint=record[0]['fingerprint']) await ctx.send(response)
def get_local_package(asset_id: str, config: YouwolEnvironment) -> Library: """ Not populated with tree items """ data_packages = parse_json(config.pathsBook.local_docdb / "cdn" / "libraries" / "data.json") raw_id = decode_id(asset_id) library_name = decode_id(raw_id) releases = [ d for d in data_packages['documents'] if d['library_name'] == library_name ] if not releases: raise HTTPException(status_code=404, detail=f'Local package {library_name} not found') return Library( libraryName=library_name, namespace=releases[0]["namespace"], releases=[ Release(version=r['version'], fingerprint=r['fingerprint'] if 'fingerprint' in r else '') for r in releases ], assetId=asset_id, rawId=raw_id, treeItems=[], )
async def create_borrowed_items(asset_id: str, tree_id: str, assets_gtw_client: AssetsGatewayClient, context: Context): env = await context.get('env', YouwolEnvironment) async with context.start(action="create_borrowed_items", with_attributes={ 'assetId': asset_id, 'treeId': tree_id }) as ctx: items_treedb = parse_json(env.pathsBook.local_treedb_docdb) tree_items = [ item for item in items_treedb['documents'] if item['related_id'] == asset_id ] borrowed_items = [ item for item in tree_items if json.loads(item['metadata'])['borrowed'] ] await asyncio.gather(*[ create_borrowed_item(item=item, borrowed_tree_id=tree_id, assets_gtw_client=assets_gtw_client, context=ctx) for item in borrowed_items ])
async def status( request: Request, config: YouwolEnvironment = Depends(yw_config) ): context = ContextFactory.get_instance( request=request, web_socket=WebSocketsStore.userChannel ) async with context.start( action="Get environment status" ) as ctx: connected = await connect_to_remote(config=config, context=context) remote_gateway_info = config.get_remote_info() if remote_gateway_info: remote_gateway_info = RemoteGatewayInfo(name=remote_gateway_info.name, host=remote_gateway_info.host, connected=connected) remotes_info = parse_json(config.pathsBook.remotesInfo)['remotes'].values() response = EnvironmentStatusResponse( users=config.get_users_list(), userInfo=config.get_user_info(), configuration=config, remoteGatewayInfo=remote_gateway_info, remotesInfo=list(remotes_info) ) await ctx.send(response) await ctx.send(ProjectsLoadingResults(results=await ProjectLoader.get_results(config, context))) return response
async def get_status( project: Project, flow_id: str, step: PipelineStep, context: Context ) -> PipelineStepStatusResponse: env = await context.get('env', YouwolEnvironment) paths: PathsBook = env.pathsBook async with context.start( action="get status", with_attributes={'projectId': project.id, 'flowId': flow_id, 'stepId': step.id} ) as _ctx: path = paths.artifacts_step(project_name=project.name, flow_id=flow_id, step_id=step.id) manifest = Manifest(**parse_json(path / 'manifest.json')) if (path / 'manifest.json').exists() else None status = await step.get_status(project=project, flow_id=flow_id, last_manifest=manifest, context=context) artifacts = [format_artifact_response(project=project, flow_id=flow_id, step=step, artifact=artifact, env=env) for artifact in step.artifacts] return PipelineStepStatusResponse( projectId=project.id, flowId=flow_id, stepId=step.id, manifest=manifest, artifactFolder=path, artifacts=artifacts, status=status )
def get_manifest(self, flow_id: FlowId, step: PipelineStep, env: YouwolEnvironment): paths_book: PathsBook = env.pathsBook manifest_path = paths_book.artifacts_manifest(project_name=self.name, flow_id=flow_id, step_id=step.id) if not manifest_path.exists(): return None return Manifest(**parse_json(manifest_path))
async def cdn_status(request: Request, project_id: str, config: YouwolEnvironment = Depends(yw_config)): context = ContextFactory.get_instance( request=request, web_socket=WebSocketsStore.userChannel) async with context.start(action="Get local cdn status", with_attributes={ 'event': 'CdnResponsePending', 'projectId': project_id }) as ctx: data = parse_json(config.pathsBook.local_cdn_docdb)['documents'] data = [d for d in data if d["library_name"] == decode_id(project_id)] def format_version(doc): storage_cdn_path = config.pathsBook.local_cdn_storage folder_path = storage_cdn_path / doc['path'] bundle_path = folder_path / doc['bundle'] files_count = sum( [len(files) for r, d, files in os.walk(folder_path)]) bundle_size = bundle_path.stat().st_size return CdnVersionResponse(name=doc['library_name'], version=doc['version'], versionNumber=doc['version_number'], filesCount=files_count, bundleSize=bundle_size, path=folder_path, namespace=doc['namespace']) response = CdnResponse(name=decode_id(project_id), versions=[format_version(d) for d in data]) await ctx.send(response) return response
def get_remote_info(self) -> Optional[RemoteGateway]: info = parse_json(self.pathsBook.remotesInfo)['remotes'] if self.selectedRemote in info: data = info[self.selectedRemote] return RemoteGateway(**data) return None
async def login(user_email: Union[str, None], selected_remote: Union[str, None], users_info_path: Path, remotes_info: Path, context: Union[Context, None]) -> (str, str): if user_email is None: users_info = parse_json(users_info_path) if 'default' in users_info['policies']: user_email = users_info['policies']["default"] if user_email is None: raise HTTPException( status_code=401, detail= f"User has not been identified, make sure it is defined in users info file ({users_info_path})" ) if user_email not in parse_json(users_info_path)['users']: context and await context.info( text= f"User {user_email} not registered in {users_info_path}: switch user", data={ "user_email": user_email, 'usersInfo': parse_json(users_info_path) }) return await login(user_email=None, selected_remote=selected_remote, users_info_path=users_info_path, remotes_info=remotes_info, context=context) if remotes_info is None: return user_email, None remotes_info = parse_json(remotes_info) remotes = remotes_info['remotes'] if selected_remote in remotes: return user_email, selected_remote if "policies" in remotes_info and "default" in remotes_info['policies']: default = remotes_info['policies']['default'] if default in remotes: return user_email, default return user_email, None
async def sync_user( request: Request, body: SyncUserBody, config: YouwolEnvironment = Depends(yw_config) ): context = ContextFactory.get_instance( request=request, web_socket=WebSocketsStore.userChannel ) async with context.start(f"Sync. user {body.email}") as ctx: try: auth_token = await get_public_user_auth_token( username=body.email, pwd=body.password, client_id=config.get_remote_info().metadata['keycloakClientId'], openid_host=config.openid_host ) except Exception: raise RuntimeError(f"Can not authorize from email/pwd @ {config.get_remote_info().host}") await ctx.info(text="Login successful") secrets = parse_json(config.pathsBook.secrets) if body.email in secrets: secrets[body.email] = {**secrets[body.email], **{"password": body.password}} else: secrets[body.email] = {"password": body.password} write_json(secrets, config.pathsBook.secrets) user_info = await retrieve_user_info(auth_token=auth_token, openid_host=config.openid_host) users_info = parse_json(config.pathsBook.usersInfo) users_info['users'][body.email] = { "id": user_info['sub'], "name": user_info['preferred_username'], "memberOf": user_info['memberof'], "email": user_info["email"] } write_json(users_info, config.pathsBook.usersInfo) await login(request=request, body=LoginBody(email=body.email), config=config) return users_info['users'][body.email]
def get_latest_local_cdn_version( env: YouwolEnvironment) -> List[TargetPackage]: db_path = parse_json(env.pathsBook.local_cdn_docdb) data = sorted(db_path['documents'], key=lambda d: d['library_name']) groups = [ list(g) for _, g in groupby(data, key=lambda d: d['library_name']) ] targets = [max(g, key=lambda d: int(d['version_number'])) for g in groups] return [TargetPackage.from_response(t) for t in targets]
def get_user_info(self) -> UserInfo: users_info = parse_json(self.pathsBook.usersInfo)['users'] if self.userEmail in users_info: data = users_info[self.userEmail] return UserInfo(**data) raise Exception( f"User '{self.userEmail}' not reference in '{str(self.pathsBook.usersInfo)}" )
async def publish_story( request: Request, file: UploadFile = File(...), configuration: Configuration = Depends(get_configuration)): headers = generate_headers_downstream(request.headers) owner = Configuration.default_owner doc_db_stories = configuration.doc_db_stories doc_db_docs = configuration.doc_db_documents storage = configuration.storage with tempfile.TemporaryDirectory() as tmp_folder: dir_path = Path(tmp_folder) zip_path = (dir_path / file.filename).with_suffix('.zip') extract_zip_file(file.file, zip_path=zip_path, dir_path=dir_path) data = parse_json(dir_path / 'data.json') story = data['story'] story_id = story['story_id'] documents = data['documents'] docs = await doc_db_stories.query(query_body=f"story_id={story_id}#1", owner=Configuration.default_owner, headers=headers) if docs['documents']: await delete_story(request, story_id=story_id, configuration=configuration) await asyncio.gather( doc_db_stories.create_document(doc=story, owner=owner, headers=headers), *[ doc_db_docs.create_document(doc=doc, owner=owner, headers=headers) for doc in documents ], *[ storage.post_object( path=doc['content_id'], content=(dir_path / doc['content_id']).read_text(), content_type=Configuration.text_content_type, owner=owner, headers=headers) for doc in documents ]) return StoryResp( storyId=story['story_id'], title=next( d for d in documents if d['document_id'] == story['root_document_id'])['title'], authors=story['authors'], rootDocumentId=story['root_document_id'])
async def get_auth_token(self, context: Context, use_cache=True): username = self.userEmail remote = self.get_remote_info() dependencies = { "username": username, "host": remote.host, "type": "auth_token" } cached_token = next( (c for c in self.tokensCache if c.is_valid(dependencies)), None) if use_cache and cached_token: return cached_token.value secrets = parse_json(self.pathsBook.secrets) if username not in secrets: raise RuntimeError( f"Can not find {username} in {str(self.pathsBook.secrets)}") pwd = secrets[username]['password'] try: access_token = await get_public_user_auth_token( username=username, pwd=pwd, client_id=remote.metadata['keycloakClientId'], openid_host=self.openid_host) except Exception as e: raise RuntimeError( f"Can not authorize from email/pwd provided in " + f"{str(self.pathsBook.secrets)} (error:{e})") deadline = datetime.timestamp(datetime.now()) + 1 * 60 * 60 * 1000 self.tokensCache.append( DeadlinedCache(value=access_token, deadline=deadline, dependencies=dependencies)) await context.info(text="Access token renewed", data={ "host": remote.host, "access_token": access_token }) return access_token
def get_users_list(self) -> List[str]: users = list(parse_json(self.pathsBook.usersInfo)['users'].keys()) return users