def getFiles(self, tree, context, path_prefix=""): logger.debug(f"getFiles for {path_prefix}") path, contents = self.getFolderContents(context, path_prefix) if "files" in contents: for key2, dandi_file in contents["files"].items(): # we save the version in the url query param for later usage in the download task download_url = f"{self.api_url}/assets/{dandi_file['asset_id']}?folder={context}/{path_prefix}" add_to_tree( tree=tree, tree_path=dandi_file["path"].split("/"), path=download_url, size=dandi_file["size"], timestamp_modified=dandi_file["modified"], osbrepository_id=self.osbrepository.id, ) if "folders" in contents: with concurrent.futures.ThreadPoolExecutor( max_workers=MAX_WORKERS) as executor: for key, dandi_folder in contents["folders"].items(): new_path_prefix = f"{path_prefix}/{key}".strip("/") folder_url = f"{self.api_url}/dandisets/{self.dandiset_id}/versions/{context}/assets/paths/?path_prefix={new_path_prefix}&folder={context}/{new_path_prefix}" add_to_tree( tree=tree, tree_path=f"{new_path_prefix}".split("/"), path=folder_url, osbrepository_id=self.osbrepository.id, ) self.futures.append( executor.submit( self.getFiles, context=context, tree=tree, path_prefix=new_path_prefix, ))
def get_tags(self, context): try: result = self._get_dandi_info(context) return result["metadata"]["keywords"] except Exception as e: logger.debug("unable to get the tags from Dandi, %", str(e)) return []
def handler(app, event_client, message): log.debug("Handler received message: %s", message) for nh in [ nh for nh in NotificationsController._notification_handlers if nh.message_type == message.get("message_type") ]: nh.handle_event(message)
def get_tags(self, context): try: result = self._get_figshare_info(context) return result["tags"] except Exception as e: logger.debug("unable to get the tags from Figshare, %", str(e)) return []
def get_description(self, context): try: result = self._get_dandi_info(context) return result["metadata"]["description"] except Exception as e: logger.debug("unable to get the description from Dandi, %", str(e)) return ""
def get_description(self, context): try: result = self._get_figshare_info(context) return result["description"] except Exception as e: logger.debug("unable to get the description from Figshare, %", str(e)) return ""
def get_description(self, context): try: result = self.get_json(f"{self.api_url}readme?ref={context}") description = base64.b64decode(result["content"]).decode("utf-8") return description except Exception as e: logger.debug("unable to get the description from github, %", str(e)) return ""
def submit(self): """Created and submits the Argo workflow""" op = self.to_workflow() log.debug("Submitting workflow\n" + pyaml.dump(op)) # TODO use rest api for that? Include this into cloudharness.workflows? self.persisted = argo.submit_workflow(op) return self.persisted
def get_json(self, uri): logger.debug(f"Getting: {uri}") try: r = requests.get(uri, ) if r.status_code == 200: return r.json() else: raise FigShareException( f"Unexpected requests status code: {r.status_code}") except Exception as e: raise FigShareException("Unexpected error:", sys.exc_info()[0])
def get_keycloak_data(): bearer = request.headers.get("Authorization", None) logger.debug(f"Bearer: {bearer}") if not bearer or bearer == "Bearer undefined": decoded_token = None keycloak_id = -1 # No authorization --> no user --> only publicable workspaces else: token = bearer.split(" ")[1] decoded_token = AuthClient.decode_token(token) keycloak_id = decoded_token["sub"] return keycloak_id, decoded_token
def post_commit(self, workspace): # Create a new Persistent Volume Claim for this workspace logger.debug(f"Post Commit for workspace id: {workspace.id}") wsrr = WorkspaceResourceRepository() for workspace_resource in workspace.resources: wsr = wsrr.post_commit(workspace_resource) if wsr: db.session.add(wsr) db.session.commit() return workspace
def pre_commit(self, workspace_resource): # Check if we can determine the resource type logger.debug( f"Pre Commit for workspace resource id: {workspace_resource.id}") if not workspace_resource.resource_type or workspace_resource.resource_type == "u": origin = json.loads(workspace_resource.origin) workspace_resource.resource_type = self.guess_resource_type( origin.get("path")) if workspace_resource.folder is None or len(workspace_resource.folder) == 0: workspace_resource.folder = workspace_resource.name return workspace_resource
def submit_workflow(spec) -> Workflow: log.debug(f"Submitting workflow %s", spec) service = WorkflowServiceApi(api_client=get_api_client()) req = V1alpha1WorkflowCreateRequest(workflow=spec, instance_id=namespace, namespace=namespace) # pprint(service.list_workflows('ch', V1alpha1WorkflowList())) wf = service.create_workflow(namespace, req) return Workflow(wf)
def execute(self, timeout=None): self.persisted = self.submit() start_time = time.time() while not self.persisted.is_finished(): time.sleep(POLLING_WAIT_SECONDS) log.debug(f"Polling argo workflow {self.persisted.name}") self.persisted = argo.get_workflow(self.persisted.name) log.debug( f"Polling succeeded for {self.persisted.name}. Current phase: {self.persisted.status}" ) if timeout and time.time() - start_time > timeout: log.error("Timeout exceeded while polling for results") return self.persisted return self.persisted
def start_handlers(self): """ Start consuming incomming messages """ self._init_handlers() # use a sleep loop to not frustrate the cpu nap_time = 30 try: while True: time.sleep(nap_time) # sleep xx seconds log.debug("Running...") except Exception as e: log.error( 'Notification Controller threw an error, stopping handlers.', exc_info=True) finally: self.stop_handlers()
def delete_topic(self) -> bool: log.debug("Deleting topic " + self.topic_id) ## Connect to kafka admin_client = KafkaAdminClient( bootstrap_servers=self._get_bootstrap_servers(), client_id=self._get_client_id()) ## Delete topic try: admin_client.delete_topics([self.topic_id]) return True except UnknownTopicOrPartitionError as e: log.error(f"Topic {self.topic_id} does not exists.") raise EventTopicDeleteException from e except Exception as e: log.error(f"Error deleting the Topic {self.topic_id} --> {e}", exc_info=True) raise EventGeneralException from e
def get(self, id_): workspace_entity: TWorkspaceEntity = super().get(id_) workspace = dao_entity2dict(workspace_entity) if workspace: resources = workspace.get("resources") if resources: for r in resources: r.update({"origin": json.loads(r.get("origin"))}) else: workspace.update({"resources": []}) # check if there are running import tasks logger.debug("Post get, check workflows for workspace %....", workspace.get("id")) workflows = argo.get_workflows(status="Running", limit=9999) if workflows and workflows.items: for workflow in workflows.items: try: if workflow.status == "Running" and workflow.raw.spec.templates[ 0].metadata.labels.get("workspace").strip() == str( workspace["id"]): fake_path = f"Importing resources, progress {workflow.raw.status.progress}".replace( "/", " of ") workspace["resources"].append({ "id": -1, "name": "Importing resources into workspace", "origin": { "path": fake_path }, "resource_type": ResourceType.E, "workspace_id": workspace["id"], }) break except Exception as e: # probably not a workspace import workflow job --> skip it pass return workspace
def post_commit(self, workspace_resource): # Create a load WorkspaceResource workflow task logger.debug( f"Post Commit for workspace resource id: {workspace_resource.id}") workspace = WorkspaceRepository().get(workspace_resource.workspace_id) if workspace_resource.folder is None: logger.debug( f"Pre Commit for workspace resource id: {workspace_resource.id} setting folder from file name") workspace_resource.folder = workspace_resource.name if ( workspace is not None and workspace_resource.status == "p" and workspace_resource.origin and len(workspace_resource.origin) > 0 ): from workspaces.helpers.etl_helpers import copy_workspace_resource copy_workspace_resource(workspace_resource) return workspace_resource
def delete(self, id): resource_repository = WorkspaceResourceRepository() workspace = super().get(id) for resource in workspace.resources: logger.debug("deleting resource %s", resource.id) resource_repository.delete(resource.id) logger.info("deleting workspace %s", id) super().delete(id) logger.info("deleted workspace %s", id) logger.info("deleting volume %s", id) delete_persistent_volume_claim(f"workspace-{id}") logger.info("deleted volume %s", id) folder = os.path.join(Config.WORKSPACES_DIR, f"{id}") if os.path.exists(os.path.join(Config.STATIC_DIR, folder)): logger.info("deleting workspace files") shutil.rmtree(os.path.join(Config.STATIC_DIR, folder)) logger.info("deleted workspace files")
def search(self, page=1, per_page=20, *args, **kwargs) -> Pagination: """ Query the model and return all records Args: page: The page number per_page: The number of records per page to limit Returns: dict with all records from the model current page number of pages """ logger.debug("Search args %s", args) logger.debug("Search kwargs %s", kwargs) current_user_id = keycloak_user_id() if current_user_id is not None: # Admins see all workspaces, non admin users can see only their own workspaces if not get_auth_client().user_has_realm_role( user_id=current_user_id, role="administrator"): objects = self.repository.search(page=page, per_page=per_page, user_id=current_user_id, *args, **kwargs) else: objects = self.repository.search(page=page, per_page=per_page, user_id=current_user_id, show_all=True, *args, **kwargs) else: objects = self.repository.search(page=page, per_page=per_page, *args, **kwargs) for obj in objects.items: self._calculated_fields_populate(obj) return objects
def search(self, page=1, per_page=20, *args, **kwargs) -> Pagination: """ Query the model and return all records Args: page: The page number per_page: The number of records per page to limit Returns: dict with all records from the model current page number of pages """ logger.debug("Search args %s", args) logger.debug("Search kwargs %s", kwargs) objects = self.repository.search(page=page, per_page=per_page, *args, **kwargs) for obj in objects.items: self._calculated_fields_populate(obj) return objects
def async_consume(self, app=None, handler=None, group_id='default'): log.debug('creating thread') if app: log.debug('get current object from app') app = app._get_current_object() self._consumer_thread = threading.Thread(target=self._consume_task, kwargs={ 'app': app, 'group_id': group_id, 'handler': handler }) self._consumer_thread.daemon = True self._consumer_thread.start() log.debug('thread started')
def pre_commit(self, obj): logger.debug(f"Pre Commit for {obj} id: {obj.id}") return obj