def getFiles(self, tree, context, path_prefix=""):
logger.debug(f"getFiles for {path_prefix}")
path, contents = self.getFolderContents(context, path_prefix)
if "files" in contents:
for key2, dandi_file in contents["files"].items():
# we save the version in the url query param for later usage in the download task
download_url = f"{self.api_url}/assets/{dandi_file['asset_id']}?folder={context}/{path_prefix}"
add_to_tree(
tree=tree,
tree_path=dandi_file["path"].split("/"),
path=download_url,
size=dandi_file["size"],
timestamp_modified=dandi_file["modified"],
osbrepository_id=self.osbrepository.id,
)
if "folders" in contents:
with concurrent.futures.ThreadPoolExecutor(
max_workers=MAX_WORKERS) as executor:
for key, dandi_folder in contents["folders"].items():
new_path_prefix = f"{path_prefix}/{key}".strip("/")
folder_url = f"{self.api_url}/dandisets/{self.dandiset_id}/versions/{context}/assets/paths/?path_prefix={new_path_prefix}&folder={context}/{new_path_prefix}"
add_to_tree(
tree=tree,
tree_path=f"{new_path_prefix}".split("/"),
path=folder_url,
osbrepository_id=self.osbrepository.id,
)
self.futures.append(
executor.submit(
self.getFiles,
context=context,
tree=tree,
path_prefix=new_path_prefix,
))
def get_tags(self, context):
try:
result = self._get_dandi_info(context)
return result["metadata"]["keywords"]
except Exception as e:
logger.debug("unable to get the tags from Dandi, %", str(e))
return []
def handler(app, event_client, message):
log.debug("Handler received message: %s", message)
for nh in [
nh for nh in NotificationsController._notification_handlers
if nh.message_type == message.get("message_type")
]:
nh.handle_event(message)
def get_tags(self, context):
try:
result = self._get_figshare_info(context)
return result["tags"]
except Exception as e:
logger.debug("unable to get the tags from Figshare, %", str(e))
return []
def get_description(self, context):
try:
result = self._get_dandi_info(context)
return result["metadata"]["description"]
except Exception as e:
logger.debug("unable to get the description from Dandi, %", str(e))
return ""
def get_description(self, context):
try:
result = self._get_figshare_info(context)
return result["description"]
except Exception as e:
logger.debug("unable to get the description from Figshare, %",
str(e))
return ""
def get_description(self, context):
try:
result = self.get_json(f"{self.api_url}readme?ref={context}")
description = base64.b64decode(result["content"]).decode("utf-8")
return description
except Exception as e:
logger.debug("unable to get the description from github, %",
str(e))
return ""
def submit(self):
"""Created and submits the Argo workflow"""
op = self.to_workflow()
log.debug("Submitting workflow\n" + pyaml.dump(op))
# TODO use rest api for that? Include this into cloudharness.workflows?
self.persisted = argo.submit_workflow(op)
return self.persisted
def get_json(self, uri):
logger.debug(f"Getting: {uri}")
try:
r = requests.get(uri, )
if r.status_code == 200:
return r.json()
else:
raise FigShareException(
f"Unexpected requests status code: {r.status_code}")
except Exception as e:
raise FigShareException("Unexpected error:", sys.exc_info()[0])
def get_keycloak_data():
bearer = request.headers.get("Authorization", None)
logger.debug(f"Bearer: {bearer}")
if not bearer or bearer == "Bearer undefined":
decoded_token = None
keycloak_id = -1 # No authorization --> no user --> only publicable workspaces
else:
token = bearer.split(" ")[1]
decoded_token = AuthClient.decode_token(token)
keycloak_id = decoded_token["sub"]
return keycloak_id, decoded_token
def post_commit(self, workspace):
# Create a new Persistent Volume Claim for this workspace
logger.debug(f"Post Commit for workspace id: {workspace.id}")
wsrr = WorkspaceResourceRepository()
for workspace_resource in workspace.resources:
wsr = wsrr.post_commit(workspace_resource)
if wsr:
db.session.add(wsr)
db.session.commit()
return workspace
def pre_commit(self, workspace_resource):
# Check if we can determine the resource type
logger.debug(
f"Pre Commit for workspace resource id: {workspace_resource.id}")
if not workspace_resource.resource_type or workspace_resource.resource_type == "u":
origin = json.loads(workspace_resource.origin)
workspace_resource.resource_type = self.guess_resource_type(
origin.get("path"))
if workspace_resource.folder is None or len(workspace_resource.folder) == 0:
workspace_resource.folder = workspace_resource.name
return workspace_resource
def submit_workflow(spec) -> Workflow:
log.debug(f"Submitting workflow %s", spec)
service = WorkflowServiceApi(api_client=get_api_client())
req = V1alpha1WorkflowCreateRequest(workflow=spec,
instance_id=namespace,
namespace=namespace)
# pprint(service.list_workflows('ch', V1alpha1WorkflowList()))
wf = service.create_workflow(namespace, req)
return Workflow(wf)
def execute(self, timeout=None):
self.persisted = self.submit()
start_time = time.time()
while not self.persisted.is_finished():
time.sleep(POLLING_WAIT_SECONDS)
log.debug(f"Polling argo workflow {self.persisted.name}")
self.persisted = argo.get_workflow(self.persisted.name)
log.debug(
f"Polling succeeded for {self.persisted.name}. Current phase: {self.persisted.status}"
)
if timeout and time.time() - start_time > timeout:
log.error("Timeout exceeded while polling for results")
return self.persisted
return self.persisted
def start_handlers(self):
"""
Start consuming incomming messages
"""
self._init_handlers()
# use a sleep loop to not frustrate the cpu
nap_time = 30
try:
while True:
time.sleep(nap_time) # sleep xx seconds
log.debug("Running...")
except Exception as e:
log.error(
'Notification Controller threw an error, stopping handlers.',
exc_info=True)
finally:
self.stop_handlers()
def delete_topic(self) -> bool:
log.debug("Deleting topic " + self.topic_id)
## Connect to kafka
admin_client = KafkaAdminClient(
bootstrap_servers=self._get_bootstrap_servers(),
client_id=self._get_client_id())
## Delete topic
try:
admin_client.delete_topics([self.topic_id])
return True
except UnknownTopicOrPartitionError as e:
log.error(f"Topic {self.topic_id} does not exists.")
raise EventTopicDeleteException from e
except Exception as e:
log.error(f"Error deleting the Topic {self.topic_id} --> {e}",
exc_info=True)
raise EventGeneralException from e
def get(self, id_):
workspace_entity: TWorkspaceEntity = super().get(id_)
workspace = dao_entity2dict(workspace_entity)
if workspace:
resources = workspace.get("resources")
if resources:
for r in resources:
r.update({"origin": json.loads(r.get("origin"))})
else:
workspace.update({"resources": []})
# check if there are running import tasks
logger.debug("Post get, check workflows for workspace %....",
workspace.get("id"))
workflows = argo.get_workflows(status="Running", limit=9999)
if workflows and workflows.items:
for workflow in workflows.items:
try:
if workflow.status == "Running" and workflow.raw.spec.templates[
0].metadata.labels.get("workspace").strip() == str(
workspace["id"]):
fake_path = f"Importing resources, progress {workflow.raw.status.progress}".replace(
"/", " of ")
workspace["resources"].append({
"id":
-1,
"name":
"Importing resources into workspace",
"origin": {
"path": fake_path
},
"resource_type":
ResourceType.E,
"workspace_id":
workspace["id"],
})
break
except Exception as e:
# probably not a workspace import workflow job --> skip it
pass
return workspace
def post_commit(self, workspace_resource):
# Create a load WorkspaceResource workflow task
logger.debug(
f"Post Commit for workspace resource id: {workspace_resource.id}")
workspace = WorkspaceRepository().get(workspace_resource.workspace_id)
if workspace_resource.folder is None:
logger.debug(
f"Pre Commit for workspace resource id: {workspace_resource.id} setting folder from file name")
workspace_resource.folder = workspace_resource.name
if (
workspace is not None
and workspace_resource.status == "p"
and workspace_resource.origin
and len(workspace_resource.origin) > 0
):
from workspaces.helpers.etl_helpers import copy_workspace_resource
copy_workspace_resource(workspace_resource)
return workspace_resource
def delete(self, id):
resource_repository = WorkspaceResourceRepository()
workspace = super().get(id)
for resource in workspace.resources:
logger.debug("deleting resource %s", resource.id)
resource_repository.delete(resource.id)
logger.info("deleting workspace %s", id)
super().delete(id)
logger.info("deleted workspace %s", id)
logger.info("deleting volume %s", id)
delete_persistent_volume_claim(f"workspace-{id}")
logger.info("deleted volume %s", id)
folder = os.path.join(Config.WORKSPACES_DIR, f"{id}")
if os.path.exists(os.path.join(Config.STATIC_DIR, folder)):
logger.info("deleting workspace files")
shutil.rmtree(os.path.join(Config.STATIC_DIR, folder))
logger.info("deleted workspace files")
def search(self, page=1, per_page=20, *args, **kwargs) -> Pagination:
"""
Query the model and return all records
Args:
page: The page number
per_page: The number of records per page to limit
Returns:
dict with all records from the model
current page
number of pages
"""
logger.debug("Search args %s", args)
logger.debug("Search kwargs %s", kwargs)
current_user_id = keycloak_user_id()
if current_user_id is not None:
# Admins see all workspaces, non admin users can see only their own workspaces
if not get_auth_client().user_has_realm_role(
user_id=current_user_id, role="administrator"):
objects = self.repository.search(page=page,
per_page=per_page,
user_id=current_user_id,
*args,
**kwargs)
else:
objects = self.repository.search(page=page,
per_page=per_page,
user_id=current_user_id,
show_all=True,
*args,
**kwargs)
else:
objects = self.repository.search(page=page,
per_page=per_page,
*args,
**kwargs)
for obj in objects.items:
self._calculated_fields_populate(obj)
return objects
def search(self, page=1, per_page=20, *args, **kwargs) -> Pagination:
"""
Query the model and return all records
Args:
page: The page number
per_page: The number of records per page to limit
Returns:
dict with all records from the model
current page
number of pages
"""
logger.debug("Search args %s", args)
logger.debug("Search kwargs %s", kwargs)
objects = self.repository.search(page=page,
per_page=per_page,
*args,
**kwargs)
for obj in objects.items:
self._calculated_fields_populate(obj)
return objects
def async_consume(self, app=None, handler=None, group_id='default'):
log.debug('creating thread')
if app:
log.debug('get current object from app')
app = app._get_current_object()
self._consumer_thread = threading.Thread(target=self._consume_task,
kwargs={
'app': app,
'group_id': group_id,
'handler': handler
})
self._consumer_thread.daemon = True
self._consumer_thread.start()
log.debug('thread started')
def pre_commit(self, obj):
logger.debug(f"Pre Commit for {obj} id: {obj.id}")
return obj
