def update(call: APICall): """ update :summary: Update project information. See `project.create` for parameters. :return: updated - `int` - number of projects updated fields - `[string]` - updated fields """ project_id = call.data["project"] with translate_errors_context(): project = Project.get_for_writing(company=call.identity.company, id=project_id) if not project: raise errors.bad_request.InvalidProjectId(id=project_id) fields = parse_from_call(call.data, create_fields, Project.get_fields(), discard_none_values=False) conform_tag_fields(call, fields, validate=True) fields["last_update"] = datetime.utcnow() with TimingContext("mongo", "projects_update"): updated = project.update(upsert=False, **fields) conform_output_tags(call, fields) call.result.data_model = UpdateResponse(updated=updated, fields=fields)
def validate(cls, task: Task, force=False): assert isinstance(task, Task) if task.parent and not Task.get(company=task.company, id=task.parent, _only=("id", ), include_public=True): raise errors.bad_request.InvalidTaskId("invalid parent", parent=task.parent) if task.project: Project.get_for_writing(company=task.company, id=task.project) model = cls.validate_execution_model(task) if model and not force and not model.ready: raise errors.bad_request.ModelNotReady("can't be used in a task", model=model.id) if task.execution: if task.execution.parameters: cls._validate_execution_parameters(task.execution.parameters) if task.output and task.output.destination: parsed_url = urlparse(task.output.destination) if parsed_url.scheme not in OutputDestinationField.schemes: raise errors.bad_request.FieldsValueError( "unsupported scheme for output destination", dest=task.output.destination, )
def delete(call): assert isinstance(call, APICall) project_id = call.data["project"] force = call.data.get("force", False) with translate_errors_context(): project = Project.get_for_writing(company=call.identity.company, id=project_id) if not project: raise errors.bad_request.InvalidProjectId(id=project_id) # NOTE: from this point on we'll use the project ID and won't check for company, since we assume we already # have the correct project ID. # Find the tasks which belong to the project for cls, error in ( (Task, errors.bad_request.ProjectHasTasks), (Model, errors.bad_request.ProjectHasModels), ): res = cls.objects( project=project_id, system_tags__nin=[EntityVisibility.archived.value]).only("id") if res and not force: raise error("use force=true to delete", id=project_id) updated_count = res.update(project=None) project.delete() call.result.data = {"deleted": 1, "disassociated_tasks": updated_count}
def validate(cls, task: Task): assert isinstance(task, Task) if task.parent and not Task.get( company=task.company, id=task.parent, _only=("id",), include_public=True ): raise errors.bad_request.InvalidTaskId("invalid parent", parent=task.parent) if task.project: Project.get_for_writing(company=task.company, id=task.project) cls.validate_execution_model(task) if task.execution: if task.execution.parameters: cls._validate_execution_parameters(task.execution.parameters)
def validate( cls, task: Task, validate_model=True, validate_parent=True, validate_project=True, ): if ( validate_parent and task.parent and not Task.get( company=task.company, id=task.parent, _only=("id",), include_public=True ) ): raise errors.bad_request.InvalidTaskId("invalid parent", parent=task.parent) if ( validate_project and task.project and not Project.get_for_writing(company=task.company, id=task.project) ): raise errors.bad_request.InvalidProjectId(id=task.project) if validate_model: cls.validate_execution_model(task)
def _resolve_entities( cls, experiments: List[str] = None, projects: List[str] = None ) -> Dict[Type[mongoengine.Document], Set[mongoengine.Document]]: from database.model.project import Project from database.model.task.task import Task entities = defaultdict(set) if projects: print("Reading projects...") entities[Project].update(cls._resolve_type(Project, projects)) print("--> Reading project experiments...") objs = Task.objects( project__in=list(set(filter(None, (p.id for p in entities[Project])))) ) entities[Task].update(o for o in objs if o.id not in (experiments or [])) if experiments: print("Reading experiments...") entities[Task].update(cls._resolve_type(Task, experiments)) print("--> Reading experiments projects...") objs = Project.objects( id__in=list(set(filter(None, (p.project for p in entities[Task])))) ) project_ids = {p.id for p in entities[Project]} entities[Project].update(o for o in objs if o.id not in project_ids) return entities
def update_featured_projects_order(cls): featured_order = config.get("services.projects.featured_order", []) def get_index(p: Project): for index, entry in enumerate(featured_order): if ( entry.get("id", None) == p.id or entry.get("name", None) == p.name or ("name_regex" in entry and re.match(entry["name_regex"], p.name)) ): return index return 999 for project in Project.get_many_public(projection=["id", "name"]): featured_index = get_index(project) Project.objects(id=project.id).update(featured=featured_index)
def create(call): assert isinstance(call, APICall) identity = call.identity with translate_errors_context(): fields = parse_from_call(call.data, create_fields, Project.get_fields()) conform_tag_fields(call, fields, validate=True) now = datetime.utcnow() project = Project(id=database.utils.id(), user=identity.user, company=identity.company, created=now, last_update=now, **fields) with TimingContext("mongo", "projects_save"): project.save() call.result.data = {"id": project.id}
def get_all(call): assert isinstance(call, APICall) with translate_errors_context(), TimingContext("mongo", "projects_get_all"): res = Project.get_many( company=call.identity.company, query_dict=call.data, query_options=get_all_query_options, parameters=call.data, allow_public=True, ) call.result.data = {"projects": res}
def get_all(call: APICall): conform_tag_fields(call, call.data) with translate_errors_context(), TimingContext("mongo", "projects_get_all"): projects = Project.get_many( company=call.identity.company, query_dict=call.data, query_options=get_all_query_options, parameters=call.data, allow_public=True, ) conform_output_tags(call, projects) call.result.data = {"projects": projects}
def get_by_id(call): assert isinstance(call, APICall) project_id = call.data["project"] with translate_errors_context(): with TimingContext("mongo", "projects_by_id"): query = Q(id=project_id) & get_company_or_none_constraint( call.identity.company) project = Project.objects(query).first() if not project: raise errors.bad_request.InvalidProjectId(id=project_id) project_dict = project.to_proper_dict() conform_output_tags(call, project_dict) call.result.data = {"project": project_dict}
def _resolve_entities( cls, experiments: Sequence[str] = None, projects: Sequence[str] = None, task_statuses: Sequence[str] = None, ) -> Dict[Type[mongoengine.Document], Set[mongoengine.Document]]: entities = defaultdict(set) if projects: print("Reading projects...") entities[Project].update(cls._resolve_type(Project, projects)) print("--> Reading project experiments...") query = Q( project__in=list( set(filter(None, (p.id for p in entities[Project])))), system_tags__nin=[EntityVisibility.archived.value], ) if task_statuses: query &= Q(status__in=list(set(task_statuses))) objs = Task.objects(query) entities[Task].update(o for o in objs if o.id not in (experiments or [])) if experiments: print("Reading experiments...") entities[Task].update(cls._resolve_type(Task, experiments)) print("--> Reading experiments projects...") objs = Project.objects(id__in=list( set(filter(None, (p.project for p in entities[Task]))))) project_ids = {p.id for p in entities[Project]} entities[Project].update(o for o in objs if o.id not in project_ids) model_ids = { model_id for task in entities[Task] for model_id in (task.output.model, task.execution.model) if model_id } if model_ids: print("Reading models...") entities[Model] = set(Model.objects(id__in=list(model_ids))) return entities
def update_project_time(project_id): if project_id: Project.objects(id=project_id).update(last_update=datetime.utcnow())
def status_report( self, company_id: str, user_id: str, ip: str, report: StatusReportRequest, tags: Sequence[str] = None, ) -> None: """ Write worker status report :param company_id: worker's company ID :param user_id: user_id ID under which this worker is running :param ip: worker IP :param report: the report itself :param tags: tags for this worker :raise bad_request.InvalidTaskId: the reported task was not found :return: worker entry instance """ entry = self._get_worker(company_id, user_id, report.worker) try: entry.ip = ip now = datetime.utcnow() entry.last_activity_time = now if tags is not None: entry.tags = tags if report.machine_stats: self._log_stats_to_es( company_id=company_id, company_name=entry.company.name, worker=report.worker, timestamp=report.timestamp, task=report.task, machine_stats=report.machine_stats, ) entry.queue = report.queue if report.queues: entry.queues = report.queues if not report.task: entry.task = None entry.project = None else: with translate_errors_context(): query = dict(id=report.task, company=company_id) update = dict( last_worker=report.worker, last_worker_report=now, last_update=now, ) # modify(new=True, ...) returns the modified object task = Task.objects(**query).modify(new=True, **update) if not task: raise bad_request.InvalidTaskId(**query) entry.task = IdNameEntry(id=task.id, name=task.name) entry.project = None if task.project: project = Project.objects(id=task.project).only("name").first() if project: entry.project = IdNameEntry(id=project.id, name=project.name) entry.last_report_time = now except APIError: raise except Exception as e: msg = "Failed processing worker status report" log.exception(msg) raise server_error.DataError(msg, err=e.args[0]) finally: self._save_worker(entry)
def _cleanup_project(cls, project: Project): project.user = "" project.company = "" project.tags = cls._filter_out_export_tags(project.tags)
org_bll = OrgBLL() task_bll = TaskBLL() archived_tasks_cond = { "$in": [EntityVisibility.archived.value, "$system_tags"] } create_fields = { "name": None, "description": None, "tags": list, "system_tags": list, "default_output_destination": None, } get_all_query_options = Project.QueryParameterOptions( pattern_fields=("name", "description"), list_fields=("tags", "system_tags", "id"), ) @endpoint("projects.get_by_id", required_fields=["project"]) def get_by_id(call): assert isinstance(call, APICall) project_id = call.data["project"] with translate_errors_context(): with TimingContext("mongo", "projects_by_id"): query = Q(id=project_id) & get_company_or_none_constraint( call.identity.company) project = Project.objects(query).first() if not project: raise errors.bad_request.InvalidProjectId(id=project_id)
def get_all_ex(call: APICall): include_stats = call.data.get("include_stats") stats_for_state = call.data.get("stats_for_state", EntityVisibility.active.value) allow_public = not call.data.get("non_public", False) if stats_for_state: try: specific_state = EntityVisibility(stats_for_state) except ValueError: raise errors.bad_request.FieldsValueError( stats_for_state=stats_for_state) else: specific_state = None conform_tag_fields(call, call.data) with translate_errors_context(), TimingContext("mongo", "projects_get_all"): projects = Project.get_many_with_join( company=call.identity.company, query_dict=call.data, query_options=get_all_query_options, allow_public=allow_public, ) conform_output_tags(call, projects) if not include_stats: call.result.data = {"projects": projects} return ids = [project["id"] for project in projects] status_count_pipeline, runtime_pipeline = make_projects_get_all_pipelines( call.identity.company, ids, specific_state=specific_state) default_counts = dict.fromkeys(get_options(TaskStatus), 0) def set_default_count(entry): return dict(default_counts, **entry) status_count = defaultdict(lambda: {}) key = itemgetter(EntityVisibility.archived.value) for result in Task.aggregate(status_count_pipeline): for k, group in groupby(sorted(result["counts"], key=key), key): section = (EntityVisibility.archived if k else EntityVisibility.active).value status_count[result["_id"]][section] = set_default_count({ count_entry["status"]: count_entry["count"] for count_entry in group }) runtime = { result["_id"]: {k: v for k, v in result.items() if k != "_id"} for result in Task.aggregate(runtime_pipeline) } def safe_get(obj, path, default=None): try: return dpath.get(obj, path) except KeyError: return default def get_status_counts(project_id, section): path = "/".join((project_id, section)) return { "total_runtime": safe_get(runtime, path, 0), "status_count": safe_get(status_count, path, default_counts), } report_for_states = [ s for s in EntityVisibility if not specific_state or specific_state == s ] for project in projects: project["stats"] = { task_state.value: get_status_counts(project["id"], task_state.value) for task_state in report_for_states } call.result.data = {"projects": projects}
def make_public(call: APICall, company_id, request: MakePublicRequest): with translate_errors_context(): call.result.data = Project.set_public(company_id, ids=request.ids, invalid_cls=InvalidProjectId, enabled=False)