def _is_valid_json(text: str) -> bool: """Check str for valid json""" if not text: return False try: loads(text) except Exception: return False return True
def _update_call_data(self, call, req): """ Use request payload/form to fill call data or batched data """ if req.content_type == "application/json-lines": items = [] for i, line in enumerate(req.data.splitlines()): try: event = json.loads(line) if not isinstance(event, dict): raise BadRequest( f"json lines must contain objects, found: {type(event).__name__}" ) items.append(event) except ValueError as e: msg = f"{e} in batch item #{i}" req.on_json_loading_failed(msg) call.batched_data = items else: json_body = req.get_json(force=True, silent=False) if req.data else None # merge form and args form = req.form.copy() form.update(req.args) form = form.to_dict() # convert string numbers to floats for key in form: if form[key].replace(".", "", 1).isdigit(): if "." in form[key]: form[key] = float(form[key]) else: form[key] = int(form[key]) elif form[key].lower() == "true": form[key] = True elif form[key].lower() == "false": form[key] = False call.data = json_body or form or {}
def _check_for_update(cls, map_file: Path, entities: dict, metadata_hash: str) -> Tuple[bool, Sequence[str]]: if not map_file.is_file(): return True, [] files = [] try: map_data = json.loads(map_file.read_text()) files = map_data.get("files", []) for file in files: if not Path(file).is_file(): return True, files new_times = { item.id: cls._get_last_update_time(item).replace(tzinfo=timezone.utc) for item in chain.from_iterable(entities.values()) } old_times = map_data.get("entities", {}) if set(new_times.keys()) != set(old_times.keys()): return True, files for id_, new_timestamp in new_times.items(): if new_timestamp != old_times[id_]: return True, files if metadata_hash != map_data.get("metadata_hash", ""): return True, files except Exception as ex: print("Error reading map file. " + str(ex)) return True, files return False, files
def get_user_preferences(call: APICall, company_id): user_id = call.identity.user preferences = get_user(call, company_id, user_id, only=["preferences"]).get("preferences") if preferences and isinstance(preferences, str): preferences = loads(preferences) return preferences or {}
def _import_events(cls, f: IO[bytes], full_name: str, company_id: str, _): _, _, task_id = full_name[0 : -len(cls.events_file_suffix)].rpartition("_") print(f"Writing events for task {task_id} into database") for events_chunk in chunked_iter(cls.json_lines(f), 1000): events = [json.loads(item) for item in events_chunk] cls.event_bll.add_events( company_id, events=events, worker="", allow_locked_tasks=True )
def _import_entity( cls, f: IO[bytes], full_name: str, company_id: str, user_id: str, metadata: Mapping[str, Any], ) -> Optional[Sequence[Task]]: cls_ = cls._get_entity_type(full_name) print(f"Writing {cls_.__name__.lower()}s into database") tasks = [] override_project_count = 0 data_upgrade_funcs: Mapping[Type, Callable] = { cls.task_cls: cls._upgrade_task_data, cls.model_cls: cls._upgrade_model_data, } for item in cls.json_lines(f): upgrade_func = data_upgrade_funcs.get(cls_) if upgrade_func: item = json.dumps(upgrade_func(json.loads(item))) doc = cls_.from_json(item, created=True) if hasattr(doc, "user"): doc.user = user_id if hasattr(doc, "company"): doc.company = company_id if isinstance(doc, cls.project_cls): override_project_name = metadata.get("project_name", None) if override_project_name: if override_project_count: override_project_name = ( f"{override_project_name} {override_project_count + 1}" ) override_project_count += 1 doc.name = override_project_name doc.logo_url = metadata.get("logo_url", None) doc.logo_blob = metadata.get("logo_blob", None) cls_.objects( company=company_id, name=doc.name, id__ne=doc.id ).update( set__name= f"{doc.name}_{datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')}" ) doc.save() if isinstance(doc, cls.task_cls): tasks.append(doc) cls.event_bll.delete_task_events(company_id, doc.id, allow_locked=True) if tasks: return tasks
def import_from_zip( cls, filename: str, artifacts_path: str, company_id: Optional[str] = None, user_id: str = "", user_name: str = "", ): cls._init_entity_types() metadata = None with ZipFile(filename) as zfile: try: with zfile.open(cls.metadata_filename) as f: metadata = json.loads(f.read()) meta_public = metadata.get("public") if company_id is None and meta_public is not None: company_id = "" if meta_public else get_default_company( ) if not user_id: meta_user_id = metadata.get("user_id", "") meta_user_name = metadata.get("user_name", "") user_id, user_name = meta_user_id, meta_user_name except Exception: pass if not user_id: user_id, user_name = "__allegroai__", "Allegro.ai" # Make sure we won't end up with an invalid company ID if company_id is None: company_id = "" existing_user = cls.user_cls.objects(id=user_id).only("id").first() if not existing_user: cls.user_cls(id=user_id, name=user_name, company=company_id).save() cls._import(zfile, company_id, user_id, metadata) if artifacts_path and os.path.isdir(artifacts_path): artifacts_file = Path(filename).with_suffix(cls.artifacts_ext) if artifacts_file.is_file(): print(f"Unzipping artifacts into {artifacts_path}") with ZipFile(artifacts_file) as zfile: zfile.extractall(artifacts_path)
def _update_call_data(self, call, req): """ Use request payload/form to fill call data or batched data """ if req.content_type == "application/json-lines": items = [] for i, line in enumerate(req.data.splitlines()): try: event = json.loads(line) if not isinstance(event, dict): raise BadRequest( f"json lines must contain objects, found: {type(event).__name__}" ) items.append(event) except ValueError as e: msg = f"{e} in batch item #{i}" req.on_json_loading_failed(msg) call.batched_data = items else: body = (req.get_json(force=True, silent=False) if req.data else None) or {} if req.args: self._apply_multi_dict(body, req.args) if req.form: self._apply_multi_dict(body, req.form) call.data = body
def from_json(cls: Type[ModelBase], s): return cls(**loads(s))
def _import_entity( cls, f: IO[bytes], full_name: str, company_id: str, user_id: str, metadata: Mapping[str, Any], ) -> Optional[Sequence[Task]]: cls_ = cls._get_entity_type(full_name) print(f"Writing {cls_.__name__.lower()}s into database") tasks = [] override_project_count = 0 for item in cls.json_lines(f): if cls_ == cls.task_cls: task_data = json.loads(item) artifacts_path = ("execution", "artifacts") artifacts = nested_get(task_data, artifacts_path) if isinstance(artifacts, list): nested_set( task_data, artifacts_path, value={get_artifact_id(a): a for a in artifacts}, ) item = json.dumps(task_data) doc = cls_.from_json(item, created=True) if hasattr(doc, "user"): doc.user = user_id if hasattr(doc, "company"): doc.company = company_id if isinstance(doc, cls.project_cls): override_project_name = metadata.get("project_name", None) if override_project_name: if override_project_count: override_project_name = ( f"{override_project_name} {override_project_count + 1}" ) override_project_count += 1 doc.name = override_project_name doc.logo_url = metadata.get("logo_url", None) doc.logo_blob = metadata.get("logo_blob", None) cls_.objects( company=company_id, name=doc.name, id__ne=doc.id ).update( set__name= f"{doc.name}_{datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')}" ) doc.save() if isinstance(doc, cls.task_cls): tasks.append(doc) cls.event_bll.delete_task_events(company_id, doc.id, allow_locked=True) if tasks: return tasks