def start_sender(cls): if not cls.supported: return url = config.get("apiserver.statistics.url") retries = config.get("apiserver.statistics.max_retries", 5) max_backoff = config.get("apiserver.statistics.max_backoff_sec", 5) session = requests.Session() adapter = HTTPAdapter(max_retries=Retry(retries)) session.mount("http://", adapter) session.mount("https://", adapter) session.headers["Content-type"] = "application/json" WarningFilter.attach() while not ThreadsManager.terminating: try: report = cls.send_queue.get() # Set a random backoff factor each time we send a report adapter.max_retries.backoff_factor = random.random( ) * max_backoff session.post(url, data=dumps(report)) except Exception as ex: pass
def generate(): scroll_id = None batch_size = 1000 while True: log_events, scroll_id, _ = event_bll.scroll_task_events( task.get_index_company(), task_id, order="asc", event_type=EventType.task_log, batch_size=batch_size, scroll_id=scroll_id, ) if not log_events: break for ev in log_events: ev["asctime"] = ev.pop("timestamp") if is_json: ev.pop("type") ev.pop("task") yield json.dumps(ev) + "\n" else: try: yield line_format.format(**ev) except KeyError as ex: raise errors.bad_request.FieldsValueError( "undefined placeholders in line format", placeholders=[str(ex)], ) if len(log_events) < batch_size: break
def _import_entity( cls, f: IO[bytes], full_name: str, company_id: str, user_id: str, metadata: Mapping[str, Any], ) -> Optional[Sequence[Task]]: cls_ = cls._get_entity_type(full_name) print(f"Writing {cls_.__name__.lower()}s into database") tasks = [] override_project_count = 0 data_upgrade_funcs: Mapping[Type, Callable] = { cls.task_cls: cls._upgrade_task_data, cls.model_cls: cls._upgrade_model_data, } for item in cls.json_lines(f): upgrade_func = data_upgrade_funcs.get(cls_) if upgrade_func: item = json.dumps(upgrade_func(json.loads(item))) doc = cls_.from_json(item, created=True) if hasattr(doc, "user"): doc.user = user_id if hasattr(doc, "company"): doc.company = company_id if isinstance(doc, cls.project_cls): override_project_name = metadata.get("project_name", None) if override_project_name: if override_project_count: override_project_name = ( f"{override_project_name} {override_project_count + 1}" ) override_project_count += 1 doc.name = override_project_name doc.logo_url = metadata.get("logo_url", None) doc.logo_blob = metadata.get("logo_blob", None) cls_.objects( company=company_id, name=doc.name, id__ne=doc.id ).update( set__name= f"{doc.name}_{datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')}" ) doc.save() if isinstance(doc, cls.task_cls): tasks.append(doc) cls.event_bll.delete_task_events(company_id, doc.id, allow_locked=True) if tasks: return tasks
def _write_update_file( cls, map_file: Path, entities: dict, created_files: Sequence[str], metadata_hash: str, ): map_file.write_text( json.dumps( dict( files=created_files, entities={ entity.id: cls._get_last_update_time(entity) for entity in chain.from_iterable(entities.values()) }, metadata_hash=metadata_hash, )))
def set_preferences(call: APICall, company_id, request: SetPreferencesRequest): changes = request.preferences def invalid_key(_, key, __): if not isinstance(key, str): return True elif key.startswith("$") or "." in key: raise errors.bad_request.FieldsValueError( f"Key {key} is invalid. Keys cannot start with '$' or contain '.'." ) return True remap(changes, visit=invalid_key) base_preferences = get_user_preferences(call, company_id) new_preferences = deepcopy(base_preferences) for key, value in changes.items(): try: dpath.new(new_preferences, key, value, separator=".") except Exception: log.exception( 'invalid preferences update for user "{}": key=`%s`, value=`%s`', key, value, ) raise errors.bad_request.InvalidPreferencesUpdate(key=key, value=value) if new_preferences == base_preferences: updated, fields = 0, {} else: with translate_errors_context("updating user preferences"): updated = User.objects(id=call.identity.user, company=company_id).update( upsert=False, preferences=dumps(new_preferences)) return { "updated": updated, "fields": { "preferences": new_preferences } if updated else {}, }
def _export_task_events(cls, task: Task, base_filename: str, writer: ZipFile, hash_) -> Sequence[str]: artifacts = [] filename = f"{base_filename}_{task.id}{cls.events_file_suffix}.json" print(f"Writing task events into {writer.filename}:{filename}") with BytesIO() as f: with cls.JsonLinesWriter(f) as w: scroll_id = None while True: res = cls.event_bll.get_task_events( company_id=task.company, task_id=task.id, event_type=EventType.all, scroll_id=scroll_id, ) if not res.events: break scroll_id = res.next_scroll_id for event in res.events: event_type = event.get("type") if event_type == EventType.metrics_image.value: url = cls._get_fixed_url(event.get("url")) if url: event["url"] = url artifacts.append(url) elif event_type == EventType.metrics_plot.value: plot_str: str = event.get("plot_str", "") for match in cls.img_source_regex.findall( plot_str): url = cls._get_fixed_url(match) if match != url: plot_str = plot_str.replace(match, url) artifacts.append(url) w.write(json.dumps(event)) data = f.getvalue() hash_.update(data) writer.writestr(filename, data) return artifacts
def to_json(self: ModelBase): return dumps(self.to_struct())
def export_to_zip( cls, filename: str, experiments: Sequence[str] = None, projects: Sequence[str] = None, artifacts_path: str = None, task_statuses: Sequence[str] = None, tag_exported_entities: bool = False, metadata: Mapping[str, Any] = None, ) -> Sequence[str]: cls._init_entity_types() if task_statuses and not set(task_statuses).issubset( get_options(TaskStatus)): raise ValueError("Invalid task statuses") file = Path(filename) entities = cls._resolve_entities(experiments=experiments, projects=projects, task_statuses=task_statuses) hash_ = hashlib.md5() if metadata: meta_str = json.dumps(metadata) hash_.update(meta_str.encode()) metadata_hash = hash_.hexdigest() else: meta_str, metadata_hash = "", "" map_file = file.with_suffix(".map") updated, old_files = cls._check_for_update(map_file, entities=entities, metadata_hash=metadata_hash) if not updated: print(f"There are no updates from the last export") return old_files for old in old_files: old_path = Path(old) if old_path.is_file(): old_path.unlink() with ZipFile(file, **cls.zip_args) as zfile: if metadata: zfile.writestr(cls.metadata_filename, meta_str) artifacts = cls._export( zfile, entities=entities, hash_=hash_, tag_entities=tag_exported_entities, ) file_with_hash = file.with_name( f"{file.stem}_{hash_.hexdigest()}{file.suffix}") file.replace(file_with_hash) created_files = [str(file_with_hash)] artifacts = cls._filter_artifacts(artifacts) if artifacts and artifacts_path and os.path.isdir(artifacts_path): artifacts_file = file_with_hash.with_suffix(cls.artifacts_ext) with ZipFile(artifacts_file, **cls.zip_args) as zfile: cls._export_artifacts(zfile, artifacts, artifacts_path) created_files.append(str(artifacts_file)) cls._write_update_file( map_file, entities=entities, created_files=created_files, metadata_hash=metadata_hash, ) return created_files
def get_response(self, include_stack: bool = False ) -> Tuple[Union[dict, str], str]: """ Get the response for this call. :param include_stack: If True, stack trace stored in this call's result should be included in the response (default is False) :return: Response data (encoded according to self.content_type) and the data's content type """ def make_version_number( version: PartialVersion) -> Union[None, float, str]: """ Client versions <=2.0 expect expect endpoint versions in float format, otherwise throwing an exception """ if version is None: return None if self.requested_endpoint_version < PartialVersion("2.1"): return float(str(version)) return str(version) if self.result.raw_data and not self.failed: # endpoint returned raw data and no error was detected, return raw data, no fancy dicts return self.result.raw_data, self.result.content_type else: res = { "meta": { "id": self.id, "trx": self.trx, "endpoint": { "name": self.endpoint_name, "requested_version": make_version_number(self.requested_endpoint_version), "actual_version": make_version_number(self.actual_endpoint_version), }, "result_code": self.result.code, "result_subcode": self.result.subcode, "result_msg": self.result.msg, "error_stack": self.result.traceback if include_stack else None, "error_data": self.result.error_data, }, "data": self.result.data, } if self.content_type.lower() == JSON_CONTENT_TYPE: try: res = json.dumps(res) except Exception as ex: # JSON serialization may fail, probably problem with data or error_data so pop it and try again if not (self.result.data or self.result.error_data): raise self.result.data = None self.result.error_data = None msg = "Error serializing response data: " + str(ex) self.set_error_result(code=500, subcode=0, msg=msg, include_stack=False) return self.get_response() return res, self.content_type
def as_json(self) -> str: return json.dumps(self.to_dict(), indent=2)
def _import_entity( cls, f: IO[bytes], full_name: str, company_id: str, user_id: str, metadata: Mapping[str, Any], ) -> Optional[Sequence[Task]]: cls_ = cls._get_entity_type(full_name) print(f"Writing {cls_.__name__.lower()}s into database") tasks = [] override_project_count = 0 for item in cls.json_lines(f): if cls_ == cls.task_cls: task_data = json.loads(item) artifacts_path = ("execution", "artifacts") artifacts = nested_get(task_data, artifacts_path) if isinstance(artifacts, list): nested_set( task_data, artifacts_path, value={get_artifact_id(a): a for a in artifacts}, ) item = json.dumps(task_data) doc = cls_.from_json(item, created=True) if hasattr(doc, "user"): doc.user = user_id if hasattr(doc, "company"): doc.company = company_id if isinstance(doc, cls.project_cls): override_project_name = metadata.get("project_name", None) if override_project_name: if override_project_count: override_project_name = ( f"{override_project_name} {override_project_count + 1}" ) override_project_count += 1 doc.name = override_project_name doc.logo_url = metadata.get("logo_url", None) doc.logo_blob = metadata.get("logo_blob", None) cls_.objects( company=company_id, name=doc.name, id__ne=doc.id ).update( set__name= f"{doc.name}_{datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')}" ) doc.save() if isinstance(doc, cls.task_cls): tasks.append(doc) cls.event_bll.delete_task_events(company_id, doc.id, allow_locked=True) if tasks: return tasks