def start_sender(cls): if not cls.supported: return url = config.get("apiserver.statistics.url") retries = config.get("apiserver.statistics.max_retries", 5) max_backoff = config.get("apiserver.statistics.max_backoff_sec", 5) session = requests.Session() adapter = HTTPAdapter(max_retries=Retry(retries)) session.mount("http://", adapter) session.mount("https://", adapter) session.headers["Content-type"] = "application/json" WarningFilter.attach() while not ThreadsManager.terminating: try: report = cls.send_queue.get() # Set a random backoff factor each time we send a report adapter.max_retries.backoff_factor = random.random( ) * max_backoff session.post(url, data=dumps(report)) except Exception as ex: pass
def generate(): scroll_id = None batch_size = 1000 while True: log_events, scroll_id, _ = event_bll.scroll_task_events( task.company, task_id, order="asc", event_type="log", batch_size=batch_size, scroll_id=scroll_id, ) if not log_events: break for ev in log_events: ev["asctime"] = ev.pop("timestamp") if is_json: ev.pop("type") ev.pop("task") yield json.dumps(ev) + "\n" else: try: yield line_format.format(**ev) except KeyError as ex: raise errors.bad_request.FieldsValueError( "undefined placeholders in line format", placeholders=[str(ex)], ) if len(log_events) < batch_size: break
def generate(): scroll_id = None batch_size = 1000 while True: log_events, scroll_id, _ = event_bll.scroll_task_events( company_id, task_id, order="asc", event_type="log", batch_size=batch_size, scroll_id=scroll_id) if not log_events: break for ev in log_events: ev['asctime'] = ev.pop('@timestamp') if is_json: ev.pop('type') ev.pop('task') yield json.dumps(ev) + '\n' else: try: yield line_format.format(**ev) except KeyError as ex: raise errors.bad_request.FieldsValueError( 'undefined placeholders in line format', placeholders=[str(ex)]) if len(log_events) < batch_size: break
def get_tags(self, company, include_system: bool = False, filter_: Sequence[str] = None) -> dict: """ Get tags and optionally system tags for the company Return the dictionary of tags per tags field name The function retrieves both cached values from Redis in one call and re calculates any of them if missing in Redis """ fields = [ self._tags_field, *([self._system_tags_field] if include_system else []), ] redis_keys = [ self._get_tags_cache_key(company, f, filter_) for f in fields ] cached = self.redis.mget(redis_keys) ret = {} for field, tag_data, key in zip(fields, cached, redis_keys): if tag_data is not None: tags = json.loads(tag_data) else: tags = list(self._get_tags_from_db(company, field, filter_)) self.redis.setex( key, time=self._tags_cache_expiration_seconds, value=json.dumps(tags), ) ret[field] = tags return ret
def _export_task_events( cls, task: Task, base_filename: str, writer: ZipFile, hash_ ) -> Sequence[str]: artifacts = [] filename = f"{base_filename}_{task.id}{cls.events_file_suffix}.json" print(f"Writing task events into {writer.filename}:{filename}") with BytesIO() as f: with cls.JsonLinesWriter(f) as w: scroll_id = None while True: res = cls.event_bll.get_task_events( task.company, task.id, scroll_id=scroll_id ) if not res.events: break scroll_id = res.next_scroll_id for event in res.events: if event.get("type") == "training_debug_image": url = cls._get_fixed_url(event.get("url")) if url: event["url"] = url artifacts.append(url) w.write(json.dumps(event)) data = f.getvalue() hash_.update(data) writer.writestr(filename, data) return artifacts
def _write_update_file(cls, map_file: Path, entities: dict, created_files: Sequence[str]): map_data = cls._MapData( files=created_files, entities={ entity.id: cls._get_last_update_time(entity) for entity in chain.from_iterable(entities.values()) }, ) map_file.write_text(json.dumps(attr.asdict(map_data)))
def get_response(self): def make_version_number(version): """ Client versions <=2.0 expect expect endpoint versions in float format, otherwise throwing an exception """ if version is None: return None if self.requested_endpoint_version < PartialVersion("2.1"): return float(str(version)) return str(version) if self.result.raw_data and not self.failed: # endpoint returned raw data and no error was detected, return raw data, no fancy dicts return self.result.raw_data, self.result.content_type else: res = { "meta": { "id": self.id, "trx": self.trx, "endpoint": { "name": self.endpoint_name, "requested_version": make_version_number(self.requested_endpoint_version), "actual_version": make_version_number(self.actual_endpoint_version), }, "result_code": self.result.code, "result_subcode": self.result.subcode, "result_msg": self.result.msg, "error_stack": self.result.traceback, }, "data": self.result.data, } if self.content_type.lower() == JSON_CONTENT_TYPE: with TimingContext("json", "serialization"): try: res = json.dumps(res) except Exception as ex: # JSON serialization may fail, probably problem with data so pop it and try again if not self.result.data: raise self.result.data = None msg = "Error serializing response data: " + str(ex) self.set_error_result(code=500, subcode=0, msg=msg, include_stack=False) return self.get_response() return res, self.content_type
def set_preferences(call, company_id, req_model): # type: (APICall, str, SetPreferencesRequest) -> Dict assert isinstance(call, APICall) changes = req_model.preferences def invalid_key(_, key, __): if not isinstance(key, str): return True elif key.startswith("$") or "." in key: raise errors.bad_request.FieldsValueError( f"Key {key} is invalid. Keys cannot start with '$' or contain '.'." ) return True remap(changes, visit=invalid_key) base_preferences = get_user_preferences(call) new_preferences = deepcopy(base_preferences) for key, value in changes.items(): try: dpath.new(new_preferences, key, value, separator=".") except Exception: log.exception( 'invalid preferences update for user "{}": key=`%s`, value=`%s`', key, value, ) raise errors.bad_request.InvalidPreferencesUpdate(key=key, value=value) if new_preferences == base_preferences: updated, fields = 0, {} else: with translate_errors_context("updating user preferences"): updated = User.objects(id=call.identity.user, company=company_id).update( upsert=False, preferences=dumps(new_preferences)) return { "updated": updated, "fields": { "preferences": new_preferences } if updated else {}, }
def _write_update_file( cls, map_file: Path, entities: dict, created_files: Sequence[str], metadata_hash: str, ): map_file.write_text( json.dumps( dict( files=created_files, entities={ entity.id: cls._get_last_update_time(entity) for entity in chain.from_iterable(entities.values()) }, metadata_hash=metadata_hash, ) ) )
def to_json(self: ModelBase): return dumps(self.to_struct())
def export_to_zip( cls, filename: str, experiments: Sequence[str] = None, projects: Sequence[str] = None, artifacts_path: str = None, task_statuses: Sequence[str] = None, tag_exported_entities: bool = False, metadata: Mapping[str, Any] = None, ) -> Sequence[str]: if task_statuses and not set(task_statuses).issubset(get_options(TaskStatus)): raise ValueError("Invalid task statuses") file = Path(filename) entities = cls._resolve_entities( experiments=experiments, projects=projects, task_statuses=task_statuses ) hash_ = hashlib.md5() if metadata: meta_str = json.dumps(metadata) hash_.update(meta_str.encode()) metadata_hash = hash_.hexdigest() else: meta_str, metadata_hash = "", "" map_file = file.with_suffix(".map") updated, old_files = cls._check_for_update( map_file, entities=entities, metadata_hash=metadata_hash ) if not updated: print(f"There are no updates from the last export") return old_files for old in old_files: old_path = Path(old) if old_path.is_file(): old_path.unlink() with ZipFile(file, **cls.zip_args) as zfile: if metadata: zfile.writestr(cls.metadata_filename, meta_str) artifacts = cls._export( zfile, entities=entities, hash_=hash_, tag_entities=tag_exported_entities, ) file_with_hash = file.with_name(f"{file.stem}_{hash_.hexdigest()}{file.suffix}") file.replace(file_with_hash) created_files = [str(file_with_hash)] artifacts = cls._filter_artifacts(artifacts) if artifacts and artifacts_path and os.path.isdir(artifacts_path): artifacts_file = file_with_hash.with_suffix(cls.artifacts_ext) with ZipFile(artifacts_file, **cls.zip_args) as zfile: cls._export_artifacts(zfile, artifacts, artifacts_path) created_files.append(str(artifacts_file)) cls._write_update_file( map_file, entities=entities, created_files=created_files, metadata_hash=metadata_hash, ) return created_files
def to_json(self): return dumps(self.to_struct())