def start_sender(cls):
if not cls.supported:
return
url = config.get("apiserver.statistics.url")
retries = config.get("apiserver.statistics.max_retries", 5)
max_backoff = config.get("apiserver.statistics.max_backoff_sec", 5)
session = requests.Session()
adapter = HTTPAdapter(max_retries=Retry(retries))
session.mount("http://", adapter)
session.mount("https://", adapter)
session.headers["Content-type"] = "application/json"
WarningFilter.attach()
while not ThreadsManager.terminating:
try:
report = cls.send_queue.get()
# Set a random backoff factor each time we send a report
adapter.max_retries.backoff_factor = random.random(
) * max_backoff
session.post(url, data=dumps(report))
except Exception as ex:
pass
def generate():
scroll_id = None
batch_size = 1000
while True:
log_events, scroll_id, _ = event_bll.scroll_task_events(
task.get_index_company(),
task_id,
order="asc",
event_type=EventType.task_log,
batch_size=batch_size,
scroll_id=scroll_id,
)
if not log_events:
break
for ev in log_events:
ev["asctime"] = ev.pop("timestamp")
if is_json:
ev.pop("type")
ev.pop("task")
yield json.dumps(ev) + "\n"
else:
try:
yield line_format.format(**ev)
except KeyError as ex:
raise errors.bad_request.FieldsValueError(
"undefined placeholders in line format",
placeholders=[str(ex)],
)
if len(log_events) < batch_size:
break
def _import_entity(
cls,
f: IO[bytes],
full_name: str,
company_id: str,
user_id: str,
metadata: Mapping[str, Any],
) -> Optional[Sequence[Task]]:
cls_ = cls._get_entity_type(full_name)
print(f"Writing {cls_.__name__.lower()}s into database")
tasks = []
override_project_count = 0
data_upgrade_funcs: Mapping[Type, Callable] = {
cls.task_cls: cls._upgrade_task_data,
cls.model_cls: cls._upgrade_model_data,
}
for item in cls.json_lines(f):
upgrade_func = data_upgrade_funcs.get(cls_)
if upgrade_func:
item = json.dumps(upgrade_func(json.loads(item)))
doc = cls_.from_json(item, created=True)
if hasattr(doc, "user"):
doc.user = user_id
if hasattr(doc, "company"):
doc.company = company_id
if isinstance(doc, cls.project_cls):
override_project_name = metadata.get("project_name", None)
if override_project_name:
if override_project_count:
override_project_name = (
f"{override_project_name} {override_project_count + 1}"
)
override_project_count += 1
doc.name = override_project_name
doc.logo_url = metadata.get("logo_url", None)
doc.logo_blob = metadata.get("logo_blob", None)
cls_.objects(
company=company_id, name=doc.name, id__ne=doc.id
).update(
set__name=
f"{doc.name}_{datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')}"
)
doc.save()
if isinstance(doc, cls.task_cls):
tasks.append(doc)
cls.event_bll.delete_task_events(company_id,
doc.id,
allow_locked=True)
if tasks:
return tasks
def _write_update_file(
cls,
map_file: Path,
entities: dict,
created_files: Sequence[str],
metadata_hash: str,
):
map_file.write_text(
json.dumps(
dict(
files=created_files,
entities={
entity.id: cls._get_last_update_time(entity)
for entity in chain.from_iterable(entities.values())
},
metadata_hash=metadata_hash,
)))
def set_preferences(call: APICall, company_id, request: SetPreferencesRequest):
changes = request.preferences
def invalid_key(_, key, __):
if not isinstance(key, str):
return True
elif key.startswith("$") or "." in key:
raise errors.bad_request.FieldsValueError(
f"Key {key} is invalid. Keys cannot start with '$' or contain '.'."
)
return True
remap(changes, visit=invalid_key)
base_preferences = get_user_preferences(call, company_id)
new_preferences = deepcopy(base_preferences)
for key, value in changes.items():
try:
dpath.new(new_preferences, key, value, separator=".")
except Exception:
log.exception(
'invalid preferences update for user "{}": key=`%s`, value=`%s`',
key,
value,
)
raise errors.bad_request.InvalidPreferencesUpdate(key=key,
value=value)
if new_preferences == base_preferences:
updated, fields = 0, {}
else:
with translate_errors_context("updating user preferences"):
updated = User.objects(id=call.identity.user,
company=company_id).update(
upsert=False,
preferences=dumps(new_preferences))
return {
"updated": updated,
"fields": {
"preferences": new_preferences
} if updated else {},
}
def _export_task_events(cls, task: Task, base_filename: str,
writer: ZipFile, hash_) -> Sequence[str]:
artifacts = []
filename = f"{base_filename}_{task.id}{cls.events_file_suffix}.json"
print(f"Writing task events into {writer.filename}:{filename}")
with BytesIO() as f:
with cls.JsonLinesWriter(f) as w:
scroll_id = None
while True:
res = cls.event_bll.get_task_events(
company_id=task.company,
task_id=task.id,
event_type=EventType.all,
scroll_id=scroll_id,
)
if not res.events:
break
scroll_id = res.next_scroll_id
for event in res.events:
event_type = event.get("type")
if event_type == EventType.metrics_image.value:
url = cls._get_fixed_url(event.get("url"))
if url:
event["url"] = url
artifacts.append(url)
elif event_type == EventType.metrics_plot.value:
plot_str: str = event.get("plot_str", "")
for match in cls.img_source_regex.findall(
plot_str):
url = cls._get_fixed_url(match)
if match != url:
plot_str = plot_str.replace(match, url)
artifacts.append(url)
w.write(json.dumps(event))
data = f.getvalue()
hash_.update(data)
writer.writestr(filename, data)
return artifacts
def to_json(self: ModelBase):
return dumps(self.to_struct())
def export_to_zip(
cls,
filename: str,
experiments: Sequence[str] = None,
projects: Sequence[str] = None,
artifacts_path: str = None,
task_statuses: Sequence[str] = None,
tag_exported_entities: bool = False,
metadata: Mapping[str, Any] = None,
) -> Sequence[str]:
cls._init_entity_types()
if task_statuses and not set(task_statuses).issubset(
get_options(TaskStatus)):
raise ValueError("Invalid task statuses")
file = Path(filename)
entities = cls._resolve_entities(experiments=experiments,
projects=projects,
task_statuses=task_statuses)
hash_ = hashlib.md5()
if metadata:
meta_str = json.dumps(metadata)
hash_.update(meta_str.encode())
metadata_hash = hash_.hexdigest()
else:
meta_str, metadata_hash = "", ""
map_file = file.with_suffix(".map")
updated, old_files = cls._check_for_update(map_file,
entities=entities,
metadata_hash=metadata_hash)
if not updated:
print(f"There are no updates from the last export")
return old_files
for old in old_files:
old_path = Path(old)
if old_path.is_file():
old_path.unlink()
with ZipFile(file, **cls.zip_args) as zfile:
if metadata:
zfile.writestr(cls.metadata_filename, meta_str)
artifacts = cls._export(
zfile,
entities=entities,
hash_=hash_,
tag_entities=tag_exported_entities,
)
file_with_hash = file.with_name(
f"{file.stem}_{hash_.hexdigest()}{file.suffix}")
file.replace(file_with_hash)
created_files = [str(file_with_hash)]
artifacts = cls._filter_artifacts(artifacts)
if artifacts and artifacts_path and os.path.isdir(artifacts_path):
artifacts_file = file_with_hash.with_suffix(cls.artifacts_ext)
with ZipFile(artifacts_file, **cls.zip_args) as zfile:
cls._export_artifacts(zfile, artifacts, artifacts_path)
created_files.append(str(artifacts_file))
cls._write_update_file(
map_file,
entities=entities,
created_files=created_files,
metadata_hash=metadata_hash,
)
return created_files
def get_response(self,
include_stack: bool = False
) -> Tuple[Union[dict, str], str]:
"""
Get the response for this call.
:param include_stack: If True, stack trace stored in this call's result should
be included in the response (default is False)
:return: Response data (encoded according to self.content_type) and the data's content type
"""
def make_version_number(
version: PartialVersion) -> Union[None, float, str]:
"""
Client versions <=2.0 expect expect endpoint versions in float format, otherwise throwing an exception
"""
if version is None:
return None
if self.requested_endpoint_version < PartialVersion("2.1"):
return float(str(version))
return str(version)
if self.result.raw_data and not self.failed:
# endpoint returned raw data and no error was detected, return raw data, no fancy dicts
return self.result.raw_data, self.result.content_type
else:
res = {
"meta": {
"id": self.id,
"trx": self.trx,
"endpoint": {
"name":
self.endpoint_name,
"requested_version":
make_version_number(self.requested_endpoint_version),
"actual_version":
make_version_number(self.actual_endpoint_version),
},
"result_code": self.result.code,
"result_subcode": self.result.subcode,
"result_msg": self.result.msg,
"error_stack":
self.result.traceback if include_stack else None,
"error_data": self.result.error_data,
},
"data": self.result.data,
}
if self.content_type.lower() == JSON_CONTENT_TYPE:
try:
res = json.dumps(res)
except Exception as ex:
# JSON serialization may fail, probably problem with data or error_data so pop it and try again
if not (self.result.data or self.result.error_data):
raise
self.result.data = None
self.result.error_data = None
msg = "Error serializing response data: " + str(ex)
self.set_error_result(code=500,
subcode=0,
msg=msg,
include_stack=False)
return self.get_response()
return res, self.content_type
def as_json(self) -> str:
return json.dumps(self.to_dict(), indent=2)
def _import_entity(
cls,
f: IO[bytes],
full_name: str,
company_id: str,
user_id: str,
metadata: Mapping[str, Any],
) -> Optional[Sequence[Task]]:
cls_ = cls._get_entity_type(full_name)
print(f"Writing {cls_.__name__.lower()}s into database")
tasks = []
override_project_count = 0
for item in cls.json_lines(f):
if cls_ == cls.task_cls:
task_data = json.loads(item)
artifacts_path = ("execution", "artifacts")
artifacts = nested_get(task_data, artifacts_path)
if isinstance(artifacts, list):
nested_set(
task_data,
artifacts_path,
value={get_artifact_id(a): a
for a in artifacts},
)
item = json.dumps(task_data)
doc = cls_.from_json(item, created=True)
if hasattr(doc, "user"):
doc.user = user_id
if hasattr(doc, "company"):
doc.company = company_id
if isinstance(doc, cls.project_cls):
override_project_name = metadata.get("project_name", None)
if override_project_name:
if override_project_count:
override_project_name = (
f"{override_project_name} {override_project_count + 1}"
)
override_project_count += 1
doc.name = override_project_name
doc.logo_url = metadata.get("logo_url", None)
doc.logo_blob = metadata.get("logo_blob", None)
cls_.objects(
company=company_id, name=doc.name, id__ne=doc.id
).update(
set__name=
f"{doc.name}_{datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')}"
)
doc.save()
if isinstance(doc, cls.task_cls):
tasks.append(doc)
cls.event_bll.delete_task_events(company_id,
doc.id,
allow_locked=True)
if tasks:
return tasks
