def emit(self, record): # pylint:disable=inconsistent-return-statements
if settings.IS_MANAGED or not self.can_record(record):
return
try:
return self._send_logs(self.format_record(record))
except Exception:
logger.warning("Polyaxon failed creating log record")
def log_data_ref(self, data, data_name='data', reset=False):
try:
params = {data_name: hash_value(data)}
patch_dict = {'data_refs': params}
if reset is False:
patch_dict['merge'] = True
self._update(patch_dict)
except Exception as e:
logger.warning('Could create data hash %s', e)
def upload(self,
url,
files,
files_size,
params=None,
json_data=None,
timeout=None,
headers=None,
session=None):
if files_size > settings.WARN_UPLOAD_SIZE:
logger.warning(
"You are uploading %s, there's a hard limit of %s.\n"
"If you have data files in the current directory, "
"please make sure to add them to .polyaxonignore or "
"add them directly to your data volume, or upload them "
"separately using `polyaxon data` command and remove them from here.\n",
self.format_sizeof(settings.WARN_UPLOAD_SIZE),
self.format_sizeof(settings.MAX_UPLOAD_SIZE))
if files_size > settings.MAX_UPLOAD_SIZE:
raise PolyaxonShouldExitError(
"Files too large to sync, please keep it under {}.\n"
"If you have data files in the current directory, "
"please add them directly to your data volume, or upload them "
"separately using `polyaxon data` command and remove them from here.\n"
.format(self.format_sizeof(settings.MAX_UPLOAD_SIZE)))
files = to_list(files)
if json_data:
files.append(('json', json.dumps(json_data)))
multipart_encoder = MultipartEncoder(fields=files)
request_headers = headers or {}
request_headers.update(
{"Content-Type": multipart_encoder.content_type})
# Attach progress bar
progress_callback, callback_bar = self.create_progress_callback(
multipart_encoder)
multipart_encoder_monitor = MultipartEncoderMonitor(
multipart_encoder, progress_callback)
timeout = timeout if timeout is not None else settings.LONG_REQUEST_TIMEOUT
try:
response = self.put(url=url,
params=params,
data=multipart_encoder_monitor,
headers=request_headers,
timeout=timeout,
session=session)
finally:
# always make sure we clear the console
callback_bar.done()
return response
def log_data_ref(self, data, data_name='data', reset=False):
if settings.NO_OP:
return
try:
import hashlib
params = {
data_name: hashlib.md5(str(data).encode("utf-8")).hexdigest()[:settings.HASH_LENGTH]
}
patch_dict = {'data_refs': params}
if reset is False:
patch_dict['merge'] = True
self.client.experiment.update_experiment(username=self.username,
project_name=self.project_name,
experiment_id=self.experiment_id,
patch_dict=patch_dict,
background=True)
except Exception as e:
logger.warning('Could create data hash %s', e)