def __init__(self, title):
self.group_name = ''
self.tag = ''
self.title = title
assert title != ''
# experiment config
self.project = mlogger.getEnv().dashboard.project
self.experiment_name = mlogger.getEnv().dashboard.experiment_name
self.experiment_uuid = mlogger.getEnv().dashboard.experiment_uuid
self.folder = ''
if self.experiment_uuid is None:
return
def handle_exception(exc_type, exc_value, exc_traceback):
global __exit_flag
__exit_flag = True
if mlogger.getEnv() is None or mlogger.getEnv().dashboard is None:
return
project = mlogger.getEnv().dashboard.project
experiment = mlogger.getEnv().dashboard.experiment_name
if project is not None and experiment is not None:
# error
mlogger.error()
# print error info
print(exc_traceback)
# print log
print('error {}/{} experiment logger'.format(project, experiment))
def handle_exit():
global __exit_flag
if not __exit_flag:
# project/experiment
if mlogger.getEnv() is None or mlogger.getEnv().dashboard is None:
return
project = mlogger.getEnv().dashboard.project
experiment = mlogger.getEnv().dashboard.experiment_name
if project is not None and experiment is not None:
# exit
mlogger.exit()
# print log
print('finish {}/{} experiment logger'.format(project, experiment))
def get(self):
# 文件下载
# 1.step 获取日志文件
tag_str = self.tag if self.group_name == '' else '%s/%s' % (
self.group_name, self.tag)
response = \
mlogger.getEnv().dashboard.\
rpc.experiment.file.get(experiment_uuid=self.experiment_uuid,
key='{}/{}'.format(tag_str, self.title))
if response['status'] == 'ERROR':
logging.error('couldnt get log file path')
return
log_file = response['content']['path']
log_name = log_file.split('/')[-1]
tmpSecretId = response['content']['tmpSecretId']
tmpSecretKey = response['content']['tmpSecretKey']
sessionToken = response['content']['sessionToken']
region = response['content']['region']
bucket = response['content']['bucket']
# 2.step 下载日志文件
try:
config = CosConfig(Region=region,
SecretId=tmpSecretId,
SecretKey=tmpSecretKey,
Token=sessionToken,
Scheme='https')
client = CosS3Client(config)
response = client.get_object(Bucket=bucket, Key=log_file)
file_content = response['Body'].get_raw_stream()
with open('./{}'.format(log_name), 'wb') as fp:
fp.write(file_content.read())
except:
logging.error('download {} error'.format(log_file))
def __init__(self, plot_title, **kwargs):
super(Simple, self).__init__(plot_title, **kwargs)
self.chart_x_axis = 'time'
self.chart_y_axis = ''
self.channel = mlogger.getEnv().create_channel(str(uuid.uuid4()),
channel_type='LINE',
**self.channel_config)
def get(self, kind='current'):
assert (kind in ['current', 'history'])
# 1.step 优先使用本地数据
if self.value is not None and kind == 'current':
return {'value': self.value, 'type': 'current'}
# 2.step 使用远程数据
channel_name = self.channel.channel_name
chart_name = self.chart.chart_title
data = \
mlogger.getEnv().dashboard.\
rpc.experiment.chart.get(chart_name=chart_name,
channel_name=channel_name)
if data['status'] == 'OK':
return {'value': self._get(data['content']), 'type': 'history'}
return None
def name(self, val):
self.channel.channel_name = val
group_name = ''
if '/' in val:
group_name, _ = val.split('/')
if self.chart is None:
group_plot_title = self.plot_title
if group_name != '':
group_plot_title = '{}/{}'.format(group_name, self.plot_title)
if group_plot_title not in Base.charts:
Base.charts[group_plot_title] = mlogger.getEnv().create_chart(
[], group_plot_title)
self.chart = Base.charts[group_plot_title]
self.chart.chart_x_axis = self.chart_x_axis
self.chart.chart_y_axis = self.chart_y_axis
self.chart.bind_channel(self.channel)
def __init__(self, plot_title, **kwargs):
super(Heatmap, self).__init__(plot_title, 'complex', **kwargs)
self.channel = mlogger.getEnv().create_channel(str(uuid.uuid4()),
channel_type='HEATMAP',
**self.channel_config)
def __init__(self, plot_title, **kwargs):
super(Histogram, self).__init__(plot_title, 'complex', **kwargs)
self.channel = mlogger.getEnv().create_channel(
str(uuid.uuid4()), channel_type='HISTOGRAM', **self.channel_config)
def __init__(self, plot_title, **kwargs):
super(Scatter, self).__init__(plot_title, 'complex', **kwargs)
self.channel = mlogger.getEnv().create_channel(str(uuid.uuid4()),
channel_type='SCATTER',
time_series=False,
**self.channel_config)
def update(self, *args, **kwargs):
# 支持WEBDAV, MLTALKER
# antvis/experiment_uuid/group/%s_(latest_%d).xxx
if len(args) == 0:
return
local_file_path = os.path.normpath(args[0])
if not os.path.exists(local_file_path):
return
# 扩展名
ext_name = local_file_path.split('.')[-1]
# 文件名
file_name = local_file_path.split('/')[-1]
# 单位M
fsize = os.path.getsize(local_file_path)
fsize = fsize / float(1024 * 1024)
fsize = round(fsize, 2)
# 1.step 获得临时秘钥
response = \
mlogger.getEnv().dashboard.rpc.cos.experiment.get(file_size=fsize,
operator='upload')
if response['status'] == 'ERROR':
logging.error('could get cos credential')
return
tmpSecretId = response['content']['tmpSecretId']
tmpSecretKey = response['content']['tmpSecretKey']
sessionToken = response['content']['sessionToken']
region = response['content']['region']
bucket = response['content']['bucket']
# 2.step 上传文件
tag_str = self.tag if self.group_name == '' else '%s/%s' % (
self.group_name, self.tag)
try:
config = CosConfig(Region=region,
SecretId=tmpSecretId,
SecretKey=tmpSecretKey,
Token=sessionToken,
Scheme='https')
client = CosS3Client(config)
with open(local_file_path, 'rb') as fp:
client.put_object(Bucket=bucket,
Body=fp,
Key='{}/{}/{}/{}'.format(
self.experiment_uuid, tag_str,
self.title, file_name),
StorageClass='STANDARD',
EnableMD5=False)
except:
logging.error('upload {} error'.format(local_file_path))
return
# 3.step 更新平台记录
mlogger.getEnv().dashboard.experiment.patch(
**{
'experiment_name':
self.experiment_name,
'experiment_uuid':
self.experiment_uuid,
'experiment_stage':
mlogger.getEnv().dashboard.experiment_stage,
'experiment_data':
json.dumps({
'FILE_ABSTRACT': {
'group':
tag_str,
'title':
self.title,
'backend':
'ANTVIS',
'path':
'{}/{}/{}/{}'.format(self.experiment_uuid, tag_str,
self.title, file_name),
'size':
fsize,
},
'APP_STAGE':
mlogger.getEnv().dashboard.experiment_stage
})
})