def test_get_projects_with_given_namespace(self, _):
# given
session = Session(API_TOKEN)
# and
api_projects = [a_project(), a_project()]
# and
session._client.get_projects.return_value = api_projects
# and
custom_namespace = 'custom_namespace'
# when
projects = session.get_projects(custom_namespace)
# then
expected_projects = {
custom_namespace + '/' + p.name: Project(session._client, p.id,
custom_namespace, p.name)
for p in api_projects
}
self.assertEqual(expected_projects, projects)
# and
session._client.get_projects.assert_called_with(custom_namespace)
def test_get_projects_with_given_namespace(self, _):
# given
api_projects = [a_project(), a_project()]
# and
backend = MagicMock()
leaderboard = MagicMock()
backend.get_projects.return_value = api_projects
backend.create_leaderboard_backend.return_value = leaderboard
# and
session = Session(backend=backend)
# and
custom_namespace = "custom_namespace"
# when
projects = session.get_projects(custom_namespace)
# then
expected_projects = OrderedDict((
custom_namespace + "/" + p.name,
Project(leaderboard, p.id, custom_namespace, p.name),
) for p in api_projects)
self.assertEqual(expected_projects, projects)
# and
backend.get_projects.assert_called_with(custom_namespace)
def test_get_projects_with_given_namespace(self, _):
# given
api_projects = [a_project(), a_project()]
# and
backend = MagicMock()
backend.get_projects.return_value = api_projects
# and
session = Session(backend=backend)
# and
custom_namespace = 'custom_namespace'
# when
projects = session.get_projects(custom_namespace)
# then
expected_projects = OrderedDict(
(custom_namespace + '/' + p.name,
Project(backend, p.id, custom_namespace, p.name))
for p in api_projects)
self.assertEqual(expected_projects, projects)
# and
backend.get_projects.assert_called_with(custom_namespace)
class TelegramBot:
def __init__(self, telegram_api_token, neptune_api_token):
self.session = Session(api_token=neptune_api_token)
self.updater = Updater(token=telegram_api_token)
self.dispatcher = self.updater.dispatcher
self.neptune_project = None
self.project_name = None
self.dispatcher.add_handler(CommandHandler('project', self.project, pass_args=True))
self.dispatcher.add_handler(CommandHandler('experiments', self.experiments, pass_args=True))
self.dispatcher.add_handler(CommandHandler('experiment', self.experiment, pass_args=True))
self.dispatcher.add_handler(MessageHandler(Filters.command, self.unknown))
def run(self):
self.updater.start_polling()
def project(self, bot, update, args):
if args:
if args[0] == 'select':
self._project_select(bot, update, args)
elif args[0] == 'list':
self._project_list(bot, update, args)
else:
self._project_help(bot, update)
else:
self._project_help(bot, update)
def experiments(self, bot, update, args):
if not self.neptune_project:
self._no_project_selected(bot, update)
else:
if args:
if args[0] == 'last':
self._experiments_last(bot, update, args)
elif args[0] == 'best':
self._experiments_best(bot, update, args)
elif args[0] == 'state':
self._experiments_state(bot, update, args)
else:
self._experiments_help(bot, update)
else:
self._experiments_help(bot, update)
def experiment(self, bot, update, args):
if not self.neptune_project:
self._no_project_selected(bot, update)
else:
if args:
if args[0] == 'link':
self._experiment_link(bot, update, args)
elif args[0] == 'plot':
self._experiment_plot(bot, update, args)
else:
self._experiment_help(bot, update)
else:
self._experiment_help(bot, update)
def unknown(self, bot, update):
bot.send_message(chat_id=update.message.chat_id,
text="Sorry, I only undestand /project, /experiments, /experiment")
def _project_list(self, bot, update, args):
if len(args) != 2:
msg = ['message should have a format:',
'/project list NAMESPACE',
'for example:',
'/project list neptune-ai']
msg = '\n'.join(msg)
else:
namespace = args[1]
project_names = self.session.get_projects(namespace).keys()
msg = '\n'.join(project_names)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _project_select(self, bot, update, args):
if len(args) != 2:
msg = ['message should have a format:',
'/project select NAMESPACE/PROJECT_NAME',
'for example:',
'/project select neptune-ai/neptune-examples']
msg = '\n'.join(msg)
else:
self.project_name = args[1]
namespace = self.project_name.split('/')[0]
self.neptune_project = self.session.get_projects(namespace)[self.project_name]
msg = 'Selected a project: {}'.format(self.project_name)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _project_help(self, bot, update):
msg = """Available options are:\n
/project list NAMESPACE
/project select NAMESPACE/PROJECT_NAME
"""
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiments_last(self, bot, update, args):
if len(args) != 3:
msg = ['message should have a format:',
'/experiments last NR_EXPS TIMESTAMP',
'for example:',
'/experiments last 5 finished']
msg = '\n'.join(msg)
else:
nr_exps = int(args[1])
timestamp = args[2]
if timestamp not in ['created', 'finished']:
msg = 'choose created or finished as timestamp'
else:
leaderboard = self.neptune_project.get_leaderboard()
leaderboard[timestamp] = pd.to_datetime(leaderboard[timestamp])
leaderboard.sort_values(timestamp, ascending=False, inplace=True)
ids = leaderboard['id'].tolist()[:nr_exps]
ids = ['id'] + ids
msg = '\n'.join(ids)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiments_best(self, bot, update, args):
if len(args) != 3:
msg = ['message should have a format:',
'/experiments best METRIC NR_EXPS',
'for example:',
'/experiments best log_loss 3']
msg = '\n'.join(msg)
else:
metric_name = args[1]
metric_column = 'channel_' + metric_name
nr_exps = int(args[2])
leaderboard = self.neptune_project.get_leaderboard()
scores = leaderboard.sort_values([metric_column], ascending=False)[['id', metric_column]]
msg = 'id | {}\n'.format(metric_name)
for idx, metric in scores.values[:nr_exps]:
msg = msg + '{} | {}\n'.format(idx, metric)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiments_state(self, bot, update, args):
if len(args) != 3:
msg = ['message should have a format:',
'/experiments state STATE NR_EXPS',
'for example:',
'/experiments state running 4']
msg = '\n'.join(msg)
else:
state = args[1]
nr_exps = int(args[2])
leaderboard = self.neptune_project.get_leaderboard(state=state)
leaderboard['created'] = pd.to_datetime(leaderboard['created'])
leaderboard.sort_values('created', ascending=False, inplace=True)
ids = leaderboard['id'].tolist()[:nr_exps]
ids = ['id'] + ids
msg = '\n'.join(ids)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiments_help(self, bot, update):
msg = """Available options are:\n
/experiments last NR_EXPS
/experiments best METRIC_NAME NR_EXPS(optional)
/experiments state STATE NR_EXPS(optional)
"""
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiment_link(self, bot, update, args):
if len(args) != 2:
msg = ['message should have a format:',
'/experiment link SHORT_ID',
'for example:',
'/experiment link NEP-508']
msg = '\n'.join(msg)
else:
short_id = args[1]
namespace, project = self.project_name.split('/')
msg = 'https://ui.neptune.ai/o/{}/org/{}/e/{}/details'.format(namespace, project, short_id)
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _experiment_plot(self, bot, update, args):
if len(args) < 3:
msg = ['message should have a format:',
'/experiment plot SHORT_ID METRIC_NAME OTHER_METRIC_NAME',
'for example:',
'/experiment plot NEP-508 train_loss valid_loss']
msg = '\n'.join(msg)
bot.send_message(chat_id=update.message.chat_id, text=msg)
else:
short_id = args[1]
if len(args) == 2:
metric_names = [args[2]]
else:
metric_names = args[2:]
experiment = self.neptune_project.get_experiments(id=short_id)[0]
data = experiment.get_numeric_channels_values(*metric_names)
fig = plt.figure()
for channel_name in data.columns:
if channel_name != 'x':
plt.plot('x', channel_name, data=data,
marker='', linewidth=2, label=channel_name)
plt.legend()
buffer = BytesIO()
fig.savefig(buffer, format='png')
buffer.seek(0)
update.message.reply_photo(buffer)
def _experiment_help(self, bot, update):
msg = """Available options are:\n
/experiment link SHORT_ID
/experiments plot SHORT_ID METRIC_NAME (OTHER_METRIC_NAME)
"""
bot.send_message(chat_id=update.message.chat_id, text=msg)
def _no_project_selected(self, bot, update):
msg = ["You haven't selected your project.",
"Do so by running:\n"
"/project select NAMESPACE/PROJECT_NAME",
"For example:",
"/project select neptune-ai/neptune-examples"]
msg = '\n'.join(msg)
bot.send_message(chat_id=update.message.chat_id, text=msg)