def __init__(self, notifiers: bool = True):
self.config = Config(config_file) if path.isfile(
config_file) else Config()
if self.config.debug:
# pylint: disable=E1103
loggers = [
logging.getLogger(name)
for name in logging.root.manager.loggerDict
]
# pylint: enable=E1103
for logger in loggers:
logger.setLevel(logging.DEBUG)
log.info("Debugging mode enabled")
self.metrics = Metrics()
self.item_ids = self.config.item_ids
self.amounts = {}
try:
self.tgtg_client = TgtgClient(
email=self.config.tgtg["username"],
timeout=self.config.tgtg["timeout"],
access_token_lifetime=self.config.
tgtg["access_token_lifetime"],
max_polling_tries=self.config.tgtg["max_polling_tries"],
polling_wait_time=self.config.tgtg["polling_wait_time"],
access_token=self.config.tgtg["access_token"],
refresh_token=self.config.tgtg["refresh_token"],
user_id=self.config.tgtg["user_id"])
self.tgtg_client.login()
self.config.save_tokens(self.tgtg_client.access_token,
self.tgtg_client.refresh_token,
self.tgtg_client.user_id)
except TgtgAPIError as err:
raise err
except Error as err:
log.error(err)
raise TGTGConfigurationError() from err
if notifiers:
if self.config.metrics:
self.metrics.enable_metrics()
self.notifiers = Notifiers(self.config)
if not self.config.disable_tests:
log.info("Sending test Notifications ...")
self.notifiers.send(self._test_item)
def loss(self, trainable_params: hk.Params,
non_trainable_params: hk.Params, rng_key: PRNGKey, batch: Batch,
training: int) -> Tuple[Array, Tuple[Metrics, ItoGeneralOutput]]:
params = hk.data_structures.merge(trainable_params,
non_trainable_params)
rng_key_list = random.split(rng_key, batch[0].shape[0])
metrics: Metrics
output: ItoGeneralOutput
metrics, output = \
jax.vmap(self.model.apply, (None, 0, 0, 0, None))(params, rng_key_list, batch[0], batch[1], training)
mean_metrics = Metrics(jnp.mean(metrics.elbo),
jnp.mean(metrics.elbo_generated),
jnp.mean(metrics.mse))
return -mean_metrics.elbo, (mean_metrics, output)
def save_metrics(self, dataframe):
session = self.manager.db.Session()
#TODO Id and other politics to define
session.execute("delete from Metrics where Name = '{}'".format(
dataframe['Name'][0]))
#NOTE This function is not generic AT ALL
metrics_object = [
Metrics(dataframe['Name'][i], dataframe['Period'][i],
dataframe['Sharpe.Ratio'][i], dataframe['Sortino.Ratio'],
dataframe['Information'], dataframe['Returns'][i],
dataframe['Max.Drawdown'][i], dataframe['Volatility'][i],
dataframe['Beta'][i], dataframe['Alpha'][i],
dataframe['Excess.Returns'][i],
dataframe['Benchmark.Returns'][i],
dataframe['Benchmark.Volatility'][i],
dataframe['Treasury.Returns'][i])
for i in range(len(dataframe.index))
]
session.add_all(metrics_object)
session.commit()
session.close()
class Scanner():
def __init__(self, notifiers: bool = True):
self.config = Config(config_file) if path.isfile(
config_file) else Config()
if self.config.debug:
# pylint: disable=E1103
loggers = [
logging.getLogger(name)
for name in logging.root.manager.loggerDict
]
# pylint: enable=E1103
for logger in loggers:
logger.setLevel(logging.DEBUG)
log.info("Debugging mode enabled")
self.metrics = Metrics()
self.item_ids = self.config.item_ids
self.amounts = {}
try:
self.tgtg_client = TgtgClient(
email=self.config.tgtg["username"],
timeout=self.config.tgtg["timeout"],
access_token_lifetime=self.config.
tgtg["access_token_lifetime"],
max_polling_tries=self.config.tgtg["max_polling_tries"],
polling_wait_time=self.config.tgtg["polling_wait_time"],
access_token=self.config.tgtg["access_token"],
refresh_token=self.config.tgtg["refresh_token"],
user_id=self.config.tgtg["user_id"])
self.tgtg_client.login()
self.config.save_tokens(self.tgtg_client.access_token,
self.tgtg_client.refresh_token,
self.tgtg_client.user_id)
except TgtgAPIError as err:
raise err
except Error as err:
log.error(err)
raise TGTGConfigurationError() from err
if notifiers:
if self.config.metrics:
self.metrics.enable_metrics()
self.notifiers = Notifiers(self.config)
if not self.config.disable_tests:
log.info("Sending test Notifications ...")
self.notifiers.send(self._test_item)
@property
def _test_item(self) -> Item:
"""
Returns an item for test notifications
"""
items = sorted(self._get_favorites(),
key=lambda x: x.items_available,
reverse=True)
if items:
return items[0]
items = sorted([
Item(item)
for item in self.tgtg_client.get_items(favorites_only=False,
latitude=53.5511,
longitude=9.9937,
radius=50)
],
key=lambda x: x.items_available,
reverse=True)
return items[0]
def _job(self) -> None:
"""
Job iterates over all monitored items
"""
for item_id in self.item_ids:
try:
if item_id != "":
item = Item(self.tgtg_client.get_item(item_id))
self._check_item(item)
except Exception:
log.error("itemID %s Error! - %s", item_id, sys.exc_info())
for item in self._get_favorites():
try:
self._check_item(item)
except Exception:
log.error("check item error! - %s", sys.exc_info())
log.debug("new State: %s", self.amounts)
if len(self.amounts) == 0:
log.warning("No items in observation! Did you add any favorites?")
self.config.save_tokens(self.tgtg_client.access_token,
self.tgtg_client.refresh_token,
self.tgtg_client.user_id)
def _get_favorites(self) -> list[Item]:
"""
Get favorites as list of Items
"""
items = []
page = 1
page_size = 100
error_count = 0
while error_count < 5:
try:
new_items = self.tgtg_client.get_items(favorites_only=True,
page_size=page_size,
page=page)
items += new_items
if len(new_items) < page_size:
break
page += 1
except Exception:
log.error("get item error! - %s", sys.exc_info())
error_count += 1
self.metrics.get_favorites_errors.inc()
return [Item(item) for item in items]
def _check_item(self, item: Item) -> None:
"""
Checks if the available item amount raised from zero to something and triggers notifications.
"""
try:
if self.amounts[
item.item_id] == 0 and item.items_available > self.amounts[
item.item_id]:
self._send_messages(item)
self.metrics.send_notifications.labels(
item.item_id, item.display_name).inc()
self.metrics.item_count.labels(
item.item_id, item.display_name).set(item.items_available)
except Exception:
self.amounts[item.item_id] = item.items_available
finally:
if self.amounts[item.item_id] != item.items_available:
log.info("%s - new amount: %s", item.display_name,
item.items_available)
self.amounts[item.item_id] = item.items_available
def _send_messages(self, item: Item) -> None:
"""
Send notifications for Item
"""
log.info("Sending notifications for %s - %s bags available",
item.display_name, item.items_available)
self.notifiers.send(item)
def run(self) -> NoReturn:
"""
Main Loop of the Scanner
"""
log.info("Scanner started ...")
while True:
try:
self._job()
if self.tgtg_client.captcha_error_count > 10:
log.warning("Too many 403 Errors. Sleeping for 1 hour.")
sleep(60 * 60)
log.info("Continuing scanning.")
self.tgtg_client.captcha_error_count = 0
except Exception:
log.error("Job Error! - %s", sys.exc_info())
finally:
sleep(self.config.sleep_time * (0.9 + 0.2 * random()))
def __del__(self) -> None:
"""
Cleanup on shutdown
"""
try:
if hasattr(self, 'notifiers') and self.notifiers.telegram.updater:
self.notifiers.telegram.updater.stop()
except Exception as exc:
log.warning(exc)
class Scanner():
def __init__(self, notifiers=True):
self.config = Config(config_file) if path.isfile(
config_file) else Config()
if self.config.debug:
# pylint: disable=E1103
loggers = [
logging.getLogger(name)
for name in logging.root.manager.loggerDict
]
# pylint: enable=E1103
for logger in loggers:
logger.setLevel(logging.DEBUG)
log.info("Debugging mode enabled")
self.metrics = Metrics()
if self.config.metrics:
self.metrics.enable_metrics()
self.item_ids = self.config.item_ids
self.amounts = {}
try:
self.tgtg_client = TgtgClient(
email=self.config.tgtg["username"],
timeout=self.config.tgtg["timeout"],
access_token_lifetime=self.config.
tgtg["access_token_lifetime"],
max_polling_tries=self.config.tgtg["max_polling_tries"],
polling_wait_time=self.config.tgtg["polling_wait_time"],
access_token=self.config.tgtg["access_token"],
refresh_token=self.config.tgtg["refresh_token"],
user_id=self.config.tgtg["user_id"])
self.tgtg_client.login()
except TgtgAPIError as err:
raise
except Error as err:
log.error(err)
raise TGTGConfigurationError() from err
if notifiers:
self.notifiers = Notifiers(self.config)
def _job(self):
for item_id in self.item_ids:
try:
if item_id != "":
data = self.tgtg_client.get_item(item_id)
self._check_item(Item(data))
except Exception:
log.error("itemID %s Error! - %s", item_id, sys.exc_info())
for data in self._get_favorites():
try:
self._check_item(Item(data))
except Exception:
log.error("check item error! - %s", sys.exc_info())
log.debug("new State: %s", self.amounts)
self.config.save_tokens(self.tgtg_client.access_token,
self.tgtg_client.refresh_token,
self.tgtg_client.user_id)
def _get_favorites(self):
items = []
page = 1
page_size = 100
error_count = 0
while True and error_count < 5:
try:
new_items = self.tgtg_client.get_items(favorites_only=True,
page_size=page_size,
page=page)
items += new_items
if len(new_items) < page_size:
break
page += 1
except Exception:
log.error("get item error! - %s", sys.exc_info())
error_count += 1
self.metrics.get_favorites_errors.inc()
return items
def _check_item(self, item: Item):
try:
if self.amounts[
item.item_id] == 0 and item.items_available > self.amounts[
item.item_id]:
self._send_messages(item)
self.metrics.send_notifications.labels(
item.item_id, item.display_name).inc()
self.metrics.item_count.labels(
item.item_id, item.display_name).set(item.items_available)
except Exception:
self.amounts[item.item_id] = item.items_available
finally:
if self.amounts[item.item_id] != item.items_available:
log.info("%s - new amount: %s", item.display_name,
item.items_available)
self.amounts[item.item_id] = item.items_available
def _send_messages(self, item: Item):
log.info("Sending notifications for %s - %s bags available",
item.display_name, item.items_available)
self.notifiers.send(item)
def run(self):
log.info("Scanner started ...")
while True:
try:
self._job()
except Exception:
log.error("Job Error! - %s", sys.exc_info())
finally:
sleep(self.config.sleep_time * (0.9 + 0.2 * random()))
def __del__(self):
try:
if self.notifiers.telegram.updater:
self.notifiers.telegram.updater.stop()
except:
pass
def __init__(self, team_names):
self.model = MetricsModel()
class Metrics:
def __init__(self, team_names):
self.model = MetricsModel()
def mk_bau_overview_figure(self):
df = self.model.sprint_bau_report_df()
df = df.sort_values(['team_name', 'start_date'])
team_names = df.team_name.unique()
plots = len(team_names)
cols = 3
rows, rem = divmod(plots, cols)
rows += rem
fig = make_subplots(rows=rows,
cols=cols,
subplot_titles=[name for name in team_names],
specs=[[{
'type': 'domain'
} for _ in range(cols)] for _ in range(rows)])
for i, (_, group) in enumerate(df.groupby("team_name")):
div, rem = divmod(i, cols)
fig.add_trace(_mk_sub_pie_trace(
group.bau_summary.explode("bau_summary").dropna()),
row=div + 1,
col=rem + 1)
fig.update_layout(legend_x=1,
legend_y=1,
margin=dict(t=0, b=0, r=0, l=0))
fig.update_traces(textinfo='percent', showlegend=True)
fig.update_layout(transition_duration=500)
return fig
@staticmethod
def add_sprint_delivery_traces(df, row, col, show_legend, fig):
fig.add_trace(_mk_sub_line_trace(df,
name='BAU %',
x_col='end_date',
y_col='bau_issues_percentage',
color='orange',
show_legend=show_legend),
row=row,
col=col)
fig.add_trace(_mk_sub_line_trace(
df,
name='Delivery %',
x_col='end_date',
y_col='roadmap_delivered_issues_percentage',
color=GOOD,
show_legend=show_legend),
row=row,
col=col)
fig.add_trace(_mk_sub_bar_trace(df,
name='Goal completed',
x_col='end_date',
y_col='goal_completed',
color='#90ee90',
show_legend=show_legend),
row=row,
col=col)
@staticmethod
def add_gauge_trace(df, row, col, fig):
fig.add_trace(mk_gauge_trace(df), row, col)
@staticmethod
def _gen_titles(df, cols):
teams = df['team_name'].unique()
for i in range(0, len(teams), cols):
yield list(teams[i:i + 3])
yield [None] * cols
@staticmethod
def _gen_specs(df, cols):
teams = df['team_name'].unique()
for _ in range(0, len(teams), cols):
yield [{}] * cols
yield [{'type': 'indicator'}] * cols
@staticmethod
def _gen_heights(df, cols):
teams = df['team_name'].unique()
for _ in range(0, len(teams), cols):
yield 0.7
yield 0.3
def mk_delivery_summary_figure(self):
df = self.model.sprint_performance_report_df()
df = df.sort_values(['team_name', 'start_date'])
cols = 3
titles = list(self._gen_titles(df, cols))
specs = list(self._gen_specs(df, cols))
heights = list(self._gen_heights(df, cols))
fig = make_subplots(rows=len(specs),
cols=cols,
subplot_titles=list(chain.from_iterable(titles)),
specs=specs,
row_heights=heights)
for row_base, groups_chunk in enumerate(chunk(df.groupby('team_name'),
cols),
start=1):
row_above = row_base + row_base - 1
row_below = row_above + 1
for col, (_, group_df) in enumerate(groups_chunk, start=1):
if row_above == 1 and col == 1:
show_legend = True
else:
show_legend = False
self.add_sprint_delivery_traces(group_df, row_above, col,
show_legend, fig)
self.add_gauge_trace(group_df, row_below, col, fig)
fig.update_layout(
legend_x=1,
legend_y=1,
height=600,
margin=dict(t=20, b=20, r=0, l=0),
)
fig.update_layout(transition_duration=500)
for ax in fig['layout']:
if ax.startswith('yaxis'):
fig['layout'][ax]['ticksuffix'] = '%'
return fig
def render(self):
return dbc.Col([
singleColRow(html.H2('KPIs')),
singleColRow(dcc.Graph(figure=self.mk_delivery_summary_figure())),
singleColRow(html.H2('BAU overview')),
singleColRow(dcc.Graph(figure=self.mk_bau_overview_figure()))
])