def __init__(self, dbman, log, accountman, initialcmd, colourmode):
"""
:param dbman: :class:`~alot.db.DBManager`
:param log: :class:`logging.Logger`
:param accountman: :class:`~alot.account.AccountManager`
:param initialcmd: commandline applied after setting up interface
:type initialcmd: str
:param colourmode: determines which theme to chose
:type colourmode: int in [1,16,256]
"""
self.dbman = dbman
self.dbman.ui = self # register ui with dbman
self.logger = log
self.accountman = accountman
if not colourmode:
colourmode = config.getint('general', 'colourmode')
self.logger.info('setup gui in %d colours' % colourmode)
self.mainframe = urwid.Frame(urwid.SolidFill())
self.inputwrap = InputWrap(self, self.mainframe)
self.mainloop = urwid.MainLoop(self.inputwrap,
config.get_palette(),
handle_mouse=False,
event_loop=urwid.TwistedEventLoop(),
unhandled_input=self.unhandeled_input)
self.mainloop.screen.set_terminal_properties(colors=colourmode)
self.show_statusbar = config.getboolean('general', 'show_statusbar')
self.notificationbar = None
self.mode = 'global'
self.commandprompthistory = []
self.logger.debug('fire first command')
self.apply_command(initialcmd)
self.mainloop.run()
def __init__(self):
self.mhashes = {} # metric -> metric hash
#
self.hash_width = config.getint("pm_storage", "hash_width")
self.key_mask = "!%dsL" % self.hash_width
# Set key-value store class
kvtype = config.get("pm_storage", "type")
logger.info("Initializing %s storage. %d-byte wide hash", kvtype,
self.hash_width)
self.kvcls = load_name("noc.pm.db.storage", kvtype, KVStorage)
if not self.kvcls:
logger.critical("Invalid storage type: '%s'", kvtype)
raise ValueError("Invalid storage type: '%s'" % kvtype)
# Set partitioning scheme
ps = config.get("pm_storage", "partition")
logger.info("Setting %s partitioning scheme", ps)
self.partition = load_name("noc.pm.db.partition", ps, Partition)
# Index collection
self.metrics = get_db()["noc.ts.metrics"]
self.metrics_batch = self.metrics.initialize_ordered_bulk_op()
self.new_metrics = 0
self.flush_lock = threading.Lock()
self.epoch = int(
time.mktime(
time.strptime(config.get("pm_storage", "epoch"), "%Y-%m-%d")))
self.zero_hash = Binary("\x00" * self.hash_width)
def __init__(self, dbman, log, accountman, initialquery, colourmode):
self.dbman = dbman
self.dbman.ui = self # register ui with dbman
self.logger = log
self.accountman = accountman
if not colourmode:
colourmode = config.getint('general', 'colourmode')
self.logger.info('setup gui in %d colours' % colourmode)
self.mainframe = MainWidget(self)
self.mainloop = urwid.MainLoop(self.mainframe,
config.get_palette(),
handle_mouse=False,
unhandled_input=self.keypress)
self.mainloop.screen.set_terminal_properties(colors=colourmode)
self.show_statusbar = config.getboolean('general', 'show_statusbar')
self.notificationbar = None
self.mode = ''
self.commandprompthistory = []
self.logger.debug('setup bindings')
cmd = commandfactory('search', query=initialquery)
self.apply_command(cmd)
self.mainloop.run()
def starter(bot):
for group in config.sections()[1:]:
poster.updater(bot, group, config.getint(group, 'last_id'))
log.info('Очистка кэша')
for data in listdir('.'):
remove(data)
log.info('Переход в режим сна')
def init():
from settings import config
host = config.get('restAPI', 'host', fallback='0.0.0.0')
port = config.getint('restAPI', 'port', fallback=5000)
debug = config.getboolean('restAPI', 'debug', fallback=False)
app.debug = debug
global server
server = WebSocketServer((host, port), app, handler_class=WebSocketHandler)
server.serve_forever()
def notify(self, message, priority='normal', timeout=0, block=False):
"""
opens notification popup
:param message: message to print
:type message: str
:param priority: priority string, used to format the popup: currently,
'normal' and 'error' are defined. If you use 'X' here,
the attribute 'global_notify_X' is used to format the
popup.
:type priority: str
:param timeout: seconds until message disappears. Defaults to the value
of 'notify_timeout' in the general config section.
A negative value means never time out.
:type timeout: int
:param block: this notification blocks until a keypress is made
:type block: bool
:returns: an urwid widget (this notification) that can be handed to
:meth:`clear_notify` for removal
"""
def build_line(msg, prio):
cols = urwid.Columns([urwid.Text(msg)])
return urwid.AttrMap(cols, 'global_notify_' + prio)
msgs = [build_line(message, priority)]
if not self.notificationbar:
self.notificationbar = urwid.Pile(msgs)
else:
newpile = self.notificationbar.widget_list + msgs
self.notificationbar = urwid.Pile(newpile)
self.update()
def clear(*args):
self.clear_notify(msgs)
if block:
# put "cancel to continue" widget as overlay on main widget
txt = urwid.Text('(cancel continues)')
overlay = urwid.Overlay(txt, self.mainframe,
('fixed left', 0),
('fixed right', 0),
('fixed bottom', 0),
None)
self.show_as_root_until_keypress(overlay, 'cancel',
relay_rest=False,
afterwards=clear)
else:
if timeout >= 0:
if timeout == 0:
timeout = config.getint('general', 'notify_timeout')
self.mainloop.set_alarm_in(timeout, clear)
return msgs[0]
def rebuild(self):
cols = []
formatstring = config.get('general', 'timestamp_format')
newest = self.thread.get_newest_date()
if formatstring:
datestring = newest.strftime(formatstring)
else:
datestring = pretty_datetime(newest).rjust(10)
self.date_w = urwid.AttrMap(urwid.Text(datestring), 'threadline_date')
cols.append(('fixed', len(datestring), self.date_w))
mailcountstring = "(%d)" % self.thread.get_total_messages()
self.mailcount_w = urwid.AttrMap(urwid.Text(mailcountstring),
'threadline_mailcount')
cols.append(('fixed', len(mailcountstring), self.mailcount_w))
tags = self.thread.get_tags()
tags.sort()
for tag in tags:
tw = TagWidget(tag)
self.tag_widgets.append(tw)
cols.append(('fixed', tw.width(), tw))
authors = self.thread.get_authors() or '(None)'
maxlength = config.getint('general', 'authors_maxlength')
authorsstring = shorten(authors, maxlength).strip()
self.authors_w = urwid.AttrMap(urwid.Text(authorsstring),
'threadline_authors')
cols.append(('fixed', len(authorsstring), self.authors_w))
subjectstring = self.thread.get_subject().strip()
self.subject_w = urwid.AttrMap(urwid.Text(subjectstring, wrap='clip'),
'threadline_subject')
if subjectstring:
cols.append(('fixed', len(subjectstring), self.subject_w))
if self.display_content:
msgs = self.thread.get_messages().keys()
msgs.sort()
lastcontent = ' '.join([m.get_text_content() for m in msgs])
contentstring = lastcontent.replace('\n', ' ').strip()
self.content_w = urwid.AttrMap(urwid.Text(contentstring,
wrap='clip'),
'threadline_content')
cols.append(self.content_w)
self.columns = urwid.Columns(cols, dividechars=1)
self.original_widget = self.columns
def notify(self, message, priority='normal', timeout=0, block=False):
"""notify popup
:param message: message to print
:type message: str
:param priority: priority string, used to format the popup: currently,
'normal' and 'error' are defined. If you use 'X' here,
the attribute 'notify_X' is used to format the popup.
:type priority: str
:param timeout: seconds until message disappears. Defaults to the value
of 'notify_timeout' in the general config section.
A negative value means never time out.
:type timeout: int
:param block: this notification blocks until a keypress is made
:type block: boolean
"""
def build_line(msg, prio):
cols = urwid.Columns([urwid.Text(msg)])
return urwid.AttrMap(cols, 'notify_' + prio)
msgs = [build_line(message, priority)]
if timeout == -1 and block:
msgs.append(build_line('(hit any key to proceed)', 'normal'))
footer = self.mainframe.get_footer()
if not self.notificationbar:
self.notificationbar = urwid.Pile(msgs)
else:
newpile = self.notificationbar.widget_list + msgs
self.notificationbar = urwid.Pile(newpile)
self.update()
def clear(*args):
self.clear_notify(msgs)
self.mainloop.draw_screen()
if block:
keys = self.mainloop.screen.get_input()
clear()
else:
if timeout >= 0:
if timeout == 0:
timeout = config.getint('general', 'notify_timeout')
self.mainloop.set_alarm_in(timeout, clear)
return msgs[0]
def string_sanitize(string, tab_width=None):
r"""
strips, and replaces non-printable characters
:param tab_width: number of spaces to replace tabs with. Read from
`globals.tabwidth` setting if `None`
:type tab_width: int or `None`
>>> string_sanitize(' foo\rbar ', 8)
'foobar'
>>> string_sanitize('foo\tbar', 8)
'foo bar'
>>> string_sanitize('foo\t\tbar', 8)
'foo bar'
"""
if tab_width == None:
tab_width = config.getint('general', 'tabwidth')
string = string.strip()
string = string.replace('\r', '')
lines = list()
for line in string.split('\n'):
tab_count = line.count('\t')
if tab_count > 0:
line_length = 0
new_line = list()
for i, chunk in enumerate(line.split('\t')):
line_length += len(chunk)
new_line.append(chunk)
if i < tab_count:
next_tab_stop_in = tab_width - (line_length % tab_width)
new_line.append(' ' * next_tab_stop_in)
line_length += next_tab_stop_in
lines.append(''.join(new_line))
else:
lines.append(line)
return '\n'.join(lines)
def buffer_close(self, buf):
"""
closes given :class:`~alot.buffers.Buffer`.
This it removes it from the bufferlist and calls its cleanup() method.
"""
buffers = self.buffers
if buf not in buffers:
string = 'tried to close unknown buffer: %s. \n\ni have:%s'
self.logger.error(string % (buf, self.buffers))
elif self.current_buffer == buf:
self.logger.debug('UI: closing current buffer %s' % buf)
index = buffers.index(buf)
buffers.remove(buf)
offset = config.getint('general', 'bufferclose_focus_offset')
nextbuffer = buffers[(index + offset) % len(buffers)]
self.buffer_focus(nextbuffer)
buf.cleanup()
else:
string = 'closing buffer %d:%s'
self.logger.debug(string % (buffers.index(buf), buf))
buffers.remove(buf)
buf.cleanup()
def rebuild(self):
cols = []
# DATE
formatstring = config.get('general', 'timestamp_format')
newest = self.thread.get_newest_date()
if formatstring:
datestring = newest.strftime(formatstring)
else:
datestring = pretty_datetime(newest).rjust(10)
self.date_w = urwid.AttrMap(urwid.Text(datestring), 'threadline_date')
# SIZE
thread_size = self.thread.get_total_messages()
# Show number of messages only if there are at least 2 mails
# (save space in the line)
if thread_size>1 and thread_size<=20:
charcode = 0x2474 + thread_size
mailcountstring = unichr(charcode)
elif thread_size>1 and thread_size>20:
mailcountstring = "(%d)" % thread_size
else:
mailcountstring = " "
# TAGS
tags = self.thread.get_tags()
tags.sort()
tagstrings = []
for tag in tags:
tw = TagWidget(tag)
self.tag_widgets.append(tw)
tagstrings.append(('fixed', tw.width(), tw))
# AUTHORS
authors_string = self.thread.get_authors() or '(None)'
maxlength = config.getint('general', 'authors_maxlength')
authorsstring = shorten_author_string(authors_string, maxlength - len(mailcountstring))
offset = maxlength - len(authorsstring)
mailcountstring = mailcountstring.rjust(offset)
self.mailcount_w = urwid.AttrMap(urwid.Text(mailcountstring),
'threadline_mailcount')
self.authors_w = urwid.AttrMap(urwid.Text(authorsstring),
'threadline_authors')
# SUBJECT
subjectstring = self.thread.get_subject().strip()
self.subject_w = urwid.AttrMap(urwid.Text(subjectstring, wrap='clip'),
'threadline_subject')
# BODY
if self.display_content:
msgs = self.thread.get_messages().keys()
msgs.sort()
lastcontent = ' '.join([m.get_text_content() for m in msgs])
contentstring = lastcontent.replace('\n', ' ').strip()
self.content_w = urwid.AttrMap(urwid.Text(contentstring,
wrap='clip'),
'threadline_content')
# Set column order
#self.select = urwid.AttrMap(urwid.Text("[ ] ", wrap='clip'),
# 'threadline_subject')
#cols.append(('fixed', 4, self.select))
cols.append(('fixed', len(datestring), self.date_w))
cols.append(('fixed', len(authorsstring), self.authors_w))
cols.append(('fixed', len(mailcountstring), self.mailcount_w))
cols.extend(tagstrings)
if subjectstring:
cols.append(('fixed', len(subjectstring), self.subject_w))
if self.display_content:
cols.append(self.content_w)
self.columns = urwid.Columns(cols, dividechars=1)
self.original_widget = self.columns
class RenderApplication(ExtApplication):
"""
Graphite-compatible render
"""
title = "Render"
DEFAULT_GRAPH_WIDTH = 330
DEFAULT_GRAPH_HEIGTH = 250
# Empty space around the borders of chart
X_PADDING = 10
Y_PADDING = 10
#
@view(url="^$", method=["GET"], access="launch",
validate={
"graphType": StringParameter(
default="line",
choices=GraphTypes.keys()
),
"pieMode": StringParameter(
default="average",
# @todo: Specify all modes
choices=["average"]
),
"cacheTimeout": IntParameter(
min_value=0,
default=config.getint("pm_render", "cache_duration")
),
"target": ListOfParameter(
element=StringParameter(),
convert=True, default=[]
),
"localOnly": StringParameter(default="0"),
"tz": StringParameter(default=TIME_ZONE),
"pickle": StringParameter(required=False),
"rawData": StringParameter(required=False),
"jsonp": StringParameter(required=False),
"format": StringParameter(required=False),
"noCache": StringParameter(required=False),
"maxDataPoints": IntParameter(required=False)
},
api=True)
def api_render(self, request,
graphType=None, pieMode=None, cacheTimeout=None,
target=None, localOnly=None, tz=None, pickle=None,
rawData=None, jsonp=None,
noCache=None, format=None,
maxDataPoints=None,
**kwargs):
# Get timezone info
try:
tz = pytz.timezone(tz)
except pytz.UnknownTimeZoneError:
tz = pytz.timezone(TIME_ZONE)
# Get format
if pickle is not None:
format = "pickle"
elif rawData is not None:
format = "raw"
# Get time range
try:
t0 = parseATTime(kwargs.get("from", "-1d"))
t1 = parseATTime(kwargs.get("until", "now"))
except Exception, why:
return self.response_bad_request(
"Cannot parse time: %s" % why
)
if t0 == t1:
return self.response_bad_request("Empty time range")
# Collect parameters
request_opts = {
"graphType": graphType,
"graphClass": GraphTypes[graphType],
"pieMode": pieMode,
"targets": target or [],
"localOnly": localOnly == "1",
"tzinfo": tz,
"format": format,
"noCache": noCache is not None,
"startTime": min(t0, t1),
"endTime": max(t0, t1),
"cacheTimeout": cacheTimeout
}
if format:
request_opts["format"] = format
if jsonp is not None:
request_opts["jsonp"] = jsonp
# Fill possible graph options
graph_opts = {
"width": self.DEFAULT_GRAPH_WIDTH,
"height": self.DEFAULT_GRAPH_HEIGTH,
}
if format == "svg":
graph_opts["outputFormat"] = "svg"
for opt in request_opts["graphClass"].customizable:
if opt in kwargs:
v = kwargs[opt]
if opt not in ("fgcolor", "bgcolor", "fontColor"):
try:
graph_opts[opt] = int(v)
continue
except ValueError:
pass
try:
graph_opts[opt] = float(v)
continue
except ValueError:
pass
if v.lower() in ("true", "false"):
graph_opts[opt] = v.lower() == "true"
continue
if not v or v.lower() == "default":
continue
graph_opts[opt] = v
use_cache = not request_opts["noCache"]
cache_timeout = request_opts["cacheTimeout"]
ctx = {
"startTime": request_opts["startTime"],
"endTime": request_opts["endTime"],
"localOnly": request_opts["localOnly"],
"maxDataPoints": maxDataPoints,
"data": []
}
data = ctx["data"]
# Try to use cached response
if use_cache:
request_key = hashRequest(request)
cached_response = cache.get(request_key)
if cached_response:
return cached_response
else:
request_opts["requestKey"] = request_key
# Cache miss, prepare requested data
if graphType == "pie":
for t in request_opts["targets"]:
if ":" in t:
try:
name, value = t.split(":", 1)
data += [(name, float(value))]
except ValueError:
raise ValueError("Invalid target: '%s'" % t)
else:
for series in evaluateTarget(ctx, t):
f = PieFunctions(request_opts["pieMode"])
data += [(series.name, f(ctx, series) or 0)]
elif graphType == "line":
if use_cache:
# Store cached data
data_key = hashData(request_opts["targets"],
request_opts["startTime"],
request_opts["endTime"])
cached_data = cache.get(data_key)
else:
cached_data = None
if cached_data is None:
for t in request_opts["targets"]:
if not t.strip():
continue
data.extend(evaluateTarget(ctx, t))
if use_cache:
cache.set(
data_key,
[d.get_info() for d in data],
cache_timeout
)
else:
# Convert cached data to Time Series
data = [TimeSeries(**a) for a in cached_data]
# Return data in requested format
h = getattr(self, "get_%s_response" % request_opts["format"], None)
if h:
r = h(data, request_opts)
else:
graph_opts["data"] = data
r = self.render_graph(request_opts, graph_opts)
r["Pragma"] = "no-cache"
r["Cache-Control"] = "no-cache"
return r
def rebuild(self):
cols = []
if self.thread:
newest = self.thread.get_newest_date()
else:
newest = None
if newest == None:
datestring = u' ' * 10
else:
formatstring = config.get('general', 'timestamp_format')
if formatstring:
datestring = newest.strftime(formatstring)
else:
datestring = pretty_datetime(newest).rjust(10)
self.date_w = urwid.AttrMap(urwid.Text(datestring),
'search_thread_date')
cols.append(('fixed', len(datestring), self.date_w))
if self.thread:
mailcountstring = "(%d)" % self.thread.get_total_messages()
else:
mailcountstring = "(?)"
self.mailcount_w = urwid.AttrMap(urwid.Text(mailcountstring),
'search_thread_mailcount')
cols.append(('fixed', len(mailcountstring), self.mailcount_w))
if self.thread:
self.tag_widgets = [TagWidget(t) for t in self.thread.get_tags()]
else:
self.tag_widgets = []
self.tag_widgets.sort(tag_cmp,
lambda tag_widget: tag_widget.translated)
for tag_widget in self.tag_widgets:
cols.append(('fixed', tag_widget.width(), tag_widget))
if self.thread:
authors = self.thread.get_authors() or '(None)'
else:
authors = '(None)'
maxlength = config.getint('general', 'authors_maxlength')
authorsstring = shorten_author_string(authors, maxlength)
self.authors_w = urwid.AttrMap(urwid.Text(authorsstring),
'search_thread_authors')
cols.append(('fixed', len(authorsstring), self.authors_w))
if self.thread:
subjectstring = self.thread.get_subject().strip()
else:
subjectstring = ''
self.subject_w = urwid.AttrMap(urwid.Text(subjectstring, wrap='clip'),
'search_thread_subject')
if subjectstring:
cols.append(('weight', 2, self.subject_w))
if self.display_content:
if self.thread:
msgs = self.thread.get_messages().keys()
else:
msgs = []
msgs.sort()
lastcontent = ' '.join([m.get_text_content() for m in msgs])
contentstring = lastcontent.replace('\n', ' ').strip()
self.content_w = urwid.AttrMap(urwid.Text(contentstring,
wrap='clip'),
'search_thread_content')
cols.append(self.content_w)
self.columns = urwid.Columns(cols, dividechars=1)
self.original_widget = self.columns
from time import sleep
from datetime import datetime, time
from crawler import WebsiteCrawler
from helpers import convert_string_to_time
from settings import config
from telegram_bot import send_telegram
interval_minutes = config.getint("general", "interval_minutes")
send_live_bits = config.getboolean("general", "send_live_bits")
live_bit_only_on_weekdays = [1, 2, 3, 4, 5, 6, 7]
raw_start_time = config.get("general", "start_time")
start_time = convert_string_to_time(config.get("general", "start_time"))
end_time = convert_string_to_time(config.get("general", "end_time"))
if __name__ == '__main__':
crawler = WebsiteCrawler()
send_telegram("Service gestartet")
previous_day = -1
while True:
try:
now_time = datetime.now().time()
# check website between this time
