def merge_dicts(in_dicts_list, out_dict):
result_dict = []
for dict in in_dicts_list:
path_in = "{0}/Thesis/senti/dict/{1}".format(config.get("home_path"), dict)
path_out = "{0}/Thesis/senti/dict/{1}".format(config.get("home_path"), out_dict)
with open(path_in, 'r',) as dict_file:
for num, line in enumerate(dict_file):
if line.startswith(codecs.BOM_UTF8):
line = line[3:]
line = line.strip()
print num, line
word = line.split('\t')[0].strip().decode("utf8")
score = int(line.split('\t')[1])
result_dict.append((word, score))
'''Удаляем дубликаты'''
seen = set()
output = []
for item in result_dict:
if item[0] not in seen:
output.append(item)
seen.add(item[0])
output.sort(key=lambda tup: tup[0])
with open(path_out, 'w') as dict_out_file:
for item in output:
dict_out_file.write((item[0] + "\t" + str(item[1]) + "\n").encode('utf-8'))
def __init__(self):
self.mhashes = {} # metric -> metric hash
#
self.hash_width = config.getint("pm_storage", "hash_width")
self.key_mask = "!%dsL" % self.hash_width
# Set key-value store class
kvtype = config.get("pm_storage", "type")
logger.info("Initializing %s storage. %d-byte wide hash", kvtype,
self.hash_width)
self.kvcls = load_name("noc.pm.db.storage", kvtype, KVStorage)
if not self.kvcls:
logger.critical("Invalid storage type: '%s'", kvtype)
raise ValueError("Invalid storage type: '%s'" % kvtype)
# Set partitioning scheme
ps = config.get("pm_storage", "partition")
logger.info("Setting %s partitioning scheme", ps)
self.partition = load_name("noc.pm.db.partition", ps, Partition)
# Index collection
self.metrics = get_db()["noc.ts.metrics"]
self.metrics_batch = self.metrics.initialize_ordered_bulk_op()
self.new_metrics = 0
self.flush_lock = threading.Lock()
self.epoch = int(
time.mktime(
time.strptime(config.get("pm_storage", "epoch"), "%Y-%m-%d")))
self.zero_hash = Binary("\x00" * self.hash_width)
def main():
try:
logging.basicConfig(level=logging.DEBUG)
parser = argparse.ArgumentParser()
parser.add_argument('--host', default='127.0.0.1', type=str)
parser.add_argument('--port', default=8000, type=int)
args = parser.parse_args()
config['HOST'] = config.get('HOST') or args.host
config['PORT'] = config.get('PORT') or args.port
middlewares = [
basic_auth_middleware,
]
app = web.Application(middlewares=middlewares)
executor = concurrent.futures.ThreadPoolExecutor(
max_workers=config['NUM_THREADS'])
loop = asyncio.get_event_loop()
app['config'] = config
setup_storage(loop, executor)
setup_handlers(app, loop, executor)
setup_routes(app)
web.run_app(app, host=config['HOST'], port=config['PORT'])
finally:
logging.info('Exiting app')
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
# postgresql://postgres:postgres@localhost/oracles_vision
# url = context_config.get_main_option("sqlalchemy.url")
init_settings()
url = 'postgresql://%s:%s@%s/%s' % (config.get(
'DB', 'user'), config.get('DB', 'password'), config.get(
'DB', 'host'), config.get('DB', 'database'))
context.configure(url=url,
target_metadata=target_metadata,
literal_binds=True)
with context.begin_transaction():
context.run_migrations()
def configure(cls):
rules = {}
for opt in config.options(cls.CONFIG_SECTION):
if opt.startswith("map."):
# Process metric mappings
_, model_id = opt.split(".", 1)
v = config.get(cls.CONFIG_SECTION, opt)
logger.info("Configuring metric mappings %s -> %s",
model_id, v)
try:
cls._TEMPLATES[model_id] = Template(v)
except TemplateSyntaxError, why:
logging.error("Template syntax error: %s", why)
elif opt.startswith("metric."):
# Process metric rules
n = int(opt.split(".")[1])
v = config.get(cls.CONFIG_SECTION, opt)
try:
rx = re.compile(v)
except re.error, why:
logging.error(
"Invalid regular expression in rule %s: %s",
n, why
)
continue
def avg_comments():
width = 0.8
data = {
"bk55": 8.5272094345,
"gorod55": 10.6726493011,
"ngs55": 9.88509836752,
"omskinform": 2.71181391085
}
sources = data.keys()
data = [(source, round(avg, 2)) for source, avg in sorted(data.items(), key=lambda x: x[1])]
sources_x = np.arange(len(data))
numbers = [number for source, number in data]
fig, ax = plt.subplots()
ax.set_xlabel(u'Источник')
ax.set_ylabel(u'Среднее количество комментариев')
rects = ax.bar(sources_x, numbers, width)
plt.xticks(sources_x + 0.4, sources)
for rect, label in zip(rects, sources):
height = rect.get_height()
ax.text(rect.get_x() + rect.get_width()/2, height, height, ha='center', va='bottom')
for ext in config.get("tex_image_format"):
plt.savefig(config.get("tex_image_path") + "avg_comments." + ext, bbox_inches='tight', format=ext, dpi=1200)
plt.show()
def is_admin(bot, update):
if update.message.text == config.get('global', 'bot_token'):
if config.get('global', 'admin') != update.message.from_user.id:
update_parameter('global', 'admin', update.message.from_user.id)
log.info('[TG] Установлен администратор бота с ID {0}'.format(
update.message.from_user.id))
update.message.reply_text(
'Доступ получен.\n\nКоманда просмотра всех доступных команд: /help'
)
def mallet_plot():
## Строим график из данных Mallet
topics = [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100]
perp = [514.1302184469, 525.0241590983, 534.4977260582, 538.5513127672, 541.8989344152, 541.0094559616, 538.4206752508, 535.3505225192, 532.7629698852, 533.9200808978, 540.3049166531, 531.4757292891, 538.0214941918, 546.4364835315, 538.0214941918, 540.9982061032, 532.1429352736, 535.5732148468, 536.4983783528, 539.0330790911]
plt.plot(topics, perp, 'o-')
plt.ylabel(u'Перплексия')
plt.xlabel(u'Количество тем')
plt.grid(True)
plt.xlim(0, 105)
for ext in config.get("tex_image_format"):
plt.savefig(config.get("tex_image_path") + "perplexity_mallet." + ext,
format=ext, bbox_inches='tight', dpi=1200)
def send_telegram(message: str):
""" Send a Telegram message to a bot
first a telegram bot must be created
https://www.giga.de/apps/telegram/tipps/telegram-bot-erstellen-loeschen-andere-befehle-so-geht-s/
"""
bot_token = config.get("telegram", "bot_token")
bot_chat_id = config.get("telegram", "bot_chat_id")
send_text = f"https://api.telegram.org/bot{bot_token}/sendMessage?chat_id={bot_chat_id}&parse_mode=Markdown&text={message}"
response = requests.get(send_text)
return response.json()
def mongo_factory():
"""
wraps getting a mongo client reading defaults from the settings
"""
from settings import config
hosts = config.get('mongo.hosts', 'localhost:27017')
database = config.get('mongo.database', 'test')
user = config.get('mongo.user', '')
password = config.get('mongo.password', '')
hosts = hosts.split(',')
return mongo_client(hosts, database, user, password)
def send_post(bot, domain, post):
log.info("[TG] Отправка поста...")
if post.text:
if post.photos:
pass
else:
bot.sendMessage(chat_id=config.get(domain, 'channel'),
text=post.text,
parse_mode='HTML',
disable_web_page_preview=True)
if post.photos:
# noinspection PyBroadException
try:
if post.text:
if len(post.photos) == 1 and len(post.text) < 200:
bot.sendPhoto(chat_id=config.get(domain, 'channel'),
photo=post.photos[0]['media'],
caption=post.text,
parse_mode='HTML')
else:
bot.sendMessage(chat_id=config.get(domain, 'channel'),
text=post.text,
parse_mode='HTML',
disable_web_page_preview=True)
bot.sendMediaGroup(chat_id=config.get(domain, 'channel'),
media=post.photos)
else:
bot.sendMediaGroup(chat_id=config.get(domain, 'channel'),
media=post.photos)
except Exception:
log.warning('[TG] Невозможно отправить фото: {0}.'.format(
sys.exc_info()[1]))
for m in post.videos:
try:
bot.sendVideo(chat_id=config.get(domain, 'channel'),
video=open(m, 'rb'),
timeout=60)
except Exception:
pass
for m in post.docs:
bot.sendDocument(chat_id=config.get(domain, 'channel'),
document=open(m, 'rb'),
timeout=60)
for (m, n) in post.tracks:
try:
if getsize(m) > 52428800:
remove(m)
else:
try:
bot.sendAudio(chat_id=config.get(domain, 'channel'),
audio=open(m, 'rb'),
duration=int(n),
timeout=60)
except:
pass
remove(m)
except FileNotFoundError:
continue
def compare(self):
login_data = {"pin": config.get("website", "pin")}
session = requests.Session()
header = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0",
}
raw_request = session.post(self.__url, data=login_data, headers=header)
if raw_request.status_code != 200:
send_telegram(
"Fehler beim Anmelden auf der Seite des Testergebnisses. Das Programm wird beendet"
)
exit()
request = raw_request.json()
message = str()
try:
message = request["results"][0]["headline"]
message += " Das Programm wird nun automatisch beendet."
except:
send_telegram(
"Testergebnis hat sich verändert. Das Programm wird nun automatisch beendet."
)
exit()
if message != "Ihre Probe ist auf dem Weg ins Labor.":
send_telegram(message)
exit()
return
def send_post(bot, update):
if str(config.get('global', 'admin')) == str(update.message.from_user.id):
global chat
chat = update.message['text'][11:]
update.message.reply_text(
'Чтобы отправить ваш пост в канал/группу, ответьте на это сообщение.'
)
def GET(self):
params = web.input(raw_msg="", msisdn="", form="")
web.header("Content-Type", "application/json; charset=utf-8")
year, week = get_current_week()
myparams = {
"username": config["dispatcher2_username"],
"password": config["dispatcher2_password"],
"ctype": "text",
"raw_msg": params.raw_msg,
"year": year,
"week": week,
"msisdn": params.msisdn,
"is_qparams": 't',
"extras": "{}",
"source": "mtrackpro",
"destination": KEYWORD_SERVER_MAPPINGS.get(params.form, "localhost")
}
payload = "message={0}&originator={1}".format(params.raw_msg, params.msisdn)
queueEndpoint = config.get("dispatcher2_queue_url", "http://localhost:9191/queue?")
# print("Call=>", queueEndpoint)
requests.post(
queueEndpoint,
data=payload,
params=myparams,
headers={"Content-type": "text/plain"})
# print("RESP:===>", resp.text)
return json.dumps({"status": "success"})
def source_read(app, docname, source):
""" prepend header to every file """
section = "manual-%s" % app.config.language
header_file = config.get(section, "header-file")
if header_file and os.path.exists(header_file):
with open(header_file, "rb") as f:
source[0] = "%s\n%s" % (f.read().decode('utf-8'), source[0])
def file_every_1000():
for num, doc in enumerate(db.find()):
if fmod(num, 1000) == 0:
corpus = etree.Element("corpus")
begin = num
end = begin + 1000
if end > db.find().count():
end = db.find().count() - 1
print "begin, end ", begin, end
for doc in db.find()[begin:end]:
document = etree.SubElement(corpus, "document")
etree.SubElement(document, "title").text = doc["title"]
etree.SubElement(document, "content").text = doc["content"]
etree.SubElement(document, "date").text = str(doc["date"])
etree.SubElement(document, "url").text = doc["url"]
etree.SubElement(document, "source").text = doc["source"]
comments = etree.SubElement(document, "comments", amount=str(len(doc["comments"])))
for comment in doc["comments"]:
etree.SubElement(comments, "comment").text = comment
xml_str = etree.tostring(corpus, pretty_print=True)
with open("{0}/Thesis/omsk_media_{1}-{2}.xml".format(config.get("home_path"), begin, end), 'w') as outFile:
outFile.write(xml_str)
def status(bot, update):
if str(config.get('global', 'admin')) == str(update.message.from_user.id):
stat = 'Автопостинг остановлен'
if job_status:
if job_status.enabled:
stat = 'Автопостинг запущен'
update.message.reply_text(stat, quote=True)
def update_price_history(self, c, timestamp, price):
monitored_at = datetime.datetime.fromtimestamp(timestamp).strftime('%H:%M:%S')
if len(c['history_live']) >= config.get('history_live_window'):
c['history_live'] = c['history_live'][1:]
c['history_live'].append({'timestamp': monitored_at, 'price': price})
self.store({'stock': c.get('stock'), 'timestamp': timestamp, 'price': price})
return c
def __init__(self, tag):
self.tag = tag
self.translated = config.get('tag-translate', tag, fallback=tag)
self.txt = urwid.Text(self.translated.encode('utf-8'), wrap='clip')
normal = config.get_tagattr(tag)
focus = config.get_tagattr(tag, focus=True)
urwid.AttrMap.__init__(self, self.txt, normal, focus)
def send_post(bot, update, args):
global chat
if len(args) != 0:
chat = args[0]
else:
chat = config.get('global', 'main_group')
update.message.reply_text('Чтобы отправить ваш пост в канал/группу {}, ответьте на это сообщение.'.format(chat),
reply_markup=ForceReply())
def stop(bot, update):
if str(config.get('global', 'admin')) == str(update.message.from_user.id):
global job_status
if job_status:
if job_status.enabled:
job_status.enabled = False
update.message.reply_text('Бот остановлен.', quote=True)
log.info('Stopping a job_status...')
def source_list(bot, update):
sources_list = config.sections()[1:]
sources = 'Список источников:\nИсточник' + '\x20'*8 + '---->' + '\x20'*8 + 'Назначение\n\n'
for source in sources_list:
sources += 'https://vk.com/' + source + '\x20'*8 + '---->' + '\x20'*8 + config.get(source, 'channel') + '\n'
sources += 'Для удаления источника отправьте команду /remove <домен группы вк>\nНапример, /remove ' +\
sources_list[randint(0, len(sources_list) - 1)]
update.message.reply_text(sources, disable_web_page_preview=True)
def get_datestring(self):
"""returns formated datestring"""
formatstring = config.get('general', 'timestamp_format')
if formatstring:
res = self._datetime.strftime(formatstring)
else:
res = helper.pretty_datetime(self._datetime)
return res
def print_result(evaluate_set, predict, accuracy, f1, precision, recall):
print("accuracy: ", str(accuracy * 100) + "%")
print("f1:", f1)
print("precision:", precision)
print("recall:", recall)
if config.get("print-predict?", False):
for i in range(10):
print(evaluate_set[i], "predict:", predict[i])
def expand_words(path_in, path_out):
path_in = "{0}/Thesis/senti/dict/{1}/EmotionLookupTable.txt".format(config.get("home_path"), path_in)
path_out = "{0}/Thesis/senti/dict/{1}/EmotionLookupTable.txt".format(config.get("home_path"), path_out)
morph = pymorphy2.MorphAnalyzer()
with open(path_in, "r") as infile, open(path_out, 'w') as outfile:
#Очищаем файл
outfile.truncate()
for num, line in enumerate(infile):
# http://en.wikipedia.org/wiki/Byte_order_mark#UTF-8
nline = line
if nline.startswith(codecs.BOM_UTF8):
nline = nline[3:]
nline = nline.strip()
word = nline.split('\t')[0].strip().decode("utf8")
score = int(nline.split('\t')[1])
if "*" not in word:
""" Берём слова """
stemmed = stem_snowball(word)
exclude_words = [u"зря"]
if len(stemmed) < 4:
"""Слишком короткие заменяем всеми словоформами"""
if word in exclude_words:
outfile.write(word.encode('utf-8') + "\t" + str(score).encode('utf-8') + "\n")
else:
for form in get_all_forms(word):
outfile.write(form.encode('utf-8') + "\t" + str(score).encode('utf-8') + "\n")
if len(stemmed) >= 4:
"""Для некоротких создаём правило на основе стемминга"""
if len(stemmed) < len(word):
""" Если стемминг имел место, заменяем усечённую часть * """
result_word = stemmed + "*"
outfile.write(result_word.encode('utf-8') + "\t" + str(score).encode('utf-8') + "\n")
else:
""" Иначе оставляем как было """
result_word = word # == stemmed
outfile.write(result_word.encode('utf-8') + "\t" + str(score).encode('utf-8') + "\n")
else:
outfile.write((word + "\t" + str(score) + "\n").encode('utf-8'))
def init_locale(self):
self.locale = self.state.document.settings.env.config.language
t = gettext.translation('messages',
localedir=os.path.join(
root_dir, config.get("DEFAULT", "locale")),
languages=[self.locale])
t.install(**kwargs)
self.init_type_mapping()
def _create_connection(self):
if not self.connection:
pgsql = config.get('pgsql', {})
self.connection = psycopg2.connect(dbname=pgsql.get('name'),
user=pgsql.get('user'),
password=pgsql.get('password'),
host=pgsql.get('host'),
port=pgsql.get('port'))
return self.connection
def get_last_logs(bot, update, args):
if str(config.get('global', 'admin')) == str(update.message.from_user.id):
if args:
string = int(args[0])
else:
string = 15
with open('../bot_log.log', 'r', encoding='utf-8') as f:
last_logs = ''.join(f.readlines()[-string:])
update.message.reply_text('Последние {} строк логов:\n\n'.format(str(string)) + last_logs, quote=True)
def __new__(cls, *arguments, **keywords):
'''
The class that will be created will be a subclass in the config file:
General -> exporter
'''
for subclass in Exporter.__subclasses__():
class_str = subclass.__module__ + "." + subclass.__name__
if class_str == config.get('General','exporter'):
return super(cls, subclass).__new__(subclass)
raise Exception, 'Invalid Exporter! Use one of: %s'%([c.__name__ for c in Exporter.__subclasses__()])
def explain(self):
p = subprocess.Popen(shlex.split("java -jar {0}/Thesis/senti/SentiStrengthCom.jar stdin noDictionary illegalDoubleLettersInWordMiddle"
" ёйухцчщьыъ illegalDoubleLettersAtWordEnd абвгджзйкоуфхцчщэ UTF8 urlencoded explain sentidata "
"{0}/Thesis/senti/dict/{1}/".format(config.get("home_path"), self.dict_name)),
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_text, stderr_text = p.communicate(self.text.replace(" ", "+"))
print self.text.replace(" ", "+")
if stderr_text:
raise Exception("Ошибка в модуле оценки эмоций:" + stderr_text)
return stdout_text
def init():
from settings import config
host = config.get('restAPI', 'host', fallback='0.0.0.0')
port = config.getint('restAPI', 'port', fallback=5000)
debug = config.getboolean('restAPI', 'debug', fallback=False)
app.debug = debug
global server
server = WebSocketServer((host, port), app, handler_class=WebSocketHandler)
server.serve_forever()
def comments_count_by_day_complete():
date_dict = comments_count_by_day()
weekend_comments = []
weekday_comments = []
a = comments_count_by_day().items()
a = sorted(a, key=lambda x: x[0], reverse=False)
a = [(date, scores[1]) for date, scores in a]
fig, ax = plt.subplots()
ax.plot(*zip(*a))
for i, single_date in enumerate(daterange()):
print i
if single_date.weekday() == 5:
x_list = [single_date, single_date + datetime.timedelta(1)]
y_list = [date_dict[single_date][1], date_dict[single_date + datetime.timedelta(1)][1]]
weekend_comments.extend(y_list)
print x_list
ax.fill_between(x_list, y_list, alpha=0.3, color='#FF8E8E')
elif (single_date.weekday() != 5) and (single_date.weekday() != 6):
weekday_comments.append(date_dict[single_date][1])
print single_date.weekday()
ax.set_ylabel(u'Количество комментариев за день')
ax.set_xlabel(u'Дата')
red_patch = mpatches.Patch(color='#FF8E8E', alpha=0.4, label=u"Суббота и воскресенье")
plt.legend(handles=[red_patch], fontsize=14)
ax.format_xdata = mdates.DateFormatter('%Y-%m-%d')
fig.autofmt_xdate() # поворачивает надписи даты
ax.grid(True)
for ext in config.get("tex_image_format"):
plt.savefig(config.get("tex_image_path") + "comments_by_day." + ext,
format=ext, bbox_inches='tight', dpi=1200)
plt.show()
print weekend_comments
print weekday_comments
print sum(weekend_comments), len(weekend_comments)
print sum(weekday_comments), len(weekday_comments)
def button(bot, update):
# TODO Доделать управление источниками через меню
query = update.callback_query
if query.data == '3':
bot.forward_message(chat_id=config.get('global', 'main_group'), from_chat_id=message.chat.id,
message_id=message.message_id)
bot.edit_message_text(text='Принято', chat_id=query.message.chat_id,
message_id=query.message.message_id)
elif query.data == '4':
bot.edit_message_text(text='Отклонено', chat_id=query.message.chat_id,
message_id=query.message.message_id)
def check_if_taskqueue(self, *args, **kwargs):
"""
Check if it is executed by Taskqueue in Staging or Production
Allow run in localhost calling the url
"""
if self.request.headers.get('X-AppEngine-TaskName') is None \
and config.get('environment') == "production" \
and not users.is_current_user_admin():
return self.error(403)
else:
return handler(self, *args, **kwargs)
def perplexity():
# split into train and test - random sample, but preserving order
perwordbound_list = []
perplexity2_list = []
topics_list = []
train_size = int(round(len(corpus)*0.9))
train_index = sorted(random.sample(xrange(len(corpus)), train_size))
test_index = sorted(set(xrange(len(corpus)))-set(train_index))
train_corpus = [corpus[i] for i in train_index]
test_corpus = [corpus[j] for j in test_index]
for num_topics in range(5, 101, 5):
model = LDA(dictionary, train_corpus, num_topics, "lda20/lda_training_{num_topics}"
.format(num_topics=str(num_topics)), passes=20)
perwordbound = model.log_perplexity(test_corpus)
perplexity2 = np.exp2(-perwordbound)
perwordbound_list.append(perwordbound)
perplexity2_list.append(perplexity2)
topics_list.append(num_topics)
plt.plot(topics_list, perwordbound_list, 'o-')
plt.ylabel(u'Per word bound')
plt.xlabel(u'Количество тем')
plt.grid(True)
plt.xlim(0, 105)
for ext in config.get("tex_image_format"):
plt.savefig(config.get("tex_image_path") + "perplexity_perwordbound." + ext,
bbox_inches='tight', format=ext, dpi=1200)
plt.close()
plt.plot(topics_list, perplexity2_list, 'o-')
plt.ylabel(u'Перплексия')
plt.xlabel(u'Количество тем')
plt.grid(True)
plt.xlim(0, 105)
for ext in config.get("tex_image_format"):
plt.savefig(config.get("tex_image_path") + "perplexity_exp2." + ext,
format=ext, bbox_inches='tight', dpi=1200)
plt.close()
def render(filename, **args):
""" define mako render function """
try:
mytemplate = _lookup.get_template(filename)
args["sitename"] = config.get("sitename")
args["cdate"] = datetime.datetime.now().strftime("%Y-%m-%d")
args['session'] = web.ctx.session
args["ctx"] = web.ctx
args["current_css"] = current_css
args["is_select"] = is_select
return mytemplate.render(**args)
except:
return exceptions.text_error_template().render()
def sending(bot, update):
global chat, message
if str(config.get('global', 'admin')) == str(update.message.from_user.id) and chat:
if update.message.text:
bot.send_message(chat_id=chat, text=update.message.text, parse_mode='Markdown')
elif update.message.document:
bot.send_document(chat_id=chat, document=update.message.document.file_id, caption=update.message.caption,
parse_mode='Markdown')
elif update.message.photo:
bot.send_photo(chat_id=chat, photo=update.message.photo[0].file_id, caption=update.message.caption,
parse_mode='Markdown')
elif update.message.video:
bot.send_video(chat_id=chat, video=update.message.video.file_id, caption=update.message.caption,
parse_mode='Markdown')
bot.send_media_group(chat_id=chat, media=update.message.media_group_id)
chat = None
else:
button_list = [InlineKeyboardButton('Принять', callback_data="3"), InlineKeyboardButton('Отклонить', callback_data="4")]
reply_markup = InlineKeyboardMarkup(build_menu(button_list, n_cols=2))
message = bot.forward_message(chat_id=config.get('global', 'admin'), from_chat_id=update.message.chat.id,
message_id=update.message.message_id)
bot.send_message(chat_id=config.get('global', 'admin'), text='Что сделать с сообщением выше?', reply_markup=reply_markup)
def watch(self):
while True:
for idx, c in enumerate(self.watchlist):
try:
timestamp = time.time()
stat = urllib2.urlopen(self.source % c.get('stock')).read().strip()
self.watchlist[idx] = self.update_price_history(c, timestamp, float(stat))
self.watchlist[idx] = self.update_market_values_for_holdings(c)
except Exception as ex:
print (ex.message)
sys.exit(1)
self.present()
time.sleep(config.get('update_frequency_sec'))
def render(filename,**args):
""" define mako render function """
try:
mytemplate = _lookup.get_template(filename)
args["sitename"] = config.get("sitename")
args["cdate"] = datetime.datetime.now().strftime( "%Y-%m-%d")
args['session'] = web.ctx.session
args["ctx"] = web.ctx
args["current_css"] = current_css
args["is_select"] = is_select
return mytemplate.render(**args)
except:
return exceptions.text_error_template().render()
def run(bot, update):
if str(config.get('global', 'admin')) == str(update.message.from_user.id):
try:
global job_status
if job_status:
if job_status.enabled:
return 0
job_status = job_queue.run_repeating(job_repeated, interval=5 * 60, first=0)
log.info('Running a job_status...')
update.message.reply_text('Бот запущен', quote=True)
except Exception:
log.info('Got an error while running a job_status: %s.' % sys.exc_info()[0])
update.message.reply_text('Не удалось запустить бота: {}'.format(sys.exc_info()[1]), quote=True)
def build_portfolio(self):
portfolio_data = json.load(open(config.get('paths').get('portfolio_file_path'), 'r'))
portfolio = []
for stock in portfolio_data:
portfolio.append({
'stock': stock.get('code'),
'history_live': [{'time': '', 'price': 0.00},],
'holdings': map(self.stock_record, stock.get('holdings')),
'announcement_dates': {},
'news': {},
})
return portfolio
def number_of_docs_by_day():
date_dict = {date: db.docs_topics.find({"date": date}).count() for date in daterange()}
print date_dict
weekend = []
weekday = []
fig, ax = plt.subplots()
for i, single_date in enumerate(daterange()):
print i
if single_date.weekday() == 5:
x_list = [single_date, single_date + datetime.timedelta(1)]
y_list = [date_dict[single_date], date_dict[single_date + datetime.timedelta(1)]]
weekend.extend(y_list)
print x_list
ax.fill_between(x_list, y_list, alpha=0.3, color='#FF8E8E')
elif (single_date.weekday() != 5) and (single_date.weekday() != 6):
weekday.append(date_dict[single_date])
print single_date.weekday()
red_patch = mpatches.Patch(color='#FF8E8E', alpha=0.4, label=u"Суббота и воскресенье")
plt.legend(handles=[red_patch], fontsize=14)
a = sorted(date_dict.items(), key=lambda x: x[0], reverse=False)
ax.plot(*zip(*a))
ax.set_ylabel(u'Количество статей')
ax.set_xlabel(u'Дата')
ax.format_xdata = mdates.DateFormatter('%Y-%m-%d')
fig.autofmt_xdate() # поворачивает надписи даты
fig.set_figwidth(11)
ax.grid(True)
for ext in config.get("tex_image_format"):
plt.savefig(config.get("tex_image_path") + "docs_by_day." + ext,
format=ext, bbox_inches='tight', dpi=1200)
plt.show()
print sum(weekend), len(weekend), sum(weekend)/len(weekend)
print sum(weekday), len(weekday), sum(weekday)/len(weekday)
def __init__(self):
self.profile = global_settings.get('general', 'profile')
if os.path.exists(os.path.join(os.getcwd(), 'profiles', self.profile)):
try:
import importlib
self.settings = importlib.import_module(
"profiles.%s.settings" % self.profile)
self.parameters = self.buildRobotParameters()
logging.info('profile: %s', self.profile)
except ImportError as ex:
self.settings = None
logging.error('%s: %s', self.profile, ex.message)
else:
logging.error('directory <%s> does not exists')
def GET(self, id):
web.header("Content-Type", "application/json; charset=utf-8")
r = db.query(
"SELECT waybill, destination_id, quantity_bales FROM distribution_log_w2sc_view "
"WHERE id = $id", {'id': id})
if r:
res = r[0]
msg = 'Please send "REC %s %s" to %s if you received %s bales of nets with waybill %s.'
to_subcounty = res['destination_id']
msg = msg % (res['waybill'], res['quantity_bales'],
config.get('shortcode', '6400'),
res['quantity_bales'], res['waybill'])
ret = {"sms": msg, "to_subcounty": to_subcounty}
return json.dumps(ret)
def validate_access_token(access_token):
"""
This request requires validation. To get an access token use the ``/authenticate`` endpoint.
"""
"""
Check to see if the access_token is valid
The access token will invalidate if the user changes their password.
parameters
==========
* access_token - a token created with create_access_token
returns
=======
returns True or False depending on if the access_token is valid
"""
from settings import config
validated = False
try:
decoded = jwt.decode(access_token, config.get('pepper', ''), algorithms=['HS256'])
except jwt.DecodeError:
logger.debug('jwt DecodeError')
else:
now = datetime.utcnow()
then = datetime.fromtimestamp(decoded['iat'])
age = now - then
user = User.get_by_id(decoded['user_id'])
# TODO: make the age configurable
# for now the access_token is valid for 5 hours
if age.seconds > 60 * 60 * 5:
logger.debug('stale access token, timestamp expired')
elif user and decoded['password'] == user.password:
validated = {
'user': user,
}
logger.debug('Valid access token for user:{}'.format(user.id))
else:
logger.debug('access token failed to validate')
return validated
def __init__(self, tag):
self.tag = tag
#self.translated = self.translated.encode('utf-8')
#self.txt = urwid.Text(self.translated, wrap='clip')
#normal = config.get_tagattr(tag)
normal = config.get_tagattr(tag)
focus = config.get_tagattr(tag, focus=True)
self.translated = config.get('tag-translate', tag, fallback=tag)
self.txt = urwid.Text(self.translated)#, wrap='space')
self.focus_palette = normal
self.unfocus_palette = normal
urwid.AttrMap.__init__(self, self.txt, self.unfocus_palette, self.focus_palette)
def senti_tuple(self):
#open a subprocess using shlex to get the command line string into the correct args list format
# добавить параметр explain для объяснения вывода изменить на
p = subprocess.Popen(shlex.split("java -jar {0}/Thesis/senti/SentiStrengthCom.jar stdin noDictionary illegalDoubleLettersInWordMiddle"
" ёйухцчщьыъ illegalDoubleLettersAtWordEnd абвгджзйкоуфхцчщэ UTF8 urlencoded sentidata "
"{0}/Thesis/senti/dict/{1}/".format(config.get("home_path"), self.dict_name)),
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#communicate via stdin the string to be rated. Note that all spaces are replaced with +
stdout_text, stderr_text = p.communicate(self.text.replace(" ", "+"))
#remove the tab spacing between the positive and negative ratings. e.g. 1 -5 -> 1-5
stdout_text = stdout_text.rstrip().replace("\t","")
if stderr_text:
raise Exception("Ошибка в модуле оценки эмоций:" + stderr_text)
senti_tuple = (int(stdout_text.split("-")[0]), -int(stdout_text.split("-")[1]))
return senti_tuple
def doc_number():
width = 0.8
data = {
"bk55": 14078,
"gorod55": 6302,
"ngs55": 4780,
"omskinform": 8727
}
sources_x = np.arange(len(data))
numbers = [number for source, number in data.items()]
fig, ax = plt.subplots()
ax.set_xlabel(u'Источник')
ax.set_ylabel(u'Количество статей')
rects = ax.bar(sources_x, numbers, width)
plt.xticks(sources_x + 0.4, data.keys())
for rect, label in zip(rects, data.keys()):
height = rect.get_height()
ax.text(rect.get_x() + rect.get_width()/2, height + 5, height, ha='center', va='bottom')
for ext in config.get("tex_image_format"):
plt.savefig(config.get("tex_image_path") + "doc_number." + ext, bbox_inches='tight', format=ext, dpi=1200)
plt.show()
def get_datestring(self):
"""
returns reformated datestring for this messages.
It uses the format spacified by `timestamp_format` in
the general section of the config.
"""
if self._datetime == None:
return None
if config.has_option('general', 'timestamp_format'):
formatstring = config.get('general', 'timestamp_format')
res = self._datetime.strftime(formatstring)
else:
res = helper.pretty_datetime(self._datetime)
return res
def update(request_kwargs=None):
global updater, dp, job_queue, job
updater = Updater(config.get('global', 'bot_token'),
request_kwargs=request_kwargs)
dp = updater.dispatcher
job_queue = updater.job_queue
dp.add_error_handler(error)
dp.add_handler(CommandHandler('start', commands.start))
dp.add_handler(CommandHandler('help', commands.help))
dp.add_handler(CommandHandler('run', commands.run))
dp.add_handler(CommandHandler('stop', commands.stop))
dp.add_handler(CommandHandler('get_full_logs', commands.get_full_logs))
dp.add_handler(CommandHandler('get_last_logs', commands.get_last_logs))
dp.add_handler(CommandHandler('status', commands.status))
dp.add_handler(MessageHandler(callback=commands.is_admin,
filters=Filters.regex(
config.get('global', 'bot_token'))),
group=1)
dp.add_handler(CommandHandler('send_post', commands.send_post))
dp.add_handler(
MessageHandler(callback=commands.sending, filters=Filters.reply))
job = job_queue.run_repeating(commands.job_repeated,
interval=5 * 60,
first=0)
def rebuild(self):
cols = []
formatstring = config.get('general', 'timestamp_format')
newest = self.thread.get_newest_date()
if formatstring:
datestring = newest.strftime(formatstring)
else:
datestring = pretty_datetime(newest).rjust(10)
self.date_w = urwid.AttrMap(urwid.Text(datestring), 'threadline_date')
cols.append(('fixed', len(datestring), self.date_w))
mailcountstring = "(%d)" % self.thread.get_total_messages()
self.mailcount_w = urwid.AttrMap(urwid.Text(mailcountstring),
'threadline_mailcount')
cols.append(('fixed', len(mailcountstring), self.mailcount_w))
tags = self.thread.get_tags()
tags.sort()
for tag in tags:
tw = TagWidget(tag)
self.tag_widgets.append(tw)
cols.append(('fixed', tw.width(), tw))
authors = self.thread.get_authors() or '(None)'
maxlength = config.getint('general', 'authors_maxlength')
authorsstring = shorten(authors, maxlength).strip()
self.authors_w = urwid.AttrMap(urwid.Text(authorsstring),
'threadline_authors')
cols.append(('fixed', len(authorsstring), self.authors_w))
subjectstring = self.thread.get_subject().strip()
self.subject_w = urwid.AttrMap(urwid.Text(subjectstring, wrap='clip'),
'threadline_subject')
if subjectstring:
cols.append(('fixed', len(subjectstring), self.subject_w))
if self.display_content:
msgs = self.thread.get_messages().keys()
msgs.sort()
lastcontent = ' '.join([m.get_text_content() for m in msgs])
contentstring = lastcontent.replace('\n', ' ').strip()
self.content_w = urwid.AttrMap(urwid.Text(contentstring,
wrap='clip'),
'threadline_content')
cols.append(self.content_w)
self.columns = urwid.Columns(cols, dividechars=1)
self.original_widget = self.columns
class WikipediaAPI(object):
API_HOST = config.get('api_host')
def __init__(self):
self._http_client = HttpClient(api_host=self.API_HOST)
@pytest.allure.step
def suggest(self, query=None, limit=None):
data = dict(action='opensearch')
if query:
data.update({'search': query})
if limit:
data.update({'limit': limit})
return self._http_client.request('/w/api.php', method='GET', data=data)
def make_celery():
from settings import app, config
global cel
CELERY = config.get("CELERY")
celery = Celery("twitter", **CELERY)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
if cel is None:
cel = celery
return cel
def single_file():
corpus = etree.Element("corpus")
for doc in db.find():
document = etree.SubElement(corpus, "document")
etree.SubElement(document, "title").text = doc["title"]
etree.SubElement(document, "content").text = doc["content"]
etree.SubElement(document, "date").text = str(doc["date"])
etree.SubElement(document, "url").text = doc["url"]
etree.SubElement(document, "source").text = doc["source"]
comments = etree.SubElement(document, "comments", amount=str(len(doc["comments"])))
for comment in doc["comments"]:
etree.SubElement(comments, "comment").text = comment
xml_str = etree.tostring(corpus, pretty_print=True)
outFile = open("{0}/Thesis/omsk_media.xml".format(config.get("home_path")), 'w')
outFile.write(xml_str)
outFile.close()
def most_commented_topics3():
"""
Как most_commented_topics2, только умножаем на распространённость темы
for num, i in enumerate(most_commented_topics3()):
print "{num} & {topic} & {percent}\\% \\\\".format(num=num+1, topic=i[0], percent=round(i[1]/466794.4839104599 * 100, 1))
"""
path = "{0}/Thesis/code/output/topics/most_commented_topic".format(config.get("home_path"))
data_exists = os.path.isfile(path)
if data_exists:
fileObject = open(path, 'rb')
result = pickle.load(fileObject)
else:
struct = {}
result = {}
topics_distr = general_topic_distribution()
for doc in db.docs_topics.find():
for topic, prob in doc["topics"]:
if topic not in struct:
struct[topic] = prob * doc["commentsCount"]
else:
struct[topic] += prob * doc["commentsCount"]
struct = struct.items()
for i in struct:
print i, topics_distr[i[0]]
result[i[0]] = i[1] / topics_distr[i[0]]
result = sorted(result.items(), key=lambda x: x[1], reverse=True)
pickle.dump(result, open(path, "wb"))
most_commented_value = result[0][1]
for num, i in enumerate(result):
print "{num} & {topic_id}. {topic_name} & {percent}\\% \\\\".format(num=num+1,
topic_id=i[0],
topic_name=config.topics_by_id(i[0]),
percent=round(i[1]/most_commented_value * 100, 1))
return result
def remove_duplicates(path):
#TODO не обрабатывается ситуация с разными баллами у одного слова
path = "{0}/Thesis/senti/dict/{1}/EmotionLookupTable.txt".format(config.get("home_path"), path)
dict = []
seen = set()
output = []
with open(path, 'r',) as dict_file:
for num, line in enumerate(dict_file):
if line.startswith(codecs.BOM_UTF8):
line = line[3:]
line = line.strip()
word = line.split('\t')[0].strip().decode("utf8")
score = int(line.split('\t')[1])
dict.append((word, score))
for item in dict:
if item[0] not in seen:
output.append(item)
seen.add(item[0])
with open(path, 'w') as dict_file:
for item in output:
dict_file.write((item[0] + "\t" + str(item[1]) + "\n").encode('utf-8'))
def create_access_token(user):
"""
Creates an access token for the user
parameters
==========
* user (auth.models.User) - a user object
returns
=======
access_token (string)
"""
from settings import config
data = {
'user_id': str(user.id),
'password': user.password,
'iat': datetime.utcnow(),
}
access_token = jwt.encode(data, config.get('pepper', ''), algorithm='HS256')
return access_token
def rebuild(self):
cols = []
# DATE
formatstring = config.get('general', 'timestamp_format')
newest = self.thread.get_newest_date()
if formatstring:
datestring = newest.strftime(formatstring)
else:
datestring = pretty_datetime(newest).rjust(10)
self.date_w = urwid.AttrMap(urwid.Text(datestring), 'threadline_date')
# SIZE
thread_size = self.thread.get_total_messages()
# Show number of messages only if there are at least 2 mails
# (save space in the line)
if thread_size>1 and thread_size<=20:
charcode = 0x2474 + thread_size
mailcountstring = unichr(charcode)
elif thread_size>1 and thread_size>20:
mailcountstring = "(%d)" % thread_size
else:
mailcountstring = " "
# TAGS
tags = self.thread.get_tags()
tags.sort()
tagstrings = []
for tag in tags:
tw = TagWidget(tag)
self.tag_widgets.append(tw)
tagstrings.append(('fixed', tw.width(), tw))
# AUTHORS
authors_string = self.thread.get_authors() or '(None)'
maxlength = config.getint('general', 'authors_maxlength')
authorsstring = shorten_author_string(authors_string, maxlength - len(mailcountstring))
offset = maxlength - len(authorsstring)
mailcountstring = mailcountstring.rjust(offset)
self.mailcount_w = urwid.AttrMap(urwid.Text(mailcountstring),
'threadline_mailcount')
self.authors_w = urwid.AttrMap(urwid.Text(authorsstring),
'threadline_authors')
# SUBJECT
subjectstring = self.thread.get_subject().strip()
self.subject_w = urwid.AttrMap(urwid.Text(subjectstring, wrap='clip'),
'threadline_subject')
# BODY
if self.display_content:
msgs = self.thread.get_messages().keys()
msgs.sort()
lastcontent = ' '.join([m.get_text_content() for m in msgs])
contentstring = lastcontent.replace('\n', ' ').strip()
self.content_w = urwid.AttrMap(urwid.Text(contentstring,
wrap='clip'),
'threadline_content')
# Set column order
#self.select = urwid.AttrMap(urwid.Text("[ ] ", wrap='clip'),
# 'threadline_subject')
#cols.append(('fixed', 4, self.select))
cols.append(('fixed', len(datestring), self.date_w))
cols.append(('fixed', len(authorsstring), self.authors_w))
cols.append(('fixed', len(mailcountstring), self.mailcount_w))
cols.extend(tagstrings)
if subjectstring:
cols.append(('fixed', len(subjectstring), self.subject_w))
if self.display_content:
cols.append(self.content_w)
self.columns = urwid.Columns(cols, dividechars=1)
self.original_widget = self.columns
# -*- coding: utf-8 -*-
from __future__ import division
from settings import config
from gensim import corpora, models, similarities, matutils
import numpy as np
import scipy.stats as stats
import pickle
import matplotlib.pyplot as plt
from matplotlib import rc
font = {
'family': config.get("tex_font_family"),
'weight': 'normal',
'size': config.get("tex_font_size")
}
rc('font', **font)
import scipy.sparse
from sparsesvd import sparsesvd
from pymongo import MongoClient
db = MongoClient().thesis
raw_tokens = db.raw_tokens
from analysis import LDA, get_corpus, get_dictionary
# Define KL function
def sym_kl(p, q):
return np.sum([stats.entropy(p, q), stats.entropy(q, p)])
def rebuild(self):
cols = []
if self.thread:
newest = self.thread.get_newest_date()
else:
newest = None
if newest == None:
datestring = u' ' * 10
else:
formatstring = config.get('general', 'timestamp_format')
if formatstring:
datestring = newest.strftime(formatstring)
else:
datestring = pretty_datetime(newest).rjust(10)
self.date_w = urwid.AttrMap(urwid.Text(datestring),
'search_thread_date')
cols.append(('fixed', len(datestring), self.date_w))
if self.thread:
mailcountstring = "(%d)" % self.thread.get_total_messages()
else:
mailcountstring = "(?)"
self.mailcount_w = urwid.AttrMap(urwid.Text(mailcountstring),
'search_thread_mailcount')
cols.append(('fixed', len(mailcountstring), self.mailcount_w))
if self.thread:
self.tag_widgets = [TagWidget(t) for t in self.thread.get_tags()]
else:
self.tag_widgets = []
self.tag_widgets.sort(tag_cmp,
lambda tag_widget: tag_widget.translated)
for tag_widget in self.tag_widgets:
cols.append(('fixed', tag_widget.width(), tag_widget))
if self.thread:
authors = self.thread.get_authors() or '(None)'
else:
authors = '(None)'
maxlength = config.getint('general', 'authors_maxlength')
authorsstring = shorten_author_string(authors, maxlength)
self.authors_w = urwid.AttrMap(urwid.Text(authorsstring),
'search_thread_authors')
cols.append(('fixed', len(authorsstring), self.authors_w))
if self.thread:
subjectstring = self.thread.get_subject().strip()
else:
subjectstring = ''
self.subject_w = urwid.AttrMap(urwid.Text(subjectstring, wrap='clip'),
'search_thread_subject')
if subjectstring:
cols.append(('weight', 2, self.subject_w))
if self.display_content:
if self.thread:
msgs = self.thread.get_messages().keys()
else:
msgs = []
msgs.sort()
lastcontent = ' '.join([m.get_text_content() for m in msgs])
contentstring = lastcontent.replace('\n', ' ').strip()
self.content_w = urwid.AttrMap(urwid.Text(contentstring,
wrap='clip'),
'search_thread_content')
cols.append(self.content_w)
self.columns = urwid.Columns(cols, dividechars=1)
self.original_widget = self.columns
