def merge_dicts(in_dicts_list, out_dict): result_dict = [] for dict in in_dicts_list: path_in = "{0}/Thesis/senti/dict/{1}".format(config.get("home_path"), dict) path_out = "{0}/Thesis/senti/dict/{1}".format(config.get("home_path"), out_dict) with open(path_in, 'r',) as dict_file: for num, line in enumerate(dict_file): if line.startswith(codecs.BOM_UTF8): line = line[3:] line = line.strip() print num, line word = line.split('\t')[0].strip().decode("utf8") score = int(line.split('\t')[1]) result_dict.append((word, score)) '''Удаляем дубликаты''' seen = set() output = [] for item in result_dict: if item[0] not in seen: output.append(item) seen.add(item[0]) output.sort(key=lambda tup: tup[0]) with open(path_out, 'w') as dict_out_file: for item in output: dict_out_file.write((item[0] + "\t" + str(item[1]) + "\n").encode('utf-8'))
def __init__(self): self.mhashes = {} # metric -> metric hash # self.hash_width = config.getint("pm_storage", "hash_width") self.key_mask = "!%dsL" % self.hash_width # Set key-value store class kvtype = config.get("pm_storage", "type") logger.info("Initializing %s storage. %d-byte wide hash", kvtype, self.hash_width) self.kvcls = load_name("noc.pm.db.storage", kvtype, KVStorage) if not self.kvcls: logger.critical("Invalid storage type: '%s'", kvtype) raise ValueError("Invalid storage type: '%s'" % kvtype) # Set partitioning scheme ps = config.get("pm_storage", "partition") logger.info("Setting %s partitioning scheme", ps) self.partition = load_name("noc.pm.db.partition", ps, Partition) # Index collection self.metrics = get_db()["noc.ts.metrics"] self.metrics_batch = self.metrics.initialize_ordered_bulk_op() self.new_metrics = 0 self.flush_lock = threading.Lock() self.epoch = int( time.mktime( time.strptime(config.get("pm_storage", "epoch"), "%Y-%m-%d"))) self.zero_hash = Binary("\x00" * self.hash_width)
def main(): try: logging.basicConfig(level=logging.DEBUG) parser = argparse.ArgumentParser() parser.add_argument('--host', default='127.0.0.1', type=str) parser.add_argument('--port', default=8000, type=int) args = parser.parse_args() config['HOST'] = config.get('HOST') or args.host config['PORT'] = config.get('PORT') or args.port middlewares = [ basic_auth_middleware, ] app = web.Application(middlewares=middlewares) executor = concurrent.futures.ThreadPoolExecutor( max_workers=config['NUM_THREADS']) loop = asyncio.get_event_loop() app['config'] = config setup_storage(loop, executor) setup_handlers(app, loop, executor) setup_routes(app) web.run_app(app, host=config['HOST'], port=config['PORT']) finally: logging.info('Exiting app')
def run_migrations_offline(): """Run migrations in 'offline' mode. This configures the context with just a URL and not an Engine, though an Engine is acceptable here as well. By skipping the Engine creation we don't even need a DBAPI to be available. Calls to context.execute() here emit the given string to the script output. """ # postgresql://postgres:postgres@localhost/oracles_vision # url = context_config.get_main_option("sqlalchemy.url") init_settings() url = 'postgresql://%s:%s@%s/%s' % (config.get( 'DB', 'user'), config.get('DB', 'password'), config.get( 'DB', 'host'), config.get('DB', 'database')) context.configure(url=url, target_metadata=target_metadata, literal_binds=True) with context.begin_transaction(): context.run_migrations()
def configure(cls): rules = {} for opt in config.options(cls.CONFIG_SECTION): if opt.startswith("map."): # Process metric mappings _, model_id = opt.split(".", 1) v = config.get(cls.CONFIG_SECTION, opt) logger.info("Configuring metric mappings %s -> %s", model_id, v) try: cls._TEMPLATES[model_id] = Template(v) except TemplateSyntaxError, why: logging.error("Template syntax error: %s", why) elif opt.startswith("metric."): # Process metric rules n = int(opt.split(".")[1]) v = config.get(cls.CONFIG_SECTION, opt) try: rx = re.compile(v) except re.error, why: logging.error( "Invalid regular expression in rule %s: %s", n, why ) continue
def avg_comments(): width = 0.8 data = { "bk55": 8.5272094345, "gorod55": 10.6726493011, "ngs55": 9.88509836752, "omskinform": 2.71181391085 } sources = data.keys() data = [(source, round(avg, 2)) for source, avg in sorted(data.items(), key=lambda x: x[1])] sources_x = np.arange(len(data)) numbers = [number for source, number in data] fig, ax = plt.subplots() ax.set_xlabel(u'Источник') ax.set_ylabel(u'Среднее количество комментариев') rects = ax.bar(sources_x, numbers, width) plt.xticks(sources_x + 0.4, sources) for rect, label in zip(rects, sources): height = rect.get_height() ax.text(rect.get_x() + rect.get_width()/2, height, height, ha='center', va='bottom') for ext in config.get("tex_image_format"): plt.savefig(config.get("tex_image_path") + "avg_comments." + ext, bbox_inches='tight', format=ext, dpi=1200) plt.show()
def is_admin(bot, update): if update.message.text == config.get('global', 'bot_token'): if config.get('global', 'admin') != update.message.from_user.id: update_parameter('global', 'admin', update.message.from_user.id) log.info('[TG] Установлен администратор бота с ID {0}'.format( update.message.from_user.id)) update.message.reply_text( 'Доступ получен.\n\nКоманда просмотра всех доступных команд: /help' )
def mallet_plot(): ## Строим график из данных Mallet topics = [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100] perp = [514.1302184469, 525.0241590983, 534.4977260582, 538.5513127672, 541.8989344152, 541.0094559616, 538.4206752508, 535.3505225192, 532.7629698852, 533.9200808978, 540.3049166531, 531.4757292891, 538.0214941918, 546.4364835315, 538.0214941918, 540.9982061032, 532.1429352736, 535.5732148468, 536.4983783528, 539.0330790911] plt.plot(topics, perp, 'o-') plt.ylabel(u'Перплексия') plt.xlabel(u'Количество тем') plt.grid(True) plt.xlim(0, 105) for ext in config.get("tex_image_format"): plt.savefig(config.get("tex_image_path") + "perplexity_mallet." + ext, format=ext, bbox_inches='tight', dpi=1200)
def send_telegram(message: str): """ Send a Telegram message to a bot first a telegram bot must be created https://www.giga.de/apps/telegram/tipps/telegram-bot-erstellen-loeschen-andere-befehle-so-geht-s/ """ bot_token = config.get("telegram", "bot_token") bot_chat_id = config.get("telegram", "bot_chat_id") send_text = f"https://api.telegram.org/bot{bot_token}/sendMessage?chat_id={bot_chat_id}&parse_mode=Markdown&text={message}" response = requests.get(send_text) return response.json()
def mongo_factory(): """ wraps getting a mongo client reading defaults from the settings """ from settings import config hosts = config.get('mongo.hosts', 'localhost:27017') database = config.get('mongo.database', 'test') user = config.get('mongo.user', '') password = config.get('mongo.password', '') hosts = hosts.split(',') return mongo_client(hosts, database, user, password)
def send_post(bot, domain, post): log.info("[TG] Отправка поста...") if post.text: if post.photos: pass else: bot.sendMessage(chat_id=config.get(domain, 'channel'), text=post.text, parse_mode='HTML', disable_web_page_preview=True) if post.photos: # noinspection PyBroadException try: if post.text: if len(post.photos) == 1 and len(post.text) < 200: bot.sendPhoto(chat_id=config.get(domain, 'channel'), photo=post.photos[0]['media'], caption=post.text, parse_mode='HTML') else: bot.sendMessage(chat_id=config.get(domain, 'channel'), text=post.text, parse_mode='HTML', disable_web_page_preview=True) bot.sendMediaGroup(chat_id=config.get(domain, 'channel'), media=post.photos) else: bot.sendMediaGroup(chat_id=config.get(domain, 'channel'), media=post.photos) except Exception: log.warning('[TG] Невозможно отправить фото: {0}.'.format( sys.exc_info()[1])) for m in post.videos: try: bot.sendVideo(chat_id=config.get(domain, 'channel'), video=open(m, 'rb'), timeout=60) except Exception: pass for m in post.docs: bot.sendDocument(chat_id=config.get(domain, 'channel'), document=open(m, 'rb'), timeout=60) for (m, n) in post.tracks: try: if getsize(m) > 52428800: remove(m) else: try: bot.sendAudio(chat_id=config.get(domain, 'channel'), audio=open(m, 'rb'), duration=int(n), timeout=60) except: pass remove(m) except FileNotFoundError: continue
def compare(self): login_data = {"pin": config.get("website", "pin")} session = requests.Session() header = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0", } raw_request = session.post(self.__url, data=login_data, headers=header) if raw_request.status_code != 200: send_telegram( "Fehler beim Anmelden auf der Seite des Testergebnisses. Das Programm wird beendet" ) exit() request = raw_request.json() message = str() try: message = request["results"][0]["headline"] message += " Das Programm wird nun automatisch beendet." except: send_telegram( "Testergebnis hat sich verändert. Das Programm wird nun automatisch beendet." ) exit() if message != "Ihre Probe ist auf dem Weg ins Labor.": send_telegram(message) exit() return
def send_post(bot, update): if str(config.get('global', 'admin')) == str(update.message.from_user.id): global chat chat = update.message['text'][11:] update.message.reply_text( 'Чтобы отправить ваш пост в канал/группу, ответьте на это сообщение.' )
def GET(self): params = web.input(raw_msg="", msisdn="", form="") web.header("Content-Type", "application/json; charset=utf-8") year, week = get_current_week() myparams = { "username": config["dispatcher2_username"], "password": config["dispatcher2_password"], "ctype": "text", "raw_msg": params.raw_msg, "year": year, "week": week, "msisdn": params.msisdn, "is_qparams": 't', "extras": "{}", "source": "mtrackpro", "destination": KEYWORD_SERVER_MAPPINGS.get(params.form, "localhost") } payload = "message={0}&originator={1}".format(params.raw_msg, params.msisdn) queueEndpoint = config.get("dispatcher2_queue_url", "http://localhost:9191/queue?") # print("Call=>", queueEndpoint) requests.post( queueEndpoint, data=payload, params=myparams, headers={"Content-type": "text/plain"}) # print("RESP:===>", resp.text) return json.dumps({"status": "success"})
def source_read(app, docname, source): """ prepend header to every file """ section = "manual-%s" % app.config.language header_file = config.get(section, "header-file") if header_file and os.path.exists(header_file): with open(header_file, "rb") as f: source[0] = "%s\n%s" % (f.read().decode('utf-8'), source[0])
def file_every_1000(): for num, doc in enumerate(db.find()): if fmod(num, 1000) == 0: corpus = etree.Element("corpus") begin = num end = begin + 1000 if end > db.find().count(): end = db.find().count() - 1 print "begin, end ", begin, end for doc in db.find()[begin:end]: document = etree.SubElement(corpus, "document") etree.SubElement(document, "title").text = doc["title"] etree.SubElement(document, "content").text = doc["content"] etree.SubElement(document, "date").text = str(doc["date"]) etree.SubElement(document, "url").text = doc["url"] etree.SubElement(document, "source").text = doc["source"] comments = etree.SubElement(document, "comments", amount=str(len(doc["comments"]))) for comment in doc["comments"]: etree.SubElement(comments, "comment").text = comment xml_str = etree.tostring(corpus, pretty_print=True) with open("{0}/Thesis/omsk_media_{1}-{2}.xml".format(config.get("home_path"), begin, end), 'w') as outFile: outFile.write(xml_str)
def status(bot, update): if str(config.get('global', 'admin')) == str(update.message.from_user.id): stat = 'Автопостинг остановлен' if job_status: if job_status.enabled: stat = 'Автопостинг запущен' update.message.reply_text(stat, quote=True)
def update_price_history(self, c, timestamp, price): monitored_at = datetime.datetime.fromtimestamp(timestamp).strftime('%H:%M:%S') if len(c['history_live']) >= config.get('history_live_window'): c['history_live'] = c['history_live'][1:] c['history_live'].append({'timestamp': monitored_at, 'price': price}) self.store({'stock': c.get('stock'), 'timestamp': timestamp, 'price': price}) return c
def __init__(self, tag): self.tag = tag self.translated = config.get('tag-translate', tag, fallback=tag) self.txt = urwid.Text(self.translated.encode('utf-8'), wrap='clip') normal = config.get_tagattr(tag) focus = config.get_tagattr(tag, focus=True) urwid.AttrMap.__init__(self, self.txt, normal, focus)
def send_post(bot, update, args): global chat if len(args) != 0: chat = args[0] else: chat = config.get('global', 'main_group') update.message.reply_text('Чтобы отправить ваш пост в канал/группу {}, ответьте на это сообщение.'.format(chat), reply_markup=ForceReply())
def stop(bot, update): if str(config.get('global', 'admin')) == str(update.message.from_user.id): global job_status if job_status: if job_status.enabled: job_status.enabled = False update.message.reply_text('Бот остановлен.', quote=True) log.info('Stopping a job_status...')
def source_list(bot, update): sources_list = config.sections()[1:] sources = 'Список источников:\nИсточник' + '\x20'*8 + '---->' + '\x20'*8 + 'Назначение\n\n' for source in sources_list: sources += 'https://vk.com/' + source + '\x20'*8 + '---->' + '\x20'*8 + config.get(source, 'channel') + '\n' sources += 'Для удаления источника отправьте команду /remove <домен группы вк>\nНапример, /remove ' +\ sources_list[randint(0, len(sources_list) - 1)] update.message.reply_text(sources, disable_web_page_preview=True)
def get_datestring(self): """returns formated datestring""" formatstring = config.get('general', 'timestamp_format') if formatstring: res = self._datetime.strftime(formatstring) else: res = helper.pretty_datetime(self._datetime) return res
def print_result(evaluate_set, predict, accuracy, f1, precision, recall): print("accuracy: ", str(accuracy * 100) + "%") print("f1:", f1) print("precision:", precision) print("recall:", recall) if config.get("print-predict?", False): for i in range(10): print(evaluate_set[i], "predict:", predict[i])
def expand_words(path_in, path_out): path_in = "{0}/Thesis/senti/dict/{1}/EmotionLookupTable.txt".format(config.get("home_path"), path_in) path_out = "{0}/Thesis/senti/dict/{1}/EmotionLookupTable.txt".format(config.get("home_path"), path_out) morph = pymorphy2.MorphAnalyzer() with open(path_in, "r") as infile, open(path_out, 'w') as outfile: #Очищаем файл outfile.truncate() for num, line in enumerate(infile): # http://en.wikipedia.org/wiki/Byte_order_mark#UTF-8 nline = line if nline.startswith(codecs.BOM_UTF8): nline = nline[3:] nline = nline.strip() word = nline.split('\t')[0].strip().decode("utf8") score = int(nline.split('\t')[1]) if "*" not in word: """ Берём слова """ stemmed = stem_snowball(word) exclude_words = [u"зря"] if len(stemmed) < 4: """Слишком короткие заменяем всеми словоформами""" if word in exclude_words: outfile.write(word.encode('utf-8') + "\t" + str(score).encode('utf-8') + "\n") else: for form in get_all_forms(word): outfile.write(form.encode('utf-8') + "\t" + str(score).encode('utf-8') + "\n") if len(stemmed) >= 4: """Для некоротких создаём правило на основе стемминга""" if len(stemmed) < len(word): """ Если стемминг имел место, заменяем усечённую часть * """ result_word = stemmed + "*" outfile.write(result_word.encode('utf-8') + "\t" + str(score).encode('utf-8') + "\n") else: """ Иначе оставляем как было """ result_word = word # == stemmed outfile.write(result_word.encode('utf-8') + "\t" + str(score).encode('utf-8') + "\n") else: outfile.write((word + "\t" + str(score) + "\n").encode('utf-8'))
def init_locale(self): self.locale = self.state.document.settings.env.config.language t = gettext.translation('messages', localedir=os.path.join( root_dir, config.get("DEFAULT", "locale")), languages=[self.locale]) t.install(**kwargs) self.init_type_mapping()
def _create_connection(self): if not self.connection: pgsql = config.get('pgsql', {}) self.connection = psycopg2.connect(dbname=pgsql.get('name'), user=pgsql.get('user'), password=pgsql.get('password'), host=pgsql.get('host'), port=pgsql.get('port')) return self.connection
def get_last_logs(bot, update, args): if str(config.get('global', 'admin')) == str(update.message.from_user.id): if args: string = int(args[0]) else: string = 15 with open('../bot_log.log', 'r', encoding='utf-8') as f: last_logs = ''.join(f.readlines()[-string:]) update.message.reply_text('Последние {} строк логов:\n\n'.format(str(string)) + last_logs, quote=True)
def __new__(cls, *arguments, **keywords): ''' The class that will be created will be a subclass in the config file: General -> exporter ''' for subclass in Exporter.__subclasses__(): class_str = subclass.__module__ + "." + subclass.__name__ if class_str == config.get('General','exporter'): return super(cls, subclass).__new__(subclass) raise Exception, 'Invalid Exporter! Use one of: %s'%([c.__name__ for c in Exporter.__subclasses__()])
def explain(self): p = subprocess.Popen(shlex.split("java -jar {0}/Thesis/senti/SentiStrengthCom.jar stdin noDictionary illegalDoubleLettersInWordMiddle" " ёйухцчщьыъ illegalDoubleLettersAtWordEnd абвгджзйкоуфхцчщэ UTF8 urlencoded explain sentidata " "{0}/Thesis/senti/dict/{1}/".format(config.get("home_path"), self.dict_name)), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout_text, stderr_text = p.communicate(self.text.replace(" ", "+")) print self.text.replace(" ", "+") if stderr_text: raise Exception("Ошибка в модуле оценки эмоций:" + stderr_text) return stdout_text
def init(): from settings import config host = config.get('restAPI', 'host', fallback='0.0.0.0') port = config.getint('restAPI', 'port', fallback=5000) debug = config.getboolean('restAPI', 'debug', fallback=False) app.debug = debug global server server = WebSocketServer((host, port), app, handler_class=WebSocketHandler) server.serve_forever()
def comments_count_by_day_complete(): date_dict = comments_count_by_day() weekend_comments = [] weekday_comments = [] a = comments_count_by_day().items() a = sorted(a, key=lambda x: x[0], reverse=False) a = [(date, scores[1]) for date, scores in a] fig, ax = plt.subplots() ax.plot(*zip(*a)) for i, single_date in enumerate(daterange()): print i if single_date.weekday() == 5: x_list = [single_date, single_date + datetime.timedelta(1)] y_list = [date_dict[single_date][1], date_dict[single_date + datetime.timedelta(1)][1]] weekend_comments.extend(y_list) print x_list ax.fill_between(x_list, y_list, alpha=0.3, color='#FF8E8E') elif (single_date.weekday() != 5) and (single_date.weekday() != 6): weekday_comments.append(date_dict[single_date][1]) print single_date.weekday() ax.set_ylabel(u'Количество комментариев за день') ax.set_xlabel(u'Дата') red_patch = mpatches.Patch(color='#FF8E8E', alpha=0.4, label=u"Суббота и воскресенье") plt.legend(handles=[red_patch], fontsize=14) ax.format_xdata = mdates.DateFormatter('%Y-%m-%d') fig.autofmt_xdate() # поворачивает надписи даты ax.grid(True) for ext in config.get("tex_image_format"): plt.savefig(config.get("tex_image_path") + "comments_by_day." + ext, format=ext, bbox_inches='tight', dpi=1200) plt.show() print weekend_comments print weekday_comments print sum(weekend_comments), len(weekend_comments) print sum(weekday_comments), len(weekday_comments)
def button(bot, update): # TODO Доделать управление источниками через меню query = update.callback_query if query.data == '3': bot.forward_message(chat_id=config.get('global', 'main_group'), from_chat_id=message.chat.id, message_id=message.message_id) bot.edit_message_text(text='Принято', chat_id=query.message.chat_id, message_id=query.message.message_id) elif query.data == '4': bot.edit_message_text(text='Отклонено', chat_id=query.message.chat_id, message_id=query.message.message_id)
def check_if_taskqueue(self, *args, **kwargs): """ Check if it is executed by Taskqueue in Staging or Production Allow run in localhost calling the url """ if self.request.headers.get('X-AppEngine-TaskName') is None \ and config.get('environment') == "production" \ and not users.is_current_user_admin(): return self.error(403) else: return handler(self, *args, **kwargs)
def perplexity(): # split into train and test - random sample, but preserving order perwordbound_list = [] perplexity2_list = [] topics_list = [] train_size = int(round(len(corpus)*0.9)) train_index = sorted(random.sample(xrange(len(corpus)), train_size)) test_index = sorted(set(xrange(len(corpus)))-set(train_index)) train_corpus = [corpus[i] for i in train_index] test_corpus = [corpus[j] for j in test_index] for num_topics in range(5, 101, 5): model = LDA(dictionary, train_corpus, num_topics, "lda20/lda_training_{num_topics}" .format(num_topics=str(num_topics)), passes=20) perwordbound = model.log_perplexity(test_corpus) perplexity2 = np.exp2(-perwordbound) perwordbound_list.append(perwordbound) perplexity2_list.append(perplexity2) topics_list.append(num_topics) plt.plot(topics_list, perwordbound_list, 'o-') plt.ylabel(u'Per word bound') plt.xlabel(u'Количество тем') plt.grid(True) plt.xlim(0, 105) for ext in config.get("tex_image_format"): plt.savefig(config.get("tex_image_path") + "perplexity_perwordbound." + ext, bbox_inches='tight', format=ext, dpi=1200) plt.close() plt.plot(topics_list, perplexity2_list, 'o-') plt.ylabel(u'Перплексия') plt.xlabel(u'Количество тем') plt.grid(True) plt.xlim(0, 105) for ext in config.get("tex_image_format"): plt.savefig(config.get("tex_image_path") + "perplexity_exp2." + ext, format=ext, bbox_inches='tight', dpi=1200) plt.close()
def render(filename, **args): """ define mako render function """ try: mytemplate = _lookup.get_template(filename) args["sitename"] = config.get("sitename") args["cdate"] = datetime.datetime.now().strftime("%Y-%m-%d") args['session'] = web.ctx.session args["ctx"] = web.ctx args["current_css"] = current_css args["is_select"] = is_select return mytemplate.render(**args) except: return exceptions.text_error_template().render()
def sending(bot, update): global chat, message if str(config.get('global', 'admin')) == str(update.message.from_user.id) and chat: if update.message.text: bot.send_message(chat_id=chat, text=update.message.text, parse_mode='Markdown') elif update.message.document: bot.send_document(chat_id=chat, document=update.message.document.file_id, caption=update.message.caption, parse_mode='Markdown') elif update.message.photo: bot.send_photo(chat_id=chat, photo=update.message.photo[0].file_id, caption=update.message.caption, parse_mode='Markdown') elif update.message.video: bot.send_video(chat_id=chat, video=update.message.video.file_id, caption=update.message.caption, parse_mode='Markdown') bot.send_media_group(chat_id=chat, media=update.message.media_group_id) chat = None else: button_list = [InlineKeyboardButton('Принять', callback_data="3"), InlineKeyboardButton('Отклонить', callback_data="4")] reply_markup = InlineKeyboardMarkup(build_menu(button_list, n_cols=2)) message = bot.forward_message(chat_id=config.get('global', 'admin'), from_chat_id=update.message.chat.id, message_id=update.message.message_id) bot.send_message(chat_id=config.get('global', 'admin'), text='Что сделать с сообщением выше?', reply_markup=reply_markup)
def watch(self): while True: for idx, c in enumerate(self.watchlist): try: timestamp = time.time() stat = urllib2.urlopen(self.source % c.get('stock')).read().strip() self.watchlist[idx] = self.update_price_history(c, timestamp, float(stat)) self.watchlist[idx] = self.update_market_values_for_holdings(c) except Exception as ex: print (ex.message) sys.exit(1) self.present() time.sleep(config.get('update_frequency_sec'))
def render(filename,**args): """ define mako render function """ try: mytemplate = _lookup.get_template(filename) args["sitename"] = config.get("sitename") args["cdate"] = datetime.datetime.now().strftime( "%Y-%m-%d") args['session'] = web.ctx.session args["ctx"] = web.ctx args["current_css"] = current_css args["is_select"] = is_select return mytemplate.render(**args) except: return exceptions.text_error_template().render()
def run(bot, update): if str(config.get('global', 'admin')) == str(update.message.from_user.id): try: global job_status if job_status: if job_status.enabled: return 0 job_status = job_queue.run_repeating(job_repeated, interval=5 * 60, first=0) log.info('Running a job_status...') update.message.reply_text('Бот запущен', quote=True) except Exception: log.info('Got an error while running a job_status: %s.' % sys.exc_info()[0]) update.message.reply_text('Не удалось запустить бота: {}'.format(sys.exc_info()[1]), quote=True)
def build_portfolio(self): portfolio_data = json.load(open(config.get('paths').get('portfolio_file_path'), 'r')) portfolio = [] for stock in portfolio_data: portfolio.append({ 'stock': stock.get('code'), 'history_live': [{'time': '', 'price': 0.00},], 'holdings': map(self.stock_record, stock.get('holdings')), 'announcement_dates': {}, 'news': {}, }) return portfolio
def number_of_docs_by_day(): date_dict = {date: db.docs_topics.find({"date": date}).count() for date in daterange()} print date_dict weekend = [] weekday = [] fig, ax = plt.subplots() for i, single_date in enumerate(daterange()): print i if single_date.weekday() == 5: x_list = [single_date, single_date + datetime.timedelta(1)] y_list = [date_dict[single_date], date_dict[single_date + datetime.timedelta(1)]] weekend.extend(y_list) print x_list ax.fill_between(x_list, y_list, alpha=0.3, color='#FF8E8E') elif (single_date.weekday() != 5) and (single_date.weekday() != 6): weekday.append(date_dict[single_date]) print single_date.weekday() red_patch = mpatches.Patch(color='#FF8E8E', alpha=0.4, label=u"Суббота и воскресенье") plt.legend(handles=[red_patch], fontsize=14) a = sorted(date_dict.items(), key=lambda x: x[0], reverse=False) ax.plot(*zip(*a)) ax.set_ylabel(u'Количество статей') ax.set_xlabel(u'Дата') ax.format_xdata = mdates.DateFormatter('%Y-%m-%d') fig.autofmt_xdate() # поворачивает надписи даты fig.set_figwidth(11) ax.grid(True) for ext in config.get("tex_image_format"): plt.savefig(config.get("tex_image_path") + "docs_by_day." + ext, format=ext, bbox_inches='tight', dpi=1200) plt.show() print sum(weekend), len(weekend), sum(weekend)/len(weekend) print sum(weekday), len(weekday), sum(weekday)/len(weekday)
def __init__(self): self.profile = global_settings.get('general', 'profile') if os.path.exists(os.path.join(os.getcwd(), 'profiles', self.profile)): try: import importlib self.settings = importlib.import_module( "profiles.%s.settings" % self.profile) self.parameters = self.buildRobotParameters() logging.info('profile: %s', self.profile) except ImportError as ex: self.settings = None logging.error('%s: %s', self.profile, ex.message) else: logging.error('directory <%s> does not exists')
def GET(self, id): web.header("Content-Type", "application/json; charset=utf-8") r = db.query( "SELECT waybill, destination_id, quantity_bales FROM distribution_log_w2sc_view " "WHERE id = $id", {'id': id}) if r: res = r[0] msg = 'Please send "REC %s %s" to %s if you received %s bales of nets with waybill %s.' to_subcounty = res['destination_id'] msg = msg % (res['waybill'], res['quantity_bales'], config.get('shortcode', '6400'), res['quantity_bales'], res['waybill']) ret = {"sms": msg, "to_subcounty": to_subcounty} return json.dumps(ret)
def validate_access_token(access_token): """ This request requires validation. To get an access token use the ``/authenticate`` endpoint. """ """ Check to see if the access_token is valid The access token will invalidate if the user changes their password. parameters ========== * access_token - a token created with create_access_token returns ======= returns True or False depending on if the access_token is valid """ from settings import config validated = False try: decoded = jwt.decode(access_token, config.get('pepper', ''), algorithms=['HS256']) except jwt.DecodeError: logger.debug('jwt DecodeError') else: now = datetime.utcnow() then = datetime.fromtimestamp(decoded['iat']) age = now - then user = User.get_by_id(decoded['user_id']) # TODO: make the age configurable # for now the access_token is valid for 5 hours if age.seconds > 60 * 60 * 5: logger.debug('stale access token, timestamp expired') elif user and decoded['password'] == user.password: validated = { 'user': user, } logger.debug('Valid access token for user:{}'.format(user.id)) else: logger.debug('access token failed to validate') return validated
def __init__(self, tag): self.tag = tag #self.translated = self.translated.encode('utf-8') #self.txt = urwid.Text(self.translated, wrap='clip') #normal = config.get_tagattr(tag) normal = config.get_tagattr(tag) focus = config.get_tagattr(tag, focus=True) self.translated = config.get('tag-translate', tag, fallback=tag) self.txt = urwid.Text(self.translated)#, wrap='space') self.focus_palette = normal self.unfocus_palette = normal urwid.AttrMap.__init__(self, self.txt, self.unfocus_palette, self.focus_palette)
def senti_tuple(self): #open a subprocess using shlex to get the command line string into the correct args list format # добавить параметр explain для объяснения вывода изменить на p = subprocess.Popen(shlex.split("java -jar {0}/Thesis/senti/SentiStrengthCom.jar stdin noDictionary illegalDoubleLettersInWordMiddle" " ёйухцчщьыъ illegalDoubleLettersAtWordEnd абвгджзйкоуфхцчщэ UTF8 urlencoded sentidata " "{0}/Thesis/senti/dict/{1}/".format(config.get("home_path"), self.dict_name)), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) #communicate via stdin the string to be rated. Note that all spaces are replaced with + stdout_text, stderr_text = p.communicate(self.text.replace(" ", "+")) #remove the tab spacing between the positive and negative ratings. e.g. 1 -5 -> 1-5 stdout_text = stdout_text.rstrip().replace("\t","") if stderr_text: raise Exception("Ошибка в модуле оценки эмоций:" + stderr_text) senti_tuple = (int(stdout_text.split("-")[0]), -int(stdout_text.split("-")[1])) return senti_tuple
def doc_number(): width = 0.8 data = { "bk55": 14078, "gorod55": 6302, "ngs55": 4780, "omskinform": 8727 } sources_x = np.arange(len(data)) numbers = [number for source, number in data.items()] fig, ax = plt.subplots() ax.set_xlabel(u'Источник') ax.set_ylabel(u'Количество статей') rects = ax.bar(sources_x, numbers, width) plt.xticks(sources_x + 0.4, data.keys()) for rect, label in zip(rects, data.keys()): height = rect.get_height() ax.text(rect.get_x() + rect.get_width()/2, height + 5, height, ha='center', va='bottom') for ext in config.get("tex_image_format"): plt.savefig(config.get("tex_image_path") + "doc_number." + ext, bbox_inches='tight', format=ext, dpi=1200) plt.show()
def get_datestring(self): """ returns reformated datestring for this messages. It uses the format spacified by `timestamp_format` in the general section of the config. """ if self._datetime == None: return None if config.has_option('general', 'timestamp_format'): formatstring = config.get('general', 'timestamp_format') res = self._datetime.strftime(formatstring) else: res = helper.pretty_datetime(self._datetime) return res
def update(request_kwargs=None): global updater, dp, job_queue, job updater = Updater(config.get('global', 'bot_token'), request_kwargs=request_kwargs) dp = updater.dispatcher job_queue = updater.job_queue dp.add_error_handler(error) dp.add_handler(CommandHandler('start', commands.start)) dp.add_handler(CommandHandler('help', commands.help)) dp.add_handler(CommandHandler('run', commands.run)) dp.add_handler(CommandHandler('stop', commands.stop)) dp.add_handler(CommandHandler('get_full_logs', commands.get_full_logs)) dp.add_handler(CommandHandler('get_last_logs', commands.get_last_logs)) dp.add_handler(CommandHandler('status', commands.status)) dp.add_handler(MessageHandler(callback=commands.is_admin, filters=Filters.regex( config.get('global', 'bot_token'))), group=1) dp.add_handler(CommandHandler('send_post', commands.send_post)) dp.add_handler( MessageHandler(callback=commands.sending, filters=Filters.reply)) job = job_queue.run_repeating(commands.job_repeated, interval=5 * 60, first=0)
def rebuild(self): cols = [] formatstring = config.get('general', 'timestamp_format') newest = self.thread.get_newest_date() if formatstring: datestring = newest.strftime(formatstring) else: datestring = pretty_datetime(newest).rjust(10) self.date_w = urwid.AttrMap(urwid.Text(datestring), 'threadline_date') cols.append(('fixed', len(datestring), self.date_w)) mailcountstring = "(%d)" % self.thread.get_total_messages() self.mailcount_w = urwid.AttrMap(urwid.Text(mailcountstring), 'threadline_mailcount') cols.append(('fixed', len(mailcountstring), self.mailcount_w)) tags = self.thread.get_tags() tags.sort() for tag in tags: tw = TagWidget(tag) self.tag_widgets.append(tw) cols.append(('fixed', tw.width(), tw)) authors = self.thread.get_authors() or '(None)' maxlength = config.getint('general', 'authors_maxlength') authorsstring = shorten(authors, maxlength).strip() self.authors_w = urwid.AttrMap(urwid.Text(authorsstring), 'threadline_authors') cols.append(('fixed', len(authorsstring), self.authors_w)) subjectstring = self.thread.get_subject().strip() self.subject_w = urwid.AttrMap(urwid.Text(subjectstring, wrap='clip'), 'threadline_subject') if subjectstring: cols.append(('fixed', len(subjectstring), self.subject_w)) if self.display_content: msgs = self.thread.get_messages().keys() msgs.sort() lastcontent = ' '.join([m.get_text_content() for m in msgs]) contentstring = lastcontent.replace('\n', ' ').strip() self.content_w = urwid.AttrMap(urwid.Text(contentstring, wrap='clip'), 'threadline_content') cols.append(self.content_w) self.columns = urwid.Columns(cols, dividechars=1) self.original_widget = self.columns
class WikipediaAPI(object): API_HOST = config.get('api_host') def __init__(self): self._http_client = HttpClient(api_host=self.API_HOST) @pytest.allure.step def suggest(self, query=None, limit=None): data = dict(action='opensearch') if query: data.update({'search': query}) if limit: data.update({'limit': limit}) return self._http_client.request('/w/api.php', method='GET', data=data)
def make_celery(): from settings import app, config global cel CELERY = config.get("CELERY") celery = Celery("twitter", **CELERY) TaskBase = celery.Task class ContextTask(TaskBase): abstract = True def __call__(self, *args, **kwargs): with app.app_context(): return TaskBase.__call__(self, *args, **kwargs) celery.Task = ContextTask if cel is None: cel = celery return cel
def single_file(): corpus = etree.Element("corpus") for doc in db.find(): document = etree.SubElement(corpus, "document") etree.SubElement(document, "title").text = doc["title"] etree.SubElement(document, "content").text = doc["content"] etree.SubElement(document, "date").text = str(doc["date"]) etree.SubElement(document, "url").text = doc["url"] etree.SubElement(document, "source").text = doc["source"] comments = etree.SubElement(document, "comments", amount=str(len(doc["comments"]))) for comment in doc["comments"]: etree.SubElement(comments, "comment").text = comment xml_str = etree.tostring(corpus, pretty_print=True) outFile = open("{0}/Thesis/omsk_media.xml".format(config.get("home_path")), 'w') outFile.write(xml_str) outFile.close()
def most_commented_topics3(): """ Как most_commented_topics2, только умножаем на распространённость темы for num, i in enumerate(most_commented_topics3()): print "{num} & {topic} & {percent}\\% \\\\".format(num=num+1, topic=i[0], percent=round(i[1]/466794.4839104599 * 100, 1)) """ path = "{0}/Thesis/code/output/topics/most_commented_topic".format(config.get("home_path")) data_exists = os.path.isfile(path) if data_exists: fileObject = open(path, 'rb') result = pickle.load(fileObject) else: struct = {} result = {} topics_distr = general_topic_distribution() for doc in db.docs_topics.find(): for topic, prob in doc["topics"]: if topic not in struct: struct[topic] = prob * doc["commentsCount"] else: struct[topic] += prob * doc["commentsCount"] struct = struct.items() for i in struct: print i, topics_distr[i[0]] result[i[0]] = i[1] / topics_distr[i[0]] result = sorted(result.items(), key=lambda x: x[1], reverse=True) pickle.dump(result, open(path, "wb")) most_commented_value = result[0][1] for num, i in enumerate(result): print "{num} & {topic_id}. {topic_name} & {percent}\\% \\\\".format(num=num+1, topic_id=i[0], topic_name=config.topics_by_id(i[0]), percent=round(i[1]/most_commented_value * 100, 1)) return result
def remove_duplicates(path): #TODO не обрабатывается ситуация с разными баллами у одного слова path = "{0}/Thesis/senti/dict/{1}/EmotionLookupTable.txt".format(config.get("home_path"), path) dict = [] seen = set() output = [] with open(path, 'r',) as dict_file: for num, line in enumerate(dict_file): if line.startswith(codecs.BOM_UTF8): line = line[3:] line = line.strip() word = line.split('\t')[0].strip().decode("utf8") score = int(line.split('\t')[1]) dict.append((word, score)) for item in dict: if item[0] not in seen: output.append(item) seen.add(item[0]) with open(path, 'w') as dict_file: for item in output: dict_file.write((item[0] + "\t" + str(item[1]) + "\n").encode('utf-8'))
def create_access_token(user): """ Creates an access token for the user parameters ========== * user (auth.models.User) - a user object returns ======= access_token (string) """ from settings import config data = { 'user_id': str(user.id), 'password': user.password, 'iat': datetime.utcnow(), } access_token = jwt.encode(data, config.get('pepper', ''), algorithm='HS256') return access_token
def rebuild(self): cols = [] # DATE formatstring = config.get('general', 'timestamp_format') newest = self.thread.get_newest_date() if formatstring: datestring = newest.strftime(formatstring) else: datestring = pretty_datetime(newest).rjust(10) self.date_w = urwid.AttrMap(urwid.Text(datestring), 'threadline_date') # SIZE thread_size = self.thread.get_total_messages() # Show number of messages only if there are at least 2 mails # (save space in the line) if thread_size>1 and thread_size<=20: charcode = 0x2474 + thread_size mailcountstring = unichr(charcode) elif thread_size>1 and thread_size>20: mailcountstring = "(%d)" % thread_size else: mailcountstring = " " # TAGS tags = self.thread.get_tags() tags.sort() tagstrings = [] for tag in tags: tw = TagWidget(tag) self.tag_widgets.append(tw) tagstrings.append(('fixed', tw.width(), tw)) # AUTHORS authors_string = self.thread.get_authors() or '(None)' maxlength = config.getint('general', 'authors_maxlength') authorsstring = shorten_author_string(authors_string, maxlength - len(mailcountstring)) offset = maxlength - len(authorsstring) mailcountstring = mailcountstring.rjust(offset) self.mailcount_w = urwid.AttrMap(urwid.Text(mailcountstring), 'threadline_mailcount') self.authors_w = urwid.AttrMap(urwid.Text(authorsstring), 'threadline_authors') # SUBJECT subjectstring = self.thread.get_subject().strip() self.subject_w = urwid.AttrMap(urwid.Text(subjectstring, wrap='clip'), 'threadline_subject') # BODY if self.display_content: msgs = self.thread.get_messages().keys() msgs.sort() lastcontent = ' '.join([m.get_text_content() for m in msgs]) contentstring = lastcontent.replace('\n', ' ').strip() self.content_w = urwid.AttrMap(urwid.Text(contentstring, wrap='clip'), 'threadline_content') # Set column order #self.select = urwid.AttrMap(urwid.Text("[ ] ", wrap='clip'), # 'threadline_subject') #cols.append(('fixed', 4, self.select)) cols.append(('fixed', len(datestring), self.date_w)) cols.append(('fixed', len(authorsstring), self.authors_w)) cols.append(('fixed', len(mailcountstring), self.mailcount_w)) cols.extend(tagstrings) if subjectstring: cols.append(('fixed', len(subjectstring), self.subject_w)) if self.display_content: cols.append(self.content_w) self.columns = urwid.Columns(cols, dividechars=1) self.original_widget = self.columns
# -*- coding: utf-8 -*- from __future__ import division from settings import config from gensim import corpora, models, similarities, matutils import numpy as np import scipy.stats as stats import pickle import matplotlib.pyplot as plt from matplotlib import rc font = { 'family': config.get("tex_font_family"), 'weight': 'normal', 'size': config.get("tex_font_size") } rc('font', **font) import scipy.sparse from sparsesvd import sparsesvd from pymongo import MongoClient db = MongoClient().thesis raw_tokens = db.raw_tokens from analysis import LDA, get_corpus, get_dictionary # Define KL function def sym_kl(p, q): return np.sum([stats.entropy(p, q), stats.entropy(q, p)])
def rebuild(self): cols = [] if self.thread: newest = self.thread.get_newest_date() else: newest = None if newest == None: datestring = u' ' * 10 else: formatstring = config.get('general', 'timestamp_format') if formatstring: datestring = newest.strftime(formatstring) else: datestring = pretty_datetime(newest).rjust(10) self.date_w = urwid.AttrMap(urwid.Text(datestring), 'search_thread_date') cols.append(('fixed', len(datestring), self.date_w)) if self.thread: mailcountstring = "(%d)" % self.thread.get_total_messages() else: mailcountstring = "(?)" self.mailcount_w = urwid.AttrMap(urwid.Text(mailcountstring), 'search_thread_mailcount') cols.append(('fixed', len(mailcountstring), self.mailcount_w)) if self.thread: self.tag_widgets = [TagWidget(t) for t in self.thread.get_tags()] else: self.tag_widgets = [] self.tag_widgets.sort(tag_cmp, lambda tag_widget: tag_widget.translated) for tag_widget in self.tag_widgets: cols.append(('fixed', tag_widget.width(), tag_widget)) if self.thread: authors = self.thread.get_authors() or '(None)' else: authors = '(None)' maxlength = config.getint('general', 'authors_maxlength') authorsstring = shorten_author_string(authors, maxlength) self.authors_w = urwid.AttrMap(urwid.Text(authorsstring), 'search_thread_authors') cols.append(('fixed', len(authorsstring), self.authors_w)) if self.thread: subjectstring = self.thread.get_subject().strip() else: subjectstring = '' self.subject_w = urwid.AttrMap(urwid.Text(subjectstring, wrap='clip'), 'search_thread_subject') if subjectstring: cols.append(('weight', 2, self.subject_w)) if self.display_content: if self.thread: msgs = self.thread.get_messages().keys() else: msgs = [] msgs.sort() lastcontent = ' '.join([m.get_text_content() for m in msgs]) contentstring = lastcontent.replace('\n', ' ').strip() self.content_w = urwid.AttrMap(urwid.Text(contentstring, wrap='clip'), 'search_thread_content') cols.append(self.content_w) self.columns = urwid.Columns(cols, dividechars=1) self.original_widget = self.columns