def _process_url( url: str ) -> Tuple[Optional[str], Optional[ArticleMeta], str, str, Optional[str]]: """ Returns: - Link (str) - Article Meta - User Notification (str) - Inline Start Param (str) - Detailed Error Message """ fetcher = get("rpc", "stub") # type: WxFetcherStub url = html.unescape(url) if not _is_url_supported(url): return None, None, "不支持的链接", "error_unsupported", None try: fetch_req = FetchURLRequest(url=url) fetch_resp = fetcher.FetchURL(fetch_req) # type: FetchURLResponse if fetch_resp.error == FetchURLError.Value("OK"): link = "{}/{}".format(get("prefix"), fetch_resp.key) return link, fetch_resp.meta, "嗨,别来无恙啊!", "bielaiwuyang", None if fetch_resp.error == FetchURLError.Value("UNSUPPORTED"): return None, None, "不支持的链接", "error_unsupported", None if fetch_resp.error == FetchURLError.Value("NETWORK"): return None, None, "网络错误,快去锤 @mutong", "error_network chui", fetch_resp.msg if fetch_resp.error == FetchURLError.Value("PARSE"): return None, None, fetch_resp.msg.capitalize(), "error_parse", None if fetch_resp.error == FetchURLError.Value("INTERNAL"): return None, None, "出错了,快去锤 @mutong", "error_internal chui", fetch_resp.msg except RpcError as e: return None, None, "服务器炸了,快去锤 @mutong", "error_rpc chui", str(e)
def get(mode=Mode.avg): client = Client.from_str(request.args.get('client', default='', type=str)) hours = request.args.get('hours', default=0, type=int) days = request.args.get('days', default=0, type=int) weeks = request.args.get('weeks', default=0, type=int) json = 'json' in request.args clients = [client] if client else Client.items data = storage.get(LOCAL_ENV, timedelta( hours=hours, days=days, weeks=weeks, ), mode, clients) if json: return jsonify(data) else: return render_template( 'get.html', readouts=data, header=','.join([c['client'] for c in data]), units=Sensor.get_map(), )
def confirm_single_item_exists(item_name, item_id=None, item_slug=None): item_id = get_single_item_id(item_name, item_id, item_slug) if item_id is None: return False item = storage.get(get_single_item_storage_key(item_name, item_id)) return item is not None
def test_elasticache(self): storage.init( "ELASTICACHE", "redis://test2.0vgvbw.ng.0001.usw1.cache.amazonaws.com:6379") self.assertEqual(storage.elasticache_client.ping(), True) msg = b"test msg" storage.put("elasticache-test-key", msg) self.assertEqual(storage.get("elasticache-test-key"), msg)
def get_category(name, parent_id = None): category_ids = get_item_ids("categories") for category_id in category_ids: category = storage.get(get_single_item_storage_key("categories", category_id)) correct_name = (name == category["name"]) correct_parent = ((parent_id is None and category["parent"] == 0) or (parent_id is not None and category["parent"] == parent_id)) if correct_name and correct_parent: return category
def get_single_item_id(item_name, item_id=None, item_slug=None): if item_id is None: if item_slug is None: raise Exception("Please provide either an item ID or a slug") slugs = storage.get(get_slugs_storage_key(item_name)) if item_slug in slugs: item_id = slugs[item_slug] return item_id
def uncache_gravity_forms_entry(gravity_forms_entry_id): gravity_forms_entry = storage.get( get_gravity_forms_entry_storage_key(gravity_forms_entry_id)) if not gravity_forms_entry: gravity_forms_entry = gf.get_entry(gravity_forms_entry_id) if not gravity_forms_entry: cache_gravity_forms_entry(gravity_forms_entry_id, gravity_forms_entry) return gravity_forms_entry
def wxmpbot_callback_query_callback(update: Update, context: CallbackContext): cb = update.callback_query # type: CallbackQuery if cb.data == "chui": context.bot.send_message( get("tg", "admin"), "<a href=\"tg://user?id={}\">{}</a> 来锤你了!".format( cb.from_user.id, cb.from_user.full_name), parse_mode="HTML") cb.message.edit_text("锤到 @mutong 了!")
def load_contacts_store(): _domainContacts = DomainContacts() for e in storage.get(storage.SID_CONTACTS): logging.debug(str(e)) _domainContacts.append(e) logging.debug("Local loaded %d contacts" % len(_domainContacts)) if _domainContacts == None: log.debug("Reset Contacts") _domainContacts = DomainContacts()
def p_get(self): self.response.headers['Content-Type'] = "text/json" pp = storage.get(self.request.get('key')) p = { "title": pp.title, "content": pp.content, "key": str(pp.key()), "format": pp.format } self.response.out.write(json.dumps(p))
def get(channel, fromDate, toDate, id=None): messages = storage.get(channel) if messages == None: return None ret = [] for message in messages: if fromDate <= message["time"] <= toDate and id == None or message[ "id"] == id: ret.append(message) return ret
def crawl(): """ Main method for executing a report. Calls other modules to provide a result :return: void """ crawler = Crawler() run(crawler) authors, articles = get(crawler.get_authors(), crawler.get_articles()) save(authors, articles) visualize(authors, articles)
def poll(): storage = AmazonSQS() clients = storage.create_queue("flash_ad_clients") while 1: print "Looking for clients..." client = storage.get(clients) if (client): print "Found a client: %s"%client post_client(client) time.sleep(5)
def get(self, key): try: post = storage.get(key) except storage.DBException: self.error(404) return if post: self.response.headers['Content-Type'] = "text/html" self.response.out.write(render.html_render([post])) else: self.error(404)
def cleaning_fb(): if request.headers['X-Appengine-Cron']: # here we check how long passed after last message to akinator # if it is long enough delete the document from collection docs = storage.coll_content(bot_akinator.AKINATOR_COLLECTION) for doc in docs: get_doc = storage.get(bot_akinator.AKINATOR_COLLECTION, doc) # if more than 5 min passed we delete aki history if (datetime.now(timezone.utc) - get_doc['last_active']).seconds > 300: storage.delete(bot_akinator.AKINATOR_COLLECTION, doc) return 'ok'
def __runSim(self): reader = TraceReader() stor = storage.get() disk = stor.disk for req in reader: disk.process_queue(req.ts) stor.request(req) disk.process_rest() # handle rest requests in the disk queue stor.flush() stor.report()
def wxmpbot_inline_query_callback(update: Update, context: CallbackContext): query = update.inline_query # type: InlineQuery try: if _REGEX_URL.fullmatch(query.query): link, meta, notification, param, detail = _process_url(query.query) answer = [ InlineQueryResultArticle( id="{}|link".format(link), title="带预览的链接", description=link, thumb_url=meta.image, input_message_content=InputTextMessageContent(link)), InlineQueryResultArticle( id="{}|title".format(link), title="文章标题", description=meta.title, thumb_url=meta.image, input_message_content=InputTextMessageContent( "<a href=\"{}\">{}</a>".format( link, html.escape(meta.title)), parse_mode="HTML", disable_web_page_preview=True)), InlineQueryResultArticle( id="{}|brief".format(link), title="标题与摘要", description=meta.brief, thumb_url=meta.image, input_message_content=InputTextMessageContent( "<a href=\"{}\">{}</a>\n<pre>{}</pre>".format( link, html.escape(meta.title), html.escape(meta.brief)), parse_mode="HTML", disable_web_page_preview=True)) ] if link is not None else [] else: answer, notification, param, detail = [], "嗨,别来无恙啊!", "bielaiwuyang", None except Exception: logger.exception("Unexpected error in inline_query_callback.") answer, notification, param, detail = [], "出错了,快去锤 @mutong", "error_unexpected chui", traceback.format_exc( ) # Send bot answer query.answer(results=answer, switch_pm_text=notification, switch_pm_parameter=param, cache_time=10) if detail is not None: context.bot.send_message( get("tg", "admin"), "Error in <code>inline_query_callback</code>\n\nQuery:\n<pre>{}</pre>\n\nError:\n<pre>{}</pre>" .format(html.escape(query.query), html.escape(detail)), parse_mode="HTML")
def processing_news(): if request.headers['X-Appengine-Cron']: session = vk.Session(access_token=auth.user) api = vk.API(session, v=5.95) # retrieving last post date last = storage.get(u'db', u'news')['threshold'] for article in scrap.get_news(): if article['datetime'] > last: response = scrap.post(auth, article, api) if response != 'error': # here we update last posted news date in the file last = article['datetime'] storage.update(u'db', u'news', {u'threshold': last}) return 'ok'
def test_crawler(self): crawler = Crawler() process = CrawlerProcess({ 'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', # 'LOG_LEVEL': 'DEBUG' }) process.crawl(crawler) process.start() authors, articles = get(crawler.get_authors(), crawler.get_articles()) self.assertTrue(len(authors) > 20) self.assertTrue(len(articles) > 20)
def test_read_save(self): self.prepare_storage() save([{ 'name': 'author' }], [{ 'title': 'article', 'date': 1541631600.0 }]) authors, articles = get([{ 'name': 'author2' }], [{ 'title': 'article2', 'date': 1542236400.0 }]) self.assertEqual(len(articles), 2) self.assertEqual(len(authors), 2)
def game(from_id, text, time, returning): if text.lower() == 'stop': # deleting the saved game storage.delete(AKINATOR_COLLECTION, str(from_id)) return {"text": "Thank you for playing! If you want to play again just type and send Akinator!", "image_url": None, 'win': True} if returning: # resuming the game fields = storage.get(AKINATOR_COLLECTION, str(from_id)) # getting saved game aki = load(fields) # creating akinator instance if text.lower() in ['back', 'b']: # we need to go back try: response = {"text": aki.back(), "image_url": None, 'win': False} aki.last_active = time storage.update(AKINATOR_COLLECTION, str(from_id), dump(aki)) return response except akinator.exceptions.CantGoBackAnyFurther: return {"text": "Cannot go back! If you want to stop send Stop", "image_url": None, 'win': False} try: response = {"text": aki.answer(text), "image_url": None, 'win': False} # passing users answer to akinator except akinator.exceptions.InvalidAnswerError: return {"text": """You put "{}", which is an invalid answer. The answer must be one of these: - "yes" OR "y" OR "0" for YES - "no" OR "n" OR "1" for NO - "i" OR "idk" OR "i dont know" OR "i don't know" OR "2" for I DON'T KNOW - "probably" OR "p" OR "3" for PROBABLY - "probably not" OR "pn" OR "4" for PROBABLY NOT If you want to stop playing send word Stop. """.format(text), 'win': False, "image_url": None} # checking if we are close to make prediction if aki.progression >= 90: # we can make a prediction aki.win() response = {'text': "It's {} ({})!".format(aki.name, aki.description), 'win': True} if aki.picture: response['image_url'] = aki.picture storage.delete(AKINATOR_COLLECTION, str(from_id)) # deleting document when the game is over else: # we need to save current progression aki.last_active = time d = dump(aki) storage.update(AKINATOR_COLLECTION, str(from_id), d) else: # creating the new game aki = akinator.Akinator() # starting game and asking user first question response = {"text": aki.start_game(), "image_url": None, 'win': False} # save current progress aki.last_active = time storage.add(AKINATOR_COLLECTION, str(from_id), dump(aki)) return response
def get_products(categories=None, page_num=1, per_page=10): product_ids = get_item_ids("products") offset_first_product = (page_num - 1) * per_page offset_last_product = page_num * per_page if categories is None: return { "num_total": len(product_ids), "products": ids_to_items("products", product_ids[offset_first_product:offset_last_product + 1]) } else: products_skipped = 0 products_added = 0 total_matching_products_found = 0 products_found = [] if not isinstance(categories, list): categories = [categories] for product_id in product_ids: product = storage.get(get_single_item_storage_key("products", product_id)) num_categories_product_found_in = 0 for category in categories: for product_category in product["categories"]: if product_category["id"] == category["id"]: num_categories_product_found_in += 1 break if num_categories_product_found_in == len(categories): total_matching_products_found += 1 if products_added == per_page: continue if products_skipped == offset_first_product: products_found.append(product) products_added += 1 else: products_skipped += 1 return { "num_total": total_matching_products_found, "products": products_found }
def wxmpbot_text_message_callback(update: Update, context: CallbackContext): msg = update.message # type: Message try: if _REGEX_URL.fullmatch(msg.text): link, meta, notification, param, detail = _process_url(msg.text) answer = link if link is not None else notification else: answer, detail = "这看起来不是一个链接啦!", None except Exception: logger.exception("Unexpected error in text_message_callback.") answer, detail = "出错了,快去锤 @mutong", traceback.format_exc() msg.reply_text(answer) if detail is not None: context.bot.send_message( get("tg", "admin"), "Error in <code>text_message_callback</code>\n\nText:\n<pre>{}</pre>\n\nError:\n<pre>{}</pre>" .format(html.escape(msg.text), html.escape(detail)), parse_mode="HTML")
def write_blob(file_handle_or_path): """Write a single file testcase to GCS.""" blobs_bucket = storage.blobs_bucket() blob_name = generate_new_blob_name() if storage.get(storage.get_cloud_storage_file_path(blobs_bucket, blob_name)): raise BlobsException('UUID collision found: %s' % blob_name) if isinstance(file_handle_or_path, basestring): filename = os.path.basename(file_handle_or_path) else: filename = file_handle_or_path.name metadata = { storage.BLOB_FILENAME_METADATA_KEY: filename, } gcs_path = '/%s/%s' % (blobs_bucket, blob_name) if storage.copy_file_to(file_handle_or_path, gcs_path, metadata=metadata): return blob_name raise BlobsException('Failed to write blob %s.' % blob_name)
def _get_unpacked_rows(self, rows: List[dict]) -> Iterable[dict]: main: List[dict] = [] for row in rows: uniques: Dict[str, Any] = { key: value for key, value in row.items() if key != 'reports' } for rep in row['reports']: yield {**uniques, **rep} main.append(uniques) self.uniques = pd.DataFrame(main) if self._costs_file_id is None: return self.uniques = self.uniques.join(other=pd.read_excel( storage.get(storage.Bucket.files, self._costs_file_id).data).groupby('nm_id').max(), on='nm_id', how='left')
def signal_handler(signum, frame): logger.warning("Signal received, exiting...") get("tg", "updater").stop() get("rpc", "channel").close()
def get_item_ids(item_name): ids = storage.get(item_name) if ids is None: ids = [] return ids
def get_form(id): storage_key = get_single_item_storage_key("forms", item_id=id) return storage.get(storage_key)
def get_product(id=None, slug=None): storage_key = get_single_item_storage_key("products", item_id=id, item_slug=slug) return storage.get(storage_key)
#!/usr/bin/env python3 from htmlgen import * import sys import storage html=htmlgen.HTMLgen(pagelayout.getLayoutXML().decode('utf-8'),"Home Page") count=storage.count("articles") for i in range(count): try: article=storage.get("articles",i) if "anime" in article["tags"]: html.addArticle(**article) except: html.addArticle(**storage.get("articles",i), aid=i) sys.stdout.buffer.write("Content-type: text/html\r\n\r\n".encode('utf8')) sys.stdout.buffer.write("<!DOCTYPE html>".encode('utf8')) sys.stdout.buffer.write(html.renderSite().encode('utf8')) sys.stdout.flush() #print(html.renderSite())
def ids_to_items(item_name, ids): items = [] for item_id in ids: items.append(storage.get(get_single_item_storage_key(item_name, item_id))) return items
def test_get(self): person = storage.put(Person(name="alabama")) person_from_db = storage.get(Person, uuid=person.uuid) self.assertEqual(person.name, person_from_db.name) self.assertEqual(person.uuid, person_from_db.uuid)
timestamp=int(time.time()) data={"name":cgi.escape(form["title"].value),"markdown":cgi.escape(message),"author":cgi.escape(username),"date":timestamp} storage.append("comments-%i"%aid,data) except KeyError: pass seed=random.SystemRandom().randint(0,2**24) random.seed(seed) checkstr="".join(random.choice(string.digits+string.ascii_lowercase) for _ in range(5)) image = ImageCaptcha() capt = image.generate(checkstr) html=htmlgen.HTMLgen(pagelayout.getLayoutXML().decode('utf-8'),"Comments") html.appendHTML("<form action=\"comments.py?aid=%i\" method=\"POST\">"%aid) html.appendHTML("<input type=\"hidden\" name=\"seed\" value=\"%i\" />"%seed) html.appendHTML("<input placeholder=\"Username\" name=\"username\" /><br />") html.appendHTML("<input placeholder=\"Title\" name=\"title\" /><br />") html.appendHTML("<textarea rows=\"25\" cols=\"80\" name=\"message\" placeholder=\"Compose your message. Markdown is enabled.\" ></textarea><br />") html.appendHTML("<img src=\"data:image/png;base64,%s\" alt=\"Captcha image\" />" % base64.b64encode(capt.getvalue()).decode("UTF-8")) html.appendHTML("<input placeholder=\"Captcha. lowercase only. case sensitive\" name=\"checkstr\" />") html.appendHTML("<input type=\"submit\" /></form>") count=storage.count("comments-%i"%aid) for i in range(count): html.addArticle(aid=aid,**(storage.get("comments-%i"%aid,i))) sys.stdout.buffer.write("Content-type: text/html\r\n\r\n".encode('utf8')) sys.stdout.buffer.write("<!DOCTYPE html>".encode('utf8')) sys.stdout.buffer.write(html.renderSite(True).encode('utf8')) sys.stdout.flush() #print(html.renderSite(True))
#!/usr/bin/env python3 from htmlgen import * import sys import storage html=htmlgen.HTMLgen(pagelayout.getLayoutXML().decode('utf-8'),"Home Page") count=storage.count("articles") for i in range(count): try: article=storage.get("articles",i) if "homepage" in article["tags"]: html.addArticle(**article) except: html.addArticle(aid=i, **storage.get("articles",i)) sys.stdout.buffer.write("Content-type: text/html\r\n\r\n".encode('utf8')) sys.stdout.buffer.write("<!DOCTYPE html>".encode('utf8')) sys.stdout.buffer.write(html.renderSite().encode('utf8')) sys.stdout.flush() #print(html.renderSite())
def p_get(self): self.response.headers['Content-Type'] = "text/json" pp = storage.get(self.request.get('key')) p = {"title":pp.title, "content":pp.content,"key":str(pp.key()),"format":pp.format} self.response.out.write(json.dumps(p))
def test_s3(self): storage.init("S3", "storage-module-test") msg = bytes(random.randint(1, 10000)) storage.put("aws-test-key", msg) self.assertEqual(storage.get("aws-test-key"), msg)