def _process_url(
url: str
) -> Tuple[Optional[str], Optional[ArticleMeta], str, str, Optional[str]]:
"""
Returns:
- Link (str)
- Article Meta
- User Notification (str)
- Inline Start Param (str)
- Detailed Error Message
"""
fetcher = get("rpc", "stub") # type: WxFetcherStub
url = html.unescape(url)
if not _is_url_supported(url):
return None, None, "不支持的链接", "error_unsupported", None
try:
fetch_req = FetchURLRequest(url=url)
fetch_resp = fetcher.FetchURL(fetch_req) # type: FetchURLResponse
if fetch_resp.error == FetchURLError.Value("OK"):
link = "{}/{}".format(get("prefix"), fetch_resp.key)
return link, fetch_resp.meta, "嗨，别来无恙啊！", "bielaiwuyang", None
if fetch_resp.error == FetchURLError.Value("UNSUPPORTED"):
return None, None, "不支持的链接", "error_unsupported", None
if fetch_resp.error == FetchURLError.Value("NETWORK"):
return None, None, "网络错误，快去锤 @mutong", "error_network chui", fetch_resp.msg
if fetch_resp.error == FetchURLError.Value("PARSE"):
return None, None, fetch_resp.msg.capitalize(), "error_parse", None
if fetch_resp.error == FetchURLError.Value("INTERNAL"):
return None, None, "出错了，快去锤 @mutong", "error_internal chui", fetch_resp.msg
except RpcError as e:
return None, None, "服务器炸了，快去锤 @mutong", "error_rpc chui", str(e)
def get(mode=Mode.avg):
client = Client.from_str(request.args.get('client', default='', type=str))
hours = request.args.get('hours', default=0, type=int)
days = request.args.get('days', default=0, type=int)
weeks = request.args.get('weeks', default=0, type=int)
json = 'json' in request.args
clients = [client] if client else Client.items
data = storage.get(LOCAL_ENV,
timedelta(
hours=hours,
days=days,
weeks=weeks,
), mode, clients)
if json:
return jsonify(data)
else:
return render_template(
'get.html',
readouts=data,
header=','.join([c['client'] for c in data]),
units=Sensor.get_map(),
)
def confirm_single_item_exists(item_name, item_id=None, item_slug=None): item_id = get_single_item_id(item_name, item_id, item_slug) if item_id is None: return False item = storage.get(get_single_item_storage_key(item_name, item_id)) return item is not None
def test_elasticache(self):
storage.init(
"ELASTICACHE",
"redis://test2.0vgvbw.ng.0001.usw1.cache.amazonaws.com:6379")
self.assertEqual(storage.elasticache_client.ping(), True)
msg = b"test msg"
storage.put("elasticache-test-key", msg)
self.assertEqual(storage.get("elasticache-test-key"), msg)
def get_category(name, parent_id = None):
category_ids = get_item_ids("categories")
for category_id in category_ids:
category = storage.get(get_single_item_storage_key("categories", category_id))
correct_name = (name == category["name"])
correct_parent = ((parent_id is None and category["parent"] == 0) or (parent_id is not None and category["parent"] == parent_id))
if correct_name and correct_parent:
return category
def get_single_item_id(item_name, item_id=None, item_slug=None):
if item_id is None:
if item_slug is None:
raise Exception("Please provide either an item ID or a slug")
slugs = storage.get(get_slugs_storage_key(item_name))
if item_slug in slugs:
item_id = slugs[item_slug]
return item_id
def uncache_gravity_forms_entry(gravity_forms_entry_id):
gravity_forms_entry = storage.get(
get_gravity_forms_entry_storage_key(gravity_forms_entry_id))
if not gravity_forms_entry:
gravity_forms_entry = gf.get_entry(gravity_forms_entry_id)
if not gravity_forms_entry:
cache_gravity_forms_entry(gravity_forms_entry_id, gravity_forms_entry)
return gravity_forms_entry
def wxmpbot_callback_query_callback(update: Update, context: CallbackContext):
cb = update.callback_query # type: CallbackQuery
if cb.data == "chui":
context.bot.send_message(
get("tg", "admin"),
"<a href=\"tg://user?id={}\">{}</a> 来锤你了！".format(
cb.from_user.id, cb.from_user.full_name),
parse_mode="HTML")
cb.message.edit_text("锤到 @mutong 了！")
def load_contacts_store():
_domainContacts = DomainContacts()
for e in storage.get(storage.SID_CONTACTS):
logging.debug(str(e))
_domainContacts.append(e)
logging.debug("Local loaded %d contacts" % len(_domainContacts))
if _domainContacts == None:
log.debug("Reset Contacts")
_domainContacts = DomainContacts()
def p_get(self):
self.response.headers['Content-Type'] = "text/json"
pp = storage.get(self.request.get('key'))
p = {
"title": pp.title,
"content": pp.content,
"key": str(pp.key()),
"format": pp.format
}
self.response.out.write(json.dumps(p))
def get(channel, fromDate, toDate, id=None):
messages = storage.get(channel)
if messages == None:
return None
ret = []
for message in messages:
if fromDate <= message["time"] <= toDate and id == None or message[
"id"] == id:
ret.append(message)
return ret
def crawl():
"""
Main method for executing a report. Calls other modules to provide a result
:return: void
"""
crawler = Crawler()
run(crawler)
authors, articles = get(crawler.get_authors(), crawler.get_articles())
save(authors, articles)
visualize(authors, articles)
def poll():
storage = AmazonSQS()
clients = storage.create_queue("flash_ad_clients")
while 1:
print "Looking for clients..."
client = storage.get(clients)
if (client):
print "Found a client: %s"%client
post_client(client)
time.sleep(5)
def get(self, key):
try:
post = storage.get(key)
except storage.DBException:
self.error(404)
return
if post:
self.response.headers['Content-Type'] = "text/html"
self.response.out.write(render.html_render([post]))
else:
self.error(404)
def get(self, key):
try:
post = storage.get(key)
except storage.DBException:
self.error(404)
return
if post:
self.response.headers['Content-Type'] = "text/html"
self.response.out.write(render.html_render([post]))
else:
self.error(404)
def cleaning_fb():
if request.headers['X-Appengine-Cron']:
# here we check how long passed after last message to akinator
# if it is long enough delete the document from collection
docs = storage.coll_content(bot_akinator.AKINATOR_COLLECTION)
for doc in docs:
get_doc = storage.get(bot_akinator.AKINATOR_COLLECTION, doc)
# if more than 5 min passed we delete aki history
if (datetime.now(timezone.utc) - get_doc['last_active']).seconds > 300:
storage.delete(bot_akinator.AKINATOR_COLLECTION, doc)
return 'ok'
def __runSim(self):
reader = TraceReader()
stor = storage.get()
disk = stor.disk
for req in reader:
disk.process_queue(req.ts)
stor.request(req)
disk.process_rest() # handle rest requests in the disk queue
stor.flush()
stor.report()
def wxmpbot_inline_query_callback(update: Update, context: CallbackContext):
query = update.inline_query # type: InlineQuery
try:
if _REGEX_URL.fullmatch(query.query):
link, meta, notification, param, detail = _process_url(query.query)
answer = [
InlineQueryResultArticle(
id="{}|link".format(link),
title="带预览的链接",
description=link,
thumb_url=meta.image,
input_message_content=InputTextMessageContent(link)),
InlineQueryResultArticle(
id="{}|title".format(link),
title="文章标题",
description=meta.title,
thumb_url=meta.image,
input_message_content=InputTextMessageContent(
"<a href=\"{}\">{}</a>".format(
link, html.escape(meta.title)),
parse_mode="HTML",
disable_web_page_preview=True)),
InlineQueryResultArticle(
id="{}|brief".format(link),
title="标题与摘要",
description=meta.brief,
thumb_url=meta.image,
input_message_content=InputTextMessageContent(
"<a href=\"{}\">{}</a>\n<pre>{}</pre>".format(
link, html.escape(meta.title),
html.escape(meta.brief)),
parse_mode="HTML",
disable_web_page_preview=True))
] if link is not None else []
else:
answer, notification, param, detail = [], "嗨，别来无恙啊！", "bielaiwuyang", None
except Exception:
logger.exception("Unexpected error in inline_query_callback.")
answer, notification, param, detail = [], "出错了，快去锤 @mutong", "error_unexpected chui", traceback.format_exc(
)
# Send bot answer
query.answer(results=answer,
switch_pm_text=notification,
switch_pm_parameter=param,
cache_time=10)
if detail is not None:
context.bot.send_message(
get("tg", "admin"),
"Error in <code>inline_query_callback</code>\n\nQuery:\n<pre>{}</pre>\n\nError:\n<pre>{}</pre>"
.format(html.escape(query.query), html.escape(detail)),
parse_mode="HTML")
def processing_news():
if request.headers['X-Appengine-Cron']:
session = vk.Session(access_token=auth.user)
api = vk.API(session, v=5.95)
# retrieving last post date
last = storage.get(u'db', u'news')['threshold']
for article in scrap.get_news():
if article['datetime'] > last:
response = scrap.post(auth, article, api)
if response != 'error':
# here we update last posted news date in the file
last = article['datetime']
storage.update(u'db', u'news', {u'threshold': last})
return 'ok'
def test_crawler(self):
crawler = Crawler()
process = CrawlerProcess({
'USER_AGENT':
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
# 'LOG_LEVEL': 'DEBUG'
})
process.crawl(crawler)
process.start()
authors, articles = get(crawler.get_authors(), crawler.get_articles())
self.assertTrue(len(authors) > 20)
self.assertTrue(len(articles) > 20)
def test_read_save(self):
self.prepare_storage()
save([{
'name': 'author'
}], [{
'title': 'article',
'date': 1541631600.0
}])
authors, articles = get([{
'name': 'author2'
}], [{
'title': 'article2',
'date': 1542236400.0
}])
self.assertEqual(len(articles), 2)
self.assertEqual(len(authors), 2)
def game(from_id, text, time, returning):
if text.lower() == 'stop': # deleting the saved game
storage.delete(AKINATOR_COLLECTION, str(from_id))
return {"text": "Thank you for playing! If you want to play again just type and send Akinator!",
"image_url": None, 'win': True}
if returning: # resuming the game
fields = storage.get(AKINATOR_COLLECTION, str(from_id)) # getting saved game
aki = load(fields) # creating akinator instance
if text.lower() in ['back', 'b']: # we need to go back
try:
response = {"text": aki.back(), "image_url": None, 'win': False}
aki.last_active = time
storage.update(AKINATOR_COLLECTION, str(from_id), dump(aki))
return response
except akinator.exceptions.CantGoBackAnyFurther:
return {"text": "Cannot go back! If you want to stop send Stop", "image_url": None, 'win': False}
try:
response = {"text": aki.answer(text), "image_url": None, 'win': False} # passing users answer to akinator
except akinator.exceptions.InvalidAnswerError:
return {"text": """You put "{}", which is an invalid answer.
The answer must be one of these:
- "yes" OR "y" OR "0" for YES
- "no" OR "n" OR "1" for NO
- "i" OR "idk" OR "i dont know" OR "i don't know" OR "2" for I DON'T KNOW
- "probably" OR "p" OR "3" for PROBABLY
- "probably not" OR "pn" OR "4" for PROBABLY NOT
If you want to stop playing send word Stop.
""".format(text), 'win': False, "image_url": None}
# checking if we are close to make prediction
if aki.progression >= 90: # we can make a prediction
aki.win()
response = {'text': "It's {} ({})!".format(aki.name, aki.description), 'win': True}
if aki.picture:
response['image_url'] = aki.picture
storage.delete(AKINATOR_COLLECTION, str(from_id)) # deleting document when the game is over
else: # we need to save current progression
aki.last_active = time
d = dump(aki)
storage.update(AKINATOR_COLLECTION, str(from_id), d)
else: # creating the new game
aki = akinator.Akinator()
# starting game and asking user first question
response = {"text": aki.start_game(), "image_url": None, 'win': False}
# save current progress
aki.last_active = time
storage.add(AKINATOR_COLLECTION, str(from_id), dump(aki))
return response
def get_products(categories=None, page_num=1, per_page=10):
product_ids = get_item_ids("products")
offset_first_product = (page_num - 1) * per_page
offset_last_product = page_num * per_page
if categories is None:
return {
"num_total": len(product_ids),
"products": ids_to_items("products", product_ids[offset_first_product:offset_last_product + 1])
}
else:
products_skipped = 0
products_added = 0
total_matching_products_found = 0
products_found = []
if not isinstance(categories, list):
categories = [categories]
for product_id in product_ids:
product = storage.get(get_single_item_storage_key("products", product_id))
num_categories_product_found_in = 0
for category in categories:
for product_category in product["categories"]:
if product_category["id"] == category["id"]:
num_categories_product_found_in += 1
break
if num_categories_product_found_in == len(categories):
total_matching_products_found += 1
if products_added == per_page:
continue
if products_skipped == offset_first_product:
products_found.append(product)
products_added += 1
else:
products_skipped += 1
return {
"num_total": total_matching_products_found,
"products": products_found
}
def wxmpbot_text_message_callback(update: Update, context: CallbackContext):
msg = update.message # type: Message
try:
if _REGEX_URL.fullmatch(msg.text):
link, meta, notification, param, detail = _process_url(msg.text)
answer = link if link is not None else notification
else:
answer, detail = "这看起来不是一个链接啦！", None
except Exception:
logger.exception("Unexpected error in text_message_callback.")
answer, detail = "出错了，快去锤 @mutong", traceback.format_exc()
msg.reply_text(answer)
if detail is not None:
context.bot.send_message(
get("tg", "admin"),
"Error in <code>text_message_callback</code>\n\nText:\n<pre>{}</pre>\n\nError:\n<pre>{}</pre>"
.format(html.escape(msg.text), html.escape(detail)),
parse_mode="HTML")
def write_blob(file_handle_or_path):
"""Write a single file testcase to GCS."""
blobs_bucket = storage.blobs_bucket()
blob_name = generate_new_blob_name()
if storage.get(storage.get_cloud_storage_file_path(blobs_bucket, blob_name)):
raise BlobsException('UUID collision found: %s' % blob_name)
if isinstance(file_handle_or_path, basestring):
filename = os.path.basename(file_handle_or_path)
else:
filename = file_handle_or_path.name
metadata = {
storage.BLOB_FILENAME_METADATA_KEY: filename,
}
gcs_path = '/%s/%s' % (blobs_bucket, blob_name)
if storage.copy_file_to(file_handle_or_path, gcs_path, metadata=metadata):
return blob_name
raise BlobsException('Failed to write blob %s.' % blob_name)
def _get_unpacked_rows(self, rows: List[dict]) -> Iterable[dict]:
main: List[dict] = []
for row in rows:
uniques: Dict[str, Any] = {
key: value
for key, value in row.items() if key != 'reports'
}
for rep in row['reports']:
yield {**uniques, **rep}
main.append(uniques)
self.uniques = pd.DataFrame(main)
if self._costs_file_id is None:
return
self.uniques = self.uniques.join(other=pd.read_excel(
storage.get(storage.Bucket.files,
self._costs_file_id).data).groupby('nm_id').max(),
on='nm_id',
how='left')
def signal_handler(signum, frame):
logger.warning("Signal received, exiting...")
get("tg", "updater").stop()
get("rpc", "channel").close()
def get_item_ids(item_name): ids = storage.get(item_name) if ids is None: ids = [] return ids
def get_form(id):
storage_key = get_single_item_storage_key("forms", item_id=id)
return storage.get(storage_key)
def get_product(id=None, slug=None):
storage_key = get_single_item_storage_key("products", item_id=id, item_slug=slug)
return storage.get(storage_key)
#!/usr/bin/env python3
from htmlgen import *
import sys
import storage
html=htmlgen.HTMLgen(pagelayout.getLayoutXML().decode('utf-8'),"Home Page")
count=storage.count("articles")
for i in range(count):
try:
article=storage.get("articles",i)
if "anime" in article["tags"]:
html.addArticle(**article)
except:
html.addArticle(**storage.get("articles",i), aid=i)
sys.stdout.buffer.write("Content-type: text/html\r\n\r\n".encode('utf8'))
sys.stdout.buffer.write("<!DOCTYPE html>".encode('utf8'))
sys.stdout.buffer.write(html.renderSite().encode('utf8'))
sys.stdout.flush()
#print(html.renderSite())
def ids_to_items(item_name, ids): items = [] for item_id in ids: items.append(storage.get(get_single_item_storage_key(item_name, item_id))) return items
def test_get(self):
person = storage.put(Person(name="alabama"))
person_from_db = storage.get(Person, uuid=person.uuid)
self.assertEqual(person.name, person_from_db.name)
self.assertEqual(person.uuid, person_from_db.uuid)
timestamp=int(time.time())
data={"name":cgi.escape(form["title"].value),"markdown":cgi.escape(message),"author":cgi.escape(username),"date":timestamp}
storage.append("comments-%i"%aid,data)
except KeyError:
pass
seed=random.SystemRandom().randint(0,2**24)
random.seed(seed)
checkstr="".join(random.choice(string.digits+string.ascii_lowercase) for _ in range(5))
image = ImageCaptcha()
capt = image.generate(checkstr)
html=htmlgen.HTMLgen(pagelayout.getLayoutXML().decode('utf-8'),"Comments")
html.appendHTML("<form action=\"comments.py?aid=%i\" method=\"POST\">"%aid)
html.appendHTML("<input type=\"hidden\" name=\"seed\" value=\"%i\" />"%seed)
html.appendHTML("<input placeholder=\"Username\" name=\"username\" /><br />")
html.appendHTML("<input placeholder=\"Title\" name=\"title\" /><br />")
html.appendHTML("<textarea rows=\"25\" cols=\"80\" name=\"message\" placeholder=\"Compose your message. Markdown is enabled.\" ></textarea><br />")
html.appendHTML("<img src=\"data:image/png;base64,%s\" alt=\"Captcha image\" />" % base64.b64encode(capt.getvalue()).decode("UTF-8"))
html.appendHTML("<input placeholder=\"Captcha. lowercase only. case sensitive\" name=\"checkstr\" />")
html.appendHTML("<input type=\"submit\" /></form>")
count=storage.count("comments-%i"%aid)
for i in range(count):
html.addArticle(aid=aid,**(storage.get("comments-%i"%aid,i)))
sys.stdout.buffer.write("Content-type: text/html\r\n\r\n".encode('utf8'))
sys.stdout.buffer.write("<!DOCTYPE html>".encode('utf8'))
sys.stdout.buffer.write(html.renderSite(True).encode('utf8'))
sys.stdout.flush()
#print(html.renderSite(True))
#!/usr/bin/env python3
from htmlgen import *
import sys
import storage
html=htmlgen.HTMLgen(pagelayout.getLayoutXML().decode('utf-8'),"Home Page")
count=storage.count("articles")
for i in range(count):
try:
article=storage.get("articles",i)
if "homepage" in article["tags"]:
html.addArticle(**article)
except:
html.addArticle(aid=i, **storage.get("articles",i))
sys.stdout.buffer.write("Content-type: text/html\r\n\r\n".encode('utf8'))
sys.stdout.buffer.write("<!DOCTYPE html>".encode('utf8'))
sys.stdout.buffer.write(html.renderSite().encode('utf8'))
sys.stdout.flush()
#print(html.renderSite())
def p_get(self):
self.response.headers['Content-Type'] = "text/json"
pp = storage.get(self.request.get('key'))
p = {"title":pp.title, "content":pp.content,"key":str(pp.key()),"format":pp.format}
self.response.out.write(json.dumps(p))
def test_s3(self):
storage.init("S3", "storage-module-test")
msg = bytes(random.randint(1, 10000))
storage.put("aws-test-key", msg)
self.assertEqual(storage.get("aws-test-key"), msg)
