import datetime
import re
from ramjet.settings import logger as ramjet_logger
logger = ramjet_logger.getChild('tasks.twitter')
twitter_surl_regex = re.compile('https?://t\.co/[A-z0-9]*')
def twitter_api_parser(tweet):
reg_topic = re.compile(r"[\b|\s]#(\S+)")
tweet['topics'] = reg_topic.findall(tweet['text'].replace('.', '_'))
tweet['created_at'] = datetime.datetime.strptime(
tweet['created_at'], '%a %b %d %H:%M:%S +0000 %Y')
# replace url
t = tweet['text']
if 't.co' in t:
# parse entities media
if 'media' in tweet['entities']:
for media in tweet['entities']['media']:
surl = media['url']
eurl = media.get('media_url_https') or media['media_url']
t = t.replace(surl, eurl)
# parse entities urls
if 'urls' in tweet['entities']:
for d in tweet['entities']['urls']:
surl = d['url']
eurl = d['expanded_url']
t = t.replace(surl, eurl)
"""Task 示例
"""
import gc
from ramjet.settings import logger
from ramjet.engines import ioloop
logger = logger.getChild('tasks.heart')
def bind_task():
def callback(*args, **kw):
logger.info('tasks heart!')
gc.collect()
ioloop.call_later(60, callback, *args, **kw)
ioloop.call_later(0, callback)
from ramjet.settings import logger as ramjet_logger
logger = ramjet_logger.getChild('tasks.monitors')
"""
Web HTTP Hanle 的示例
访问：/webdemo/
"""
from aiohttp import web
from aiohttp_session import get_session
from ramjet.settings import logger
logger = logger.getChild('tasks.web_demo')
def bind_task():
logger.info("run web_demo")
def bind_handle(add_route):
logger.info('bind_handle')
add_route('/', DemoHandle)
class DemoHandle(web.View):
async def get(self):
logger.info('get DemoHandle')
s = await get_session(self.request)
if 'skey' in s:
logger.info('session work ok')
from aiohttp import web
from kipp.utils import EmailSender
from functools import partial
from ramjet.settings import logger as root_logger
from ramjet.engines import thread_executor, ioloop
logger = root_logger.getChild("tasks.email_proxy")
def bind_task():
logger.info("run email proxy")
def bind_handle(add_route):
logger.info("bind handle")
add_route("/", EmailProxyHandle)
class EmailProxyHandle(web.View):
async def get(self):
return web.Response(text="email proxy")
async def post(self):
data = await self.request.json()
sender = EmailSender(
host=data.pop('host'),
username=data.pop('username'),
passwd=data.pop('passwd'),
use_tls=data.pop('use_tls', None),
import time
from io import BytesIO
import aiohttp
import boto3
import aiohttp_jinja2
from ramjet.engines import ioloop, thread_executor
from ramjet.settings import AWS_ACCESS_KEY, AWS_SECRET_KEY, logger
logger = logger.getChild('tasks.aws.rekognition')
def bind_handle(add_route):
logger.info('bind_handle aws.rekognition')
add_route('/detect/', DemoHandle)
async def download_img(url):
logger.info('download image {}'.format(url))
with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
return BytesIO(await resp.read())
def _load_img_labels(aws_conn, im):
logger.info('request to deteck image...')
return aws_conn.detect_labels(Image={'Bytes': im.read()}, MaxLabels=10)
async def load_img_labels(aws_conn, im):
import re
import heapq
from operator import itemgetter
from collections import defaultdict
import jieba.analyse
from ramjet.settings import logger
from ramjet.engines import ioloop, process_executor
from ramjet.utils import utcnow, get_conn
logger = logger.getChild('tasks.keyword')
N_CHINESE_KEYWORDS = 30
N_ENGLISH_KEYWORDS = 20
DB_HOST = 'localhost'
DB_PORT = 27016
DB = 'statistics'
STOP_WORDS = set([
'strong',
'class',
'style',
'uploads',
'align',
'height',
'image',
'center',
'aligncenter',
'target',
'blockquote',
'blank',
"""
Web HTTP Hanle 的示例
访问：/apps/
"""
from aiohttp import web
import aiohttp_jinja2
from ramjet.settings import logger
logger = logger.getChild('tasks.webapps')
def bind_task():
logger.info("run webapps")
def bind_handle(add_route):
logger.info('bind_handle')
add_route('/', WebApps)
class WebApps(web.View):
@aiohttp_jinja2.template('static/dist/index.html')
async def get(self):
logger.info('get WebApps')
return None