def __init__(self, params):
from_ = params.get('from') or body()
to = params.get('to', None)
aggregate = params.get('aggregate', None)
assert to is not None, 'split processor requires to argument.'
assert aggregate is None or callable(
aggregate
), 'split processor requires aggregate argument as function.'
consumer = to.get_consumer()
async def split_processor(exchange):
import copy
to_split = evaluate_expression(from_, exchange)
import collections
if isinstance(to_split, str):
to_split = to_split.split()
assert isinstance(
to_split,
collections.Iterable), 'split target is not iterable.'
gatherd = await asyncio.gather(*[
consumer.produce(exchange.create_child(Exchange(sp)))
for sp in to_split
])
if aggregate:
exchange.set_body(aggregate(gatherd))
return exchange
self.split_processor = split_processor
def markdown(params=None):
import markdown as markdown_module
expression = body()
if params is not None and params.get('expression'):
expression = params.get('expression')
async def processor(exchange):
value = evaluate_expression(expression, exchange)
exchange.set_body(markdown_module.markdown(value))
return exchange
return processor
def file(params):
mode = params.get('mode') or 'read'
file_name_expression = params.get('file_name') or body()
#TBD: other mode
#TBD: file cache
async def processor(exchange):
if mode == 'read':
file_name = evaluate_expression(file_name_expression, exchange)
f = open(file_name, 'r')
exchange.set_body(f.read())
f.close()
return exchange
return processor
def aiohttp_request(params={}):
url_expression = params.get('url') or body()
response_type = params.get('response_type', 'text')
isValid = params.get('isValid', False)
async def processor(exchange):
url = evaluate_expression(url_expression, exchange)
import aiohttp
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
if isValid:
exchange.set_header('validate', resp.status == 200)
else:
if response_type == 'text':
exchange.set_body(await resp.text())
elif response_type == 'data':
exchange.set_body(await resp.read())
exchange.set_header('request_url', url) #buggy
return exchange
return processor
# channel_2: get image url from entry page and put queue to channel_3
'channel_2': To(aiohttp_request())
.to(soup(lambda soup:[element.get('src')
for element in soup.find_all("img",
src=re.compile("^http.*user_images.*?/o"))]))
.put_queue('channel_3', queue_name='myqueue'),
# channel_3: get image from image url and write to zip
'channel_3':To(aiohttp_request({'response_type':'data'}))
.to(zipper({
'mode': 'write',
'file_name': lambda ex:'{}_{}'
.format(*(ex.get_header('request_url').split('?')[0].split('/')[i] for i in [-7, -1]))
}))
},
'init_queue': {
'channel_1':body()
},
'queue_name': 'myqueue',
'maxsize': 30
})
.to(zipper({'mode':'close'}))
.process(set_body(get_header('zip_file_name')))
.to(log({'name':'ameblo_images_main_end'}))
)
Aiohttp().application().router.add_static(
prefix='/public/tmp', path='../public/tmp')
from consumer import Aiohttp
from components import file, markdown, jinja2
from evaluator import body, set_header
(Aiohttp('/markdown')
.to(file({'file_name': lambda ex: '../public/static/' + ex.get_header('file')}))
.to(markdown())
.to(jinja2({
'template': 'markdown.html',
'data': {
'markdown_body': body()
}
}))
) #yapf: disable
Aiohttp().application().router.add_static(prefix='/public/static',
path='../public/static')
async def simple_update_success_count_to_none(exchange):
id_ = int(exchange.get_header('id'))
for item in exchange.get_body():
if item['id'] == id_:
if 'cnt_sc' in item:
del item['cnt_sc']
if 'prev_sc' in item:
del item['prev_sc']
return exchange
(Aiohttp('/antenna')
.to(cache_get_processor)
.to(jinja2_({
'template': 'antenna.html',
'data': {
'items': body(),
'span_list': span_option
},
'util': create_util
}))
) #yapf: disable
async def merge_item(exchange):
id_ = int(exchange.get_header('id', '-1'))
for index, item in enumerate(exchange.get_body()):
if item['id'] == id_:
exchange.get_headers().update(item)
exchange.set_header('index', index)
break
exchange.set_body('success')
return exchange