Beispiel #1
0
    def __init__(self, params):
        from_ = params.get('from') or body()
        to = params.get('to', None)
        aggregate = params.get('aggregate', None)
        assert to is not None, 'split processor requires to argument.'
        assert aggregate is None or callable(
            aggregate
        ), 'split processor requires aggregate argument as function.'
        consumer = to.get_consumer()

        async def split_processor(exchange):
            import copy
            to_split = evaluate_expression(from_, exchange)
            import collections
            if isinstance(to_split, str):
                to_split = to_split.split()
            assert isinstance(
                to_split,
                collections.Iterable), 'split target is not iterable.'
            gatherd = await asyncio.gather(*[
                consumer.produce(exchange.create_child(Exchange(sp)))
                for sp in to_split
            ])
            if aggregate:
                exchange.set_body(aggregate(gatherd))
            return exchange

        self.split_processor = split_processor
Beispiel #2
0
def markdown(params=None):
    import markdown as markdown_module
    expression = body()
    if params is not None and params.get('expression'):
        expression = params.get('expression')

    async def processor(exchange):
        value = evaluate_expression(expression, exchange)
        exchange.set_body(markdown_module.markdown(value))
        return exchange

    return processor
Beispiel #3
0
def file(params):
    mode = params.get('mode') or 'read'
    file_name_expression = params.get('file_name') or body()

    #TBD: other mode
    #TBD: file cache
    async def processor(exchange):
        if mode == 'read':
            file_name = evaluate_expression(file_name_expression, exchange)
            f = open(file_name, 'r')
            exchange.set_body(f.read())
            f.close()
        return exchange

    return processor
Beispiel #4
0
def aiohttp_request(params={}):
    url_expression = params.get('url') or body()
    response_type = params.get('response_type', 'text')
    isValid = params.get('isValid', False)

    async def processor(exchange):
        url = evaluate_expression(url_expression, exchange)
        import aiohttp
        async with aiohttp.ClientSession() as session:
            async with session.get(url) as resp:
                if isValid:
                    exchange.set_header('validate', resp.status == 200)
                else:
                    if response_type == 'text':
                        exchange.set_body(await resp.text())
                    elif response_type == 'data':
                        exchange.set_body(await resp.read())
                    exchange.set_header('request_url', url)  #buggy
            return exchange

    return processor
Beispiel #5
0
            # channel_2: get image url from entry page and put queue to channel_3
            'channel_2': To(aiohttp_request())
                .to(soup(lambda soup:[element.get('src')
                        for element in soup.find_all("img",
                            src=re.compile("^http.*user_images.*?/o"))]))

                .put_queue('channel_3', queue_name='myqueue'),

            # channel_3: get image from image url and write to zip
            'channel_3':To(aiohttp_request({'response_type':'data'}))
                .to(zipper({
                    'mode': 'write',
                    'file_name': lambda ex:'{}_{}'
                        .format(*(ex.get_header('request_url').split('?')[0].split('/')[i] for i in [-7, -1]))
                }))
        },
        'init_queue': {
            'channel_1':body()
        },
        'queue_name': 'myqueue',
        'maxsize': 30
    })
    .to(zipper({'mode':'close'}))
    .process(set_body(get_header('zip_file_name')))
    .to(log({'name':'ameblo_images_main_end'}))
 )

Aiohttp().application().router.add_static(
    prefix='/public/tmp', path='../public/tmp')
Beispiel #6
0
from consumer import Aiohttp
from components import file, markdown, jinja2
from evaluator import body, set_header

(Aiohttp('/markdown')
    .to(file({'file_name': lambda ex: '../public/static/' + ex.get_header('file')}))
    .to(markdown())
    .to(jinja2({
        'template': 'markdown.html',
        'data': {
            'markdown_body': body()
        }
    }))
) #yapf: disable

Aiohttp().application().router.add_static(prefix='/public/static',
                                          path='../public/static')
Beispiel #7
0
async def simple_update_success_count_to_none(exchange):
    id_ = int(exchange.get_header('id'))
    for item in exchange.get_body():
        if item['id'] == id_:
            if 'cnt_sc' in item:
                del item['cnt_sc']
            if 'prev_sc' in item:
                del item['prev_sc']
    return exchange

(Aiohttp('/antenna')
    .to(cache_get_processor)
    .to(jinja2_({
        'template': 'antenna.html',
        'data': {
            'items': body(),
            'span_list': span_option
        },
        'util': create_util
    }))
) #yapf: disable

async def merge_item(exchange):
    id_ = int(exchange.get_header('id', '-1'))
    for index, item in enumerate(exchange.get_body()):
        if item['id'] == id_:
            exchange.get_headers().update(item)
            exchange.set_header('index', index)
            break
    exchange.set_body('success')
    return exchange