Пример #1
0
#coding:utf-8
import re
from copy import deepcopy

import requests
import pymongo
from bson import ObjectId, Binary
from pymongo.errors import OperationFailure
import thread
import time

from base import MessageQueue, to_zip, to_zip64, logging

log = logging.getLogger('spider')


class SpiderWorker(MessageQueue):
    def __init__(self, ident):
        super(SpiderWorker, self).__init__(ident)
        self.crawl_handler = CrawlHandler(self)

    def on_receive_json(self, caller, request, request_id):
        cmd = request.get('cmd')
        log.debug('receive cmd: %s', request)

        if cmd == 'crawl':
            result = self.crawl_handler.process(request)
        elif cmd == 'who':
            result = {'role': 'worker'}
        else:
            log.warning('invalid cmd: %s - %s', cmd, request)
Пример #2
0
#coding:utf-8
from copy import deepcopy
import random
import time
from datetime import datetime

from base import MessageQueue, logging

log = logging.getLogger('server')

class User(object):
    def __init__(self, name):
        self.name = name
        self.role = 'user'
        self.waiting_reply = False

    def waiting(self, flag=None):
        if flag is None:
            return self.waiting_reply
        else:
            self.waiting_reply = True if flag else False

    def __repr__(self):
        return '<User:%s(%s) waiting_reply=%s>' % (self.name, self.role, self.waiting_reply)


class Request(object):
    def __init__(self, request_id, cmd, params, source=None, ttl=600.0):
        self.request_id = request_id
        self.cmd = cmd
        self.params = params
Пример #3
0
#coding:utf-8
import thread
import time

from base import MessageQueue, logging

log = logging.getLogger('client')


class SpiderClient(MessageQueue):
    def send_request(self, cmd, **kwargs):
        kwargs["type"] = 'request'
        kwargs['cmd'] = cmd
        self.send_json('server', kwargs)

    def on_receive_json(self, ident, obj, mid64):
        log.debug('on_receive_json <- [%s]: %s %s', ident, repr(mid64), obj)


if __name__ == '__main__':
    client = SpiderClient('client')
    client.connect('getf5.com', 44444)

    thread.start_new_thread(client.loop, ())

    while 1:
        print 'sleep'
        time.sleep(10)