def __init__(self, *args, **kwargs):
''' '''
ClientServiceBase.__init__(self, *args, **kwargs)
self.max_count = MAX_COUNT,
self.count = MAX_COUNT # 每小时最大请求次数
self.last_call = 0
self.users = []
self.user_cursor = 0
self.api = None
self.ready = False
def __init__(self, *args, **kwargs):
''' '''
ClientServiceBase.__init__(self, *args, **kwargs)
cfg = kwargs['cfg']
self.name = socket.gethostname() + cfg.prefix # node name
self.proxy = cfg.http_proxy # not used
self.user_agent = cfg.http_agent
self.max_agent = cfg.max_agent
self.agent_pool = TimedAgentPool()
self.last_clear = 0
self.ready = True
def __init__(self, *args, **kwargs):
''' '''
ClientServiceBase.__init__(self, *args, **kwargs)
cfg = kwargs['cfg']
self.name = socket.gethostname() + cfg.prefix # node name
self.proxy = cfg.http_proxy # not used
self.user_agent = cfg.http_agent
self.max_agent = cfg.max_agent
self.agent_pool = TimedAgentPool()
self.last_clear = 0
self.ready = True
def __init__(self, *args, **kwargs):
ClientServiceBase.__init__(self, *args, **kwargs)
self.count = 4000
self.dbconn = None
self._users = None
cfg = kwargs['cfg']
self.dbhost = cfg.userdb_host
self.dbport = cfg.userdb_port
self.dbname = cfg.userdb_dbname
self.dbprobecollection = cfg.userdb_idcollection
self.dbcollection = cfg.userdb_collection
self._ids = None
self.users = []
def __init__(self, *args, **kwargs):
''' '''
ClientServiceBase.__init__(self, *args, **kwargs)
cfg = kwargs['cfg']
self.name = socket.gethostname() + cfg.prefix # node name
self.proxy = cfg.http_proxy # not used
self.userAgent = cfg.http_agent
self.max_agent = cfg.max_agent
self.agentPool = TimedAgentPool(self.interval_min,
self.interval_max,
self.login_interval)
self.token = self.get_token()
self.last_clear = 0
self.ready = True
def __init__(self, *args, **kwargs):
''' '''
ClientServiceBase.__init__(self, *args, **kwargs)
cfg = kwargs['cfg']
self.version_major = 1
self.version_minor = 1
self.max_slots = 10
self.ip = cfg.ip
self.userAgent = cfg.http_agent
self.users = cfg.weibo_users
self.agentPool = TimeAgentPool(
cfg.http_interval_min,
cfg.http_interval_max,
)
self.proxy = cfg.http_proxy
def startService(self):
yield ClientServiceBase.startService(self)
self.dbconn = pymongo.connection.Connection(
host = self.dbhost,
port = self.dbport)
self._users = self.dbconn[self.dbname][self.dbcollection]
self._ids = self.dbconn[self.dbname][self.dbprobecollection]
self.refresh_users()
def refresh_users(self):
try:
self.users = ClientServiceBase.controllerUsers(self)
self.user_cursor = self.user_cursor % len(self.users)
if not self.ready:
self.ready = True
yield self.updateStatus()
except pb.RemoteError:
pass
reactor.callLater(60.0, self.refresh_users)
def __init__(self, *args, **kwargs):
''' '''
ClientServiceBase.__init__(self, *args, **kwargs)
cfg = kwargs['cfg']
self.name = socket.gethostname() + cfg.prefix # node name
self.proxy = cfg.http_proxy # not used
self.userAgent = cfg.http_agent
self.interval_min = cfg.http_interval_min
self.interval_max = cfg.http_interval_max
self.avg_interval = (self.interval_min + self.interval_max) / 2.0
self.login_interval = cfg.login_interval
self.max_agent = cfg.max_agent
self.agentPool = TimedAgentPool(self.interval_min,
self.interval_max,
self.login_interval)
self.last_clear = 0
self.count = int(3600.0*2 / self.avg_interval)
self.ready = True
def startService(self):
''' '''
for username, password in self.users.iteritems():
cookies = CookieJar()
agent = getAgent(
username,
password,
self.proxy,
self.userAgent,
cookies,
self.ip,
)
self.agentPool.initAgent(agent)
os.environ['TZ'] = "PRC"
time.tzset()
return ClientServiceBase.startService(self)
def startService(self):
''' '''
yield ClientServiceBase.startService(self)
self.api = API()
self.deferred_call = []
self.clear_count()
def startService(self):
''' start the fetch service '''
os.environ['TZ'] = 'PRC'
time.tzset()
yield ClientServiceBase.startService(self)
self.fillAgents()
def startService(self):
''' start the fetch service '''
os.environ['TZ'] = 'PRC'
time.tzset()
yield ClientServiceBase.startService(self)
self.fillAgents()
def stopService(self):
self.dbconn.close()
self.dbconn = None
self._users = None
self.users = None
yield ClientServiceBase.stopService(self)