def worker(tasks={}):
try:
logger.debug('work is Start !')
for refresh_task in tasks.values():
refresh_task["URL_OVERLOAD_PER_HOUR"] = URL_OVERLOAD_PER_HOUR
refresh_task["DIR_OVERLOAD_PER_HOUR"] = DIR_OVERLOAD_PER_HOUR
splitter.process(db, refresh_task)
logger.debug('splitter.process task : %s!' % refresh_task)
logger.debug('work is Stop !')
except Exception, e:
logger.warn("worker error. expected:%s " % e)
def test_process_tencent(self):
task = {
"username":
"******",
"urls":
["http://www.chinacache.com/a.jpg", "http://www.error.com/b.jpg"],
"update_urls": ["http://www.chinacache.com/u.jpg"],
"dirs": ["http://www.chinacache.com/"],
"callback": {
"url": "http://callback"
},
"remote_addr":
"0.0.0.0",
'serial_num':
'123456',
'request_time':
'2012-01-01 01:01:01'
}
when(splitter.COUNTER_CACHE).get('%s_URL_chinacache' %
self.hour).thenReturn('-10')
when(splitter.COUNTER_CACHE).get('%s_DIR_chinacache' %
self.hour).thenReturn('-1')
when(splitter.datetime).strptime('2012-01-01 01:01:01',
'%Y-%m-%d %X').thenReturn(
datetime(2012, 1, 1, 1, 1, 1))
when(splitter.submit).delay(task).thenReturn(True)
when(bson).ObjectId().thenReturn(ObjectId("4e4c7b9f5bc89412ec000004"))
self.assertEqual({"r_id": "4e4c7b9f5bc89412ec000004"},
splitter.process(self.db, task))
verify(splitter.submit).delay(task)
def test_process_not_overload(self):
task = {
"username":
"******",
"urls":
["http://www.chinacache.com/a.jpg", "http://www.error.com/b.jpg"],
"update_urls": ["http://www.chinacache.com/u.jpg"],
"preload_urls": ["http://www.chinacache.com/c.jpg"],
"dirs": ["http://www.chinacache.com/"],
"callback": {
"url": "http://callback"
}
}
when(splitter.submit).delay(task).thenReturn(True)
when(splitter.COUNTER_CACHE).get('%s_URL_chinacache' %
self.hour).thenReturn('-10')
when(splitter.COUNTER_CACHE).get('%s_DIR_chinacache' %
self.hour).thenReturn('-1')
self.db.request = mock()
when(self.db.request).insert({
"username": task.get("username"),
"callback": task.get("callback"),
"status": "PROGRESS",
"created_time": self.dt,
'remote_addr': '',
'serial_num': ''
}).thenReturn(ObjectId("4e4c7b9f5bc89412ec000004"))
self.assertEqual({"r_id": "4e4c7b9f5bc89412ec000004"},
splitter.process(self.db, task, True))
verify(splitter.submit).delay(task)
def test_process_overload(self):
task = {
"username":
"******",
"urls":
["http://www.chinacache.com/a.jpg", "http://www.error.com/b.jpg"],
"update_urls": ["http://www.chinacache.com/u.jpg"],
"preload_urls": ["http://www.chinacache.com/c.jpg"],
"dirs": ["http://www.chinacache.com/"],
"callback": {
"url": "http://callback"
}
}
when(splitter.submit).delay(task).thenReturn(True)
when(splitter.COUNTER_CACHE).get('%s_URL_chinacache' %
self.hour).thenReturn('2')
when(splitter.COUNTER_CACHE).get('%s_DIR_chinacache' %
self.hour).thenReturn('1')
task1 = {
'username': '******',
'preload_urls': ['http://www.chinacache.com/c.jpg'],
'r_id': ObjectId(mock_object_id),
'update_urls': [],
'callback': {
'url': 'http://callback'
},
'urls': [],
'dirs': ['http://www.chinacache.com/']
}
task2 = {
'username': '******',
'preload_urls': ['http://www.chinacache.com/c.jpg'],
'r_id': ObjectId(mock_object_id),
'update_urls': [],
'callback': {
'url': 'http://callback'
},
'urls': [],
'dirs': []
}
when(splitter).setCounterCache(task1, 3, 'URL').thenReturn(None)
when(splitter).setCounterCache(task2, 1, 'DIR').thenReturn(None)
self.assertEqual(
{
'dirExceed': 1,
"urlExceed": 3,
'r_id': mock_object_id
}, splitter.process(self.db, task, True))
def test_process_url_overload_without_dir(self):
task = {
"username":
"******",
"urls":
["http://www.chinacache.com/a.jpg", "http://www.error.com/b.jpg"],
"update_urls": ["http://www.chinacache.com/u.jpg"],
"callback": {
"url": "http://callback"
}
}
when(splitter.submit).delay(task).thenReturn(True)
when(splitter.COUNTER_CACHE).get('%s_URL_chinacache' %
self.hour).thenReturn('2')
when(splitter.COUNTER_CACHE).get('%s_DIR_chinacache' %
self.hour).thenReturn('-4')
when(splitter).setCounterCache('chinacache', 3, 'URL').thenReturn(5)
self.assertEqual({
'urlExceed': 3,
'r_id': mock_object_id
}, splitter.process(self.db, task, True))