def test_it(data):
setup_log('netboy')
boy = NetBoy()
boy.use_triggers(['test.functional.netboy.test_pycurl_get_header.trig_it'])
boy.use_spider('pycurl').use_filter([
'url', 'effect', 'title', 'header', 'cookie'
]).use_mode('process').use_timeout(10, 5, 5, 5).use_workers(1, 1, 1)
# boy.info['cookie'] = bytes('test=value',"utf8")
resp = boy.run(data)
return resp
def test_it(data):
setup_log('netboy')
boy = NetBoy()
boy.use_spider(
'pycurl'
).use_filter(
['url', 'effect', 'title', 'charset']
).use_mode('process').use_timeout(10, 5, 5, 5).use_workers(4, 2, 2)
resp = boy.run(data)
return resp
def test_it(data):
setup_log('netboy')
boy = NetBoy()
boy.use_spider('pycurl').use_filter(
['url', 'title', 'effect', 'data']).use_triggers([
'test.functional.netboy.test_baidu_subdomain.trig_sub',
'test.functional.netboy.test_baidu_subdomain.trig_print',
]).use_mode('process').use_timeout(10, 5, 5, 5).use_workers(4, 2, 2)
resp = boy.run(data)
return resp
def test_it(data):
setup_log('netboy')
boy = NetBoy()
boy.use_triggers(
['test.functional.netboy.test_pycurl_server_set_cookie.trig_it'])
boy.use_spider('pycurl').use_filter(
['url', 'effect', 'title',
'cookie']).use_mode('process').use_timeout(10, 5, 5,
5).use_workers(1, 1, 1)
resp = boy.run(data)
return resp
def test_it(data):
setup_log('netboy')
boy = NetBoy()
boy.use_spider('pycurl').use_filter([
'url', 'title', 'effect', 'data'
]).use_workers().use_triggers([
'test.functional.netboy.test_thread_pycurl_trigger_redirect.trigger_it',
'test.functional.netboy.test_thread_pycurl_trigger_redirect.trigger_it2'
]).use_mode('thread').use_timeout(15, 5)
resp = boy.run(data)
return resp
def test_it(data):
setup_log('netboy')
boy = NetBoy()
boy.use_spider('pycurl').use_filter([
'url', 'title', 'effect'
]).use_triggers([
'test.functional.netboy.test_thread_pycurl_basic.trig_it'
]).use_mode('thread').use_timeout(10, 5, 5,
5).use_workers(4, 2,
2) #.use_queue('worker')
resp = boy.run(data)
return resp
def test_it(data):
setup_log('netboy')
boy = NetBoy()
boy.use_spider('pycurl').use_filter(
['url', 'title', 'effect', 'data']).use_triggers([
'test.functional.netboy.test_baidu_subdomain_location.trig_sub',
'test.functional.netboy.test_baidu_subdomain_location.trig_print',
]).use_mode('celery').use_timeout(10, 5, 5, 5).use_workers(
8, 2, 2).use_queue('worker').use_final(
'test.functional.netboy.test_baidu_subdomain_location.final')
boy.info['job_id'] = 'test'
resp = boy.run(data)
return resp
from store.seaweed import SeaweedStore
store = SeaweedStore({
'host':
payload.get('weed_host', '192.168.199.203'),
'port':
payload.get('weed_port', 9333)
})
res = store.create(url + '.webp', file.getvalue())
# print(res)
# print(file.getvalue()[:100])
else:
print(response)
if __name__ == '__main__':
setup_log('netboy')
data = [
# "http://172.30.0.3:9992",
"http://ip.cn",
# "http://www.cnbeta.com",
# "http://www.hxsjjq.gov.cn",
"http://www.douban.com"
# "http://www.bing.com",
# "http://www.hbcs.gov.cn",
# "http://www.hnxcdj.com",
# "http://www.ryzj.gov.cn",
# "http://www.lhsajj.com",
# "http://www.nysylj.com",
# "http://www.nyszglc.com",
# "http://www.ayjtj.gov.cn",
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
setup_log('netboy')
cls._instances[cls] = super(Singleton,
cls).__call__(*args, **kwargs)
return cls._instances[cls]