def resolve_one(record_type, domain, server_type, server_ip, server_port,
timeout, strategy, queue):
server = (server_type, server_ip, server_port)
begin_time = time.time()
answers = []
try:
if 'udp' == server_type:
answers = resolve_over_udp(record_type, domain, server_ip,
server_port, timeout, strategy)
elif 'tcp' == server_type:
answers = resolve_over_tcp(record_type, domain, server_ip,
server_port, timeout)
else:
LOGGER.error('unsupported server type: %s' % server_type)
except NoSuchDomain as e:
queue.put((server, e))
return
except:
LOGGER.exception('failed to resolve one: %s' % domain)
if answers:
if answers[0] in WRONG_ANSWERS:
LOGGER.info('%s://%s:%s resolved %s => %s' %
(server_type, server_ip, server_port, domain, answers))
LOGGER.critical('!!! should not resolve wrong answer')
return
queue.put((server, answers))
elapsed_seconds = time.time() - begin_time
LOGGER.info('%s://%s:%s resolved %s => %s, took %0.2f' %
(server_type, server_ip, server_port, domain, answers,
elapsed_seconds))
def yielding_checked_fnwalk(path, fn, sleep_interval=0.01):
try:
parent, name = os.path.split(path)
entry = scandir.GenericDirEntry(parent, name)
if fn(entry):
yield entry
queue = gevent.queue.LifoQueue()
if entry.is_dir():
queue.put(path)
while True:
try:
path = queue.get(timeout=0)
except gevent.queue.Empty:
break
else:
for entry in scandir.scandir(path):
if fn(entry):
if entry.is_dir():
queue.put(entry.path)
yield entry
gevent.sleep(sleep_interval)
except Exception as e:
logging.exception(
'Exception while directory walking: {}'.format(str(e)))
def _channel_dispatcher(self):
while True:
try:
event = self._events.recv()
except Exception as e:
logger.error( \
'zerorpc.ChannelMultiplexer, ' + \
'ignoring error on recv: {0}'.format(e))
continue
channel_id = event.header.get('response_to', None)
queue = None
if channel_id is not None:
channel = self._active_channels.get(channel_id, None)
if channel is not None:
queue = channel._queue
elif self._broadcast_queue is not None:
queue = self._broadcast_queue
if queue is None:
logger.error( \
'zerorpc.ChannelMultiplexer, ' + \
'unable to route event: ' + \
event.__str__(ignore_args=True))
else:
queue.put(event)
def writer(seconds, bucket, name=None, queue=None, quantity=1, file_size=1, file_stddev=0, file_name_seed=None):
with gevent.Timeout(seconds, False):
while (1):
r = random.randint(0, 65535)
r2 = r
if file_name_seed != None:
r2 = file_name_seed
files = generate_objects.get_random_files(
quantity=quantity,
mean=1024 * file_size,
stddev=1024 * file_stddev,
seed=r,
)
start = time.clock()
generate_objects.upload_objects(bucket, files, r2)
end = time.clock()
elapsed = end - start
if queue:
queue.put(Result(name,
type=Result.TYPE_WRITER,
time=elapsed,
size=sum(f.size/1024 for f in files),
)
)
def test_single_proxy(my_ip, ip, port, retry_count=0):
if retry_count == 3:
logging.debug('Could not connect to: {}:{}'.format(ip, port))
return
retry_count += 1
proxy_handler = urllib2.ProxyHandler({'http': '{}:{}'.format(ip, port)})
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
try:
response = urllib2.urlopen(LOCAL_SERVER)
except (urllib2.URLError, httplib.BadStatusLine) as e:
message = e.message or 'something wrong happened, retring...'
logging.debug('Error ({}), with proxy: {}:{}'.format(message, ip, port))
return test_single_proxy(my_ip, ip, port, retry_count)
except socket.timeout as e:
message = e.message or 'something wrong happened, retring...'
logging.debug('Error ({}), with proxy: {}:{}'.format(message, ip, port))
return False
response_info = response.info()
leaking_headers = []
for header_key in privacy_headers:
info = response_info
header_value = info.get(header_key, info.get(header_key.lower()))
if not header_value:
continue
if my_ip in header_value:
leaking_headers.append(header_key)
if leaking_headers:
if verbose:
headers = ', '.join(leaking_headers)
logging.debug('Proxy leaking header {}: {}'.format(headers, ip))
return False
if verbose:
logging.debug('Proxy safe: {}:{}'.format(ip, port))
queue.put('{}:{}'.format(ip, port))
def wrapper(*a,**kw):
if gevent.getcurrent() == drainQueue:
print("calling queue in queue...")
return f(*a,**kw)
resume = gevent.event.AsyncResult()
queue.put((f,a,kw,resume))
return resume.get()
def reader(bucket, objname, queue):
key = bucket.new_key(objname)
fp = realistic.FileVerifier()
result = dict(type="r", bucket=bucket.name, key=key.name)
start = time.time()
try:
key.get_contents_to_file(fp)
except gevent.GreenletExit:
raise
except Exception as e:
# stop timer ASAP, even on errors
end = time.time()
result.update(error=dict(msg=str(e), traceback=traceback.format_exc()))
# certain kinds of programmer errors make this a busy
# loop; let parent greenlet get some time too
time.sleep(0)
else:
end = time.time()
if not fp.valid():
result.update(error=dict(msg="md5sum check failed"))
elapsed = end - start
result.update(start=start, duration=int(round(elapsed * NANOSECOND)), chunks=fp.chunks)
queue.put(result)
def parallelPost(self, payload):
queue = gevent.queue.Queue()
gList = []
if type(self.resourceList) is list:
for i in self.resourceList:
queue.put(i)
for t in range(len(self.resourceList)):
gl = gevent.spawn(postRequest, queue, payload)
gList.append(gl)
print str(gList)
gevent.joinall(gList)
return GreenletRequests.NodeResponsesPost
if type(self.resourceList) is dict:
for k, v in self.resourceList.iteritems():
print k
queue.put(k)
for k, v in self.resourceList.iteritems():
gl = gevent.spawn(postRequest, queue, v)
gList.append(gl)
print str(gList)
gevent.joinall(gList)
return GreenletRequests.NodeResponsesPost
def do_resolve(host, dnsservers, queue):
try:
iplist = dns_remote_resolve(host, dnsservers, GC.DNS_BLACKLIST, timeout=2)
queue.put((host, dnsservers, iplist or []))
except (socket.error, OSError) as e:
logging.error(u'远程解析失败：host=%r，%r', host, e)
queue.put((host, dnsservers, []))
def add(queue, args):
name = optdict.get('-q')
if name: queue.use(name)
funcname = optdict.get('-f', 'main')
for url in args:
queue.put(httputils.ReqInfo(funcname, url).pack())
print 'put:', url
def writer(bucket, objname, fp, queue):
key = bucket.new_key(objname)
result = dict(
type='w',
bucket=bucket.name,
key=key.name,
)
start = time.time()
try:
key.set_contents_from_file(fp, rewind=True)
except gevent.GreenletExit:
raise
except Exception as e:
# stop timer ASAP, even on errors
end = time.time()
result.update(error=dict(
msg=str(e),
traceback=traceback.format_exc(),
), )
# certain kinds of programmer errors make this a busy
# loop; let parent greenlet get some time too
time.sleep(0)
else:
end = time.time()
elapsed = end - start
result.update(
start=start,
duration=int(round(elapsed * NANOSECOND)),
chunks=fp.last_chunks,
)
queue.put(result)
def add(queue, args):
name = optdict.get('-q')
if name: queue.use(name)
funcname = optdict.get('-f', 'main')
for url in args:
queue.put(httputils.ReqInfo(funcname, url).pack())
print 'put:', url
def _channel_dispatcher(self):
while True:
try:
event = self._events.recv()
except Exception as e:
logger.error( \
'zerorpc.ChannelMultiplexer, ' + \
'ignoring error on recv: {0}'.format(e))
continue
channel_id = event.header.get('response_to', None)
queue = None
if channel_id is not None:
channel = self._active_channels.get(channel_id, None)
if channel is not None:
queue = channel._queue
elif self._broadcast_queue is not None:
queue = self._broadcast_queue
if queue is None:
logger.error( \
'zerorpc.ChannelMultiplexer, ' + \
'unable to route event: ' + \
event.__str__(ignore_args=True))
else:
queue.put(event)
def resolve_one(record_type,
domain,
server_type,
server_ip,
server_port,
timeout,
strategy,
wrong_answer,
queue=None):
answers = []
try:
# LOGGER.info('%s resolve %s at %s:%s' % (server_type, domain, server_ip, server_port))
if 'udp' == server_type:
wrong_answers = set(wrong_answer) if wrong_answer else set()
wrong_answers |= BUILTIN_WRONG_ANSWERS()
answers = resolve_over_udp(record_type, domain, server_ip,
server_port, timeout, strategy,
wrong_answers)
elif 'tcp' == server_type:
answers = resolve_over_tcp(record_type, domain, server_ip,
server_port, timeout)
else:
LOGGER.error('unsupported server type: %s' % server_type)
except NoSuchDomain as e:
queue.put((domain, e))
return
except:
LOGGER.exception('failed to resolve one: %s' % domain)
if answers and queue:
queue.put((domain, answers))
LOGGER.info(
'%s resolved %s at %s:%s => %s' %
(server_type, domain, server_ip, server_port, json.dumps(answers)))
return
def _cacher(begin, end):
put = False
# Hack (1)
old_owner, lock._owner = lock._owner, gevent.getcurrent()
with lock:
try:
for chunk_begin, chunk_end, chunk in _downloader():
# Ensure that the chunk we have downloaded is a chunk that
# we are interested in. For instance, we may need the
# middle part of a song, but this will mean that we the
# beginning should be downloaded (and saved to file) first.
if (chunk_begin <= begin < chunk_end) or \
(chunk_begin <= end < chunk_end):
put = not put
if put:
queue.put((chunk_begin, chunk_end, chunk))
finally:
# Make sure the streamer stops
queue.put(StopIteration)
# Hack (2)
lock._owner = old_owner
def writer(seconds,
bucket,
name=None,
queue=None,
quantity=1,
file_size=1,
file_stddev=0,
file_name_seed=None):
with gevent.Timeout(seconds, False):
while (1):
r = random.randint(0, 65535)
r2 = r
if file_name_seed != None:
r2 = file_name_seed
files = generate_objects.get_random_files(
quantity=quantity,
mean=1024 * file_size,
stddev=1024 * file_stddev,
seed=r,
)
start = time.clock()
generate_objects.upload_objects(bucket, files, r2)
end = time.clock()
elapsed = end - start
if queue:
queue.put(
Result(
name,
type=Result.TYPE_WRITER,
time=elapsed,
size=sum(f.size / 1024 for f in files),
))
def _channel_dispatcher(self): # channel 调度器, 给 给定的 channel 分发 event
while True:
try:
event = self._events.recv()
except Exception:
logger.exception(
'zerorpc.ChannelMultiplexer ignoring error on recv')
continue
channel_id = event.header.get(
u'response_to', None
) # 获取目标 ～～identity～～ zmq.ROUTER? channel_id 是 message_id n122行
queue = None
if channel_id is not None:
channel = self._active_channels.get(
channel_id, None
) # 根据 ～～identity～～ 获取 channel, channel_id 是 message_id n122行
if channel is not None:
queue = channel._queue
elif self._broadcast_queue is not None: # 客户端来的第一个 event 包， 此时没有创建 channel，所以没有channel_id None
queue = self._broadcast_queue
if queue is None:
logger.warning('zerorpc.ChannelMultiplexer,'
' unable to route event: {0}'.format(
event.__str__(ignore_args=True)))
else:
queue.put(event)
def reader(data_file: str, queue: queue.Queue):
with open('./data/' + data_file, 'rb') as f:
data = f.readline()
while data:
data = f.readline()
#print(data)
queue.put(data, block=True)
gevent.sleep(0.01)
def put(self, data):
try:
self._put(data, False, .001)
except gevent.queue.Full:
self.queue = queue = Buffer(self.queue)
self._put = queue.put
self.close = queue.close
queue.put(data)
def sleepy(time, queue):
num = 2
while True:
print("done like a good script")
num_str = "Ok-" + str(num)
queue.put(num_str)
num += 2
gevent.sleep(time)
def do_local_resolve(host, queue):
assert isinstance(host, basestring)
for _ in xrange(3):
try:
queue.put((host, socket.gethostbyname_ex(host)[-1]))
except (socket.error, OSError) as e:
logging.warning('socket.gethostbyname_ex host=%r failed: %s', host, e)
time.sleep(0.1)
def paas_fileobj_pipe(fileobj, queue, bufsize=8192):
logging.info('paas_fileobj_pipe(fileobj=%r, queue)', fileobj)
while 1:
data = fileobj.read(bufsize)
if data:
queue.put(data)
else:
queue.put(StopIteration)
break
def test_proxy_list(http_proxies, pool_size, server_port):
pool = gevent.pool.Pool(pool_size)
my_ip = whats_my_ip()
globals()['LOCAL_SERVER'] = 'http://{}:{}/'.format(my_ip, server_port)
for proxy in http_proxies:
ip, port = proxy.rsplit(':')
pool.spawn(test_single_proxy, my_ip, ip, port)
pool.join()
queue.put(StopIteration)
def __call__(self, env, _):
ws = env['wsgi.websocket']
while True:
data = ws.receive()
if data is None:
break
for queue in Messenger.queues.itervalues():
queue.put(data)
ws.close()
def download(hashes):
for h in hashes:
queue.put(h)
workers = [
gevent.spawn(_download_helper)
for _ in xrange(128)
]
gevent.joinall(workers)
def handle_incoming_records(records, queue, unit_conf):
n_handled = 0
for record in records:
logger.debug('Got journal record: %s', record)
queue.put(
Record(unit_conf, record['__REALTIME_TIMESTAMP'],
record['__CURSOR'], unit_conf.format_func(record)))
n_handled += 1
return n_handled
def download(hashes, workercount=32):
for idx, h in enumerate(hashes):
if idx and idx % 500 == 0:
for _ in xrange(workercount):
queue.put("wait")
queue.put(h)
workers = [gevent.spawn(_download_helper) for _ in xrange(workercount)]
gevent.joinall(workers)
def do_remote_resolve(host, dnsserver, queue):
assert isinstance(dnsserver, basestring)
for dnslib_resolve in (dnslib_resolve_over_udp, dnslib_resolve_over_tcp):
try:
time.sleep(random.random())
iplist = dnslib_record2iplist(dnslib_resolve(host, [dnsserver], timeout=4, blacklist=()))
queue.put((host, dnsserver, iplist))
except (socket.error, OSError) as e:
logging.warning('%r remote host=%r failed: %s', dnslib_resolve, host, e)
time.sleep(1)
def publish(self, topic, body):
"""
Publish a message with the given body to the queue identified by topic.
"""
queue = _get_queue(topic)
msg = Message(body)
try:
queue.put(msg)
except gevent.queue.Full:
log.warn("queue full, dropping message")
def handle_request(self, proto_input, queue, request_meta):
"""
从 proto_input中读取数据，然后调用processor处理请求，结果暂时缓存在内存中, 最后一口气交给 queue,
由专门的 greenlet将数据写回到socket上
request_meta = (name, type, seqid, start_time)
"""
start_time0 = time.time()
# 1. 获取一个可用的trans_output
if len(self.out_protocols) > 0:
trans_output, proto_output = self.out_protocols.popleft()
trans_output.prepare_4_frame() # 预留4个字节的Frame Size
else:
trans_output = TCyMemoryBuffer()
trans_output.prepare_4_frame()
proto_output = TCyBinaryProtocol(trans_output,
client=False) # 无状态的
try:
# 2.1 处理正常的请求
self.processor.process(proto_input, proto_output)
queue.put(trans_output)
except Exception as e:
# 2.2 处理异常(主要是结果序列化时参数类型不对的情况)
trans_output.prepare_4_frame()
name = request_meta[0]
seqId = request_meta[2]
msg = '%s, Exception: %s, Trace: %s' % (name, e,
traceback.format_exc())
x = TApplicationException(TApplicationException.INVALID_PROTOCOL,
msg)
proto_output.writeMessageBegin(name, TMessageType.EXCEPTION, seqId)
x.write(proto_output)
proto_output.writeMessageEnd()
proto_output.trans.flush()
queue.put(trans_output)
finally:
start_time = request_meta[3]
now = time.time()
elapsed = now - start_time
if elapsed > 2:
# 做异常记录
exception_logger.info(
"Exception Request: %s %s seqId: %s_%s, Elaspsed: %.3f, Execute: %.3f",
request_meta[0], request_meta[1], self.pid,
request_meta[2], elapsed, now - start_time0)
# 3. 回收 transport 和 protocol
self.out_protocols.append((trans_output, proto_output))
def crawler():
global handler
global crawled
global DATA
global ALREADY_CRAWLED
global MAX_CRAWLS
global ITEMS_COUNT
handler.log("job started...")
print "job started..."
while 1:
try:
url = queue.get(timeout=1)
if url in ALREADY_CRAWLED:
continue
content = handler.loadPage(url)
content = content.decode('utf-8')
doc = lxml.html.fromstring(content)
imgs = doc.xpath("//div[@class='outletProductImage']/a/img")
for img in imgs:
img_url = main_domain + img.attrib['src']
if img_url:
handler.addResults(img_url)
ITEMS_COUNT += 1
print img_url
handler.log(img_url)
#add the next pages to crawl to the queue
if crawled < MAX_CRAWLS:
crawled += 1
ALREADY_CRAWLED.append(url)
hrefs = doc.xpath("//div[@class='browsePageControls']/a[@class='control next']")
for href in hrefs:
href = href.attrib['href']
if href:
next_url = main_domain+href
if next_url not in ALREADY_CRAWLED:
queue.put(next_url)
break #take only the first nav on the top page
else:
raise gevent.queue.Empty
except gevent.queue.Empty:
break
print "job done"
handler.log("job done")
print "so far crawled %s pages" % crawled
handler.log("so far crawled %s pages" % crawled)
def reader(bucket, worker_id, file_names, queue, rand):
while True:
objname = rand.choice(file_names)
key = bucket.new_key(objname)
fp = realistic.FileVerifier()
result = dict(
type='r',
bucket=bucket.name,
key=key.name,
worker=worker_id,
)
start = time.time()
try:
key.get_contents_to_file(fp)
except gevent.GreenletExit:
raise
except Exception as e:
# stop timer ASAP, even on errors
end = time.time()
result.update(
error=dict(
msg=str(e),
traceback=traceback.format_exc(),
),
)
# certain kinds of programmer errors make this a busy
# loop; let parent greenlet get some time too
time.sleep(0)
else:
end = time.time()
if not fp.valid():
m='md5sum check failed start={s} ({se}) end={e} size={sz} obj={o}'.format(s=time.ctime(start), se=start, e=end, sz=fp.size, o=objname)
result.update(
error=dict(
msg=m,
traceback=traceback.format_exc(),
),
)
print "ERROR:", m
else:
elapsed = end - start
result.update(
start=start,
duration=int(round(elapsed * NANOSECOND)),
chunks=fp.chunks,
)
queue.put(result)
def reader(bucket, worker_id, file_names, queue, rand):
while True:
objname = rand.choice(file_names)
key = bucket.new_key(objname)
fp = realistic.FileValidator()
result = dict(
type='r',
bucket=bucket.name,
key=key.name,
worker=worker_id,
)
start = time.time()
try:
key.get_contents_to_file(fp._file)
except gevent.GreenletExit:
raise
except Exception as e:
# stop timer ASAP, even on errors
end = time.time()
result.update(error=dict(
msg=str(e),
traceback=traceback.format_exc(),
), )
# certain kinds of programmer errors make this a busy
# loop; let parent greenlet get some time too
time.sleep(0)
else:
end = time.time()
if not fp.valid():
m = 'md5sum check failed start={s} ({se}) end={e} size={sz} obj={o}'.format(
s=time.ctime(start),
se=start,
e=end,
sz=fp._file.tell(),
o=objname)
result.update(error=dict(
msg=m,
traceback=traceback.format_exc(),
), )
print "ERROR:", m
else:
elapsed = end - start
result.update(
start=start,
duration=int(round(elapsed * NANOSECOND)),
)
queue.put(result)
def val(self, value):
new_val = bool(value)
if new_val != self.__val:
self.__val0 = self.__val
self.__val = new_val
if not self.__val0 and self.__val:
edge = self.__r_edge_data()
else:
edge = self.__f_edge_data()
for queue in self.__usage_points__.values():
queue.put(edge)
def feed():
queue = self.queue
while self.size > 0:
file.seek(self.read_position)
data = load(file)
self.read_position = file.tell()
queue.put(data)
if self.size > 0:
# We check the size here, in case the queue was closed
if data:
self.size_bytes -= len(data)
self.size -= 1
else:
assert self.size == -1
def parallelPut(self, payload):
queue = gevent.queue.Queue()
for i in self.resourceList:
queue.put(i)
gList = []
for p in range(len(self.resourceList)):
gl = gevent.spawn(putRequest, queue, payload)
gList.append(gl)
print str(gList)
gevent.joinall(gList)
return GreenletRequests.NodeResponsesPut
def parallelDelete(self):
queue = gevent.queue.Queue()
for i in self.resourceList:
queue.put(i)
gList = []
for t in range(len(self.resourceList)):
gl = gevent.spawn(deleteRequest, queue)
gList.append(gl)
print str(gList)
gevent.joinall(gList)
return GreenletRequests.NodeResponsesDelete
def handle_request(self, proto_input, queue, request_meta):
"""
从 proto_input中读取数据，然后调用processor处理请求，结果暂时缓存在内存中, 最后一口气交给 queue,
由专门的 greenlet将数据写回到socket上
request_meta = (name, type, seqid, start_time)
"""
start_time0 = time.time()
# 1. 获取一个可用的trans_output
if len(self.out_protocols) > 0:
trans_output, proto_output = self.out_protocols.popleft()
trans_output.prepare_4_frame() # 预留4个字节的Frame Size
else:
trans_output = TCyMemoryBuffer()
trans_output.prepare_4_frame()
proto_output = TCyBinaryProtocol(trans_output, client=False) # 无状态的
try:
# 2.1 处理正常的请求
self.processor.process(proto_input, proto_output)
queue.put(trans_output)
except Exception as e:
# 2.2 处理异常(主要是结果序列化时参数类型不对的情况)
trans_output.prepare_4_frame()
name = request_meta[0]
seqId = request_meta[2]
msg = '%s, Exception: %s, Trace: %s' % (name, e, traceback.format_exc())
x = TApplicationException(TApplicationException.INVALID_PROTOCOL, msg)
proto_output.writeMessageBegin(name, TMessageType.EXCEPTION, seqId)
x.write(proto_output)
proto_output.writeMessageEnd()
proto_output.trans.flush()
queue.put(trans_output)
finally:
start_time = request_meta[3]
now = time.time()
elapsed = now - start_time
if elapsed > 2:
# 做异常记录
exception_logger.info("Exception Request: %s %s seqId: %s_%s, Elaspsed: %.3f, Execute: %.3f", request_meta[0], request_meta[1], self.pid, request_meta[2], elapsed, now - start_time0)
# 3. 回收 transport 和 protocol
self.out_protocols.append((trans_output, proto_output))
def reader(bucket, worker_id, file_names, queue, rand):
while True:
objname = rand.choice(file_names)
key = bucket.new_key(objname)
fp = realistic.FileVerifier()
result = dict(
type='r',
bucket=bucket.name,
key=key.name,
worker=worker_id,
)
start = time.time()
try:
key.get_contents_to_file(fp)
except gevent.GreenletExit:
raise
except Exception as e:
# stop timer ASAP, even on errors
end = time.time()
result.update(
error=dict(
msg=str(e),
traceback=traceback.format_exc(),
),
)
# certain kinds of programmer errors make this a busy
# loop; let parent greenlet get some time too
time.sleep(0)
else:
end = time.time()
if not fp.valid():
result.update(
error=dict(
msg='md5sum check failed',
),
)
elapsed = end - start
result.update(
start=start,
duration=int(round(elapsed * NANOSECOND)),
chunks=fp.chunks,
)
queue.put(result)
def crawler():
global crawled
while 1:
try:
u = queue.get(timeout=0)
response = requests.get(u)
print response.status_code, u
for link in re.findall('<a href="(http.*?)"', response.content):
if crawled < 10:
crawled += 1
queue.put(link)
except gevent.queue.Empty:
break
def stop(self):
"""Stop the greenlet workers and empty all queues."""
with self._state_change:
if not self._running:
return
self._running = False
for queue in (self.callback_queue, ):
queue.put(_STOP)
while self._workers:
worker = self._workers.pop()
worker.join()
# Clear the queues
self.callback_queue = Queue() # pragma: nocover
def crawler():
global crawled
while 1:
try:
u = queue.get(timeout=1)
response = requests.get(u)
print response.status_code, u
for link in re.findall('<a href="(http.*?)"', response.content):
if crawled < 10:
crawled += 1
queue.put(link)
except gevent.queue.Empty:
break
def stop(self):
"""Stop the greenlet workers and empty all queues."""
with self._state_change:
if not self._running:
return
self._running = False
for queue in (self.callback_queue,):
queue.put(_STOP)
while self._workers:
worker = self._workers.pop()
worker.join()
# Clear the queues
self.callback_queue = Queue() # pragma: nocover
def resolve_one(record_type, domain, server_type, server_ip, server_port, timeout, strategy, queue):
server = (server_type, server_ip, server_port)
answers = []
try:
if 'udp' == server_type:
answers = resolve_over_udp(record_type, domain, server_ip, server_port, timeout, strategy)
elif 'tcp' == server_type:
answers = resolve_over_tcp(record_type, domain, server_ip, server_port, timeout)
else:
LOGGER.error('unsupported server type: %s' % server_type)
except NoSuchDomain as e:
queue.put((server, e))
return
except:
LOGGER.exception('failed to resolve one: %s' % domain)
if answers:
queue.put((server, answers))
LOGGER.info('%s://%s:%s resolved %s => %s' % (server_type, server_ip, server_port, domain, answers))
def parallelFileGet(self):
queue = gevent.queue.Queue()
for i in self.resourceList:
queue.put(i)
gList = []
for t in range(len(self.resourceList)):
gl = gevent.spawn(getrequestFile, queue, self.outputDir)
gList.append(gl)
# print str(gList)
app.logger.info(
'[%s] : [INFO] gFileList %s',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'),
str(gList))
gevent.joinall(gList)
return GreenletRequests.NodeResponsesGet
def parallelPut(self, payload):
queue = gevent.queue.Queue()
for i in self.resourceList:
queue.put(i)
gList = []
for p in range(len(self.resourceList)):
gl = gevent.spawn(putRequest, queue, payload)
gList.append(gl)
# print str(gList)
app.logger.info(
'[%s] : [INFO] gList %s',
datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S'),
str(gList))
gevent.joinall(gList)
return GreenletRequests.NodeResponsesPut
def resolve_one(record_type, domain, server_type, server_ip, server_port, timeout, strategy, wrong_answer, queue=None):
answers = []
try:
LOGGER.info('%s resolve %s at %s:%s' % (server_type, domain, server_ip, server_port))
if 'udp' == server_type:
wrong_answers = set(wrong_answer) if wrong_answer else set()
wrong_answers |= BUILTIN_WRONG_ANSWERS()
answers = resolve_over_udp(
record_type, domain, server_ip, server_port, timeout, strategy, wrong_answers)
elif 'tcp' == server_type:
answers = resolve_over_tcp(record_type, domain, server_ip, server_port, timeout)
else:
LOGGER.error('unsupported server type: %s' % server_type)
except:
LOGGER.exception('failed to resolve one: %s' % domain)
if answers and queue:
queue.put((domain, answers))
LOGGER.info('%s resolved %s at %s:%s => %s' % (server_type, domain, server_ip, server_port, json.dumps(answers)))
return answers
def _job_spawner(func, seq, queue, pool, link_value_func, link_exception_func):
"""Generator that populates queue with output from func(args)
for args in seq. Calls link_value_func() when successful
and link_exception_func() when error raised
"""
for args in seq:
job = pool.spawn(lambda args: queue.put((args, func(args))), args)
job.link_value(link_value_func)
job.link_exception(link_exception_func)
yield job
def crawler():
'''A very simple queued gevent web crawler'''
global crawled
while 1:
try:
u = queue.get(timeout=1)
response = requests.get(u)
print(response.status_code)
# Extract some links to follow
for link in re.findall('<a href="(http.*?)"', response.content):
# Limit to 10 pages (ignores links when the pool is already full)
if crawled < 10:
crawled += 1
queue.put(link)
except gevent.queue.Empty:
break
def ip_delay(index, ip):
start = time.time()
if ip is None:
proxies = None
else:
proxies = {
'http': 'http://' + ip,
}
try:
r = requests.get('http://store.nike.com/cn/zh_cn/', proxies=proxies, timeout=(3, 1))
except requests.exceptions.ConnectionError:
return
except requests.exceptions.ReadTimeout:
return
end = time.time()
delay = '{:.0f}ms'.format((end-start)*1000)
queue.put([index, delay])
def _channel_dispatcher(self):
while True:
event = self._events.recv()
channel_id = event.header.get("response_to", None)
queue = None
if channel_id is not None:
channel = self._active_channels.get(channel_id, None)
if channel is not None:
queue = channel._queue
elif self._broadcast_queue is not None:
queue = self._broadcast_queue
if queue is None:
print >>sys.stderr, "zerorpc.ChannelMultiplexer, ", "unable to route event:", event.__str__(
ignore_args=True
)
else:
queue.put(event)
def runloop(self):
i = self.i
queue = self.queue
multiplier = self.options.multiplier
rate_limit = self.options.rate_limit
location_grep = self.options.location_grep
incoming_requests_counter = Counter('input')
record_file = None
if self.options.record_file:
record_file = open(self.options.record_file, 'w')
if statsd_client:
def on_tick():
statsd_client.incr('backlog', queue.qsize())
incoming_requests_counter.on_tick = on_tick
drop_counter = Counter('dropped')
multiplied_output_counter = Counter()
logger.info('Listener %d started', i)
len_limit = self.options.backlog - self.options.backlog_breathing_space
while self.running:
q = self.next_query()
if not q:
continue
if location_grep and not self.filter_by_location(q, location_grep):
continue
if record_file:
print >> record_file, q
incoming_requests_counter.count()
logger.debug('Listener %d got %s', i, q)
if queue:
for _ in xrange(multiplier):
multiplied_output_counter.count()
if rate_limit > 0 and multiplied_output_counter.v >= rate_limit:
continue
if queue.qsize() > len_limit:
drop_counter.count()
else:
queue.put(q)
def crawler():
while 1:
try:
url = queue.get(timeout=0)
print('Fetching ... %s' % url)
response = requests.get(url)
if response.status_code == requests.codes.ok:
# Extract some links to follow
for link in re.findall(URLREGEX, response.text):
if link not in tranvered:
tranvered[link] = True
queue.put(getUrl(link))
else:
print('\x1b[0;30;41mFAILED\x1b[0m with %d ... %s' %
(response.status_code, url))
except gevent.queue.Empty:
print('queue empty')
break
def _channel_dispatcher(self):
while True:
event = self._events.recv()
channel_id = event.header.get('response_to', None)
queue = None
if channel_id is not None:
channel = self._active_channels.get(channel_id, None)
if channel is not None:
queue = channel._queue
elif self._broadcast_queue is not None:
queue = self._broadcast_queue
if queue is None:
print >> sys.stderr, \
'zerorpc.ChannelMultiplexer, ', \
'unable to route event:', \
event.__str__(ignore_args=True)
else:
queue.put(event)
def parallelPost(self, payload):
queue = gevent.queue.Queue()
gList = []
if type(self.resourceList) is list:
for i in self.resourceList:
queue.put(i)
for t in range(len(self.resourceList)):
gl = gevent.spawn(postRequest, queue, payload)
gList.append(gl)
# print str(gList)
app.logger.info(
'[%s] : [INFO] gList %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), str(gList))
gevent.joinall(gList)
return GreenletRequests.NodeResponsesPost
if type(self.resourceList) is dict:
for k, v in self.resourceList.iteritems():
# print k
app.logger.info(
'[%s] : [INFO] Key %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), str(k))
queue.put(k)
for k, v in self.resourceList.iteritems():
gl = gevent.spawn(postRequest, queue, v)
gList.append(gl)
# print str(gList)
app.logger.info(
'[%s] : [INFO] gList %s',
datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), str(gList))
gevent.joinall(gList)
return GreenletRequests.NodeResponsesPost
