def extractor(filename):
interpreter = POPEN(binary + [filename] + append,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
timeout = Timeout(getattr(settings, 'EXTRACTION_TIMEOUT', 120),
ChildTimeout)
timeout.start()
try:
output, run_error = interpreter.communicate('')
timeout.cancel()
except ChildTimeout:
print 'killing %s' % filename
interpreter.kill()
raise
if (output_type == 'text' and not output.strip()) or (
output_type == 'html' and html_is_empty(output)) or (
error and (error in output or error in run_error)):
raise ExtractionFailed()
elif output_type == 'html':
# strip non-breaking spaces
return _nbsp.sub(' ', output)
else:
return output
def _methodExecute(self, method_, id_="", timeout_=0, **args): # added the _ to make sure we have no conflicts with the code we execute
timeoutObj = Timeout(timeout_)
timeoutObj.start()
if "lock_" in args:
lock = args["lock_"]
if lock != None:
while not lock.checkCanExecute(id_):
# print "sleep for lock:%s for methodid:%s"% (lock,id_)
gevent.sleep(0.05)
args.pop("lock_")
try:
result = method_(**args)
except Exception as e:
timeoutObj.cancel()
self.methodError(id_, e)
return None
except Timeout as t:
if t is not timeoutObj:
raise RuntimeError("not my timeout")
self.methodTimeout(id_)
return None
timeoutObj.cancel()
if id_ in self.locksActive:
self.locksActive.pop(id_)
if lock != None:
if id_ in lock.greenletsActive:
lock.greenletsActive.pop(id_) # unlock the lock for this greenlet
else:
print "Could not find lock for id %s" % id_
return result
def send_api_request(self, init_params):
"""Send request to api.
:param dict init_params: Request params
:return: http request response
:rtype: str
:raises ApiError: raise :class:`ApiError` if there are some error during connection.
"""
# gevent based api request
if self._gevent_async:
timeout = Timeout(self._timeout)
timeout.start()
try:
#self.logger.debug('START')
res = self._send_api_request(init_params)
#self.logger.debug('STOP')
except Timeout:
err = 'Cloudstack api call timeout after : %ss' % self._timeout
self.logger.error(err)
raise ApiError(err)
except ApiError:
raise
finally:
timeout.cancel()
# blocking api request
else:
res = self._send_api_request(init_params, timeout=self._timeout)
return res
def create(self, usernames, passwords, target_iqn, target_lun, size,
initiator_iqn_list):
# NB: initiator_iqn_list needs to be a comma separated list of initiator iqn strings
self.logger.debug("Preparing to execute create()")
timeout = Timeout(self.script_timeout)
process = Popen(self.scriptfile_path + " -c -q" + " -u " + usernames +
" -p " + passwords + " -s " + size + " -m " +
target_lun + " -t " + target_iqn + " -i " +
initiator_iqn_list,
stdout=PIPE,
shell=True)
output = "Create operation exceeded execution timeout.\n"
returncode = 1
timeout.start()
try:
output = process.communicate()[0]
returncode = process.returncode
except Timeout:
process.kill()
self.logger.warn(
"Process %s servicing create() " +
"exceeded execution timeout and was terminated.", process.pid)
if process.returncode is not None:
returncode = process.returncode
finally:
timeout.cancel()
return [output, returncode]
def _get_message_data(self):
max_size = self.extensions.getparam('SIZE', filter=int)
reader = DataReader(self.io, max_size)
err = None
timeout = Timeout(self.data_timeout)
timeout.start()
try:
data = reader.recv()
except ConnectionLost:
raise
except SmtpError as e:
data = None
err = e
finally:
timeout.cancel()
reply = Reply('250', '2.6.0 Message Accepted for Delivery')
self._call_custom_handler('HAVE_DATA', reply, data, err)
self.io.send_reply(reply)
self.io.flush_send()
self.have_mailfrom = None
self.have_rcptto = None
def _read_output_buffer(self, _buffer, timeout=None):
timer = GTimeout(seconds=timeout, exception=Timeout)
remainder = b""
remainder_len = 0
timer.start()
try:
for data in _buffer:
pos = 0
size = len(data)
while pos < size:
linesep, new_line_pos = find_eol(data, pos)
if linesep == -1:
remainder += data[pos:]
remainder_len = len(remainder)
break
end_of_line = pos+linesep
if remainder_len > 0:
line = remainder + data[pos:end_of_line]
remainder = b""
remainder_len = 0
else:
line = data[pos:end_of_line]
yield line
pos += linesep + new_line_pos
if remainder_len > 0:
# Finished reading without finding ending linesep
yield remainder
finally:
timer.close()
def generate_iso2pose(folder_name, label, args):
outdir = os.path.join(args.output_file, folder_name)
if not os.path.exists(outdir):
os.mkdir(outdir)
vps = labels[label]
print('vps: ', vps)
converter = Model2SVG(width=args.width, height=args.height, tol=args.tol,
margin_left=args.margin_left, margin_top=args.margin_top,
line_width=args.line_width, line_width_hidden=args.line_width_hidden)
try:
# generate frt
seconds = 10 # the worker can run 60 seconds.
timeout = Timeout(seconds)
timeout.start()
fname = glob.glob(os.path.join(args.file, folder_name, '*.step'))[0]
shp = read_step_file(fname)
for vp in vps[:-1]:
fname_out = os.path.join(outdir, '{}_{}.svg'.format(folder_name, vp))
converter.export_shape_to_svg(shape=shp, filename=fname_out, proj_ax=converter.DIRS[vp])
# generate answer
fname_out = os.path.join(outdir, 'answer.svg')
print("vp-1:", vps[-1])
converter.export_shape_to_svg(shape=shp, filename=fname_out, proj_ax=converter.DIRS[vps[-1]])
return True
except Exception as re:
shutil.rmtree(outdir)
print('{} failed, due to: {}'.format(fname, re))
return False
def _read_output(self, read_func, timeout=None):
remainder = b""
remainder_len = 0
size, data = read_func()
t = GTimeout(seconds=timeout, exception=Timeout)
t.start()
try:
while size == LIBSSH2_ERROR_EAGAIN or size > 0:
while size == LIBSSH2_ERROR_EAGAIN:
self.poll(timeout=timeout)
size, data = read_func()
while size > 0:
pos = 0
while pos < size:
linesep, new_line_pos = find_eol(data, pos)
if linesep == -1:
remainder += data[pos:]
remainder_len = len(remainder)
break
end_of_line = pos + linesep
if remainder_len > 0:
line = remainder + data[pos:end_of_line]
remainder = b""
remainder_len = 0
else:
line = data[pos:end_of_line]
yield line
pos += linesep + new_line_pos
size, data = read_func()
if remainder_len > 0:
# Finished reading without finding ending linesep
yield remainder
finally:
t.close()
def initial_test(self, address):
try:
timeout = Timeout(self.TIMEOUT, TestTimeout('The server timed out on the first command.'))
timeout.start()
TestClient(address).put('key', 'value')
finally:
timeout.cancel()
def __init__(self, url):
self.url = url
self.protocol, self.domain = self.url.split("://") #e.g. news.bbc.co.uk
self.domain = self.domain.split('/')[0]
self.site_data = sites[self.domain]
self.total_words = {}
timeout = Timeout(30, TimeoutError)
timeout.start()
try:
self.html = self.read_url()
except TimeoutError:
print url + " timed out"
return
finally:
timeout.cancel()
self.text = self.boiler_extract()
self.soup = BeautifulSoup(self.html, 'lxml')
self.article = self.is_article()
if self.article:
self.calc_total_words()
articles.put(self)
self.find_links()
def _recv_command(self):
timeout = Timeout(self.command_timeout)
timeout.start()
try:
return self.io.recv_command()
finally:
timeout.cancel()
def main(socket, address):
global client_mgr
print "one client", address
logger.debug("one client %s" % str(address))
client = Client(socket)
hbTimer = None
while True:
try:
hbTimer = Timeout(ONE_MOVE_MAX_TIME)
hbTimer.start()
client.read_and_deal_cmd()
hbTimer.cancel()
except Timeout, t:
if t == hbTimer:
print "client lose"
client.lose_hb()
client.cancel_timeout()
if client.latitude != None:
client_mgr.remove_client(client)
client = None
break
else:
print "other timeout"
hbTimer.cancel()
client.deal_timeout()
except:
def sample():
timeout = Timeout(5)
timeout.start()
try:
gevent.spawn(wait).join()
except Timeout:
print 'Could not complete'
def __init__(self, url):
self.url = url
self.protocol, self.domain = self.url.split(
"://") #e.g. news.bbc.co.uk
self.domain = self.domain.split('/')[0]
self.site_data = sites[self.domain]
self.total_words = {}
timeout = Timeout(30, TimeoutError)
timeout.start()
try:
self.html = self.read_url()
except TimeoutError:
print url + " timed out"
return
finally:
timeout.cancel()
self.text = self.boiler_extract()
self.soup = BeautifulSoup(self.html, 'lxml')
self.article = self.is_article()
if self.article:
self.calc_total_words()
articles.put(self)
self.find_links()
def func1():
utc = arrow.utcnow()
local = utc.to('Asia/Shanghai')
ts = local.timestamp
print arrow.get(ts)
#print local.format('YYYY-MM-DD HH:mm:ss ZZ')
"""function and heartbeat"""
ex = TimeoutException("timeout ex")
#gevent timeout
timeout = Timeout(6, ex)
#start
timeout.start()
try:
# exception will be raised here, after *seconds*
# passed since start() call
gevent.sleep(3 * random.randint(1,4))
#print "f1 heart beat"
heartbeat("f1")
except TimeoutException as ex:
print ex
finally:
#cancel timeout
timeout.cancel()
def make_conn(url,timelimit):
global SUCCESS_RECORD,FAIL_RECORD,TOTAL_SIZE,REQTIME_ARR
time_start = time.time()
if timelimit:
timeout = Timeout(timelimit)
timeout.start()
try:
f = urlopen(url)
if f.getcode() == 200:
time_end = time.time()
server_info = f.info()
content_type = server_info['content-type'].split(";")[0]
if content_type == "text/html":
data = f.read()
size = int(server_info['content-length'])
else:
size = int(server_info['content-length'])
REQTIME_ARR.append((time_end - time_start) * 1000)
TOTAL_SIZE = TOTAL_SIZE + size
SUCCESS_RECORD += 1
else:
FAIL_RECORD += 1
except Timeout:
FAIL_RECORD += 1
return
except Exception,e:
FAIL_RECORD += 1
return
def execute(self, name, args, func_type='PY'):
module_path = settings['MODULE_PATH'] #"c:/mtp/mabotech/maboss1.1"
info = "[%s]%s:%s" % (module_path, func_type, name)
log.debug( info)
t = time.time()
if name == "time":
#for reconnection testing
return t
#Sync Code Here !!!
timeout = Timeout(5, Exception)
timeout.start()
try:
#... # exception will be raised here, after *seconds* passed since start() call
rtn = "OK"
rtn = py_executor.execute(name, args, module_path)
#gevent.sleep(0.02)
pf_log.debug("%10.5f,%s,%s" %(time.time()-t, func_type, name ) )
return rtn
except Exception, e:
log.error(e.message)
def read_url(url):
timeout = Timeout(10)
timeout.start()
try:
response = urllib2.urlopen(url)
reason, other = response.getcode(), response.msg
except Timeout, t:
reason, other = 'gevent timeout', 0
def test1():
timeout = Timeout(10) # constrain executing greenlet
timeout.start()
try:
gevent.spawn(task).join()
except Timeout:
print('could not complete')
def run_cmd(args, timeout=None):
_init()
args = list(args)
for i, x in enumerate(args):
if isinstance(x, unicode):
args[i] = x.encode("utf-8")
sp = socket.socketpair()
pid = os.fork()
if pid == 0:
# client
try:
os.dup2(sp[1].fileno(), 1)
os.dup2(sp[1].fileno(), 2)
sp[0].close()
sp[1].close()
os.execvp(args[0], args)
except:
stderr = os.fdopen(2, "w", 0)
os.write(2, "failed to exec child process: %r\nPATH=%r" % (args, os.environ.get('PATH')))
traceback.print_exc(file=stderr)
finally:
os._exit(97)
pid2status[pid] = event.AsyncResult()
if not _nochild:
def cb():
pid2status[pid].set(child_watcher.rstatus)
child_watcher.stop()
child_watcher = get_hub().loop.child(pid)
child_watcher.start(cb)
sp[1].close()
chunks = []
# prevent loopexit. see test_run_cmd_trigger_loopexit in test_proc.py
if timeout is None:
timeout = 2 ** 30
timeout = Timeout(timeout)
timeout.start()
try:
while 1:
chunk = sp[0].recv(4096)
if not chunk:
break
chunks.append(chunk)
st = pid2status[pid].get()
del pid2status[pid]
return st, "".join(chunks)
except Timeout, t:
if t is not timeout:
raise
def timeout_wrapper(*args, **kwargs):
t = Timeout(seconds,
TestTimeout('Timed out after %d seconds' % seconds))
t.start()
try:
ret = func(*args, **kwargs)
finally:
t.cancel()
return ret
def reposts_crawler():
'''
greenlet reposts crawler
'''
while not reposts_fetch_queue.empty():
IS_NEED_REFETCH = False #when timeout or errors occur,put the url back into the task queue and the make sure the task is not set to done!
try:
wait_time = Timeout(MAX_WAIT_TIME)
wait_time.start()
url = reposts_fetch_queue.get()
gevent.sleep(0.0)
reposts_time = _http_call(url)
for status in reposts_time['reposts']:
if not status.get('deleted'):
weibo_created_at = datetime.strptime(status.get('created_at'), '%a %b %d %H:%M:%S +0800 %Y')
user_created_at = datetime.strptime(status.get('user').get('created_at'),
'%a %b %d %H:%M:%S +0800 %Y')
reposts_status_id = -1
if status.get('retweeted_status') is not None:
reposts_status = status['retweeted_status']
reposts_status_id = reposts_status['id']
weibo_params = (
status['id'], status['user']['id'], status['text'], status['source'], weibo_created_at,
reposts_status_id)
user_params = (
status['user']['id'], status['user']['screen_name'], status['user']['name'],
status['user']['province'],
status['user']['city'], status['user']['location'], status['user']['description'],
status['user']['profile_image_url'], status['user']['domain'], status['user']['gender'],
status['user']['followers_count'], status['user']['friends_count'],
status['user']['statuses_count']
,
status['user']['favourites_count'], user_created_at, status['user']['verified'],
status['user']['verified_type'], status['user']['verified_reason'],
status['user']['bi_followers_count'] )
cursor.execute(REPOSTS_WEIBO_INSERT_SQL, weibo_params)
cursor.execute(REPOSTS_USER_INSERT_SQL, user_params)
except Timeout as t:
if t is wait_time:
# print '处理超时，等待重新抓取!'
#put timeout url back into the task queue
IS_NEED_REFETCH = True
except Exception as e:
IS_NEED_REFETCH = True
logger.error(traceback.format_exc())
finally:
wait_time.cancel()
if IS_NEED_REFETCH is not True:
reposts_fetch_queue.task_done()
# print url + ' 抓取完成 --- 转发'
else:
reposts_fetch_queue.put(url)
print status
print url + ' 抓取失败 --- 转发'
def serve_for_test(self):
timeout = Timeout(10)
timeout.start()
try:
while self.is_connected():
if len(self.re_schedule_events) == 10 and len(self.heartbeat_events) == 10:
break
gevent.sleep(0.01)
finally:
timeout.cancel()
def worker(sleep_time):
try:
seconds = 5 # max time the worker may run
timeout = Timeout(seconds)
timeout.start()
time.sleep(sleep_time)
print("%s is a early bird" % sleep_time)
except Timeout:
print("%s is late(time out)" % sleep_time)
def timeout_wrapper(*args, **kwargs):
t = Timeout(seconds,
TestTimeout('Timed out after %d seconds' % seconds)
)
t.start()
try:
ret = func(*args, **kwargs)
finally:
t.cancel()
return ret
def serve_for_test(self):
timeout = Timeout(10)
timeout.start()
try:
while self.is_connected():
if len(self.re_schedule_events) == 10 and len(
self.heartbeat_events) == 10:
break
gevent.sleep(0.01)
finally:
timeout.cancel()
def requestGet(self, url):
wait = random.random() * (wait_time[1] - wait_time[0])
sleep(wait)
timeout = Timeout(request_timeout)
timeout.start()
try:
req = requests.get(url=url, verify=True, headers=headers, proxies=proxies)
except IncompleteRead:
pass
# todo:未知错误，暂还未查清
timeout.cancel()
return req
class FDSafeHandler(WSGIHandler):
'''Our WSGI handler. Doesn't do much non-default except timeouts'''
def handle(self):
self.timeout = Timeout(120, Exception)
self.timeout.start()
try:
WSGIHandler.handle(self)
except Timeout as ex:
if ex is self.timeout:
pass
else:
raise
def foo1():
timeout = Timeout(seconds)
timeout.start()
def wait():
gevent.sleep(10)
try:
gevent.spawn(wait).join()
except Timeout:
print('Could not complete')
else:
print('Complete!')
def query_documents_with_timeout(*args, **kwargs):
timeout = Timeout(30)
timeout.start()
try:
gevent.sleep(0.0001)
return query_documents(*args, **kwargs)
except:
return [[], 0]
finally:
timeout.cancel()
def query_documents_with_timeout(*args, **kwargs):
timeout = Timeout(30)
timeout.start()
try:
gevent.sleep(0.0001)
return query_documents(*args, **kwargs)
except:
return [[], 0]
finally:
timeout.cancel()
def foo1():
timeout = Timeout(seconds)
timeout.start()
def wait():
gevent.sleep(10)
try:
gevent.spawn(wait).join()
except Timeout:
print('Could not complete')
else:
print('Complete!')
def Generate_task(fname, Path_output=pathwrite, args=args):
converter = Model2SVG(width=args.width,
height=args.height,
tol=args.tol,
margin_left=args.margin_left,
margin_top=args.margin_top,
line_width=args.line_width,
line_width_hidden=args.line_width_hidden)
index_list = [
'1', '9', '2', '11', '3', '13', '4', '15', '5', '10', '6', '12', '7',
'14', '8', '16', '17', '18', '19', '20'
]
path_list = fname.split(os.sep)
model_number = path_list[-1].replace(".step", "")
model_number = model_number[0:8]
MotherDic = Path_output + "/" + model_number + "/"
if not os.path.exists(MotherDic):
os.makedirs(MotherDic)
viewpoints = ['f', 'r', 't']
try:
seconds = 60
timeout = Timeout(seconds)
timeout.start()
shp = read_step_file(fname)
boundbox = get_boundingbox(shp, use_mesh=False)
max_3d_eadge = max(boundbox[6], boundbox[7], boundbox[8])
# sc=min(args.width, args.height)/max_3d_eadge
#### generate F R T views
for vp in viewpoints:
converter.export_shape_to_svg(shape=shp,
filename=MotherDic + model_number +
"_" + vp + ".svg",
proj_ax=converter.DIRS[vp],
max_eadge=max_3d_eadge)
for Vp in index_list:
converter.export_shape_to_svg(shape=shp,
filename=MotherDic + Vp + ".svg",
proj_ax=converter.DIRS[Vp],
max_eadge=max_3d_eadge)
return 1
except Exception as re:
shutil.rmtree(MotherDic)
print(fname + ' failed, due to: {}'.format(re))
return 0
def get_commits():
print('Start - {0}'.format(datetime.datetime.now()))
timeout = Timeout(10)
timeout.start()
try:
job_stack = [gevent.spawn(download(url)) for url in urls]
gevent.joinall(job_stack)
except Timeout:
pass
finally:
timeout.cancel()
cntx = OrderedDict(sorted(result.items()))
return render_template('start.html', cntx=cntx)
print('End - {0}'.format(datetime.datetime.now()))
def _get_message_data(self):
max_size = self.extensions.getparam('SIZE', filter=int)
reader = DataReader(self.io, max_size)
err = None
timeout = Timeout(self.data_timeout)
timeout.start()
try:
data = reader.recv()
except ConnectionLost:
raise
except SmtpError, e:
data = None
err = e
def generate_iso2pose(folder_name, label, args):
outdir = os.path.join(args.output_file, folder_name)
if not os.path.exists(outdir):
os.mkdir(outdir)
vps = labels[label]
print('vps: ', vps)
converter = Model2SVG(width=args.width,
height=args.height,
tol=args.tol,
margin_left=args.margin_left,
margin_top=args.margin_top,
line_width=args.line_width,
line_width_hidden=args.line_width_hidden)
try:
# generate frt
seconds = 10 # the worker can run 60 seconds.
timeout = Timeout(seconds)
timeout.start()
fname = os.path.join(args.file, folder_name + '.step')
shp = read_step_file(fname)
boundbox = get_boundingbox(
shp
) #3D- return xmax, xmin, ymax, ymin, zmax, zmin, abs(xmax-xmin), abs(ymax-ymin), abs(zmax-zmin)
max_3d_eadge = max(boundbox[6], boundbox[7], boundbox[8])
for vp in vps[:-1]:
fname_out = os.path.join(outdir,
'{}_{}.svg'.format(folder_name, vp))
converter.export_shape_to_svg(shape=shp,
filename=fname_out,
proj_ax=converter.DIRS[vp],
scale=sc,
max_eadge=max_3d_eadge)
# generate answer
fname_out = os.path.join(outdir, 'answer.svg')
print("vp-1:", vps[-1])
converter.export_shape_to_svg(shape=shp,
filename=fname_out,
proj_ax=converter.DIRS[vps[-1]],
scale=sc,
max_eadge=max_3d_eadge)
return True
except Exception as re:
shutil.rmtree(outdir)
print('{} failed, due to: {}'.format(fname, re))
return False
def start(self):
task = self.task_queue.get(block=False)
# [target, (vid1, [name1, class1])]
target = task[0]
poc_vid = task[1][0]
poc_name = task[1][1][0].split(".")[-1]
poc = task[1][1][1]()
poc.scan_info = {
'TaskId': self.task_id,
'Target': target,
'Verbose': self.verbose,
'Error': '',
'Mode': self.mode,
'Success': False,
'Ret': tree(),
"risk_category": poc.scan_info.get('risk_category', '')
}
poc.poc_info["poc"]["Class"] = task[1][1][1].__name__
timeout = Timeout(self.fb.poc_setting.timeout)
timeout.start()
try:
log.info("{} - {} start...".format(poc_vid, target))
poc.run(fb=self.fb)
log.info("{} - {} finish.".format(poc_vid, target))
except Timeout:
poc.scan_info['Error'] = "PoC run timeout."
poc.scan_info['Success'] = False
log.error("{} - {} error: PoC run timeout.".format(
poc_vid, target))
except (requests.exceptions.Timeout,
requests.exceptions.ConnectionError) as e:
poc.scan_info['Error'] = str(e)
poc.scan_info['Success'] = False
log.error("{} - {} error: {}.".format(poc_vid, target, e))
except Exception:
import traceback
err = traceback.format_exc()
poc.scan_info['Error'] = err
poc.scan_info['Success'] = False
log.error("{} - {} error: {}.".format(poc_vid, target, err))
finally:
timeout.cancel()
if not poc.scan_info.get("Success", False):
return
if self.fb.poc_setting.return_resp:
poc.scan_info["req_resp"] = self._get_http_data(poc_vid, target)
self.result.put_nowait([poc_name, poc.poc_info, poc.scan_info])
def _exec_pipe(the_pipe):
logging.info('----Begin to process batch----')
startTime= datetime.now()
timeout = Timeout(10, False)
error_code = S_OK
pipe_result = []
timeout.start()
try:
pipe_result = the_pipe.execute()
except Timeout, t:
if t is not timeout:
raise # not my timeout
logging.error('unable to execute the_pipe (possibly gevent.Timeout)')
error_code = S_ERR
def requestGet(self, url):
wait = random.random() * (wait_time[1] - wait_time[0])
sleep(wait)
timeout = Timeout(request_timeout)
timeout.start()
try:
req = requests.get(url=url,
verify=True,
headers=headers,
proxies=proxies)
except IncompleteRead:
pass
# todo:未知错误，暂还未查清
timeout.cancel()
return req
def wrapper(*args, **kwargs):
timeout = Timeout(650)
timeout.start()
try:
token = kwargs.get("token", None)
result = func(*args, **kwargs)
worker_db.Update_Token_DB(token, result, "success")
except Timeout:
LOG.exception('%s: Gevent task %s TIMEOUT!!' %
(func.__name__, token))
worker_db.Update_Token_DB(token, "token timed out!!", "failed")
except Exception, e:
LOG.exception('%s: api_call exception: %s' %
(func.__name__, str(e)))
worker_db.Update_Token_DB(token, str(e), "failed")
def generate():
result = None
while result is None:
try:
timeout = Timeout(25)
timeout.start()
result = json.dumps(client.get_events(
queue_id=queue_id,
last_event_id=last_event_id))
logging.debug('got a response')
except Timeout:
pass
finally:
timeout.cancel()
yield result or ' '
def thread_timeout():
seconds = 10
timeout = Timeout(seconds)
timeout.start()
def wait():
gevent.sleep(10)
try:
wait_gt = gevent.spawn(wait)
gevent.joinall([wait_gt])
print(wait_gt.exception)
except Timeout:
print("Could not complete")
def read(self, nbytes):
if self.timeout is None:
timeout = None
else:
timeout = Timeout(self.timeout)
timeout.start()
try:
buf = fd.read(self.fd, nbytes)
except Timeout as e:
if e is not timeout:
raise
raise TIMEOUT('Timeout reading from fd')
else:
if timeout is not None:
timeout.cancel()
return buf
def curl(ip):
url = 'http://' + ip
request = urllib2.Request(url=url)
reason, other = None, 0
timeout = Timeout(CONNECT_TIMEOUT + DATA_TIMEOUT)
timeout.start()
try:
rsp = urllib2.urlopen(request)
print rsp.read()
reason, other = rsp.getcode(), rsp.msg
except Timeout, t:
if t is timeout:
reason, other = 'gevent timeout', 0
else:
reason, other= 'gevent timeout 2', 0
def handle(self, body):
t = int((self.timestamp + self.expiration) - time.time())
worker = self.get_worker(self.routing_key)
log.debug("Running {0} with timeout {1} sec.".format(self.w_name, t))
timeout = Timeout(t, TimeoutError)
timeout.start()
try:
res = worker(body)
log.debug('Task finished.')
return res
except Exception as e:
log.debug(traceback.format_exc())
log.error('Task error: {0}'.format(unicode(e)))
return e
finally:
timeout.cancel()
def _read_result():
timeout = Timeout(self._read_timeout, Timeout)
timeout.start()
try:
result = self._read_result(cmd)
result_channel.put(result)
except Timeout:
raise
except:
self.log.exception("read error in defer_command")
result_channel.put((MemcacheResult.ERROR, error_value))
self.log.warn("Error communicating with Memcache %s, disconnecting", self._address)
self.disconnect()
finally:
timeout.cancel()
class TimeoutMixin(object):
"""超时
"""
def __init__(self, secs=None, exception=None, ref=True, priority=-1):
self.secs = secs
self.timeout = Timeout(secs)
def stop_timeout(self):
self.timeout.cancel()
def reset_timeout(self):
self.timeout.cancel()
self.timeout = Timeout(self.secs, False)
self.timeout.start()
def start_timeout(self):
self.timeout.start()
def _write_command():
timeout = Timeout(self._write_timeout, Timeout)
timeout.start()
try:
if not self.is_connected():
self.connect()
self._write_command(cmd, args, True)
self._read_queue.defer(_read_result)
except Timeout:
raise
except:
result_channel.put((MemcacheResult.ERROR, error_value))
self.log.warn("Error communicating with Memcache %s, disconnecting", self._address)
self.disconnect()
finally:
timeout.cancel()
def execute(self, call):
""" Calls a method for an RPC call (part of ``ConnectionHandler``'s
``call_handler`` interface).
"""
callable = self.get_call_callable(call)
timeout = None
if self.call_timeout is not None:
timeout = Timeout(getattr(callable, "_timeout", self.call_timeout))
call_semaphore = self._get_call_semaphore(call)
if call_semaphore.locked():
log.warning("too many concurrent callers (%r); call %r will block",
self.max_concurrent_calls, call)
call_semaphore.acquire()
def finished_callback(is_error):
self.active_calls.remove(call)
self.call_stats["completed"] += 1
if is_error:
self.call_stats["errors"] += 1
call_semaphore.release()
if timeout is not None:
timeout.cancel()
got_err = True
result_is_generator = False
try:
if timeout is not None:
timeout.start()
time_in_queue = time.time() - call.meta.get("time_received", 0)
call.meta["time_in_queue"] = time_in_queue
self.active_calls.append(call)
result = callable(*call.args, **call.kwargs)
if isiter(result):
result = self.wrap_generator_result(call, result,
finished_callback)
result_is_generator = True
got_err = False
finally:
if not result_is_generator:
finished_callback(is_error=got_err)
return result
def _send(self, message, receiveTimeout=_RECEIVE_TIMEOUT):
'''
Tries to send a message to the server and waits a fixed amount of
seconds for it to answer.
@param message: message to send to the server
@type message: str
@param receiveTimeout: max time in seconds in which the server should respond
@type receiveTimeout: int
'''
if not self._isConnected:
self._connect()
try:
self._socket.send(message)
except Exception as e:
self._disconnect()
printInDebugMode('Couldn\'t send message, unexpected exception'
' while sending to server: %s' % e)
result = None
timeout = Timeout(receiveTimeout)
timeout.start()
try:
result = self._socket.recv()
except Timeout as timeoutException:
if timeoutException == timeout:
printInDebugMode(
'Couldn\'t send message, server response timed'
' out after %d seconds' % receiveTimeout)
self._disconnect()
except Exception as e:
printInDebugMode('Couldn\'t send message, unexpected exception'
' while receiving response from server: %s' % e)
self._disconnect()
finally:
timeout.cancel()
return result == '1'
def _send(self, message, receiveTimeout=_RECEIVE_TIMEOUT):
'''
Tries to send a message to the server and waits a fixed amount of
seconds for it to answer.
@param message: message to send to the server
@type message: str
@param receiveTimeout: max time in seconds in which the server should respond
@type receiveTimeout: int
'''
if not self._isConnected:
self._connect()
try:
self._socket.send(message)
except Exception as e:
self._disconnect()
printInDebugMode('Couldn\'t send message, unexpected exception'
' while sending to server: %s' % e)
result = None
timeout = Timeout(receiveTimeout)
timeout.start()
try:
result = self._socket.recv()
except Timeout as timeoutException:
if timeoutException == timeout:
printInDebugMode('Couldn\'t send message, server response timed'
' out after %d seconds' % receiveTimeout)
self._disconnect()
except Exception as e:
printInDebugMode('Couldn\'t send message, unexpected exception'
' while receiving response from server: %s' % e)
self._disconnect()
finally:
timeout.cancel()
return result == '1'
def curl(ip):
'''
使用urllib2探测IP是否可以访问，并抽取应答码
和错误原因
'''
url = 'http://' + ip
request = urllib2.Request(url=url)
reason, other = None, 0
timeout = Timeout(CONNECT_TIMEOUT + DATA_TIMEOUT)
timeout.start()
try:
rsp = urllib2.urlopen(request)
reason, other = rsp.getcode(), rsp.msg
except Timeout, t:
if t is timeout:
reason, other = 'gevent timeout', 0
else:
reason, other = 'gevent timeout 2', 0
def probe_proxy_ip(proxy_ip):
"""代理检测"""
proxy = urllib2.ProxyHandler(proxy_ip)
opener = urllib2.build_opener(proxy)
urllib2.install_opener(opener)
timeout = Timeout(30)
timeout.start()
try:
html = urllib2.urlopen('http://1212.ip138.com/ic.asp')
if html:
return True
else:
return False
except Exception as e:
print 'URLopen error'
return False
except Timeout:
print 'Timeout'
return False
def get_git_refs():
if DISABLE_NEW_EXTENSIONS:
return 'Disabled', 403
git_endpoint = request.values.get('ep', None)
if git_endpoint is None:
return jsonify(error={'message': 'Missing endpoint'}), 400
if not git_endpoint.endswith('.git'):
return jsonify(error={'message': 'Invalid git endpoint'}), 400
git_path = config.get('MINEMELD_GIT_PATH', None)
if git_path is None:
return jsonify(error={'message': 'MINEMELD_GIT_PATH not set'}), 500
git_args = [git_path, 'ls-remote', '-t', '-h', git_endpoint]
git_process = Popen(args=git_args,
close_fds=True,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
timeout = Timeout(20.0)
timeout.start()
try:
git_stdout, git_stderr = git_process.communicate()
except Timeout:
git_process.kill()
return jsonify(error={'message': 'Timeout accessing git repo'}), 400
finally:
timeout.cancel()
if git_process.returncode != 0:
LOG.error('Error running {}: {}'.format(git_args, git_stderr))
return jsonify(
error={'message': 'Error running git: {}'.format(git_stderr)}), 400
return jsonify(
result=[line.rsplit('/', 1)[-1] for line in git_stdout.splitlines()])
def write(self, buf):
if self.timeout is None:
timeout = None
else:
timeout = Timeout(self.timeout)
timeout.start()
buf = compat.buffer(buf)
byteswritten = 0
try:
while byteswritten != len(buf):
nbytes = fd.write(self.fd, buf[byteswritten:])
assert nbytes != 0
byteswritten += nbytes
except Timeout as e:
if e is not timeout:
raise
raise TIMEOUT('Timeout writing to fd')
else:
if timeout is not None:
timeout.cancel()
return len(buf)
def delete(self, name):
self.logger.debug("Preparing to execute delete()")
timeout = Timeout(self.script_timeout)
process = Popen(self.scriptfile_path + " -d -q" + " -n " + name,
stdout=PIPE,
shell=True)
output = "Delete operation exceeded execution timeout.\n"
returncode = 1
timeout.start()
try:
output = process.communicate()[0]
returncode = process.returncode
except Timeout:
process.kill()
self.logger.warn(
"Process %s servicing delete() " +
"exceeded execution timeout and was terminated.", process.pid)
if process.returncode is not None:
returncode = process.returncode
finally:
timeout.cancel()
return [output, returncode]
def run_thread(classifier_name, classifier_params, fold, n_genes, run_delay,
timeout_sec):
try:
try:
time.sleep((np.random.random_sample()) + run_delay * 10)
timeout = Timeout(timeout_sec)
timeout.start()
print('Starting {} Classifier fold {} run: {}'.format(
classifier_name, fold, run_delay))
out = subprocess.call([
python_interpeter, './{}'.format(experiment_file),
str(classifier_name),
str(classifier_params),
str(n_genes)
])
print('Done with classifier {} fold {} run: {}'.format(
classifier_name, fold, run_delay))
if out == 0:
return 'Success_{}_{}_{}'.format(classifier_name, fold,
run_delay)
elif out == 1:
return 'Error_{}_{}_{}'.format(classifier_name, fold,
run_delay)
return out
except TimeoutException as te:
print('Set timeout of {} has been reached in '.format(te))
print('Classifier {}, fold {}, run {}'.format(
classifier_name, fold, run_delay))
print('Probably caused by a locked thread')
return 'Timeout_{}_{}_{}'.format(classifier_name, fold, run_delay)
except Exception as e:
print('Exception {} occured during threading of:'.format(e))
print('Classifier {}, fold {}, run {}'.format(classifier_name, fold,
run_delay))
return 'Catched Exception: {}, Classifier: {}, fold: {}, run: {}'.format(
e, classifier_name, fold, run_delay)
