def startTest(self, test):
"""
Start the test
"""
if ((self.cumulative_tc_failure_threshold != 0) and
(self.__tf_count >= self.cumulative_tc_failure_threshold)):
_msg = 'Total testcases failure count exceeded cumulative'
_msg += ' testcase failure threshold '
_msg += '(%d)' % self.cumulative_tc_failure_threshold
self.logger.error(_msg)
raise KeyboardInterrupt
if ((self.tc_failure_threshold != 0) and
(self.__failed_tc_count >= self.tc_failure_threshold)):
if self.__failed_tc_count_msg:
raise TCThresholdReached
_msg = 'Testcases failure for this testsuite count exceeded'
_msg += ' testcase failure threshold '
_msg += '(%d)' % self.tc_failure_threshold
self.logger.error(_msg)
self.__failed_tc_count_msg = True
raise TCThresholdReached
timeout = self.__get_timeout(test)
rv = self.__are_requirements_matching(self.param_dict, test)
if rv is False:
# Below method call is needed in order to get the test case
# details in the output and to have the skipped test count
# included in total run count of the test run
self.result.startTest(test)
raise SkipTest('Test requirements are not matching')
def timeout_handler(signum, frame):
raise TimeOut('Timed out after %s second' % timeout)
old_handler = signal.signal(signal.SIGALRM, timeout_handler)
setattr(test, 'old_sigalrm_handler', old_handler)
signal.alarm(timeout)
def rpc(self, method, *args, **kwargs):
"""
Send a command to the queue. Timeout after 10 seconds
"""
pubsub = self._redis.pubsub()
channel = "%s%s" % (platform.node(), int(time.time()))
pubsub.subscribe(channel)
self._redis.rpush(RedisTransport.COMMAND_KEY,
cPickle.dumps(RedisMessage(channel, method, args, kwargs)))
resp = pubsub.listen()
signal.signal(signal.SIGALRM, self.shutdown)
signal.alarm(10)
resp.next() # clear subscribe message
response = resp.next()
pubsub.unsubscribe()
try:
return cPickle.loads(response['data'])
except: # pylint: disable=W0702
msg = "%s: Failed to receive response: %s" % \
(self.__class__.__name__,
traceback.format_exc().splitlines()[-1])
self.logger.error(msg)
return None
def terminate(self):
"""
Terminate all the child processes of this ProcessManager, bringing the
loop() to an end.
"""
if self._terminating:
return False
self._terminating = True
print("sending SIGTERM to all processes", file=self.system_printer)
for proc in self.processes:
if proc.poll() is None:
print("sending SIGTERM to pid {0:d}".format(proc.pid), file=self.system_printer)
proc.terminate()
def kill(signum, frame):
# If anything is still alive, SIGKILL it
for proc in self.processes:
if proc.poll() is None:
print("sending SIGKILL to pid {0:d}".format(proc.pid), file=self.system_printer)
proc.kill()
if ON_WINDOWS:
# SIGALRM is not supported on Windows: just kill instead
kill(None, None)
else:
# the default is POSIX
signal.signal(signal.SIGALRM, kill) # @UndefinedVariable
signal.alarm(5) # @UndefinedVariable
def run(args, input=None, cwd = None, shell = False, kill_tree = True, timeout = -1, env = None):
'''
Run a command with a timeout after which it will be forcibly
killed.
'''
class Alarm(Exception):
pass
def alarm_handler(signum, frame):
raise Alarm
p = Popen(args, shell = shell, cwd = cwd, stdout = PIPE, stderr = PIPE, env = env)
if timeout != -1:
if alarm_available:
signal(SIGALRM, alarm_handler)
alarm(timeout)
try:
t0 = time.time()
stdout, stderr = p.communicate()
print("Ran for", time.time()-t0, "seconds.")
if timeout != -1:
alarm(0)
except Alarm:
pids = [p.pid]
if kill_tree:
pids.extend(get_process_children(p.pid))
for pid in pids:
# process might have died before getting to this line
# so wrap to avoid OSError: no such process
try:
kill(pid, SIGKILL)
except OSError:
pass
return -9, '', ''
return p.returncode, stdout, stderr
def kill_workers(self, timeout=5):
"""
Send a suicide message to all workers, with some kind of timeout.
"""
logging.info('Killing workers, taking up to %d seconds.', int(timeout))
poller = zmq.Poller()
poller.register(self.results_pull, zmq.POLLIN)
while True:
# Seems to get stuck gevent-blocking in the work_push.send() after
# all the workers have died. Also, gevent.Timeout() doesn't seem
# to work here?!
signal.alarm(int(timeout))
self.work_push.send(msgpack.dumps([{'type': 'PING'}]))
socks = dict(poller.poll(timeout * 1500))
if self.results_pull in socks \
and socks[self.results_pull] == zmq.POLLIN:
result_packed = self.results_pull.recv()
result = msgpack.loads(result_packed)
logging.info('Heard from worker id=%d; sending SUICIDE',
result['worker_id'])
self.work_push.send(msgpack.dumps([{'type': 'SUICIDE'}]))
gevent.sleep(0.1)
else:
break
signal.alarm(0)
def run(self):
"""Runs the handler, flushes the streams, and ends the request."""
# If there is a timeout
if self._timeout:
old_alarm = signal.signal(signal.SIGALRM, self.timeout_handler)
signal.alarm(self._timeout)
try:
protocolStatus, appStatus = self.server.handler(self)
except:
traceback.print_exc(file=self.stderr)
self.stderr.flush()
if not self.stdout.dataWritten:
self.server.error(self)
protocolStatus, appStatus = FCGI_REQUEST_COMPLETE, 0
if __debug__: _debug(1, 'protocolStatus = %d, appStatus = %d' %
(protocolStatus, appStatus))
# Restore old handler if timeout was given
if self._timeout:
signal.alarm(0)
signal.signal(signal.SIGALRM, old_alarm)
try:
self._flush()
self._end(appStatus, protocolStatus)
except socket.error, e:
if e[0] != errno.EPIPE:
raise
def local(cmd, retry_ttl=0, retry_interval=3, capture=True, timeout=60, **kwargs):
def retry(*args, **kwargs):
log.info('failed cmd: {0}, ttl: {1}, sleep: {2}'.format(
cmd, retry_ttl, retry_interval))
if retry_ttl > 0:
time.sleep(retry_interval)
local(cmd, retry_ttl - 1, retry_interval, capture)
log_cmd = 'local> ' + cmd
api.env.cmd_history.append(log_cmd)
log.info(log_cmd)
print_for_test(log_cmd)
if api.env.is_test:
result = test_cmd(cmd)
else:
signal.signal(signal.SIGALRM, retry)
signal.alarm(timeout)
try:
result = api.local(cmd, capture=capture, **kwargs)
finally:
signal.alarm(0)
result_msg = 'return> {0}'.format(result.return_code)
log.info(result_msg)
print_for_test(result_msg)
return result
def main():
mbean, attr = getMbeans(args.mbeantype, args.keyspace, args.columnfamily)
jmx = JmxQuery(args.host, args.port, args.username, args.password, context=args.context)
state = jmx.isNodeActive()
if state == 1:
signal.alarm(args.timeout)
jmx_data = jmx.getData(mbean, attr)
signal.alarm(0)
for mbean_result in jmx_data:
if mbean_result['status'] != 200:
pprint.pprint(mbean_result)
sys.exit(1)
for mbean, mbean_value in mbean_result['value'].items():
for attribute, attribute_value in mbean_value.items():
if type(attribute_value).__name__ == 'dict':
for path, path_value in attribute_value.items():
print "%s_%s:%s" % (attribute, path, path_value),
elif args.mbeantype == 'Stages':
prefix = mbean.split('=',1)[1]
print "%s_%s:%s" % (prefix, attribute, attribute_value),
else:
print "%s:%s" % (attribute, attribute_value),
else:
print state
def timeLimit(seconds):
"""
http://stackoverflow.com/a/601168
Use to limit the execution time of a function. Raises an exception if the execution of the
function takes more than the specified amount of time.
:param seconds: maximum allowable time, in seconds
>>> import time
>>> with timeLimit(5):
... time.sleep(4)
>>> import time
>>> with timeLimit(5):
... time.sleep(6)
Traceback (most recent call last):
...
RuntimeError: Timed out
"""
def signal_handler(signum, frame):
raise RuntimeError('Timed out')
signal.signal(signal.SIGALRM, signal_handler)
signal.alarm(seconds)
try:
yield
finally:
signal.alarm(0)
def _sigint(sig, frame):
print
# Sometimes a request hangs and prevents the interpreter from exiting.
# By setting an 1-second alarm we avoid this
if hasattr(signal, 'alarm'):
signal.alarm(1)
sys.exit(0)
def run(self):
class Alarm(Exception):
pass
def alarm_handler(signum, frame):
raise Alarm
try:
self.process = Popen(self.args, shell=True, stdout=PIPE, stderr=PIPE)
if self.timeout != -1:
signal(SIGALRM, alarm_handler)
alarm(self.timeout)
try:
self.stdout, self.stderr = self.process.communicate()
if self.timeout != -1:
alarm(0)
except Alarm:
os.kill(self.process.pid, SIGKILL)
raise CloudRuntimeException("Timeout during command execution")
self.success = self.process.returncode == 0
except:
raise CloudRuntimeException(formatExceptionInfo())
if not self.success:
logging.debug("Failed to execute:" + self.getErrMsg())
def get_packet(self, num_bytes):
# set an alarm to timeout in 5 seconds and signal a PacketTimeout if we have not gotten a full packet
signal.alarm(TIMEOUT_SECS)
self.packet = []
try:
while True:
self.packet.append(self.ser.read(1))
if DEBUG:
print("self packet = " + str(self.packet[0]))
print("current command = " + str(self.current_command))
# if the first value of the packet is not the command byte, pop values off until it is
if ord(self.packet[0]) != self.current_command and self.packet:
self.packet.pop(0)
# check if we have enough bytes for a complete packet
if len(self.packet) >= num_bytes:
# try to get a complete packet
int_list = self.parse_packet(num_bytes)
if int_list:
self.good_packets += 1
signal.alarm(0)
return int_list
else:
while self.packet and self.packet[0] != self.current_command:
# remove values at the head of the packet until you find another potential header
self.packet.pop(0)
except Timeout:
self.packet = []
return False
def testInterruptedTimeout(self):
# XXX I don't know how to do this test on MSWindows or any other
# plaform that doesn't support signal.alarm() or os.kill(), though
# the bug should have existed on all platforms.
if not hasattr(signal, "alarm"):
return # can only test on *nix
self.serv.settimeout(5.0) # must be longer than alarm
class Alarm(Exception):
pass
def alarm_handler(signal, frame):
raise Alarm
old_alarm = signal.signal(signal.SIGALRM, alarm_handler)
try:
signal.alarm(2) # POSIX allows alarm to be up to 1 second early
try:
foo = self.serv.accept()
except socket.timeout:
self.fail("caught timeout instead of Alarm")
except Alarm:
pass
except:
self.fail("caught other exception instead of Alarm")
else:
self.fail("nothing caught")
signal.alarm(0) # shut off alarm
except Alarm:
self.fail("got Alarm in wrong place")
finally:
# no alarm can be pending. Safe to restore old handler.
signal.signal(signal.SIGALRM, old_alarm)
def timeout(timeout_secs, func, *args, **kwargs):
"""Enforce a maximum time for a callable to complete.
Not yet implemented on Windows.
"""
default_return = kwargs.pop('default_return', None)
if on_win:
# Why does Windows have to be so difficult all the time? Kind of gets old.
# Guess we'll bypass Windows timeouts for now.
try:
return func(*args, **kwargs)
except KeyboardInterrupt: # pragma: no cover
return default_return
else:
class TimeoutException(Exception):
pass
def interrupt(signum, frame):
raise TimeoutException()
signal.signal(signal.SIGALRM, interrupt)
signal.alarm(timeout_secs)
try:
ret = func(*args, **kwargs)
signal.alarm(0)
return ret
except (TimeoutException, KeyboardInterrupt): # pragma: no cover
return default_return
def set_continuous_command(self, command):
# The command sent to the IMU will be result in a continuous data stream
signal.alarm(TIMEOUT_SECS)
try:
self.packet = []
self.send_byte(0x10)
self.send_byte(0x00)
self.send_byte(command)
self.current_command = command
# setting a continuous command will send back a 7 byte response
self.packet = list(self.ser.read(7))
# read back the packet and make sure it is correct by parsing else return false
if not self.parse_packet(7):
if DEBUG:
print("failed setting command")
return False
else:
if DEBUG:
print("successly set command!:")
except IOError:
print("cannot write and set to continuous mode")
return False
except Timeout:
return False
signal.alarm(0)
return True
def ttyUSB_transfer(ttyUSB, write_data = None, timeout = -1, terminate_word = None, no_print = None):
log = []
class TimeoutAlarm(Exception):
pass
def timeout_handler(signum, frame):
raise TimeoutAlarm
if timeout > 0:
signal(SIGALRM, timeout_handler)
alarm(timeout)
try:
tty = open(ttyUSB,"r+")
if write_data != None:
tty.write(write_data)
if timeout > 0:
while 1:
line = tty.readline()
if no_print == None:
print line,
log.append(line)
if terminate_word != None and line.find(terminate_word) != -1:
break
tty.close()
if timeout > 0:
alarm(0)
except TimeoutAlarm:
tty.close()
return ''.join(log)
def reap(self):
"""Collect the dead jobs."""
while self._running:
dead = set()
for job in self._running:
st = job.state(self._cache)
if st == _RUNNING: continue
if st == _FAILURE or st == _KILLED:
self._failures += 1
if self._stop_on_failure:
self._cancelled = True
for job in self._running:
job.kill()
dead.add(job)
break
for job in dead:
self._completed += 1
self._running.remove(job)
if dead: return
if (not self._travis):
message('WAITING', '%d jobs running, %d complete, %d failed' % (
len(self._running), self._completed, self._failures))
if platform.system() == 'Windows':
time.sleep(0.1)
else:
global have_alarm
if not have_alarm:
have_alarm = True
signal.alarm(10)
signal.pause()
def drive_unit(exper_body,
block_id, agent_id, treatment_id, timeout,
log_file, treatment_names, configuration):
# exper_body(agent_id, treatment_id)
def signal_handler(signum, frame):
print "Timeout!"
fo = open(log_file, "a")
fo.write(str(datetime.now())+"||error||block timeout||Error||"+str(treatment_id)+"||"+str(agent_id)+"\n")
fo.close()
print "Killing process", os.getpid()
raise TimeoutException("Timed out!")
old_handler = signal.signal(signal.SIGALRM, signal_handler)
signal.alarm(timeout)
try:
exper_body(agent_id, treatment_id, configuration)
except TimeoutException:
return
finally:
print "Instance", agent_id, "exiting!"
signal.signal(signal.SIGALRM, old_handler)
# os.kill(os.getpid(), 1)
signal.alarm(0)
def GetAttachedDevices(adb_cmd):
"""Gets the device list from adb.
We use an alarm in this function to avoid deadlocking from an external
dependency.
Args:
adb_cmd: binary to run adb
Returns:
list of devices or an empty list on timeout
"""
signal.alarm(2)
try:
out, err = subprocess.Popen([adb_cmd, 'devices'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if err:
logging.warning('adb device error %s', err.strip())
return re.findall('^(\w+)\tdevice$', out, re.MULTILINE)
except TimeoutException:
logging.warning('"adb devices" command timed out')
return []
except (IOError, OSError):
logging.exception('Exception from "adb devices"')
return []
finally:
signal.alarm(0)
def time_limit(seconds):
"""Limit function execution time.
>>> try:
with time_limit(10):
long_function_call()
except TimeoutException:
print "Timeout"
>>>
Args:
seconds: An integer indicating how long limit the function
execution time.
"""
def signal_handler(signum, frame):
raise TimeLimitException("timeout")
signal.signal(signal.SIGALRM, signal_handler)
signal.alarm(seconds)
try:
yield
finally:
signal.alarm(0)
def wrapper(*args, **kwargs):
signal.signal(signal.SIGALRM, _handler)
signal.alarm(seconds) # Set the alarm.
try:
return func(*args, **kwargs)
finally:
signal.alarm(0) # Turn the alarm off.
def timeout_call_old(cmd, timeout):
""" Invokes command specified by @cmd.
Raises 'Alarm' Exception if cmd does
not return withint @timeout (in secs)
Can only be used in main thread ;-(
"""
call = None
def timeout_handler(signum, frame):
if call:
os.kill(call.pid, signal.SIGKILL)
os.waitpid(-1, os.WNOHANG)
raise Alarm
signal.signal(signal.SIGALRM, timeout_handler)
call = subprocess.Popen(cmd,
shell = True, close_fds = True,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
signal.alarm(timeout)
res = call.communicate()
return_code = call.returncode
signal.alarm(0) # switch off alarm
return res, return_code
def fork_and_check(constr):
constr = simplify(constr)
parent_conn, child_conn = multiprocessing.Pipe()
p = multiprocessing.Process(target=fork_and_check_worker,
args=(constr, child_conn))
p.start()
child_conn.close()
## timeout after a while..
def sighandler(signo, stack):
print "Timed out.."
# print z3expr(constr, True).sexpr()
p.terminate()
signal.signal(signal.SIGALRM, sighandler)
signal.alarm(z3_timeout)
try:
res = parent_conn.recv()
except EOFError:
res = (z3.unknown, None)
finally:
signal.alarm(0)
p.join()
return res
def start(self):
if self.with_signaling:
signal.signal(signal.SIGALRM, self.murder_connections)
signal.alarm(self.timeout)
else:
self._reaper = ConnectionReaper(self, delay=self.timeout)
self._reaper.ensure_started()
def run(self, command, interpreter="/bin/bash", forward_ssh_agent=False):
"""
Execute ``command`` using ``interpreter``
Consider this roughly as::
echo "command" | ssh root@server "/bin/interpreter"
Raise SSHError if server is unreachable
Hint: Try interpreter='/usr/bin/python'
"""
ssh_command = self.ssh_command(interpreter, forward_ssh_agent)
pipe = subprocess.Popen(
ssh_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.get_env()
)
try:
signal.signal(signal.SIGALRM, _timeout_handler)
except ValueError: # signal only works in main thread
pass
signal.alarm(self.timeout)
out = err = ""
try:
out, err = pipe.communicate(command)
except IOError, exc:
# pipe.terminate() # only in python 2.6 allowed
os.kill(pipe.pid, signal.SIGTERM)
signal.alarm(0) # disable alarm
raise SSHError(str(exc))
def __init__(self, alarm_time = 1, silent = False):
self.running_thread = []
self.paused_thread = []
self.alarm_time = alarm_time
self.silent = silent
signal.signal(signal.SIGALRM, self)
signal.alarm(self.alarm_time)
def write_dot_graph(filename, dot_graph,
run_graphviz=True,
quiet=False,
timeout=DEFAULT_TIMEOUT):
"""
Write a dot graph and possibly run graphviz on it
"""
makedirs(fp.dirname(filename))
dot_file = filename + '.dot'
svg_file = filename + '.svg'
with codecs.open(dot_file, 'w', encoding='utf-8') as dotf:
print(dot_graph.to_string(), file=dotf)
if run_graphviz:
if not quiet:
print("Creating %s" % svg_file, file=sys.stderr)
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(timeout) # half a minute
try:
subprocess.call(["dot",
"-T", "svg",
"-o", svg_file,
dot_file])
signal.alarm(0) # reset the alarm
except Alarm:
print("Killed graphviz because it was taking too long",
file=sys.stderr)
def run(js, testfile):
if "%s" in js:
cmd = js.replace("%s", testfile)
else:
cmd = js+" "+testfile
class TimeException(Exception):
pass
def timeout_handler(signum, frame):
raise TimeException()
signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(2) # triger alarm in 3 seconds
try:
proc = subprocess.Popen( cmd, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, error = proc.communicate('through stdin to stdout')
signal.alarm(0)
return error+output
except TimeException:
#os.kill(proc.pid+1, signal.SIGINT) # no idea why + 1, but the reported pid isn't right ?!
subprocess.Popen("killall js", shell=True)
return -1
def playback(refdes, event_url, particle_url, missing_dates):
"""
Put together the directory glob and other options for the playback command.
This will then playback cabled data.
"""
main_directory = '/rsn_cabled/rsn_data/DVT_Data'
driver = get_driver(refdes)
reader_list = get_reader_type(refdes)
node = refdes.split('-')[1].lower()
instrument = refdes.split('-')[3]
signal.signal(signal.SIGALRM, timeout_handler)
for reader in reader_list:
for date in missing_dates:
directory = '/'.join([main_directory, node, instrument])
directory += '*'.join(['', date, ''])
# Check to see if this particular file exists before executing callback
if glob.glob(directory):
playback_command = ' '.join(['playback', reader, driver, refdes, event_url, particle_url, directory])
logging.info("%s", playback_command)
# Some playback reader types may not ingest the data at all.
# Timeout after 90 seconds and continue to the next reader type or
# next data.
signal.alarm(90)
try:
call(playback_command, shell=True)
except TimeoutException:
logging.warning('%s Took more than 90 seconds. Timing out this ingestion.', playback_command)
continue
else:
signal.alarm(0)
def test_from_arguments(self):
options = Options()
options.collecting_data = True
options.display_data = False
options.filename = '/tmp/%s' % (str(uuid.uuid4()))
options.file_max_size = 0.01
options.sample_interval_secs = 1
self.addCleanup(lambda: os.remove(options.filename))
signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(3)
try:
with suppress_stdout():
# still get the os system clear call which blanks the screen
main(options)
except Exception:
pass
data = []
with open(options.filename, 'r') as data_file:
for line in data_file:
data.append(json.loads(line))
self.assertTrue(len(data) > 0)
self.assertTrue('cpu_times' in data[0])
self.assertTrue('utc_time' in data[0])
self.assertTrue('sample_time' in data[0])
self.assertTrue('num_threads' in data[0])
def lambda_handler(event, context):
def run_local(requestDict):
#namespace = UsedNamespace()
#Add a try here to catch errors and return a better response
solution = requestDict['solution']
tests = requestDict['tests']
import StringIO
import sys
# Store App Engine's modified stdout so we can restore it later
gae_stdout = sys.stdout
# Redirect stdout to a StringIO object
new_stdout = StringIO.StringIO()
sys.stdout = new_stdout
try:
namespace = {}
compiled = compile('import json', 'submitted code', 'exec')
exec compiled in namespace
compiled = compile(solution, 'submitted code', 'exec')
exec compiled in namespace
namespace['YOUR_SOLUTION'] = solution.strip()
namespace['LINES_IN_YOUR_SOLUTION'] = len(
solution.strip().splitlines())
test_cases = doctest.DocTestParser().get_examples(tests)
results, solved = execute_test_cases(test_cases, namespace)
# Get whatever was printed to stdout using the `print` statement (if necessary)
printed = new_stdout.getvalue()
# Restore App Engine's original stdout
sys.stdout = gae_stdout
responseDict = {
"solved": solved,
"results": results,
"printed": printed
}
#Add a try here can catch errors.
responseJSON = json.dumps(responseDict)
logging.info("Python verifier returning %s", responseJSON)
return responseJSON
except:
sys.stdout = gae_stdout
errors = traceback.format_exc()
logging.info("Python verifier returning errors =%s", errors)
if len(errors) > 500:
lines = errors.splitlines()
i = len(lines) - 1
errors = lines[i]
i -= 1
while i >= 0:
line = lines[i]
s = '%s\n%s' % (line, errors)
if len(s) > 490:
break
errors = s
i -= 1
errors = '...\n%s' % errors
responseDict = {'errors': '%s' % errors}
responseJSON = json.dumps(responseDict)
#logging.info("######## Returning json encoded errors %s", responseJSON)
return responseJSON
#Refactor for better readability.
def execute_test_cases(testCases, namespace):
resultList = []
solved = True
for e in testCases:
if not e.want:
exec e.source in namespace
continue
call = e.source.strip()
logging.warning('call: %s', (call, ))
got = eval(call, namespace)
expected = eval(e.want, namespace)
correct = True
if got == expected:
correct = True
else:
correct = False
solved = False
resultDict = {
'call': call,
'expected': expected,
'received': "%(got)s" % {
'got': got
},
'correct': correct
}
resultList.append(resultDict)
return resultList, solved
method = event.get('httpMethod', {})
indexPage = """
<html>
<head>
<meta charset="utf-8">
<meta content="width=device-width,initial-scale=1,minimal-ui" name="viewport">
<link rel="stylesheet" href="https://unpkg.com/vue-material@beta/dist/vue-material.min.css">
<link rel="stylesheet" href="https://unpkg.com/vue-material@beta/dist/theme/default.css">
</head>
<body>
<h1>Python Doctest Custom Activity</h1>
<div id="app" class="md-layout md-gutter">
<div id="cardGroupCreator" class="md-layout-item md-size-50">
<md-card>
<md-card-header>
<md-card-header-text>
<div class="md-title">Tests</div>
<div class="md-subhead">Add your doctest below</div>
<div class="md-subhead">View <a href="https://pymotw.com/3/doctest/">this link</a> for more info</div>
</md-card-header-text>
</md-card-header>
<md-card-content>
<md-field>
<md-textarea v-model="testCases"></md-textarea>
</md-field>
</md-card-content>
</md-card>
<md-card>
<md-card-header>
<md-card-header-text>
<div class="md-title">Hidden Code Block</div>
<div class="md-subhead">Your code goes below</div>
</md-card-header-text>
</md-card-header>
<md-card-content>
<md-field>
<md-textarea v-model="hidden"></md-textarea>
</md-field>
</md-card-content>
</md-card>
<md-card>
<md-card-header>
<md-card-header-text>
<div class="md-title">Editable Code Block</div>
<div class="md-subhead">Your code goes below</div>
</md-card-header-text>
</md-card-header>
<md-card-content>
<md-field>
<md-textarea v-model="solution"></md-textarea>
</md-field>
</md-card-content>
</md-card>
<button v-on:click="staygo">Submit</button>
</div>
<div id="cardGroupPreview" class="md-layout-item md-size-50">
<md-card>
<md-card-header>
<md-card-header-text>
<div class="md-title">Output</div>
<div class="md-subhead">Test results</div>
</md-card-header-text>
</md-card-header>
<md-card-content>
<md-field>
<md-tabs>
<md-tab id="tab-htmlResults" md-label="HTML results">
<div v-html="answer.htmlFeedback"></div>
</md-tab>
<md-tab id="tab-jsonResults" md-label="JSON results">
<md-textarea v-model="answer.jsonFeedback" readonly></md-textarea>
</md-tab>
<md-tab id="tab-textResults" md-label="Text results">
<md-textarea v-model="answer.textFeedback" readonly></md-textarea>
</md-tab>
</md-tabs>
</md-field>
</md-card-content>
</md-card>
</div>
</div>
</body>
<script src="https://unpkg.com/vue"></script>
<script src="https://unpkg.com/vue-material@beta"></script>
<script>
Vue.use(VueMaterial.default)
new Vue({
el: '#app',
data: {
testCases: ">>> sum(1,2)\\n3 \\n>>> sum(2,2)\\n4",
solution:"def sum(a,b):\\n return b+1",
answer:"",
hidden: ">>> sum(1,2)\\n10 \\n>>> sum(2,2)\\n11"
},
methods: {
staygo: function () {
// comment: leaving the gatewayUrl empty - API will post back to itself
const gatewayUrl = '';
fetch(gatewayUrl, {
method: "POST",
headers: {
'Accept': 'application/json',
'Content-Type': 'application/json'
},
body: JSON.stringify({shown:{0:this.testCases},editable:{0:this.solution}, hidden:{0:this.hidden}})
}).then(response => {
return response.json()
}).then(data => {
this.answer = JSON.parse(JSON.stringify(data))
})
}
}
})
</script>
<style lang="scss" scoped>
.md-card {
width: 90%;
margin: 20px;
display: inline-block;
vertical-align: top;
min-height:300px
}
.md-card-content {
padding-bottom: 16px !important;
}
button {
display:block;
margin: 20px 60px 20px 60px;
width:200px !important;
}
#cardGroupCreator {
display:flex;
flex-direction:column
}
#cardGroupPreview .md-card {
width: 500px;
}
textarea {
font-size: 1rem !important;
}
.md-tabs{
width:100%;
}
html {
width:95%;
}
h1{
padding:20px;
margin:auto
}
.md-content{
min-height:300px
}
.md-tabs-container .md-tab textarea{
height:100%
}
</style>
</html>
"""
if method == 'GET':
return {
"statusCode": 200,
"headers": {
'Content-Type': 'text/html',
},
"body": indexPage
}
if method == 'POST':
bodyContent = event.get('body', {})
parsedBodyContent = json.loads(bodyContent)
testCases = parsedBodyContent["shown"]["0"]
solution = parsedBodyContent["editable"]["0"]
hidden = "\n" + parsedBodyContent["hidden"]["0"]
timeout = False
# handler function that tell the signal module to execute
# our own function when SIGALRM signal received.
def timeout_handler(num, stack):
print("Received SIGALRM")
raise Exception("processTooLong")
# register this with the SIGALRM signal
signal.signal(signal.SIGALRM, timeout_handler)
# signal.alarm(10) tells the OS to send a SIGALRM after 10 seconds from this point onwards.
signal.alarm(10)
# After setting the alarm clock we invoke the long running function.
try:
jsonResponse = run_local({
"solution": solution,
"tests": testCases + hidden
})
except Exception as ex:
if "processTooLong" in ex:
timeout = True
print("processTooLong triggered")
# set the alarm to 0 seconds after all is done
finally:
signal.alarm(0)
jsonResponseData = json.loads(jsonResponse)
solvedStatusText = ""
expectedText = ""
receivedText = ""
callText = ""
textResults = ""
overallResults = """<span class="md-subheading">All tests passed: {0}</span><br/>""".format(
str(jsonResponseData.get("solved")))
numTestCases = len(re.findall('>>>', testCases))
resultContent = jsonResponseData.get('results')
tableContentsPublic = ""
tableContentsPrivate = ""
textBackgroundColor = "#ffffff"
if resultContent:
for i in range(len(resultContent)):
expectedText = resultContent[i]["expected"]
receivedText = resultContent[i]["received"]
correctText = resultContent[i]["correct"]
callText = resultContent[i]["call"]
if numTestCases > i:
if str(expectedText) == str(receivedText):
textResults = textResults + "\nHurray! You have passed the test case. You called {0} and received {1} against the expected value of {2}.\n".format(
callText, receivedText, expectedText)
textBackgroundColor = "#b2d8b2"
else:
textResults = textResults + "\nThe test case eludes your code so far but try again! You called {0} and received {1} against the expected value of {2}.\n".format(
callText, receivedText, expectedText)
textBackgroundColor = "#ff9999"
tableContentsPublic = tableContentsPublic + """
<tr bgcolor={4}>
<td>{0}</td>
<td>{1}</td>
<td>{2}</td>
<td>{3}</td>
</tr>
""".format(callText, expectedText, receivedText,
correctText, textBackgroundColor)
else:
if str(expectedText) == str(receivedText):
textResults = textResults + "\nHurray! You have passed the hidden test case. You called {0} and received {1} against the expected value of {2}.\n".format(
callText, receivedText, expectedText)
textBackgroundColor = "#b2d8b2"
else:
textResults = textResults + "\nThe hidden test case eludes your code so far but try again! The hidden test case called {0} and received {1} against the expected value of {2}.\n".format(
callText, receivedText, expectedText)
textBackgroundColor = "#ff9999"
tableContentsPrivate = tableContentsPrivate + """
<tr bgcolor={4}>
<td>{0}</td>
<td>{1}</td>
<td>{2}</td>
<td>{3}</td>
</tr>
""".format(callText, expectedText, receivedText,
correctText, textBackgroundColor)
solvedStatusText = str(jsonResponseData.get("solved")) or "error"
textResults = """All tests passed: {0}\n""".format(
solvedStatusText) + textResults
if not resultContent:
textResults = "Your test is passing but something is incorrect..."
if timeout or jsonResponseData.get("errors"):
textResults = "An error - probably related to code syntax - has occured. Do look through the JSON results to understand the cause."
tableContentsPublic = """
<tr>
<td></td>
<td></td>
<td>error</td>
<td></td>
</tr>
"""
tableContentsPrivate = """
<tr>
<td></td>
<td></td>
<td>error</td>
<td></td>
</tr>
"""
htmlResults = """
<html>
<head>
<meta charset="utf-8">
<meta content="width=device-width,initial-scale=1,minimal-ui" name="viewport">
</head>
<body>
<div>
{0}
<span class="md-subheading tableTitle">Public tests</span>
<table>
<thead>
<tr>
<th>Called</th>
<th>Expected</th>
<th>Received</th>
<th>Correct</th>
</tr>
</thead>
<tbody>
{1}
</tbody>
</table>
<br/>
<span class="md-subheading tableTitle">Private tests</span>
<table>
<thead>
<tr>
<th>Called</th>
<th>Expected</th>
<th>Received</th>
<th>Correct</th>
</tr>
</thead>
<tbody>
{2}
</tbody>
</table>
</div>
</body>
<style>
br {{
display:block;
content:"";
margin:1rem
}}
table{{
text-align:center
}}
.tableTitle{{
text-decoration:underline
}}
</style>
</html>
""".format(overallResults, tableContentsPublic,
tableContentsPrivate)
return {
"statusCode":
200,
"headers": {
"Content-Type": "application/json",
},
"body":
json.dumps({
"isComplete": jsonResponseData.get("solved"),
"jsonFeedback": jsonResponse,
"htmlFeedback": htmlResults,
"textFeedback": textResults
})
}
def __exit__(self, *_):
if self._seconds is not None:
signal.alarm(0)
def __enter__(self):
if self._seconds is not None:
signal.signal(signal.SIGALRM, self._action)
signal.alarm(self._seconds)
def gameplay(obj1, obj2): #game simulator game_board = Board() fl1 = 'x' fl2 = 'o' old_move = (-1,-1) WINNER = '' MESSAGE = '' TIME = 15 pts1 = 0 pts2 = 0 game_board.print_board() signal.signal(signal.SIGALRM, handler) while(1): #player 1 turn temp_board_status = copy.deepcopy(game_board.board_status) temp_block_status = copy.deepcopy(game_board.block_status) signal.alarm(TIME) try: #try to get player 1's move p1_move = obj1.move(game_board, old_move, fl1) except TimedOutExc: #timeout error # print e WINNER = 'P2' MESSAGE = 'TIME OUT' pts2 = 16 break except Exception as e: WINNER = 'P2' MESSAGE = 'INVALID MOVE' pts2 = 16 break signal.alarm(0) #check if board is not modified and move returned is valid if (game_board.block_status != temp_block_status) or (game_board.board_status != temp_board_status): WINNER = 'P2' MESSAGE = 'MODIFIED THE BOARD' pts2 = 16 break if game_board.update(old_move, p1_move, fl1) == 'UNSUCCESSFUL': WINNER = 'P2' MESSAGE = 'INVALID MOVE' pts2 = 16 break status = game_board.find_terminal_state() #find if the game has ended and if yes, find the winner print status if status[1] == 'WON': #if the game has ended after a player1 move, player 1 would win pts1 = 16 WINNER = 'P1' MESSAGE = 'WON' break elif status[1] == 'DRAW': #in case of a draw, each player gets points equal to the number of blocks won WINNER = 'NONE' MESSAGE = 'DRAW' break old_move = p1_move game_board.print_board() #do the same thing for player 2 temp_board_status = copy.deepcopy(game_board.board_status) temp_block_status = copy.deepcopy(game_board.block_status) signal.alarm(TIME) try: p2_move = obj2.move(game_board, old_move, fl2) except TimedOutExc: WINNER = 'P1' MESSAGE = 'TIME OUT' pts1 = 16 break except Exception as e: WINNER = 'P1' MESSAGE = 'INVALID MOVE' pts1 = 16 break signal.alarm(0) if (game_board.block_status != temp_block_status) or (game_board.board_status != temp_board_status): WINNER = 'P1' MESSAGE = 'MODIFIED THE BOARD' pts1 = 16 break if game_board.update(old_move, p2_move, fl2) == 'UNSUCCESSFUL': print 'hey' WINNER = 'P1' MESSAGE = 'INVALID MOVE' # pts1 = 16 break status = game_board.find_terminal_state() #find if the game has ended and if yes, find the winner print status if status[1] == 'WON': #if the game has ended after a player move, player 2 would win pts2 = 16 WINNER = 'P2' MESSAGE = 'WON' break elif status[1] == 'DRAW': WINNER = 'NONE' MESSAGE = 'DRAW' break game_board.print_board() old_move = p2_move game_board.print_board() print "Winner:", WINNER print "Message", MESSAGE x = 0 d = 0 o = 0 for i in range(4): for j in range(4): if game_board.block_status[i][j] == 'x': x += 1 if game_board.block_status[i][j] == 'o': o += 1 if game_board.block_status[i][j] == 'd': d += 1 print 'x:', x, ' o:',o,' d:',d if MESSAGE == 'DRAW': pts1 = x pts2 = o return (pts1,pts2)
# form up a list of bits NOT to run, mapping from old step names
# to new step names as appropriate.
skipsteps = opts.skip.split(',')
new_skipsteps = []
for skipstep in skipsteps:
if skipstep in step_mapping:
new_skipsteps.append(step_mapping[skipstep])
else:
new_skipsteps.append(skipstep)
skipsteps = new_skipsteps
# ensure we catch timeouts
signal.signal(signal.SIGALRM, alarm_handler)
if opts.timeout is not None:
signal.alarm(opts.timeout)
if opts.list:
for step in steps:
print(step)
sys.exit(0)
if opts.list_subtests:
list_subtests()
sys.exit(0)
if opts.list_subtests_for_vehicle:
list_subtests_for_vehicle(opts.list_subtests_for_vehicle)
sys.exit(0)
if opts.list_vehicles_test:
def __exit__(self, exc_type, exc_val, exc_tb):
signal.alarm(0) # Cancel SIGALRM if it's scheduled
if self.suppress and exc_type is TimeoutError: # Suppress TimeoutError
return True
def __enter__(self):
signal.signal(signal.SIGALRM,
self._timeout_handler) # Set handler for SIGALRM
signal.alarm(self.seconds) # start countdown for SIGALRM to be raised
def __init__(self, synth, port, restore=False, timeout=10):
def timeout_handler(signum, frame):
raise TimeoutException()
if timeout > 0:
# Warning: the "signal" implementation works only on linux systems
signal.signal(signal.SIGALRM, timeout_handler)
signal.alarm(timeout) # triggers alarm in 5 seconds
try:
Synthesizer.__init__(self, port)
except valon_synth.serial.SerialException as err:
raise RuntimeError("Unable to open serial port.\nError MSG:\n%s" %
err)
except:
raise
# Add usage instructions for parent call functions
self.get_vco_range.__func__.__doc__ = "\
\nReturn the minimum and maximum frequency range of the selected synthesizer VCO. \
\nThe VCO Frequency Range information is used to limit and check the resulting VCO output frequency, \
\nentered in the set frequency request function. \
\n\t@param synth String: Synthesizer name \
\n\t@return List: (lowest VCO output frequency, highest VCO output frequency) in MHz \
\n"
self.set_vco_range.__func__.__doc__ = "\
\nSet the minimum and maximum frequency range of the selected synthesizer VCO. \
\n\t@param synth String: Synthesizer name \
\n\t@param min Int: The lowest VCO output frequency (MHz) that may be set \
\n\t@param max Int: The highest VCO output frequency (MHz) that may be set \
\n\t@return Boolean: Reply message indicating success. \
\n"
self.get_frequency.__func__.__doc__ = "\
\nReturn the current VCO output frequency for the selected synthesizer. \
\n\t@param synth String: Synthesizer name \
\n\t@return Float: Synthesizer output frequency (MHz) \
\n"
self.set_frequency.__func__.__doc__ = "\
\nRequest to set the desired frequency (MHz) for the selected synthesizer. \
\nThis is will cause the registers synthesizer to be updated and the new frequency generated. \
\nThe resulting output frequency must be within the range specified in the frequency range. \
\n\t@param synth String: Synthesizer name \
\n\t@param freq Int: Desired frequency (MHz) \
\n\t@param chan_spacing Float: Frequency Increment (KHz) to set required output frequency resolution \
\n\t@return Boolean: Reply message indicating success. \
\n"
self.get_reference.__func__.__doc__ = "\
\nRequest the input frequency of the reference oscillator. \
\n\t@param No input parameters \
\n\t@return Int: Reference input frequency (Hz) \
\n"
self.set_reference.__func__.__doc__ = "\
\nSet the input frequency of the reference oscillator. \
\nThis value must be between 5 MHz and 150 MHz according to the 5007 datasheet. \
\n\t@param freq Int: Reference input frequency (Hz) \
\n\t@return Boolean: Reply message indicating success. \
\n"
self.get_rf_level.__func__.__doc__ = "\
\nReturns the current output power level: -4, -1, 2, 5 \
\n\t@param synth String: Synthesizer name \
\n\t@return Int: Ouput power level \
\n"
self.set_rf_level.__func__.__doc__ = "\
\nAllows user to select one of four output power levels: -4, -1, 2, 5 \
\nThese levels corresponds approximately to some preset output power. \
\n\t@param synth String: Synthesizer name \
\n\t@param rf_level Int: Ouput power level \
\n\t@return Boolean: Reply message indicating success. \
\n"
self.get_ref_select.__func__.__doc__ = "\
\nReturns the currently selected reference clock. \
\n\t@param No input parameters \
\n\t@return Int: 1 for external reference and 0 otherwise \
\n"
self.set_ref_select.__func__.__doc__ = "\
\nSelects either internal or external reference clock. \
\n\t@parameter e_not_i Int: 1 for external reference and 0 for internal reference \
\n\t@return Boolean: Reply message indicating success. \
\n"
self.get_label.__func__.__doc__ = "\
\nRead the assigned synthesizer name. \
\n\t@param synth String: Synthesizer name \
\n\t@return String: Assigned synthesizer name \
\n"
self.set_label.__func__.__doc__ = "\
\nLabel the selected synthesizer to suit your application, if desired. \
\nThe maximum length of a name is 16 characters. \
\n\t@param synth String: Synthesizer name \
\n\t@param label String: Synthesizer alias to assign\
\n\t@return Boolean: Reply message indicating success. \
\n"
self.get_phase_lock.__func__.__doc__ = " \
\n Return the VCO lock status of the selected synthesizer. \
\n\t@param synth String: Synthesizer name \
\n\t@return Boolean: Reply message indicating VCO lock status. \
\n"
self.flash.__func__.__doc__ = "\
\nWrite setup parameters to non-volatile flash memory in the microcontroller board. \
\nThe next time the board is powered up, the registers will be set to the values in the non-volatile flash memory. \
\nIf the board is powered down before the write flash command command is issued, all the data in the registers will be lost. \
\n\t@param No input parameters \
\n\t@return Boolean: Reply message indicating parameters written to flash. \
\n"
self.get_options.__func__.__doc__ = "\
\nOptional VCO Settings \
\n\tprint synth.get_options(valon_synth.SYNTH_B) \
\nOutput a list of 4 parameters, which can be 0 (disabled) or 1(enabled) \
\ndouble: The reference doubler is used to enable a multiply by 2 function before the internal reference divider. \
\n\tEnable the doubler when using a 5 MHz external reference frequency. \
\n\tWhen using the internal 10 MHz reference the doubler should be disabled. \
\nhalf: The reference divide by 2 is used to enable a divide by 2 function after the intercal reference divider. \
\n\tWhen enabled, the input to phase-frequency detector will have a 50% duty cycle which will allow for faster lock up time. \
\n\tIn order to use this mode a 20 MHz external reference would have to be available. \
\n\tFor normal operations set the reference div by 2 to disabled. \
\nr: ?? \
\nspur: Low noise mode vs Low spur mode. \
\n\tLow noise mode affects the operation of the fractional synthesizer, \
\n\tand this mode will produce the lowest phase noise but there may be some spurious output signals. \
\n\tLow spur mode will reduce spurious output response but the overall phase noise will be higher. \
\n"
self.set_options.__func__.__doc__ = "\
\nOptional VCO Settings for a specified synthesizer \
\nOutput a list of 4 parameters, which can be 0 (disabled) or 1(enabled) \
\ndouble: The reference doubler is used to enable a multiply by 2 function before the internal reference divider. \
\n\tEnable the doubler when using a 5 MHz external reference frequency. \
\n\tWhen using the internal 10 MHz reference the doubler should be disabled. \
\nhalf: The reference divide by 2 is used to enable a divide by 2 function after the intercal reference divider. \
\n\tWhen enabled, the input to phase-frequency detector will have a 50% duty cycle which will allow for faster lock up time. \
\n\tIn order to use this mode a 20 MHz external reference would have to be available. \
\n\tFor normal operations set the reference div by 2 to disabled. \
\nr: ?? \
\nspur: Low noise mode vs Low spur mode. \
\n\tLow noise mode affects the operation of the fractional synthesizer, \
\n\tand this mode will produce the lowest phase noise but there may be some spurious output signals. \
\n\tLow spur mode will reduce spurious output response but the overall phase noise will be higher. \
\n"
if not restore:
## Set valon to use internal reference clock
try:
if self.get_ref_select() == "1":
self.set_rf_select(e_not_i=0)
# ## Set frequency to 2.4 GHz
# self.set_frequency(synth, freq=2465)
# ## Set rf level to be minimum available
# self.set_rf_level(synth,-4)
except TimeoutException:
raise RuntimeError('Could not connect to Valon')
self.synth = synth
def __enter__(self):
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(self.timeout_seconds)
def get_urls(f):
filename = sys.argv[1] + f
with open(filename, 'r') as json_file:
d = json.loads(json_file.read())
#html_page = d['html_content']
signal.signal(signal.SIGALRM, handler)
try:
signal.alarm(30) #timeout on page content after 30 seconds
html_page = urllib.request.urlopen(d['url']) #get html of page
except Exception:
html_page = d['html_content'] #get html of page from file
signal.alarm(0)
soup = BeautifulSoup(html_page, "html.parser")
urls_full = []
urls = []
logging.info(str(os.getpid()) + ': Getting links')
allLinks = soup.findAll('a')
c = 0
#get all the links in the html content
for link in allLinks:
signal.signal(signal.SIGALRM, handler)
try: #try validating the link
logging.info(
str(os.getpid()) + ': Validating ' + link.get('href'))
if not validators.url(link.get('href')):
continue
try: #try getting the title for the link
logging.info(
str(os.getpid()) + ': Getting title after validation')
signal.alarm(30)
title = BeautifulSoup(urllib.request.urlopen(
link.get('href')),
features="lxml").title.string
except Exception:
title = ""
except Exception:
try: #try getting the title for the link
logging.info(
str(os.getpid()) +
': Getting title without validation')
signal.alarm(30) #timeout after 30 seconds
title = BeautifulSoup(urllib.request.urlopen(
link.get('href')),
features="lxml").title.string
except Exception:
title = ""
signal.alarm(0)
logging.info(str(os.getpid()) + ': Producing final link')
final_link = link.get('href')
logging.info(str(os.getpid()) + ': Adding link')
urls_full.append({'title': title, 'url': final_link})
urls.append(final_link)
c += 1
logging.info(
str(os.getpid()) + ': Found ' + str(c) + '/' +
str(len(allLinks)) + ' links')
logging.info(str(os.getpid()) + ': Done getting links')
logging.info(str(os.getpid()) + ": Adding new URLs")
#add any URLs that were already found, if they were not found again
for u in d['found_urls']:
if u['url'] not in urls:
urls_full.append(u)
d['found_urls'] = urls_full
logging.info(str(os.getpid()) + ": Done adding new URLs")
logging.info(str(os.getpid()) + ": Writing to file")
#write the new result to file
with open('Results/' + f, 'w') as outfile:
outfile.write(json.dumps(d))
#After finishing everything, move the original file
#os.popen('rm ' + filename)
os.popen('mv ' + filename + ' Complete/' + f)
logging.info(str(os.getpid()) + ": Done writing to file")
def check_pause(self, call):
# the app sleeps for 0.5 seconds
self.curl.setopt(pycurl.URL, 'http://localhost:8380/pause')
sio = util.StringIO()
state = dict(paused=False, resumed=False)
if call:
def writefunc(data):
rv = sio.write(data)
if not state['paused']:
self.curl.pause(pycurl.PAUSE_ALL)
state['paused'] = True
return rv
else:
def writefunc(data):
if not state['paused']:
# cannot write to sio here, because
# curl takes pause return value to mean that
# nothing was written
state['paused'] = True
return pycurl.READFUNC_PAUSE
else:
return sio.write(data)
def resume(*args):
state['resumed'] = True
self.curl.pause(pycurl.PAUSE_CONT)
signal.signal(signal.SIGALRM, resume)
# alarm for 1 second which is 0.5 seconds more than the server side
# should sleep for
signal.alarm(1)
start = _time.time()
self.curl.setopt(pycurl.WRITEFUNCTION, writefunc)
m = pycurl.CurlMulti()
m.add_handle(self.curl)
# Number of seconds to wait for a timeout to happen
SELECT_TIMEOUT = 1.0
# Stir the state machine into action
while 1:
ret, num_handles = m.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
# Keep going until all the connections have terminated
while num_handles:
# The select method uses fdset internally to determine which file descriptors
# to check.
m.select(SELECT_TIMEOUT)
while 1:
if _time.time() - start > 2:
# test is taking too long, fail
assert False, 'Test is taking too long'
ret, num_handles = m.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
# Cleanup
m.remove_handle(self.curl)
m.close()
self.assertEqual('part1part2', sio.getvalue())
end = _time.time()
# check that client side waited
self.assertTrue(end - start > 1)
assert state['resumed']
def __exit__(self, etype, value, traceback):
signal.alarm(0)
def make_1nn_evaluation(x_train, y_train, x_test, y_test, U_centroids,
indicator_vector):
"""
Do the 1-nearest neighbor classification using `x_train`, `y_train` as support and `x_test`, `y_test` as
evaluation set.
The scikilearn classifiers (brute, kdtree and balltree) are called only in the case where it is the kmeans version
of the program that is called (for simplicity purposes: not do it many times).
Time is recorded.
Classification accuracy is recorded.
:param x_train: Train data set as ndarray.
:param y_train: Train labels as categories in ndarray.
:param x_test: Test data as ndarray.
:param y_test: Test labels as categories.
:param U_centroids: The matrix of centroids as ndarray or SparseFactor object
:param indicator_vector: The indicator vector for this matrix of centroids and this train data.
:return:
"""
def scikit_evaluation(str_type):
"""
Do the scikit learn version of nearest neighbor (used for comparison)
:param str_type:
:return:
"""
clf = KNeighborsClassifier(n_neighbors=1, algorithm=str_type)
clf.fit(x_train, y_train)
start_inference_time = time.time()
predictions = np.empty_like(y_test)
for obs_idx, obs_test in enumerate(x_test):
predictions[obs_idx] = clf.predict(obs_test.reshape(1, -1))[0]
stop_inference_time = time.time()
inference_time = (stop_inference_time - start_inference_time)
accuracy = np.sum(predictions == y_test) / y_test.shape[0]
results_1nn = {
"1nn_{}_inference_time".format(str_type): inference_time,
"1nn_{}_accuracy".format(str_type): accuracy
}
resprinter.add(results_1nn)
return inference_time
def kmean_tree_evaluation():
"""
Do the K-means partitioning version of nearest neighbor?=.
:return:
"""
# for each cluster, there is a sub nearest neighbor classifier for points in that cluster.
lst_clf_by_cluster = [
KNeighborsClassifier(n_neighbors=1, algorithm="brute").fit(
x_train[indicator_vector == i], y_train[indicator_vector == i])
for i in range(U_centroids.shape[0])
]
start_inference_time = time.time()
distances = get_distances(x_test, U_centroids)
indicator_vector_test = np.argmin(distances, axis=1)
predictions = np.empty_like(y_test)
for obs_idx, obs_test in enumerate(x_test):
# get the cluster to which belongs this data point and call the associated nearest neighbor classifier
idx_cluster = indicator_vector_test[obs_idx]
clf_cluster = lst_clf_by_cluster[idx_cluster]
predictions[obs_idx] = clf_cluster.predict(obs_test.reshape(1,
-1))[0]
stop_inference_time = time.time()
inference_time = (stop_inference_time - start_inference_time)
accuracy = np.sum(predictions == y_test) / y_test.shape[0]
results_1nn = {
"1nn_kmean_inference_time": inference_time,
"1nn_kmean_accuracy": accuracy
}
resprinter.add(results_1nn)
return inference_time
logger.info("1 nearest neighbor with k-means search")
kmean_tree_time = kmean_tree_evaluation()
#
if paraman["kmeans"]:
lst_knn_types = ["brute", "ball_tree", "kd_tree"]
for knn_type in lst_knn_types:
# the classification must not take more than 10 times the time taken for the K means 1 nn classification or
# it will stop.
signal.signal(signal.SIGALRM, timeout_signal_handler)
signal.alarm(int(kmean_tree_time * 10)) # start alarm
try:
logger.info(
"1 nearest neighbor with {} search".format(knn_type))
scikit_evaluation(knn_type)
except TimeoutError as te:
logger.warning(
"Timeout during execution of 1-nn with {} version: {}".
format(knn_type, te))
signal.alarm(0) # stop alarm for next evaluation
def deco(*args, **kwargs):
signal.signal(signal.SIGALRM, handler)
signal.alarm(interval)
res = func(*args, **kwargs)
signal.alarm(0)
return res
def run_val_gdb(self, testing=False):
"""
Spawns a subprocess to run tests under the control of gdb or valgrind.
INPUT:
- ``testing`` -- boolean; if True then the command to be run
will be printed rather than a subprocess started.
EXAMPLES:
Note that the command lines include unexpanded environment
variables. It is safer to let the shell expand them than to
expand them here and risk insufficient quoting. ::
sage: from sage.doctest.control import DocTestDefaults, DocTestController
sage: DD = DocTestDefaults(gdb=True)
sage: DC = DocTestController(DD, ["hello_world.py"])
sage: DC.run_val_gdb(testing=True)
exec gdb -x "$SAGE_LOCAL/bin/sage-gdb-commands" --args python "$SAGE_LOCAL/bin/sage-runtests" --serial --timeout=0 hello_world.py
::
sage: DD = DocTestDefaults(valgrind=True, optional="all", timeout=172800)
sage: DC = DocTestController(DD, ["hello_world.py"])
sage: DC.run_val_gdb(testing=True)
exec valgrind --tool=memcheck --leak-resolution=high --leak-check=full --num-callers=25 --suppressions="$SAGE_LOCAL/lib/valgrind/sage.supp" --log-file=".../valgrind/sage-memcheck.%p" python "$SAGE_LOCAL/bin/sage-runtests" --serial --timeout=172800 --optional=True hello_world.py
"""
try:
sage_cmd = self._assemble_cmd()
except ValueError:
self.log(sys.exc_info()[1])
return 2
opt = self.options
if opt.gdb:
cmd = '''exec gdb -x "$SAGE_LOCAL/bin/sage-gdb-commands" --args '''
flags = ""
if opt.logfile:
sage_cmd += " --logfile %s"%(opt.logfile)
else:
if opt.logfile is None:
default_log = os.path.join(DOT_SAGE, "valgrind")
if os.path.exists(default_log):
if not os.path.isdir(default_log):
self.log("%s must be a directory"%default_log)
return 2
else:
os.makedirs(default_log)
logfile = os.path.join(default_log, "sage-%s")
else:
logfile = opt.logfile
if opt.valgrind:
toolname = "memcheck"
flags = os.getenv("SAGE_MEMCHECK_FLAGS")
if flags is None:
flags = "--leak-resolution=high --leak-check=full --num-callers=25 "
flags += '''--suppressions="%s" '''%(os.path.join("$SAGE_LOCAL","lib","valgrind","sage.supp"))
elif opt.massif:
toolname = "massif"
flags = os.getenv("SAGE_MASSIF_FLAGS", "--depth=6 ")
elif opt.cachegrind:
toolname = "cachegrind"
flags = os.getenv("SAGE_CACHEGRIND_FLAGS", "")
elif opt.omega:
toolname = "exp-omega"
flags = os.getenv("SAGE_OMEGA_FLAGS", "")
cmd = "exec valgrind --tool=%s "%(toolname)
flags += ''' --log-file="%s" ''' % logfile
if opt.omega:
toolname = "omega"
if "%s" in flags:
flags %= toolname + ".%p" # replace %s with toolname
cmd += flags + sage_cmd
self.log(cmd)
sys.stdout.flush()
sys.stderr.flush()
if self.logfile is not None:
self.logfile.flush()
if testing:
return
# Setup Sage signal handler
_init_csage()
import signal, subprocess
p = subprocess.Popen(cmd, shell=True)
if opt.timeout > 0:
signal.alarm(opt.timeout)
try:
return p.wait()
except AlarmInterrupt:
self.log(" Timed out")
return 4
except KeyboardInterrupt:
self.log(" Interrupted")
return 128
finally:
signal.alarm(0)
if p.returncode is None:
p.terminate()
if not verify_tls_cert:
if verbosity > Verbosity.SINGLE:
print('TLS certificate verification: OFF')
# Disable SSL/TLS warnings coming from urllib,
# i.e. trust B2SHARE server even with invalid certificate
requests.packages.urllib3.disable_warnings()
if verbosity > Verbosity.SINGLE:
print('Verbosity level: {}'.format(verbosity))
if timeout and timeout > 0:
if verbosity > Verbosity.SINGLE:
print('Timeout: {} seconds'.format(timeout))
signal.signal(signal.SIGALRM, handler)
signal.alarm(timeout)
if verbosity > Verbosity.SINGLE:
print('B2SHARE URL: {}'.format(base_url))
print('Starting B2SHARE Probe...')
print('---------------------------')
try:
search_url = base_url + "/api/records/"
if verbosity > Verbosity.SINGLE:
print('Making a search.')
search_results = get_dict_from_url(search_url,
verify_tls_cert,
verbosity=verbosity)
def generate_feedback(solution):
"""
take in user submit solution and run test cases to generate feedback
:param solution: user submit solution
:return: a dict of feedback
"""
output = io.StringIO()
timeout = False
# handler function that tell the signal module to execute
# our own function when SIGALRM signal received.
def timeout_handler(num, stack):
print("Received SIGALRM")
raise Exception("processTooLong")
# register this with the SIGALRM signal
signal.signal(signal.SIGALRM, timeout_handler)
# signal.alarm(10) tells the OS to send a SIGALRM after 10 seconds from this point onwards.
signal.alarm(10)
# After setting the alarm clock we invoke the long running function.
try:
result, solved = run_test_case_session(solution)
printed = output.getvalue()
responseDict = {
"solved": solved,
"results": result,
"printed": printed
}
except Exception as ex:
if "processTooLong" in ex:
timeout = True
print("processTooLong triggered")
# set the alarm to 0 seconds after all is done
finally:
signal.alarm(0)
jsonResponseData = responseDict
textResults = tableContents = ""
overallResults = """<span class="md-subheading">All tests passed: {0}</span><br/>""".format(
str(jsonResponseData.get("solved")))
resultContent = jsonResponseData.get('results')
if resultContent:
for i in range(len(resultContent)):
expectedText = resultContent[i]["expected"]
receivedText = resultContent[i]["received"]
correct = resultContent[i]["correct"]
objective = resultContent[i]["case_objective"]
if correct:
textResults = textResults + "\nHurray! You have passed this test case {}.\n".format(
expectedText)
textBackgroundColor = "#b2d8b2"
else:
if expectedText == 'skip':
textResults = textResults + "\nYou should skip this case: {}".format(
expectedText)
textResults = textResults + "\nThe test case eludes your code so far but try again! You should match {} but received {}.\n".format(
expectedText, receivedText)
textBackgroundColor = "#ff9999"
tableContents = tableContents + """
<tr bgcolor={4}>
<td>{0}</td>
<td>{1}</td>
<td>{2}</td>
<td>{3}</td>
</tr>
""".format(objective, expectedText, receivedText,
str(correct), textBackgroundColor)
solvedStatusText = str(jsonResponseData.get("solved")) or "error"
textResults = """All tests passed: {0}\n""".format(
solvedStatusText) + textResults
if not resultContent:
textResults = "Your test is passing but something is incorrect..."
if timeout or jsonResponseData.get("errors"):
textResults = "An error - probably related to code syntax - has occured. Do look through the JSON results to understand the cause."
tableContents = """
<tr>
<td></td>
<td></td>
<td>error</td>
<td></td>
</tr>
"""
htmlResults = """
<html>
<head>
<meta charset="utf-8">
<meta content="width=device-width,initial-scale=1,minimal-ui" name="viewport">
</head>
<body>
<div>
{0}
<span class="md-subheading tableTitle">Tests</span>
<table>
<thead>
<tr>
<th>Objective</th>
<th>Case</th>
<th>Received</th>
<th>Correct</th>
</tr>
</thead>
<tbody>
{1}
</tbody>
</table>
</div>
</body>
<style>
br {{
display:block;
content:"";
margin:1rem
}}
table{{
text-align:center
}}
.tableTitle{{
text-decoration:underline
}}
</style>
</html>
""".format(overallResults, tableContents)
return {
"statusCode":
200,
"headers": {
"Content-Type": "application/json",
},
"body":
json.dumps({
"isComplete": jsonResponseData.get("solved"),
"jsonFeedback": jsonResponseData,
"htmlFeedback": htmlResults,
"textFeedback": textResults
})
}
target = process("./chall-test_tamu18-pwn1")
gdb.attach(target, execute="verify_exploit")
bof_payload = sf.BufferOverflow(arch=32)
bof_payload.set_input_start(0x37)
bof_payload.add_int32(0x20, 0xf007ba11)
bof_payload.set_ret(0x804854b)
payload = bof_payload.generate_payload()
target.sendline(payload)
# Exploit Verification starts here 15935728
def handler(signum, frame):
raise Exception("Timed out")
def check_verification_done():
while True:
if os.path.exists("pwned") or os.path.exists("rip"):
sys.exit(0)
signal.signal(signal.SIGALRM, handler)
signal.alarm(2)
try:
while True:
check_verification_done()
except Exception:
print("Exploit timed out")
def test(self):
logger = get_OTB_log()
needs_update = False
if test_name not in self.data:
needs_update = True
if test_name in self.data:
if (self.data[test_name][0] !=
a_tuple[0]) or (self.data[test_name][1] != a_tuple[1]) or (
self.data[test_name][2] is False):
needs_update = True
if needs_update:
signal.signal(signal.SIGALRM, alarm_handler)
signal.alarm(6 * 60) # 6 minutes
black_list = []
ut_command = a_tuple[0]
self.assertTrue(ut_command is not None)
self.assertTrue(ut_command != "")
ut_command_validation = a_tuple[1]
self.assertTrue(ut_command_validation is not None)
self.assertTrue(ut_command_validation != "")
if ut_command.split(" ")[0] in black_list:
raise Exception("Blacklisted test!")
args = shlex.split(ut_command)
failed = False
logger.info("Running [%s]" % ut_command)
p = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(pout, perr) = p.communicate()
if ("ERROR" in pout or "ERROR"
in perr) or ("FATAL" in pout or "FATAL" in perr) or (
"CRITICAL" in pout or "CRITICAL" in perr):
error_text = "Command [%s] returned [%s]" % (ut_command, pout)
if "Invalid image filename" in pout or "Invalid vector data filename" in pout or "Failed to open" in pout:
logger.warning(error_text)
else:
logger.error(error_text)
self.fail(error_text)
failed = True
else:
logger.info(pout)
if (len(ut_command_validation) > 0) and not failed:
new_ut_command_validation = ut_command_validation + " Execute " + ut_command
logger.info("Running Unit test [%s]" %
new_ut_command_validation)
argz = shlex.split(new_ut_command_validation)
q = subprocess.Popen(argz,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(qout, qerr) = q.communicate()
if not ("Test EXIT SUCCESS" in qout
or "Test EXIT SUCCESS" in qerr):
error_text = "Unit test [%s] returned [%s]" % (
new_ut_command_validation, qout)
if "Invalid image filename" in qout or "Invalid vector data filename" in qout or "Failed to open" in qout:
logger.warning(error_text)
else:
logger.error(error_text)
self.fail(error_text)
else:
logger.info(qout)
signal.alarm(0)
self.data[test_name] = [a_tuple[0], a_tuple[1], failed]
else:
logger.info("Passed test: %s" % test_name)
try:
token = os.environ['SLACK_BOT_TOKEN']
except Exception as err:
print("[Error] %s" % (str(err)))
# set repository
repos = (
('jhhwang4195', '', ''),
('jhhwang4195', 'TIL', 'jhhwang'),
('jhhwang4195', 'my_source', 'jhhwang'),
('jhhwang4195', 'my_config', 'jhhwang'),
('jhhwang4195', 'slackbot', 'jhhwang'),
('jhhwang4195', 'commitbot', 'jhhwang'),
('hwauni', '', ''),
('hwauni', 'MachineLearning', 'sylee'),
)
# start thread
t = SlackPost(name="slack_post",
token=token,
channels=['#dailycommit'],
repos=repos)
t.start()
while True:
time.sleep(1)
in_data = raw_input()
if in_data == "exit" or in_data == "quit":
t.signal = False
signal.alarm(1)
def task_execution(logger, # type: ...
process_name, # type: str
module, # type: ...
method_name, # type: str
time_out, # type: int
types, # type: list
values, # type: list
compss_kwargs, # type: dict
persistent_storage, # type: bool
storage_conf # type: str
):
# type: (...) -> (int, list, list, str, bool, str)
""" Task execution function.
:param logger: Logger
:param process_name: Process name
:param module: Module which contains the function
:param method_name: Function to invoke
:param time_out: Time out
:param types: List of the parameter's types
:param values: List of the parameter's values
:param compss_kwargs: PyCOMPSs keywords
:param persistent_storage: If persistent storage is enabled
:param storage_conf: Persistent storage configuration file
:return: exit_code, new_types, new_values, target_direction, timed_out
and return_message
"""
if __debug__:
logger.debug("Starting task execution")
logger.debug("module : %s " % str(module))
logger.debug("method_name: %s " % str(method_name))
logger.debug("time_out : %s " % str(time_out))
logger.debug("Types : %s " % str(types))
logger.debug("Values : %s " % str(values))
logger.debug("P. storage : %s " % str(persistent_storage))
logger.debug("Storage cfg: %s " % str(storage_conf))
try:
# WARNING: the following call will not work if a user decorator
# overrides the return of the task decorator.
# new_types, new_values = getattr(module, method_name)
# (*values, compss_types=types,
# logger=logger, **compss_kwargs)
# If the @task is decorated with a user decorator, may include more
# return values, and consequently, the new_types and new_values will
# be within a tuple at position 0.
# Force users that use decorators on top of @task to return the task
# results first. This is tested with the timeit decorator in test 19.
signal.signal(signal.SIGALRM, task_timed_out)
signal.signal(signal.SIGUSR2, task_cancel)
signal.alarm(time_out)
if persistent_storage:
with storage_task_context(logger, values,
config_file_path=storage_conf):
task_output = getattr(module, method_name)(*values,
compss_types=types,
logger=logger,
**compss_kwargs)
else:
task_output = getattr(module, method_name)(*values,
compss_types=types,
logger=logger,
**compss_kwargs)
except TimeOutError:
logger.exception("TIMEOUT ERROR IN %s - Time Out Exception" %
process_name)
logger.exception("Task has taken too much time to process")
new_values = _get_return_values_for_exception(types, values)
return task_returns(3,
types,
new_values,
None,
True,
"",
logger)
except COMPSsException as compss_exception:
logger.exception("COMPSS EXCEPTION IN %s" % process_name)
return_message = "No message"
if compss_exception.message is not None:
return_message = compss_exception.message
new_values = _get_return_values_for_exception(types, values)
return task_returns(2,
types,
new_values,
None,
False,
return_message,
logger)
except AttributeError:
# Appears with functions that have not been well defined.
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
logger.exception("WORKER EXCEPTION IN %s - Attribute Error Exception" %
process_name)
logger.exception(''.join(line for line in lines))
logger.exception("Check that all parameters have been defined with "
"an absolute import path (even if in the same file)")
# If exception is raised during the task execution, new_types and
# new_values are empty and target_direction is None
return task_returns(1,
[],
[],
None,
False,
"",
logger)
except BaseException: # noqa
# Catch any other user/decorators exception.
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
logger.exception("WORKER EXCEPTION IN %s" % process_name)
logger.exception(''.join(line for line in lines))
# If exception is raised during the task execution, new_types and
# new_values are empty and target_direction is None
return task_returns(1,
[],
[],
None,
False,
"",
logger)
finally:
signal.alarm(0)
signal.signal(signal.SIGUSR2, signal.SIG_IGN)
if isinstance(task_output[0], tuple):
# Weak but effective way to check it without doing inspect that
# another decorator has added another return thing.
# TODO: Should we consider here to create a list with all elements and
# serialize it to a file with the real task output plus the decorator
# results? == task_output[1:]
# TODO: Currently, the extra result is ignored.
new_types = task_output[0][0]
new_values = task_output[0][1]
target_direction = task_output[0][2]
else:
# The task_output is composed by the new_types and new_values returned
# by the task decorator.
new_types = task_output[0]
new_values = task_output[1]
target_direction = task_output[2]
return task_returns(0, new_types, new_values, target_direction,
False, "", logger)
def main():
f1 = open("testfile", 'w')
f2 = open("testfile", 'w')
fcntl.flock(f1, fcntl.LOCK_SH | fcntl.LOCK_NB)
"""
is flock interruptible?
"""
signal.signal(signal.SIGALRM, handler)
signal.alarm(5)
try:
fcntl.flock(f2, fcntl.LOCK_EX)
except IOError as e:
if e.errno != errno.EINTR:
raise
else:
raise RuntimeError("expect flock to block")
fcntl.flock(f1, fcntl.LOCK_UN)
lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 10, 0, 0)
try:
fcntl.fcntl(f1, fcntl.F_OFD_SETLK, lockdata)
except IOError as e:
if e.errno != errno.EINVAL:
raise
else:
print('kernel does not support fcntl.F_OFD_SETLK')
return
lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 10, 10, 0, 0)
fcntl.fcntl(f2, fcntl.F_OFD_SETLK, lockdata)
"""
is posix lock interruptible?
"""
signal.signal(signal.SIGALRM, handler)
signal.alarm(5)
try:
lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0)
fcntl.fcntl(f2, fcntl.F_OFD_SETLKW, lockdata)
except IOError as e:
if e.errno != errno.EINTR:
raise
else:
raise RuntimeError("expect posix lock to block")
"""
file handler 2 should still hold lock on 10~10
"""
try:
lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 10, 10, 0, 0)
fcntl.fcntl(f1, fcntl.F_OFD_SETLK, lockdata)
except IOError as e:
if e.errno == errno.EAGAIN:
pass
else:
raise RuntimeError("expect file handler 2 to hold lock on 10~10")
lockdata = struct.pack('hhllhh', fcntl.F_UNLCK, 0, 0, 0, 0, 0)
fcntl.fcntl(f1, fcntl.F_OFD_SETLK, lockdata)
fcntl.fcntl(f2, fcntl.F_OFD_SETLK, lockdata)
print('ok')
def FPexp_detect(fun_exe,widx_lst,bound,max_iter,num_exp):
# print bound
points_lst = get_testing_point(bound)
ovfps_lst = []
temp_bound = []
for i in bound:
if i == []:
temp_bound.append([-bf.f64max,bf.f64max])
# temp_bound.append([])
else:
temp_bound.append(i)
glob_fitness_fun = lambda x: bdary_fun(fun_exe, bf.reduce_x(temp_bound, x))
# glob_fitness_fun = lambda x: bdary_fun(fun_exe, x)
# minimizer_kwargs = {"method": "Powell"}
# minimizer_kwargs = {"method": "TNC"}
minimizer_kwargs = {"method": "Nelder-Mead"}
# max_iter = widx_lst[4].value
# print max_iter
# print num_exp
# bar = progressbar.ProgressBar(maxval=max_iter, \
# widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage()])
# bar.start()
st = time.time()
temp_points = []
temp_ovfps_lst = []
for i in points_lst:
res = glob_fitness_fun(i)
widx_lst[3].value = 0
if res != 1e100:
temp_points.append(i)
st_flag = 0
for j in range(0, max_iter):
# print j
# print ovfps_lst
for i in temp_points:
# print i
try:
signal.alarm(5)
temp_i = bf.generate_x(temp_bound, i)
# temp_i = i
res = basinhopping(glob_fitness_fun, temp_i, minimizer_kwargs=minimizer_kwargs, niter_success=1)
if res.fun == 0:
try:
inp = list(res.x)
except TypeError:
inp = [res.x]
# print "inp"
# print res.x
# ovfps_lst.append(bf.reduce_x(temp_bound,res.x))
ovfps_lst.append(bf.reduce_x(temp_bound, inp))
break
except TimeoutError:
break
signal.alarm(60)
widx_lst[2][j] = widx_lst[3].value
widx_lst[1].value = j + 1
widx_lst[3].value = 0
et = time.time() - st
ovfps_lst = reduce_lst(ovfps_lst)
if len(temp_ovfps_lst) == len(ovfps_lst):
st_flag = st_flag + 1
else:
st_flag = 0
if st_flag > 30:
break
if len(ovfps_lst)>num_exp:
break
if et > 600:
break
temp_ovfps_lst = ovfps_lst + []
# bar.finish()
temp_lst = []
cont = 0
for i in ovfps_lst:
cont = cont + 1
if i not in ovfps_lst[cont:]:
temp_lst.append(i)
return temp_lst
def request(self, host, handler, request_body, verbose=0):
# issue XML-RPC request
retry_count = 1
while True:
try:
if _python25:
# Noticed this in Hutlab environment (Windows 7 SP1)
# Activestate python 2.5, use the old method
return xmlrpclib.Transport.request(
self, host, handler, request_body, verbose=verbose)
# Follwing implementation not supported in Python <= 2.5
# FIXME: Verify with Python 2.6
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_user_agent(h)
self.send_content(h, request_body)
response = h.getresponse()
if response.status != 200:
raise xmlrpclib.ProtocolError(host + handler, response.status,
response.reason, response.msg.headers)
payload = response.read()
parser, unmarshaller = self.getparser()
parser.feed(payload)
parser.close()
return unmarshaller.close()
except SocketError, e:
if ((_ldtp_windows_env and e[0] == 10061) or \
(not _ldtp_windows_env and (e.errno == 111 or \
e.errno == 146))) \
and 'localhost' in host:
if hasattr(self, 'close'):
# On Windows XP SP3 / Python 2.5, close doesn't exist
self.close()
if retry_count == 1:
retry_count += 1
if not _ldtp_windows_env:
sigusr1 = signal.signal(signal.SIGUSR1, self._handle_signal)
sigalrm = signal.signal(signal.SIGALRM, self._handle_signal)
sigchld = signal.signal(signal.SIGCHLD, self._handle_signal)
self._spawn_daemon()
if _ldtp_windows_env:
time.sleep(5)
else:
signal.alarm(15) # Wait 15 seconds for ldtpd
signal.pause()
# restore signal handlers
signal.alarm(0)
signal.signal(signal.SIGUSR1, sigusr1)
signal.signal(signal.SIGALRM, sigalrm)
signal.signal(signal.SIGCHLD, sigchld)
continue
else:
raise
# else raise exception
raise
except xmlrpclib.Fault, e:
if hasattr(self, 'close'):
self.close()
if e.faultCode == ERROR_CODE:
raise LdtpExecutionError(e.faultString.encode('utf-8'))
else:
raise e
import signal, os
def handler(signum, frame):
print 'Signal handler called with signal', signum
raise IOError("Couldn't open device!")
# Set the signal handler and a 5-second alarm
signal.signal(signal.SIGALRM, handler)
signal.alarm(5)
# This open() may hang indefinitely
fd = os.open('/dev/ttyS0', os.O_RDWR)
signal.alarm(0) # Disable the alarm
def __enter__(self):
signal.signal(signal.SIGALRM, self._exit)
signal.alarm(self.seconds)
def execute(self, emulator=None):
"""
_execute_
"""
scramCommand = self.step.application.setup.scramCommand
scramArch = self.step.application.setup.scramArch
cmsswVersion = self.step.application.setup.cmsswVersion
scramArch = getSingleScramArch(scramArch)
# Are we using emulators again?
if emulator is not None:
return emulator.emulate(self.step, self.job)
overrides = {}
if hasattr(self.step, 'override'):
overrides = self.step.override.dictionary_()
# Set wait to over an hour
waitTime = overrides.get(
'waitTime', 3600 + (self.step.retryDelay * self.step.retryCount))
# hardcode CERN Castor T0_CH_CERN_MSS stageout parameters
castorStageOutParams = {}
castorStageOutParams['command'] = overrides.get('command', "xrdcp")
castorStageOutParams['option'] = overrides.get('option',
"--wma-cerncastor")
castorStageOutParams['phedex-node'] = overrides.get(
'phedex-node', "T2_CH_CERN")
castorStageOutParams['lfn-prefix'] = overrides.get(
'lfn-prefix', "root://castorcms.cern.ch//castor/cern.ch/cms")
# hardcode CERN EOS T2_CH_CERN stageout parameters
eosStageOutParams = {}
eosStageOutParams['command'] = overrides.get('command', "xrdcp")
eosStageOutParams['option'] = overrides.get(
'option', "--wma-disablewriterecovery")
eosStageOutParams['phedex-node'] = overrides.get(
'phedex-node', "T2_CH_CERN")
eosStageOutParams['lfn-prefix'] = overrides.get(
'lfn-prefix', "root://eoscms.cern.ch//eos/cms")
try:
castorStageOutMgr = StageOutMgr(**castorStageOutParams)
eosStageOutMgr = StageOutMgr(**eosStageOutParams)
stageInMgr = StageInMgr()
deleteMgr = DeleteMgr()
except Exception as ex:
msg = "Unable to load StageOut/Delete Impl: %s" % str(ex)
logging.error(msg)
raise WMExecutionFailure(60312, "MgrImplementationError", msg)
# prepare output tar file
taskName = self.report.getTaskName().split('/')[-1]
host = socket.gethostname().split('.')[0]
tarName = '%s-%s-%s-%i-logs.tar' % (
self.report.data.workload, taskName, host, self.job["counter"])
tarLocation = os.path.join(self.stepSpace.location, tarName)
# check if the cmsswVersion supports edmCopyUtil (min CMSSW_8_X)
result = re.match("CMSSW_([0-9]+)_([0-9]+)_([0-9]+).*", cmsswVersion)
useEdmCopyUtil = False
if result:
try:
cmssw_major = int(result.group(1))
if cmssw_major >= 8:
useEdmCopyUtil = True
elif cmssw_major < 8 and scramArch.startswith('slc6_amd64_'):
useEdmCopyUtil = True
logging.warning(
"CMSSW too old to support edmCopyUtil, using CMSSW_10_2_3 instead"
)
cmsswVersion = "CMSSW_10_2_3"
scramArch = "slc6_amd64_gcc700"
except ValueError:
pass
# setup Scram needed to run edmCopyUtil
if useEdmCopyUtil:
scram = Scram(
command=scramCommand,
version=cmsswVersion,
initialise=self.step.application.setup.softwareEnvironment,
directory=self.step.builder.workingDir,
architecture=scramArch,
)
logging.info("Running scram")
try:
projectOutcome = scram.project()
except Exception as ex:
msg = "Exception raised while running scram.\n"
msg += str(ex)
logging.critical("Error running SCRAM")
logging.critical(msg)
raise WMExecutionFailure(50513, "ScramSetupFailure", msg)
if projectOutcome > 0:
msg = scram.diagnostic()
logging.critical("Error running SCRAM")
logging.critical(msg)
raise WMExecutionFailure(50513, "ScramSetupFailure", msg)
runtimeOutcome = scram.runtime()
if runtimeOutcome > 0:
msg = scram.diagnostic()
logging.critical("Error running SCRAM")
logging.critical(msg)
raise WMExecutionFailure(50513, "ScramSetupFailure", msg)
# iterate through input files
localLogs = []
deleteLogArchives = []
if useEdmCopyUtil:
numberOfFilesPerCopy = 10
else:
numberOfFilesPerCopy = 1
for logs in grouper(self.job["input_files"], numberOfFilesPerCopy):
copyCommand = "env X509_USER_PROXY=%s edmCopyUtil" % os.environ.get(
'X509_USER_PROXY', None)
for log in logs:
copyCommand += " %s" % log['lfn']
copyCommand += " %s" % self.step.builder.workingDir
# give up after timeout of 1 minute per input file
signal.signal(signal.SIGALRM, alarmHandler)
signal.alarm(60 * numberOfFilesPerCopy)
filesCopied = False
try:
if useEdmCopyUtil:
logging.info("Running edmCopyUtil")
retval = scram(copyCommand)
if retval == 0:
filesCopied = True
else:
logging.info("Running stageIn")
for log in logs:
fileInfo = {"LFN": log['lfn']}
logArchive = stageInMgr(**fileInfo)
if logArchive:
filesCopied = True
except Alarm:
logging.error(
"Indefinite hang during edmCopyUtil/stageIn of logArchives"
)
except StageOutFailure:
logging.error("Unable to stageIn logArchives")
except Exception:
raise
signal.alarm(0)
if filesCopied:
for log in logs:
localLogs.append(
os.path.join(self.step.builder.workingDir,
os.path.basename(log['lfn'])))
deleteLogArchives.append(log)
self.report.addInputFile(sourceName="logArchives",
lfn=log['lfn'])
else:
logging.error("Unable to copy logArchives to local disk")
if useEdmCopyUtil:
with open('scramOutput.log', 'r') as f:
logging.error("Scram output: %s", f.read())
for log in logs:
self.report.addSkippedFile(log['lfn'], None)
# create tarfile if any logArchive copied in
if localLogs:
tarFile = tarfile.open(tarLocation, 'w:')
for log in localLogs:
path = log.split('/')
tarFile.add(name=log,
arcname=os.path.join(path[-3], path[-2], path[-1]))
os.remove(log)
tarFile.close()
else:
msg = "Unable to copy any logArchives to local disk"
logging.error(msg)
raise WMExecutionFailure(60312, "LogCollectError", msg)
# now staging out the LogCollect tarfile
logging.info("Staging out LogCollect tarfile to Castor and EOS")
now = datetime.datetime.now()
lfn = "/store/logs/prod/%i/%.2i/%s/%s/%s" % (
now.year, now.month, "WMAgent", self.report.data.workload,
os.path.basename(tarLocation))
tarInfo = {'LFN': lfn, 'PFN': tarLocation, 'PNN': None, 'GUID': None}
# perform mandatory stage out to CERN Castor
signal.signal(signal.SIGALRM, alarmHandler)
signal.alarm(waitTime)
try:
castorStageOutMgr(tarInfo)
except Alarm:
msg = "Indefinite hang during stageOut of LogCollect to Castor"
logging.error(msg)
raise WMExecutionFailure(60409, "LogCollectTimeout", msg)
except Exception as ex:
msg = "Unable to stageOut LogCollect to Castor:\n"
msg += str(ex)
logging.error(msg)
raise WMExecutionFailure(60408, "LogCollectStageOutError", msg)
signal.alarm(0)
# add to job report
self.report.addOutputFile(outputModule="LogCollect", aFile=tarInfo)
outputRef = getattr(self.report.data, self.stepName)
outputRef.output.pfn = tarInfo['PFN']
outputRef.output.location = tarInfo['PNN']
outputRef.output.lfn = tarInfo['LFN']
tarInfo = {'LFN': lfn, 'PFN': tarLocation, 'PNN': None, 'GUID': None}
# then, perform best effort stage out to CERN EOS
signal.signal(signal.SIGALRM, alarmHandler)
signal.alarm(waitTime)
try:
eosStageOutMgr(tarInfo)
except Alarm:
logging.error(
"Indefinite hang during stageOut of LogCollect to EOS")
except Exception as ex:
logging.error("Unable to stageOut LogCollect to EOS: %s", ex)
signal.alarm(0)
# we got this far, delete input
for log in deleteLogArchives:
# give up after timeout of 1 minutes
signal.signal(signal.SIGALRM, alarmHandler)
signal.alarm(60)
try:
fileToDelete = {
'LFN': log['lfn'],
'PFN': None,
'PNN': None,
'StageOutCommand': None
}
deleteMgr(fileToDelete=fileToDelete)
except Alarm:
logging.error("Indefinite hang during delete of logArchive")
except Exception as ex:
logging.error("Unable to delete logArchive: %s", ex)
signal.alarm(0)
return
def __enter__(self):
signal.signal(signal.SIGALRM, self.handle_timeout)
signal.alarm(self.seconds)
print('Erro ao importar o gabarito.')
############################################# Correção #############################################
for i in range(numExercicios):
nTestes = len(testes[i]) # Números de testes para o i-ésimo exercício
notaParcial = 0
if nTestes == 0:
# A função não recebe parâmetros
execucaoGabarito = 'saidaGabarito = gabarito.' + exercicios[i] + '()'
exec(execucaoGabarito)
try:
execucaoAluno = 'saidaAluno = aluno.' + exercicios[i] + '()'
signal.alarm(tempoAlarme) # Ativa o alarme
exec(execucaoAluno)
signal.alarm(0) # Cancela o alarme
assert (saidaGabarito == saidaAluno)
notaParcial = 1
except Exception:
None
else:
# A Função possui pelo menos 1 parâmetro
for j in range(nTestes):
if type(testes[i][j]) != type((0, 0)):
# A função possui somente 1 parâmetro
execucaoGabarito = 'saidaGabarito = gabarito.' + exercicios[
i] + '(' + str(testes[i][j]) + ')'
def execute(self):
signal.signal(
signal.SIGALRM,
lambda sig, frame: hard_timeout(self.module, self.want, start)
)
# setup handler before scheduling signal, to eliminate a race
signal.alarm(int(self.want.timeout))
start = datetime.datetime.utcnow()
if self.want.delay:
time.sleep(float(self.want.delay))
end = start + datetime.timedelta(seconds=int(self.want.timeout))
while datetime.datetime.utcnow() < end:
time.sleep(int(self.want.sleep))
try:
# The first test verifies that the REST API is available; this is done
# by repeatedly trying to login to it.
self.client = self._get_client_connection()
if not self.client:
continue
if self._device_is_rebooting():
# Wait for the reboot to happen and then start from the beginning
# of the waiting.
continue
if self._is_mprov_running_on_device():
self._wait_for_module_provisioning()
break
except Exception as ex:
if 'Failed to validate the SSL' in str(ex):
raise F5ModuleError(str(ex))
# The types of exception's we're handling here are "REST API is not
# ready" exceptions.
#
# For example,
#
# Typically caused by device starting up:
#
# icontrol.exceptions.iControlUnexpectedHTTPError: 404 Unexpected Error:
# Not Found for uri: https://localhost:10443/mgmt/tm/sys/
# icontrol.exceptions.iControlUnexpectedHTTPError: 503 Unexpected Error:
# Service Temporarily Unavailable for uri: https://localhost:10443/mgmt/tm/sys/
#
#
# Typically caused by a device being down
#
# requests.exceptions.SSLError: HTTPSConnectionPool(host='localhost', port=10443):
# Max retries exceeded with url: /mgmt/tm/sys/ (Caused by SSLError(
# SSLError("bad handshake: SysCallError(-1, 'Unexpected EOF')",),))
#
#
# Typically caused by device still booting
#
# raise SSLError(e, request=request)\nrequests.exceptions.SSLError:
# HTTPSConnectionPool(host='localhost', port=10443): Max retries
# exceeded with url: /mgmt/shared/authn/login (Caused by
# SSLError(SSLError(\"bad handshake: SysCallError(-1, 'Unexpected EOF')\",),)),
continue
else:
elapsed = datetime.datetime.utcnow() - start
self.module.fail_json(
msg=self.want.msg or "Timeout when waiting for BIG-IP", elapsed=elapsed.seconds
)
elapsed = datetime.datetime.utcnow() - start
self.changes.update({'elapsed': elapsed.seconds})
return False