def _timer(fm, func, title=None):
results = []
begin = time.time()
count = 0
while True:
ostart = os.times()
cstart = time.time()
r = func()
cstop = time.time()
ostop = os.times()
count += 1
a, b = ostart, ostop
results.append((cstop - cstart, b[0] - a[0], b[1]-a[1]))
if cstop - begin > 3 and count >= 100:
break
if cstop - begin > 10 and count >= 3:
break
fm.startitem()
if title:
fm.write('title', '! %s\n', title)
if r:
fm.write('result', '! result: %s\n', r)
m = min(results)
fm.plain('!')
fm.write('wall', ' wall %f', m[0])
fm.write('comb', ' comb %f', m[1] + m[2])
fm.write('user', ' user %f', m[1])
fm.write('sys', ' sys %f', m[2])
fm.write('count', ' (best of %d)', count)
fm.plain('\n')
def _DecodeFile(self, videofile, encodedfile, workdir):
tempyuvfile = os.path.join(workdir,
videofile.basename + 'tempyuvfile.yuv')
if os.path.isfile(tempyuvfile):
print "Removing tempfile before decode:", tempyuvfile
os.unlink(tempyuvfile)
commandline = self.DecodeCommandLine(videofile, encodedfile, tempyuvfile)
print commandline
with open(os.path.devnull, 'r') as nullinput:
subprocess_cpu_start = os.times()[2]
returncode = subprocess.call(commandline, shell=True,
stdin=nullinput)
if returncode:
raise Exception('Decode failed with returncode %d' % returncode)
subprocess_cpu = os.times()[2] - subprocess_cpu_start
print "Decode took %f seconds" % subprocess_cpu
commandline = encoder.Tool("psnr") + " %s %s %d %d 9999" % (
videofile.filename, tempyuvfile, videofile.width,
videofile.height)
print commandline
psnr = subprocess.check_output(commandline, shell=True, stdin=nullinput)
commandline = ['md5sum', tempyuvfile]
md5 = subprocess.check_output(commandline, shell=False)
yuv_md5 = md5.split(' ')[0]
os.unlink(tempyuvfile)
return psnr, subprocess_cpu, yuv_md5
def run(cmd, timeout=None, memory=None, status="/dev/null", stdin=None, stdout=None, stderr=None):
pid = os.fork()
if not pid:
log = open(status, "w")
print >> log, "command: %s" % cmd
print >> log, "host: %s" % socket.gethostname()
wallclock_before = datetime.datetime.now()
print >> log, "started at: %s" % wallclock_before
if timeout:
print >> log, "timeout: %d seconds" % timeout
resource.setrlimit(resource.RLIMIT_CPU, (timeout, timeout))
else:
print >> log, "timeout: none"
if memory:
print >> log, "memory limit: %d MB" % memory
bytes = memory * 1024 * 1024
resource.setrlimit(resource.RLIMIT_AS, (bytes, bytes))
else:
print >> log, "memory limit: none"
log.flush()
redirections = {}
if stdin:
redirections["stdin"] = open(stdin, "r")
if stdout:
redirections["stdout"] = open(stdout, "w")
if stderr:
redirections["stderr"] = open(stderr, "w")
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
times_before = os.times()[2] + os.times()[3]
retcode = subprocess.call(cmd, **redirections)
times_after = os.times()[2] + os.times()[3]
wallclock_after = datetime.datetime.now()
print >> log, "finished at: %s" % wallclock_after
print >> log, "wall-clock time passed: %s" % (wallclock_after - wallclock_before)
print >> log, "CPU time passed: %s seconds" % (times_after - times_before)
print >> log, "return code: %d [%s]" % (retcode, interpret_returncode(retcode))
if stderr:
if os.stat(stderr).st_size == 0:
os.unlink(stderr)
print >> log, "There was no stderr output."
else:
print >> log, "There was stderr output. See %s." % stderr
else:
print >> log, "Warning: stderr was not captured."
log.close()
# Return sucess (0) or failure (1) based on retcode.
os._exit(int(bool(retcode)))
status = os.waitpid(pid, 0)[1]
assert status in [0, 256], status
return status == 0
def postprocess(self):
"""
Cleanup and send back results.
"""
self.logger.debug('ShelxC::postprocess')
try:
#Cleanup files
rm = 'rm -rf junk*'
os.system(rm)
#Send back results
self.results = {}
if self.shelx_results:
self.results.update(self.shelx_results)
if self.results:
self.input.append(self.results)
#print self.results
self.sendBack2(self.input)
#Say strategy is complete.
self.logger.debug('-------------------------------------')
self.logger.debug('RAPD ShelxC complete.')
self.logger.debug('Total elapsed time: ' + str(os.times()[2]) + ' seconds')#Don't know if that is correct time?
self.logger.debug('-------------------------------------')
#Say strategy is complete.
print '\n-------------------------------------'
print 'RAPD ShelxC complete.'
print 'Total elapsed time: ' + str(os.times()[2]) + ' seconds'#Don't know if that is correct time?
print '-------------------------------------'
os._exit(0)
except:
self.logger.exception('**Could not send results to pipe.**')
def iterate_over_new(list, fmt):
"Iterate over list of IDs"
global total_rec
formatted_records = '' # (string-)List of formatted record of an iteration
tbibformat = 0 # time taken up by external call
tbibupload = 0 # time taken up by external call
start_date = task_get_task_param('task_starting_time') # Time at which the record was formatted
tot = len(list)
count = 0
for recID in list:
t1 = os.times()[4]
start_date = time.strftime('%Y-%m-%d %H:%M:%S')
formatted_record = zlib.compress(format_record(recID, fmt, on_the_fly=True))
if run_sql('SELECT id FROM bibfmt WHERE id_bibrec=%s AND format=%s', (recID, fmt)):
run_sql('UPDATE bibfmt SET last_updated=%s, value=%s WHERE id_bibrec=%s AND format=%s', (start_date, formatted_record, recID, fmt))
else:
run_sql('INSERT INTO bibfmt(id_bibrec, format, last_updated, value) VALUES(%s, %s, %s, %s)', (recID, fmt, start_date, formatted_record))
t2 = os.times()[4]
tbibformat += (t2 - t1)
count += 1
if (count % 100) == 0:
write_message(" ... formatted %s records out of %s" % (count, tot))
task_update_progress('Formatted %s out of %s' % (count, tot))
task_sleep_now_if_required(can_stop_too=True)
if (tot % 100) != 0:
write_message(" ... formatted %s records out of %s" % (count, tot))
return (tot, tbibformat, tbibupload)
def time_it(tb):
start = os.times()
tb.run()
stop = os.times()
delta = map((lambda a, b: a-b), stop, start)
user, sys, childrens_user, childrens_sys, real = delta
total_user = user + childrens_user
total_sys = sys + childrens_sys
res = dict()
res = {'pipes': tb.npipes, 'stages': tb.nstages, 'nsamples': tb.nsamples,
'realtime': real, 'usertime': total_user,
'systime': total_sys,
'totaltime': (total_user+total_sys)/real,
'pseudo_flop': tb.flop,
'pseudoflopreal': tb.flop/real}
if tb.machine_readable:
# print "%3d %3d %.3e %7.3f %7.3f %7.3f %7.3f %.6e %.3e" % (
# tb.npipes, tb.nstages, tb.nsamples, real, total_user, total_sys, (total_user+total_sys)/real, tb.flop, tb.flop/real)
insert_results(tb.conn, res)
else:
print "npipes %7d" % (tb.npipes,)
print "nstages %7d" % (tb.nstages,)
print "nsamples %s" % (eng_notation.num_to_str(tb.nsamples),)
print "real %7.3f" % (real,)
print "user %7.3f" % (total_user,)
print "sys %7.3f" % (total_sys,)
print "(user+sys)/real %7.3f" % ((total_user + total_sys)/real,)
print "pseudo_flop %s" % (eng_notation.num_to_str(tb.flop),)
print "pseudo_flop/real %s" % (eng_notation.num_to_str(tb.flop/real),)
def stageIn(job, jobSite, analJob, pilot_initdir, pworkdir):
""" perform the stage-in """
ec = 0
statusPFCTurl = None
usedFAXandDirectIO = False
# prepare the input files (remove non-valid names) if there are any
ins, job.filesizeIn, job.checksumIn = RunJobUtilities.prepareInFiles(job.inFiles, job.filesizeIn, job.checksumIn)
if ins:
tolog("Preparing for get command")
# get the file access info (only useCT is needed here)
useCT, oldPrefix, newPrefix, useFileStager, directIn = getFileAccessInfo()
# transfer input files
tin_0 = os.times()
ec, job.pilotErrorDiag, statusPFCTurl, job.filesWithoutFAX, job.filesWithFAX, usedFAXandDirectIO = \
mover.get_data(job, jobSite, ins, stageinretry, analysisJob=analJob, usect=useCT,\
pinitdir=pilot_initdir, proxycheck=False, inputDir=inputDir, workDir=pworkdir)
if ec != 0:
job.result[2] = ec
tin_1 = os.times()
job.timeStageIn = int(round(tin_1[4] - tin_0[4]))
return job, ins, statusPFCTurl, usedFAXandDirectIO
def test_scan_int(dev, apdev):
"""scan interval configuration"""
try:
if "FAIL" not in dev[0].request("SCAN_INTERVAL -1"):
raise Exception("Accepted invalid scan interval")
if "OK" not in dev[0].request("SCAN_INTERVAL 1"):
raise Exception("Failed to set scan interval")
dev[0].connect("not-used", key_mgmt="NONE", scan_freq="2412",
wait_connect=False)
times = {}
for i in range(0, 3):
logger.info("Waiting for scan to start")
start = os.times()[4]
ev = dev[0].wait_event(["CTRL-EVENT-SCAN-STARTED"], timeout=5)
if ev is None:
raise Exception("did not start a scan")
stop = os.times()[4]
times[i] = stop - start
logger.info("Waiting for scan to complete")
ev = dev[0].wait_event(["CTRL-EVENT-SCAN-RESULTS"], 10)
if ev is None:
raise Exception("did not complete a scan")
logger.info("times=" + str(times))
if times[0] > 1 or times[1] < 0.5 or times[1] > 1.5 or times[2] < 0.5 or times[2] > 1.5:
raise Exception("Unexpected scan timing: " + str(times))
finally:
dev[0].request("SCAN_INTERVAL 5")
def handler(self, **args):
"""Reports CPU time usage for the current thread."""
from os import times
from irclib import Event
target = args["channel"]
if args["type"] == "privmsg":
from irclib import nm_to_n
target = nm_to_n(args["source"])
return Event(
"privmsg",
"",
target,
[
"User time: "
+ str(times()[0])
+ " seconds, "
+ "system time: "
+ str(times()[1])
+ " seconds. "
+ "Childrens' user time: "
+ str(times()[2])
+ ", "
+ "childrens' system time: "
+ str(times()[3])
],
)
def startup():
global name
servisnew = 0
try:
meta = open("metadata.fue", "r")
except IOError as e:
print "NEW SERVER"
servisnew = 1
name = raw_input("Please enter a name for the server: \n")
meta = open("metadata.fue", "w")
meta.writelines(name + "\n")
meta.writelines(str(os.times()) + "\n")
meta.writelines(os.name + "\n")
meta.writelines("/////////////")
meta.close()
if servisnew == 0:
meta = open("metadata.fue", "r+")
oldmetaV = meta.readline()
endname = oldmetaV.find("\n")
name = oldmetaV[0:endname]
meta.writelines("\n" + str(os.times()) + "\n")
meta.writelines(os.name + "\n")
meta.writelines("/////////////")
try:
log = open("log.fue", "r")
except IOError as e:
log = open("log.fue", "w")
log.close()
isnew = 1
def test_autoscan_exponential(dev, apdev):
"""autoscan_exponential"""
hostapd.add_ap(apdev[0], {"ssid": "autoscan"})
try:
if "OK" not in dev[0].request("AUTOSCAN exponential:2:10"):
raise Exception("Failed to set autoscan")
dev[0].connect("not-used", key_mgmt="NONE", scan_freq="2412",
wait_connect=False)
times = {}
for i in range(0, 3):
logger.info("Waiting for scan to start")
start = os.times()[4]
ev = dev[0].wait_event(["CTRL-EVENT-SCAN-STARTED"], timeout=5)
if ev is None:
raise Exception("did not start a scan")
stop = os.times()[4]
times[i] = stop - start
logger.info("Waiting for scan to complete")
ev = dev[0].wait_event(["CTRL-EVENT-SCAN-RESULTS"], 10)
if ev is None:
raise Exception("did not complete a scan")
if times[0] > 1 or times[1] < 1 or times[1] > 3 or times[2] < 3 or times[2] > 5:
raise Exception("Unexpected scan timing: " + str(times))
finally:
dev[0].request("AUTOSCAN ")
def main():
context = zmq.Context()
socket = context.socket(zmq.REP)
socket.bind("tcp://*:36000")
print("Listening on port 36000...");
start_time = None
end_time = None
assignment_count = 0
while not end_time:
msg = socket.recv()
i = msg.find('0x')
j = msg.find('"', i)
code = eval(msg[i:j])
if code == NITRO_REQUEST_HELP:
print("\nGot enroll request. Responded with AFFIRM. Here we go...\n")
socket.send('{"body":{"code":"0x%08X"}}' % NITRO_AFFIRM_HELP)
start_time = os.times()[4]
elif code == NITRO_HERE_IS_ASSIGNMENT:
i = msg.find('"fake')
j = msg.find('"', i + 1)
cmdline = msg[i+1:j]
assignment_count += 1
if (assignment_count % 80 == 0) or (assignment_count == 1000):
print("\nGot assignment: %s" % cmdline)
else:
sys.stdout.write('.')
socket.send("ACK")
elif code == NITRO_TERMINATE_REQUEST:
end_time = os.times()[4]
print("\nBatch ended. %d launch requests. Elapsed time: %.2f secs.\n" % (assignment_count, end_time - start_time))
socket.send("ACK")
else:
print(msg)
socket.send("ACK")
def timer(os_command):
t0 = os.times()
os.system(os_command)
t1 = os.times()
elapsed_time = t1[4] - t0[4]
cpu_time_child = t1[2]-t0[2] + t1[3]-t0[3]
return cpu_time_child, elapsed_time
def iterate_over_new(recIDs, fmt):
"""
Iterate over list of IDs
@param list: the list of record IDs to format
@param fmt: the output format to use
@return: tuple (total number of records, time taken to format, time taken to insert)
"""
tbibformat = 0 # time taken up by external call
tbibupload = 0 # time taken up by external call
tot = len(recIDs)
for count, recID in enumerate(recIDs):
t1 = os.times()[4]
formatted_record, needs_2nd_pass = format_record_1st_pass(recID=recID,
of=fmt,
on_the_fly=True,
save_missing=False)
save_preformatted_record(recID=recID,
of=fmt,
res=formatted_record,
needs_2nd_pass=needs_2nd_pass,
low_priority=True)
t2 = os.times()[4]
tbibformat += t2 - t1
if count % 100 == 0:
write_message(" ... formatted %s records out of %s" % (count, tot))
task_update_progress('Formatted %s out of %s' % (count, tot))
task_sleep_now_if_required(can_stop_too=True)
if tot % 100 != 0:
write_message(" ... formatted %s records out of %s" % (tot, tot))
return tot, tbibformat, tbibupload
def optimise_with_merge(self, assignment, update=True, **kwargs):
new_assignment = self.optimise(assignment, **kwargs)
print 'Partition after {0} merges at {1}:\n{2}'.format(self.merges,
sum(os.times()[:4]), new_assignment)
opt_score = self.Scorer.score(new_assignment)
print 'Score: {0}'.format(opt_score)
self.merges += 1
split = self.split_search(new_assignment)
split = self.optimise(split, history=False, max_iter=10, **kwargs)
print 'Partition after {0} splits at {1}:\n{2}'.format(self.merges,
sum(os.times()[:4]), new_assignment)
print 'Score: {0}'.format(self.Scorer.score(split))
merged = self.merge_closest(split)
merged_score = self.Scorer.score(merged)
print 'Partition after {0} merges at {1}:\n{2}'.format(self.merges,
sum(os.times()[:4]), new_assignment)
print 'Score: {0}'.format(merged_score)
if np.abs(merged_score - opt_score) > EPS:
merged = self.optimise_with_merge(merged, **kwargs)
else:
if update is True:
self.update(merged)
return(merged)
def run(self, program, options='', inputfile=''):
"""Run program, store output on logfile."""
# the logfile is always opened in the constructor so
# we can safely append here
vfile = open(self.logfile, 'a')
vfile.write('\n#### Test: %s' % (self.scriptfilename))
vfile.write(' running %(program)s %(options)s\n' % vars())
vfile.close()
# do not use time.clock() to measure CPU time; it will not
# notice the CPU time(here waiting time) of a system command
t0 = os.times() # [user,system,cuser,csystem,elapsed]
if inputfile == '':
cmd = '%s %s >> %s' % (program,options,self.logfile)
else:
cmd = '%s %s < %s >> %s' % (program,options,inputfile,self.logfile)
failure, output = os_system(cmd, failure_handling='silent')
if failure:
vfile = open(self.logfile, 'a')
msg = 'ERROR in %s: execution failure arose from the ' \
'command\n %s\n\n%s\n\n' % (self.scriptfilename,cmd,output)
vfile.write(msg)
vfile.close()
print msg
# write CPU time of system command(user+system time
# of child processes):
t1 = os.times(); tm = t1[2] - t0[2] + t1[3] - t0[3]
vfile = open(self.logfile, 'a')
vfile.write('CPU time of %(program)s: %(tm).1f seconds' % vars())
#if re.search('(x|sun)',sys.platform):
if os.name == 'posix': # is 'posix', 'nt' or 'mac'
# unix
u = os.uname()
vfile.write(' on %s %s, %s' % (u[1],u[4],u[0]))
vfile.write('\n\n')
def run (args) :
if 'net' not in args : usage ("Parameter 'net' mandatory")
out = Trt.subs_ext (args['net'], '.ll_net', Dllola.EXT)
cmd = ['check', 'pep:lola-dl', args['net']]
t = os.times () [2]
(c, s) = runit (cmd, args['timeout'])
t = os.times () [2] - t
c = str (c)
res = Trt.init_res (Dllola)
res['test'] = out
res['stat'] = c
res['stdout'] = s
res['time'] = t
if c != '0' : return res
for l in s.splitlines () :
if 'Time needed: ' in l :
t = l.split ()
if len (t) >= 3 : res['needed'] = float (t[2])
elif 'Result: NO.' == l :
res['result'] = 'DEAD'
elif 'Result: YES.' == l :
res['result'] = 'LIVE'
res['seq'] = 'na'
elif 'Counterexample: ' in l :
res['seq'] = ' '.join (t.strip () for t in l[16:].split (','))
elif 'check: unclassified error in LoLA' == l :
res['stat'] = 'error'
return res
def sort_and_format(hits, kwargs):
kwargs = search_engine.prs_wash_arguments(**kwargs)
t1 = os.times()[4]
req = ReqStringIO() # new ver of Invenio is looking at args
kwargs['req'] = req
if 'hosted_colls_actual_or_potential_results_p' not in kwargs:
kwargs['hosted_colls_actual_or_potential_results_p'] = True # this prevents display of the nearest-term box
# search stage 4 and 5: intersection with collection universe and sorting/limiting
output = search_engine.prs_intersect_with_colls_and_apply_search_limits(hits, kwargs=kwargs, **kwargs)
if output is not None:
req.seek(0)
return req.read() + output
t2 = os.times()[4]
cpu_time = t2 - t1
kwargs['cpu_time'] = cpu_time
recids = search_engine.prs_rank_results(kwargs=kwargs, **kwargs)
if 'of' in kwargs and kwargs['of'].startswith('hc'):
output = citation_summary(intbitset(recids), kwargs['of'], kwargs['ln'], kwargs['p'], kwargs['f'])
if output:
return output
return recids
def mastermonitor_loop(self):
'''
The method test periodically the state of the ROS master. The new state will
be published as 'state_signal'.
'''
import os
current_check_hz = OwnMasterMonitoring.ROSMASTER_HZ
while (not rospy.is_shutdown()):
try:
if not self._do_pause:
cputimes = os.times()
cputime_init = cputimes[0] + cputimes[1]
if self._master_monitor.checkState():
mon_state = self._master_monitor.getState()
# publish the new state
state = MasterState(MasterState.STATE_CHANGED,
ROSMaster(str(self._local_addr),
str(self._masteruri),
mon_state.timestamp,
True,
rospy.get_name(),
''.join(['http://', str(self._local_addr),':',str(self._master_monitor.rpcport)])))
self.state_signal.emit(state)
# adapt the check rate to the CPU usage time
cputimes = os.times()
cputime = cputimes[0] + cputimes[1] - cputime_init
if current_check_hz*cputime > 0.4:
current_check_hz = float(current_check_hz)/2.0
elif current_check_hz*cputime < 0.20 and current_check_hz < OwnMasterMonitoring.ROSMASTER_HZ:
current_check_hz = float(current_check_hz)*2.0
except MasterConnectionException, e:
rospy.logwarn("Error while master check loop: %s", str(e))
except RuntimeError, e:
# will thrown on exit of the app while try to emit the signal
rospy.logwarn("Error while master check loop: %s", str(e))
def _load_modules(self, modules_to_load):
str_p = json.dumps(modules_to_load)
self.modules = []
for mod in modules_to_load:
mod_name = mod['name']
mod_params = mod['params']
mod_params['dbms'] = self.db
mod_params['drone'] = self.drone
mod_params['location'] = self.location
mod_params['run_id'] = self.run_id
mod_params['key'] = self.key
mod_params['plugs'] = str_p
mod_params['verbose'] = self.verbose
m = __import__(mod_name, fromlist="Snoop").Snoop(**mod_params)
m.setName(mod_name[8:])
self.modules.append(m)
#Start modules
#m.start()
mod_start_time = os.times()[4] #Get a system clock indepdent timer
tmp_mod_name = mod_name[8:]
if mod_name != 'plugins.run_log':
logging.info("Waiting for plugin '%s' to indicate it's ready" % tmp_mod_name)
m.start()
while not m.is_ready() and abs(os.times()[4] - mod_start_time) < self.MODULE_START_GRACE_TIME:
time.sleep(2)
if not m.is_ready():
logging.info("Plugin '%s' ran out of time to indicate its ready state, moving on to next plugin." % tmp_mod_name)
else:
if mod_name != 'plugins.run_log':
logging.info("Plugin '%s' has indicated it's ready." % tmp_mod_name)
logging.info("Done loading plugins, running...")
def main(self, *args, **kwds):
if self.inventory.dumpregistry:
self._dumpRegsitry()
return
import mcni
instrument = self._createInstrument()
geometer = self.geometer
context = self._makeSimContext()
if self.ncount < self.buffer_size:
self.buffer_size = int(self.ncount)
n = int(self.ncount / self.buffer_size)
from mcni import journal
logger = journal.logger(
'info', 'instrument', header='', footer='', format='-> %s')
for i in range(n):
logger("mpi node %s at loop %s" % (self.mpi.rank, i))
neutrons = mcni.neutron_buffer( self.buffer_size )
context.iteration_no = i
mcni.simulate( instrument, geometer, neutrons, context=context)
continue
remain = int(self.ncount % self.buffer_size)
if remain:
logger("mpi node %s at last loop" % (self.mpi.rank,))
neutrons = mcni.neutron_buffer(remain)
context.iteration_no = n
mcni.simulate( instrument, geometer, neutrons, context=context)
print os.times()
return
def run(args):
if (len(args) == 0):
n = 1000
else:
assert len(args) == 1
n = int(args[0])
#
labels = [
"gcd_int_boost",
"gcd_int_simple",
"gcd_int32_asm",
"gcd_long_boost",
"gcd_long_simple",
"gcd_unsigned_long_binary",
"gcd_long_binary",
"gcd_int64_asm"]
#
for label in labels:
compare_with_boost_rational_gcd(label=label)
#
impls = [getattr(scitbx.math, "time_%s" % label, None)
for label in labels]
for label,impl in zip(labels,impls):
if (impl is None): continue
w0 = time.time()
us0 = sum(os.times()[:2])
result = impl(n)
print "%-24s %d w=%.2f u+s=%.2f" % (
label,
result,
time.time()-w0,
sum(os.times()[:2])-us0)
print "OK"
def work(self):
# Sample CPU usage now
(prev_u, prev_s, prev_cu, prev_cs, prev_e) = os.times()
prev_clock = int(time.time())
time.sleep(self.sample_rate) # Wait 'sample_rate' seconds
clock = int(time.time()) # Sample again to compute utilization
(u, s, cu, cs, e) = os.times()
user = u - prev_u # Compute CPU utilization
system = s - prev_s
elapsed = e - prev_e
cpu_time = user + system
cpu_util = (cpu_time / elapsed) * 100
cpu_util_user = (user / elapsed) * 100
cpu_util_system = (system / elapsed) * 100
self.perf_log.append(( # Add to performance log
clock,
cpu_util,
cpu_util_user,
cpu_util_system
))
def test_runThreadsUsesNonBusyWaitingMode(self):
"""If run using the runThreads method, then the scheduler may/will block for short periods, relinquishing processor time, if all microprocesses are paused."""
# we'll measure CPU time used over 3 seconds and expect it to be <1% of 3 seconds
seconds = 3.0
s = Axon.Scheduler.scheduler()
mps = [TestPausedMProc(s) for _ in range(0,5)]
for mp in mps:
s._addThread(mp)
import os,time
import threading
def causeCompletion():
for mp in mps:
s.wakeThread(mp)
timer = threading.Timer(seconds,causeCompletion)
timer.start()
starttime = os.times()[1]
s.runThreads()
endtime = os.times()[2]
self.assert_(endtime-starttime <= 0.01*seconds, "Time consumed should have been <1% of CPU time")
def read( self, duration ):
start = os.times()[4]
lasttime = start
self.lpo1.start()
self.lpo2.start()
while os.times()[4]<start+duration:
time.sleep(1.0/1000.0)
self.gpio1.seek(0)
value = int(self.gpio1.read())
self.lpo1.read(value)
self.gpio2.seek(0)
value = int(self.gpio2.read())
self.lpo2.read(value)
ctime = os.times()[4]
if ctime>lasttime+6:
lasttime = ctime
ratio1 = round(self.lpo1.ratio()*10000)/100
ratio2 = round(self.lpo2.ratio()*10000)/100
print ("{0}: {1}({3}) {2}({4})".format(time.strftime("%H:%M:%S"),ratio1,ratio2,self.lpo1.getcount(),self.lpo2.getcount()))
return [self.lpo1.ratio(),self.lpo2.ratio()]
def run(self, benchfile):
if self.toolcmd is not None:
link = '/tmp/foo.smt2'
logging.debug("Created " + link)
os.symlink(benchfile, link)
cmd = "{0} {1}".format(self.toolcmd, link)
# logging.debug("{0}".format(cmd))
self.total += 1
self.last = benchfile
tinit = os.times()
sp = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
try:
if args.timeout is None:
output, errs = sp.communicate()
else:
output, errs = sp.communicate(timeout=int(args.timeout))
tend = os.times()
time = tend.children_user - tinit.children_user + tend.children_system - tinit.children_system
except subprocess.TimeoutExpired:
sp.kill()
output, errs = sp.communicate()
self.timeouts.insert(0, benchfile)
time = None
os.unlink(link)
logging.debug("{}, {}".format(sp.returncode, time))
self.results[benchfile] = dict()
self.results[benchfile]["output"] = output
self.results[benchfile]["errs"] = errs
self.results[benchfile]["ret"] = sp.returncode
self.results[benchfile]["time"] = time
try:
self.restypes[sp.returncode] += 1
except KeyError:
self.restypes[sp.returncode] = 1
def iterate_over_new(list, fmt):
"""
Iterate over list of IDs
@param list: the list of record IDs to format
@param fmt: the output format to use
@return: tuple (total number of records, time taken to format, time taken to insert)
"""
global total_rec
formatted_records = '' # (string-)List of formatted record of an iteration
tbibformat = 0 # time taken up by external call
tbibupload = 0 # time taken up by external call
start_date = task_get_task_param('task_starting_time') # Time at which the record was formatted
tot = len(list)
count = 0
for recID in list:
t1 = os.times()[4]
start_date = time.strftime('%Y-%m-%d %H:%M:%S')
formatted_record = zlib.compress(format_record(recID, fmt, on_the_fly=True))
run_sql('REPLACE LOW_PRIORITY INTO bibfmt (id_bibrec, format, last_updated, value) VALUES (%s, %s, %s, %s)',
(recID, fmt, start_date, formatted_record))
t2 = os.times()[4]
tbibformat += (t2 - t1)
count += 1
if (count % 100) == 0:
write_message(" ... formatted %s records out of %s" % (count, tot))
task_update_progress('Formatted %s out of %s' % (count, tot))
task_sleep_now_if_required(can_stop_too=True)
if (tot % 100) != 0:
write_message(" ... formatted %s records out of %s" % (count, tot))
return (tot, tbibformat, tbibupload)
def run(self):
startTimeReal = time.time()
startTimeUser = os.times()[0]
startTimeSys = os.times()[1]
self.longdata = ""
self.population = []
for i in range(0,self.populationSize):
self.population.append(timetable(self.problem))
self.population[i].setupRandom()
self.population[i].objective()
self.bestValue = 10000
self.best = None
self.bestcount = 0
self.genNum = 0
self.STOPCRIT = False
print "Running"
while not self.STOPCRIT:
self.bestcount += 1
self.generation()
if self.genNum == 1:
self.startState = self.bestValue
if self.bestcount > 10:
return self.bestValue, time.time() - startTimeReal, (os.times()[0] - startTimeUser) + (os.times()[1] - startTimeSys), self.genNum, self.longdata, self.startState, self.best.printMatrix()
def iterate_over_new(recIDs, fmt):
"""Iterate over list of IDs.
@param list: the list of record IDs to format
@param fmt: the output format to use
@return: tuple (total number of records, time taken to format, time taken
to insert)
"""
tbibformat = 0 # time taken up by external call
tbibupload = 0 # time taken up by external call
tot = len(recIDs)
reformat_function = _CFG_BIBFORMAT_UPDATE_FORMAT_FUNCTIONS.get(
fmt.lower(), _update_format)
for count, recID in enumerate(recIDs):
t1 = os.times()[4]
reformat_function(recID, fmt)
t2 = os.times()[4]
tbibformat += t2 - t1
if count % 100 == 0:
write_message(" ... formatted %s records out of %s" %
(count, tot))
task_update_progress('Formatted %s out of %s' % (count, tot))
task_sleep_now_if_required(can_stop_too=True)
if tot % 100 != 0:
write_message(" ... formatted %s records out of %s" % (tot, tot))
return tot, tbibformat, tbibupload
def retrieve_segmentation(self, feat_matrix):
'''
Uses the features of the feat_matrix to retrieve a segmentation of the currently visible slice
:param feat_matrix: boolean feature matrix as in opFeatureSelection.SelectionMatrix
:return: segmentation (2d numpy array), out of bag error
'''
# remember the currently selected features so that they are not changed in case the user cancels the dialog
user_defined_matrix = self.opFeatureSelection.SelectionMatrix.value
# apply new feature matrix and make sure lazyflow applies the changes
if numpy.sum(user_defined_matrix != feat_matrix) != 0:
self.opFeatureSelection.SelectionMatrix.setValue(feat_matrix)
self.opFeatureSelection.SelectionMatrix.setDirty() # this does not do anything!?!?
self.opFeatureSelection.setupOutputs()
# self.opFeatureSelection.change_feature_cache_size()
start_time = times()[4]
old_freeze_prediction_value = self.opPixelClassification.FreezePredictions.value
self.opPixelClassification.FreezePredictions.setValue(False)
# retrieve segmentation layer(s)
slice_shape = self.raw_xy_slice.shape[:2]
segmentation = numpy.zeros(slice_shape, dtype=numpy.uint8)
axisOrder = [ tag.key for tag in self.opFeatureSelection.InputImage.meta.axistags ]
bbox = self._bbox
do_transpose = axisOrder.index('x') > axisOrder.index('y')
# we need to reset the 'c' axis because it only has length 1 for segmentation
if 'c' not in list(bbox.keys()):
axisOrder += ['c']
bbox['c'] = [0, 1]
total_slicing = [slice(bbox[ai][0], bbox[ai][1]) for ai in axisOrder]
# combine segmentation layers
for i, seglayer in enumerate(self.opPixelClassification.SegmentationChannels):
single_layer_of_segmentation = numpy.squeeze(seglayer[total_slicing].wait())
if do_transpose:
single_layer_of_segmentation = single_layer_of_segmentation.transpose()
segmentation[single_layer_of_segmentation != 0] = i
end_time = times()[4]
oob_err = 100. * numpy.mean(self.opPixelClassification.opTrain.outputs['Classifier'].value.oobs)
# revert changes to matrix and other operators
if numpy.sum(user_defined_matrix != feat_matrix) != 0:
self.opFeatureSelection.SelectionMatrix.setValue(user_defined_matrix)
self.opFeatureSelection.SelectionMatrix.setDirty() # this does not do anything!?!?
self.opFeatureSelection.setupOutputs()
self.opPixelClassification.FreezePredictions.setValue(old_freeze_prediction_value)
return segmentation, oob_err, end_time-start_time
def execute(self, step_exec_attr, definitions):
""" Step execution method. step_exec_attr is array to hold parameters belong to this Step,
filled above in the workflow
"""
print 'Executing StepInstance', self.getName(
), 'of type', self.getType(), definitions.keys()
# Report the Application state if the corresponding tool is supplied
if 'JobReport' in self.workflow_commons:
if self.parent.workflowStatus['OK']:
self.workflow_commons['JobReport'].setApplicationStatus(
'Executing ' + self.getName())
# Prepare Step statistics evaluation
self.step_commons['StartTime'] = time.time()
self.step_commons['StartStats'] = os.times()
step_def = definitions[self.getType()]
step_exec_modules = {}
error_message = ''
for mod_inst in step_def.module_instances:
mod_inst_name = mod_inst.getName()
mod_inst_type = mod_inst.getType()
# print "StepInstance creating module instance ", mod_inst_name, " of type", mod_inst.getType()
step_exec_modules[mod_inst_name] = \
step_def.parent.module_definitions[mod_inst_type].main_class_obj() # creating instance
# Resolve all the linked parameter values
for parameter in mod_inst.parameters:
if parameter.preExecute():
# print '>>>> Input', parameter
if parameter.isLinked():
# print ">>>> ModuleInstance", mod_inst_name + '.' + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter()
if parameter.getLinkedModule() == 'self':
# tale value form the step_dict
setattr(
step_exec_modules[mod_inst_name],
parameter.getName(),
step_exec_attr[parameter.getLinkedParameter()])
else:
setattr(
step_exec_modules[mod_inst_name],
parameter.getName(),
getattr(
step_exec_modules[
parameter.getLinkedModule()],
parameter.getLinkedParameter()))
else:
# print ">>>> ModuleInstance", mod_inst_name + '.' + parameter.getName(), '=', parameter.getValue()
setattr(step_exec_modules[mod_inst_name],
parameter.getName(), parameter.getValue())
# print 'Step Input Parameter:', parameter.getName(), getattr( step_exec_modules[mod_inst_name], parameter.getName() )
# Set reference to the workflow and step common tools
setattr(step_exec_modules[mod_inst_name], 'workflow_commons',
self.parent.workflow_commons)
setattr(step_exec_modules[mod_inst_name], 'step_commons',
self.step_commons)
setattr(step_exec_modules[mod_inst_name], 'stepStatus',
self.stepStatus)
setattr(step_exec_modules[mod_inst_name], 'workflowStatus',
self.parent.workflowStatus)
try:
result = step_exec_modules[mod_inst_name].execute()
if not result['OK']:
if self.stepStatus['OK']:
error_message = result['Message']
if 'JobReport' in self.workflow_commons:
if self.parent.workflowStatus['OK']:
self.workflow_commons[
'JobReport'].setApplicationStatus(
error_message)
self.stepStatus = S_ERROR(result['Message'])
else:
for parameter in mod_inst.parameters:
if parameter.isOutput():
# print '<<<< Output', parameter
if parameter.isLinked():
# print "ModuleInstance self ." + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter()
if parameter.getLinkedModule() == 'self':
# this is not supposed to happen
print "Warning! Module OUTPUT attribute", parameter.getName(
),
print "refer to the attribute of the same module", parameter.getLinkedParameter(
), '=', getattr(
step_exec_modules[mod_inst_name],
parameter.getName())
step_exec_attr[
parameter.getName()] = getattr(
step_exec_modules[mod_inst_name],
parameter.getLinkedParameter(),
parameter.getValue())
# print " OUT", parameter.getLinkedParameter(), '=', getattr( step_exec_modules[mod_inst_name], parameter.getName(), parameter.getValue() )
else:
# print 'Output step_exec_attr', st_parameter.getName(), step_exec_modules[st_parameter.getLinkedModule()], parameter.getLinkedParameter()
step_exec_attr[parameter.getName()] = \
getattr( step_exec_modules[parameter.getLinkedModule()],
parameter.getLinkedParameter() )
else:
# This also does not make sense - we can give a warning
print "Warning! Module OUTPUT attribute ", parameter.getName(
),
print "assigned constant", parameter.getValue()
# print "StepInstance self." + parameter.getName(), '=', parameter.getValue()
step_exec_attr[parameter.getName(
)] = parameter.getValue()
# print 'Module Output Parameter:', parameter.getName(), step_exec_attr[parameter.getName()]
# Get output values to the step_commons dictionary
for key in result.keys():
if key != "OK":
if key != "Value":
self.step_commons[key] = result[key]
elif isinstance(result['Value'], dict):
for vkey in result['Value'].keys():
self.step_commons[vkey] = result['Value'][
vkey]
except Exception as x:
print "Exception while module execution"
print "Module", mod_inst_name, mod_inst.getType()
print str(x)
exc = sys.exc_info()
exc_type = exc[0]
value = exc[1]
print "== EXCEPTION ==\n%s: %s\n\n%s===============" % (
exc_type, value, "\n".join(traceback.format_tb(exc[2])))
print "Step status: ", self.stepStatus
print "Workflow status: ", self.parent.workflowStatus
if self.stepStatus['OK']:
# This is the error that caused the workflow disruption
# report it to the WMS
error_message = 'Exception while %s module execution: %s' % (
mod_inst_name, str(x))
if 'JobReport' in self.workflow_commons:
if self.parent.workflowStatus['OK']:
self.workflow_commons[
'JobReport'].setApplicationStatus(
'Exception in %s module' % mod_inst_name)
self.stepStatus = S_ERROR(error_message)
# now we need to copy output values to the STEP!!! parameters
for st_parameter in self.parameters:
if st_parameter.isOutput():
# print '<< Output', st_parameter
if st_parameter.isLinked():
# print "StepInstance self." + st_parameter.getName(), '=', st_parameter.getLinkedModule() + '.' + st_parameter.getLinkedParameter()
if st_parameter.getLinkedModule() == 'self':
# this is not supposed to happen
print "Warning! Step OUTPUT attribute", st_parameter.getName(
),
print "refer to the attribute of the same step", st_parameter.getLinkedParameter(
), step_exec_attr[st_parameter.getLinkedParameter()]
step_exec_attr[st_parameter.getName(
)] = step_exec_attr[st_parameter.getLinkedParameter()]
else:
# print 'Output step_exec_attr', st_parameter.getName(), step_exec_modules[st_parameter.getLinkedModule()], st_parameter.getLinkedParameter()
step_exec_attr[st_parameter.getName()] = \
getattr( step_exec_modules[st_parameter.getLinkedModule()],
st_parameter.getLinkedParameter() )
setattr(self, st_parameter.getName(),
step_exec_attr[st_parameter.getName()])
else:
# This also does not make sense - we can give a warning
print "Warning! Step OUTPUT attribute ", st_parameter.getName(
),
print "assigned constant", st_parameter.getValue()
# print "StepInstance self." + st_parameter.getName(), '=', st_parameter.getValue()
step_exec_attr[
st_parameter.getName()] = st_parameter.getValue()
print 'Step Output', st_parameter.getName(
), '=', step_exec_attr[st_parameter.getName()]
# Return the result of the first failed module or S_OK
if not self.stepStatus['OK']:
return S_ERROR(error_message)
else:
return S_OK(result['Value'])
def fast_rnn_matrix_completion(gram_drop, seqs,
rnn,
rnn_units,
dense_units,
dropout,
implementation,
bidirectional,
batchnormalization,
loss_function,
siamese_arms_activation,
siamese_joint_method,
trained_modelfile_hdf5=None):
"""Fill in Gram matrix with dropped elements with Keras Siamese RNN.
Trains the network on given part of Gram matrix and the corresponding sequences
Fills in missing elements by network prediction
:param gram_drop: Gram matrix with dropped elements
:param seqs: List of time series
:param epochs: Number of passes over data set
:param patience: Early Stopping parameter
:param logfile_loss: Log file name for results
:param modelfile_hdf5: Log file name for network structure and weights in HDF5 format
:param rnn_units: Recurrent layer sizes
:param dense_units: Dense layer sizes
:param rnn: Recurrent Layer type (Vanilla, LSTM or GRU)
:param dropout: Dropout probability
:param implementation: RNN implementation (0: CPU, 2: GPU, 1: any)
:param bidirectional: Flag to switch between Forward and Bidirectional RNN
:param batchnormalization: Flag to switch Batch Normalization on/off
:param gram_drop: Keras Siamese RNN to be tested
:param test_indices: Testing 2-tuples of time series index pairs
:param gram_drop: Gram matrix with dropped elements
:param seqs: List of time series
:param load_pretrained: Flag to switch training from training set/use pretrained weights in HDF5 format
:type gram_drop: np.ndarrays
:type seqs: list of np.ndarrays
:type epochs: int
:type patience: int
:type logfile_loss: str
:type modelfile_hdf5: str
:type rnn: str
:type rnn_units: list of int
:type dense_units: list of int
:type rnn: str
:type dropout: float
:type implementation: int
:type bidirectional: bool
:type batchnormalization: bool
:type load_pretrained: bool
:returns: Filled in Gram matrix, training and prediction start and end times
:rtype: np.ndarray, float, float, float, float
"""
# pre-processing
num_seqs = len(seqs)
time_dim = max([seq.shape[0] for seq in seqs])
pad_value = -4444
seqs = pad_sequences([seq.tolist() for seq in seqs],
maxlen=time_dim, dtype='float32',
padding='post', value=pad_value)
feat_dim = seqs[0].shape[1]
input_shape = (time_dim, feat_dim)
K.clear_session()
# build network
model = siamese_rnn_branch.SiameseRnnBranch(input_shape, pad_value,
rnn_units,
dense_units,
rnn,
dropout,
implementation,
bidirectional,
batchnormalization,
loss_function,
siamese_joint_method,
trained_modelfile_hdf5,
siamese_arms_activation=siamese_arms_activation)
test_indices = [i
for i in range(num_seqs)
if all(np.isnan(gram_drop[i]))]
train_indices = np.delete(np.arange(len(seqs)), test_indices)
train_seqs = seqs[train_indices]
test_seqs = seqs[test_indices]
train_features = model.predict(train_seqs)
time_pred_start = os.times()
test_features = model.predict(test_seqs)
# fill in
gram_completed = copy.deepcopy(gram_drop)
for i, test_feature in zip(test_indices, test_features):
for j, train_feature in zip(train_indices, train_features):
prediction = np.inner(test_feature, train_feature)
assert np.isnan(gram_completed[i][j])
gram_completed[i][j] = prediction
gram_completed[j][i] = prediction
assert not np.isnan(gram_completed[i][j])
for j, test_feature_ in zip(test_indices, test_features):
if j > i:
continue
prediction = np.inner(test_feature, test_feature_)
assert np.isnan(gram_completed[i][j])
gram_completed[i][j] = prediction
gram_completed[j][i] = prediction
assert not np.isnan(gram_completed[i][j])
time_pred_end = os.times()
assert not np.any(np.isnan(np.array(gram_completed)))
assert not np.any(np.isinf(np.array(gram_completed)))
return gram_completed, time_pred_start, time_pred_end
def take(cls) -> "Snapshot":
return cls(os.times(), time.time())
def _monothonic_time():
return os.times()[4]
def methode_naive(
#initialisation des variables
grille: Grille, # grille modélisant le cas à résoudre
map: TypeMap, # le cas donn à résoudre (sélection dans fichier 'main')
MAX_STUCK: int,
FACTEUR_DISTANCE_RETRACTATION: int,
MAX_DIST_PM: int,
ADDITION:
Tuple, # un tuple qui contient les valeurs de pondération pour chaque cas
FACTEUR_CONCENTRATION_BRAS: float = 1,
affichage_graphique: bool = True,
affichage_console: bool = True):
"""Une méthode très simple pour résoudre le problème"""
random.shuffle(grille.point_montages)
###################################################################################################
# Choix des points de montage pour chaque robot
point_montage_deja_pris = []
for idx_robot in range(len(grille.robots)):
point_montage_min: PointMontage = None
total_facteur_min = 9999999
for point_montage in grille.point_montages:
ok = True
for pm in point_montage_deja_pris:
if point_montage.distance(pm) < math.sqrt(
grille.longueur * grille.hauteur /
len(grille.robots)) * FACTEUR_CONCENTRATION_BRAS:
ok = False
if pm == point_montage:
ok = False
# f_decentralized
if map == TypeMap.F:
if 40 < min(point_montage.x, grille.longueur - point_montage.x,
point_montage.y, grille.hauteur - point_montage.y):
ok = False
if ok:
total_facteur = 0
for tache in grille.taches:
total_facteur += point_montage.distance(
tache.centre_gravite) * tache.distance / (
tache.points + 1)
if total_facteur_min > total_facteur:
point_montage_min = point_montage
total_facteur_min = total_facteur
if point_montage_min is not None:
point_montage_deja_pris.append(point_montage_min)
grille.robots[idx_robot].point_montage = point_montage_min
nb_robot_sans_pm = 0
for robot in grille.robots:
if robot.point_montage is None:
nb_robot_sans_pm += 1
if nb_robot_sans_pm:
raise ValueError(
f'Il y a {nb_robot_sans_pm} robot(s) qui n\'ont pas de point de montage assigné. '
f'Pensez à réduire la valeur de FACTEUR_DISTANCE_RETRACTATION.')
# for i in range(len(grille.robots) - 100):
# grille.robots.pop(random.randint(0, len(grille.robots) - 1))
# la grille qui va en live tester les solutions
grille_live = copy.deepcopy(grille)
# la grille qui contien la solution
grille_solution = copy.deepcopy(grille)
if affichage_graphique:
debug_canvas: DebugCanvas = DebugCanvas(
grille_live,
cells_size=int(1200 /
(grille_live.longueur + grille_live.hauteur)))
stated_time = os.times()[0]
# assigne les taches et fait les démonstration de nos fichiers
while grille_live.step_simulation > 0:
coordonnees_prochain_mouvement = []
for idx_robot, robot in enumerate(grille_live.robots):
if affichage_graphique and grille_live.step_simulation == grille.step_simulation:
debug_canvas.update()
pince: ItemCase = robot.coordonnees_pince()
if len(robot.bras) > (grille_live.longueur +
grille_live.hauteur) * 1:
robot.stucks = MAX_STUCK
if robot.stucks == MAX_STUCK:
robot.mouvements.clear()
###################################################################################################
# Choix de la prochaine tâche
if not len(robot.taches) and len(grille_live.taches):
# oblige la rétractation après finition d'une tâche
# prochaine tâche
# prochaine_tache = robot.get_tache_plus_rentable(grille_live)
intersection_min = 9999999
prochaine_tache: Tache = None
facteur_max = 999999
distance_pince_min = 999999
# calcule
# for robot_for_tache in grille_live.robots:
# if len(robot_for_tache.taches):
# tache = robot_for_tache.taches[0]
# tache.etapes[0].temps = robot_for_tache.coordonnees_pince().distance(tache.etapes[0])
# for etape_from, etape_to in zip(tache.etapes[0::1], tache.etapes[1::1]):
# etape_to.temps = etape_from.distance(etape_to) + etape_from.temps
for tache in grille_live.taches:
intersection = 0
if tache.distance * 1.4 > grille_live.step_simulation:
intersection += 99999
facteur += 999999
# intersection entre la tâche et les étapes
# tache.etapes[0].temps = pince.distance(tache.etapes[0])
# for etape_from, etape_to in zip(tache.etapes[0::1], tache.etapes[1::1]):
# etape_to.temps = etape_from.distance(etape_to) + etape_from.temps
# tache.etapes[-1].temps_aller_retour = tache.etapes[-1].temps
# if len(tache.etapes) >= 2:
# for etape_to, etape_from in zip(tache.etapes[-1:1:-1], tache.etapes[-2:0:-1]):
# etape_from.temps_aller_retour = etape_to.temps_aller_retour \
# + etape_to.temps - etape_from.temps
# tache.etapes[0].temps_aller_retour = tache.etapes[1].temps_aller_retour \
# + tache.etapes[1].temps - tache.etapes[0].temps
# etapes = []
# for etape in robot.etapes_done:
# etapes.append(etape)
# etapes.append(pince)
# for etape in tache.etapes:
# etapes.append(etape)
#
# for robot_for_tache in grille_live.robots:
# robot_for_tache_pince = robot_for_tache.coordonnees_pince()
#
# etapes_robot = []
# for etape in robot_for_tache.etapes_done:
# etapes_robot.append(etape)
# etapes_robot.append(robot_for_tache_pince)
# for tache_robot in robot_for_tache.taches:
# for etape_robot in tache_robot.etapes:
# etapes_robot.append(etape_robot)
#
# for etape_from, etape_to in zip(etapes[0::1], etapes[1::1]):
# for robot_etape_from, robot_etape_to in zip(etapes_robot[0::1], etapes_robot[1::1]):
# if intersect_rectangle(etape_from, etape_to, robot_etape_from, robot_etape_to):
# # if robot_etape_from.temps < etape_from.temps_aller_retour + 1:
# intersection += 1
# for etape_from_1, etape_to_1 in zip(tache.etapes[0::1], tache.etapes[1::1]):
# for etape_from_2, etape_to_2 in zip(tache.etapes[0::1], tache.etapes[1::1]):
# if etape_to_1 != etape_from_2 and etape_to_2 != etape_from_1 and etape_from_1 != etape_from_2:
# if intersect_rectangle(etape_from_1, etape_to_1, etape_from_2, etape_to_2):
# intersection += 1
# for etape_from, etape_to in zip(tache.etapes[0::1], tache.etapes[1::1]):
# for robot_for_tache in grille_live.robots:
# robot_for_tache_pince = robot_for_tache.coordonnees_pince()
# for tache_robot in robot_for_tache.taches:
# for robot_etape_from, robot_etape_to in zip(tache_robot.etapes[0::1],
# tache_robot.etapes[1::1]):
# if intersect_rectangle(etape_from, etape_to, robot_etape_from, robot_etape_to):
# # if robot_etape_from.temps < etape_from.temps_aller_retour + 1:
# intersection += 1
# # pince vers première étape
# if intersect_rectangle(pince, tache.etapes[0], robot_etape_from, robot_etape_to):
# intersection += 1
# if len(robot_for_tache.taches) and intersect_rectangle(pince, tache.etapes[0], robot_for_tache_pince, robot_for_tache.taches[0].etapes[0]):
# intersection += 1
# # etapes déjà faites pour chaque robots
# for robot_etape_from, robot_etape_to in zip(robot_for_tache.etapes_done[0::1],
# robot_for_tache.etapes_done[1::1]):
# if intersect_rectangle(etape_from, etape_to, robot_etape_from, robot_etape_to):
# intersection += 1
# if etape_from == etape_to:
# intersection += 1
# # au niveau du bras
# if (len(robot_for_tache.taches) and intersect_rectangle(etape_from, etape_to,
# robot_for_tache.coordonnees_pince(),
# robot_for_tache.taches[0].etapes[0])) \
# or (len(robot_for_tache.etapes_done) and intersect_rectangle(etape_from, etape_to,
# robot_for_tache.coordonnees_pince(),
# robot_for_tache.etapes_done[-1])):
# intersection += 1
# for etape_from_1, etape_to_1 in zip(tache.etapes[0::1], tache.etapes[1::1]):
# for etape_from_2, etape_to_2 in zip(tache.etapes[0::1], tache.etapes[1::1]):
# if etape_to_1 != etape_from_2 and etape_to_2 != etape_from_1 and etape_from_1 != etape_from_2:
# if intersect_rectangle(etape_from_1, etape_to_1, etape_from_2, etape_to_2):
# intersection += 1
# facteur = tache.etapes[0].surface(robot.point_montage)
facteur = tache.distance + tache.etapes[0].distance(
robot.point_montage)
facteur = facteur / tache.points
max_distance_from_pm = 0
for etape in tache.etapes:
distance = etape.distance(robot.point_montage)
if distance > max_distance_from_pm:
max_distance_from_pm = distance
facteur = (facteur+ADDITION[0])\
* (tache.centre_gravite.distance(robot.point_montage) + ADDITION[1])\
* (tache.distance_centre_gravite+ADDITION[2])\
* (intersection+ADDITION[3])
if facteur_max > facteur\
and max_distance_from_pm < MAX_DIST_PM :
intersection_min = intersection
facteur_max = facteur
distance_pince_min = facteur
prochaine_tache = tache
if prochaine_tache is not None:
if affichage_console:
print(
" --> Nouvelle tâche assignée : distance: ",
str(prochaine_tache.etapes[0].distance(
robot.point_montage)), ", etapes: ",
len(prochaine_tache.etapes),
", min intersection: ", intersection_min,
", facteur: ", int(facteur_max), ", ",
prochaine_tache.points, " points")
# ajoute une nouvelle tâche au robot
robot.add_tache(prochaine_tache, grille_live)
# on essaye de retrouver la tache corrspondante dans grille_solve
for tache in grille_solution.taches:
if tache == prochaine_tache:
grille_solution.robots[idx_robot].add_tache(
prochaine_tache, grille_solution)
###################################################################################################
# Choix du prochain mouvement du robot
prochain_mouvement: Mouvement = Mouvement.ATTENDRE
if len(robot.taches) and robot.stucks < MAX_STUCK:
# Reste-t-il des mouvements déjà déterminés pour le robot ?
if len(robot.mouvements):
# Recherche d'une collision ? Si oui, il faut retrouver un autre chemin
collision = False
x_new, y_new = robot.coordonnees_pince().get_position(
robot.mouvements[0])
# le prochain mouvement d'un autre bras a déjà réservé cette case ?
for idx_coordonnees in range(
0, len(coordonnees_prochain_mouvement), 2):
if x_new == coordonnees_prochain_mouvement[idx_coordonnees] and \
y_new == coordonnees_prochain_mouvement[idx_coordonnees + 1]:
collision = True
# il y a un bras ou un point de montage présent sur cette case ?
for item in grille_live.cases[y_new][x_new]:
if isinstance(item, Bras) or isinstance(
item, PointMontage):
collision = True
#si collision, la liste des mouvements est effacées
if collision:
robot.mouvements.clear()
else:
# si pas de collision alors on fait le prochain mouvement
prochain_mouvement = robot.mouvements[0]
# se rétracter est plus intéressant ?
retractation = False
if len(robot.taches) and not len(robot.mouvements):
# recherche de la distance entre la pince à l'étape
distance_pince_etape = pince.distance(
robot.taches[0].etapes[0])
# si une autre case occupée par le bras du robot est plus proche de la tâche alors le robot se rétracte
for idx_bras, bras in enumerate(
robot.bras[::(len(robot.bras) // 10 + 1)]):
if bras.distance(robot.taches[0].etapes[0]) + (
len(robot.bras) - idx_bras
) / FACTEUR_DISTANCE_RETRACTATION < distance_pince_etape:
retractation = True
#########################
# path-finding simple
# y a-t-il des mouvements pré-déffini ou une rétractation ?
if not len(robot.mouvements) and not retractation:
# il y a une tâche à accomplir
# bouge vers l'etape correspondante
x_diff: int = robot.taches[0].etapes[0].x - pince.x
y_diff: int = robot.taches[0].etapes[0].y - pince.y
# chemin vers la tâche est recherché en fonction de la distance entre les coordonées
# du robot et les coordonées de l'étape
if abs(x_diff) > abs(y_diff):
# en x
if x_diff > 0:
prochain_mouvement = Mouvement.DROITE
else:
prochain_mouvement = Mouvement.GAUCHE
else:
# en y
if y_diff > 0:
prochain_mouvement = Mouvement.HAUT
elif y_diff > 0:
prochain_mouvement = Mouvement.BAS
else:
prochain_mouvement = Mouvement.ATTENDRE
# collision ?
x_new, y_new = robot.coordonnees_pince().get_position(
prochain_mouvement)
collision = False
# le prochain mouvement d'un autre bras a déjà réservé cette case ?
for idx_coordonnees in range(
0, len(coordonnees_prochain_mouvement), 2):
if x_new == coordonnees_prochain_mouvement[idx_coordonnees] and \
y_new == coordonnees_prochain_mouvement[idx_coordonnees + 1]:
collision = True
# il y a un bras ou un point de montage présent sur cette case ?
for item in grille_live.cases[y_new][x_new]:
if isinstance(item, Bras) or isinstance(
item, PointMontage):
collision = True
# en cas de collision, attendre pendant un tour
if collision:
prochain_mouvement = Mouvement.ATTENDRE
#########################
# path-finding A*
# recherche d'un chemin plus optimisé, toujours à l'aide de la distance entre le robot et l'étape
if prochain_mouvement == Mouvement.ATTENDRE:
x_min = max(
(pince.x if pince.x < robot.taches[0].etapes[0].x
else robot.taches[0].etapes[0].x) -
(1 + robot.elargissement), 0)
y_min = max(
(pince.y if pince.y < robot.taches[0].etapes[0].y
else robot.taches[0].etapes[0].y) -
(1 + robot.elargissement), 0)
x_max = min(
(pince.x if pince.x >= robot.taches[0].etapes[0].x
else robot.taches[0].etapes[0].x) +
(1 + robot.elargissement),
grille_live.longueur - 1)
y_max = min(
(pince.y if pince.y >= robot.taches[0].etapes[0].y
else robot.taches[0].etapes[0].y) +
(1 + robot.elargissement), grille_live.hauteur - 1)
matrix = grille_live.to_matrix(x_min, y_min, x_max,
y_max, robot)
# ajoute les prochains mouvements déjà programmé des autres bras
for idx_coordonnees in range(
0, len(coordonnees_prochain_mouvement), 2):
if x_min <= coordonnees_prochain_mouvement[idx_coordonnees] <= x_max and \
y_min <= coordonnees_prochain_mouvement[idx_coordonnees + 1] <= y_max:
matrix[coordonnees_prochain_mouvement[idx_coordonnees + 1] - y_min] \
[coordonnees_prochain_mouvement[idx_coordonnees] - x_min] = -1
start = robot.coordonnees_pince()
end = robot.taches[0].etapes[0]
path = grille_live.pathfinder(start, end, robot.bras,
matrix, x_min, y_min)
if path is not None:
for index_path in range(0, len(path)):
prochain_mouvement: Mouvement
x_diff = path[index_path][0] - (
path[index_path -
1][0] if index_path != 0 else start.x)
y_diff = path[index_path][1] - (
path[index_path -
1][1] if index_path != 0 else start.y)
if x_diff < 0 and y_diff < 0:
robot.mouvements.append(Mouvement.GAUCHE)
prochain_mouvement = Mouvement.BAS
elif x_diff < 0 and y_diff == 0:
prochain_mouvement = Mouvement.GAUCHE
elif x_diff < 0:
robot.mouvements.append(Mouvement.GAUCHE)
prochain_mouvement = Mouvement.HAUT
elif x_diff == 0 and y_diff < 0:
prochain_mouvement = Mouvement.BAS
elif x_diff == 0:
prochain_mouvement = Mouvement.HAUT
elif x_diff > 0 and y_diff < 0:
robot.mouvements.append(Mouvement.DROITE)
prochain_mouvement = Mouvement.BAS
elif x_diff > 0 and y_diff == 0:
prochain_mouvement = Mouvement.DROITE
else:
robot.mouvements.append(Mouvement.DROITE)
prochain_mouvement = Mouvement.HAUT
robot.mouvements.append(prochain_mouvement)
else:
# pas de tache -> attente
prochain_mouvement = Mouvement.ATTENDRE
if len(robot.mouvements):
prochain_mouvement = robot.mouvements[0]
if prochain_mouvement == Mouvement.ATTENDRE:
#########################
# rétractation
# supprime les étapes déjà visité lors de la rétractation
if len(robot.bras) and len(
robot.etapes_done
) and robot.bras[-1] == robot.etapes_done[-1]:
robot.etapes_done.pop()
if len(robot.bras) >= 2:
if robot.bras[-2].x - robot.bras[-1].x < 0:
prochain_mouvement = Mouvement.GAUCHE
elif robot.bras[-2].x - robot.bras[-1].x > 0:
prochain_mouvement = Mouvement.DROITE
elif robot.bras[-2].y - robot.bras[-1].y < 0:
prochain_mouvement = Mouvement.BAS
else:
prochain_mouvement = Mouvement.HAUT
elif len(robot.bras) == 1:
if robot.point_montage.x - robot.bras[-1].x < 0:
prochain_mouvement = Mouvement.GAUCHE
elif robot.point_montage.x - robot.bras[-1].x > 0:
prochain_mouvement = Mouvement.DROITE
elif robot.point_montage.y - robot.bras[-1].y < 0:
prochain_mouvement = Mouvement.BAS
else:
prochain_mouvement = Mouvement.HAUT
else:
robot.stucks = 0
robot.etapes_done.clear()
robot.etapes_done.append(robot.point_montage)
#changement des coordonées du robot
x_new, y_new = robot.coordonnees_pince().get_position(
prochain_mouvement)
coordonnees_prochain_mouvement.append(x_new)
coordonnees_prochain_mouvement.append(y_new)
if not len(robot.mouvements):
robot.mouvements.append(prochain_mouvement)
grille_solution.robots[idx_robot].mouvements.append(
prochain_mouvement)
###################################################################################################
# Avancement dans la simulation
grille_live.one_step_simulation()
# pourcentage de progression
if int(math.floor((grille_live.step_simulation) / grille_solution.step_simulation * 1000)) \
> int(math.floor((grille_live.step_simulation - 1) / grille_solution.step_simulation * 1000)):
#pourcentage d'avancement de la simulation
pourcentage = (1000 - int(
math.floor((grille_live.step_simulation - 1) /
grille_solution.step_simulation * 1000))) / 10
#temps estimé pour la simulation
temps_estime = math.floor(
(os.times()[0] - stated_time) / (pourcentage + 0.000000001) *
(100 - pourcentage))
#points gagnés
total_point_en_cours = 0
#tache à réaliser
total_tache_restante = len(grille_live.taches)
for robot in grille_live.robots:
for tache in robot.taches:
total_tache_restante += 1
total_point_en_cours += tache.points
if affichage_console:
print(pourcentage, " %, ", temps_estime // 60, " min ",
str(temps_estime % 60), "sec, ",
grille_live.points // 1000, " K points, ",
total_tache_restante, " taches restantes, ",
total_point_en_cours, " points en cours")
if affichage_graphique:
debug_canvas.update()
###################################################################################################
# Fin de la simulation
# supprime les tâches non finies
for index_robot in range(len(grille_live.robots)):
if len(grille_live.robots[index_robot].taches) > 0:
grille_solution.robots[index_robot].taches.pop()
if affichage_console:
print(grille_live)
print("Points : ", grille_live.points)
grille_solution.points = grille_live.points
return grille_solution
def strottle(self, tbytes, total):
span = tbytes / self.bytes_ps
d1, d2, d3, d4, now = os.times()
rspan = now - self.tbegin
if span > rspan:
time.sleep(span - rspan)
def sftp_put_trottle(self, file, destName):
d1, d2, d3, d4, now = os.times()
self.tbegin = now + 0.0
self.sftp.put(file, destName, self.strottle)
def send(self, files):
# process with file sending
currentFTPDir = ''
for filex in files:
file = filex
self.partialfile = None
# priority 0 is retransmission and is never suppressed
priority = file.split('/')[-3]
# if in cache than it was already sent... nothing to do
# caching is always done on original file for early check (before fx)
if self.client.nodups and priority != '0' and self.in_cache(
True, file):
self.logger.info("suppressed duplicate send %s",
os.path.basename(file))
os.unlink(file)
continue
# applying the fx_script if defined redefine the file list
if self.client.fx_execfile != None:
fxfile = self.client.run_fx_script(file, self.logger)
if fxfile == None:
self.logger.warning("FX script ignored the file : %s" %
os.path.basename(file))
os.unlink(file)
continue
elif fxfile == file:
self.logger.warning("FX script kept the file as is : %s" %
os.path.basename(file))
else:
self.logger.info("FX script modified %s to %s " %
(os.path.basename(file), fxfile))
os.unlink(file)
file = fxfile
# get files ize
nbBytes = 0
try:
nbBytes = os.stat(file)[stat.ST_SIZE]
except:
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to stat %s, Type: %s, Value: %s" %
(file, type, value))
continue
# get/check destination Name
basename = os.path.basename(file)
destName, destDir = self.client.getDestInfos(basename)
if not destName:
os.unlink(file)
self.logger.info('No destination name: %s has been erased' %
file)
continue
# run destfn
ldestName = self.client.run_destfn_script(destName)
if ldestName != destName:
self.logger.info("destfn_script : %s becomes %s " %
(destName, ldestName))
destName = ldestName
# check protocol
if not self.client.protocol in ['file', 'ftp', 'sftp']:
self.logger.critical("Unknown protocol: %s" %
self.client.protocol)
sys.exit(2)
# We remove the first / (if there was only one => relative path, if there was two => absolute path)
destDir = destDir[1:]
if self.client.dir_pattern == True:
destDir = self.dirPattern(file, basename, destDir, destName)
if destDir == '':
destDirString = '/'
elif destDir == '/':
destDirString = '//'
else:
destDirString = '/' + destDir + '/'
# file protocol means file renaming
if self.client.protocol == 'file':
try:
self.logger.start_timer()
if self.client.dir_mkdir == True and not os.path.isdir(
destDirString):
os.makedirs(destDirString)
os.rename(file, destDirString + destName)
self.logger.delivered(
"(%i Bytes) File %s delivered to %s://%s@%s%s%s" %
(nbBytes, file, self.client.protocol, self.client.user,
self.client.host, destDirString, destName),
destDirString + destName)
# add data to cache if needed
if self.client.nodups and self.cacheMD5 != None:
self.cacheManager.find(self.cacheMD5, 'standard')
except:
(type, value, tb) = sys.exc_info()
self.logger.error(
"Unable to move to: %s, Type: %s, Value:%s" %
((destDirString + destName), type, value))
time.sleep(1)
continue
# ftp or sftp protocol
if self.client.protocol == 'ftp' or self.client.protocol == 'sftp':
try:
self.logger.start_timer()
# the alarm timeout is set at that level
# it means that everything done to a file must be done
# within "timeout_send" seconds
timex = AlarmFTP(self.client.protocol + ' timeout')
if self.client.timeout_send > 0:
timex.alarm(self.client.timeout_send)
# we are not in the proper directory
if currentFTPDir != destDir:
# create and cd to the directory
if self.client.dir_mkdir:
try:
self.dirMkdir(destDir)
currentFTPDir = destDir
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error(
"Unable to mkdir: %s, Type: %s, Value:%s" %
(destDir, type, value))
time.sleep(1)
continue
# just cd to the directory
else:
try:
self.chdir(self.originalDir)
self.chdir(destDir)
currentFTPDir = destDir
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error(
"Unable to chdir to: %s, Type: %s, Value:%s"
% (destDir, type, value))
time.sleep(1)
continue
# try to write the file to the client
try:
d1, d2, d3, d4, tbegin = os.times()
self.partialfile = destName
self.send_file(file, destName)
self.partialfile = None
d1, d2, d3, d4, tend = os.times()
self.logger.delivered("(%i Bytes) File %s delivered to %s://%s@%s%s%s" % \
(nbBytes, file, self.client.protocol, self.client.user, \
self.client.host, destDirString, destName),file)
os.unlink(file)
if self.client.kbytes_ps > 0.0:
tspan = tend - tbegin
if tspan >= 1.0:
tspan = tspan * 1024.0
tspeed = nbBytes / tspan
self.logger.debug("Speed %f Kb/s" % tspeed)
# add data to cache if needed
if self.client.nodups and self.cacheMD5 != None:
self.cacheManager.find(self.cacheMD5, 'standard')
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to deliver to %s://%s@%s%s%s, Type: %s, Value: %s" %
(self.client.protocol, self.client.user, self.client.host, \
destDirString, destName, type, value))
# preventive delete when problem
if self.partialfile != None:
timex.alarm(5)
try:
self.logger.debug("Trying preventive delete")
self.rm(self.partialfile)
except:
self.logger.debug(
"Preventive delete timed out")
timex.cancel()
time.sleep(1)
return
timex.cancel()
except FtpTimeoutException:
timex.cancel()
self.logger.warning("SEND TIMEOUT (%i Bytes) File %s going to %s://%s@%s%s%s" % \
(nbBytes, file, self.client.protocol, self.client.user, \
self.client.host, destDirString, destName))
return
and value2_minimax == value2_ab == correctmoves_2[i]):
correct += 1
print(
'Testing Minimax and Alphabeta Moves Equality (with Depth Limit of 6)')
print("You computed correct moves for {} of {} tests".format(
correct, len(correctmoves_1)))
if test_caching_big:
print('Testing Caching Big')
check_1 = 0
check_2 = 0
for i in range(0, len(bigboards)):
start_time_1 = os.times()[0]
no_cache = select_move_alphabeta(bigboards[i], 1, 6)
end_time_1 = os.times()[0]
start_time_2 = os.times()[0]
with_cache = select_move_alphabeta(bigboards[i], 1, 6, 1)
end_time_2 = os.times()[0]
if (end_time_1 - start_time_1) >= (end_time_2 - start_time_2):
check_1 += 1
if (with_cache == no_cache):
check_2 += 1
print(
"State caching improved the time of your alpha-beta for {} of {} boards"
def rnn_matrix_completion(gram_drop, seqs,
epochs, patience,
epoch_start_from,
logfile_loss,
new_modelfile_hdf5,
rnn,
rnn_units,
dense_units,
dropout,
implementation,
bidirectional,
batchnormalization,
mode,
loss_function,
loss_weight_ratio,
labels,
siamese_joint_method,
siamese_arms_activation,
trained_modelfile_hdf5=None):
"""Fill in Gram matrix with dropped elements with Keras Siamese RNN.
Trains the network on given part of Gram matrix and the corresponding sequences
Fills in missing elements by network prediction
:param gram_drop: Gram matrix with dropped elements
:param seqs: List of time series
:param epochs: Number of passes over data set
:param patience: Early Stopping parameter
:param logfile_loss: Log file name for results
:param modelfile_hdf5: Log file name for network structure and weights in HDF5 format
:param rnn_units: Recurrent layer sizes
:param dense_units: Dense layer sizes
:param rnn: Recurrent Layer type (Vanilla, LSTM or GRU)
:param dropout: Dropout probability
:param implementation: RNN implementation (0: CPU, 2: GPU, 1: any)
:param bidirectional: Flag to switch between Forward and Bidirectional RNN
:param batchnormalization: Flag to switch Batch Normalization on/off
:param gram_drop: Keras Siamese RNN to be tested
:param test_indices: Testing 2-tuples of time series index pairs
:param gram_drop: Gram matrix with dropped elements
:param seqs: List of time series
:param load_pretrained: Flag to switch training from training set/use pretrained weights in HDF5 format
:type gram_drop: np.ndarrays
:type seqs: list of np.ndarrays
:type epochs: int
:type patience: int
:type logfile_loss: str
:type modelfile_hdf5: str
:type rnn: str
:type rnn_units: list of int
:type dense_units: list of int
:type rnn: str
:type dropout: float
:type implementation: int
:type bidirectional: bool
:type batchnormalization: bool
:type load_pretrained: bool
:returns: Filled in Gram matrix, training and prediction start and end times
:rtype: np.ndarray, float, float, float, float
"""
# pre-processing
num_seqs = len(seqs)
time_dim = max([seq.shape[0] for seq in seqs])
pad_value = -4444
seqs = pad_sequences([seq.tolist() for seq in seqs],
maxlen=time_dim, dtype='float32',
padding='post', value=pad_value)
feat_dim = seqs[0].shape[1]
input_shape = (time_dim, feat_dim)
K.clear_session()
# build network
model = siamese_rnn.SiameseRnn(input_shape, pad_value,
rnn_units, dense_units,
rnn,
dropout,
implementation,
bidirectional,
batchnormalization,
loss_function,
siamese_joint_method,
siamese_arms_activation)
# training
# make 90% + 10% training validation random split
train_and_validation_indices = np.random.permutation([(i, j)
for i in range(num_seqs)
for j in range(i, num_seqs)
if not np.isnan(gram_drop[i][j])])
train_indices = train_and_validation_indices[:int(len(train_and_validation_indices) * 0.9)]
validation_indices = train_and_validation_indices[int(len(train_and_validation_indices) * 0.9):]
time_train_start = os.times()
if mode == 'train':
assert epoch_start_from == 1
model.train_and_validate(train_indices, validation_indices,
gram_drop,
seqs,
labels,
epochs,
patience,
epoch_start_from,
loss_weight_ratio,
logfile_loss,
new_modelfile_hdf5)
elif mode == 'continue_train':
assert trained_modelfile_hdf5 != None
print("load from hdf5 file: %s" % trained_modelfile_hdf5)
model.load_weights(trained_modelfile_hdf5)
model.train_and_validate(train_indices, validation_indices,
gram_drop,
seqs,
labels,
epochs,
patience,
epoch_start_from,
loss_weight_ratio,
logfile_loss,
new_modelfile_hdf5)
elif mode == 'fine_tuning':
assert trained_modelfile_hdf5 != None
print("Fine Tuning, load from hdf5 file: %s" % trained_modelfile_hdf5)
model.load_weights(trained_modelfile_hdf5)
model.train_and_validate(train_indices, validation_indices,
gram_drop,
seqs,
labels,
epochs,
patience,
epoch_start_from,
loss_weight_ratio,
logfile_loss,
new_modelfile_hdf5)
elif mode == 'load_pretrained':
print("load from hdf5 file: %s" % trained_modelfile_hdf5)
model.load_weights(trained_modelfile_hdf5)
else:
print('Unsupported mode.')
exit(-1)
time_train_end = os.times()
# prediction
test_indices = [(i, j)
for i in range(num_seqs)
for j in range(i, num_seqs)
if np.isnan(gram_drop[i][j])]
time_pred_start = os.times()
predictions = model.predict(test_indices, gram_drop, seqs)
time_pred_end = os.times()
# fill in
gram_completed = copy.deepcopy(gram_drop)
for k in range(len(test_indices)):
prediction = predictions[k][0]
i, j = test_indices[k]
assert np.isnan(gram_completed[i][j])
gram_completed[i][j] = prediction
gram_completed[j][i] = prediction
assert not np.isnan(gram_completed[i][j])
assert not np.any(np.isnan(np.array(gram_completed)))
assert not np.any(np.isinf(np.array(gram_completed)))
return gram_completed, time_train_start, time_train_end,\
time_pred_start, time_pred_end
disc_sys_labels=sys_modes
)
## Create convex proposition preserving partition
ppp = abstract.prop2part(cont_ss, cont_props)
ppp, new2old = abstract.part2convex(ppp)
ax = ppp.plot_props()
ax.figure.savefig(imgpath + 'cprops.pdf')
ax = ppp.plot()
ax.figure.savefig(imgpath + 'ppp.pdf')
## Discretize to establish transitions
if os.name == "posix":
start = os.times()[2]
logger.info('start time: ' + str(start))
else:
logger.info('Timing currently only available for POSIX platforms (not Windows)')
disc_params = {}
disc_params[('normal', 'fly')] = {'N':N, 'trans_length':3}
disc_params[('refuel', 'fly')] = {'N':N, 'trans_length':3}
sys_ts = abstract.multiproc_discretize_switched(
ppp, switched_dynamics, disc_params, plot=True
)
if os.name == "posix":
end = os.times()[2]
logger.info('end time: ' + str(end))
# This is a longer version of the bootstrap script given at the end of
# faqwin.py; it prints timing statistics at the end of the regular CGI
# script's output (so you can monitor how it is doing).
# This script should be placed in your cgi-bin directory and made
# executable.
# You need to edit the first line and the lines that define FAQDIR and
# SRCDIR, below: change /usr/local/bin/python to where your Python
# interpreter lives, change the value for FAQDIR to where your FAQ
# lives, and change the value for SRCDIR to where your faqwiz.py
# module lives. The faqconf.py and faqcust.py files live there, too.
import os
t1 = os.times() # If this doesn't work, just get rid of the timing code!
try:
FAQDIR = "/usr/people/guido/python/FAQ"
SRCDIR = "/usr/people/guido/python/src/Tools/faqwiz"
import os, sys
os.chdir(FAQDIR)
sys.path.insert(0, SRCDIR)
import faqwiz
except SystemExit, n:
sys.exit(n)
except:
t, v, tb = sys.exc_info()
print
import cgi
cgi.print_exception(t, v, tb)
myproc = comm.Get_rank()
nprocs = comm.Get_size()
hostname = os.uname()[1]
ham_dir = "/homes/daveturner/apps/pyxaid.mpi" # Where res.tar.gz is
scr_dir = "/tmp" # Use /tmp on each node's local disk
#scr_dir = os.getwcd() + "/scratch" # No scratch, just use a directory on CWD
#os.system( "mkdir -p " + scr_dir )
# gunzip and untar the hamiltonian directory to scratch, or /tmp on each node
# This is fast and easy to do, but may only help on some file systems
# NOTE: You therefore need res.tar.gz in the Hamiltonian directory ham_dir
t0 = os.times()[4]
for i in range(0, nprocs):
comm.barrier()
if myproc == i:
os.system("mkdir -p " + scr_dir +
"/marker.dir") # Dummy dir for tests later
#print("Proc " + str(myproc) + " says hello from " + hostname )
if not os.path.isdir(scr_dir + "/res"):
print(
str(myproc) + " loading res onto " + scr_dir + " on " +
hostname)
os.system("tar -xzf " + ham_dir + "/res.tar.gz -C " + scr_dir)
comm.barrier()
tres = os.times()[4] - t0
params = {}
max(min(int(iterations), 10), 1)
except (TypeError, ValueError), x:
return S_ERROR(x)
# This number of iterations corresponds to 250 HS06 seconds
n = int(1000 * 1000 * 12.5)
calib = 250.0 / UNITS[reference]
m = 0L
m2 = 0L
p = 0
p2 = 0
# Do one iteration extra to allow CPUs with variable speed
for i in range(iterations + 1):
if i == 1:
start = os.times()
# Now the iterations
for _j in range(n):
t = random.normalvariate(10, 1)
m += t
m2 += t * t
p += t
p2 += t * t
end = os.times()
cput = sum(end[:4]) - sum(start[:4])
wall = end[4] - start[4]
if not cput:
return S_ERROR('Can not get used CPU')
# Instalação flake8: pip install flake8
import sys
import os
a = []
b = ('cdsichsdjaioujc dsacsdoaic saidojcsoadijc ssdaoicjsdaoii '
'sojcosdijcoisadi saoidcj sodi')
idade = 20
if idade > 18:
print("maior idade")
print(sys.argv)
print(os.times())
nome = 'Élysson MR'
print(f'Nome: {nome}')
log.exception('Exception raised out of drivers.run:')
except Exception, e:
print 'Exception raised in log.exception. This is *really*'
print 'bad. Hopefully it won\'t happen again, but tell us'
print 'about it anyway, this is a significant problem.'
print 'Anyway, here\'s the exception: %s' % \
utils.gen.exnToString(e)
except:
print 'Man, this really sucks. Not only did log.exception'
print 'raise an exception, but freaking-a, it was a string'
print 'exception. People who raise string exceptions should'
print 'die a slow, painful death.'
now = time.time()
seconds = now - world.startedAt
log.info('Total uptime: %s.', utils.gen.timeElapsed(seconds))
(user, system, _, _, _) = os.times()
log.info('Total CPU time taken: %s seconds.', user + system)
log.info('No more Irc objects, exiting.')
version = '0.83.4.1'
if __name__ == '__main__':
###
# Options:
# -p (profiling)
# -n, --nick (nick)
# --startup (commands to run onStart)
# --connect (commands to run afterConnect)
# --config (configuration values)
parser = optparse.OptionParser(usage='Usage: %prog [options] configFile',
version='Supybot %s' % version)
def clock():
times = os.times()
return (times[0] + times[1], times[4])
def cputime():
user, system, childUser, childSystem, wall = os.times()
return user
def stageOutFile(self, source, destination, token=None, outputDir=None):
"""Stage out the file. Should be implementated by different site mover"""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
# determine which timeout option to use
timeout_option = "-t %d" % (self.timeout)
#mkdir
_cmd_str = '%s gfal-mkdir --verbose %s -p %s' % (self._setup, timeout_option, os.path.dirname(destination))
self.log("Executing command: %s" % (_cmd_str))
status, output = commands.getstatusoutput(_cmd_str)
self.log("status: %s, output: %s" % (status, output.replace("\n"," ")))
if status != 0:
outputRet["errorLog"] = output
outputRet["report"]["clientState"] = "ERR_MKDIR"
return PilotErrors.ERR_MKDIR, outputRet
# cleanup the SURL if necessary (remove port and srm substring)
if token:
# Special case for GROUPDISK (do not remove dst: bit before this stage, needed in several places)
if "dst:" in token:
token = token[len('dst:'):]
tolog("Dropped dst: part of space token descriptor; token=%s" % (token))
if 'DATADISK' in token:
token = "ATLASDATADISK"
else:
token = "ATLASGROUPDISK"
tolog("Space token descriptor reset to: %s" % (token))
_cmd_str = '%s gfal-copy --verbose %s -D "SRM PLUGIN:TURL_PROTOCOLS=gsiftp" -S %s file://%s %s' % (self._setup, timeout_option, token, os.path.abspath(source), destination)
else:
# surl is the same as putfile
_cmd_str = '%s gfal-copy --verbose %s -D "SRM PLUGIN:TURL_PROTOCOLS=gsiftp" file://%s %s' % (self._setup, timeout_option, os.path.abspath(source), destination)
ec = -1
t0 = os.times()
o = '(not defined)'
if outputDir and outputDir.endswith("PilotMVOutputDir"):
timeStart = time()
outputFile = os.path.join(outputDir, os.path.basename(source))
mvCmd = "cp -f %s %s" % (source, outputFile)
tolog("Executing command: %s" % (mvCmd))
lstatus, loutput = commands.getstatusoutput(mvCmd)
if lstatus != 0:
ec = lstatus
o = loutput
else:
outputFileCmd = outputFile + ".gfalcmd"
handle = open(outputFileCmd, 'w')
handle.write(_cmd_str.replace(source, outputFile))
handle.close()
tolog("Write command %s to %s" % (_cmd_str.replace(source, outputFile), outputFileCmd))
tolog("Waiting remote to finish transfer")
o = "Remote timeout to transfer out file"
while (time() - timeStart) < self.timeout:
sleep(5)
if os.path.exists(outputFile + ".gfalcmdfinished"):
ec = 0
o = "Remote finished transfer"
tolog(o)
os.remove(outputFile + ".gfalcmdfinished")
break
if os.path.exists(outputFile + ".gfalcmdfailed"):
ec = 0
o = "Remote finished transfer"
tolog(o)
os.remove(outputFile + ".gfalcmdfailed")
break
else:
tolog("Executing command: %s" % (_cmd_str))
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
ec, o = commands.getstatusoutput(_cmd_str)
except Exception, e:
tolog("!!WARNING!!2999!! gfal-copy threw an exception: %s" % (o))
o = str(e)
def clock():
times = os.times()
return times[0] + times[1]
ec = 0
o = "Remote finished transfer"
tolog(o)
os.remove(outputFile + ".gfalcmdfailed")
break
else:
tolog("Executing command: %s" % (_cmd_str))
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
ec, o = commands.getstatusoutput(_cmd_str)
except Exception, e:
tolog("!!WARNING!!2999!! gfal-copy threw an exception: %s" % (o))
o = str(e)
outputRet["report"]['validateStart'] = time()
t1 = os.times()
t = t1[4] - t0[4]
tolog("Command finished after %f s" % (t))
tolog("ec = %d, output = %s" % (ec, o.replace("\n"," ")))
if ec != 0:
tolog("!!WARNING!!2990!! Command failed: %s" % (_cmd_str))
#check_syserr(ec, o)
tolog('!!WARNING!!2990!! Stage Out failed: Status=%d Output=%s' % (ec, str(o.replace("\n"," "))))
status, output = self.errorToReport(o, t, source, stageMethod="stageOut")
if status == PilotErrors.ERR_FILEEXIST:
return status, output
# check if file was partially transferred, if so, remove it
_ec, removeOutput = self.removeRemoteFile(destination)
class GFAL2SiteMover(SiteMover.SiteMover):
""" SiteMover that uses gfal-copy for both get and put """
# no registration is done
copyCommand = "gfal-copy"
checksum_command = "adler32"
has_mkdir = False
has_df = False
has_getsize = False
has_md5sum = True
has_chmod = False
timeout = 3600
def __init__(self, setup_path, *args, **kwrds):
self._setup = setup_path.strip()
self.__isSetuped = False
self._defaultSetup = None
def get_timeout(self):
return self.timeout
def log(self, errorLog):
tolog(errorLog)
def getLocalEMISetup(self, si):
""" Build command to prepend the xrdcp command [xrdcp will in general not be known in a given site] """
return si.getLocalEMISetup()
def getSetup(self):
""" Return the setup string (pacman setup os setup script) for the copy command used by the mover """
_setup_str = ""
self._setup = self._setup.strip()
tolog("self setup: %s" % self._setup)
if self._setup and self._setup != "" and self._setup.strip() != "":
if not self._setup.endswith(";"):
self._setup += ";"
if not "alias" in self._setup:
if "atlasLocalSetup.sh" in self._setup and "--quiet" not in self._setup:
self._setup = self._setup.replace("atlasLocalSetup.sh", "atlasLocalSetup.sh --quiet")
if self._setup.startswith("export") or self._setup.startswith("source"):
_setup_str = "%s" % self._setup
else:
_setup_str = "source %s" % self._setup
else:
_setup_str = self._setup
if _setup_str != "":
tolog("Using setup: %s" % (_setup_str))
return _setup_str
def verifySetupCommand(self, _setupStr):
""" Make sure the setup command exists """
statusRet = 0
outputRet={}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
# remove any '-signs
_setupStr = _setupStr.replace("'", "")
self.log("Will verify: %s" % (_setupStr))
if _setupStr != "" and "source " in _setupStr:
# first extract the file paths from the source command(s)
setupPaths = extractFilePaths(_setupStr)
# only run test if string begins with an "/"
if setupPaths:
# verify that the file paths actually exists
for setupPath in setupPaths:
if "-" in setupPath:
continue
if os.path.exists(setupPath):
self.log("File %s has been verified" % (setupPath))
else:
outputRet["errorLog"] = errorLog = "No such file or directory: %s" % (setupPath)
self.log('!!WARNING!!2991!! %s' % (errorLog))
statusRet = PilotErrors.ERR_NOSUCHFILE
break
else:
# nothing left to test
pass
else:
self.log("Nothing to verify in setup: %s (either empty string or no source command)" % (_setupStr))
return statusRet, outputRet
def verifySetupProxy(self, _setupStr, experiment):
#check do we have a valid proxy
# get the experiment object
thisExperiment = getExperiment(experiment)
status, output = thisExperiment.verifyProxy(envsetup=_setupStr, limit=2)
return status, output
def verifySetup(self, _setupStr, experiment, proxycheck=True):
statusRet, outputRet = self.verifySetupCommand(_setupStr)
if statusRet != 0:
#self.prepareReport('RFCP_FAIL', self._variables['report'])
outputRet["report"]["clientState"] = "RFCP_FAIL"
return statusRet, outputRet
command = _setupStr
if command != "" and not command.endswith(';'):
command = command + ";"
command += " which " + self.copyCommand
status, output = commands.getstatusoutput(command)
self.log("Execute command: %s" % command)
self.log("Status: %s, Output: %s" % (status, output))
if status != 0:
self.log(self.copyCommand +" is not found in envsetup: " + _setupStr)
#self.prepareReport('RFCP_FAIL', self._variables['report'])
outputRet["report"]["clientState"] = "RFCP_FAIL"
outputRet["errorLog"] = output
return status, outputRet
if proxycheck:
status, outputLog = self.verifySetupProxy(_setupStr, experiment)
if status != 0:
outputRet["errorLog"] = outputLog
outputRet["report"]["clientState"] = 'PROXYFAIL'
return status, outputRet
return status, outputRet
def setup(self, experiment):
""" setup env """
if self.__isSetuped:
return 0, None
thisExperiment = getExperiment(experiment)
self.useTracingService = thisExperiment.useTracingService()
si = getSiteInformation(experiment)
self._defaultSetup = self.getLocalEMISetup(si)
_setupStr = self.getSetup()
# get the user proxy if available
envsetupTest = _setupStr.strip()
if envsetupTest != "" and not envsetupTest.endswith(';'):
envsetupTest += ";"
if os.environ.has_key('X509_USER_PROXY'):
envsetupTest += " export X509_USER_PROXY=%s;" % (os.environ['X509_USER_PROXY'])
self.log("to verify site setup: %s " % envsetupTest)
status, output = self.verifySetup(envsetupTest, experiment)
self.log("site setup verifying: status: %s, output: %s" % (status, output["errorLog"]))
if status == 0:
self._setup = envsetupTest
self.__isSetuped = True
return status, output
else:
if self._defaultSetup:
#try to use default setup
self.log("Try to use default envsetup")
envsetupTest = self._defaultSetup.strip()
if envsetupTest != "" and not envsetupTest.endswith(';'):
envsetupTest += ";"
if os.environ.has_key('X509_USER_PROXY'):
envsetupTest += " export X509_USER_PROXY=%s;" % (os.environ['X509_USER_PROXY'])
self.log("verify default setup: %s " % envsetupTest)
status, output = self.verifySetup(envsetupTest, experiment)
self.log("default setup verifying: status: %s, output: %s" % (status, output["errorLog"]))
if status == 0:
self._setup = envsetupTest
self.__isSetuped = True
return status, output
return status, output
def fixStageInPath(self, path):
"""Fix the path"""
if path[:3] == "srm" and '?SFN=' in path:
self.log("Found SFN part in file path: %s" % (path))
elif path[:3] == "srm":
try:
hostname = path.split('/',3)[2]
except Exception as e:
self.log("'!!WARNING!!2999!! Could not extract srm protocol for replacement, keeping path variable as it is: %s (%s)' %\
(path, str(e))")
else:
# srm = 'srm://head01.aglt2.org'
srm = 'srm://' + hostname
# does seopt contain any matching srm's?
sematch = self.getSEMatchFromSEOpt(srm)
if sematch != "":
path = path.replace(srm, sematch)
self.log("Replaced %s with %s (from seopt) in path: %s" % (srm, sematch, path))
else:
se = readpar('se').split(",")[0]
_dummytoken, se = self.extractSE(se)
tolog("Using SE: %s" % (se))
path = path.replace(srm, se)
self.log("Replaced %s with %s (from se) in path: %s" % (srm, se, path))
# add port number from se to getfile if necessary
path = self.addPortToPath(se, path)
siteInformation = SiteInformation()
path = siteInformation.getCopyPrefixPath(path, stageIn=True)
return path
def getStageInMode(self, lfn, prodDBlockToken, transferType):
# should the root file be copied or read directly by athena?
status = 0
output={}
output["errorLog"] = None
output["report"] = {}
output["report"]["clientState"] = None
output["transfer_mode"] = None
isRootFileName = self.isRootFileName(lfn)
siteInformation = SiteInformation()
directIn, transfer_mode = siteInformation.getDirectInAccessMode(prodDBlockToken, isRootFileName, transferType)
if transfer_mode:
output["transfer_mode"] = transfer_mode
if directIn:
output["report"]["clientState"] = 'FOUND_ROOT'
output["report"]['relativeStart'] = None
output["report"]['transferStart'] = None
return PilotErrors.ERR_DIRECTIOFILE, output
return 0, output
def stageInFile(self, source, destination):
"""StageIn the file. should be implementated by different site mover."""
statusRet = 0
outputRet = {}
outputRet["errorLog"] = None
outputRet["report"] = {}
outputRet["report"]["clientState"] = None
self.log("StageIn files started.")
_cmd_str = '%s gfal-copy --verbose -t %s -D "SRM PLUGIN:TURL_PROTOCOLS=gsiftp" %s file://%s' % (self._setup, self.timeout, source, os.path.abspath(destination))
self.log('Executing command: %s' % (_cmd_str))
s = -1
o = '(not defined)'
t0 = os.times()
outputRet["report"]['relativeStart'] = time()
outputRet["report"]['transferStart'] = time()
try:
s, o = commands.getstatusoutput(_cmd_str)
except Exception, e:
tolog("!!WARNING!!2990!! Exception caught by stageInFile(): %s" % (str(e)))
o = str(e)
t1 = os.times()
t = t1[4] - t0[4]
self.log("Command finished after %f s: %s" % (t, o.replace('\n', ' ')))
if s == 0:
self.log("Stagein succeeded")
else:
self.log("!!WARNING!!2990!! Command failed: %s" % (_cmd_str))
o = o.replace('\n', ' ')
#check_syserr(s, o)
self.log("!!WARNING!!2990!! get_data failed. Status=%s Output=%s" % (s, str(o)))
# remove the local file before any get retry is attempted
_status = self.removeLocal(destination)
if not _status:
self.log("!!WARNING!!1112!! Failed to remove local file, get retry will fail")
status, output = self.errorToReport(o, t, source, stageMethod="stageIN")
return status, output
return statusRet, outputRet
def time_func():
tup = times()
#just user time
return tup[0] # + tup[1]
def optimal_subtraction(new_fits, ref_fits):
"""Function that takes in a new and a reference image, finds their WCS
solution using Astrometry.net, runs SExtractor (inside
Astrometry.net), PSFex to extract the PSF of the images, and
performs Barak's optimal subtraction to produce the subtracted
image (D), the significance image (S), and the corrected
significance image (Scorr - see Zackay, Ofek & Gal-Yam 2016, ApJ,
830, 27).
Written by Paul Vreeswijk ([email protected])
"""
start_time1 = os.times()
# define the base names of input fits files, base_new and
# base_ref, as global so they can be used in any function in this
# module
global base_new, base_ref
base_new = new_fits[0:-5]
base_ref = ref_fits[0:-5]
# read in header of new_fits
t = time.time()
with pyfits.open(new_fits) as hdulist:
header_new = hdulist[0].header
keywords = ['NAXIS2','NAXIS1','GAIN','RDNOISE','SATURATE','RA','DEC']
ysize, xsize, gain_new, readnoise_new, satlevel_new, ra_new, dec_new = read_header(header_new, keywords)
if verbose:
print keywords
print read_header(header_new, keywords)
# read in header of ref_fits
with pyfits.open(ref_fits) as hdulist:
header_ref = hdulist[0].header
ysize_ref, xsize_ref, gain_ref, readnoise_ref, satlevel_ref, ra_ref, dec_ref = read_header(header_ref, keywords)
if verbose:
print keywords
print read_header(header_ref, keywords)
if docosmics:
# clean new image of cosmic rays
new_fits_crr = base_new+'_crr.fits'
new_fits_crrmask = base_new+'_crrmask.fits'
if not os.path.isfile(new_fits_crr) or redo:
result = clean_cosmics(new_fits, new_fits, new_fits_crrmask,
gain_new, readnoise_new, 5.0, 0.3, 5.0, 4, -1., False)
# clean ref image of cosmic rays
ref_fits_crr = base_ref+'_crr.fits'
ref_fits_crrmask = base_ref+'_crrmask.fits'
if not os.path.isfile(ref_fits_crr) or redo:
result = clean_cosmics(ref_fits, ref_fits, ref_fits_crrmask,
gain_ref, readnoise_ref, 5.0, 0.3, 5.0, 4, -1., False)
# determine WCS solution of new_fits
new_fits_wcs = base_new+'_wcs.fits'
if not os.path.isfile(new_fits_wcs) or redo:
result = run_wcs(base_new+'.fits', new_fits_wcs, ra_new, dec_new,
gain_new, readnoise_new)
# determine WCS solution of ref_fits
ref_fits_wcs = base_ref+'_wcs.fits'
if not os.path.isfile(ref_fits_wcs) or redo:
result = run_wcs(base_ref+'.fits', ref_fits_wcs, ra_ref, dec_ref,
gain_ref, readnoise_ref)
# remap ref to new
ref_fits_remap = base_ref+'_wcs_remap.fits'
if not os.path.isfile(ref_fits_remap) or redo:
result = run_remap(base_new+'_wcs.fits', base_ref+'_wcs.fits', ref_fits_remap,
[ysize, xsize], gain=gain_new, config='Config/swarp.config')
# initialize full output images
data_D_full = np.ndarray((ysize, xsize), dtype='float32')
data_S_full = np.ndarray((ysize, xsize), dtype='float32')
data_Scorr_full = np.ndarray((ysize, xsize), dtype='float32')
if addfakestar:
data_new_full = np.ndarray((ysize, xsize), dtype='float32')
data_ref_full = np.ndarray((ysize, xsize), dtype='float32')
# determine cutouts
centers, cuts_ima, cuts_ima_fft, cuts_fft, sizes = centers_cutouts(subimage_size, ysize, xsize)
ysize_fft = subimage_size + 2*subimage_border
xsize_fft = subimage_size + 2*subimage_border
nsubs = centers.shape[0]
if verbose:
print 'nsubs', nsubs
for i in range(nsubs):
print 'i', i
print 'cuts_ima[i]', cuts_ima[i]
print 'cuts_ima_fft[i]', cuts_ima_fft[i]
print 'cuts_fft[i]', cuts_fft[i]
# prepare cubes with shape (nsubs, ysize_fft, xsize_fft) with new,
# ref, psf and background images
data_new, psf_new, psf_orig_new, data_new_bkg = prep_optimal_subtraction(base_new+'_wcs.fits',
nsubs, 'new')
data_ref, psf_ref, psf_orig_ref, data_ref_bkg = prep_optimal_subtraction(base_ref+'_wcs.fits',
nsubs, 'ref')
# determine corresponding variance images
var_new = data_new + readnoise_new**2
var_ref = data_ref + readnoise_ref**2
if verbose:
print 'readnoise_new, readnoise_ref', readnoise_new, readnoise_ref
# get x, y and fratios from matching PSFex stars across entire frame
x_fratio, y_fratio, fratio, dra, ddec = get_fratio_radec(base_new+'_wcs.psfexcat',
base_ref+'_wcs.psfexcat',
base_new+'_wcs.sexcat',
base_ref+'_wcs.sexcat')
dx = dra / pixelscale
dy = ddec / pixelscale
dr = np.sqrt(dx**2 + dy**2)
if verbose: print 'standard deviation dr over the full frame:', np.std(dr)
dr_full = np.sqrt(np.median(dr)**2 + np.std(dr)**2)
dx_full = np.sqrt(np.median(dx)**2 + np.std(dx)**2)
dy_full = np.sqrt(np.median(dy)**2 + np.std(dy)**2)
#dr_full = np.std(dr)
#dx_full = np.std(dx)
#dy_full = np.std(dy)
if verbose:
print 'np.median(dr), np.std(dr)', np.median(dr), np.std(dr)
print 'np.median(dx), np.std(dx)', np.median(dx), np.std(dx)
print 'np.median(dy), np.std(dy)', np.median(dy), np.std(dy)
print 'dr_full, dx_full, dy_full', dr_full, dx_full, dy_full
#fratio_median, fratio_std = np.median(fratio), np.std(fratio)
fratio_mean, fratio_median, fratio_std = sigma_clipped_stats(fratio, sigma=2.)
if verbose:
print 'fratio_mean, fratio_median, fratio_std', fratio_mean, fratio_median, fratio_std
if makeplots:
# plot dy vs dx
plt.axis((-1,1,-1,1))
plt.plot(dx, dy, 'go')
plt.xlabel('dx (pixels)')
plt.ylabel('dy (pixels)')
plt.title(new_fits+'\n vs '+ref_fits, fontsize=12)
plt.savefig('dxdy.png')
plt.show()
plt.close()
# plot dr vs x_fratio
plt.axis((0,xsize,0,1))
plt.plot(x_fratio, dr, 'go')
plt.xlabel('x (pixels)')
plt.ylabel('dr (pixels)')
plt.title(new_fits+'\n vs '+ref_fits, fontsize=12)
plt.savefig('drx.png')
plt.show()
plt.close()
# plot dr vs y_fratio
plt.axis((0,ysize,0,1))
plt.plot(y_fratio, dr, 'go')
plt.xlabel('y (pixels)')
plt.ylabel('dr (pixels)')
plt.title(new_fits+'\n vs '+ref_fits, fontsize=12)
plt.savefig('dry.png')
plt.show()
plt.close()
# plot dx vs x_fratio
plt.axis((0,xsize,-1,1))
plt.plot(x_fratio, dx, 'go')
plt.xlabel('x (pixels)')
plt.ylabel('dx (pixels)')
plt.title(new_fits+'\n vs '+ref_fits, fontsize=12)
plt.savefig('dxx.png')
plt.show()
plt.close()
# plot dy vs y_fratio
plt.axis((0,ysize,-1,1))
plt.plot(y_fratio, dy, 'go')
plt.xlabel('y (pixels)')
plt.ylabel('dy (pixels)')
plt.title(new_fits+'\n vs '+ref_fits, fontsize=12)
plt.savefig('dyy.png')
plt.show()
plt.close()
start_time2 = os.times()
for nsub in range(nsubs):
if timing: tloop = time.time()
if verbose:
print '\nNsub:', nsub+1
print '----------'
# determine clipped mean, median and stddev
#mean_new, median_new, stddev_new = sigma_clipped_stats(data_new[nsub], sigma=3.)
#print 'mean_new, median_new, stddev_new', mean_new, median_new, stddev_new
median_new = np.median(data_new[nsub])
stddev_new = np.sqrt(median_new + readnoise_new**2)
if verbose:
print 'median_new, stddev_new', median_new, stddev_new
#mean_ref, median_ref, stddev_ref = sigma_clipped_stats(data_ref[nsub], sigma=3.)
#print 'mean_ref, median_ref, stddev_ref', mean_ref, median_ref, stddev_ref
median_ref = np.median(data_ref[nsub])
stddev_ref = np.sqrt(median_ref + readnoise_ref**2)
if verbose:
print 'median_ref, stddev_ref', median_ref, stddev_ref
show = False
if makeplots and show:
print 'data_new[nsub] data type:', data_new[nsub].dtype
range_new = (median_new-3.*stddev_new, median_new+3.*stddev_new)
bins = np.linspace(range_new[0], range_new[1], 100)
plt.hist(np.ravel(data_new[nsub]), bins, color='green')
plt.xlabel('pixel value (e-)')
plt.ylabel('number')
plt.title('subsection of '+new_fits)
plt.show()
plt.close()
print 'data_ref[nsub] data type:', data_ref[nsub].dtype
range_ref = (median_ref-3.*stddev_ref, median_ref+3.*stddev_ref)
bins = np.linspace(range_ref[0], range_ref[1], 100)
plt.hist(np.ravel(data_ref[nsub]), bins, color='green')
plt.xlabel('pixel value (e-)')
plt.ylabel('number')
plt.title('subsection of '+ref_fits)
plt.show()
plt.close()
# replace low values in subimages
data_new[nsub][data_new[nsub] <= 0.] = median_new
data_ref[nsub][data_ref[nsub] <= 0.] = median_ref
# replace low values in variance subimages
#var_new[nsub][var_new[nsub] < stddev_new**2] = stddev_new**2
#var_ref[nsub][var_ref[nsub] < stddev_ref**2] = stddev_ref**2
if addfakestar:
# add fake star to new image
# first normalize psf_orig_new
psf_orig_new[nsub] /= np.amax(psf_orig_new[nsub])
psf_orig_new[nsub] *= 3.*stddev_new
# place it at the center of the new image
xpos = xsize_fft/2
ypos = ysize_fft/2
data_new[nsub][ypos-50/2:ypos+50/2,xpos-50/2:xpos+50/2] += psf_orig_new[nsub]
if background_sex:
# use background subimages
bkg_new = data_new_bkg[nsub]
bkg_ref = data_ref_bkg[nsub]
else:
# or median values of subimages
bkg_new = median_new
bkg_ref = median_ref
# subtract the background
data_new[nsub] -= bkg_new
data_ref[nsub] -= bkg_ref
# replace saturated pixel values with zero
#data_new[nsub][data_new[nsub] > 0.95*satlevel_new] = 0.
#data_ref[nsub][data_ref[nsub] > 0.95*satlevel_ref] = 0.
# get median fratio from PSFex stars across subimage
subcut = cuts_ima[nsub]
index_sub = ((y_fratio > subcut[0]) & (y_fratio < subcut[1]) &
(x_fratio > subcut[2]) & (x_fratio < subcut[3]))
# take local or full-frame values for fratio
if fratio_local and any(index_sub):
#fratio_mean, f_new, f_new_std = sigma_clipped_stats(fratio[index_sub], sigma=2.5)
f_new, f_new_std = np.median(fratio[index_sub]), np.std(fratio[index_sub])
else:
f_new, f_new_std = fratio_median, fratio_std
# and the same for dx and dy
if dxdy_local and any(index_sub):
dx_sub = np.sqrt(np.median(dx[index_sub])**2 + np.std(dx[index_sub])**2)
dy_sub = np.sqrt(np.median(dy[index_sub])**2 + np.std(dy[index_sub])**2)
if dx_sub > 2.*dx_full or not np.isfinite(dx_sub):
dx_sub = dx_full
if dy_sub > 2.*dy_full or not np.isfinite(dy_sub):
dy_sub = dy_full
else:
dx_sub = dx_full
dy_sub = dy_full
# f_ref is set to one - could also opt to set f_new to unity instead
f_ref = 1.
if verbose:
print 'f_new, f_new_std, f_ref', f_new, f_new_std, f_ref
print 'dx_sub, dy_sub', dx_sub, dy_sub
# call Barak's function: optimal_binary_image_subtraction
data_D, data_S, data_Scorr = optimal_binary_image_subtraction_pmv(data_ref[nsub],
data_new[nsub],
psf_ref[nsub],
psf_new[nsub],
stddev_ref,
stddev_new,
f_ref,
f_new,
var_ref[nsub],
var_new[nsub],
dx_sub,
dy_sub)
# check that robust stddev of Scorr is around unity
if verbose:
mean_Scorr, median_Scorr, stddev_Scorr = sigma_clipped_stats(data_Scorr, sigma=3.)
print 'mean_Scorr, median_Scorr, stddev_Scorr', mean_Scorr, median_Scorr, stddev_Scorr
# put sub images into output frames
subcut = cuts_ima[nsub]
fftcut = cuts_fft[nsub]
y1 = subimage_border
x1 = subimage_border
y2 = subimage_border+subimage_size
x2 = subimage_border+subimage_size
data_D_full[subcut[0]:subcut[1],subcut[2]:subcut[3]] = data_D[y1:y2,x1:x2] / gain_ref
data_S_full[subcut[0]:subcut[1],subcut[2]:subcut[3]] = data_S[y1:y2,x1:x2]
data_Scorr_full[subcut[0]:subcut[1],subcut[2]:subcut[3]] = data_Scorr[y1:y2,x1:x2]
if addfakestar:
if background_sex:
data_new_full[subcut[0]:subcut[1],subcut[2]:subcut[3]] = (data_new[nsub][y1:y2,x1:x2]
+ bkg_new[y1:y2,x1:x2]) / gain_new
data_ref_full[subcut[0]:subcut[1],subcut[2]:subcut[3]] = (data_ref[nsub][y1:y2,x1:x2]
+ bkg_ref[y1:y2,x1:x2]) / gain_ref
else:
data_new_full[subcut[0]:subcut[1],subcut[2]:subcut[3]] = (data_new[nsub][y1:y2,x1:x2]
+ bkg_new) / gain_new
data_ref_full[subcut[0]:subcut[1],subcut[2]:subcut[3]] = (data_ref[nsub][y1:y2,x1:x2]
+ bkg_ref) / gain_ref
if display and (nsub == 65 or nsub==0):
# just for displaying purpose:
pyfits.writeto('D.fits', data_D, clobber=True)
pyfits.writeto('S.fits', data_S, clobber=True)
pyfits.writeto('Scorr.fits', data_Scorr, clobber=True)
#pyfits.writeto('Scorr_1sigma.fits', data_Scorr_1sigma, clobber=True)
# write new and ref subimages to fits
subname = '_sub'+str(nsub)
newname = base_new+'_wcs'+subname+'.fits'
pyfits.writeto(newname, data_new[nsub]+bkg_new, clobber=True)
refname = base_ref+'_wcs'+subname+'.fits'
pyfits.writeto(refname, data_ref[nsub]+bkg_ref, clobber=True)
# variance images
pyfits.writeto('Vnew.fits', var_new[nsub], clobber=True)
pyfits.writeto('Vref.fits', var_ref[nsub], clobber=True)
# and display
cmd = ['ds9','-zscale',newname,refname,'D.fits','S.fits','Scorr.fits']
cmd = ['ds9','-zscale',newname,refname,'D.fits','S.fits','Scorr.fits',
'Vnew.fits', 'Vref.fits', 'VSn.fits', 'VSr.fits',
'VSn_ast.fits', 'VSr_ast.fits', 'Sn.fits', 'Sr.fits', 'kn.fits', 'kr.fits']
result = call(cmd)
if timing: print 'wall-time spent in nsub loop', time.time()-tloop
end_time = os.times()
dt_usr = end_time[2] - start_time2[2]
dt_sys = end_time[3] - start_time2[3]
dt_wall = end_time[4] - start_time2[4]
print
print "Elapsed user time in {0}: {1:.3f} sec".format("optsub", dt_usr)
print "Elapsed CPU time in {0}: {1:.3f} sec".format("optsub", dt_sys)
print "Elapsed wall time in {0}: {1:.3f} sec".format("optsub", dt_wall)
dt_usr = end_time[2] - start_time1[2]
dt_sys = end_time[3] - start_time1[3]
dt_wall = end_time[4] - start_time1[4]
print
print "Elapsed user time in {0}: {1:.3f} sec".format("total", dt_usr)
print "Elapsed CPU time in {0}: {1:.3f} sec".format("total", dt_sys)
print "Elapsed wall time in {0}: {1:.3f} sec".format("total", dt_wall)
# write full new, ref, D and S images to fits
if addfakestar:
pyfits.writeto('new.fits', data_new_full, header_new, clobber=True)
pyfits.writeto('ref.fits', data_ref_full, header_ref, clobber=True)
pyfits.writeto('D.fits', data_D_full, clobber=True)
pyfits.writeto('S.fits', data_S_full, clobber=True)
pyfits.writeto('Scorr.fits', data_Scorr_full, clobber=True)
# and display
if addfakestar:
cmd = ['ds9','-zscale','new.fits','ref.fits','D.fits','S.fits','Scorr.fits']
else:
cmd = ['ds9','-zscale',new_fits,ref_fits_remap,'D.fits','S.fits','Scorr.fits']
result = call(cmd)
def check_finished(self, max_plugins_output_length):
# pylint: disable=too-many-branches
"""Handle action if it is finished (get stdout, stderr, exit code...)
:param max_plugins_output_length: max plugin data length
:type max_plugins_output_length: int
:return: None
"""
self.last_poll = time.time()
_, _, child_utime, child_stime, _ = os.times()
# Not yet finished...
if self.process.poll() is None:
# We must wait, but checks are variable in time so we do not wait the same
# for a little check or a long ping. So we do like TCP: slow start with a very
# shot time (0.0001 s) increased *2 but do not wait more than 0.5 s.
self.wait_time = min(self.wait_time * 2, 0.5)
now = time.time()
# This log is really spamming... uncomment if you really need this information :)
# logger.debug("%s - Process pid=%d is still alive", now, self.process.pid)
# Get standard outputs in non blocking mode from the process streams
stdout = no_block_read(self.process.stdout)
stderr = no_block_read(self.process.stderr)
try:
self.stdoutdata += stdout.decode("utf-8")
self.stderrdata += stderr.decode("utf-8")
except AttributeError:
pass
if (now - self.check_time) > self.timeout:
logger.warning(
"Process pid=%d spent too much time: %.2f seconds",
self.process.pid, now - self.check_time)
self._in_timeout = True
self._kill()
self.status = ACT_STATUS_TIMEOUT
self.execution_time = now - self.check_time
self.exit_status = 3
if self.log_actions:
if os.environ['ALIGNAK_LOG_ACTIONS'] == 'WARNING':
logger.warning("Action '%s' exited on timeout (%d s)",
self.command, self.timeout)
else:
logger.info("Action '%s' exited on timeout (%d s)",
self.command, self.timeout)
# Do not keep the process objcet
del self.process
# Replace stdout with stderr if stdout is empty
self.stdoutdata = self.stdoutdata.strip()
if not self.stdoutdata:
self.stdoutdata = self.stderrdata
# Now grep what we want in the output
self.get_outputs(self.stdoutdata, max_plugins_output_length)
# We can clean the useless properties now
del self.stdoutdata
del self.stderrdata
# Get the user and system time
_, _, n_child_utime, n_child_stime, _ = os.times()
self.u_time = n_child_utime - child_utime
self.s_time = n_child_stime - child_stime
return
return
logger.debug("Process pid=%d exited with %d", self.process.pid,
self.process.returncode)
if fcntl:
# Get standard outputs in non blocking mode from the process streams
stdout = no_block_read(self.process.stdout)
stderr = no_block_read(self.process.stderr)
else:
# Get standard outputs from the communicate function
(stdout, stderr) = self.process.communicate()
try:
self.stdoutdata += stdout.decode("utf-8")
self.stderrdata += stderr.decode("utf-8")
except AttributeError:
pass
self.exit_status = self.process.returncode
if self.log_actions:
if os.environ['ALIGNAK_LOG_ACTIONS'] == 'WARNING':
logger.warning("Action '%s' exited with code %d", self.command,
self.exit_status)
else:
logger.info("Action '%s' exited with code %d", self.command,
self.exit_status)
# We do not need the process now
del self.process
# check for bad syntax in command line:
if (self.stderrdata.find('sh: -c: line 0: unexpected EOF') >= 0 or
(self.stderrdata.find('sh: -c: ') >= 0
and self.stderrdata.find(': Syntax') >= 0 or
self.stderrdata.find('Syntax error: Unterminated quoted string')
>= 0)):
logger.warning("Bad syntax in command line!")
# Very, very ugly. But subprocess._handle_exitstatus does
# not see a difference between a regular "exit 1" and a
# bailing out shell. Strange, because strace clearly shows
# a difference. (exit_group(1) vs. exit_group(257))
self.stdoutdata = self.stdoutdata + self.stderrdata
self.exit_status = 3
# Make sure that exit code is a valid exit code
if self.exit_status not in VALID_EXIT_STATUS:
self.exit_status = 3
# Replace stdout with stderr if stdout is empty
self.stdoutdata = self.stdoutdata.strip()
if not self.stdoutdata:
self.stdoutdata = self.stderrdata
# Now grep what we want in the output
self.get_outputs(self.stdoutdata, max_plugins_output_length)
# We can clean the useless properties now
del self.stdoutdata
del self.stderrdata
self.status = ACT_STATUS_DONE
self.execution_time = time.time() - self.check_time
# Also get the system and user times
_, _, n_child_utime, n_child_stime, _ = os.times()
self.u_time = n_child_utime - child_utime
self.s_time = n_child_stime - child_stime
def _searchOpen(self, goal_fn, heur_fn, fval_function, costbound):
"""
Search, starting from self.open.
@param goal_fn: the goal function.
@param heur_fn: the heuristic function.
@param fval_function: the f-value function (only relevant when using a custom search strategy).
@param costbound: the cost bound 3-tuple, as described in the assignment.
"""
#BEGIN TRACING
if self.trace:
print(" TRACE: Initial OPEN: ", self.open.print_open())
if self.cycle_check == _CC_FULL:
print(" TRACE: Initial CC_Dict:", self.cc_dictionary)
#END TRACING
while not self.open.empty():
node = self.open.extract()
#BEGIN TRACING
if self.trace:
print(" TRACE: Next State to expand: <S{}:{}:{}, g={}, h={}, f=g+h={}>".format(
node.state.index, node.state.action, node.state.hashable_state(), node.gval, node.hval, node.gval + node.hval))
if node.state.gval != node.gval:
print("ERROR: Node gval not equal to state gval!")
#END TRACING
if goal_fn(node.state):
#node at front of OPEN is a goal...search is completed.
return node
if self.search_stop_time: #timebound check
if os.times()[0] > self.search_stop_time:
#exceeded time bound, must terminate search
print("TRACE: Search has exceeeded the time bound provided")
return False
#All states reached by a search node on OPEN have already
#been hashed into the self.cc_dictionary. However,
#before expanding a node we might have already expanded
#an equivalent state with lower g-value. So only expand
#the node if the hashed g-value is no greater than the
#node's current g-value.
#BEGIN TRACING
if self.trace:
if self.cycle_check == _CC_FULL: print(" TRACE: CC_dict gval={}, node.gval={}".format(
self.cc_dictionary[node.state.hashable_state()], node.gval))
#END TRACING
if self.cycle_check == _CC_FULL and self.cc_dictionary[node.state.hashable_state()] < node.gval:
continue
successors = node.state.successors()
#BEGIN TRACING
if self.trace:
print(" TRACE: Expanding Node. Successors = {", end="")
for ss in successors:
print("<S{}:{}:{}, g={}, h={}, f=g+h={}>, ".format(
ss.index, ss.action, ss.hashable_state(), ss.gval, heur_fn(ss), ss.gval+heur_fn(ss)), end="")
print("}")
#END TRACING
for succ in successors:
hash_state = succ.hashable_state()
if self.trace > 1:
if self.cycle_check == _CC_FULL and hash_state in self.cc_dictionary:
print(" TRACE: Already in CC_dict, CC_dict gval={}, successor state gval={}".format(
self.cc_dictionary[hash_state], succ.gval))
#BEGIN TRACING
if self.trace > 1:
print(" TRACE: Successor State:", end="")
succ.print_state()
print(" TRACE: Heuristic Value:", heur_fn(succ))
if self.cycle_check == _CC_FULL and hash_state in self.cc_dictionary:
print(" TRACE: Already in CC_dict, CC_dict gval={}, successor state gval={}".format(
self.cc_dictionary[hash_state], succ.gval))
if self.cycle_check == _CC_PATH and succ.has_path_cycle():
print(" TRACE: On cyclic path")
#END TRACING
prune_succ = (self.cycle_check == _CC_FULL and
hash_state in self.cc_dictionary and
succ.gval > self.cc_dictionary[hash_state]
) or (
self.cycle_check == _CC_PATH and
succ.has_path_cycle()
)
if prune_succ :
self.cycle_check_pruned = self.cycle_check_pruned + 1
#BEGIN TRACING
if self.trace > 1:
print(" TRACE: Successor State pruned by cycle checking")
print("\n")
#END TRACING
continue
succ_hval = heur_fn(succ)
if costbound is not None and (succ.gval > costbound[0] or
succ_hval > costbound[1] or
succ.gval + succ_hval > costbound[2]) :
self.cost_bound_pruned = self.cost_bound_pruned + 1
if self.trace > 1:
print(" TRACE: Successor State pruned, over current cost bound of {}", costbound)
print("\n")
continue
#passed all cycle checks and costbound checks ...add to open
self.open.insert(sNode(succ, succ_hval, node.fval_function))
#BEGIN TRACING
if self.trace > 1:
print(" TRACE: Successor State added to OPEN")
print("\n")
#END TRACING
#record cost of this path in dictionary.
if self.cycle_check == _CC_FULL:
self.cc_dictionary[hash_state] = succ.gval
#end of while--OPEN is empty and no solution
return False
def __init__(self):
self.temp_dir = os.path.join(tempfile.gettempdir(),
str(hash(os.times())))
if not os.path.exists(self.temp_dir):
os.makedirs(self.temp_dir)
def setUpClass(self):
self.temp_parent = os.path.join(tempfile.gettempdir(), format(hash(os.times())))
self.count = 0
def test_ap_cipher_tkip_countermeasures_sta2(dev, apdev, params):
"""WPA-PSK/TKIP countermeasures (detected by two STAs) [long]"""
if not params['long']:
raise HwsimSkip(
"Skip test case with long duration due to --long not specified")
skip_with_fips(dev[0])
params = {
"ssid": "tkip-countermeasures",
"wpa_passphrase": "12345678",
"wpa": "1",
"wpa_key_mgmt": "WPA-PSK",
"wpa_pairwise": "TKIP"
}
hapd = hostapd.add_ap(apdev[0], params)
testfile = "/sys/kernel/debug/ieee80211/%s/netdev:%s/tkip_mic_test" % (
hapd.get_driver_status_field("phyname"), apdev[0]['ifname'])
if hapd.cmd_execute(["ls", testfile])[0] != 0:
raise HwsimSkip("tkip_mic_test not supported in mac80211")
dev[0].connect("tkip-countermeasures",
psk="12345678",
pairwise="TKIP",
group="TKIP",
scan_freq="2412")
dev[0].dump_monitor()
id = dev[1].connect("tkip-countermeasures",
psk="12345678",
pairwise="TKIP",
group="TKIP",
scan_freq="2412")
dev[1].dump_monitor()
hapd.cmd_execute(["echo", "-n", "ff:ff:ff:ff:ff:ff", ">", testfile],
shell=True)
ev = dev[0].wait_disconnected(
timeout=10, error="No disconnection after two Michael MIC failure")
if "reason=14" not in ev:
raise Exception("Unexpected disconnection reason: " + ev)
ev = dev[1].wait_disconnected(
timeout=5, error="No disconnection after two Michael MIC failure")
if "reason=14" not in ev:
raise Exception("Unexpected disconnection reason: " + ev)
ev = dev[0].wait_event(["CTRL-EVENT-CONNECTED"], timeout=1)
if ev is not None:
raise Exception("Unexpected connection during TKIP countermeasures")
ev = dev[1].wait_event(["CTRL-EVENT-CONNECTED"], timeout=1)
if ev is not None:
raise Exception("Unexpected connection during TKIP countermeasures")
dev[0].request("REMOVE_NETWORK all")
logger.info("Waiting for TKIP countermeasures to end")
connected = False
start = os.times()[4]
while True:
now = os.times()[4]
if start + 70 < now:
break
dev[0].connect("tkip-countermeasures",
psk="12345678",
pairwise="TKIP",
group="TKIP",
scan_freq="2412",
wait_connect=False)
ev = dev[0].wait_event(
["CTRL-EVENT-AUTH-REJECT", "CTRL-EVENT-CONNECTED"], timeout=10)
if ev is None:
raise Exception("No connection result")
if "CTRL-EVENT-CONNECTED" in ev:
connected = True
break
if "status_code=1" not in ev:
raise Exception(
"Unexpected connection failure reason during TKIP countermeasures: "
+ ev)
dev[0].request("REMOVE_NETWORK all")
time.sleep(1)
dev[0].dump_monitor()
dev[1].dump_monitor()
if not connected:
raise Exception("No connection after TKIP countermeasures terminated")
ev = dev[1].wait_event(["CTRL-EVENT-CONNECTED"], timeout=1)
if ev is None:
dev[1].request("DISCONNECT")
dev[1].select_network(id)
dev[1].wait_connected()
def __enter__(self):
self.temp_path = os.path.join(self.temp_dir, str(hash(os.times())))
return self.temp_path
