def getClosestFood(currentGameState, successorGameState,posX,posY):
dict = {(posX,posY):1}
posList = Queue()
posList._put(((posX,posY),0))
walls = successorGameState.getWalls()
food = currentGameState.getFood()
if food[posX][posY] == True:
return 1
check = False
closestFood = 99999
max = 20
current = 0
#from the position of pacman, expands on all sides with a valid path. efficient for dense food positions
while not posList.empty():
((currX, currY),x) = posList._get()
current = x+1
if(food[currX][currY] == True):
check = True
closestFood = min(closestFood,current)
if check == False and (currX-1,currY) not in dict.keys() and walls[currX-1][currY] != True:
posList._put(((currX-1,currY),current))
dict[(currX-1,currY)] = 1
if check == False and (currX,currY-1) not in dict.keys()and walls[currX][currY-1] != True:
posList._put(((currX,currY-1),current))
dict[(currX,currY-1)] = 1
if check == False and (currX+1,currY) not in dict.keys() and walls[currX+1][currY] != True:
posList._put(((currX+1,currY),current))
dict[(currX+1,currY)] = 1
if check == False and(currX,currY+1) not in dict.keys() and walls[currX][currY+1] != True:
posList._put(((currX,currY+1),current))
dict[(currX,currY+1)] = 1
return 1.0/closestFood
def _put(self, item):
delay, item = item
if delay:
if self.task.running:
heapq.heappush(self.delayed, (time.time() + delay, item))
else:
warn("TimeDelayQueue.put called with a delay parameter without background task having been started")
else:
Queue._put(self, item)
def _put_and_notify(self):
with self.not_empty:
while self.delayed:
when, item = heapq.heappop(self.delayed)
if when <= time.time():
Queue._put(self, item)
self.not_empty.notify()
else:
heapq.heappush(self.delayed, (when, item))
break
def getClosestFood(currentGameState, successorGameState,posX,posY):
dict = {(posX,posY):1}
posList = Queue()
posList._put(((posX,posY),0))
walls = successorGameState.getWalls()
food = currentGameState.getFood()
if food[posX][posY] == True:
return 1
check = False
closestFood = 99999
max = 20
current = 0
while not posList.empty():
#print "entered"
((currX, currY),x) = posList._get()
current = x+1
# if current >10:
# break
if(food[currX][currY] == True):
check = True
closestFood = min(closestFood,current)
if check == False and (currX-1,currY) not in dict.keys() and walls[currX-1][currY] != True:
posList._put(((currX-1,currY),current))
dict[(currX-1,currY)] = 1
if check == False and (currX,currY-1) not in dict.keys()and walls[currX][currY-1] != True:
posList._put(((currX,currY-1),current))
dict[(currX,currY-1)] = 1
if check == False and (currX+1,currY) not in dict.keys() and walls[currX+1][currY] != True:
posList._put(((currX+1,currY),current))
dict[(currX+1,currY)] = 1
if check == False and(currX,currY+1) not in dict.keys() and walls[currX][currY+1] != True:
posList._put(((currX,currY+1),current))
dict[(currX,currY+1)] = 1
# if closestFood <3:
# closestFood = 7-closestFood
# elif closestFood < 10:
# closestFood = 5-closestFood/2
#
# elif closestFood >10:
# closestFood = 5-closestFood/4
return 1.0/closestFood
eval_sess.ctlr.done("unexpected eval error")
def _put(self, (eval_sess, is_reeval)):
# Only consider re-evaluation if we are still on the same eval
# session.
if is_reeval and self._curr_eval_sess is not eval_sess:
return
# We only allow *one* eval session at a time.
# - Drop a possible accumulated eval session.
if len(self.queue):
self.queue.clear()
# - Abort the current eval session.
if not is_reeval and self._curr_eval_sess is not None:
self._curr_eval_sess.ctlr.abort()
# Lazily start the eval thread.
if not self.isAlive():
self.start()
Queue._put(self, (eval_sess, is_reeval))
assert len(self.queue) == 1
def _get(self):
eval_sess, is_reeval = Queue._get(self)
if is_reeval:
assert self._curr_eval_sess is eval_sess
else:
self._curr_eval_sess = eval_sess
return eval_sess, is_reeval
def _put(self, item):
Queue._put(self, item)
self.unfinished_tasks += 1
def _put(self, item):
pkg_name = item['name']
if pkg_name not in self.all_items:
self.all_items.add(pkg_name)
Queue._put(self, item)
def _put(self, item):
Queue._put(self, item)
self.unfinished_tasks += 1
def _put(self, item): if item not in self.all_items: Queue._put(self, item) self.all_items.add(item)
def mirror_tree_non_r(root, root2):
s1 = Queue()
s1._put(root)
s2 = Queue()
s2._put(root2)
while s1.empty() != True and s2.empty() != True:
if s1.empty() or s2.empty():
return False
root = s1._get()
root2 = s2._get()
if root.data != root2.data:
return False
if root.left:
s1._put(root.left)
if root2.right:
s2._put(root2.right)
if root.right:
s1._put(root.right)
if root2.left:
s2._put(root2.left)
return True
class mysqlManager:
TYPE_INSERT = "insert"
TYPE_DELETE = "delete"
def __init__(self, dbconfig):
self.accountId = dbconfig["accountId"]
self.db = baseMysql(dbconfig)
self.__queue = Queue()
self.__thread = Thread(target=self.__run) ##io操作另起线程跑
self.__active = True
self.__thread.start()
#----------------------------------------------------------------------
def __run(self):
"""引擎运行"""
while self.__active == True:
try:
event = self.__queue.get(block=True,
timeout=1) # 获取事件的阻塞时间设为1秒
self.__process(event)
except Empty:
pass
#----------------------------------------------------------------------
def __process(self, reqData):
tableName = reqData.table
type = reqData.type
if type == self.TYPE_INSERT:
self.dbInsert_(tableName, reqData.dict_["data"])
if type == self.TYPE_DELETE:
self.dbDelete_(tableName, reqData.dict_["data"])
def putReq(self, req):
self.__queue._put(req)
#----------------------------------------------------------------------
def dbInsert(self, tableName, d):
"""向MongoDB中插入数据，d是具体数据"""
req = DatabaseReq(tableName, self.TYPE_INSERT)
req.dict_['data'] = d
self.putReq(req)
def dbInsert_(self, tableName, d):
sql = getInsertSql(tableName, d, self.accountId)
self.db.dml(sql)
def dbDelete(self, tableName, d):
req = DatabaseReq(tableName, self.TYPE_DELETE)
req.dict_['data'] = d
self.putReq(req)
def dbDelete_(self, tableName, d):
sql = getDeleteSql(tableName, d, self.accountId)
self.db.dml(sql)
##select不做异步，因为一般都是启动加载
def dbSelect(self, tableName, d, ret_type):
sql = getSelectSql(tableName, d, self.accountId, ret_type)
return self.db.query(sql, ret_type)
def find_parameters(options, mse_callback):
def update_param(c, g, p, mse, best_c, best_g, best_p, best_mse, worker,
resumed):
if (mse < best_mse):
best_mse, best_c, best_g, best_p = mse, c, g, p
stdout_str = '[{0}] {1} {2} (best '.format\
(worker,' '.join(str(x) for x in [c,g,p] if x is not None),mse)
output_str = ''
if c != None:
stdout_str += 'c={0}, '.format(best_c)
output_str += 'c={0} '.format(c)
if g != None:
stdout_str += 'g={0}, '.format(best_g)
output_str += 'g={0} '.format(g)
if p != None:
stdout_str += 'p={0}, '.format(best_p)
output_str += 'p={0} '.format(p)
stdout_str += 'mse={0})'.format(best_mse)
print(stdout_str)
return best_c, best_g, best_p, best_mse
options = GridOption(options)
# put jobs in queue
jobs, resumed_jobs = calculate_jobs(options)
job_queue = Queue(0)
result_queue = Queue(0)
for (c, g, p) in resumed_jobs:
result_queue.put(('resumed', c, g, p, resumed_jobs[(c, g, p)]))
for line in jobs:
for (c, g, p) in line:
if (c, g, p) not in resumed_jobs:
job_queue.put((c, g, p))
# hack the queue to become a stack --
# this is important when some thread
# failed and re-put a job. It we still
# use FIFO, the job will be put
# into the end of the queue, and the graph
# will only be updated in the end
job_queue._put = job_queue.queue.appendleft
# fire telnet workers
if telnet_workers:
nr_telnet_worker = len(telnet_workers)
username = getpass.getuser()
password = getpass.getpass()
for host in telnet_workers:
worker = TelnetWorker(host, job_queue, result_queue, host,
username, password, options)
worker.start()
# fire ssh workers
if ssh_workers:
for host in ssh_workers:
worker = SSHWorker(host, job_queue, result_queue, host, options)
worker.start()
# fire local workers
for i in range(nr_local_worker):
worker = LocalWorker('local thread-' + str(i), job_queue, result_queue,
options, mse_callback)
worker.start()
# gather results
done_jobs = {}
db = []
best_mse = float('+inf')
best_c, best_g, best_p = None, None, None
for (c, g, p) in resumed_jobs:
mse = resumed_jobs[(c, g, p)]
best_c, best_g, best_p, best_mse = update_param(
c, g, p, mse, best_c, best_g, best_p, best_mse, 'resumed', True)
for line in jobs:
for (c, g, p) in line:
while (c, g, p) not in done_jobs:
(worker, c1, g1, p1, mse1) = result_queue.get()
done_jobs[(c1, g1, p1)] = mse1
if (c1, g1, p1) not in resumed_jobs:
best_c, best_g, best_p, best_mse = update_param(
c1, g1, p1, mse1, best_c, best_g, best_p, best_mse,
worker, False)
db.append((c, g, p, done_jobs[(c, g, p)]))
job_queue.put((WorkerStopToken, None, None))
best_param, best_cgp = {}, []
if best_c != None:
best_param['c'] = best_c
best_cgp += [best_c]
if best_g != None:
best_param['g'] = best_g
best_cgp += [best_g]
if best_p != None:
best_param['p'] = best_p
best_cgp += [best_p]
print('{0} {1}'.format(' '.join(map(str, best_cgp)), best_mse))
return best_mse, best_param
def find_parameters(dataset_pathname, options=''):
options = GridOption(dataset_pathname, options);
if options.gnuplot_pathname:
gnuplot = Popen(options.gnuplot_pathname,stdin = PIPE,stdout=PIPE,stderr=PIPE).stdin
else:
gnuplot = None
# put jobs in queue
jobs,resumed_jobs = calculate_jobs(options)
job_queue = Queue(0)
result_queue = Queue(0)
for (c,g) in resumed_jobs:
result_queue.put(('resumed',c,g,resumed_jobs[(c,g)]))
for line in jobs:
for (c,g) in line:
if (c,g) not in resumed_jobs:
job_queue.put((c,g))
# hack the queue to become a stack --
# this is important when some thread
# failed and re-put a job. It we still
# use FIFO, the job will be put
# into the end of the queue, and the graph
# will only be updated in the end
job_queue._put = job_queue.queue.appendleft
# Broadcast the options
for i in range(1, total_mpi):
comm.send(options.__dict__, dest=i, tag=0)
njobs_per_mpi = job_queue.qsize() / total_mpi
job_queue0 = Queue(0)
for i in range(0, njobs_per_mpi):
job_queue0.put(job_queue.get())
# Dispatch the remained jobs
id = 0
rnd = [0] * total_mpi
while not job_queue.empty():
id = (id + 1) % total_mpi
if id == 0:
continue
rnd[id] += 1
(cexp, gexp) = job_queue.get()
comm.isend((cexp, gexp), dest=id, tag=rnd[id])
for i in range(1, total_mpi):
comm.isend((None, None), dest=i, tag=(rnd[i]+1))
while not job_queue0.empty():
(cexp0, gexp0) = job_queue0.get()
worker = Worker(options.__dict__, cexp0, gexp0)
(cexp, gexp, rate) = worker.run()
result_queue.put(('0', cexp, gexp, rate))
for id in range(1, total_mpi):
for i in range(1, rnd[id]+1):
(workerid, cexp, gexp, rate) = comm.recv(source=id, tag=i)
result_queue.put((workerid, cexp, gexp, rate))
best_rate = -1
best_c,best_g = None, None
while not result_queue.empty():
(workerid, c, g, rate) = result_queue.get()
best_c, best_g, best_rate = update_param(options, c, g, rate, best_c, best_g, best_rate, 'mpi'+workerid, False)
if gnuplot and options.grid_with_c and options.grid_with_g:
redraw(db,[best_c, best_g, best_rate],gnuplot,options)
redraw(db,[best_c, best_g, best_rate],gnuplot,options,True)
if options.out_pathname:
result_file.close()
best_param, best_cg = {}, []
if best_c != None:
best_param['c'] = 2.0**best_c
best_cg += [2.0**best_c]
if best_g != None:
best_param['g'] = 2.0**best_g
best_cg += [2.0**best_g]
print('{0} {1}'.format(' '.join(map(str,best_cg)), best_rate))
return best_rate, best_param
def find_parameters(dataset_pathname, options=""):
def update_param(c, g, p, mse, best_c, best_g, best_p, best_mse, worker, resumed):
if mse < best_mse:
best_mse, best_c, best_g, best_p = mse, c, g, p
stdout_str = "[{0}] {1} {2} (best ".format(worker, " ".join(str(x) for x in [c, g, p] if x is not None), mse)
output_str = ""
if c != None:
stdout_str += "c={0}, ".format(2.0 ** best_c)
output_str += "log2c={0} ".format(c)
if g != None:
stdout_str += "g={0}, ".format(2.0 ** best_g)
output_str += "log2g={0} ".format(g)
if p != None:
stdout_str += "p={0}, ".format(2.0 ** best_p)
output_str += "log2p={0} ".format(p)
stdout_str += "mse={0})".format(best_mse)
print(stdout_str)
if options.out_pathname and not resumed:
output_str += "mse={0}\n".format(mse)
result_file.write(output_str)
result_file.flush()
return best_c, best_g, best_p, best_mse
options = GridOption(dataset_pathname, options)
if options.gnuplot_pathname:
gnuplot = Popen(options.gnuplot_pathname, stdin=PIPE, stdout=PIPE, stderr=PIPE).stdin
else:
gnuplot = None
# put jobs in queue
jobs, resumed_jobs = calculate_jobs(options)
job_queue = Queue(0)
result_queue = Queue(0)
for (c, g, p) in resumed_jobs:
result_queue.put(("resumed", c, g, p, resumed_jobs[(c, g, p)]))
for line in jobs:
for (c, g, p) in line:
if (c, g, p) not in resumed_jobs:
job_queue.put((c, g, p))
# hack the queue to become a stack --
# this is important when some thread
# failed and re-put a job. It we still
# use FIFO, the job will be put
# into the end of the queue, and the graph
# will only be updated in the end
job_queue._put = job_queue.queue.appendleft
# fire telnet workers
if telnet_workers:
nr_telnet_worker = len(telnet_workers)
username = getpass.getuser()
password = getpass.getpass()
for host in telnet_workers:
worker = TelnetWorker(host, job_queue, result_queue, host, username, password, options)
worker.start()
# fire ssh workers
if ssh_workers:
for host in ssh_workers:
worker = SSHWorker(host, job_queue, result_queue, host, options)
worker.start()
# fire local workers
for i in range(nr_local_worker):
worker = LocalWorker("local", job_queue, result_queue, options)
worker.start()
# gather results
done_jobs = {}
if options.out_pathname:
if options.resume_pathname:
result_file = open(options.out_pathname, "a")
else:
result_file = open(options.out_pathname, "w")
db = []
best_mse = float("+inf")
best_c, best_g, best_p = None, None, None
for (c, g, p) in resumed_jobs:
mse = resumed_jobs[(c, g, p)]
best_c, best_g, best_p, best_mse = update_param(c, g, p, mse, best_c, best_g, best_p, best_mse, "resumed", True)
for line in jobs:
for (c, g, p) in line:
while (c, g, p) not in done_jobs:
(worker, c1, g1, p1, mse1) = result_queue.get()
done_jobs[(c1, g1, p1)] = mse1
if (c1, g1, p1) not in resumed_jobs:
best_c, best_g, best_p, best_mse = update_param(
c1, g1, p1, mse1, best_c, best_g, best_p, best_mse, worker, False
)
db.append((c, g, p, done_jobs[(c, g, p)]))
if gnuplot and options.grid_with_c and options.grid_with_g:
redraw(db, [best_c, best_g, best_rate], gnuplot, options)
redraw(db, [best_c, best_g, best_rate], gnuplot, options, True)
if options.out_pathname:
result_file.close()
job_queue.put((WorkerStopToken, None, None))
best_param, best_cgp = {}, []
if best_c != None:
best_param["c"] = 2.0 ** best_c
best_cgp += [2.0 ** best_c]
if best_g != None:
best_param["g"] = 2.0 ** best_g
best_cgp += [2.0 ** best_g]
if best_p != None:
best_param["p"] = 2.0 ** best_p
best_cgp += [2.0 ** best_p]
print("{0} {1}".format(" ".join(map(str, best_cgp)), best_mse))
return best_mse, best_param
def _put(self, item):
if isinstance(item, basestring) and self.task is not None:
self.task.updateStatus(item)
else:
Queue._put(self, item)
def _put(self, item):
if item not in self.all_items:
Queue._put(self, item)
self.all_items.add(item)
def _put(self, item):
Queue._put(self, item)
self._put_done()
def find_parameters(dataset_pathname, options=''):
def update_param(c, g, rate, best_c, best_g, best_rate, worker, resumed):
if (rate > best_rate) or (rate == best_rate and g == best_g
and c < best_c):
best_rate, best_c, best_g = rate, c, g
stdout_str = '[{0}] {1} {2} (best '.format\
(worker,' '.join(str(x) for x in [c,g] if x is not None),rate)
output_str = ''
if c != None:
stdout_str += 'c={0}, '.format(2.0**best_c)
output_str += 'log2c={0} '.format(c)
if g != None:
stdout_str += 'g={0}, '.format(2.0**best_g)
output_str += 'log2g={0} '.format(g)
stdout_str += 'rate={0})'.format(best_rate)
print(stdout_str)
if options.out_pathname and not resumed:
output_str += 'rate={0}\n'.format(rate)
result_file.write(output_str)
result_file.flush()
return best_c, best_g, best_rate
options = GridOption(dataset_pathname, options)
if options.gnuplot_pathname:
gnuplot = Popen(options.gnuplot_pathname,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE).stdin
else:
gnuplot = None
# put jobs in queue
jobs, resumed_jobs = calculate_jobs(options)
job_queue = Queue(0)
result_queue = Queue(0)
for (c, g) in resumed_jobs:
result_queue.put(('resumed', c, g, resumed_jobs[(c, g)]))
for line in jobs:
for (c, g) in line:
if (c, g) not in resumed_jobs:
job_queue.put((c, g))
# hack the queue to become a stack --
# this is important when some thread
# failed and re-put a job. It we still
# use FIFO, the job will be put
# into the end of the queue, and the graph
# will only be updated in the end
job_queue._put = job_queue.queue.appendleft
# fire telnet workers
if telnet_workers:
nr_telnet_worker = len(telnet_workers)
username = getpass.getuser()
password = getpass.getpass()
for host in telnet_workers:
worker = TelnetWorker(host, job_queue, result_queue, host,
username, password, options)
worker.start()
# fire ssh workers
if ssh_workers:
for host in ssh_workers:
worker = SSHWorker(host, job_queue, result_queue, host, options)
worker.start()
# fire local workers
for i in range(nr_local_worker):
worker = LocalWorker('local', job_queue, result_queue, options)
worker.start()
# gather results
done_jobs = {}
if options.out_pathname:
if options.resume_pathname:
result_file = open(options.out_pathname, 'a')
else:
result_file = open(options.out_pathname, 'w')
db = []
best_rate = -1
best_c, best_g = None, None
for (c, g) in resumed_jobs:
rate = resumed_jobs[(c, g)]
best_c, best_g, best_rate = update_param(c, g, rate, best_c, best_g,
best_rate, 'resumed', True)
for line in jobs:
for (c, g) in line:
while (c, g) not in done_jobs:
(worker, c1, g1, rate1) = result_queue.get()
done_jobs[(c1, g1)] = rate1
if (c1, g1) not in resumed_jobs:
best_c, best_g, best_rate = update_param(
c1, g1, rate1, best_c, best_g, best_rate, worker,
False)
db.append((c, g, done_jobs[(c, g)]))
if gnuplot and options.grid_with_c and options.grid_with_g:
redraw(db, [best_c, best_g, best_rate], gnuplot, options)
redraw(db, [best_c, best_g, best_rate], gnuplot, options, True)
if options.out_pathname:
result_file.close()
job_queue.put((WorkerStopToken, None))
best_param, best_cg = {}, []
if best_c != None:
best_param['c'] = 2.0**best_c
best_cg += [2.0**best_c]
if best_g != None:
best_param['g'] = 2.0**best_g
best_cg += [2.0**best_g]
print('{0} {1}'.format(' '.join(map(str, best_cg)), best_rate))
return [best_rate, best_param]
def _put(self, item):
Queue._put(self, item)
self._put_done()
def _put(self, item):
Queue._put(self, item)
self.sort()
def _put(self, item):
Queue._put(self, item)
self.sort()
if len(self.queue):
pending = list(self.queue)
self.queue.clear()
for evalr, _ in pending:
try:
evalr.close()
except:
log.exception("Failed to close evalr")
## - Abort the current eval session.
if not is_reeval and self._curr_eval_sess is not None:
self._curr_eval_sess.ctlr.abort()
# Lazily start the eval thread.
if not self.isAlive():
self.start()
Queue._put(self, (eval_sess, is_reeval))
if replace:
assert len(self.queue) == 1
def _get(self):
eval_sess, is_reeval = Queue._get(self)
if is_reeval:
assert self._curr_eval_sess is eval_sess
else:
self._curr_eval_sess = eval_sess
return eval_sess, is_reeval
def find_parameters(dataset_pathname, options=''):
options = GridOption(dataset_pathname, options)
if options.gnuplot_pathname:
gnuplot = Popen(options.gnuplot_pathname,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE).stdin
else:
gnuplot = None
# put jobs in queue
jobs, resumed_jobs = calculate_jobs(options)
job_queue = Queue(0)
result_queue = Queue(0)
for (c, g) in resumed_jobs:
result_queue.put(('resumed', c, g, resumed_jobs[(c, g)]))
for line in jobs:
for (c, g) in line:
if (c, g) not in resumed_jobs:
job_queue.put((c, g))
# hack the queue to become a stack --
# this is important when some thread
# failed and re-put a job. It we still
# use FIFO, the job will be put
# into the end of the queue, and the graph
# will only be updated in the end
job_queue._put = job_queue.queue.appendleft
# Broadcast the options
for i in range(1, total_mpi):
comm.send(options.__dict__, dest=i, tag=0)
njobs_per_mpi = job_queue.qsize() / total_mpi
job_queue0 = Queue(0)
for i in range(0, njobs_per_mpi):
job_queue0.put(job_queue.get())
# Dispatch the remained jobs
id = 0
rnd = [0] * total_mpi
while not job_queue.empty():
id = (id + 1) % total_mpi
if id == 0:
continue
rnd[id] += 1
(cexp, gexp) = job_queue.get()
comm.isend((cexp, gexp), dest=id, tag=rnd[id])
for i in range(1, total_mpi):
comm.isend((None, None), dest=i, tag=(rnd[i] + 1))
while not job_queue0.empty():
(cexp0, gexp0) = job_queue0.get()
worker = Worker(options.__dict__, cexp0, gexp0)
(cexp, gexp, rate) = worker.run()
result_queue.put(('0', cexp, gexp, rate))
for id in range(1, total_mpi):
for i in range(1, rnd[id] + 1):
(workerid, cexp, gexp, rate) = comm.recv(source=id, tag=i)
result_queue.put((workerid, cexp, gexp, rate))
best_rate = -1
best_c, best_g = None, None
while not result_queue.empty():
(workerid, c, g, rate) = result_queue.get()
best_c, best_g, best_rate = update_param(options, c, g, rate,
best_c, best_g, best_rate,
'mpi' + workerid, False)
if gnuplot and options.grid_with_c and options.grid_with_g:
redraw(db, [best_c, best_g, best_rate], gnuplot, options)
redraw(db, [best_c, best_g, best_rate], gnuplot, options, True)
if options.out_pathname:
result_file.close()
best_param, best_cg = {}, []
if best_c != None:
best_param['c'] = 2.0**best_c
best_cg += [2.0**best_c]
if best_g != None:
best_param['g'] = 2.0**best_g
best_cg += [2.0**best_g]
print('{0} {1}'.format(' '.join(map(str, best_cg)), best_rate))
return best_rate, best_param
def find_parameters(dataset_pathname, options=''):
def update_param(c, g, rate, best_c, best_g, best_rate, worker, resumed):
if (rate > best_rate) or (rate == best_rate and g == best_g and c < best_c):
best_rate, best_c, best_g = rate, c, g
stdout_str = '[{0}] {1} {2} (best '.format \
(worker, ' '.join(str(x) for x in [c, g] if x is not None), rate)
output_str = ''
if c != None:
stdout_str += 'c={0}, '.format(2.0 ** best_c)
output_str += 'log2c={0} '.format(c)
if g != None:
stdout_str += 'g={0}, '.format(2.0 ** best_g)
output_str += 'log2g={0} '.format(g)
stdout_str += 'rate={0})'.format(best_rate)
print(stdout_str)
if options.out_pathname and not resumed:
output_str += 'rate={0}\n'.format(rate)
result_file.write(output_str)
result_file.flush()
return best_c, best_g, best_rate
options = GridOption(dataset_pathname, options);
if options.gnuplot_pathname:
gnuplot = Popen(options.gnuplot_pathname, stdin=PIPE, stdout=PIPE, stderr=PIPE).stdin
else:
gnuplot = None
# put jobs in queue
jobs, resumed_jobs = calculate_jobs(options)
job_queue = Queue(0)
result_queue = Queue(0)
for (c, g) in resumed_jobs:
result_queue.put(('resumed', c, g, resumed_jobs[(c, g)]))
for line in jobs:
for (c, g) in line:
if (c, g) not in resumed_jobs:
job_queue.put((c, g))
# hack the queue to become a stack --
# this is important when some thread
# failed and re-put a job. It we still
# use FIFO, the job will be put
# into the end of the queue, and the graph
# will only be updated in the end
job_queue._put = job_queue.queue.appendleft
# fire telnet workers
if telnet_workers:
nr_telnet_worker = len(telnet_workers)
username = getpass.getuser()
password = getpass.getpass()
for host in telnet_workers:
worker = TelnetWorker(host, job_queue, result_queue,
host, username, password, options)
worker.start()
# fire ssh workers
if ssh_workers:
for host in ssh_workers:
worker = SSHWorker(host, job_queue, result_queue, host, options)
worker.start()
# fire local workers
for i in range(nr_local_worker):
worker = LocalWorker('local', job_queue, result_queue, options)
worker.start()
# gather results
done_jobs = {}
if options.out_pathname:
if options.resume_pathname:
result_file = open(options.out_pathname, 'a')
else:
result_file = open(options.out_pathname, 'w')
db = []
best_rate = -1
best_c, best_g = None, None
for (c, g) in resumed_jobs:
rate = resumed_jobs[(c, g)]
best_c, best_g, best_rate = update_param(c, g, rate, best_c, best_g, best_rate, 'resumed', True)
for line in jobs:
for (c, g) in line:
while (c, g) not in done_jobs:
(worker, c1, g1, rate1) = result_queue.get()
done_jobs[(c1, g1)] = rate1
if (c1, g1) not in resumed_jobs:
best_c, best_g, best_rate = update_param(c1, g1, rate1, best_c, best_g, best_rate, worker, False)
db.append((c, g, done_jobs[(c, g)]))
if gnuplot and options.grid_with_c and options.grid_with_g:
redraw(db, [best_c, best_g, best_rate], gnuplot, options)
redraw(db, [best_c, best_g, best_rate], gnuplot, options, True)
if options.out_pathname:
result_file.close()
job_queue.put((WorkerStopToken, None))
best_param, best_cg = {}, []
if best_c != None:
best_param['c'] = 2.0 ** best_c
best_cg += [2.0 ** best_c]
if best_g != None:
best_param['g'] = 2.0 ** best_g
best_cg += [2.0 ** best_g]
print('{0} {1}'.format(' '.join(map(str, best_cg)), best_rate))
return best_rate, best_param
def _put(self,item):
Queue._put(self,item)
print "emmiting signal"
self.emit(qtcore.SIGNAL("datainqueue()"))
