def parallel_map(func, items, max_workers=1, single_thread_fallback=True):
"""
Our own implementation for parallel processing
which handles gracefully CTRL+C and reverts to
single thread processing in case of errors
:param items list of objects
:param func function to execute on each object
"""
global error
error = None
def process_one(q):
func(q)
def worker():
global error
while True:
(num, q) = pq.get()
if q is None or error is not None:
pq.task_done()
break
try:
process_one(q)
except Exception as e:
error = e
finally:
pq.task_done()
if max_workers > 1:
use_single_thread = False
pq = queue.PriorityQueue()
threads = []
for i in range(max_workers):
t = threading.Thread(target=worker)
t.start()
threads.append(t)
i = 1
for t in items:
pq.put((i, t.copy()))
i += 1
def stop_workers():
for i in range(len(threads)):
pq.put((-1, None))
for t in threads:
t.join()
# block until all tasks are done
try:
while pq.unfinished_tasks > 0:
time.sleep(0.5)
except KeyboardInterrupt:
print("CTRL+C terminating...")
stop_workers()
sys.exit(1)
stop_workers()
if error is not None and single_thread_fallback:
# Try to reprocess using a single thread
# in case this was a memory error
log.ODM_WARNING("Failed to run process in parallel, retrying with a single thread...")
use_single_thread = True
else:
use_single_thread = True
if use_single_thread:
# Boring, single thread processing
for q in items:
process_one(q)
def AStar_Search(edges, start, goal):
frontier = Q.PriorityQueue()
currentCost = {}
path = {}
initial = [m[1] for m in edges.keys() if m[0] == goal]
for next in initial:
cfg = (goal, next)
# print(next.getASVPositions())
currentCost[cfg] = 0
path[cfg] = None
frontier.put((0, cfg, edges[cfg]))
# j = 0
while not frontier.empty():
# print(j)
# j = j + 1
element = frontier.get()
cfg = element[1] #cfg[0] is current, cfg[1] is destination config
node = element[2]
if node != None:
# print(cfg[0].getASVPositions())
# print(cfg[1].getASVPositions())
if cfg[0].totalDistance(start) <= (0.5 * 2.) \
or cfg[1].totalDistance(start) <= (0.5 * 2.):
print("Solution reached")
c = cfg
route = []
route.extend(edges[cfg])
cost = sum(currentCost.values())
while path[c] != None:
route.insert(0, edges[c])
c = path[c]
i += 1
return cost, route
adjacent = [
n for n in edges.keys() if (n[0] == cfg[1] and n[1] != cfg[0])
]
for next in adjacent:
# if (next[1], next[0]) not in currentCost:
# print(next[0].getASVPositions())
# print(next[1].getASVPositions())
estimatedCost = cfg[1].totalDistance(
next[1]) + currentCost[cfg]
if (next not in currentCost \
and (next[1], next[0])) not in currentCost \
or estimatedCost < currentCost[next]:
# print('hello')
currentCost[next] = estimatedCost
priority = estimatedCost + heuristic(goal, next[0])
frontier.put((priority, next, edges[next]))
path[next] = cfg
# print("nothing")
return None, None
def __init__(self, title, occupancyGrid):
CellBasedForwardSearch.__init__(self, title, occupancyGrid)
self.greedyQueue = Queue.PriorityQueue()
if u not in g.nodes():
g.add_node(u)
temp[u] = 1
score.append(temp)
Graphs.append(g)
for node in g.nodes():
flags[i][node] = 0
if g.degree(node) > DN[i]:
DN[i] = g.degree(node)
return Graphs
IL = []
H_measure = []
max_h_measure = []
K_Shell_measure = []
I_U_value = []
que = Q.PriorityQueue()
ques = []
seed = set()
seeds = []
def reset():
global IL
global H_measure
global max_h_measure
global K_Shell_measure
global I_U_value
global seeds
global flag
for i in range(m):
flags.append(dict())
self.completed = True
env.updateState(self.status, source, destination)
return self.completed
#load road network, starting time, and paths for vehicles
road = np.array(pk.load(open("road", "r")))
time = np.array(pk.load(open("time", "r")))
paths = np.array(pk.load(open("vehicle", "r")))
#store the total number of cars
n = len(time)
#create a city and place vehicles in the city with the given information in a priority queue
#the queue takes next instant the vehicle changes the road as priority
city = Environment(road)
vehicle = Q.PriorityQueue()
for i in range(n):
vehicle.put(Agent(time[i], paths[i], i))
#create an empty output matrix (to be filled as vehicles complete their journey)
output = np.zeros((n, 5))
#while the vehicles who have not completed the journey exist
while not vehicle.empty():
#get the vehicle with next time instance
min = vehicle.get()
#take action and check whether it has completed it's journey
min_completed = min.takeAction(city)
#if not, put it back in the queue
if (not min_completed):
vehicle.put(min)
def document_selection_task(topicId, topic_initial_info, per_topic_train_index_list, al_protocol, batch_size, collection_size='qrels'):
per_topic_X, per_topic_y, _, _, per_topic_seed_one_counter, per_topic_seed_zero_counter = topic_initial_info[topicId]
# this train_index_list will be used to calculate the acuracy of the classifier
train_index_list = copy.deepcopy(per_topic_train_index_list)
test_index_list = []
# it means everything in the train list and we do not need to predict
# so we do not need any training of the model
# so return here
if len(per_topic_train_index_list) == len(per_topic_y):
return (per_topic_train_index_list, 1.0, 1.0, 1.0, 0, True) # 0 means no new document selected, as f1, precision, recall all reached 1.0, True means this topic is completed
# print isPredictable.count(1)
total_documents = len(per_topic_y)
train_size_controller = len(per_topic_train_index_list)
size_limit = train_size_controller + batch_size
number_of_document_selected = batch_size
# boundary checking
if size_limit > len(per_topic_y):
size_limit = len(per_topic_y)
number_of_document_selected = len(per_topic_y) - len(per_topic_train_index_list)
per_topic_initial_X_test = []
per_topic_test_index_dictionary = {}
test_index_counter = 0
for train_index in xrange(0, total_documents):
if train_index not in per_topic_train_index_list:
per_topic_initial_X_test.append(per_topic_X[train_index])
per_topic_test_index_dictionary[test_index_counter] = train_index
test_index_counter = test_index_counter + 1
predictableSize = len(per_topic_initial_X_test)
isPredictable = [1] * predictableSize # initially we will predict all
# here modeling is utilizing the document selected in previous
# iteration for training
# when loopCounter == 0
# model is utilizing all the seed document collected at the begining
model = None
if collection_size == 'qrels':
model = LogisticRegression(solver=small_data_solver, C=small_data_C_parameter)
model.fit(per_topic_X[per_topic_train_index_list], per_topic_y[per_topic_train_index_list])
queueSize = isPredictable.count(1)
queue = Queue.PriorityQueue(queueSize)
# these are used for SPL
randomArray = []
for counter in xrange(0, predictableSize):
if isPredictable[counter] == 1:
# model.predict returns a list of values in so we need index [0] as we
# have only one element in the list
y_prob = model.predict_proba(per_topic_initial_X_test[counter])[0]
val = 0
if al_protocol == 'CAL':
val = y_prob[1]
queue.put(relevance(val, counter))
elif al_protocol == 'SAL':
val = calculate_entropy(y_prob[0], y_prob[1])
queue.put(relevance(val, counter))
elif al_protocol == 'SPL':
randomArray.append(counter)
if al_protocol == 'SPL':
random.shuffle(randomArray)
batch_counter = 0
#for batch_counter in xrange(0, batch_size):
# if batch_counter > len(randomArray) - 1:
# break
while True:
if train_size_controller == size_limit:
break
itemIndex = randomArray[batch_counter]
batch_counter = batch_counter + 1
isPredictable[itemIndex] = 0
per_topic_train_index_list.append(per_topic_test_index_dictionary[itemIndex])
# test_index_list will be used for calculating the accuracy of the classiifer
test_index_list.append(per_topic_test_index_dictionary[itemIndex])
train_size_controller = train_size_controller + 1
else:
while not queue.empty():
if train_size_controller == size_limit:
break
item = queue.get()
isPredictable[item.index] = 0 # not predictable
per_topic_train_index_list.append(per_topic_test_index_dictionary[item.index])
# test_index_list will be used for calculating the accuracy of the classiifer
test_index_list.append(per_topic_test_index_dictionary[item.index])
train_size_controller = train_size_controller + 1
f1score, precision, recall, _ = calculate_accuracy_classifier(per_topic_X, per_topic_y, train_index_list, test_index_list,collection_size)
return (per_topic_train_index_list, f1score, precision, recall, number_of_document_selected, False)
N = raw_input('How many processes you have : ')
print N
ls = []
i = 1
while (True):
if (int(i) > int(N)):
break
n = raw_input('Enter Name of process %d: ' % i)
at = raw_input('Enter Arrival Time of process %d: ' % i)
d = raw_input('Enter Duration of process %d: ' % i)
obj = ProcessA(n, at, d)
ls.append(obj)
i += 1
print "Gantt Chart"
queueA = Q.PriorityQueue()
for x in ls:
queueA.put(x)
var = queueA.get()
ob = ProcessR(var.name, var.arrivalTime, var.duration)
queueR = Q.PriorityQueue()
queueR.put(ob)
j = 0
while (not queueA.empty()):
var = queueA.get()
if (int(var.arrivalTime) == j):
ob = ProcessR(var.name, var.arrivalTime, var.duration)
queueR.put(ob)
else:
queueA.put(var)
break
'''
下例 为我们展示了 PriorityQueue 的使用.
如果参数为元组, 元组的第一个成员表示优先级(数值越小优先级越高).
如果参数为数字或字符串, 则按照数字字母的ASCII码排序, 值越小优先级越高.
'''
import Queue
import bisect
Empty = Queue.Empty
#
# try it
queue = Queue.PriorityQueue(0)
# add items out of order
queue.put((20, "second"))
queue.put((10, "first"))
queue.put((30, "third"))
# print queue contents
try:
while 1:
print queue.get_nowait()
except Empty:
pass
scores = []
Graphs = []
status = []
for i in range(m):
scores.append([])
graph = nx.Graph()
for u in nodes:
for v in nodes:
if u == v:
continue
if random.randint(1, 1000) < 996:
continue
graph.add_edge(u, v, weight=round(random.uniform(0, 1), 2))
Graphs.append(graph)
pq = []
PQ = Q.PriorityQueue()
infl = []
ttl_infl = []
for i in range(m):
t2 = []
t3 = []
for u in range(max(nodes) + 1):
t3.append(0)
t1 = []
for v in range(max(nodes) + 1):
t1.append(0)
t2.append(t1)
infl.append(t2)
ttl_infl.append(t3)
seed = set()
seeds = []
def solve(road_segments, idsDepth=sys.maxsize):
closed_list = []
if initial_state == goal_state:
return initial_state
if routing_algo == 'bfs':
fringe = q.Queue()
elif routing_algo == 'dfs' or routing_algo == 'ids':
fringe = q.LifoQueue()
else:
fringe = q.PriorityQueue()
heurSegment, heurDistance, heurTime, heurCost = getHeuristic(
initial_state, goal_state, cost_function)
# dic of all the elements in fringe and explored elements
previoslyExists = {}
# heurCost is to return the heuristic as distance/time when inital and goal state are same.
fringe.put(State(initial_state, 0, 0, 0, heurCost, None, cost_function))
previoslyExists[(initial_state)] = 1
while not fringe.empty():
best = fringe.get()
closed_list.append(best.cityName)
if best.cityName == goal_state:
return best
if routing_algo == 'ids' and best.segment > idsDepth:
continue
if (best.cityName in adjacency_list.keys()):
successor_list = adjacency_list[best.cityName]
for i in successor_list:
if i not in closed_list:
from_city = best.cityName
to_city = i
city_Pair = (from_city, to_city)
costToCurrent = best.cost
segmentSucc, distanceSucc, timeSucc, costSucc = getCostToSuccessor(
from_city, to_city, cost_function, city_Pair)
segmentHeur, distanceHeur, timeHeur, costHeur = getHeuristic(
to_city, goal_state, cost_function)
# costSucc is to generalize. to avoid any loop. This will automatically use this value dependent on cost function
totalCostToSuccessor = costToCurrent + costSucc
currentHash = (to_city)
flag = False
if currentHash in previoslyExists:
for succ in list(fringe.queue):
if to_city in succ.cityName:
if succ.cost > totalCostToSuccessor:
fringe.queue.remove(succ)
else:
flag = True
if not flag:
fringe.put(
State(to_city, best.segment + segmentSucc,
best.distance + distanceSucc,
best.time + timeSucc, costHeur, best,
cost_function))
previoslyExists[currentHash] = 1
return False
def __init__(self, start, goal):
self.path=[]
self.visited_queue=[]
self.priority_queue=Q.PriorityQueue()
self.start=start
self.goal=goal
def rq():
return Queue.PriorityQueue()
def astar(maze):
# TODO: Write your code here
cur_pos = maze.getStart()
dimensions = maze.getDimensions()
visited = [[False for x in range(dimensions[1])]
for y in range(dimensions[0])]
pellet_arr = [[0 for x in range(dimensions[1])]
for y in range(dimensions[0])]
came_from = [[() for x in range(dimensions[1])]
for y in range(dimensions[0])]
num_states_explored = 0
frontier = queue.PriorityQueue()
distance = heuristic(maze, cur_pos)
frontier.put([distance, cur_pos])
came_from[cur_pos[0]][cur_pos[1]] = None
cost_so_far = {}
cost_so_far[cur_pos] = 0
prev = cur_pos
prev_goal = cur_pos
optimal = []
prim_list = maze.getObjectives()
prim_list.insert(0, cur_pos)
obj_list = maze.getObjectives()
pellets = 0
cur_goal = closest_goal(cur_pos, obj_list)
mst_sum = prim(cur_pos, prim_list)
while not frontier.empty():
cur_pos = frontier.get()
location = cur_pos[1] #set location to the coordinates
neighbors = maze.getNeighbors(location[0], location[1])
if (location in obj_list):
obj_list.remove(
location
) #if location is an objective then we move it from the objective list
frontier.queue.clear(
) #clear the priority queue since we will move on to another objective
if (obj_list):
cur_goal = closest_goal(
location, obj_list
) #if obj_list is not empty, we set the current goal to the closest goal
del prim_list[0] #remove the first item from prim list
prim_list.remove(location) #remove the objective we found
prim_list.insert(
0, location) #add the objective we found to the start
mst_sum = prim(
location,
prim_list) #calculate the sum of minimum spanning tree
prev = location
pellets += 1 #increment pellets if we reached a goal
while prev != prev_goal:
prev = came_from[prev[0]][prev[
1]] #while prev is not equal to the previous goal, then we set prev to what it came from
optimal.append(
(prev[0], prev[1])) #append prev to the optimal path
prev_goal = location
if (pellets == len(maze.getObjectives())):
optimal.append(
(location[0], location[1])
) #if pellets equals the # of objectives, we append the location to the optimal path
return (optimal, num_states_explored)
if (visited[location[0]][location[1]] == False):
visited[location[0]][location[
1]] = True #if we have not visited if before, set it to visited
num_states_explored += 1
for i in neighbors:
new_cost = cost_so_far[
location] + 1 #add 1 since each move is 1 unit cost
if i not in cost_so_far or new_cost < cost_so_far[
i]: #if it's not an old location or if the new cost is lower than the cost so far
cost_so_far[i] = new_cost
distance = new_cost + manhattan(i, cur_goal) + mst_sum
frontier.put([
distance, i
]) #put the neighbor and its distance into the priority queue
came_from[i[0]][i[1]] = location
if pellet_arr[i[0]][i[
1]] < pellets: # used to check the current state and the state at the i location
pellet_arr[i[0]][i[1]] = pellets
cost_so_far[i] = new_cost
distance = new_cost + manhattan(i, cur_goal) + mst_sum
frontier.put([distance, i])
came_from[i[0]][i[1]] = location
return [], 0
def __init__(self, start=State(), end=State(), graph=defaultdict(list)):
super(GreedySearch, self).__init__(start, end, graph)
self.heap = Q.PriorityQueue()
# if any(n[0] == i or n[1] == i for n in featureIndex):
# continue
# for j in range(i+1, len(ellipses)):
# if any(n[0] == j or n[1] == j for n in featureIndex):
# continue
# distance, isCentroid = target(ellipses[i],ellipses[j], threshD)
# if isCentroid:
# flags[k]=1
# featureIndex[k] = (i,j)
# threshD = distance
# if flags[k] == 1:
# k += 1
# if k > 3:
# break
featureQueue = Queue.PriorityQueue()
for i in range(0,
len(ellipses)): # for each blob find the best matching pair
threshD = THRESHD
for j in range(i + 1, len(ellipses)):
distance, isCentroid = target(ellipses[i], ellipses[j], threshD)
if isCentroid:
featureQueue.put((distance, (i, j)))
threshD = distance
#if featureQueue.qsize() > 4:
# THRESHD = (THRESHD - 0.25) if THRESHD > 5.0 else 5.0
#elif featureQueue.qsize() < 4:
# THRESHD = (THRESHD + 0.25) if THRESHD < 25.0 else 25.0
def fib(n):
q = Queue.PriorityQueue()
q.get()
def fetch(self):
response_status = self.response.status
response_headers = dict(
(k.title(), v) for k, v in self.response.getheaders())
content_range = response_headers['Content-Range']
#content_length = response_headers['Content-Length']
start, end, length = tuple(
int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
if start == 0:
response_status = 200
response_headers['Content-Length'] = str(length)
del response_headers['Content-Range']
else:
response_headers['Content-Range'] = 'bytes %s-%s/%s' % (start, end,
length)
response_headers['Content-Length'] = str(length - start)
logging.info('>>>>>>>>>>>>>>> RangeFetch started(%r) %d-%d', self.url,
start, end)
self.wfile.write(
('HTTP/1.1 %s\r\n%s\r\n' %
(response_status, ''.join('%s: %s\r\n' % (k, v)
for k, v in response_headers.items()))))
data_queue = Queue.PriorityQueue()
range_queue = Queue.PriorityQueue()
range_queue.put((start, end, self.response))
for begin in range(end + 1, length, self.maxsize):
range_queue.put((begin, min(begin + self.maxsize - 1,
length - 1), None))
#thread.start_new_thread(self.__fetchlet, (range_queue, data_queue, 0))
p = threading.Thread(target=self.__fetchlet,
args=(range_queue, data_queue, 0))
p.daemon = True
p.start()
t0 = time.time()
cur_threads = 1
has_peek = hasattr(data_queue, 'peek')
peek_timeout = 90
self.expect_begin = start
while self.expect_begin < length - 1:
while cur_threads < self.threads and time.time(
) - t0 > cur_threads * config.AUTORANGE_MAXSIZE / 1048576:
#thread.start_new_thread(self.__fetchlet, (range_queue, data_queue, cur_threads * config.AUTORANGE_MAXSIZE))
p = threading.Thread(
target=self.__fetchlet,
args=(range_queue, data_queue,
cur_threads * config.AUTORANGE_MAXSIZE))
p.daemon = True
p.start()
cur_threads += 1
try:
if has_peek:
begin, data = data_queue.peek(timeout=peek_timeout)
if self.expect_begin == begin:
data_queue.get()
elif self.expect_begin < begin:
time.sleep(0.1)
continue
else:
logging.error(
'RangeFetch Error: begin(%r) < expect_begin(%r), quit.',
begin, self.expect_begin)
break
else:
begin, data = data_queue.get(timeout=peek_timeout)
if self.expect_begin == begin:
pass
elif self.expect_begin < begin:
data_queue.put((begin, data))
time.sleep(0.1)
continue
else:
logging.error(
'RangeFetch Error: begin(%r) < expect_begin(%r), quit.',
begin, self.expect_begin)
break
except Queue.Empty:
logging.error('data_queue peek timeout, break')
break
try:
self.wfile.write(data)
self.expect_begin += len(data)
except Exception as e:
logging.info('RangeFetch client connection aborted(%s).', e)
break
self._stopped = True
self.N = float(N or sum(self.itervalues()))
self.missingfn = missingfn or (lambda k, N: 1. / N)
def __call__(self, key):
if key in self: return float(self[key]) / float(self.N)
elif digitRegex.match(key): return 0.1
else : return self.missingfn(key, self.N)
# the default segmenter does not use any probabilities, but you could ...
# Pw = Pdist(opts.counts1w)
Pw = PdistUnigram(opts.counts2w)
PwJoint = PdistJoint(opts.counts2w)
old = sys.stdout
sys.stdout = codecs.lookup('utf-8')[-1](sys.stdout)
# ignoring the dictionary provided in opts.counts
pq = Q.PriorityQueue()
with open(opts.input) as f:
for line in f:
utf8line = unicode(line.strip(), 'utf-8')
input = [i for i in utf8line]
chart = {}
for j in range(1, min(1 + Pw.maxlen, len(input)) + 1):
newWord = "".join(input[:j])
prev_word = unicode("<S>", 'utf-8');
joint_tuple = " ".join([prev_word, newWord]);
joint_prob = PwJoint(joint_tuple)
unigramPrevProb = Pw(prev_word)
if joint_prob is None:
# print "Inside"
def active_learning_multi_processing(topicId, al_protocol, al_classifier, document_collection, topic_seed_info, topic_complete_qrels_address, train_per_centage, use_pooled_budget, per_topic_budget_from_trec_qrels):
train_index_list = topic_seed_info[topicId]
#print topicId
#print type(train_index_list)
#print "train_index_list", train_index_list
#print len(topic_complete_qrels[topicId][0]), len(topic_complete_qrels[topicId][1]), len(topic_complete_qrels[topicId][2])
topic_complete_qrels = pickle.load(open(topic_complete_qrels_address + topicId + '.pickle', 'rb'))
original_labels = topic_complete_qrels[0]
predicted_label = topic_complete_qrels[1]
original_predicted_merged_dict = {}
original_labels_list = []
for k, v in original_labels.iteritems():
original_predicted_merged_dict[k] = v
original_labels_list.append(v)
#exit(0)
#print "tmp_l1:",original_labels_list.count(1)
predicted_labels_list = []
for k, v in predicted_label.iteritems():
original_predicted_merged_dict[k] = v
predicted_labels_list.append(v)
#print "tmp_l2:",predicted_labels_list.count(1)
#print "sum", original_labels_list.count(1) + predicted_labels_list.count(1)
original_predicted_merged_list = []
for k in sorted(original_predicted_merged_dict.iterkeys()):
#print k, original_predicted_merged_dict[k]
original_predicted_merged_list.append(original_predicted_merged_dict[k])
#print "again sum", original_predicted_merged_list.count(1)
# need to convert y to np.array the Y otherwise Y[train_index_list] does not work directly on a list
y = np.array(original_predicted_merged_list) # 2 is complete labels of all documents in document collection
# type needed because y is an object need and throws error Unknown label type: 'unknown'
y = y.astype('int')
#print "numpy sum", np.count_nonzero(y)
#print y
#print y.shape
#print train_index_list
#print y[train_index_list]
#exit(0)
total_documents = len(y)
total_document_set = set(np.arange(0, total_documents, 1))
initial_X_test = []
test_index_dictionary = {}
test_index_counter = 0
#print "Starting Test Set Generation:"
#start = time.time()
for train_index in xrange(0, total_documents):
if train_index not in train_index_list:
initial_X_test.append(document_collection[train_index])
test_index_dictionary[test_index_counter] = train_index
test_index_counter = test_index_counter + 1
#print "Finshed Building Test Set:", time.time() - start
predictableSize = len(initial_X_test)
isPredictable = [1] * predictableSize # initially we will predict all
# initializing the train_size controller
train_size_controller = len(train_index_list)
loopCounter = 1 # loop starts from 1 because 0 is for seed_set
topic_all_info = {} # key is the loopCounter
while True:
#print "iteration:", loopCounter
# here modeling is utilizing the document selected in previous
# iteration for training
# when loopCounter == 0
# model is utilizing all the seed document collected at the begining
if al_classifier == 'LR':
model = LogisticRegression(solver=large_data_solver, C=large_data_C_parameter, max_iter=200)
elif al_classifier == 'SVM':
model = SVC(C=1.0, kernel='linear', degree=3, gamma='auto', probability = True)
elif al_classifier == 'RF':
model = RandomForestClassifier(n_estimators=10, max_depth=10, random_state=0)
elif al_classifier == 'RFN':
model = RandomForestClassifier(n_estimators=10, max_depth=None, random_state=0)
elif al_classifier == 'RFN100':
model = RandomForestClassifier(n_estimators=100, max_depth=None, random_state=0)
elif al_classifier == 'NB':
model = MultinomialNB()
elif al_classifier == 'Ada':
# base model is decision tree
# logistic regression will not help
model = AdaBoostClassifier(n_estimators=50,
learning_rate=1)
elif al_classifier == 'Xgb':
model = XGBClassifier(random_state=1, learning_rate=0.01)
elif al_classifier == 'BagLR':
LRmodel = LogisticRegression(solver=large_data_solver, C=large_data_C_parameter, max_iter=200)
model = BaggingClassifier(LRmodel, n_estimators = 5, max_samples = 1) # If float, then draw max_samples * X.shape[0] samples. 1 means use all samples
elif al_classifier == 'BagNB':
model = BaggingClassifier(MultinomialNB(), n_estimators = 5, max_samples = 0.5) # If float, then draw max_samples * X.shape[0] samples. 1 means use all samples
elif al_classifier == 'Vot':
LRmodel = LogisticRegression(solver=large_data_solver, C=large_data_C_parameter, max_iter=200)
NBmodel = MultinomialNB()
model = VotingClassifier(estimators=[('lr', LRmodel), ('nb', NBmodel)], voting = 'soft')
model.fit(document_collection[train_index_list], y[train_index_list])
test_index_list = list(total_document_set - set(train_index_list))
pooled_document_count = len(set(train_index_list).intersection(set(original_labels_list)))
non_pooled_document_count = len(set(train_index_list).intersection(set(predicted_labels_list)))
y_actual = None
y_pred = None
y_pred_all = []
if isPredictable.count(1) != 0:
y_pred = model.predict(document_collection[test_index_list])
start = time.time()
#print 'Statred y_pred_all'
y_actual = np.concatenate((y[train_index_list], y[test_index_list]), axis=None)
y_pred_all = np.concatenate((y[train_index_list], y_pred), axis=None)
'''
for doc_index in xrange(0,total_documents):
if doc_index in train_index_list:
y_pred_all.append(y[doc_index])
else:
# result_index in test_set
# test_index_list is a list of doc_index
# test_Index_list [25, 9, 12]
# test_index_list[0] = 25 and its prediction in y_pred[0] --one to one mapping
# so find the index of doc_index in test_index_list using
pred_index = test_index_list.index(doc_index)
y_pred_all.append(y_pred[pred_index])
'''
#print "Finsh y_pred_all", time.time() - start
else: # everything in trainset
y_pred = y
y_actual = y
y_pred_all = y
test_index_list = train_index_list
f1score = f1_score(y_actual, y_pred_all, average='binary')
precision = precision_score(y_actual, y_pred_all, average='binary')
recall = recall_score(y_actual, y_pred_all, average='binary')
#print f1score, precision, recall, len(train_index_list), len(test_index_list), len(y_pred_all)
# save all info using (loopCounter - 1)
# list should be deep_copy otherwise all will point to final referecne at final iterraion
topic_all_info[loopCounter - 1] = (topicId, f1score, precision, recall, copy.deepcopy(train_index_list), test_index_list, y_pred, pooled_document_count, non_pooled_document_count)
# it means everything in the train list and we do not need to predict
# so we do not need any training of the model
# so break here
if isPredictable.count(1) == 0:
break
#print isPredictable.count(1)
# suppose original budget is 5,
# then when train_index_list is 5, we cannot just turn off Active learning
# we need to use that AL with train_index_list of size 5 to train use that to predict the rest
# so we cannot exit at 5, we should exit at 5 + 1
# that is the reason we set per_topic_budget_from_trec_qrels[topicId] + 1 where 1 is the batch_size
# it means everything of pooled_budget size in the train_list so we need not tany training of the model
# so break here
if use_pooled_budget == 1 and per_topic_budget_from_trec_qrels[topicId] == len(train_index_list):
break
queueSize = isPredictable.count(1)
queue = Queue.PriorityQueue(queueSize)
# these are used for SPL
randomArray = []
for counter in xrange(0, predictableSize):
if isPredictable[counter] == 1:
# model.predict returns a list of values in so we need index [0] as we
# have only one element in the list
y_prob = model.predict_proba(initial_X_test[counter])[0]
val = 0
if al_protocol == 'CAL':
val = y_prob[1]
queue.put(relevance(val, counter))
elif al_protocol == 'SAL':
val = calculate_entropy(y_prob[0], y_prob[1])
queue.put(relevance(val, counter))
elif al_protocol == 'SPL':
randomArray.append(counter)
if use_pooled_budget == 1:
#print "use pooled budget"
size_limit = math.ceil(train_per_centage[loopCounter] * per_topic_budget_from_trec_qrels[topicId])
print "size limit:", size_limit, "total_docs:", per_topic_budget_from_trec_qrels[topicId]
else:
size_limit = math.ceil(train_per_centage[loopCounter] * total_documents)
print "size limit:", size_limit, "total_docs:", total_documents
if al_protocol == 'SPL':
random.shuffle(randomArray)
batch_counter = 0
# for batch_counter in xrange(0, batch_size):
# if batch_counter > len(randomArray) - 1:
# break
while True:
if train_size_controller == size_limit:
break
itemIndex = randomArray[batch_counter]
isPredictable[itemIndex] = 0
train_index_list.append(test_index_dictionary[itemIndex])
train_size_controller = train_size_controller + 1
batch_counter = batch_counter + 1
else:
while not queue.empty():
if train_size_controller == size_limit:
break
item = queue.get()
isPredictable[item.index] = 0 # not predictable
train_index_list.append(test_index_dictionary[item.index])
train_size_controller = train_size_controller + 1
loopCounter = loopCounter + 1
return topic_all_info
def __init__(self, cameraMaxX, cameraMaxY, dummy=False):
""" port = usb port of the arduino controling the motors
set to "" on a computer without arduino
"""
self.dummy = dummy
# this variable is True when no gestures are being executed
self.isIdle = True
self.headDataUpdated = False
self.tracking_disabled = False
# TODO: hardcoded configs?
gesture_files = {
"neutral": "gestures/neutral_gestures.csv",
"fear": "gestures/fear_gestures.csv",
"longing": "gestures/longing_gestures.csv",
"surprise": "gestures/surprise_gestures.csv",
"shame": "gestures/shame_gestures.csv"
}
# contains a dictionary of emotions to sequences
self.gestureNameToSeq = {}
# reading gesture files
for gesture_type, filename in gesture_files.iteritems():
with open(filename, "r") as f:
reader = csv.reader(f)
reader.next()
for row in reader:
self.gestureNameToSeq[gesture_type + "_" +
row[0]] = (float(row[1]), row[2:])
# Read the positions from the Positions.json file
self.positions = {}
# Read the positions from the json files
fileList = ["Positions.json"]
try:
os.chdir("./Positions")
for filename in glob.glob("*.json"):
fileList.append("Positions/" + filename)
os.chdir("..")
except:
pass
for filename in fileList:
self.loadPositionsFromFile(filename)
self.timeInterval = 0.25 # (1/4 second)
# Initialize the angles to the marionette's default (0 everywhere)
marionette = Marionette()
self.currentAngles = marionette.getAngles()
self.currentTargetAngles = marionette.getAngles()
self.targetReached = False
# Get min/max head angle values
self.headMinAngle = marionette.motor['H'].minAngle
self.headMaxAngle = marionette.motor['H'].maxAngle
# Head IMU angles:
self.roll = 0
self.pitch = 0
self.yaw = 0
self.pitchMax = 0
self.pitchMin = 0
self.yawMax = 0
self.yawMin = 0
# Angle delta above which a head motion will be triggered
self.headMotionThreshold = 0.0
# Max x and y in camera coordinates space
self.cameraCoordMaxX = predictor.PROCESSING_SIZE
self.cameraCoordMaxY = self.cameraCoordMaxX * cameraMaxY / cameraMaxX
# Read the calibration file
self.calibration = []
filename = "IMUCameraCalibration.json"
with open(filename, "r") as read_file:
self.calibration = json.load(read_file)
print self.calibration
self.pitchMax = self.calibration["up"][1]
self.pitchMin = self.calibration["down"][1]
self.yawMax = self.calibration["left"][2]
self.yawMin = self.calibration["right"][2]
# motor name to Arduino motor id
self.arduinoID = {}
self.arduinoID['motorH'] = 'head'
self.arduinoID['motorS'] = 'shoulder'
self.arduinoID['motorHR'] = 0 # "Right head"
self.arduinoID['motorHL'] = 1 # "Left head"
self.arduinoID['motorSR'] = 9 # "Right shoulder"
self.arduinoID['motorSL'] = 8 # "Left shoulder"
self.arduinoID['motorAR'] = -1 # "Right arm"
self.arduinoID['motorAL'] = 7 # "Left arm"
self.arduinoID['motorWR'] = 2 # "Right hand"
self.arduinoID['motorWL'] = 3 # "Left hand"
self.arduinoID['motorFR'] = 5 # "Right foot"
self.arduinoID['motorFL'] = 4 # "Left foot"
self.arduinoID['motorEX'] = 'eyeX' # "Eye horizontal"
self.arduinoID['motorEY'] = 'eyeY' # "Eye vertical"
# Arduino motor id to index of the angle in the angles list
# motor: 'S' 'SR' 'SL' 'AR' 'AL' 'H' 'HR' 'HL' 'FR' 'FL' 'WR' 'WL' 'EX' 'EY'
# angle index: 0 1 2 3 4 5 6 7 8 9 10 11 12 13
# arduino id: s m,9 m,8 -1 m,7 h m,0 m,1 m,5 m,4 m,2 m,3 e
self.arduinoIDToAngleIndex = {}
self.arduinoIDToAngleIndex['h'] = 5
self.arduinoIDToAngleIndex['s'] = 0
self.arduinoIDToAngleIndex['0'] = 6 # "Right head"
self.arduinoIDToAngleIndex['1'] = 7 # "Left head"
self.arduinoIDToAngleIndex['9'] = 1 # "Right shoulder"
self.arduinoIDToAngleIndex['8'] = 2 # "Left shoulder"
#self.arduinoIDToAngleIndex['6'] = 3 # "Right arm"
self.arduinoIDToAngleIndex['7'] = 4 # "Left arm"
self.arduinoIDToAngleIndex['2'] = 10 # "Right hand"
self.arduinoIDToAngleIndex['3'] = 11 # "Left hand"
self.arduinoIDToAngleIndex['5'] = 8 # "Right foot"
self.arduinoIDToAngleIndex['4'] = 9 # "Left foot"
self.arduinoIDToAngleIndex['e,x'] = 12 # "Eye horizontal"
self.arduinoIDToAngleIndex['e,y'] = 13 # "Eye vertical"
# Thread related variables
self.movementCount = long(0)
self.busy_executing = False
self.qMotorCmds = Queue.PriorityQueue()
self.running = False
self.arduino_thread = None
self.start()
def FCFS(plist,n):
p=process()
q=Q.PriorityQueue(maxsize=n)
x=0
Pdone=0
time_lap=0
numer_completed=0
templ=[]
btl=dict()
for y in plist:
btl[y.pid]=y.bt
pass
print("in srt")
while(1):
print "this is iteration number %d"%(x)
if(x<n):
print "going to push %d "%(x)
while(plist[x].at<=time_lap):
# print "pushed %d"%(x)
q.put(plist[x])
x+=1
if(x>=n):
break
#print(q.empty())
if(q.empty()==False):
#time_lap+=1
p=q.get()
print "printing in q"
p.disp()
time_lap=time_lap+p.bt
p.bt=0
if(p.bt==0):
p.ct=time_lap
p.wat=time_lap-btl[p.pid]-p.at
if p.wat<0:
p.wat=0
pass
p.tat=p.ct-p.at
p.bt=btl[p.pid]
templ.append(p)
Pdone+=1
else:
q.put(p)
else :
time_lap+=1
if(Pdone==n):
break
print "PRnp schduling done !"
return templ
pass
def resolution(clauses,out_file):
global globalvar
global lineNum
global maxQueue
candidates = Q.PriorityQueue()
lengthC1 = len(clauses)
i = 0
while i<lengthC1:
j=i+1
while j<lengthC1:
if resolvable(clauses[i],clauses[j]):
candidates.put(ResClauses(clauses[i],clauses[j]))
j+=1
i+=1
iter = 1
while not candidates.empty():
curPair = candidates.get()
list1 = curPair.clause1.clause_list
list2 = curPair.clause2.clause_list
resolvant = Clause()
for p in list1:
stripped = p
stripped = stripped.strip("-")
if(p == stripped and ("-"+p) in list2) or (p!=stripped and stripped in list2):
print "Iteration %s, queue size %s, resolution on %s and %s" % (iter, candidates.qsize()+1,curPair.clause1.index,curPair.clause2.index)
out_file.write("Iteration %s, queue size %s, resolution on %s and %s" % (iter, candidates.qsize()+1,curPair.clause1.index,curPair.clause2.index))
out_file.write("\n")
iter+=1
if maxQueue < candidates.qsize()+1:
maxQueue=candidates.qsize()+1
print "resolving %s and %s" %(curPair.clause1.getList(),curPair.clause2.getList())
out_file.write("resolving %s and %s" %(curPair.clause1.getList(),curPair.clause2.getList()))
out_file.write("\n")
resolvant=resolve(curPair.clause1, curPair.clause2, p)
if len(resolvant.clause_list)==0:
resolvant.parent1=curPair.clause1.index
resolvant.parent2=curPair.clause2.index
resolvant.index=globalvar
print "success empty clause found!"
print "Max Queue Size %d"% maxQueue
print "Total Number of iterations %d" %iter
out_file.write("\nSuccess empty clause found! \n")
out_file.write("Max Queue Size %d \n"% maxQueue)
out_file.write("Total Number of iterations %d" %iter)
return resolvant
if(not clauseExists(clauses, resolvant)):
print "%d: %s generated from %d and %d" %(lineNum, resolvant.getList(),curPair.clause1.index,curPair.clause2.index)
print
out_file.write("%d: %s generated from %d and %d" %(lineNum, resolvant.getList(),curPair.clause1.index,curPair.clause2.index))
out_file.write("\n\n")
lineNum+=1
resolvant.parent1 = curPair.clause1.index
resolvant.parent2 = curPair.clause2.index
resolvant.index = globalvar
clauses.append(resolvant)
globalvar+=1
i=0
while i<globalvar-1:
if resolvable(clauses[i], resolvant):
candidates.put(ResClauses(resolvant, clauses[i]))
i+=1
else:
print "Discard %s generated as it already exists in clauses" %resolvant.getList()
print
out_file.write("Discard %s generated as it already exists in clauses" %resolvant.getList())
out_file.write("\n")
out_file.write("\n")
return False
print candidates.qsize()
money = np.random.randint(100000)
#stock = 0
price = startprice
currentstock = stock[num]
arrlock.acquire()
stock[num] = money
arrlock.release()
sellqueue.put_nowait((-money, num))
threads = []
numpeople = 500
sells = Queue.PriorityQueue()
buys = Queue.PriorityQueue()
startprice = 100
arrlock = threading.Lock()
money = [np.random.randint(10000) for i in range(numpeople)]
stock = [np.random.randint(50) for i in range(numpeople)]
print sum(money), max(money), min(money)
lastprice = 10.
multiplier = .75
for roundi in range(100):
offers = [
np.random.normal(multiplier * lastprice, 5) for i in range(numpeople)
]
sellprices = []
def __init__(self):
self.pq = Queue.PriorityQueue(maxsize=0)
def __init__(self):
self.nodes = {}
self.startNode = None
self.goalNode = None
self.queue = Queue.PriorityQueue()
def __init__(self):
self.q = Q.PriorityQueue()
import time
import Queue
import threading
numbers = [23, 23, 23, 23]
q = Queue.PriorityQueue()
threads = []
queue_condition = threading.Condition()
shared_queue = Queue.Queue()
def fibonacci(condition, priority):
with condition:
while shared_queue.empty():
condition.wait()
num = shared_queue.get()
a, b = 0, 1
for i in range(num):
a, b = b, a + b
q.put((priority, a))
return
def queue_task(condition):
with condition:
for item in numbers:
shared_queue.put(item)
def astar2d(start, goal, obstacle_map, epsilon=1):
#print "Planning From", start, "to", goal
#Error Check Trivial Paths
if not isFree(start, obstacle_map):
# print "Invalid Start"
return [[], float('Inf'), 0]
if not isFree(goal, obstacle_map):
# print "Invalid Goal"
return [[], float('Inf'), 0]
if start == goal:
# print "Astar Complete"
return [[goal], 0.0, 0]
#Initialize Distance Matrix
[x, y] = obstacle_map.shape
distances = {}
for ii in range(x):
for jj in range(y):
distances[ii, jj] = float('inf')
#Expand Start Node
expanded_node = start
distances[start[0], start[1]] = 0.0
open_list = Queue.PriorityQueue()
closed_list = []
current_node = start
num_expanded = 1
for node in getNeighbors(start, obstacle_map):
distances[node[0], node[1]] = min(
distances[current_node[0], current_node[1]] + 1,
distances[node[0], node[1]])
if node == goal:
#print "Astar Complete"
#print distances[goal[0], goal[1]]
return getPath(start, goal, distances,
obstacle_map) + [len(closed_list)]
scored_node = (f(node, goal, distances, epsilon), num_expanded, node)
#print scored_node
open_list.put(scored_node)
closed_list.append(start)
while not open_list.empty():
current_scored_node = open_list.get(block=False)
current_node = current_scored_node[2]
closed_list.append(current_node)
#print "Expanding", current_node, f(current_node, goal, distances)
num_expanded += 1
for node in getNeighbors(current_node, obstacle_map, closed_list):
distances[node[0], node[1]] = min(
distances[current_node[0], current_node[1]] + 1,
distances[node[0], node[1]]) #4-connected graph assumption
if node == goal:
#print "Astar Complete"
#print distances[goal[0], goal[1]]
return getPath(start, goal, distances,
obstacle_map) + [len(closed_list)]
scored_node = (f(node, goal, distances,
epsilon), num_expanded, node)
updatePQueue(scored_node, open_list)
print "astar Failed"
def bfs(grid, car_list):
"""
Breadth First Search solver for the Rush Hour board game
"""
# Init vars etc.
queue = Queue.PriorityQueue()
queue.put((0, grid, car_list))
pre_grid = {}
pre_grid[str(grid)] = "finished"
# Main loop
while queue:
# Take first grid from queue
get_grid = queue.get()
gridObj = Grid(get_grid[1], get_grid[2])
# Check for all cars in the grid if there are moves possible
for car in get_grid[2]:
# Skip the placeholder car
if car != 'placeholder':
# Find the car number
car_n = gridObj.retrieve_value(car[2], car[3])
# Function for moving the car, checking for solution and creating new states
def move_car(move):
# Check if move is possible
if gridObj.check_move_car(car_n, move):
# Create copy of grid and move the car in the new grid
newGridObj = Grid([x[:] for x in get_grid[1]],
get_grid[2][:])
newGridObj.move_car(car_n, move)
newGridObjGridStr = str(newGridObj.grid)
# Check if grid has already existed
if newGridObjGridStr not in pre_grid:
cost = get_grid[0]
# Heuristic: Cost of blocking car is higher than blocking truck
for i in range(
newGridObj.car_list[1][2] +
newGridObj.car_list[1][1],
len(newGridObj.grid)):
if newGridObj.retrieve_value(
i, newGridObj.car_list[1][3]) != 0:
if newGridObj.car_list[1][1] == 3:
cost += 10
if newGridObj.car_list[1][1] == 2:
cost += 100
# Check for solution (clear path to endpoint)
if newGridObj.check_solution():
# Check if red car is at the endpoint
if newGridObj.car_list[1][2] != (
len(newGridObj.grid[0]) - 2):
# Add grid to queue to be further processed
queue.put((cost, newGridObj.grid,
newGridObj.car_list))
pre_grid[newGridObjGridStr] = get_grid[
1][:]
# Return and finish algorithm
else:
pre_grid[newGridObjGridStr] = get_grid[
1][:]
return (newGridObj, pre_grid)
# Add state to queue to be further processed
queue.put(
(cost, newGridObj.grid, newGridObj.car_list))
pre_grid[newGridObjGridStr] = get_grid[1][:]
# Try to move selected car both ways
returned = move_car(1)
if returned:
return returned
returned = move_car(-1)
if returned:
return returned
return "No solution"
def main():
opts = parse_opts()
if opts.verbose:
log_level = logging.DEBUG
else:
log_level = logging.INFO
should_test_modules = opts.testnames is None
logging.basicConfig(stream=sys.stdout, level=log_level, format="%(message)s")
LOGGER.info("Running PySpark tests. Output is in %s", LOG_FILE)
if os.path.exists(LOG_FILE):
os.remove(LOG_FILE)
python_execs = opts.python_executables.split(',')
LOGGER.info("Will test against the following Python executables: %s", python_execs)
if should_test_modules:
modules_to_test = []
for module_name in opts.modules.split(','):
if module_name in python_modules:
modules_to_test.append(python_modules[module_name])
else:
print("Error: unrecognized module '%s'. Supported modules: %s" %
(module_name, ", ".join(python_modules)))
sys.exit(-1)
LOGGER.info("Will test the following Python modules: %s", [x.name for x in modules_to_test])
else:
testnames_to_test = opts.testnames.split(',')
LOGGER.info("Will test the following Python tests: %s", testnames_to_test)
task_queue = Queue.PriorityQueue()
for python_exec in python_execs:
# Check if the python executable has coverage installed when 'COVERAGE_PROCESS_START'
# environmental variable is set.
if "COVERAGE_PROCESS_START" in os.environ:
_check_coverage(python_exec)
python_implementation = subprocess_check_output(
[python_exec, "-c", "import platform; print(platform.python_implementation())"],
universal_newlines=True).strip()
LOGGER.debug("%s python_implementation is %s", python_exec, python_implementation)
LOGGER.debug("%s version is: %s", python_exec, subprocess_check_output(
[python_exec, "--version"], stderr=subprocess.STDOUT, universal_newlines=True).strip())
if should_test_modules:
for module in modules_to_test:
if python_implementation not in module.blacklisted_python_implementations:
for test_goal in module.python_test_goals:
heavy_tests = ['pyspark.streaming.tests', 'pyspark.mllib.tests',
'pyspark.tests', 'pyspark.sql.tests', 'pyspark.ml.tests']
if any(map(lambda prefix: test_goal.startswith(prefix), heavy_tests)):
priority = 0
else:
priority = 100
task_queue.put((priority, (python_exec, test_goal)))
else:
for test_goal in testnames_to_test:
task_queue.put((0, (python_exec, test_goal)))
# Create the target directory before starting tasks to avoid races.
target_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'target'))
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
def process_queue(task_queue):
while True:
try:
(priority, (python_exec, test_goal)) = task_queue.get_nowait()
except Queue.Empty:
break
try:
run_individual_python_test(target_dir, test_goal, python_exec)
finally:
task_queue.task_done()
start_time = time.time()
for _ in range(opts.parallelism):
worker = Thread(target=process_queue, args=(task_queue,))
worker.daemon = True
worker.start()
try:
task_queue.join()
except (KeyboardInterrupt, SystemExit):
print_red("Exiting due to interrupt")
sys.exit(-1)
total_duration = time.time() - start_time
LOGGER.info("Tests passed in %i seconds", total_duration)
for key, lines in sorted(SKIPPED_TESTS.items()):
pyspark_python, test_name = key
LOGGER.info("\nSkipped tests in %s with %s:" % (test_name, pyspark_python))
for line in lines:
LOGGER.info(" %s" % line.rstrip())
