def report_for_file(self, file_name):
print 'report for: ' + file_name
windows = self._windows
methods = self._methods
report = Report(file_name, windows, methods)
raw_txt = Reader.readFromFile(file_name)
#print raw_txt
words = Reader.extractWords(raw_txt, "russian")
keywords = Reader.meter(words)
self._keywords = keywords
self._terms = words
#инициализация отчета термами с tf
for term in self._terms:
report.add_term_tf(term, keywords[term])
for window in windows:
for method in methods:
print method, window
(array, graph) = self.get_rw_for(method, window)
report._graph = graph #todo graph как св-во, пересмотреть логику
for v in array:
term = v.term_value
report.add_term_rw_stats(term, method, window, v.term_weight_rw)
self._reports[file_name] = report
def sign_up(new_username,new_password,gender,Countryin,Targetcountry):
previousCredentials=read.read_file('./util/credentials.txt')
new_credentials='%s,%s,%s,%s,%s'%(new_username,new_password,gender,Countryin,Targetcountry)
if new_username=='' or new_password=='' or new_username in previousCredentials or new_username.find(',')!=-1 or new_password.find(',')!=-1:
return 'Invalid signup credentials'
else:
read.write_file('./util/credentials.txt','\n'+new_credentials,'a')
return 'Successfully signed up'
def read_data():
global patients_log
Reader.readLog(file='Input/Log-completo.csv', start=2014, finish=2015, cv_frequency=0)
patients_log = Reader.patients
global patients_dm, patients_dm_periods, patients_age, patients_gender
patients_dm, patients_dm_periods, patients_age, patients_gender = DMCompensation_filter.set_data()
patients_official_list()
def file_report():
raw_txt = Reader.readFromFile('data/test1')
words = Reader.extractWords(raw_txt)
keywords = Reader.meter(words)
window = 3
graph_rw = TermGraph()
stopwords = nltk.corpus.stopwords.words('english')
words_clean = []
for word in words:
if word not in stopwords:
words_clean.append(word)
words = words_clean
length = words.__len__()
# print length, len(keywords), window
# print words
for i in xrange(0, length-1, 2):
print '----', i, '----'
print words[i:i+window]
v1 = Vertex(words[i], keywords[words[i]])
v2 = Vertex(words[i+1], keywords[words[i+1]])
v3 = Vertex(words[i+2], keywords[words[i+2]])
e1 = Edge(v1, v2)
e2 = Edge(v1, v3)
e3 = Edge(v2, v3)
graph_rw.add_edge(e1)
graph_rw.add_edge(e2)
graph_rw.add_edge(e3)
graph_rw.recalc_edges()
mode = 'rw'
graph_rw.recalc_vert_weights(mode)
for i in xrange(0,100):
graph_rw.recalc_vert_weights(mode)
array = graph_rw._verticles.values()
array.sort(comparator)
#report
file = './test_results/' + mode + '-win3.csv'
toCsv = ''
for v in array:
num_joins = graph_rw._verticlesJoins[v.term_value].__len__()
toCsv += str(v.term_value) + ',' +str(v.term_weight_rw) + ',' + str(v.term_weight_tf) + '\n'
print v, v.term_weight_rw, num_joins, v.term_weight_tf
print graph_rw._verticlesJoins[v.term_value]
def task1_2(ap=0.5):
reader = Reader(PATH_DATASET, ['car'], "perFrame")
gt = reader.get_annotations()
det = reader.get_det("yolo")
grouped = defaultdict(list)
for box in gt:
grouped[box.frame].append(box)
orderedict = OrderedDict(sorted(grouped.items()))
rec, prec, ap = voc_eval(det, orderedict, ap, is_confidence=True)
print(ap)
return
def file_report():
raw_txt = Reader.readFromFile('data/test1')
words = Reader.extractWords(raw_txt)
keywords = Reader.meter(words)
window = 3
graph_rw = TermGraph()
stopwords = nltk.corpus.stopwords.words('english')
words_clean = []
for word in words:
if word not in stopwords:
words_clean.append(word)
words = words_clean
length = words.__len__()
# print length, len(keywords), window
# print words
for i in xrange(0, length - 1, 2):
print '----', i, '----'
print words[i:i + window]
v1 = Vertex(words[i], keywords[words[i]])
v2 = Vertex(words[i + 1], keywords[words[i + 1]])
v3 = Vertex(words[i + 2], keywords[words[i + 2]])
e1 = Edge(v1, v2)
e2 = Edge(v1, v3)
e3 = Edge(v2, v3)
graph_rw.add_edge(e1)
graph_rw.add_edge(e2)
graph_rw.add_edge(e3)
graph_rw.recalc_edges()
mode = 'rw'
graph_rw.recalc_vert_weights(mode)
for i in xrange(0, 100):
graph_rw.recalc_vert_weights(mode)
array = graph_rw._verticles.values()
array.sort(comparator)
#report
file = './test_results/' + mode + '-win3.csv'
toCsv = ''
for v in array:
num_joins = graph_rw._verticlesJoins[v.term_value].__len__()
toCsv += str(v.term_value) + ',' + str(v.term_weight_rw) + ',' + str(
v.term_weight_tf) + '\n'
print v, v.term_weight_rw, num_joins, v.term_weight_tf
print graph_rw._verticlesJoins[v.term_value]
def run(self, flag):
nx_graphs, _ = Reader.multi_readG(self.path)
if flag == "LN":
r_t = Reader.true_cluster(self.path).tolist()
print(clustering(r_t))
cluster_true = [r[0] - 1 for r in r_t]
k_list = [k for k in range(2, 11)]
else:
cluster_true = []
k_list = [2, 3, 6, 8]
for i in range(29):
if i < 12:
cluster_true.append(0)
else:
cluster_true.append(1)
w_dict = Reader.weight(self.path)
print(nx_graphs[0])
MK_G = Node2Vec_LayerSelect.Graph(nx_graphs, self.p, self.q, 0.5)
MK_G.preprocess_transition_probs(w_dict, 2)
MK_walks = MK_G.simulate_walks(self.num_walks, self.walk_length)
MK_words = []
for walk in MK_walks:
MK_words.extend([str(step) for step in walk])
M_L = Word2Vec.Learn(MK_words)
M_matrix, M_mapping = M_L.train()
result = {}
for k in k_list:
cluster_trained = KMeans(
n_clusters=k, random_state=0).fit_predict(M_matrix).tolist()
length = min(len(cluster_true), len(cluster_trained))
r = normalized_mutual_info_score(cluster_true[0:length],
cluster_trained[0:length])
f = f1_score(cluster_true[0:length],
cluster_trained[0:length],
average='micro')
print(cluster_trained)
print(cluster_true)
result[k] = (r, f)
#pickle.dump(cluster_trained, open(self.path+str(k)+'.pickle', '+wb'))
print(result)
def signup():
previousCredentials=Reader.read_file('credentials.txt')
if request.method=="GET":
return render_template('signup.html')
elif request.method=="POST":
new_user=request.form['nuser']
new_pswd=request.form['npswd']
new_credentials='%s,%s'%(new_user,new_pswd)
if new_user=='' or new_pswd=='' or new_user in previousCredentials or new_user.find(',')!=-1 or new_pswd.find(',')!=-1:
return render_template('form.html', error='Invalid signup credentials')
else:
Reader.write_file('credentials.txt',new_credentials,'a')
return render_template('form.html', error='Successfully signed up')
else:
return 'yo'
class BackTestor:
def __init__(self, order_path):
self.r = Reader()
self.t = Trader()
self.LoadOrder(order_path)
def LoadOrder(self, order_path):
self.r.load_order_file(order_path)
for i in range(self.r.get_ordersize()):
o = self.r.read_border(i)
if o.price > 0:
self.t.RegisterOneTrade(o.contract, o.size if o.side == 1 else -o.size, o.price)
def Plot(self):
self.t.PlotStratPnl()
def write_entity_test(verbose):
relative_file_path = "../../resources/sql/accountSchema.sql"
file_contents = Reader.open_file(relative_file_path)
parser = Reader.SQL_Parser(file_contents)
table_name = parser.parse_table()
table_attributes = parser.parse_attributes()
t = Table.Table(table_name, file_contents, table_attributes)
w = Writer.Entity_Writer(t)
w.write_entity("some_file")
return 0
def open_file_test(verbose):
relative_file_path = "../../resources/sql/accountSchema.sql"
file_contents = Reader.open_file(relative_file_path)
if (len(file_contents) != 0):
return 1
return 0
def clear(self):
self.reader = Reader()
self.order_file_size = 0
self.shot_file_size = 0
self.pos = {}
self.net_pnl = {}
self.gross_pnl = {}
self.ticker_strat_map = {}
self.avgcost = {}
self.gross_time_allpnl_map = {}
self.net_time_allpnl_map = {}
self.strat_data_map = {}
self.gross_strat_pnl_map = {}
self.net_strat_pnl_map = {}
self.pnl_contract = set([])
self.Caler = CALER(contract_config_path)
def run_epoch(session, model, data, eval_op, epoch_num):
"""Runs the model on the given data."""
epoch_size = ((len(data) // model.batch_size) - 1) // model.num_steps
start_time = time.time()
costs = 0.0
iters = 0
state = session.run(model.initial_state)
for step, (x, y) in enumerate(
Reader.ptb_iterator(data, model.batch_size, model.num_steps)):
fetches = [model.cost, model.final_state, model.logits, eval_op]
feed_dict = {}
feed_dict[model.input_data] = x
feed_dict[model.targets] = y
for i, (c, h) in enumerate(model.initial_state):
feed_dict[c] = state[i].c
feed_dict[h] = state[i].h
cost, state, logits, _ = session.run(fetches, feed_dict)
costs += cost
iters += model.num_steps
# Rani: show the actual prediction
# decodedWordId = int(np.argmax(logits))
# print(" ".join([inverseDictionary[int(x1)] for x1 in np.nditer(x)]) + \
# " got:" + inverseDictionary[decodedWordId] + " expected:" + inverseDictionary[int(y)])
#if verbose and step % (epoch_size // 10) == 10:
if step % 10 == 0:
print("epoch: %d\t%.3f perplexity: %.3f speed: %.0f wps" %
(epoch_num, step * 1.0 / epoch_size, np.exp(costs / iters),
iters * model.batch_size / (time.time() - start_time)))
print("costs is %d and iters is %d" % (costs, iters))
return np.exp(costs / iters)
def main():
#modeOfSubstitution=0-> delete none values
#modeOfSubstitution=1 -> substitute for mean
#modeOfSubstitution=2 -> substitute for mode
modeOfSubstition = 2
numPartitions = 6
containID = 1
reader = Reader.Reader('breastCancer.data', 'breastCancerInfo.data',
modeOfSubstition, containID)
#Different methods for generating the set
trainSet, testSet = reader.leaveOneOut()
#partitions=reader.crossValidation(numPartitions)
#testSet,trainSet=generateSets(partitions)
if testSet[0] == [-1]:
print "Numero de particiones erroneo"
else:
attributesUsed = initListOfAttributes(reader.canUse)
#fullTreeID3=treeGenerationID3(trainSet,attributesUsed)
fullTreeID45 = treeGenerationID45(trainSet, attributesUsed)
print fullTreeID45.childs['9'].connectionName
solveForLeave1Out(fullTreeID45, testSet, attributesUsed)
#solveForCrossValidation(fullTreeID3,testSet,attributesUsed)
#printTree(fullTreeID3)
print "Generacion del arbol completada"
def worker(chunk, sect_dir, opcode_dir, dll_dir):
widgets = [
'Worker {}: '.format(chunk[0]), ' ',
Percentage(), ' ',
Bar(), ' ',
ETA()
]
pbar = ProgressBar(widgets=widgets, maxval=len(chunk[1])).start()
for idx, item in enumerate(chunk[1]):
# Filenames
fn = item.split(os.sep)[-1].split(".")[0]
# Read asm code
asm = r.readAsmCode(item)
# Extract features
opcodeDict, DLLDict, sectionDict = extractAssemblyFeatures(asm)
# Save features
pickle.dump(opcodeDict, open(os.path.join(opcode_dir, fn), "wb"))
pickle.dump(DLLDict, open(os.path.join(dll_dir, fn), "wb"))
pickle.dump(sectionDict, open(os.path.join(sect_dir, fn), "wb"))
# Update progressbar
pbar.update(idx + 1)
pbar.finish()
def run():
target = 50047984
data = []
for number in Reader.read("input"):
data.append(int(number))
sums = set()
for i in range(len(data)):
if (target - data[i]) in sums:
group, sum = [], 0
for j in range(i):
group.append(data[i - j])
sum += data[i - j]
if sum == target:
return min(group) + max(group)
else:
newSums = set()
for number in sums:
newSums.add(number + data[i])
newSums.add(data[i])
sums = newSums
return "Invalid"
def run():
seen = set()
highest = 0
lowest = 1000000
for data in Reader.read("input"):
lower, upper = 0, 127
for i in range(7):
h = (upper - lower) // 2
if data[i] == "F":
upper = lower + h
elif data[i] == "B":
lower = upper - h
left, right = 0, 7
for i in range(7, 10):
h = (right - left) // 2
if data[i] == "L":
right = left + h
elif data[i] == "R":
left = right - h
seat = (lower * 8) + left
highest = max(highest, seat)
lowest = min(lowest, seat)
seen.add(seat)
for i in range(lowest + 1, highest):
if i not in seen:
return i
def train_from_grid(self,
outputSamples="training/samples.data",
outputResponses="training/responses.data"):
dimCase = self.stockDim
self.samples = np.empty((0, self.stockDim**2))
self.responses = []
for i in range(6, 10):
reader = Reader.Reader()
reader.load_image("data/sudokus/sudoku" + str(i) + ".png")
reader.clean_image()
reader.rectify_perspective()
reader.cut_image_from_clean()
for img in reader.cases:
contours, hierarchy = cv2.findContours(copy.deepcopy(img),
cv2.RETR_LIST,
cv2.CHAIN_APPROX_SIMPLE)
maxArea = -1
biggest = None
for ctn in contours:
area = cv2.arcLength(ctn, True)
if area > maxArea and area > 25:
maxArea = area
biggest = ctn
x, y, w, h = cv2.boundingRect(biggest)
img = img[y:y + h, x:x + w]
img = cv2.resize(img, (self.stockDim, self.stockDim))
img = img.reshape((1, img.size))
img = np.float32(img)
self.responses.append(i)
self.samples = np.append(self.samples, img, 0)
def run(self):
print("[log]Generator start...")
while (not globcfg.event.is_set()):
print(
"[log]currentRunThread: Reader= {readCount}, Writer= {writeCount}"
.format(readCount=globcfg.currentRunThreadCount['Reader'],
writeCount=globcfg.currentRunThreadCount['Writer']))
genterate_time = getRandomInterval(globcfg.lamGen)
globcfg.generateTime_lock.acquire()
globcfg.generate_time_globalCopy = genterate_time
globcfg.generateTime_lock.release()
globcfg.event.wait(genterate_time)
choice = random.randint(0, 1)
# generate a new thread
if (choice):
print("[log]Generate thread {number} : {name}".format(
number=globcfg.threadNumber, name="Reader"))
self.gui.change_state("R", globcfg.threadNumber,
self.gui.nowhere, self.gui.scheduling)
globcfg.waitingList.append(
Reader.Reader(self.book, self.lock, globcfg.threadNumber,
self.gui)) #new Reader
else:
print("[log]Generate thread {number} : {name}".format(
number=globcfg.threadNumber, name="Writer"))
self.gui.change_state("W", globcfg.threadNumber,
self.gui.nowhere, self.gui.scheduling)
globcfg.waitingList.append(
Writer.Writer(self.book, self.lock, globcfg.threadNumber,
self.gui)) #new Writer
globcfg.threadNumber += 1
def call(self, *arg, **kw):
if debugCall: print "Call:", arg, kw
id = thread.get_ident()
if id not in self.readlocks:
if debugThreads: print "Call: Making readlock for", id
self.readlocks[id] = threading.Condition()
if debugThreads: print "Call: Locking readlock for", id
self.readlocks[id].acquire()
if debugThreads: print "Call: Locked readlock for", id
if self.err is not None:
if debugThreads: print "Call: Releasing readlock for", id
self.readlocks[id].release()
if debugThreads: print "Call: Released readlock for", id
raise self.err
try:
self.write(('RPCCall', id, arg, kw))
while self.err is None and id not in self.res:
if debugThreads: print "Call: Waiting on readlock for", id
self.readlocks[id].wait()
if debugThreads: print "Call: Waiting on readlock for", id, "done"
res = self.res[id]
del self.res[id]
err = self.err
finally:
if debugThreads: print "Call: Releasing readlock for", id
self.readlocks[id].release()
if debugThreads: print "Call: Released readlock for", id
if err:
raise err
res = Reader.extend(res, self.extension.parse)
if debugResult: print "Result:", res
return res
def RemoveFromList(ListSize):
MsgToReturn = ""
if ListSize > 1:
IndexToRemove = 0
while True:
ItemToRemove = r.intReader("POSICION DE ELEMENTO A REMOVER: ",
"VALOR NO VALIDO")
for x in range(ListSize):
if ItemToRemove == ListIndexes[
x] and ItemToRemove > 100 and ItemToRemove <= 200:
if (x + 1) < ListSize and ItemToRemove == ListIndexes[x +
1]:
IndexToRemove = x + 1
break
else:
IndexToRemove = x
break
if IndexToRemove > 0:
break
else:
print("ID DE POSICION FUERA DE RANGO")
# deletes items requested on IndexToRemove at lists
ListStorage.pop(IndexToRemove)
ListIndexes.pop(IndexToRemove)
else:
MsgToReturn = " UNDERFLOW ERROR"
# Returns string
return MsgToReturn
def main():
"""This is the main function for the program, input the selected method you want to run"""
programchoose = input(
"Type 1 for single day, variable, hmo analysis, 2 for average across HMO analysis and 3 for average across timeslot analysis"
)
if (programchoose == 1):
ch.chooseafile()
elif (programchoose == 2):
dc.findDaysInCommon()
elif (programchoose == 3):
rd.main()
else:
return 0
return
class SavingReader(Reader.Reader):
def __init__(self, context, url, *args, **kw):
self.__filename = kw['filename']
del kw['filename']
apply(Reader.Reader.__init__, (self, context, '') + args, kw)
context.rmreader(self)
self.url = url
self.restart(url)
def handle_meta(self, errcode, errmsg, headers):
if not self.handle_meta_prelim(errcode, errmsg, headers):
return
# now save:
self.stop()
try:
self.save_file = open(self.__filename, "wb")
except IOError, msg:
self.context.error_dialog(IOError, msg)
return
#
# add to history without destroying any title already known:
#
history = grailutil.get_grailapp().global_history
title, when = history.lookup_url(self.url)
history.remember_url(self.url, title or '')
#
Reader.TransferDisplay(self.last_context, self.__filename, self)
def GlobalLoop(pathparking,pathdemand):
[stamps,parking]=R.datareader(pathparking,pathdemand)
robots=SU.setuprobots(2,stamps[0])
customers=SU.Get_customers(pathdemand)
# print(stamps)
# print(customers)
for tf in stamps:
# print("tf")
typeaction = GI.CheckTypeAction(customers, tf)
asignedspot=SS.asignswapspot(parking)
target = GI.GetCustomerId(customers,tf)
# print(typeaction)
if(typeaction):
parking[asignedspot]=target
print("depose")
place=SF.Findplace(parking, stamps, customers,tf)
neworder=MO.Task(asignedspot,place,tf,target)
GO.giveorder(robots,neworder)
parking[asignedspot]="none"
parking[place]=target
elif(typeaction == False):
print("retrieve")
place=GI.Retrievelocation(parking,target)
print(place)
print("asingnedSPOT IS",asignedspot)
print("PLACEIS",place)
EC.extractcar(customers,asignedspot,parking,robots,place,tf,stamps,target)
parking[asignedspot]= target
parking[place]="none"
def grep(inputFn, outputFn, pattern):
file = r.readAsmCode(inputFn)
out = open(outputFn, "w")
for row in file:
if pattern in row:
out.write(row)
out.write("\n")
out.close()
def eval_ast(ast, env_):
_type = Reader.getType(ast)
if _type == "symbol":
return env_.get(ast)
if _type == "list": # call eval on all using list comprehension
return [EVAL(element, env_) for element in ast]
else:
return ast
def signup():
previousCredentials = Reader.read_file('credentials.txt')
if request.method == "GET":
return render_template('signup.html')
elif request.method == "POST":
new_user = request.form['nuser']
new_pswd = request.form['npswd']
new_credentials = '%s,%s' % (new_user, new_pswd)
if new_user == '' or new_pswd == '' or new_user in previousCredentials or new_user.find(
',') != -1 or new_pswd.find(',') != -1:
return render_template('form.html',
error='Invalid signup credentials')
else:
Reader.write_file('credentials.txt', new_credentials, 'a')
return render_template('form.html', error='Successfully signed up')
else:
return 'yo'
def merge_g(path):
nx_graphs, _ = Reader.multi_readG(path)
m_g = nx.Graph()
for g in nx_graphs:
m_g.add_nodes_from(g.nodes())
m_g.add_edges_from(g.edges())
pickle.dump(m_g, open(path+'\\'+'merged_graph.txtnx_graph.pickle', '+wb'))
def validate(username, password):
if username=='' or password=='':
return False
creds=read.getCsvDict('./util/credentials.txt')
if username in creds.keys() and creds[username][0]==password:
return True
else:
return False
def load_data(self):
train_edges = []
path_pk = "baselines.pkl"
if os.path.exists(path_pk):
print("The pkl file has existed!")
with open(path_pk, 'rb') as f:
(nx_graph, merge_graph, pos_edge_list, neg_edge_list,
nodes_attr) = pk.load(f)
else:
path = "Sampling_graph/Datasets_With_Attributes/" + os.path.basename(
self.path) + ".graph"
nx_graph, merge_graph, pos_edge_list, neg_edge_list, nodes_attr = Reader.data_load(
path)
with open(path_pk, 'wb') as f:
pk.dump((nx_graph, merge_graph, pos_edge_list, neg_edge_list,
nodes_attr), f)
# 对网络中的节点标签进行修改，需要进行排序
test_edges, test_labels = get_selected_edges(pos_edge_list,
neg_edge_list)
nodes = sorted(list(merge_graph.nodes()))
if nodes[0] > 0:
train_edges.extend([[i, e[0] - 1, e[1] - 1, 1]
for i in range(len(nx_graph))
for e in nx_graph[i].edges()])
train_merge = nx.relabel_nodes(merge_graph, lambda x: int(x) - 1)
train_nxgraph = [
nx.relabel_nodes(g, lambda x: int(x) - 1) for g in nx_graph
]
test_edges = [[e[0] - 1, e[1] - 1] for e in test_edges]
nodes = list(train_merge.nodes())
else:
train_edges.extend([[i, e[0], e[1], 1]
for i in range(len(nx_graph))
for e in nx_graph[i].edges()])
train_nxgraph = copy.deepcopy(nx_graph)
train_merge = copy.deepcopy(merge_graph)
# 有的节点编号并不是连续的，下面语句是为了使节点的编号连续
if isinstance(test_edges[0], list):
restru_test_edges = [(e[0], e[1]) for e in test_edges]
else:
restru_test_edges = [(nodes.index(e[0]), nodes.index(e[1]))
for e in test_edges]
str_graph = nx.relabel_nodes(train_merge, lambda x: str(x))
# 下面操作的是opennet定义的网络，为了使用现有的单层网络算法做对比
G = opgraph.Graph()
DG = str_graph.to_directed()
G.read_g(DG)
nx_para_graph = []
for g in train_nxgraph:
str_graph = nx.relabel_nodes(g, lambda x: str(x))
G = opgraph.Graph()
DG = str_graph.to_directed()
G.read_g(DG)
nx_para_graph.append(G)
return train_nxgraph, restru_test_edges, train_merge, test_edges, train_edges, test_labels
def read_data(self, file_name):
"""
将读取的数据保存在array中
:return:
"""
data = Reader.read_data(file_name)
for i in range(self.n):
self.x[i] = np.array(data[i][:15])
self.y[i] = np.array(data[i][15:])
def run():
data = [0]
for number in Reader.read("input"):
data.append(int(number))
data = sorted(data)
target = data[-1] + 3
return trace(len(data), target, data)
def getCallNames(fn):
text = r.readAsmCode(fn)
callDict = {}
for row in text:
if "(" in row:
callName = row.split("__stdcall")[1].strip().split("(")[0]
callDict[callName] = callDict.get(callName, 0) + 1
return callDict
def __init__(self):
'''controls and connects everything'''
self.Instructions_memory = Reader.CardReader()
print("Ingresa nombre del codigo (sin el .code)")
file_name = input()
data = [""] * 8 #creating data
self.Instructions_memory.change_file(file_name)
self.ALU = ALU.ALU()
self.ram = RAM()
self.registerA = Registers(data)
self.registerB = Registers(data)
self.registerC = Registers(data)
self.registerD = Registers(data)
self.instruction_register = Registers(
data, 8) #shows the selected data that was located in RAM
self.output_register = Registers(data)
data_bios = Reader.YamlReader().yaml_loader()
### variables to assign to CPU parts
ramdata = data_bios.get('RAM_NUMBERS')
ramdata = ramdata.split(' ')
clockdata = data_bios.get('clock')
self.clock_time = int(clockdata)
visualizationdata = data_bios.get('visualization')
##asigns all ram data to each ram space
for i in range(16):
adress = bin(i)[2:]
if len(adress) < 4:
if len(adress) == 1:
adress = "000" + adress
if len(adress) == 2:
adress = "00" + adress
if len(adress) == 3:
adress = "0" + adress
self.ram.write_enable(adress, ramdata[i])
self.visualization_code = visualizationdata['code']
self.visualization_ram = visualizationdata['RAM']
self.visualization_registers = visualizationdata['Registers']
self.visualization_clock = visualizationdata['clock']
self.visualization_alu = visualizationdata['ALU']
def eval_ast(ast, env_):
if callable(ast): # if it's a function just return it
return ast
_type = Reader.getType(ast)
if _type == "symbol":
return env_.get(ast)
if _type == "list": # call eval on all using list comprehension
return [EVAL(element, env_) for element in ast]
else:
return ast # literal value
def getFunctionNames(fn):
text = r.readAsmCode(fn)
funcDict = {}
for row in text:
if "PRESS" in row:
funcName = row[row.index("FUNCTION"):row.index("PRESS")].split(
".")[0].split(" ")[-1]
funcDict[funcName] = funcDict.get(funcName, 0) + 1
return funcDict
def parse_text_plain(*args, **kw):
headers = args[0].context.get_headers()
ctype = headers.get('content-type')
if ctype:
ctype, opts = grailutil.conv_mimetype(ctype)
if opts.get('format'):
how = str.lower(opts['format'])
if how == "flowed":
from filetypes import FlowingText
return FlowingText.FlowingTextParser(args, kw)
return Reader.TextParser(args, kw)
def main(args):
if '.h5' in args.mask_path:
args.mask_path = args.mask_path.replace('.h5', '')
reader = nd.Reader("", args.mask_path, args.image_path)
LaunchInstanceSegmentation(reader, args.image_type, args.fov,
args.range_of_frames[0],
args.range_of_frames[1], args.threshold,
args.min_seed_dist, args.path_to_weights)
def generatePassports():
current = {}
for data in Reader.read("input"):
if data == "":
yield current
current = {}
else:
for pair in data.split(" "):
key, value = pair.split(":")
current[key] = pair
yield current
def dispatchThread(msg, callindex):
msg = Reader.extend(msg, self.extension.parse)
if debugDispatch or profileDispatch: print "Dispatch " + str(callindex[0]) + ":", msg
try:
res = self.dispatch(*msg[1], **msg[2])
except:
exc_type, exc_value = sys.exc_info()[:2]
e = exc_value or exc_type
if Grimoire.Utils.isInstance(e, *debugExceptions) and not Grimoire.Utils.isInstance(e, *dontDebugExceptions):
import traceback
traceback.print_exc()
res = Types.RaiseException()
self.write(('RPCCallReturn', msg[0], res))
def readerThread(self):
unknown = Exception('Unknown exception in reader thread')
reads = 0
while 1:
reads += 1
try:
try:
if profileRead:
print "Read:", reads
p = hotshot.Profile(profileRead + str(reads))
try:
msg = p.runcall(Reader.read, self.buffer)
finally:
p.close()
else:
msg = Reader.read(self.buffer)
self.err = None
if debugRead: print "Read", reads, ":", msg
except:
self.err = sys.exc_value
raise sys.exc_type, sys.exc_value, sys.exc_traceback
try:
if msg[0] == 'RPCCall':
th = threading.Thread(
target = self.dispatchThread,
args = (msg[1:],),
name = 'Dispatch handler for %s' % threading.currentThread().getName())
if debugThreads: print "Starting dispatch thread %s" % th
th.start()
elif msg[0] == 'RPCCallReturn':
self.res[msg[1]] = msg[2]
if debugThreads: print "Reader thread: Locking readlock", msg[1]
self.readlocks[msg[1]].acquire()
if debugThreads: print "Reader thread: Notifying readlock", msg[1]
self.readlocks[msg[1]].notify()
if debugThreads: print "Reader thread: Unlocking readlock", msg[1]
self.readlocks[msg[1]].release()
if debugThreads: print "Reader thread: Unlocked readlock", msg[1]
else:
raise IOError('Unknown rpc message type', msg)
except:
self.err = sys.exc_value
finally:
if self.err:
if debugThreads: print "Reader thread: Notifying all readlocks"
for readlock in self.readlocks.values():
readlock.acquire()
readlock.notify()
readlock.release()
if debugThreads: print "Reader thread: Notifying all readlocks done"
def main():
#host = "localhost"
gui = GUI()
port = int(textport)
addr = (host, port)
buf = 1024
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(addr)
#sock.send("C1200");
r = Reader(sock, gui)
print "are u sure?"
r.start()
print "did u block?"
gui.root.mainloop()
#write as well as read from socket.
#while(1):
# k = raw_input()
# sock.send(k)
sys.exit()
def changeSettings(old_user,new_username,new_password,gender,Countryin,Targetcountry):
previousCredentials=read.getCsvDict('./util/credentials.txt')
newCredentials=''
if new_username=='':
newCredentials+=old_user+','
else:
newCredentials+=new_username+','
if new_password=='':
newCredentials+=previousCredentials[old_user][0]+','
else:
newCredentials+=new_password+','
if gender=='blank':
newCredentials+=previousCredentials[old_user][1]+','
else:
newCredentials+=gender+','
if Countryin=='':
newCredentials+=previousCredentials[old_user][2]+','
else:
newCredentials+=Countryin+','
if Targetcountry=='':
newCredentials+=previousCredentials[old_user][3]
else:
newCredentials+=Targetcountry
del previousCredentials[old_user]
newCredentials=newCredentials.split(',')
previousCredentials[newCredentials[0]]=newCredentials[1:]
Credentials=''
for i in previousCredentials.keys():
Credentials+=i+','
for u in previousCredentials[i]:
Credentials+=u+','
Credentials=Credentials[:-1]+'\n'
read.write_file('./util/credentials.txt',Credentials,'w')
def log_in():
request_data=request.form#Takes the immutable dictionary of the user's inputs and saves it in a variable
if request.method=="GET":
return render_template('form.html', error='Please enter from the form on the main page')#Tells the user to go through the main page if they entered directly
elif request.method=="POST":#Verifies the username and password against constants at the top
credentials=Reader.getCsvDict('credentials.txt')
if request_data['user'] in credentials and request_data['user']!='':
if request_data['pswd']==credentials[request_data['user']][0]:
return 'success'
else:
return render_template('form.html', error='Invalid username or password')
else:
return render_template('form.html', error='Invalid username or password')
else:
return 'yo'
def open(self):
Assert(self.state == WAIT)
resturl, method, params, data = self.args
if data:
Assert(method=="POST")
else:
Assert(method in ("GET", "POST"))
if type(resturl) == type(()):
host, selector = resturl # For proxy interface
else:
host, selector = splithost(resturl)
if not host:
raise IOError, "no host specified in URL"
i = string.find(host, '@')
if i >= 0:
user_passwd, host = host[:i], host[i+1:]
else:
user_passwd = None
if user_passwd:
import base64
auth = string.strip(base64.encodestring(user_passwd))
else:
auth = None
self.h = MyHTTP(host)
self.h.putrequest(method, selector)
self.h.putheader('User-agent', GRAILVERSION)
if auth:
self.h.putheader('Authorization', 'Basic %s' % auth)
if not params.has_key('host'):
self.h.putheader('Host', host)
if not params.has_key('accept-encoding'):
encodings = Reader.get_content_encodings()
if encodings:
encodings.sort()
self.h.putheader(
'Accept-Encoding', string.join(encodings, ", "))
for key, value in params.items():
if key[:1] != '.':
self.h.putheader(key, value)
self.h.putheader('Accept', '*/*')
self.h.endheaders()
if data:
self.h.send(data)
self.readahead = ""
self.state = META
self.line1seen = 0
if self.reader_callback:
self.reader_callback()
def generate_chains( fname, key_size ):
chains = {}
text = reader.read_file( fname )
words = text.split()
i = 0
while i < len(words) - key_size:
key = ' '.join( words[i : i+key_size] )
value = words[i + key_size]
if key in chains:
chains[ key ].append( value )
else:
new_list = []
new_list.append( value )
chains[ key ] = new_list
i+= 1
return chains
def similiarity_of_texts(self, txt1, txt2):
#print 'report for: ' + txt1
#print 'report for: ' + txt2
window = self._windows[0]
method = self._methods[0]
raw_txt_1 = Reader.readFromFile(txt1)
raw_txt_2 = Reader.readFromFile(txt2)
words_1 = Reader.extractWords(raw_txt_1, "russian")
words_2 = Reader.extractWords(raw_txt_2, "russian")
keywords_1 = Reader.meter(words_1)
keywords_2 = Reader.meter(words_2)
(gr_1) = self.get_graph_for(words_1, keywords_1, method, window)
(gr_2) = self.get_graph_for(words_2, keywords_2, method, window)
mesuare_terms = TermGraph.compare_graphs_terms(gr_1, gr_2)
mesuare_edges = TermGraph.compare_graphs_edges(gr_1, gr_2)
return (mesuare_terms, mesuare_edges)
def get_category_top_tf(self, categories, cat_test, mode='rw_oc', window=3):
if not self.classifier_tf_cache:
gr_array = {}
for category in categories:
gr_array[category] = []
for category in categories:
files = Reader.batchReadReuters('training', [category])
for file_name in files:
raw_txt = Reader.readFromFile('/home/dales3d/nltk_data/corpora/reuters/' + file_name)
words = Reader.extractWords(raw_txt)
keywords = Reader.meter(words)
if file_name not in self.graph_cache.keys():
gr = self.get_graph_for(words, keywords, mode, window)
self.graph_cache[file_name] = gr
else:
#print 'tf from cache: ', file_name
gr = self.graph_cache[file_name]
gr_array[category].append(gr)
top_words_tf = set()
gr_ver_arr = []
for k in gr_array.keys():
gr_ver_arr += gr_array[k]
for gr in gr_ver_arr:
top_tf = gr.getTopVerts("tf", 100)
for w_tf in top_tf:
top_words_tf.add(w_tf)
tf_dict = {}
for v in top_words_tf:
if v.term_value in tf_dict:
tf_dict[v.term_value] += v.term_weight_tf
#if v.term_weight_tf > tf_dict[v.term_value]:
# tf_dict[v.term_value] += v.term_weight_tf
else:
tf_dict[v.term_value] = v.term_weight_tf
sort_tf = sorted(tf_dict.items(), key=lambda x: x[1], reverse=True)
sort_tf = sort_tf[:1000]
self.tf_features_cache = sort_tf
training_set = []
for category in categories:
for gr in gr_array[category]:
top_tf = gr.getTopVerts("tf", 100)
features = {}
file_top_words = set()
for w_tf in top_tf:
file_top_words.add(w_tf.term_value)
for term in sort_tf:
features[term[0]] = (term[0] in file_top_words)
training_set.append((features, category))
self.classifier_tf_cache = nltk.NaiveBayesClassifier.train(training_set)
#from cache
sort_tf = self.tf_features_cache
#test
#cat_test = "jobs"
files = Reader.batchReadReuters('test', [cat_test])
#gr_tests
results = {}
for category in categories:
results[category] = 0
for f in files:
cats = reuters.categories(f)
raw_txt = Reader.readFromFile('/home/dales3d/nltk_data/corpora/reuters/' + f)
words = Reader.extractWords(raw_txt)
keywords = Reader.meter(words)
if f not in self.graph_cache.keys():
gr = self.get_graph_for(words, keywords, mode, window)
self.graph_cache[f] = gr
else:
gr = self.graph_cache[f]
top_tf = gr.getTopVerts("tf", 1000)
features = {}
file_top_words = set()
for w_tf in top_tf:
file_top_words.add(w_tf.term_value)
for term in sort_tf:
features[term[0]] = (term[0] in file_top_words)
result = self.classifier_tf_cache.classify(features)
if result in cats:
results[cat_test] += 1
else:
results[result] += 1
print results
sum = 0
for cat_key in results.keys():
sum += results[cat_key]
print float(results[cat_test])/sum * 100
scheight_mm = float(w.winfo_screenmmheight())
scwidth_mm = float(w.winfo_screenmmwidth())
vert_pixels_per_in = scheight / (scheight_mm / 25)
horiz_pixels_per_in = scwidth / (scwidth_mm / 25)
result = (72.0 / horiz_pixels_per_in), (72.0 / vert_pixels_per_in)
## print "scaling adjustments:", result
return result
def PrintDialog(context, url, title):
try:
infp = context.app.open_url_simple(url)
except IOError, msg:
context.error_dialog(IOError, msg)
return
content_encoding, transfer_encoding = Reader.get_encodings(infp.info())
try:
ctype = infp.info()['content-type']
except KeyError:
ctype, encoding = context.app.guess_type(url)
if not content_encoding:
content_encoding = encoding
if not ctype:
MaybePrintDialog(context, url, title, infp)
return
if not Reader.support_encodings(content_encoding, transfer_encoding):
# create an alert of some sort....
return
ctype, ctype_params = grailutil.conv_mimetype(ctype)
mod = context.app.find_type_extension("printing.filetypes", ctype)
if ctype != "application/postscript" and not mod.parse:
from random import random
import Reader
Dict=Reader.getCsvDict('data/wordlist.csv')
Subjects=Dict['nouns']
Verbs=Dict['verbs']
Adjectives=Dict['adjectives']
Adverbs=Dict['adverbs']
Objects=Subjects
"""
lines=Reader.getCsvList('wordlist.csv')
Subjects=lines[0]
Subjects.pop(0)
Verbs=lines[1]
Verbs.pop(0)
Adjectives=lines[2]
Adjectives.pop(0)
Adverbs=lines[3]
Adverbs.pop(0)
Objects=Subjects
"""
#Subjects=["Stanley","Lev","Spencer","Josh","Michael","Topher","JonAlf","Chris","Abdullah","Zane","Justin"]
#Verbs=["walks","runs","strolls","saunters","travels","drives","presents","explains","projects"]
Dates = [str(c) for c in Dates]
DateTime = []
for date in Dates:
for time in searchTimes:
print int(date[0:4]),int(date[4:6]),int(date[6:8])
DateTime.append(pandas.datetime(int(date[0:4]),int(date[4:6]),int(date[6:8]),int(time[0][0:2]),0,0))
print DateTime
clusters = range(668) #import this from files and user input
times = [12,24,48,72]
analyses = ['sd','cusum']
filters = ['gaussian','uniform']
userin1 = raw_input("Input the file location of the data: ")
data = rd.readFile(userin1)
userin2 = raw_input("Where do you want to put the new data? ")
count = 0
for t1 in times:
for t2 in times:
for f1 in filters:
for f2 in filters:
with open(userin2 + '/Cluster0011_'+f1+'_'+f2+'_'+str(t1)+'_'+str(t2)+'.csv','wb') as csvfile:
sd = [str(DateTime[x]) for x in data[11][t1][f1][analyses[0]]] #sd list
cu = [str(DateTime[x]) for x in data[11][t1][f1][analyses[1]]] #cusum list
count += len(sd)+len(cu)
csvwriter = csv.writer(csvfile,delimiter=',')
csvwriter.writerow(['SD']+sd)
csvwriter.writerow(['Cusum']+cu)
def set_only_dm_data(clusters_file, clusters, dm_file, center_param, min_age, max_age, start, finish, min_number_of_total_tests):
global patients
Reader.read_clusters(clusters_file, clusters)
patients = Reader.read_dm_compensation_t90(dm_file, center_param, min_age, max_age, start, finish,
min_number_of_total_tests)
def wrap_parser(self, parser):
# handle the content-encoding and content-transfer-encoding headers
headers = self.infp.info()
content_encoding, transfer_encoding = Reader.get_encodings(headers)
return Reader.wrap_parser(parser, self.ctype,
content_encoding, transfer_encoding)
def sendData(self,command):
sock.send(command);
r = Reader(sock)
r.start()
def get_category_graph(self, categories, cat_test, mode='rw_oc', window=3):
gr_array = {}
#print '--indexing--'
for category in categories:
files = Reader.batchReadReuters('training', [category])
#print 'category: ', category
big_cat_raw_txt = ''
#print '1) read files: start'
for file_name in files:
big_cat_raw_txt += Reader.readFromFile('/home/dales3d/nltk_data/corpora/reuters/' + file_name)
#print '1) read files: finished'
#print '2) preprocess text: start'
words = Reader.extractWords(big_cat_raw_txt)
keywords = Reader.meter(words)
#print '2) preprocess text: finished'
#print '3) term weighting: start'
if category in self.graph_cache.keys():
gr = self.graph_cache[category]
else:
gr = self.get_graph_for(words, keywords, mode, window)
self.graph_cache[category] = gr
#print '3) term weighting: finished'
gr_array[category] = gr
files = Reader.batchReadReuters('test', [cat_test])
#gr_tests
results = {}
results[''] = 0
for category in categories:
results[category] = 0
for f in files:
#print '---', f, '---'
cats = reuters.categories(f)
raw_txt = Reader.readFromFile('/home/dales3d/nltk_data/corpora/reuters/' + f)
words = Reader.extractWords(raw_txt)
keywords = Reader.meter(words)
if f not in self.graph_cache.keys():
gr = self.get_graph_for(words, keywords, mode, window)
self.graph_cache[f] = gr
else:
gr = self.graph_cache[f]
sim = {}
max_res = 0.0
max_cat = ''
for category in categories:
gr_1 = gr_array[category]
gr_2 = gr
#(mes_l, mes_r) = TermGraph.compare_graphs_terms_with_weight(gr_1, gr_2)
mesuare_terms = TermGraph.compare_graphs_terms_with_weight(gr_1, gr_2)
mesuare_edges = TermGraph.compare_graphs_edges(gr_1, gr_2)
res = mesuare_terms * (1 + mesuare_edges)
if max_res < res:
max_cat = category
max_res = res
sim[category] = res
if max_cat in cats:
results[cat_test] += 1
#print max_res
else:
results[max_cat] += 1
#print max_res
print 'no ok'
print results
sum = 0
for cat_key in results.keys():
sum += results[cat_key]
print float(results[cat_test])/sum * 100
def run(self):
print "Supervisor, checking in! pid:", os.getpid()
# If image directory does not exist yet, create it!
config = Utility.getProjectConfig()
imgdir = config.get('directories','imagedirectory')
print imgdir
if not os.path.isdir(imgdir):
try:
os.makedirs(imgdir)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
guiProcess = Process(target = self.startGUI)
guiProcess.start()
# Establish whether we have stable internet
stableInternetCounter = 0
stableInternet = False
for i in range(Utility.STABLE_INTERNET_COUNT):
if Utility.checkInternetConnection():
stableInternetCounter += 1
if (stableInternetCounter >= Utility.STABLE_INTERNET_COUNT):
self.statusQueue.put(Utility.QMSG_IDLE)
stableInternet = True
else:
self.statusQueue.put(Utility.QMSG_INTERNET_NO)
stableInternet = False
# Initialize with all images currently on camera
self.statusQueue.put(Utility.QMSG_SCAN)
initialScanFail = True
try:
Reader.camera_filenames_to_file(Utility.OLD_PICS_FILE_NAME)
self.statusQueue.put(Utility.QMSG_SCAN_DONE)
initialScanFail = False
except:
self.statusQueue.put(Utility.QMSG_SCAN_FAIL)
initialScanFail = True
time.sleep(Utility.POLL_TIME)
handlerProcess = None
handlerDelayed = False
readerProcess = None
while True:
if not self.guiQueue.empty():
job = self.guiQueue.get()
if job == Utility.QMSG_START:
if initialScanFail:
try: # OH GOD PLEASE REFACTOR THIS DUPLICATED CODE !!!!!!!!!!!!!!
Reader.camera_filenames_to_file(Utility.OLD_PICS_FILE_NAME)
self.statusQueue.put(Utility.QMSG_SCAN_DONE)
initialScanFail = False
except:
self.statusQueue.put(Utility.QMSG_SCAN_FAIL)
initialScanFail = True
continue # cannot complete job as normal if no baseline scan
# REFACTOR THIS CODE, DON'T FORGET! try Supervisor.initialScan()
print "Supervisor handles Upload job here"
readerProcess = Process(target = self.startReader)
self.readerQueue.put(Utility.QMSG_SCAN)
self.statusQueue.put(Utility.QMSG_SCAN)
readerProcess.start()
# wait for reader to finish scanning
readerProcess.join()
scanMsg = Utility.readMessageQueue(self.readerQueue)
if scanMsg == Utility.QMSG_SCAN_FAIL:
self.statusQueue.put(Utility.QMSG_SCAN_FAIL)
scanMsg = 0
continue # failed, tell GUI but ignore the rest of this job
elif scanMsg == Utility.QMSG_SCAN_DONE:
self.statusQueue.put(Utility.QMSG_SCAN_DONE)
scanMsg = 0
else:
print "Something went wrong with the ReaderMsgQueue!"
if stableInternet: # only start Handler if stable connection
handlerProcess = Process(target = self.startHandler)
self.handlerQueue.put(Utility.QMSG_HANDLE)
handlerProcess.start()
handlerDelayed = False
else:
time.sleep(Utility.POLL_TIME)
self.statusQueue.put(Utility.QMSG_INTERNET_NO)
handlerDelayed = True
elif job == Utility.QMSG_SETTINGS:
print "Supervisor handles Settings job here if needed"
else:
raise Exception('Supervisor.run: unexpected object in queue')
# endif self.guiQueue.empty()
# Start upload if delayed and internet is now stable
if handlerDelayed and stableInternet:
handlerProcess = Process(target = self.startHandler)
self.handlerQueue.put(Utility.QMSG_HANDLE)
handlerProcess.start()
handlerDelayed = False
time.sleep(Utility.POLL_TIME) # wait for handlerProcess to actually start
if not self.handlerQueue.empty():
handlerMsg = Utility.readMessageQueue(self.handlerQueue)
if handlerMsg == Utility.QMSG_UPLOAD:
self.statusQueue.put(Utility.QMSG_UPLOAD)
elif handlerMsg == Utility.QMSG_UPLOAD_DONE:
self.statusQueue.put(Utility.QMSG_UPLOAD_DONE)
elif handlerMsg == Utility.QMSG_HANDLE_NONE:
self.statusQueue.put(Utility.QMSG_HANDLE_NONE)
else:
self.statusQueue.put("Unknown Message from handlerQueue")
# Check current internet connection, allowing for some fluctuation in results
if Utility.checkInternetConnection():
if stableInternetCounter < Utility.STABLE_INTERNET_COUNT:
stableInternetCounter += 1
else:
stableInternet = True
self.statusQueue.put(Utility.QMSG_INTERNET_YES)
#print 'DEBUG: checkInternetConnection() == True'
else:
if (stableInternetCounter > 0): # only count down to 0
stableInternetCounter -= 1
if stableInternetCounter < Utility.STABLE_INTERNET_COUNT/2:
stableInternet = False
self.statusQueue.put(Utility.QMSG_INTERNET_NO)
#print 'DEBUG: checkInternetConnection() == False'
print 'DEBUG: stableInternet:', stableInternet, 'stableInternetCounter:', stableInternetCounter
time.sleep(Utility.POLL_TIME)
#print 'word: ', word.encode('utf8')
if not stream.has_key(word):
stream[word] = 0
stream[word] += 1
new_word = w
word = ''
else:
break
else:
#print 'find: ', new_word.encode('utf8')
word = new_word
break
break
return stream
if __name__ == '__main__':
token = Tokenizer(Dictionary('lexdb.dat'))
reader = Reader('.')
for (filename, fullpath, filesize, create_time) in reader.txt_files():
print filename
stream = token.parse(fullpath)
print len(stream)
for key in stream.keys():
print '%s: %s' % (key, stream[key])
print '-----------------------'
