def compileClassVarDec(self):
"""
compiles a classVerDec
"""
type, token = self.get_current()
var_kind = token
self.terminal(type, token) # get 'static' or 'field'
type, token = self.get_next()
var_type = token
self.terminal(type, token) # get var type
type, token = self.get_next()
var_name = token
self.terminal(type, token) # get var name
self._class_table.append(
Symbol(var_name, var_type, var_kind, self.counters[var_kind]))
self.counters[var_kind] += 1
type, token = self.get_next()
while token == ",":
self.terminal(type, token) # get ',' symbol
type, token = self.get_next()
self.terminal(type, token) # get var name
var_name = token
self._class_table.append(
Symbol(var_name, var_type, var_kind, self.counters[var_kind]))
self.counters[var_kind] += 1
type, token = self.get_next()
self.terminal(type, token) # get ';' symbol
def compileVarDec(self):
"""
compiles a varDec.
"""
type, token = self.get_current()
self.terminal(type, token) # get 'var' keyword
type, token = self.get_next()
var_type = token
self.terminal(type, token) # get var type
type, token = self.get_next()
var_name = token
self._subroutine_table.add_symbol(
Symbol(var_name, var_type, "local",
self._subroutine_table.get_counters()["local"]))
self._subroutine_table.get_counters()["local"] += 1
self.terminal(type, token) # get var name
type, token = self.get_next()
while token == ",":
self.terminal(type, token) # get ',' symbol
type, token = self.get_next()
var_name = token
self._subroutine_table.add_symbol(
Symbol(var_name, var_type, "local",
self._subroutine_table.get_counters()["local"]))
self._subroutine_table.get_counters()["local"] += 1
self.terminal(type, token) # get var name
type, token = self.get_next()
self.terminal(type, token) # get ';' symbol
def parseMapFile(self):
"""Returns a list of Symbols instances extracted from the map file which
path is filePath (should have been generalted by gold.ld -Map <file>
"""
filePath = self.getMapFile()
res = []
# The symbols described in the parsed file can mostly have 2 forms:
# - 1 line description: "<name> <addr> <size> <alignment> <object file>"
# - 2 lines description:
# "<name>
# <addr> <size> <alignment> <object file>"
# So here I have 1 regexp for the single line scenario and 2 for the
# double lines one. This filters out a lot of uneeded stuff, but we
# still need to only keep symbols related to text/data/rodata/bss so
# an additional check is performed on the extracted symbosl before
# adding it to the result set
twoLinesRe1 = "^[\s]+(\.[texrodalcbs\.]+[\S]*)$"
twoLinesRe2 = ("^[\s]+(0x[0-9a-f]+)[\s]+(0x[0-9a-f]+)[\s]+" +
"(0x[0-9a-f]+)[\s]+(.*)$")
oneLineRe = ("^[\s]+(\.[texrodalcbs\.]+[\S]+)[\s]+(0x[0-9a-f]+)[\s]+" +
"(0x[0-9a-f]+)[\s]+(0x[0-9a-f]+)[\s]+(.*)$")
with open(filePath, "r") as mapfile:
lines = mapfile.readlines()
for index, line in enumerate(lines):
s = None
matchResult = re.match(twoLinesRe1, line)
if matchResult: # probably a 2-lines symbol description
nextLine = lines[index + 1]
matchResult2 = re.match(twoLinesRe2, nextLine)
if matchResult2:
name = matchResult.group(1)
address = int(matchResult2.group(1), 0)
size = int(matchResult2.group(2), 0)
alignment = int(matchResult2.group(3), 0)
objectFile = matchResult2.group(4)
s = Symbol.Symbol(name, address, size, alignment,
objectFile, self.getArch())
else:
er("missed a two lines symbol while parsing mapfile:\n"
)
er("line1: " + line + "\n")
er("line2: " + nextLine + "\n")
sys.exit(-1)
else:
matchResult3 = re.match(oneLineRe, line)
if matchResult3: # one line symbol description
name = matchResult3.group(1)
address = int(matchResult3.group(2), 0)
size = int(matchResult3.group(3), 0)
alignment = int(matchResult3.group(4), 0)
objectFile = matchResult3.group(5)
s = Symbol.Symbol(name, address, size, alignment,
objectFile, self.getArch())
if s:
res.append(s)
return res
def p_PrimeList(t):
'''PrimeList : PrimeList PRIME
| PRIME'''
if len(t) == 2:
t[0] = [Symbol(Symbol.PRIME)]
else:
t[0] = t[1] + [Symbol(Symbol.PRIME)]
def perform_promotion(w1_in,w2_in,w1_out,w2_out):
"""
Takes 2 isax sequences and promotes each symbol to the higher cardinality
:return:
"""
for x in xrange(len(w1_in.symbols)):
a = sy.Symbol()
b = sy.Symbol()
sy.perform_promotion(w1_in.symbols[x],w2_in.symbols[x],a,b)
w1_out.symbols.append(a)
w2_out.symbols.append(b)
def get_parse_layout(self, adj_matrix, dict_mapping_Symbol_index,
index_to_symbol, rel_classifier_obj):
'''
The function returns a dictionary of relation corresponding to two symbols, which is later used for writing it to a lg files.
'''
symbol_obj = Symbol()
dict_map_rel_to_syms = {}
tree = minimum_spanning_tree(adj_matrix)
mst = tree.toarray()
for i in xrange(mst.shape[0]):
eye_obj = index_to_symbol[i]
arr = np.where(mst[i] > 0)[0]
if arr.shape[0] == 0:
continue
else:
trace_list = []
points_eye = []
for k in xrange(len(eye_obj.symbol_list)):
points_eye += eye_obj.symbol_list[k].original_points
for j in xrange(arr.shape[0]):
other_obj = index_to_symbol[arr[j]]
total_points = []
for l in xrange(len(other_obj.symbol_list)):
total_points += other_obj.symbol_list[
l].original_points
total_points = points_eye + total_points
trace_obj = Trace(points_float=total_points)
trace_obj.normalization()
trace_list.append(trace_obj)
symbol_obj.symbol_list = trace_list
features = symbol_obj.get_features()
X_test = np.asarray(features)
label = rel_classifier_obj.predict(X_test.reshape(1, -1))
if label[0] not in dict_map_rel_to_syms:
dict_map_rel_to_syms[label[0]] = []
dict_map_rel_to_syms[label[0]].append(
(eye_obj, other_obj))
else:
dict_map_rel_to_syms[label[0]].append(
(eye_obj, other_obj))
trace_list.remove(trace_obj)
return dict_map_rel_to_syms,
def get_parse_layout(self,adj_matrix,dict_mapping_Symbol_index,index_to_symbol,rel_classifier_obj):
'''
The function returns a dictionary of relation corresponding to two symbols, which is later used for writing it to a lg files.
'''
symbol_obj=Symbol()
dict_map_rel_to_syms={}
tree=minimum_spanning_tree(adj_matrix)
mst=tree.toarray()
for i in xrange(mst.shape[0]):
eye_obj=index_to_symbol[i]
arr=np.where(mst[i]>0)[0]
if arr.shape[0]==0:
continue
else:
trace_list=[]
points_eye=[]
for k in xrange(len(eye_obj.symbol_list)):
points_eye+=eye_obj.symbol_list[k].original_points
for j in xrange(arr.shape[0]):
other_obj=index_to_symbol[arr[j]]
total_points=[]
for l in xrange(len(other_obj.symbol_list)):
total_points+=other_obj.symbol_list[l].original_points
total_points=points_eye+total_points
trace_obj=Trace(points_float=total_points)
trace_obj.normalization()
trace_list.append(trace_obj)
symbol_obj.symbol_list=trace_list
features=symbol_obj.get_features()
X_test=np.asarray(features)
label=rel_classifier_obj.predict(X_test.reshape(1,-1))
if label[0] not in dict_map_rel_to_syms:
dict_map_rel_to_syms[label[0]] = []
dict_map_rel_to_syms[label[0]].append((eye_obj,other_obj))
else:
dict_map_rel_to_syms[label[0]].append((eye_obj,other_obj) )
trace_list.remove(trace_obj)
return dict_map_rel_to_syms,
def openSymbolDialog(self):
''' Create and open the Symbol dialog '''
dialog = Symbol.SymbolDialog()
dialog.setData(self.current_dataset, self.selected_symbol,
self.selected_symbol_name)
if dialog.exec_():
result = dialog.getResult()
def p_DifferentialVariable2(t):
'''DifferentialVariable : ID CARET LBRACE LPAREN NUMBER RPAREN RBRACE LPAREN ExpressionList RPAREN
| ID CARET LBRACE LPAREN NUMBER RPAREN RBRACE'''
isOrder = False
try:
order = int(t[5].getNumber())
isOrder = True
except Exception as msg:
order = t[5]
if isOrder:
if order > 0:
primeList = [Symbol(Symbol.PRIME)]*int(order)
else:
primeList = []
if len(t) > 8:
t[0] = DifferentialVariable(Identifier(ID(t[1])), primeList, t[9])
else:
t[0] = DifferentialVariable(Identifier(ID(t[1])), primeList)
else:
if len(t) > 8:
raise SyntaxException(t.slice[5].lineno, t.slice[5].lexpos, t.slice[5].value, t.slice[5])
else:
t[0] = ExpressionWithBinaryOperation(BinaryOperator(BinaryOperator.POW), Identifier(ID(t[1])), t[5])
class TestBot:
#symbols = Symbol.Symbol("ES\H20.CM", "symbol")
symbols = Symbol.Symbol("Earnings_2020-02-04.csv", "file")
Register.Register(
"TOS",
"5555",
)
def get_symbol_test(self, soup, trace_obj_dict):
flag = True # to ignore the first tracegroup
symbols = [
] # This will store the objects for every symbol in the file.
# pdb.set_trace()
for trace_g in soup.findAll('tracegroup'):
if flag == True:
flag = False
continue
else:
# symbol_class = trace_g.annotation.text # No ground truth available
# Get symbol id
try:
symbol_id = int(trace_g.attrs[0][1])
except:
#Conversion to int nit possible
symbol_id = int(trace_g.attrs[0][1].split(":")[0])
symbol_list = []
#print ("Symbol_id: %d" ) % (symbol_id)
trace_view = trace_g.findAll('traceview')
#pdb.set_trace()
for i in xrange(len(trace_view)):
trace_id = int(trace_view[i]['tracedataref'])
symbol_list.append(trace_obj_dict[trace_id])
symbol_obj = Symbol(symbol_id, symbol_list, symbol_class)
symbols.append(symbol_obj)
return symbols
def addLabelwithLocation(Label, location):
if (str(-1) != str(Opcode_Table.CheckOpcode(Label)[1])):
return -6
symbol = Symbol.Symbol(Label, "Label")
if symbol.name not in Symbol_Table:
Symbol_Table[symbol.name] = [location, symbol.Type]
return 0
else:
Type = Symbol_Table[symbol.name][1]
if (Type == ("Variable")):
return -3
if (Type == ("Label")):
address = Symbol_Table[symbol.name][0]
if (address == -1):
Symbol_Table[symbol.name][0] = location
return 0
return -4
def segment(self, exp):
setattr(exp, "strokes", [])
setattr(exp, "segSymbols", [])
for s in exp.symbols:
for stroke in s.strokes:
exp.strokes.append(stroke)
strokeNo = 0
for stroke in exp.strokes:
s = Symbol('?')
s.addStroke(stroke, strokeNo)
s.featureVector = getFeatureVector(s.strokes)
#s.featureVector = [0,0,0,0,0,0,0,0,0,0,0,0,0,0]
strokeNo += 1
exp.segSymbols.append(s)
def __new__(cls):
if ComplexRationals.__instance == None:
obj = object.__new__(cls)
obj._polyRing = PolynomialDomain(Rationals(), Symbol.Symbol('i'))
obj._poly = obj._polyRing([1, 0, 1])
obj._ideal = BAS.PrincipalIdeal(obj._polyRing, obj._poly)
obj._ideal.isMaximal = lambda: True
super(ComplexRationals, obj).__init__(obj._polyRing, obj._ideal)
ComplexRationals.__instance = obj
return ComplexRationals.__instance
def sax_distance(self,other):
"""
Calculates SAX euclidean distance between two iSAX sequences.
:param other:
:return:
"""
alphabet = alpha.NormalAlphabet()
sqd_dist = 0
# Compare symbol by symbol
for x in xrange(len(self.symbols)):
a = sy.Symbol()
b = sy.Symbol()
sy.perform_promotion(self.symbols[x],other.symbols[x],a,b)
distance_matrix = np.array([[0],[0]])
distance_matrix = alphabet.get_distance_matrix(a.cardinality)
local_dist = distance_matrix[a.sax_character][b.sax_character]
sqd_dist += local_dist
return sqd_dist
def readExpression(filename):
tree = ET.parse(filename)
root = tree.getroot()
strokes = {}
# Collect all of the traces (strokes)
for trace in root.iter():
if(trace.tag == "{http://www.w3.org/2003/InkML}trace"):
traceid = trace.attrib['id']
tracetext = trace.text.strip().split(',')
tracecoords = []
for t in tracetext:
coords = t.strip().split(' ')
tracecoords.append( (coords[0],coords[1]) )
# Save the stroke
strokes[traceid] = tracecoords
# Go through each traceGroup and put together symbol objects.
tgroot = root.find("{http://www.w3.org/2003/InkML}traceGroup")
rawfilename = filename.split('.')[0]
newexpression = Expression(rawfilename)
first = True
for tg in tgroot.iter():
if(tg.tag == "{http://www.w3.org/2003/InkML}traceGroup"):
truth = tg.find("{http://www.w3.org/2003/InkML}annotation").text
if(first):
# traceGroup for entire segmentation
first = False
continue
newsymbol = Symbol(truth)
for elem in tg.iter():
if(elem.tag == "{http://www.w3.org/2003/InkML}traceView"):
tvindex = elem.attrib['traceDataRef']
newsymbol.addStroke(strokes[tvindex], tvindex)
newexpression.addSymbol(newsymbol)
return newexpression
def _DF(self): print "DF()" self.scope = SymbolScope.LOCAL if self.bs._symbols.defined(self.operand.toString()): sym = self.bs._symbols.get(self.operand.toString()) sym.setSymbolType(SymbolType.DEFINED_FUNCTION) else: sym = Symbol(self.operand.toString(), SymbolType.DEFINED_FUNCTION, SymbolScope.GLOBAL) self.bs._symbols.insert(sym) sym.setAddress(self.jvm.getPC()) if sym.toString() == "Main": print "Main Function Definition" struct = Structure(StructureType.MAIN) else: struct = Structure(StructureType.FUNCTION) print "Function Definition" self.structureStack.append(struct) print self.structureStack print len(self.structureStack), ": structure pushed, "
def addVariable(Variable):
if (str(-1) != str(Opcode_Table.CheckOpcode(Variable)[1])):
return -7
symbol = Symbol.Symbol(Variable, "Variable")
if symbol.name in Symbol_Table:
if (symbol.Type == ("Label")):
return -5
else:
Symbol_Table[symbol.name] = [-1, "Variable"]
return 0
def mind_dist(self,other):
alphabet = alpha.NormalAlphabet()
sqd_dist = 0
for x in xrange(len(self.symbols)):
a = sy.Symbol()
b = sy.Symbol()
sy.perform_promotion(self.symbols[x],other.symbols[x],a,b)
distance_matrix = alphabet.get_distance_matrix(a.cardinality)
local_dist = distance_matrix[a.sax_character][b.sax_character]
sqd_dist += local_dist * local_dist
# get the sq root of the sum of the squares
dist_sqrt = math.sqrt(sqd_dist)
# Calculate the coefficient
original_ts_length = self.orig_length * 1.0
derived_sax_length = len(self.symbols)
coef = math.sqrt(original_ts_length/derived_sax_length)
return dist_sqrt * coef
def _CA(self):
print "CA()"
print "calling a function: ", self.currentOp.toString(), self.operand.toString(), self.nextOp.toString()
if self.operand.toString() == "write":
print "calling the write function"
self.jvm.emit3byte(Opcode.GETSTATIC, 6)
self.compileExpression()
self.jvm.emit3byte(Opcode.INVOKEVIRTUAL, 7)
else:
if self.bs._symbols.defined(self.operand.toString()) == False:
print "function not defined"
sym = Symbol(self.operand.toString(), SymbolType.FORWARD_FUNCTION, self.scope)
self.bs._symbols.insert(sym)
sym = self.bs._symbols.get(self.operand.toString())
if sym.getSymbolType() == SymbolType.FORWARD_FUNCTION:
self.saveForwardReference(sym, self.jvm.getPC())
self.jvm.emit3byte(Opcode.JSR, 0) #jsr
elif sym.getSymbolType() == SymbolType.DEFINED_FUNCTION:
self.jvm.emit3byte(Opcode.JSR, sym.getAddress() - self.jvm.getPC()) #jsr
else:
raise CompilerException("Error, call to "+sym.toString()+" does not reference a function, "+self.bs.fln())
def test_Matrix(self):
Z5 = AS.IntegerResidueClassField(5)
m = Matrix.Matrix(2,2,Z5,[[2,1],[0,2]])
n = Matrix.Matrix(2,2,Z5,[[3,1],[0,3]])
self.assertEqual(str(m), "[2|1]\n[0|2]")
self.assertEqual(m.invert(),n)
polys = AS.PolynomialDomain(Z5,Symbol.Symbol('t'))
self.assertEqual(m.charPoly(), polys([4,1,1]))
self.assertEqual(m.determinant(), Z5.elementFromValue(4))
m2 = Matrix.Matrix(2,2,Z5,[[1,3],[2,6]])
with self.assertRaises(Matrix.MatrixInversionError):
m2.invert()
self.assertEqual(m2.determinant(), Z5(0))
k = Matrix.getUnitMatrix(2, Z5)
self.assertTrue(k.isDiagonalizable())
(p,d) = k.getDiagonalizerMatrix()
self.assertEqual(p, d)
self.assertEqual(p, k)
m = Matrix.Matrix(2,2,Z5,[[1,1],[1,1]])
self.assertTrue(m.isDiagonalizable())
(p,d) = m.getDiagonalizerMatrix()
self.assertEqual(p,Matrix.Matrix(2,2,Z5,[[4,1],[1,1]]))
Q = AS.F_Q
m = Matrix.Matrix(3,3,Q,[[1,1,0],[2,0,2],[-1,1,1]])
n = Matrix.Matrix(3,3,Q,[[2,1,-2],[4,-1,2],[-2,2,2]]).scalarMultiplication(Q(1,6))
self.assertEqual(n, m.invert())
self.assertEqual(n.invert(), m)
self.assertEqual(m.determinant(),Q(-6))
polys = AS.PolynomialDomain(Q,Symbol.Symbol("t"))
self.assertEqual(m.charPoly(), polys([6,-3,-2,1]))
m = Matrix.Matrix(3,3,Q,[[1,1,0],[0,0,1],[0,0,0]])
self.assertEqual(m.nullity(), 1)
def get_relation_probability(self,eye_obj,other_obj,relation_classfier_obj):
#pdb.set_trace()
sym_obj=Symbol()
trace_list=[]
total_points=[]
#First get all the points together
for i in xrange(len(eye_obj.symbol_list)):
#trace_list.append(eye_obj.symbol_list[i])
total_points+=eye_obj.symbol_list[i].original_points
for i in xrange(len(other_obj.symbol_list)):
total_points+=other_obj.symbol_list[i].original_points
#trace_list.append(other_obj.symbol_list[i])
trace_obj=Trace(points_float=total_points)
trace_obj.normalization()
trace_list.append(trace_obj)
sym_obj.symbol_list=trace_list
X=sym_obj.get_features()
X=np.asarray(X)
prob = np.max(relation_classfier_obj.predict_proba(X.reshape(1,-1)))
#(1-prob) this is to get the minimum spanning tree.
trace_list.remove(trace_obj)
return(1-prob)
def compileParameter(self):
"""
compiles a parameter
"""
type, token = self.get_current()
argument_type = token
self.terminal(type, token) # get parameter type
type, token = self.get_next()
argument_name = token
self._subroutine_table.add_symbol(
Symbol(argument_name, argument_type, "argument",
self._subroutine_table.get_counters()["argument"]))
self._subroutine_table.get_counters()["argument"] += 1
self.terminal(type, token) # get parameter name
def test_promote_symbol(self):
A_in = sy.Symbol(0, 2)
B_in = sy.Symbol(1, 2)
A_out = sy.Symbol(0, 0)
B_out = sy.Symbol(0, 0)
sy.perform_promotion(A_in, B_in, A_out, B_out)
self.assertEquals(A_out.cardinality, 2)
self.assertEquals(B_out.cardinality, 2)
self.assertEquals(A_out.sax_character,0)
self.assertEquals(B_out.sax_character,1)
A_in = sy.Symbol(6, 8)
B_in = sy.Symbol(0, 2)
A_out = sy.Symbol(0, 0)
B_out = sy.Symbol(0, 0)
sy.perform_promotion(A_in, B_in, A_out, B_out)
self.assertEquals(B_out.cardinality, 8)
self.assertEquals(A_out.sax_character, 6)
self.assertEquals(B_out.sax_character, 3)
A_in = sy.Symbol(1, 2)
B_in = sy.Symbol(3, 8)
A_out = sy.Symbol(0, 0)
B_out = sy.Symbol(0, 0)
sy.perform_promotion(A_in, B_in, A_out, B_out)
self.assertEquals(A_out.cardinality, 8)
self.assertEquals(B_out.cardinality, 8)
self.assertEquals(A_out.sax_character, 4)
self.assertEquals(B_out.sax_character, 3)
A_in = sy.Symbol(1, 2)
B_in = sy.Symbol(0, 8)
A_out = sy.Symbol(0, 0)
B_out = sy.Symbol(0, 0)
sy.perform_promotion(A_in, B_in, A_out, B_out)
self.assertEquals(A_out.cardinality, 8)
self.assertEquals(B_out.cardinality, 8)
self.assertEquals(A_out.sax_character, 4)
self.assertEquals(B_out.sax_character, 0)
def _declare(self, enclosing_scope, is_key = False):
self._scope = enclosing_scope
token_name = self._token.value
token_type = self._token.type.name
symbol_type = self._scope.resolve(token_type)
if is_key:
self._symbol = self._scope.resolve(token_name)
if self._symbol != None:
raise ValueError('Symbol Already Defined')
self._symbol = Symbol(token_name, symbol_type)
return self._symbol
def get_relation_probability(self, eye_obj, other_obj,
relation_classfier_obj):
#pdb.set_trace()
sym_obj = Symbol()
trace_list = []
total_points = []
#First get all the points together
for i in xrange(len(eye_obj.symbol_list)):
#trace_list.append(eye_obj.symbol_list[i])
total_points += eye_obj.symbol_list[i].original_points
for i in xrange(len(other_obj.symbol_list)):
total_points += other_obj.symbol_list[i].original_points
#trace_list.append(other_obj.symbol_list[i])
trace_obj = Trace(points_float=total_points)
trace_obj.normalization()
trace_list.append(trace_obj)
sym_obj.symbol_list = trace_list
X = sym_obj.get_features()
X = np.asarray(X)
prob = np.max(relation_classfier_obj.predict_proba(X.reshape(1, -1)))
#(1-prob) this is to get the minimum spanning tree.
trace_list.remove(trace_obj)
return (1 - prob)
def sym_segmentation(self, classifier_obj, file_path_till_traininkml,
str_opt, rel_classifier_obj):
'''
The function calls methods from MinimumSpanningTree to segment,classify and parse symbols
Input
classifier_obj - Classifier pretrained model
file_path_till_traininkml - path to inkml file
str_opt - Train or Test
rel_classifier_obj - Realationship classifier pretrained model.
'''
load_obj = loadData()
m = MinimumSpanningTree()
symbol_obj = Symbol()
lg_folder_name = "parsing_" + str_opt
file_write_obj = FileWrite(lg_folder_name)
flag = False
with open('split_files.txt', 'r') as f:
for line in f:
if flag:
files = line
files = files.strip("Set([")
files = files.strip("])\n")
list_files = files.split(', ')
break
elif line.startswith(str_opt):
flag = True
continue
count = 0
for fileName in list_files:
count = count + 1
fileName = fileName.strip("'")
print "count= %d" % (count)
fileName = fileName.replace(
"/home/sbp3624/PatternRecog/TrainINKML_v3/",
file_path_till_traininkml)
root_obj, trace_obj_dict = load_obj.loadInkml(fileName)
m.get_segmentation(trace_obj_dict, classifier_obj, symbol_obj,
file_write_obj, fileName, lg_folder_name,
rel_classifier_obj)
def sym_parsing(self, rel_classifier_obj, file_path_till_traininkml,
str_opt):
'''
The function calls methods from MinimumSpanningTree class to segment and classify symbols and then parses the symbols, finally
writing it to an lg files.
'''
load_obj = loadData()
e = Graph()
symbol_obj = Symbol()
lg_folder_name = "parsing_" + str_opt
file_write_obj = FileWrite(lg_folder_name)
flag = False
with open('split_files.txt', 'r') as f:
for line in f:
if flag:
files = line
files = files.strip("Set([")
files = files.strip("])\n")
list_files = files.split(', ')
break
elif line.startswith(str_opt):
flag = True
continue
count = 0
for fileName in list_files:
count = count + 1
fileName = fileName.strip("'")
print "count= %d" % (count)
fileName = fileName.replace(
"/home/sbp3624/PatternRecog/TrainINKML_v3/",
file_path_till_traininkml)
root_obj, trace_obj_dict = load_obj.loadInkml(fileName)
symbols = load_obj.get_symbol(root_obj, trace_obj_dict)
adj_matrix, dict_mapping_Symbol_index, index_to_symbol = e.LineOfSight(
symbols, rel_classifier_obj)
dict_map_rel_to_syms = self.get_parse_layout(
adj_matrix, dict_mapping_Symbol_index, index_to_symbol,
rel_classifier_obj)
dict_map_rel_to_syms = dict_map_rel_to_syms[
0] # because the funtion returns a tuple
self.write_to_lg(fileName, symbols, dict_map_rel_to_syms,
lg_folder_name)
def atom(token):
if token == '#t':
return True
elif token == '#f':
return False
elif token[0] == '"':
return token[1:-1]
try:
return int(token)
except ValueError:
try:
return float(token)
except ValueError:
try:
return complex(token.replace('i', 'j', 1))
except ValueError:
return Symbol.make_symbol(token)
def addLabel(Label):
if (str(-1) != str(Opcode_Table.CheckOpcode(Label)[1])):
return -6
symbol = Symbol.Symbol(Label, "Label")
if symbol.name not in Symbol_Table:
Symbol_Table[symbol.name] = [-1, symbol.Type]
return 0
else:
Type = Symbol_Table[symbol.name][1]
if (Type == ("Variable")):
return -3
return 0
def test_rootsOverFiniteFields(self):
Z7 = AS.IntegerResidueClassField(7)
Z7x = AS.PolynomialDomain(Z7,Symbol.Symbol('x'))
poly = Z7x([1,0,1])
self.assertEqual(polyTools.rootsOverFiniteField(poly), [])
poly = Z7x([-1,0,1])
self.assertEqual(polyTools.rootsOverFiniteField(poly), [[Z7(6),1],[Z7(1),1]])
Z5 = AS.IntegerResidueClassRing(5)
Z5x,x = AS.polyRing(Z5,'x')
f = x*(x+4)**3*(x**4+4)**4
#print(polyTools.SFF(f))
f = 3+2*x+3*x**2+3*x**3+3*x**4+3*x**5
print(polyTools.getBerlekampMatrix(f,5))
print(polyTools.BerlekampFactorization(f/3))
Z3x,x = AS.polyRing(AS.IntegerResidueClassField(3),'x')
f = x**11+2*x**9+2*x**8+x**6+x**5+2*x**3+2*x**2+1
def charPoly(self):
"""
returns the characteristic polynomial p of self=A, p = det(tI-A)
"""
if self.columns != self.rows:
raise MatrixInversionError("no square matrix")
polyDomain = AS.PolynomialDomain(self.basedomain, Symbol.Symbol("t"))
rationalsDomain = AS.RationalFunctionsDomain(polyDomain)
charMatrix = Matrix(self.rows, self.columns, rationalsDomain)
t = polyDomain(
[self.basedomain.zero, self.basedomain.one]
) #rationalsDomain(polyDomain([self.basedomain.zero, self.basedomain.one]),polyDomain.one)
for i in range(self.rows):
for j in range(self.columns):
charMatrix[i, j] = polyDomain([
-self[i, j]
]) #rationalsDomain(polyDomain([-self[i,j]]),polyDomain.one)
if i == j:
charMatrix[i, j] += t
return charMatrix.determinant().asBaseRingElement()
def ticketFilter(tickets=None, filerFunc='_default_'):
if tickets is None:
tickets = StockAPI.getAllTickets('hose hnx upcom')
symbols = Symbol.getAllSymbolHistory(tickets=tickets)
tradeDays = symbols['VNM'].time
if filerFunc == '_default_':
# filerFunc = lambda(x: x.len>=100 and x.sma(src='volumn',window=50).iloc[-1]>=100000)
def _defaultFilter(x):
return (
x.len>50 and
x.time.iloc[-49] == tradeDays.iloc[-49] and # remove all ticket with non-trade days
(x.sma(src='volumn',window=50).iloc[-1] * x.close.iloc[-1]) >= 7000000 and
2.0 < x.close.iloc[-1] < 110 and
len(x.name) == 3 # To remove Derative ticket...
)
filerFunc = _defaultFilter
symbols = {x.name:x for x in filter(filerFunc, symbols.values())}
tickets = sorted(symbols.keys())
pickle.dump(symbols, open('data/current_prices.pkl', 'wb'))
pickle.dump(tickets, open('data/selTickets.pkl', 'wb'))
def getAllSymbols(dayNum: int = 365 * 2 + 50, exchange: str = 'HOSE HNX'):
import StockAPI, Symbol
tickets = StockAPI.getAllTickets(exchange)
print("Getting price history of %d Stocks ..." % len(tickets),
end="",
flush=True)
syms = {}
cnt = 10
for tic in tickets:
if len(tic) != 3:
continue
symbol = Symbol.SymbolHistory(tic, dayNum=dayNum)
if symbol.len < 50 or symbol.sma(src='volumn',
window=50).iloc[-1] < 200000:
continue
syms[tic] = symbol
cnt -= 1
if cnt == 0:
print(".", end="", flush=True)
cnt = 10
print(" Done")
return syms
def compileSubroutineDec(self):
"""
compiles a subroutineDec
"""
type, token = self.get_current()
self._subroutine_table = SymbolTable(self._class_table)
self._cur_func_type = token
self.terminal(type, token) # get subroutine
type, token = self.get_next()
if self._cur_func_type == "method":
self._subroutine_table.add_symbol(
Symbol("this", self._cur_class, "argument", 0))
self._subroutine_table.get_counters()["argument"] += 1
self.terminal(type, token) # get subroutine return type
type, token = self.get_next()
self._cur_func = token
self.terminal(type, token) # get subroutine name
type, token = self.get_next()
self.terminal(type, token) # get '(' symbol
type, token = self.get_next()
self.compileParameterList()
type, token = self.get_current()
self.terminal(type, token) # get ')' symbol
self.compileSubroutineBody()
def p_Symbol(t):
'''Symbol : PI
| XI_LOWER
| CHI_LOWER
| PHI_LOWER
| PSI_LOWER
| SIGMA_LOWER
| ZETA_LOWER
| ETA_LOWER
| DELTA_LOWER
| THETA_LOWER
| LAMBDA_LOWER
| EPSILON_LOWER
| TAU_LOWER
| KAPPA_LOWER
| OMEGA_LOWER
| ALPHA_LOWER
| NU_LOWER
| RHO_LOWER
| OMICRON_LOWER
| UPSILON_LOWER
| IOTA_LOWER
| BETA_LOWER
| GAMMA_LOWER
| MU_LOWER
| PI_UPPER
| BETA
| GAMMA
| KAPPA
| OMICRON
| OMEGA
| LAMBDA
| IOTA
| PSI
| MU
| PHI
| SIGMA
| ETA
| ZETA
| THETA
| EPSILON
| TAU
| ALPHA
| XI
| CHI
| NU
| RHO
| UPSILON'''
if t.slice[1].type == "PI":
t[0] = Symbol(Symbol.PI)
elif t.slice[1].type == "PI_UPPER":
t[0] = Symbol(Symbol.PI_UPPER)
elif t.slice[1].type == "ALPHA_LOWER":
t[0] = Symbol(Symbol.ALPHA_LOWER)
elif t.slice[1].type == "CHI_LOWER":
t[0] = Symbol(Symbol.CHI_LOWER)
elif t.slice[1].type == "XI_LOWER":
t[0] = Symbol(Symbol.XI_LOWER)
elif t.slice[1].type == "PHI_LOWER":
t[0] = Symbol(Symbol.PHI_LOWER)
elif t.slice[1].type == "PSI_LOWER":
t[0] = Symbol(Symbol.PSI_LOWER)
elif t.slice[1].type == "SIGMA_LOWER":
t[0] = Symbol(Symbol.SIGMA_LOWER)
elif t.slice[1].type == "ZETA_LOWER":
t[0] = Symbol(Symbol.ZETA_LOWER)
elif t.slice[1].type == "ETA_LOWER":
t[0] = Symbol(Symbol.ETA_LOWER)
elif t.slice[1].type == "DELTA_LOWER":
t[0] = Symbol(Symbol.DELTA_LOWER)
elif t.slice[1].type == "THETA_LOWER":
t[0] = Symbol(Symbol.THETA_LOWER)
elif t.slice[1].type == "LAMBDA_LOWER":
t[0] = Symbol(Symbol.LAMBDA_LOWER)
elif t.slice[1].type == "EPSILON_LOWER":
t[0] = Symbol(Symbol.EPSILON_LOWER)
elif t.slice[1].type == "TAU_LOWER":
t[0] = Symbol(Symbol.TAU_LOWER)
elif t.slice[1].type == "KAPPA_LOWER":
t[0] = Symbol(Symbol.KAPPA_LOWER)
elif t.slice[1].type == "OMEGA_LOWER":
t[0] = Symbol(Symbol.OMEGA_LOWER)
elif t.slice[1].type == "NU_LOWER":
t[0] = Symbol(Symbol.NU_LOWER)
elif t.slice[1].type == "RHO_LOWER":
t[0] = Symbol(Symbol.RHO_LOWER)
elif t.slice[1].type == "OMICRON_LOWER":
t[0] = Symbol(Symbol.OMICRON_LOWER)
elif t.slice[1].type == "UPSILON_LOWER":
t[0] = Symbol(Symbol.UPSILON_LOWER)
elif t.slice[1].type == "IOTA_LOWER":
t[0] = Symbol(Symbol.IOTA_LOWER)
elif t.slice[1].type == "BETA_LOWER":
t[0] = Symbol(Symbol.BETA_LOWER)
elif t.slice[1].type == "GAMMA_LOWER":
t[0] = Symbol(Symbol.GAMMA_LOWER)
elif t.slice[1].type == "BETA":
t[0] = Symbol(Symbol.BETA)
elif t.slice[1].type == "GAMMA":
t[0] = Symbol(Symbol.GAMMA)
elif t.slice[1].type == "MU":
t[0] = Symbol(Symbol.MU)
elif t.slice[1].type == "KAPPA":
t[0] = Symbol(Symbol.KAPPA)
elif t.slice[1].type == "OMICRON":
t[0] = Symbol(Symbol.OMICRON)
elif t.slice[1].type == "OMEGA":
t[0] = Symbol(Symbol.OMEGA)
elif t.slice[1].type == "LAMBDA":
t[0] = Symbol(Symbol.LAMBDA)
elif t.slice[1].type == "IOTA":
t[0] = Symbol(Symbol.IOTA)
elif t.slice[1].type == "PSI":
t[0] = Symbol(Symbol.PSI)
elif t.slice[1].type == "PHI":
t[0] = Symbol(Symbol.PHI)
elif t.slice[1].type == "SIGMA":
t[0] = Symbol(Symbol.SIGMA)
elif t.slice[1].type == "ETA":
t[0] = Symbol(Symbol.ETA)
elif t.slice[1].type == "ZETA":
t[0] = Symbol(Symbol.ZETA)
elif t.slice[1].type == "THETA":
t[0] = Symbol(Symbol.THETA)
elif t.slice[1].type == "EPSILON":
t[0] = Symbol(Symbol.EPSILON)
elif t.slice[1].type == "TAU":
t[0] = Symbol(Symbol.TAU)
elif t.slice[1].type == "ALPHA":
t[0] = Symbol(Symbol.ALPHA)
elif t.slice[1].type == "XI":
t[0] = Symbol(Symbol.XI)
elif t.slice[1].type == "CHI":
t[0] = Symbol(Symbol.CHI)
elif t.slice[1].type == "NU":
t[0] = Symbol(Symbol.NU)
elif t.slice[1].type == "RHO":
t[0] = Symbol(Symbol.RHO)
elif t.slice[1].type == "UPSILON":
t[0] = Symbol(Symbol.UPSILON)
else:
t[0] = Symbol(Symbol.MU_LOWER)
def get_relationship_data(self,file_path_till_traininkml,str_opt,file_path_lg_train):
'''
The method extracts the data required for training relationship classifier.
'''
load_obj=loadData()
symbol_obj=Symbol()
X_rel_train=[]
y_rel_train=[]
flag=False
with open('split_files.txt','r') as f:
for line in f:
if flag:
files=line
files=files.strip("Set([")
files=files.strip("])\n")
list_files=files.split(', ')
break
elif line.startswith(str_opt):
flag=True
continue
count=0
for fileName in list_files:
count=count+1
fileName=fileName.strip("'")
print "count= %d" % (count)
fileName=fileName.replace("/home/sbp3624/PatternRecog/TrainINKML_v3/",file_path_till_traininkml)
fileName_lg=basename(fileName)
pos=fileName_lg.find(".")
fileName_sub=fileName_lg[:pos] + ".lg"
fileName_lg=file_path_lg_train+fileName_sub
root_obj, trace_obj_dict = load_obj.loadInkml(fileName)
dict_sym={}
list_Obj=[]
list_R=[]
with open(fileName_lg,"r") as f_read:
for line in f_read:
line=line.strip("\n")
line=line.replace(" ","")
if line.startswith("O"):
list_obj=line.split(",")
dict_sym[list_obj[1]]=list_obj[4:]
elif line.startswith("R"):
list_R=line.split(",")
list_1=dict_sym[list_R[1]]
list_1+=dict_sym[list_R[2]]
rel_label=list_R[3]
list_traceobj_rel=[]
total_points=[]
for trace_id in list_1:
#list_traceobj_rel.append(trace_obj_dict[int(trace_id)])
total_points+=trace_obj_dict[int(trace_id)].original_points
#First get the original points then normalize
trace_obj=Trace(points_float=total_points)
trace_obj.normalization()
list_traceobj_rel.append(trace_obj)
symbol_obj.symbol_list=list_traceobj_rel
features=symbol_obj.get_features()
X_rel_train.append(features)
y_rel_train.append(rel_label)
list_traceobj_rel.remove(trace_obj)
return X_rel_train,y_rel_train
def boxes2symbols(boxes):
symbols = []
for x, y, w, h in boxes:
symbols.append(Symbol(SymbolType.UNKNOWN, x, y, w, h))
return symbols
def p_DivisorFunction(t): '''DivisorFunction : SIGMA_LOWER UNDERLINE LBRACE NUMBER RBRACE LPAREN ExpressionList RPAREN''' t[0] = ExpressionWithFunction(Symbol(Symbol.SIGMA_LOWER), t[7], t[4])
def get_segmentation(self,trace_obj_dict,classifier_obj,symbol_obj,File_write_obj,fileName,lg_folder_name,rel_classifier_obj):
'''
The function is calls optimal segmenattation and then call method to write lg files
Input
trace_obj_dict: trace objects for that file
classifier_obj: classifier object
symbol_obj: Symbol class object
File_write_obj:File write object
lg_folder_name: Foldername to store lg files
'''
min_span_tree=self.get_spanning_tree(trace_obj_dict)
demo_min_span_tree=min_span_tree
temp=np.zeros(demo_min_span_tree.shape[0])
temp=temp.reshape(temp.shape[0],1)
demo_min_span_tree=np.hstack((temp,demo_min_span_tree))
temp=np.zeros(demo_min_span_tree.shape[1])
demo_min_span_tree=np.vstack((temp,demo_min_span_tree))
demo_min_span_tree_bool=demo_min_span_tree>0
# Now get best probability distribution for set of strokes
dict_trace_map={}
count=0
s=[]
# This will help in mapping which index belong to which trace
for i in xrange(1,len(trace_obj_dict)+1):
dict_trace_map[i]=trace_obj_dict[i-1]
n=len(trace_obj_dict)+1
r,s = self.memoized_initialization(n)
dict_mapping_strokes={}
r,s,dict_mapping_strokes=self.optimal_segmentation(n,1,r,s,classifier_obj,trace_obj_dict,demo_min_span_tree_bool,dict_mapping_strokes,symbol_obj)
index=n-1
list_predict_symbol=[]
symbol_list=[]
count_traces=0
#Back Track and get the list of strokes
while index>0:
x=s[index]
list_strokes=dict_mapping_strokes[index]
symbol_object=Symbol()
symbol_list.append(symbol_object) # List of symbol object , which will be used to extract features
strokes_symbol=[]
for k in list_strokes:
trace_obj=dict_trace_map[k]
strokes_symbol.append(trace_obj)
count_traces=count_traces+1
symbol_object.symbol_list=strokes_symbol
index=index-int(x)
stroke_to_pixel=[]
X_test=[]
for symbol in symbol_list:
features=symbol.get_features()
X_test.append(features)
X_test_final=np.asarray(X_test)
predict_labels=classifier_obj.predict(X_test_final)
e=Graph()
r=RelationShipClassifier()
adj_matrix,dict_mapping_Symbol_index,index_to_symbol=e.LineOfSight(symbol_list,rel_classifier_obj)
dict_map_rel_to_syms=r.get_parse_layout(adj_matrix,dict_mapping_Symbol_index,index_to_symbol,rel_classifier_obj)
dict_map_rel_to_syms=dict_map_rel_to_syms[0]
File_write_obj.write_to_lg(predict_labels,fileName,symbol_list,dict_map_rel_to_syms,count_traces,lg_folder_name)
def _DE(self):
print "DE()"
#self.suppressAdvance = True
print "declaring a variable...", self.currentOp.toString(), self.operand.toString(), self.nextOp.toString()
if self.bs._symbols.defined(self.operand.toString()):
self.setError("Error: Multiple Declaration")
raise CompilerException("Multiple Declaration of variable "+ self.operand.toString() + self.bs.fln())
#if self.nextOp.toString() == ";":
# print "default declaration"
# sym = Symbol(self.operand.toString(), SymbolType.VARIABLE, self.scope)
# sym.setAddress(self.nextLocation)
# self.nexLocation += 1
# self.pushConstantInt(0)
# self.jvm.chooseOp(Opcode.ISTORE, Opcode.ISTORE_0, sym.getAddress())
# self.advance()
# self.bs._symbols.insert(sym)
# return
#
#while self.nextOp.toString() != ";":
# print self.operand.toString(), "SYMBOL!!!"
# self.advance()
# sym.setAddress(self.nextLocation)
# if self.nextOp.toString() == "[":
# sym.setSymbolType(SymbolType.ARRAY)
# self.advance() # co = '[', no = ']'
# self.pushConstant(self.operand)
# size = self.operand.toString()
# self.jvm.emit2byte(Opcode.NEWARRAY, 10)
# sym.setAddress(self.nextLocation)
# self.jvm.chooseOp(Opcode.ASTORE, Opcode.ASTORE_0, sym.getAddress())
# self.advance() # co = ']', no = ('=' | ';')
# if self.nextOp.toString() == "=":
# self.advance() # co = '=', no = '{'
# if self.nextOp.toString() != "{":
# self.setError("Syntax Error: Missing { after = ");
#
# self.advance() #co = '{', no = ','
# arr = []
# while self.nextOp.toString() != ";":
# print "Array Declaration :", self.operand.toString()
# arr.append(self.operand)
# self.advance()
# if len(arr) > size:
# self.setError("Syntax Error: Array Subscript exceeded")
# else:
# i = 0
# for val in arr:
# self.jvm.chooseOp(Opcode.ALOAD, Opcode.ALOAD_0, sym.getAddress())
# self.pushConstantInt(i)
# self.pushConstant(val)
# self.jvm.emit1byte(Opcode.IASTORE)
# i += 1
#
# else: #if not an array
# sym.setSymbolType(SymbolType.VARIABLE)
# #sym.setAddress(self.nextLocation)
# if self.nextOp.toString() == "=":
# self.advance()
# self.pushConstant(self.operand)
# self.jvm.chooseOp(Opcode.ISTORE, Opcode.ISTORE_0, sym.getAddress())
#
# print "declared a variable"
#
# #do variable declaration things
# self.bs._symbols.insert(sym)
# self.nextLocation += 1
self.done = False
while self.done is False:
#if self.currentOp.toString() == "int":
# self.advance()
if self.nextOp.toString() == ";":
#declare an uninitialzed variable, then end
sym = Symbol(self.operand.toString(), SymbolType.VARIABLE, self.scope)
sym.setAddress(self.nextLocation)
self.nextLocation += 1
self.pushConstantInt(0)
self.jvm.chooseOp(Opcode.ISTORE, Opcode.ISTORE_0, sym.getAddress())
self.advance()
self.bs._symbols.insert(sym)
self.done = True
elif self.nextOp.toString() == ",":
#declare this uninitalized variable, then go on to the next one
if self.currentOp.toString() == "," or self.currentOp.toString() == "int":
sym = Symbol(self.operand.toString(), SymbolType.VARIABLE, self.scope)
sym.setAddress(self.nextLocation)
self.nextLocation += 1
self.pushConstantInt(0)
self.jvm.chooseOp(Opcode.ISTORE, Opcode.ISTORE_0, sym.getAddress())
self.advance()
self.bs._symbols.insert(sym)
else:
self.advance()
elif self.nextOp.toString() == "[":
#declare an array, check the nextOp, and continue or end depending
sym = Symbol(self.operand.toString(), SymbolType.ARRAY, self.scope)
self.advance() # co = '[', no = ']'
print self.operand.toString(), "OPERAND"
self.pushConstant(self.operand)
size = self.operand.toString()
print "ARRAY SIZE", size
self.jvm.emit2byte(Opcode.NEWARRAY, 10)
sym.setAddress(self.nextLocation)
self.nextLocation += 1
self.bs._symbols.insert(sym)
self.jvm.chooseOp(Opcode.ASTORE, Opcode.ASTORE_0, sym.getAddress())
self.advance() # co = ']', no = ('=' | ';' | ',')
if self.nextOp.toString() == "=":
self.advance() # co = '=', no = '{'
if self.nextOp.toString() != "{":
self.setError("Syntax Error: Missing { after = ");
self.advance() #co = '{', no = ','
arr = []
while self.currentOp.toString() != "}":
print "Array Declaration :", self.operand.toString()
arr.append(self.operand)
self.advance()
print self.nextOp.toString(), "NEXT OP ========="
print "Attempted array declaration list size", len(arr)
if len(arr) > size:
self.setError("Syntax Error: Array Subscript exceeded")
raise CompilerException("Out-of-Bounds Error, Array subscript exceeded at symbol " + sym.toString() + self.bs.fln())
else:
i = 0
for val in arr:
print "assigning value", val, "to array"
self.jvm.chooseOp(Opcode.ALOAD, Opcode.ALOAD_0, sym.getAddress())
self.pushConstantInt(i)
self.pushConstant(val)
self.jvm.emit1byte(Opcode.IASTORE)
i += 1
if self.nextOp.toString() == ";":
self.done = True
print "AFTER DECLARATION, NEXTOP =", self.nextOp.toString()
elif self.nextOp.toString() == "=":
print "declare an initialized variable"
sym = Symbol(self.operand.toString(), SymbolType.VARIABLE, self.scope)
self.advance()
print self.operand.toString()
sym.setAddress(self.nextLocation)
self.nextLocation += 1
self.pushConstant(self.operand)
self.bs._symbols.insert(sym)
self.jvm.chooseOp(Opcode.ISTORE, Opcode.ISTORE_0, sym.getAddress())
if self.nextOp.toString() == ";":
self.done = True
pass
else:
#error
self.done = True
raise CompilerException("Invalid Assignment Sequence" + self.bs.fln())
pass
print "OUT OF LOOPS"
try:
if self.scope == SymbolScope.GLOBAL and self.nextOp.toString() == "void":
print "END OF A DECLARATION STATEMENT"
sym = self.bs._symbols.get("Main")
self.saveForwardReference(sym, self.jvm.getPC())
self.jvm.emit3byte(Opcode.GOTO, 0)
except:
pass
def intradayRecheck():
import re, os, pickle
from datetime import datetime
import StockAPI, Symbol
folderName = 'data/intras'
folderName2 = 'data/intras_new'
folderName3 = 'data/intras_fixed'
fnre = re.compile(r'^(...)_(........)\.pkl$')
fileList = os.listdir(folderName)
symbols = {}
dates = StockAPI.getTradingDate(365 * 2)
dateIdx = {}
for i, v in enumerate(dates):
dateIdx[v] = i
srv = StockAPI.IntraServer('vdsc', folderName2)
errCnt = 0
fixCnt = 0
chkCnt = 20
print('Checking %d ... ' % len(fileList))
for fname in fileList:
chkCnt -= 1
if chkCnt == 0:
# print('. ', end='', flush=True)
chkCnt = 20
x = fnre.match(fname)
if not x:
continue
ticket, dateStr = x.groups()
date = int(datetime.strptime(dateStr + ' 07', '%Y%m%d %H').timestamp())
intra = pickle.load(open(folderName + '/' + fname, 'rb'))
intra2 = srv.intraday(ticket, date)
if ticket not in symbols:
symbols[ticket] = Symbol.SymbolHistory(
ticket, StockAPI.getPriceHistory(ticket, 365 * 2))
daily = symbols[ticket].atDate(date)
if daily is None:
# print("??? %s %s" % (ticket, dateStr))
os.remove(folderName + '/' + fname)
continue
def totalVol(df):
dfm = StockAPI.intraMatchedOnly(df)
return dfm['mv'].sum()
vol0 = daily.volumn
vol1 = -1 if intra is None else totalVol(intra)
volMt1 = -1 if intra is None else intra.iloc[-1]['mt']
vol2 = -1 if intra2 is None else totalVol(intra2)
volMt2 = -1 if intra2 is None else intra2.iloc[-1]['mt']
def volOk(v1, v0):
if 0.98 < (v1 / v0) < 1.02:
return True
return False
if vol0 == 0:
if vol2 == 0:
pickle.dump(intra2, open(folderName3 + '/' + fname, 'wb'))
elif vol1 == 0:
pickle.dump(intra, open(folderName3 + '/' + fname, 'wb'))
else:
print("Error: [%s] %s: vol1 %d (%d) vol2 %d (%d) vol0 %d" %
(ticket, dateStr, vol1, volMt1, vol2, volMt2, vol0))
elif volOk(vol2, vol0):
pickle.dump(intra2, open(folderName3 + '/' + fname, 'wb'))
elif volOk(vol1, vol0):
pickle.dump(intra, open(folderName3 + '/' + fname, 'wb'))
elif volOk(volMt2, vol0):
intra2['mv'] = intra2['mt'].diff()
pickle.dump(intra2, open(folderName3 + '/' + fname, 'wb'))
elif volOk(volMt1, vol0):
intra['mv'] = intra['mt'].diff()
pickle.dump(intra, open(folderName3 + '/' + fname, 'wb'))
elif vol1 != 0 and vol1 == volMt1 and (vol0 / vol1) == 10:
# VCSC sometime got this wrong in 10 folds...
intra['mv'] = intra['mv'] * 10
intra['mt'] = intra['mt'] * 10
pickle.dump(intra, open(folderName3 + '/' + fname, 'wb'))
else:
print("Error: [%s] %s: vol1 %d (%d) vol2 %d (%d) vol0 %d" %
(ticket, dateStr, vol1, volMt1, vol2, volMt2, vol0))
pickle.dump(None, open(folderName3 + '/' + fname, 'wb'))
# Kmeans에 필요한 데이터
init_center_real = tf.dtypes.cast(tf.constant([[1], [-1], [-1], [1]]),
tf.double)
init_center_imag = tf.dtypes.cast(tf.constant([[1], [1], [-1], [-1]]),
tf.double)
temp_Y = [[0] * symbol_num for i in range(4)] # 임시 심볼 값을 담을 4xn 배열
# Matrix = [[0]*5 for i in range(7)] 열 행
# 인덱스로 접근해서 값 변경
for count in range(Count_Total):
print(count)
symbol = sym.Gensymbol(QAM, symbol_num) # 초기 심볼
data_s = de.Demode(QAM, symbol) # 초기 심볼 위치 rsc err count
for snr in range(SNR):
symbol_y = sym.Addnoise(snr, symbol) # 초기 심볼 + 노이즈
# 초기화 부분
position = np.zeros(symbol_num, int) # 심볼의 사분면 위치를 담을 배열
rsc_TakeCenter = [[0] * 4 for i in range(3)]
# 계산을 위해 담음
symbol_y = np.array(symbol_y) # real imag 배열을 나누기 위해 array로 담음
# symbol_real = tf.constant(symbol_y.real)
# symbol_imag = tf.constant(symbol_y.imag)
start = timeit.default_timer()
with tf.device('/GPU:0'):