def startNodes(ami, inst_size, keyName, maxPrice, nodecnt):
GF.log("... starting " + str(nodecnt) + " node(s)", 1);
local=[]
try:
res = GF.run("ec2-request-spot-instances " + ami + " -p " + str(maxPrice) + " -instance-type " + inst_size + " -n " + str(nodecnt) + " --type one-time" + " --key " + keyName)
lines=res.split("\n")
for i in range(0,len(res.split("\n"))):
line=lines[i]
print "res: ",i,line
if line.find("SPOTINSTANCEREQUEST")>=0:
inst=line.split("\t")
local.append(CLnode.CLnode( '' ,'slave' ,inst[5], '' , '' , '' ,inst[6] ,inst[0], '' ,False,inst[1],False))
#instID,instName,status , ami , key , size , date , ntype , url ,master,sir,deployed):
if res.find("timeout")>=0:
print "TIMEOUT: ", res
sys.exit()
if res.find("InvalidAMIID")>=0:
print "INVALID AMI ID: ", res
sys.exit()
except Exception as x:
print x, "\n", res
sys.exit()
GF.addNewNodes(local)
def __init__(
self,
instID="",
instName="",
status="",
ami="",
key="",
size="",
date="",
ntype="",
url="",
master=False,
sir="",
deployed=False,
):
self.instID = instID
self.instName = instName
self.status = status
self.ami = ami
self.key = key
self.size = size
self.date = date
self.ntype = ntype
self.url = url
self.master = GF.str2bool(master)
self.sir = sir
self.deployed = GF.str2bool(deployed)
def launchCluster(ami, inst_size, keyName, maxPrice, nodes):
GF.log("Maximum Price: "+str(maxPrice), 1);
curPrice=curSpotCost(inst_size)
if curPrice == -1:
print "Error: Failed to get current spot price."
sys.exit(-1)
if curPrice > maxPrice:
print "Error: Current spot price too high."
sys.exit(-2)
GF.log("Launching "+str(nodes)+" nodes.", 1);
startNodes(ami, inst_size, keyName, maxPrice, nodes)
def divide_with_remainder(dividend, divider):
biggest_dividend_power = Utils.get_biggest_power(dividend)
biggest_divider_power = Utils.get_biggest_power(divider)
# Jeśli dzielnik mniejszy od dzielnej lub równy 0 nie trzeba dzielić
if biggest_dividend_power < biggest_divider_power or (
biggest_dividend_power >= 1 and biggest_divider_power == 0):
return [[0], dividend]
max_power = max(biggest_dividend_power, biggest_divider_power)
result = [[0] * max_power] * 2
new_dividend = list(dividend)
while True:
i = len(new_dividend) - 1
while new_dividend[i] == 0 and i > 0:
i -= 1
if i < 0:
return result
# Dzielenie przez najwyższą potęgę dzielnej
current_power = i - biggest_divider_power
# Jeśli potęga mniejsza od zera to koniec dzielenia
if current_power < 0:
if i == 0:
return [dividend, [0] * max_power]
else:
return result
# Dzielenie największego wyrazu dzielnej przez największy wyraz dzielnika
div_result = [0] * (current_power + 1)
gfs1 = GF.GFSimple(new_dividend[i], 2)
gfs2 = GF.GFSimple(divider[biggest_divider_power], 2)
gfs_result = gfs1 * (~gfs2)
div_result[current_power] = gfs_result.value
# Dodanie wyniku dzielenia do result
result[0] = Utils.add_polynomials(result[0], div_result)
# Mnożenie dzielnika przez wynik dzielenia
subtrahend = Utils.mul_polynomials(divider, div_result)
# Odejmowanie wymnożonej wartości od dzielnej
new_dividend = Utils.sub_polynomials(new_dividend, subtrahend)
# Zwrócenie wyniku jeśli nie można dalej dzielić
if Utils.get_biggest_power(new_dividend) < biggest_divider_power:
result[1] = new_dividend
return result
def sub_polynomials(poly1, poly2):
if len(poly1) >= len(poly2):
for i in range(0, len(poly2)):
gfs1 = GF.GFSimple(poly1[i], 2)
gfs2 = GF.GFSimple(poly2[i], 2)
gfs_result = gfs1 + (~gfs2)
poly1[i] = gfs_result.value
return poly1
else:
for i in range(0, len(poly1)):
gfs1 = GF.GFSimple(poly1[i], 2)
gfs2 = GF.GFSimple(poly2[i], 2)
gfs_result = gfs1 + (~gfs2)
poly2[i] = gfs_result.value
return poly2
def test_mod(self):
a = 7
print(bin(a))
b = 5
print(bin(b))
m = GF.mod(a, b)
print(bin(m))
def getRunningInstances():
try:
res = GF.run("ec2-describe-instances")
if res.find("timeout")>=0:
print "TIMEOUT: ", res
sys.exit()
for line in res.split("\n"):
if line.find("INSTANCE")>=0:
inst=line.split("\t")
if inst[5] != "terminated":
GF.nodes.append(CLnode.CLnode(inst[1],inst[1],inst[5],inst[2],inst[6],inst[9],inst[10],inst[0],inst[3],'',inst[22]))
else:
GF.log("found terminated"+line,2)
except Exception as x:
print x, "\n", res
sys.exit()
def mul_polynomials(poly1, poly2):
max_power = Utils.get_biggest_power(poly1) + Utils.get_biggest_power(
poly2)
mul_result = [0] * (max_power + 1)
for i in range(len(poly1) - 1, -1, -1):
for j in range(len(poly2) - 1, -1, -1):
gfs1 = GF.GFSimple(poly1[i], 2)
gfs2 = GF.GFSimple(poly2[j], 2)
gfs_mul_result = gfs1 * gfs2
if gfs_mul_result.value > 0:
power = i + j
gfs3 = GF.GFSimple(mul_result[power], 2)
gfs_add_result = gfs3 + gfs_mul_result
mul_result[power] = gfs_add_result.value
return mul_result
def curSpotCost(inst_size):
lt = time.localtime(time.time())
curdate = str(lt[0])+"-"+str(lt[1])+"-"+str(lt[2])+"T"+str(lt[3])+":"+str(lt[4])+":"+str(lt[5])+":"+str(lt[6])+"-0000"
try:
res = GF.run("ec2-describe-spot-price-history -d Linux/UNIX --region us-east-1 --instance-type "+inst_size+" -s "+curdate)
if res.find("timeout")>=0:
print "TIMEOUT: ", res
sys.exit()
cost=0
for i in res.split("\n"):
cost += float((i.split("\t"))[1]);
cost = cost/len(res.split("\n"))
except Exception as x:
print x, "\n", res
sys.exit()
GF.log("Current Instance Cost: "+str(cost), 1);
return cost
def test_div_mod(self):
a = 15
print(bin(a))
b = 7
print(bin(b))
q, v = GF.div_mod(a, b)
print(bin(q))
print(bin(v))
def gather(self, logsDir, sshKey):
print "\n============================================"
print "Gathering logs: ", self.instID, "/", self.instName
print "====="
logsDir += "/" + self.instID + "-" + self.instName
try:
res = GF.run("mkdir -p " + logsDir)
res = GF.run(
"scp -r -o StrictHostKeyChecking=no -i " + sshKey + " ubuntu@" + self.url + ":/var/tmp/log " + logsDir
)
print "scp -r -o StrictHostKeyChecking=no -i " + sshKey + " ubuntu@" + self.url + ":/var/tmp/log " + logsDir
except Exception as x:
print x, "\n", res
return -1
"""
def deploy(self, payload, sshKey, launch=False):
# TODO: error handeling
# remove SIG with ssh-keygen -f "/home/madmaze/.ssh/known_hosts" -R ec2-184-73-46-186.compute-1.amazonaws.com
print "\n============================================"
print "Deploing", self.instID, "/", self.instName
print "====="
if self.master is False:
try:
res = GF.run(
"scp -o StrictHostKeyChecking=no -i " + sshKey + " " + payload + " ubuntu@" + self.url + ":~/"
)
print "scp -o StrictHostKeyChecking=no -i " + sshKey + " " + payload + " ubuntu@" + self.url + ":~/"
except Exception as x:
print x, "\n", res
return -1
# EXTRACT payload
try:
res = GF.run(
"ssh -o StrictHostKeyChecking=no -i " + sshKey + " ubuntu@" + self.url + " 'tar xvf ~/bundle.tar;'"
)
print "ssh -o StrictHostKeyChecking=no -i " + sshKey + " ubuntu@" + self.url + " 'tar xvf ~/bundle.tar;'"
except Exception as x:
print x, "\n", res
return -1
if launch is True:
# LAUNCH Payload
try:
print "ssh -o StrictHostKeyChecking=no -i " + sshKey + " ubuntu@" + self.url + " 'python ~/payload/setup.py&'"
res = GF.run(
"ssh -o StrictHostKeyChecking=no -i "
+ sshKey
+ " ubuntu@"
+ self.url
+ " 'screen -dm python ~/payload/setup.py'"
)
self.deployed = True
except Exception as x:
print x, "\n", res
return -1
else:
print "Master node: No need to deploy!"
def loadState():
getRunningInstances()
nodes=[]
try:
fname="./nodeDB"
FILE = open(fname,"r")
while FILE:
line = FILE.readline().strip().split(",")
if len(line)<=1:
break
nodes.append(CLnode.CLnode(*line))
#line[0],line[1],line[2],line[3],line[4],line[5],line[6],line[7],line[8],line[9]
FILE.close()
GF.addNewNodes(nodes)
except IOError:
print "warning, nodeDB does not exists yet"
except Exception as x:
print x, "\n", res
sys.exit()
def kill(self):
if self.status != "running":
print "Instance is not running, therefore will not shutdown"
return
try:
res = ""
res = GF.run("ec2-terminate-instances " + self.instID)
print res
self.status = "terminated"
except Exception as x:
print x, "\n", res
return -1
def getSpotRequests():
try:
res = GF.run("ec2-describe-spot-instance-requests")
if res.find("timeout")>=0:
print "TIMEOUT: ", res
sys.exit()
for line in res.split("\n"):
if line.find("INSTANCE")>=0:
inst=line.split("\t")
GF.reqests.append(CLnode.CLnode(inst[1],inst[1],inst[5],'','','',inst[6],inst[0],''))
except Exception as x:
print x, "\n", res
sys.exit()
def launchMaster(ami, inst_size, keyName):
GF.log("Launching Master node..",1)
local=[]
try:
res = GF.run("ec2-run-instances " + ami + " -k " + keyName + " -t " + size)
if res.find("InvalidAMIID")>=0:
print "INVALID AMI ID: ", res
sys.exit()
print res
i=0
lines=res.split("\n")
master=CLnode.CLnode()
for l in lines:
inst=l.split("\t")
if inst[0]=="INSTANCE":
master = CLnode.CLnode(inst[1],"MASTER",inst[5],inst[2],inst[6],inst[9],inst[10],inst[0],'',True)
master.desc_detail()
local.append(master)
GF.addNewNodes(local)
except Exception as x:
print x, "\n", res
sys.exit()
def monitor(n, timeout):
allStarted=True
for i in range(0,n):
allStarted=True
getSpotRequests()
#getRunningInstances()
for n in GF.requests:
if n.status=="open":
allStarted=False
if allStarted is True:
break
time.sleep(timeout)
if allStarted is False:
print "All instances did not start during designated time."
if GF.confirmQuestion("Would you like to continue?") is True:
monitor(n, timeout)
else:
sys.exit()
else:
for i in range(0,n):
allStarted=True
getRunningInstances()
for n in GF.nodes:
if n.status=="pending":
allStarted=False
if allStarted is True:
break
if allStarted is False:
print "All instances did not start during designated time."
if GF.confirmQuestion("Would you like to continue?") is True:
monitor(n, timeout)
else:
sys.exit()
else:
#launch?!
print "stuff"
def main():
print("=====KODOWANIE I SZYFROWANIE DANYCH=====\n")
# gf1 = GF.GFSimple(1, 7)
# gf2 = GF.GFSimple(2, 7)
#
# print("=====Galois Field Simple=====")
# print("GF1: %s\nGF2: %s" % (gf1, gf2))
# print("Sum: %s" % (gf1 + gf2))
# print("Mul: %s" % (gf1 * gf2))
# print("Invert gf1: %s" % (~gf1))
# print("Invert gf2: %s" % (~gf2))
# print("Generators amount: %d" % GF.GFSimple.get_generators_amount(7))
# print("Generators: %s" % GF.GFSimple.get_generators(7))
#
# a = [3, 4, 1]
# m = [4, 5, 7]
# print("\n=====Chinese Remainder Theorem=====")
# for i in range(0, a.__len__()):
# print("x ≡ %d ( mod %d)" % (a[i], m[i]))
#
# print("Chinese remainder theorem: %d" % Utils.Utils.crt(a, m))
print("=====Galois Field Extended=====")
gfe1 = GF.GFExtended(3, [1, 1, 0])
gfe2 = GF.GFExtended(3, [1, 1, 1])
print("GFE1: %s\nGFE2: %s" % (gfe1, gfe2))
print("Sum: %s" % (gfe1 + gfe2))
print("Mul: %s" % (gfe1 * gfe2))
print("Invert gf1: %s" % (~gfe1))
print("Invert gf2: %s" % (~gfe2))
print("\nAll elements GF(2^3):")
elements = GF.GFExtended.get_all_elements(3)
for e in elements:
print(e)
def buildBundle(payload, payloadDir):
try:
#RM old bundle
res=GF.run("rm "+payload)
GF.log("rm "+payload,1)
#make new bundle
res=GF.run("tar cvf "+payload+' '+payloadDir+"/*")
GF.log("tar cvf "+payload+' '+payloadDir+"/*",1)
except Exception as x:
print x, "\n", res
sys.exit()
sys.exit(1)
if __name__ == '__main__':
ifname = 'HISTORY.P0001'
ofname = 'UVWP_t.dat'
ofname2 = 'ResE_t.dat'
if len(sys.argv) >= 2:
if sys.argv[1] == '-h' or sys.argv[1] == '--help':
print('usage: python dump_hdata.py [HISTORY [UVWP.dat [ResE.dat]]]')
sys.exit(0)
ifname = sys.argv[1]
if len(sys.argv) >= 3:
ofname = sys.argv[2]
if len(sys.argv) >= 4:
ofname2 = sys.argv[3]
h = GF.GF_FILE()
if not h.read(ifname):
sys.stderr.write("[Error] HISTORY file read failed: %s\n" % ifname)
sys.exit(2)
try:
dat = h.dataset[0].data[1].array[:,28:]
np.savetxt(ofname, dat, header='U V W P')
except:
sys.stderr.write("[Error] savetxt failed: %s\n" % ofname)
sys.exit(3)
try:
dat = h.dataset[0].data[1].array[:,19]
np.savetxt(ofname2, dat, header='ResEnergy')
except:
sys.stderr.write("[Error] savetxt failed: %s\n" % ofname2)
sys.exit(4)
output = None
verbose = False
loadState()
for o, a in opts:
if o in ("-d", "--debug"):
GF.logLevel=2
elif o in ("-i", "--info"):
GF.logLevel=1
elif o in ("-l", "--list"):
cnt=0
for node in GF.nodes:
if node.running() is True:
cnt+=1
node.desc()
GF.log("There are a total of "+str(cnt)+" instances running.",0)
saveState()
sys.exit()
elif o in ("--listblock"):
cnt=0
for node in GF.nodes:
if node.running() is True:
cnt+=1
node.desc_detail()
GF.log("There are a totoal of "+str(cnt)+" instances running.",0)
saveState()
sys.exit()
elif o in ("--listspots"):
getSpotRequests()
runcnt=0
ocnt=0
def testparser(
grammar_file,
grammar_object,
gf_language,
start, # grammar
sentences,
sentences_object,
sentences_file, # sentences
statistics,
encoding,
nr_trees,
nr_sentences,
seconds,
width,
no_header,
only_header,
verbose,
quiet,
**strategy):
table_header = (
" OK? | Grammar | Parser | Nr | Len | Chart | (rul,act,pas,pred) |"
" Time | Time/item | Trees | Unique | Correct | Sentence \n"
"----:|------------|----------|----:|-----:|--------:|:------------------:|"
"------------:|------------:|-------:|-------:|--------:|----------------"
)
if only_header:
print(table_header)
return
assert grammar_file
assert not (strategy['topdown'] and strategy['bottomup'])
assert not (gf_language and not grammar_object)
assert not (statistics and
(sentences_object or sentences_file or sentences))
if not (strategy['topdown'] or strategy['bottomup']):
strategy['topdown'] = True
starters = None
grammar_module = {}
exec(open(grammar_file).read(), grammar_module)
if not grammar_object:
grammar_object = GRAMMAR_OBJECT
if gf_language:
gf_grammar = GF.PGF(grammar_module[grammar_object])[gf_language]
grammar = gf_grammar.mcfrule_iter()
if not start:
starters = gf_grammar.cnccats[gf_grammar.abstract.start]
else:
grammar = grammar_module[grammar_object]
if isinstance(grammar, dict):
grammar = [(f, c, a, r) for (f, (c, a, r)) in grammar.items()]
if not starters:
if not start:
start = grammar[0][1]
starters = [start]
if not statistics:
if not sentences:
if sentences_file:
sentences = []
with open(sentences_file, encoding=encoding) as F:
for line in F:
line = line.strip()
if line:
sentences.append(line)
else:
if not sentences_object:
sentences_object = SENTENCES_OBJECT
sentences = grammar_module[sentences_object]
def convert_sentence(s):
if isinstance(s, str) and ":" in s:
s = tuple(s.split(":", 1))
if isinstance(s, tuple) and len(s) == 2:
n, s = s
n = int(n)
else:
n = -1
if isinstance(s, str):
s = s.split()
return n, s
if nr_sentences and nr_sentences > 0:
sentences = sentences[:nr_sentences]
sentences = list(map(convert_sentence, sentences))
grammar_name = gf_language or grammar_file
grammar_name = os.path.splitext(os.path.basename(grammar_name))[0]
parser_name = ""
if strategy["topdown"]: parser_name += "td"
if strategy["bottomup"]: parser_name += "bu"
if strategy["filtered"]: parser_name += "-lc"
if strategy["nonempty"]: parser_name += "-ne"
if quiet:
ctr = MCFParser.TracedCounter("Reading grammar '%s':" % grammar_name)
parser = MCFParser.Parser(grammar, starters, trace=not quiet, **strategy)
if quiet:
ctr.finalize()
if statistics:
parser.print_grammar_statistics()
return
if not no_header:
if not quiet:
header = "%s, %d sentences: %s = %s" % (
grammar_file, len(sentences), ", ".join(
key for key in strategy if strategy[key]), parser_name)
print(header)
print("=" * len(header))
print()
print(table_header)
if seconds:
time_multiplier = 1.0
time_suffix = "s"
else:
time_multiplier = 1e3
time_suffix = "ms"
if quiet:
ctr = MCFParser.TracedCounter("Parsing:", interval=1)
totaltime = totalsize = totalincfs = total_parsed_trees = total_unique_trees = total_correct_trees = 0
totalfail = False
for n, (correct_trees, sent) in enumerate(sentences, 1):
if correct_trees > nr_trees:
correct_trees = nr_trees
result = parser.chart_parse(sent, trace=int(verbose))
stat = parser.statistics
time = stat['Time']
chartsize = stat['Chart']['TOTAL']
try:
pct_actives = 100.0 * stat['Chart']['Active'] / chartsize
pct_found = 100.0 * stat['Chart']['Found'] / chartsize
pct_predicts = 100.0 * stat['Chart']['Symbol'] / chartsize
pct_dynrules = 100.0 * stat['Chart']['Rule'] / chartsize
item_time = time / chartsize
except ZeroDivisionError:
pct_actives = pct_found = pct_predicts = pct_dynrules = item_time = 0.0
all_parsed_trees = list(
itertools.islice(parser.extract_trees(), nr_trees))
parsed_trees = len(all_parsed_trees)
unique_trees = len(set(all_parsed_trees))
totaltime += time
totalsize += chartsize
total_parsed_trees += parsed_trees
total_unique_trees += unique_trees
total_correct_trees += correct_trees if correct_trees >= 0 else parsed_trees
fail = "F" if bool(result) != bool(parsed_trees) else ""
if parsed_trees == correct_trees: correct_trees = ""
if parsed_trees == unique_trees: unique_trees = ""
totalfail |= bool(fail or unique_trees
or (correct_trees and correct_trees >= 0))
if parsed_trees == nr_trees: parsed_trees = "%s+" % nr_trees
sentstr = " ".join(sent)
if width: sentstr = sentstr[:width]
if quiet: ctr.inc()
if not quiet or fail or unique_trees or correct_trees:
print((
'%4s | %-10s | %-8s |%4d |%5d |%8d | (%3.0f,%3.0f,%3.0f,%3.0f%%) | '
'%8.2f %-2s | %8.2f us |%7s |%7s | %7s | "%s"' %
("FAIL" if fail else "", grammar_name[:10], parser_name, n,
len(sent), chartsize, pct_dynrules, pct_actives, pct_found,
pct_predicts, time_multiplier * time, time_suffix,
1e6 * item_time, parsed_trees, unique_trees, correct_trees,
sentstr)))
if quiet:
ctr.finalize()
print(
("%4s | %-10s | %-8s |%4d | |%8d | | "
"%8.2f %-2s | %8.2f us |%7s |%7s | %7s | " %
("FAIL" if totalfail else " OK ", grammar_name[:10], parser_name,
len(sentences), 1.0 * totalsize / len(sentences), time_multiplier *
totaltime / len(sentences), time_suffix, 1e6 * totaltime / totalsize,
total_parsed_trees, total_unique_trees, total_correct_trees)))
import GF
import random
w = 8
total = 2 ** w
gf = GF.GF(w)
def vector_mul(a, b):
res = 0
for i in range(len(a)):
res = gf.add(res, gf.mul(a[i], b[i]))
return res
def matrix_mul(a, b):
a_row = len(a)
b_col = len(b[0])
b_row = len(b)
result = []
for i in range(a_row):
row_vector = []
for j in range(b_col):
row_vector.append(vector_mul(a[i], [b[m][j] for m in range(b_row)]))
result.append(row_vector)
return result
def stringToList(s):
if s == '':
return []
s = s[1:len(s)-1]
s = s.replace(' ', '')
#print(s)
def test_inv(self):
a = 0b11
b = 1 << 233 | 1 << 9 | 1 << 4 | 1 << 1 | 1
i = GF.inv(a, b)
print(bin(i))
print(bin(GF.mult(a, i ^ b, b)))
def test_mult(self):
a = 0b111
b = 0b11
m = GF.mult(a, b, 0b100000000)
print(bin(m))
import GF gf = GF.GF(8) print(gf.div(140, 172)) print(gf.mul(77, 15)) print(gf.sub(140, 172)) print(gf.add(32, 172))
import GF as GF
import time
blist = [0x02, 0x03, 0x09, 0x0B, 0x0D, 0x0E]
print("** TESTING: GF_product_p **\n")
for b in blist:
sum = 0
start = time.time()
for a in range(1, 256):
f = GF.GF_product_p(a, b)
sum = sum + f
end = time.time()
elapsed = end*1000-start*1000 #milliseconds
print("b = ", b)
print("sum = ", sum)
print(elapsed, "milliseconds")
print("\n** TESTING: GF_product_t with generation of tables**\n")
for b in blist:
sum = 0
start = time.time()
[exp, log] = GF.GF_tables()
for a in range(1, 256):
f = GF.GF_product_t(a, b, exp, log)
sum = sum + f
end = time.time()
elapsed = end*1000-start*1000 #milliseconds
print("b = ", b)
print("sum = ", sum)
print(elapsed, "milliseconds")
