def startNodes(ami, inst_size, keyName, maxPrice, nodecnt): GF.log("... starting " + str(nodecnt) + " node(s)", 1); local=[] try: res = GF.run("ec2-request-spot-instances " + ami + " -p " + str(maxPrice) + " -instance-type " + inst_size + " -n " + str(nodecnt) + " --type one-time" + " --key " + keyName) lines=res.split("\n") for i in range(0,len(res.split("\n"))): line=lines[i] print "res: ",i,line if line.find("SPOTINSTANCEREQUEST")>=0: inst=line.split("\t") local.append(CLnode.CLnode( '' ,'slave' ,inst[5], '' , '' , '' ,inst[6] ,inst[0], '' ,False,inst[1],False)) #instID,instName,status , ami , key , size , date , ntype , url ,master,sir,deployed): if res.find("timeout")>=0: print "TIMEOUT: ", res sys.exit() if res.find("InvalidAMIID")>=0: print "INVALID AMI ID: ", res sys.exit() except Exception as x: print x, "\n", res sys.exit() GF.addNewNodes(local)
def __init__( self, instID="", instName="", status="", ami="", key="", size="", date="", ntype="", url="", master=False, sir="", deployed=False, ): self.instID = instID self.instName = instName self.status = status self.ami = ami self.key = key self.size = size self.date = date self.ntype = ntype self.url = url self.master = GF.str2bool(master) self.sir = sir self.deployed = GF.str2bool(deployed)
def launchCluster(ami, inst_size, keyName, maxPrice, nodes): GF.log("Maximum Price: "+str(maxPrice), 1); curPrice=curSpotCost(inst_size) if curPrice == -1: print "Error: Failed to get current spot price." sys.exit(-1) if curPrice > maxPrice: print "Error: Current spot price too high." sys.exit(-2) GF.log("Launching "+str(nodes)+" nodes.", 1); startNodes(ami, inst_size, keyName, maxPrice, nodes)
def divide_with_remainder(dividend, divider): biggest_dividend_power = Utils.get_biggest_power(dividend) biggest_divider_power = Utils.get_biggest_power(divider) # Jeśli dzielnik mniejszy od dzielnej lub równy 0 nie trzeba dzielić if biggest_dividend_power < biggest_divider_power or ( biggest_dividend_power >= 1 and biggest_divider_power == 0): return [[0], dividend] max_power = max(biggest_dividend_power, biggest_divider_power) result = [[0] * max_power] * 2 new_dividend = list(dividend) while True: i = len(new_dividend) - 1 while new_dividend[i] == 0 and i > 0: i -= 1 if i < 0: return result # Dzielenie przez najwyższą potęgę dzielnej current_power = i - biggest_divider_power # Jeśli potęga mniejsza od zera to koniec dzielenia if current_power < 0: if i == 0: return [dividend, [0] * max_power] else: return result # Dzielenie największego wyrazu dzielnej przez największy wyraz dzielnika div_result = [0] * (current_power + 1) gfs1 = GF.GFSimple(new_dividend[i], 2) gfs2 = GF.GFSimple(divider[biggest_divider_power], 2) gfs_result = gfs1 * (~gfs2) div_result[current_power] = gfs_result.value # Dodanie wyniku dzielenia do result result[0] = Utils.add_polynomials(result[0], div_result) # Mnożenie dzielnika przez wynik dzielenia subtrahend = Utils.mul_polynomials(divider, div_result) # Odejmowanie wymnożonej wartości od dzielnej new_dividend = Utils.sub_polynomials(new_dividend, subtrahend) # Zwrócenie wyniku jeśli nie można dalej dzielić if Utils.get_biggest_power(new_dividend) < biggest_divider_power: result[1] = new_dividend return result
def sub_polynomials(poly1, poly2): if len(poly1) >= len(poly2): for i in range(0, len(poly2)): gfs1 = GF.GFSimple(poly1[i], 2) gfs2 = GF.GFSimple(poly2[i], 2) gfs_result = gfs1 + (~gfs2) poly1[i] = gfs_result.value return poly1 else: for i in range(0, len(poly1)): gfs1 = GF.GFSimple(poly1[i], 2) gfs2 = GF.GFSimple(poly2[i], 2) gfs_result = gfs1 + (~gfs2) poly2[i] = gfs_result.value return poly2
def test_mod(self): a = 7 print(bin(a)) b = 5 print(bin(b)) m = GF.mod(a, b) print(bin(m))
def getRunningInstances(): try: res = GF.run("ec2-describe-instances") if res.find("timeout")>=0: print "TIMEOUT: ", res sys.exit() for line in res.split("\n"): if line.find("INSTANCE")>=0: inst=line.split("\t") if inst[5] != "terminated": GF.nodes.append(CLnode.CLnode(inst[1],inst[1],inst[5],inst[2],inst[6],inst[9],inst[10],inst[0],inst[3],'',inst[22])) else: GF.log("found terminated"+line,2) except Exception as x: print x, "\n", res sys.exit()
def mul_polynomials(poly1, poly2): max_power = Utils.get_biggest_power(poly1) + Utils.get_biggest_power( poly2) mul_result = [0] * (max_power + 1) for i in range(len(poly1) - 1, -1, -1): for j in range(len(poly2) - 1, -1, -1): gfs1 = GF.GFSimple(poly1[i], 2) gfs2 = GF.GFSimple(poly2[j], 2) gfs_mul_result = gfs1 * gfs2 if gfs_mul_result.value > 0: power = i + j gfs3 = GF.GFSimple(mul_result[power], 2) gfs_add_result = gfs3 + gfs_mul_result mul_result[power] = gfs_add_result.value return mul_result
def curSpotCost(inst_size): lt = time.localtime(time.time()) curdate = str(lt[0])+"-"+str(lt[1])+"-"+str(lt[2])+"T"+str(lt[3])+":"+str(lt[4])+":"+str(lt[5])+":"+str(lt[6])+"-0000" try: res = GF.run("ec2-describe-spot-price-history -d Linux/UNIX --region us-east-1 --instance-type "+inst_size+" -s "+curdate) if res.find("timeout")>=0: print "TIMEOUT: ", res sys.exit() cost=0 for i in res.split("\n"): cost += float((i.split("\t"))[1]); cost = cost/len(res.split("\n")) except Exception as x: print x, "\n", res sys.exit() GF.log("Current Instance Cost: "+str(cost), 1); return cost
def test_div_mod(self): a = 15 print(bin(a)) b = 7 print(bin(b)) q, v = GF.div_mod(a, b) print(bin(q)) print(bin(v))
def gather(self, logsDir, sshKey): print "\n============================================" print "Gathering logs: ", self.instID, "/", self.instName print "=====" logsDir += "/" + self.instID + "-" + self.instName try: res = GF.run("mkdir -p " + logsDir) res = GF.run( "scp -r -o StrictHostKeyChecking=no -i " + sshKey + " ubuntu@" + self.url + ":/var/tmp/log " + logsDir ) print "scp -r -o StrictHostKeyChecking=no -i " + sshKey + " ubuntu@" + self.url + ":/var/tmp/log " + logsDir except Exception as x: print x, "\n", res return -1 """
def deploy(self, payload, sshKey, launch=False): # TODO: error handeling # remove SIG with ssh-keygen -f "/home/madmaze/.ssh/known_hosts" -R ec2-184-73-46-186.compute-1.amazonaws.com print "\n============================================" print "Deploing", self.instID, "/", self.instName print "=====" if self.master is False: try: res = GF.run( "scp -o StrictHostKeyChecking=no -i " + sshKey + " " + payload + " ubuntu@" + self.url + ":~/" ) print "scp -o StrictHostKeyChecking=no -i " + sshKey + " " + payload + " ubuntu@" + self.url + ":~/" except Exception as x: print x, "\n", res return -1 # EXTRACT payload try: res = GF.run( "ssh -o StrictHostKeyChecking=no -i " + sshKey + " ubuntu@" + self.url + " 'tar xvf ~/bundle.tar;'" ) print "ssh -o StrictHostKeyChecking=no -i " + sshKey + " ubuntu@" + self.url + " 'tar xvf ~/bundle.tar;'" except Exception as x: print x, "\n", res return -1 if launch is True: # LAUNCH Payload try: print "ssh -o StrictHostKeyChecking=no -i " + sshKey + " ubuntu@" + self.url + " 'python ~/payload/setup.py&'" res = GF.run( "ssh -o StrictHostKeyChecking=no -i " + sshKey + " ubuntu@" + self.url + " 'screen -dm python ~/payload/setup.py'" ) self.deployed = True except Exception as x: print x, "\n", res return -1 else: print "Master node: No need to deploy!"
def loadState(): getRunningInstances() nodes=[] try: fname="./nodeDB" FILE = open(fname,"r") while FILE: line = FILE.readline().strip().split(",") if len(line)<=1: break nodes.append(CLnode.CLnode(*line)) #line[0],line[1],line[2],line[3],line[4],line[5],line[6],line[7],line[8],line[9] FILE.close() GF.addNewNodes(nodes) except IOError: print "warning, nodeDB does not exists yet" except Exception as x: print x, "\n", res sys.exit()
def kill(self): if self.status != "running": print "Instance is not running, therefore will not shutdown" return try: res = "" res = GF.run("ec2-terminate-instances " + self.instID) print res self.status = "terminated" except Exception as x: print x, "\n", res return -1
def getSpotRequests(): try: res = GF.run("ec2-describe-spot-instance-requests") if res.find("timeout")>=0: print "TIMEOUT: ", res sys.exit() for line in res.split("\n"): if line.find("INSTANCE")>=0: inst=line.split("\t") GF.reqests.append(CLnode.CLnode(inst[1],inst[1],inst[5],'','','',inst[6],inst[0],'')) except Exception as x: print x, "\n", res sys.exit()
def launchMaster(ami, inst_size, keyName): GF.log("Launching Master node..",1) local=[] try: res = GF.run("ec2-run-instances " + ami + " -k " + keyName + " -t " + size) if res.find("InvalidAMIID")>=0: print "INVALID AMI ID: ", res sys.exit() print res i=0 lines=res.split("\n") master=CLnode.CLnode() for l in lines: inst=l.split("\t") if inst[0]=="INSTANCE": master = CLnode.CLnode(inst[1],"MASTER",inst[5],inst[2],inst[6],inst[9],inst[10],inst[0],'',True) master.desc_detail() local.append(master) GF.addNewNodes(local) except Exception as x: print x, "\n", res sys.exit()
def monitor(n, timeout): allStarted=True for i in range(0,n): allStarted=True getSpotRequests() #getRunningInstances() for n in GF.requests: if n.status=="open": allStarted=False if allStarted is True: break time.sleep(timeout) if allStarted is False: print "All instances did not start during designated time." if GF.confirmQuestion("Would you like to continue?") is True: monitor(n, timeout) else: sys.exit() else: for i in range(0,n): allStarted=True getRunningInstances() for n in GF.nodes: if n.status=="pending": allStarted=False if allStarted is True: break if allStarted is False: print "All instances did not start during designated time." if GF.confirmQuestion("Would you like to continue?") is True: monitor(n, timeout) else: sys.exit() else: #launch?! print "stuff"
def main(): print("=====KODOWANIE I SZYFROWANIE DANYCH=====\n") # gf1 = GF.GFSimple(1, 7) # gf2 = GF.GFSimple(2, 7) # # print("=====Galois Field Simple=====") # print("GF1: %s\nGF2: %s" % (gf1, gf2)) # print("Sum: %s" % (gf1 + gf2)) # print("Mul: %s" % (gf1 * gf2)) # print("Invert gf1: %s" % (~gf1)) # print("Invert gf2: %s" % (~gf2)) # print("Generators amount: %d" % GF.GFSimple.get_generators_amount(7)) # print("Generators: %s" % GF.GFSimple.get_generators(7)) # # a = [3, 4, 1] # m = [4, 5, 7] # print("\n=====Chinese Remainder Theorem=====") # for i in range(0, a.__len__()): # print("x ≡ %d ( mod %d)" % (a[i], m[i])) # # print("Chinese remainder theorem: %d" % Utils.Utils.crt(a, m)) print("=====Galois Field Extended=====") gfe1 = GF.GFExtended(3, [1, 1, 0]) gfe2 = GF.GFExtended(3, [1, 1, 1]) print("GFE1: %s\nGFE2: %s" % (gfe1, gfe2)) print("Sum: %s" % (gfe1 + gfe2)) print("Mul: %s" % (gfe1 * gfe2)) print("Invert gf1: %s" % (~gfe1)) print("Invert gf2: %s" % (~gfe2)) print("\nAll elements GF(2^3):") elements = GF.GFExtended.get_all_elements(3) for e in elements: print(e)
def buildBundle(payload, payloadDir): try: #RM old bundle res=GF.run("rm "+payload) GF.log("rm "+payload,1) #make new bundle res=GF.run("tar cvf "+payload+' '+payloadDir+"/*") GF.log("tar cvf "+payload+' '+payloadDir+"/*",1) except Exception as x: print x, "\n", res sys.exit()
sys.exit(1) if __name__ == '__main__': ifname = 'HISTORY.P0001' ofname = 'UVWP_t.dat' ofname2 = 'ResE_t.dat' if len(sys.argv) >= 2: if sys.argv[1] == '-h' or sys.argv[1] == '--help': print('usage: python dump_hdata.py [HISTORY [UVWP.dat [ResE.dat]]]') sys.exit(0) ifname = sys.argv[1] if len(sys.argv) >= 3: ofname = sys.argv[2] if len(sys.argv) >= 4: ofname2 = sys.argv[3] h = GF.GF_FILE() if not h.read(ifname): sys.stderr.write("[Error] HISTORY file read failed: %s\n" % ifname) sys.exit(2) try: dat = h.dataset[0].data[1].array[:,28:] np.savetxt(ofname, dat, header='U V W P') except: sys.stderr.write("[Error] savetxt failed: %s\n" % ofname) sys.exit(3) try: dat = h.dataset[0].data[1].array[:,19] np.savetxt(ofname2, dat, header='ResEnergy') except: sys.stderr.write("[Error] savetxt failed: %s\n" % ofname2) sys.exit(4)
output = None verbose = False loadState() for o, a in opts: if o in ("-d", "--debug"): GF.logLevel=2 elif o in ("-i", "--info"): GF.logLevel=1 elif o in ("-l", "--list"): cnt=0 for node in GF.nodes: if node.running() is True: cnt+=1 node.desc() GF.log("There are a total of "+str(cnt)+" instances running.",0) saveState() sys.exit() elif o in ("--listblock"): cnt=0 for node in GF.nodes: if node.running() is True: cnt+=1 node.desc_detail() GF.log("There are a totoal of "+str(cnt)+" instances running.",0) saveState() sys.exit() elif o in ("--listspots"): getSpotRequests() runcnt=0 ocnt=0
def testparser( grammar_file, grammar_object, gf_language, start, # grammar sentences, sentences_object, sentences_file, # sentences statistics, encoding, nr_trees, nr_sentences, seconds, width, no_header, only_header, verbose, quiet, **strategy): table_header = ( " OK? | Grammar | Parser | Nr | Len | Chart | (rul,act,pas,pred) |" " Time | Time/item | Trees | Unique | Correct | Sentence \n" "----:|------------|----------|----:|-----:|--------:|:------------------:|" "------------:|------------:|-------:|-------:|--------:|----------------" ) if only_header: print(table_header) return assert grammar_file assert not (strategy['topdown'] and strategy['bottomup']) assert not (gf_language and not grammar_object) assert not (statistics and (sentences_object or sentences_file or sentences)) if not (strategy['topdown'] or strategy['bottomup']): strategy['topdown'] = True starters = None grammar_module = {} exec(open(grammar_file).read(), grammar_module) if not grammar_object: grammar_object = GRAMMAR_OBJECT if gf_language: gf_grammar = GF.PGF(grammar_module[grammar_object])[gf_language] grammar = gf_grammar.mcfrule_iter() if not start: starters = gf_grammar.cnccats[gf_grammar.abstract.start] else: grammar = grammar_module[grammar_object] if isinstance(grammar, dict): grammar = [(f, c, a, r) for (f, (c, a, r)) in grammar.items()] if not starters: if not start: start = grammar[0][1] starters = [start] if not statistics: if not sentences: if sentences_file: sentences = [] with open(sentences_file, encoding=encoding) as F: for line in F: line = line.strip() if line: sentences.append(line) else: if not sentences_object: sentences_object = SENTENCES_OBJECT sentences = grammar_module[sentences_object] def convert_sentence(s): if isinstance(s, str) and ":" in s: s = tuple(s.split(":", 1)) if isinstance(s, tuple) and len(s) == 2: n, s = s n = int(n) else: n = -1 if isinstance(s, str): s = s.split() return n, s if nr_sentences and nr_sentences > 0: sentences = sentences[:nr_sentences] sentences = list(map(convert_sentence, sentences)) grammar_name = gf_language or grammar_file grammar_name = os.path.splitext(os.path.basename(grammar_name))[0] parser_name = "" if strategy["topdown"]: parser_name += "td" if strategy["bottomup"]: parser_name += "bu" if strategy["filtered"]: parser_name += "-lc" if strategy["nonempty"]: parser_name += "-ne" if quiet: ctr = MCFParser.TracedCounter("Reading grammar '%s':" % grammar_name) parser = MCFParser.Parser(grammar, starters, trace=not quiet, **strategy) if quiet: ctr.finalize() if statistics: parser.print_grammar_statistics() return if not no_header: if not quiet: header = "%s, %d sentences: %s = %s" % ( grammar_file, len(sentences), ", ".join( key for key in strategy if strategy[key]), parser_name) print(header) print("=" * len(header)) print() print(table_header) if seconds: time_multiplier = 1.0 time_suffix = "s" else: time_multiplier = 1e3 time_suffix = "ms" if quiet: ctr = MCFParser.TracedCounter("Parsing:", interval=1) totaltime = totalsize = totalincfs = total_parsed_trees = total_unique_trees = total_correct_trees = 0 totalfail = False for n, (correct_trees, sent) in enumerate(sentences, 1): if correct_trees > nr_trees: correct_trees = nr_trees result = parser.chart_parse(sent, trace=int(verbose)) stat = parser.statistics time = stat['Time'] chartsize = stat['Chart']['TOTAL'] try: pct_actives = 100.0 * stat['Chart']['Active'] / chartsize pct_found = 100.0 * stat['Chart']['Found'] / chartsize pct_predicts = 100.0 * stat['Chart']['Symbol'] / chartsize pct_dynrules = 100.0 * stat['Chart']['Rule'] / chartsize item_time = time / chartsize except ZeroDivisionError: pct_actives = pct_found = pct_predicts = pct_dynrules = item_time = 0.0 all_parsed_trees = list( itertools.islice(parser.extract_trees(), nr_trees)) parsed_trees = len(all_parsed_trees) unique_trees = len(set(all_parsed_trees)) totaltime += time totalsize += chartsize total_parsed_trees += parsed_trees total_unique_trees += unique_trees total_correct_trees += correct_trees if correct_trees >= 0 else parsed_trees fail = "F" if bool(result) != bool(parsed_trees) else "" if parsed_trees == correct_trees: correct_trees = "" if parsed_trees == unique_trees: unique_trees = "" totalfail |= bool(fail or unique_trees or (correct_trees and correct_trees >= 0)) if parsed_trees == nr_trees: parsed_trees = "%s+" % nr_trees sentstr = " ".join(sent) if width: sentstr = sentstr[:width] if quiet: ctr.inc() if not quiet or fail or unique_trees or correct_trees: print(( '%4s | %-10s | %-8s |%4d |%5d |%8d | (%3.0f,%3.0f,%3.0f,%3.0f%%) | ' '%8.2f %-2s | %8.2f us |%7s |%7s | %7s | "%s"' % ("FAIL" if fail else "", grammar_name[:10], parser_name, n, len(sent), chartsize, pct_dynrules, pct_actives, pct_found, pct_predicts, time_multiplier * time, time_suffix, 1e6 * item_time, parsed_trees, unique_trees, correct_trees, sentstr))) if quiet: ctr.finalize() print( ("%4s | %-10s | %-8s |%4d | |%8d | | " "%8.2f %-2s | %8.2f us |%7s |%7s | %7s | " % ("FAIL" if totalfail else " OK ", grammar_name[:10], parser_name, len(sentences), 1.0 * totalsize / len(sentences), time_multiplier * totaltime / len(sentences), time_suffix, 1e6 * totaltime / totalsize, total_parsed_trees, total_unique_trees, total_correct_trees)))
import GF import random w = 8 total = 2 ** w gf = GF.GF(w) def vector_mul(a, b): res = 0 for i in range(len(a)): res = gf.add(res, gf.mul(a[i], b[i])) return res def matrix_mul(a, b): a_row = len(a) b_col = len(b[0]) b_row = len(b) result = [] for i in range(a_row): row_vector = [] for j in range(b_col): row_vector.append(vector_mul(a[i], [b[m][j] for m in range(b_row)])) result.append(row_vector) return result def stringToList(s): if s == '': return [] s = s[1:len(s)-1] s = s.replace(' ', '') #print(s)
def test_inv(self): a = 0b11 b = 1 << 233 | 1 << 9 | 1 << 4 | 1 << 1 | 1 i = GF.inv(a, b) print(bin(i)) print(bin(GF.mult(a, i ^ b, b)))
def test_mult(self): a = 0b111 b = 0b11 m = GF.mult(a, b, 0b100000000) print(bin(m))
import GF gf = GF.GF(8) print(gf.div(140, 172)) print(gf.mul(77, 15)) print(gf.sub(140, 172)) print(gf.add(32, 172))
import GF as GF import time blist = [0x02, 0x03, 0x09, 0x0B, 0x0D, 0x0E] print("** TESTING: GF_product_p **\n") for b in blist: sum = 0 start = time.time() for a in range(1, 256): f = GF.GF_product_p(a, b) sum = sum + f end = time.time() elapsed = end*1000-start*1000 #milliseconds print("b = ", b) print("sum = ", sum) print(elapsed, "milliseconds") print("\n** TESTING: GF_product_t with generation of tables**\n") for b in blist: sum = 0 start = time.time() [exp, log] = GF.GF_tables() for a in range(1, 256): f = GF.GF_product_t(a, b, exp, log) sum = sum + f end = time.time() elapsed = end*1000-start*1000 #milliseconds print("b = ", b) print("sum = ", sum) print(elapsed, "milliseconds")