def diff(Neff_pos, Neff_neg, Nwindow): "a delta differencer with two arms - called Macd M1 in homework" h_pos = delta(Neff_pos, Nwindow) h_neg = delta(Neff_neg, Nwindow) h = h_pos - h_neg return h
def loss(beta, sigma, X, y, distribution='norm', loc=0, scale=1, a=None, c=None, s=None): ''' Input: beta --pd.Series. with length p + 1, including the intercept beta_0; sigma --scaler. > 0; X --pd.DataFrame. with shape n times p + 1, the elements of its first column are all 1; y --pd.Series. with length n Output: loss --scaler. the negative log likelihood. ''' n = X.shape[0] gammas = threshold(y, distribution, a=a, c=c, s=s) K = len(gammas) fitvalues = np.dot(X, beta) #return an array with shape (n, ) tmp = np.tile(gammas, n) #repeat gammas n times tmp = tmp.reshape(n, K) #transform tmp1 to a matrix with shape (n, K) #subtract each column of tmp by fitvalues. #Remark: in numpy, substract each column of a matrix by a vector only works if #the trailing axes have the same dimension. The shape of tmp is (n, K), and #the shape of fitvalues is (n, ). So we need transport tmp. You can also think #numpy can do "matrix - vector' by row #the sahpe of upper is (n, K), which is the elment in the parentheses of F_0 upper = ((tmp.T - fitvalues).T) / sigma x=np.array([-1.0 * np.inf]) x2 = gammas[:-1] lower_gammas = np.concatenate([x, x2]) tmp = np.tile(lower_gammas, n) tmp = tmp.reshape(n, K) lower = ((tmp.T - fitvalues).T) / sigma if distribution == 'norm': upper = st.norm.cdf(upper) lower = st.norm.cdf(lower) elif distribution == 'weibull': upper = st.exponweib.cdf(upper, a=a, c=c, loc=loc, scale=scale) lower = st.exponweib.cdf(lower, a=a, c=c, loc=loc, scale=scale) elif distribution == 'log': upper = st.lognorm.cdf(upper, s=s, loc=loc, scale=scale) lower = st.lognorm.cdf(lower, s=s, loc=loc, scale=scale) else: raise Exception('unvalid input for parameter distribution.') prob_y = upper - lower #probability of y equals to k log_prob_y = np.log(prob_y) #matrix with shape (n, K) deltah = np.array(delta(y)) loss = deltah * log_prob_y loss = loss.sum() loss = -1.0 * loss return loss
def circular_convolve(series, impulse): "circular convolution" return np.fft.ifft(np.fft.fft(impulse) * np.fft.fft(series)).real f, axarr = plt.subplots(3, 1) Nwindow = 256 Neff = 32 Ndelta = 192 print("Problem 11 (a): Ema and delta impulse responses") h_ema = ema(Neff, Nwindow) h_delta = delta(Ndelta, Nwindow) axarr[0].plot(h_ema) axarr[0].set_title('ema with Neff = 32') axarr[1].plot(h_delta) axarr[1].set_title('delta at k = 192') print("Problem 11 (b): Causal convolution") candidate1 = np.convolve(h_delta, h_ema) truncated1 = candidate1[:Nwindow] candidate2 = circular_convolve(h_delta, h_ema) axarr[2].plot(truncated1) axarr[2].plot(candidate2) axarr[2].set_title('Causal(blue) and Non-causal(red) convolution')
import delta import json person = { 'name': 'Andreas', 'skills': { 'matlab': 3, 'swift': 2 }, 'age': 25 } personLater = { 'name': 'Andreas', 'skills': { 'python': 3, 'swift': 3 }, 'age': 26 } print("Delta format:") print(json.dumps(delta.delta(person, personLater), indent = 4)) print("result of delta plus merge:") print(json.dumps(delta.merge(person, delta.delta(person, personLater)), indent = 4))
f, axarr = plt.subplots(3, 2) with open('jpm_trades.csv', 'r') as trades_csv: trades = np.array(list(csv.reader(trades_csv))[1:]).astype('double') prices = trades[:, 1] Nwindow = len(prices) # a) Delayed impulse Ndelay = 32 lag = 0 y_fde = apply_delayed_impulse_filter(prices, Ndelay) h_delta = delta(Ndelay, Nwindow) candidate = np.convolve(prices, h_delta) y_convolution = candidate[:len(prices)] x_axis = np.arange(Ndelay, len(prices)) axarr[0, 0].set_title("Ideal delay") axarr[0, 0].plot(x_axis, y_fde[Ndelay:]) axarr[0, 0].plot(x_axis, y_convolution[Ndelay:], 'o', markerfacecolor='none') # b) Box Nbox = 32
f, axarr = plt.subplots(3, 2) Nwindow = 256 # a) Delayed impulse Ndelay = 32 lag = 0 impulse = np.zeros(Nwindow) impulse[lag] = 1 candidate = apply_delayed_impulse_filter(impulse, Ndelay) impulse_response_fde = candidate[lag:] impulse_response_direct = delta(Ndelay, Nwindow) axarr[0, 0].set_ylim(0, 1.1) axarr[0, 0].set_title("Ideal delay") axarr[0, 0].plot(impulse_response_fde) axarr[0, 0].plot(impulse_response_direct, 'o', markerfacecolor='none') # b) Box Nbox = 32 lag = 1 impulse = np.zeros(Nwindow) impulse[lag] = 1 candidate = apply_box_filter(impulse, Nbox)
def login(argv): cmds = '' hostname = '' interval = 1 out1 = [] out2 = [] res_1 = [] res_2 = [] prompt = '' try: opts, args = getopt.getopt(argv, "hc:i:n:", ["command=", "interval=", "node="]) except getopt.GetoptError: print('deltaScript.py -n <node> -i <interval in secs> -c <command>') sys.exit(2) for opt, arg in opts: if opt == '-h': print( 'deltaScript.py -n <node> -i <interval in secs> -c <command>') sys.exit() elif opt in ("-i", "--interval"): interval = int(arg) elif opt in ("-c", "--command"): cmds = arg elif opt in ("-n", "--node"): hostname = arg #s = pxssh.pxssh() # hostname = raw_input('hostname: ') username = input('username: '******'password: '******'\n') interact.expect(prompt) interact.send('environment time-stamp') interact.expect(prompt) interact.send('environment no more') interact.expect(prompt) cmd_output = interact.current_output_clean #interact.send(cmd_output) #print(cmd_output) #s.sendline('environment time-stamp') #s.prompt() #s.sendline('environment no more') #s.prompt() for cmd in cmds.split(";"): #s.sendline(cmd) interact.send(cmd) #s.prompt() # match the prompt interact.expect(prompt) #out1.append(s.before) out1.append(interact.current_output) #print(out1[-1]) # print(s.before) # print everything before the prompt. # s.prompt () # s.sendline ('sleep ' + str(interval)) time.sleep(interval) # s.prompt () # match the prompt for cmd in cmds.split(";"): #s.sendline(cmd) interact.send(cmd) #s.prompt() # match the prompt interact.expect(prompt) #out2.append(s.before) out2.append(interact.current_output) # print len(out2) # print(s.before) # print everything before the prompt. print("The prompt is " + prompt) #s.sendline('logout') #s.close() #print (out1) #print (out2) for (aOut1, aOut2) in zip(out1, out2): #print(aOut1.decode('utf-8').split("\r")[-2]) #print(aOut2.decode('utf-8').split("\r")[-2]) #res_1.append(parse(aOut1.decode('utf-8').split("\r"))) #res_2.append(parse(aOut2.decode('utf-8').split("\r"))) #print(aOut1) #print(aOut2) print(aOut1.split("\n")[-2]) print(aOut2.split("\n")[-2]) res_1.append(parse(aOut1.split("\n"))) res_2.append(parse(aOut2.split("\n"))) # print (res_1) # print (res_2) for (aRes_1, aRes_2) in zip(res_1, res_2): delta(aRes_1, aRes_2, interval)
import delta #print delta.delta(1,10,[1,2,3,4]) delta.initMatrix([[1,2,4],[3,4,7],[8,9,10]], [[5,6,1],[7,8,0]]) s = delta.delta(3,5,[8,9,10])
from step import step from box import box from delta import delta from ema import ema from macd import macd from diff import diff Nwindow = 400 Neff_pos = 20 Neff_neg = 40 Neff = 20 Nbox = 20 h_step = step(Nwindow) h_box = box(Nbox, Nwindow) h_delta = delta(Neff_pos, Nwindow) h_ema = ema(Neff_pos, Nwindow) h_macd = macd(Neff_pos, Neff_neg, Nwindow) h_diff = diff(Neff_pos, Neff_neg, Nwindow) f, axarr = plt.subplots(3, 2) axarr[0, 0].plot(h_step) axarr[0, 0].set_title("step") axarr[0, 1].plot(h_box) axarr[0, 1].set_title("box") axarr[1, 0].plot(h_delta) axarr[1, 0].set_title("delta")
import delta a = 3 b = 4 c = delta.delta(a,b) print(c)
def login(argv): cmds = '' hostname = '' interval = 1 out1 = [] out2 = [] res_1 = [] res_2 = [] try: opts, args = getopt.getopt(argv, "hc:i:n:", ["command=", "interval=", "node="]) except getopt.GetoptError: print 'login.py -n <node> -i <interval in secs> -c <command>' sys.exit(2) for opt, arg in opts: if opt == '-h': print 'login.py -n <node> -i <interval in secs> -c <command>' sys.exit() elif opt in ("-i", "--interval"): interval = int(arg) elif opt in ("-c", "--command"): cmds = arg elif opt in ("-n", "--node"): hostname = arg s = pxssh.pxssh() # hostname = raw_input('hostname: ') username = raw_input('username: '******'password: '******'environment time-stamp') s.prompt() s.sendline('environment no more') s.prompt() for cmd in cmds.split(";"): s.sendline(cmd) s.prompt() # match the prompt out1.append(s.before) # print out1[-1] # print(s.before) # print everything before the prompt. # s.prompt () # s.sendline ('sleep ' + str(interval)) time.sleep(interval) # s.prompt () # match the prompt for cmd in cmds.split(";"): s.sendline(cmd) s.prompt() # match the prompt out2.append(s.before) # print len(out2) # print(s.before) # print everything before the prompt. print("The prompt is " + s.PROMPT) s.sendline('logout') s.close() # print (out1) # print (out2) for (aOut1, aOut2) in zip(out1, out2): print aOut1.decode('utf-8').split("\r")[-2] print aOut2.decode('utf-8').split("\r")[-2] res_1.append(parse(aOut1.decode('utf-8').split("\r"))) res_2.append(parse(aOut2.decode('utf-8').split("\r"))) # print (res_1) # print (res_2) for (aRes_1, aRes_2) in zip(res_1, res_2): delta(aRes_1, aRes_2, interval)
f, axarr = plt.subplots(3, 2) with open('jpm_trades.csv', 'r') as trades_csv: trades = np.array(list(csv.reader(trades_csv))[1:]).astype('double') prices = trades[:,1] Nwindow = len(prices) # a) Delayed impulse Ndelay = 32 lag = 0 y_fde = apply_delayed_impulse_filter(prices, Ndelay) h_delta = delta(Ndelay, Nwindow) candidate = np.convolve(prices, h_delta) y_convolution = candidate[:len(prices)] x_axis = np.arange(Ndelay, len(prices)) axarr[0, 0].set_title("Ideal delay") axarr[0, 0].plot(x_axis, y_fde[Ndelay:]) axarr[0, 0].plot(x_axis, y_convolution[Ndelay:], 'o', markerfacecolor='none') # b) Box Nbox = 32 lag = 1 y_fde = apply_box_filter(prices, Nbox)
from delta import delta def circular_convolve(series, impulse): "circular convolution" return np.fft.ifft(np.fft.fft(impulse)*np.fft.fft(series)).real f, axarr = plt.subplots(3, 1) Nwindow = 256 Neff = 32 Ndelta = 192 print("Problem 11 (a): Ema and delta impulse responses") h_ema = ema(Neff, Nwindow) h_delta = delta(Ndelta, Nwindow) axarr[0].plot(h_ema) axarr[0].set_title('ema with Neff = 32') axarr[1].plot(h_delta) axarr[1].set_title('delta at k = 192') print("Problem 11 (b): Causal convolution") candidate1 = np.convolve(h_delta, h_ema) truncated1 = candidate1[:Nwindow] candidate2 = circular_convolve(h_delta, h_ema) axarr[2].plot(truncated1) axarr[2].plot(candidate2) axarr[2].set_title('Causal(blue) and Non-causal(red) convolution')
def __mapping_callback(self, regmap, varmap): self.__varmap = varmap # save current variable mapping self.__regmap = regmap self.__ctr += 1 # increment counter # # varmap = [('argv', '*<BV64 mem_7fffffffffef148_4056_64 + 0x68>'), # ('prog', '*<BV64 mem_7fffffffffef148_4056_64 + 0x30>')] # self.__varmap = varmap # # # for a, b in SYM2ADDR.iteritems(): # print 'XXXX', a, hex(b) # # exit() # # regmap = [('__r0', 'r13'), ('__r1', 'rax')] # varmap = [('array', '*<BV64 0x621bf0>')] # self.__varmap = varmap # # regmap = [('__r0', 'rdi'), ('__r1', 'rsi')] # varmap = [('array', 6851008L)] # self.__varmap = varmap # if case that you want to apply a specific mapping, discard all others # TODO: Replace < with != (?) if self.__options['mapping-id'] != -1 and self.__ctr < self.__options[ 'mapping-id']: # dbg_prnt(DBG_LVL_1, "Discard current mapping.") return 0 # --------------------------------------------------------------------- # Pretty-print the register/variable mappings # --------------------------------------------------------------------- emph('Trying mapping #%s:' % bold(self.__ctr), DBG_LVL_1) s = ['%s <-> %s' % (bolds(virt), bolds(real)) for virt, real in regmap] emph('\tRegisters: %s' % ' | '.join(s), DBG_LVL_1) s = [ '%s <-> %s' % (bolds(var), bolds(hex(val) if isinstance(val, long) else str(val))) for var, val in varmap ] emph('\tVariables: %s' % ' | '.join(s), DBG_LVL_1) # --------------------------------------------------------------------- # Apply (any) filters to the current mapping (DEBUG) # --------------------------------------------------------------------- # if you want to enumerate mappings, don't move on if self.__options['enum']: return 0 self.__options['#mappings'] += 1 # --------------------------------------------------------------------- # Identify accepted and clobbering blocks # --------------------------------------------------------------------- ''' # We check this out on marking to be more efficient if 'rsp' in [real for _, real in regmap]: # make sure that 'rsp' is not used fatal("A virtual register cannot be mapped to %s. Discard mapping..." % bolds('rsp')) return 0 # try another mapping if not MAKE_RBP_SYMBOLIC and 'rbp' in [real for _, real in regmap]: fatal("A virtual register cannot be mapped to %s. Discard mapping..." % bolds('rbp')) return 0 ''' # given the current mapping, go back to the CFG and mark all accepted blocks accblks, rsvp = self.__mark.mark_accepted(regmap, varmap) # if there is (are) >= 1 statement(s) that don't have accepted blocks, discard mapping if not accblks: dbg_prnt( DBG_LVL_1, 'There are not enough accepted blocks. Discard mapping...') return 0 # try another mapping # if there are enough accepted blocks, go back to the CFG and mark clobbering blocks cloblks = self.__mark.mark_clobbering(regmap, varmap) # At this point you can visualize the CFG # # visualize('cfg_test', entry=self.__entry, # options=VO_DRAW_CFG | VO_DRAW_CLOBBERING | VO_DRAW_ACCEPTED | VO_DRAW_CANDIDATE) # add entry point to accblks (with min uid) to avoid special cases accblks[START_PC] = [self.__entry] # also add SPL's return address as an acceptd block for stmt in self.__IR: # return is the last statement in IR if stmt['type'] == 'return': # check that target address is a valid address of a basic block if stmt['target'] != -1 and stmt['target'] not in ADDR2NODE: fatal("Return address '0x%x' not found" % stmt['target']) accblks[stmt['uid']] = [stmt['target']] # --------------------------------------------------------------------- # Pretty-print the accepted and clobbering blocks # --------------------------------------------------------------------- dbg_prnt(DBG_LVL_2, 'Accepted block set (uid/block):') for a, b in sorted(accblks.iteritems()): dbg_prnt( DBG_LVL_2, '\t%s: %s' % (bold(a, pad=3), ', '.join(['0x%x' % x for x in b]))) dbg_prnt(DBG_LVL_3, 'Clobbering block set (uid/block):') for a, b in sorted(cloblks.iteritems()): dbg_prnt( DBG_LVL_3, '\t%s: %s' % (bold(a, pad=3), ', '.join(['0x%x' % x for x in b]))) # --------------------------------------------------------------------- # Shuflle statements and build the Delta Graph # --------------------------------------------------------------------- dbg_prnt(DBG_LVL_1, "Shuffling SPL payload...") for perm in self.__shuffle(accblks): # start shuffling IR dbg_arb(DBG_LVL_1, 'Statement order:', perm) # build the adjacency list for that order adj = self.__mk_adjacency_list(perm) self.__adj = adj # remove goto statements as they are problematic adj, rm = self.__remove_goto(accblks, adj) perm = filter(lambda x: x not in rm, perm) perm = [(y, accblks[y]) for y in perm] dbg_arb(DBG_LVL_3, "Updated SPL statement adjacency list", adj) # create the Delta Graph for the given permutation DG = D.delta(self.__cfg, self.__entry, perm, cloblks, adj) # visualise delta graph # # visualize(DG.graph, VO_TYPE_DELTA) # exit() # select the K minimum induced subgraphs Hk from the Delta Graph # Hk = a subset of accepted blocks that reconstruct the execution of the SPL payload) for size, Hk in DG.k_min_induced_subgraphs(PARAMETER_K): if size < 0: # Delta Graph disconnected? dbg_prnt(DBG_LVL_1, "Delta Graph is disconnected.") break # try another permutation (or mapping) # Paths that are too long should be discarded as it's unlikely to find a trace if size > MAX_ALLOWED_TRACE_SIZE: dbg_prnt( DBG_LVL_1, "Subgraph size is too long (%d > %d). Discard it." % (size, MAX_ALLOWED_TRACE_SIZE)) break # try another permutation (or mapping) # subgraph is ok. Flatten it and make it a "tree", to easily process it tree, pretty_tree = DG.flatten_graph(Hk) emph( 'Flattened subgraph (size %d): %s' % (size, bolds(str(pretty_tree))), DBG_LVL_2) # TODO: this check will discard "trivial" solutions (all in 1 block) if size == 0: warn('Delta graph found but it has size 0') # continue # enumerate all paths, and fork accordingly # Symbolic execution used? self.__options['simulate'] = True # visualise delta graph with Hk (induced subgraph) # visualize(DG.graph, VO_TYPE_DELTA) # exit() # # TODO: In case of conditional jump, we'll have multiple "final" states. # We should check whether those states have conflicting constraints. # dbg_prnt(DBG_LVL_2, "Enumerating Tree...") self.__simstash = [] # ------------------------------------------------------------- # Easter Egg: When entry point is -1, we skip it and we directly # start from the next statement # ------------------------------------------------------------- if self.__entry == -1: if not isinstance(tree[0], tuple): fatal('First statement is a conditional jump.') # drop first transition (from entry to the 1st statement) and start # directly from the 1st statement. There is no entry point. # # also update the entry point _, _, entry = tree.pop(0) pretty_tree.pop(0) emph("Easter Egg found! Skipping entry point") emph( 'New flattened subgraph: %s' % bolds(str(pretty_tree)), DBG_LVL_1) else: entry = self.__entry # use the regular entry point try: # create the simulation object simulation = S.simulate(self.__proj, self.__cfg, cloblks, adj, self.__IR, regmap, varmap, rsvp, entry) except Exception, e: dbg_prnt( DBG_LVL_2, "Cannot create simulation object. Discard current Hk") continue self.__sim_objs = [simulation] self.__terminals = [tree[0][1]] self.__total_path = set() self.__path = set() retn = self.__enum_tree(tree, simulation) # del simulation dbg_prnt( DBG_LVL_2, "Done. Enumeration finished with exit code %s" % bold(retn)) # visualize(self.__cfg.graph, VO_TYPE_CFG, # options=VO_CFG | VO_ACC | VO_CLOB | VO_PATHS, # func=self.__proj.kb.functions[0x41C750], entry=0x41C750, # paths=self.__total_path) # exit() if retn == 0 and self.__consistent_stashes(): self.__nsolutions += 1 self.__options['#solutions'] = self.__nsolutions # # visualise delta graph with Hk # # visualize(DG.graph, VO_TYPE_DELTA, options=VO_PATHS | VO_DRAW_INF_EDGES, # paths=self.__path) # exit() # # visualize CFG again # visualize(self.__cfg.graph, VO_TYPE_CFG, # options=VO_CFG | VO_ACC | VO_CLOB | VO_PATHS, # func=self.__proj.kb.functions[0x444A9D], entry=0x444A9D, # paths=self.__total_path) # exit() print rainbow( textwrap.dedent('''\n\n $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ *** S O L U T I O N F O U N D *** $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ ''')) emph(bolds('Solution #%d' % self.__nsolutions)) emph('Final Trace: %s' % bolds(str(pretty_tree))) output = O.output(self.__options['format']) output.comment('Solution #%d' % self.__nsolutions) output.comment('Mapping #%d' % self.__ctr) output.comment('Registers: %s' % ' | '.join( ['%s <-> %s' % (virt, real) for virt, real in regmap])) output.comment('Variables: %s' % ' | '.join([ '%s <-> %s' % (var, hex(val) if isinstance(val, long) else str(val)) for var, val in varmap ])) output.comment('') output.comment('Simulated Trace: %s' % pretty_tree) output.comment('') output.newline() # cast it to a set to drop duplicates for addr in set(self.__terminals): output.breakpoint(addr) output.newline() output.comment('Entry point') output.set('$pc', '0x%x' % entry) output.newline() # for each active stash, dump all the solutions try: for simulation in self.__simstash: simulation.dump(output, self.__options['noawp']) except Exception, e: dbg_prnt(DBG_LVL_2, "Late exception while dumping: " + str(e)) continue emph(bolds('BOPC is now happy :)')) if self.__options['noawp']: print("NOAWP solution found!") output.save(self.__options['filename']) # save state if self.__options['solutions'] == 'one': for obj in self.__sim_objs: # free memory del obj return -1 # we have a solution. No more mappings for obj in self.__sim_objs: # free memory del obj
from delta import delta f, axarr = plt.subplots(3, 2) print("Problem 7: Ema and delta delay on price series") Nwindow = 500 h_ema_1 = ema(2, Nwindow) h_ema_2 = ema(5, Nwindow) h_ema_3 = ema(10, Nwindow) h_ema_4 = ema(20, Nwindow) h_ema_5 = ema(50, Nwindow) h_ema_6 = ema(100, Nwindow) h_delta_1 = delta(2, Nwindow) h_delta_2 = delta(5, Nwindow) h_delta_3 = delta(10, Nwindow) h_delta_4 = delta(20, Nwindow) h_delta_5 = delta(50, Nwindow) h_delta_6 = delta(100, Nwindow) with open('jpm_trades.csv', 'r') as trades_csv: trades = np.array(list(csv.reader(trades_csv))[1:]).astype('double') prices = trades[:,1] candidate1 = np.convolve(prices-prices[0], h_ema_1) + prices[0] y1 = candidate1[:len(prices)] candidate11 = np.convolve(prices-prices[0], h_delta_1) + prices[0] y11 = candidate11[:len(prices)] axarr[0, 0].plot(y1)