def ffunc(w0, w1, keyyielder): t0 = gfunc(w0, keyyielder) t1 = gfunc(w1, keyyielder) f0 = (t0 + 2 * t1 + c.concat(keyyielder(), keyyielder())) % c.twosquaresixteen f1 = (2 * t0 + t1 + c.concat(keyyielder(), keyyielder())) % c.twosquaresixteen return [f0, f1]
def pool_run(target, argss): procs=[] ques=[] for args in argss: ques.append(multiprocessing.JoinableQueue()) procs.append(multiprocessing.Process(target=target,args=common.concat([[ques[-1]],args]))) for p in procs: p.daemon=True p.start() while True: logger.error("######### pool_run looop begin") for que in ques: logger.error("######### Checking QUEUE") try: while True: logger.error("######### Checking QUEUE while loop") logstd.info("gotque={0}".format(que.get(block=False))) que.task_done() except Queue.Empty as e: pass logger.error("######### sleep BEGIN") time.sleep(1) logger.error("######### sleep END") deadps=[i for (i,p) in enumerate(procs) if not p.is_alive()] logger.error("######### deadps = {0}".format(deadps)) for i in deadps: logger.error("######### joining deadps[{0}]".format(i)) procs[i].join() procs=[p for (i,p) in enumerate(procs) if not i in deadps] if not procs: break logger.error("######### pool_run loop end") return
def compress_web(out_file, input_fnames, level=DEFAULT_LEVEL, verbose=False): """ Compresses files using google's closure web service. It has caveats: no big wads of JS, and it has a quota. """ out_file = out_file or OUT_FILE p('Compressing %s ...' % out_file, verbose) s = StringIO.StringIO() # This is a very ghetto way to not compress the entire thing at once. # "What if I have big files!?" you say. Well, make them smaller. parts = 4 for i in range(0, len(input_fnames)+1, parts): if input_fnames[i:i+parts]: compressed = compress(common.concat(input_fnames[i:i+parts]), COMPILATION_LEVELS[level]) s.write(compressed) s = s.getvalue() p('Compressed to %d' % (len(s)), verbose) out = open(out_file, 'w') out.write(s) out.close() return 0
def gfunc(w, keyyielder): g1 = c.getbyteAt(w, 1) g2 = c.getbyteAt(w, 0) g3 = c.fvalue(g2 ^ keyyielder()) ^ g1 g4 = c.fvalue(g3 ^ keyyielder()) ^ g2 g5 = c.fvalue(g4 ^ keyyielder()) ^ g3 g6 = c.fvalue(g5 ^ keyyielder()) ^ g4 return c.concat(g5, g6)
def parse(rdict, backlog, pipe): cols="" for rawline in backlog: line=rawline.strip() # if not line: cotinue eval=rdict[line] if not eval or "junk" in eval[1].groupdict(): continue if "cols" in eval[1].groupdict(): cols=eval[1].groupdict()["cols"] continue if "vals" in eval[1].groupdict(): yield dict(common.concat([ zip( cols.strip().split(), eval[1].groupdict()["vals"].strip().split() ), [("time", common.gentime() if not eval[1].groupdict()["time"] else eval[1].groupdict()["time"])] ])) # #for line in pipe.stdout: while True: rawline = pipe.stdout.readline() if not rawline: if not pipe.poll(): return else: continue line=rawline.strip() logger.debug("getline=[{0}]".format(line.replace("\n","\\n").replace("\t","\\t"))) # if not line: continue eval=rdict[line] if not eval or "junk" in eval[1].groupdict(): continue if "cols" in eval[1].groupdict(): cols=eval[1].groupdict()["cols"] continue if "vals" in eval[1].groupdict(): yield dict(common.concat([ zip( cols.strip().split(), eval[1].groupdict()["vals"].strip().split() ), [("time", common.gentime() if not eval[1].groupdict()["time"] else eval[1].groupdict()["time"])] ]))
def __encrypt__(p, k, keyyielder): w = c.whitening(p, k) arr = [ c.getbyteAt(w, 3, 2), c.getbyteAt(w, 2, 2), c.getbyteAt(w, 1, 2), c.getbyteAt(w, 0, 2) ] for rnd in range(c.numround): arr = roundfunc(arr, keyyielder) y = c.concat(c.concat(arr[2], arr[3], 2), c.concat(arr[0], arr[1], 2), 4) ci = c.whitening(y, k) return [ c.getbyteAt(ci, 3, 2), c.getbyteAt(ci, 2, 2), c.getbyteAt(ci, 1, 2), c.getbyteAt(ci, 0, 2) ]
def main(opts): if not opts["args"]: sys.exit() pool_run(collector_loop, [( CMDS[arg].format(interval=opts["interval"]) if arg in CMDS else arg, #cmd common.regex_dict(dict(common.concat([ d.items() for d in [pars_vmstat.rdict,pars_sar.rdict,] ]))), "{0}.{1}".format(arg,common.nowstr("%Y%m%dT%H%M%S")).replace(" ","_") #logname ) for arg in opts["args"]])
def compress_local(out_file, input_fnames, type=TYPE_JS, verbose=False, level=None): """ """ out_file = out_file or ('compressed.'+TYPE_JS) p('Compressing %s ...' % out_file, verbose) s = common.concat(input_fnames) s = compress(s, type) out = open(out_file, 'w') out.write(s) out.close()
def _expand_tuple(left, tuple_): from pipes import Worker from common import concat def find_left_end(iter): while hasattr(iter, 'left'): iter = iter.left return iter iters = [] for it in tuple_: leftmost = find_left_end(it) if isinstance(leftmost, Worker): # When we have a worker on the left side, then it is an unbound one # so we automatically bind to the 'left' iters.append(left | concat(it)) elif not hasattr(it, '__iter__'): # If it's not iterable it is a literal value, we repeat iters.append(repeat(it)) else: # We have an iterator we can 'execute' iters.append(it) return iters
def build_lanenet2(input_shape, input_shape1=[128, 128, 3], input_shape2=[64, 64, 3], input_shape3=[32, 32, 3], input_shape4=[16, 16, 3], one_hot_label=False): print(input_shape, 'Input size shape ~') inputs1 = Input((input_shape)) inputs2 = Input((input_shape)) n_filters = 64 inputs_waves1 = Input((input_shape1)) inputs_waves2 = Input((input_shape2)) inputs_waves3 = Input((input_shape3)) inputs_waves4 = Input((input_shape4)) #wavelet1, wavelet2, wavelet3, wavelet4 = C.lanenet_wavelet(inputs) res_1_1 = resnet_block(inputs1, n_filters) skip_1_1 = downBlock(res_1_1) skip_1_1 = C.concat(skip_1_1, inputs_waves1) res_2_1 = resnet_block(inputs2, n_filters) skip_2_1 = downBlock(res_2_1) res_1_2 = resnet_block(skip_1_1, 2 * n_filters, blocks=1) skip_1_2 = downBlock(res_1_2) skip_1_2 = C.concat(skip_1_2, inputs_waves2) res_2_2 = resnet_block(skip_2_1, 2 * n_filters, blocks=1) skip_2_2 = downBlock(res_2_2) res_1_3 = resnet_block(skip_1_2, 4 * n_filters, blocks=1) skip_1_3 = downBlock(res_1_3) skip_1_3 = C.concat(skip_1_3, inputs_waves3) res_2_3 = resnet_block(skip_2_2, 4 * n_filters, blocks=1) skip_2_3 = downBlock(res_2_3) res_1_4 = resnet_block(skip_1_3, 8 * n_filters) skip_1_4 = downBlock(res_1_4) skip_1_4 = C.concat(skip_1_4, inputs_waves4) res_2_4 = resnet_block(skip_2_3, 8 * n_filters) skip_2_4 = downBlock(res_2_4) res_1_5 = resnet_block(skip_1_4, 8 * n_filters) skip_1_5 = downBlock(res_1_5) # skip_1_4 = C.concat(skip_1_4, inputs_waves4) res_2_5 = resnet_block(skip_2_4, 8 * n_filters) skip_2_5 = downBlock(res_2_5) bridge_merge = add(skip_1_5, skip_2_5) bridge = resnet_block(bridge_merge, 16 * n_filters) # Reference Architecture up_4 = upBlock(bridge, 8 * n_filters) merge_4 = concat(up_4, skip_1_4, skip_2_4, third=True) dec_4 = resnet_block(merge_4, 8 * n_filters) up_3 = upBlock(dec_4, 4 * n_filters) merge_3 = concat(up_3, skip_1_3, skip_2_3, third=True) dec_3 = resnet_block(merge_3, 4 * n_filters) up_2 = upBlock(dec_3, 2 * n_filters) merge_2 = concat(up_2, skip_1_2, skip_2_2, third=True) dec_2 = resnet_block(merge_2, 2 * n_filters) up_1 = upBlock(dec_2, n_filters) merge_1 = concat(up_1, skip_1_1, skip_2_1, third=True) dec_1 = resnet_block(merge_1, n_filters) # dec_1 = add(res_1_1, res_2_1, dec_1) up_final = upBlock(dec_1, n_filters) net = up_final #merge1 = C.conv_block(merge1, n_filters) if one_hot_label: net = Conv2D(2, 1, 1, activation='relu', border_mode='same')(net) net = Reshape((2, input_shape[0] * input_shape[1]))(net) net = Permute((2, 1))(net) net = Activation('softmax')(net) # model = Model(inputs=[inputs1, inputs2, inputs_waves1, inputs_waves2, inputs_waves3, inputs_waves4], outputs=net) else: net = Conv2D(1, kernel_size=1, strides=1, activation='sigmoid', padding='same')(net) model = Model(inputs=[ inputs1, inputs2, inputs_waves1, inputs_waves2, inputs_waves3, inputs_waves4 ], outputs=net) # model = Model(inputs=[inputs1, inputs2], outputs=net) return model
continue if "vals" in eval[1].groupdict(): yield dict(common.concat([ zip( cols.strip().split(), eval[1].groupdict()["vals"].strip().split() ), [("time", common.gentime() if not eval[1].groupdict()["time"] else eval[1].groupdict()["time"])] ])) # rdict={ # Linux 2.6.32-431.17.1.el6.x86_64 (jpn-zaq50) 04/17/15 _x86_64_ (4 CPU) #15:27:24 CPU %usr %nice %sys %iowait %steal %irq %soft %guest %idle #15:27:26 all 36.56 0.00 24.25 0.00 0.00 0.00 0.00 1.13 38.07 "^\s*(?P<uname>Linux\s+\S+)\s+(?P<host>\S+)\s+(?P<date>\S+)\s+\S+\s+\((?P<numcpu>\d+)\s+CPU\)\s*$": (parse,__name__), "^(?P<time>\d{2}:\d{2}:\d{2}) (?P<noon>AM|PM){0,1}\s+(?P<cols>(\S*[^\s0-9\.\-\+]\S*\s+){0,}\S*[^\s0-9\.\-\+]\S*)\s*$": (parse,__name__), "^(?P<time>\d{2}:\d{2}:\d{2}) (?P<noon>AM|PM){0,1}\s+(?P<vals>(\S+\s+){0,}[0-9\.\-\+]+)\s*$": (parse,__name__), "^(?P<junk>Average:.*)\s*$": (parse,__name__), } if __name__=="__main__": import json import subprocess from logging import basicConfig basicConfig(level=10) rdict = common.regex_dict(dict(common.concat([ d.items() for d in [rdict,] ]))) c=subprocess.Popen("sar -A 2 2", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) gen=parse(rdict, [], c) for dat in gen: print json.dumps(dat)
def build_lanenet_att(input_shape, input_shape1=[128, 128, 3], input_shape2=[64, 64, 3], input_shape3=[32, 32, 3], input_shape4=[16, 16, 3], one_hot_label=False): print(input_shape, 'Input size shape ~') inputs = Input((input_shape)) n_filters = 64 inputs_waves1 = Input((input_shape1)) inputs_waves2 = Input((input_shape2)) inputs_waves3 = Input((input_shape3)) inputs_waves4 = Input((input_shape4)) #wavelet1, wavelet2, wavelet3, wavelet4 = C.lanenet_wavelet(inputs) enc1 = C.encoder_block(inputs, n_filters, blocks=2) skip1 = C.pool(enc1) _skip1 = C.concat(skip1, inputs_waves1) enc2 = C.encoder_block(_skip1, 2 * n_filters, blocks=2) skip2 = C.pool(enc2) _skip2 = C.concat(skip2, inputs_waves2) enc3 = C.encoder_block(_skip2, 4 * n_filters) skip3 = C.pool(enc3) _skip3 = C.concat(skip3, inputs_waves3) enc4 = C.encoder_block(_skip3, 8 * n_filters) skip4 = C.pool(enc4) _skip4 = C.concat(skip4, inputs_waves4) enc5 = C.encoder_block(_skip4, 8 * n_filters) skip5 = C.pool(enc5) bridge = C.encoder_block(skip5, 32 * n_filters, blocks=2) #bridge = C.encoder_block(bridge, 16 * n_filters) up4 = C.desconv(bridge, 8 * n_filters) att4 = attention(up4, skip4, 8 * n_filters) merge4 = C.add(up4, att4) merge4 = C.conv_block(merge4, 8 * n_filters, 1) up3 = C.desconv(merge4, 4 * n_filters) att3 = attention(up3, skip3, 4 * n_filters) merge3 = C.add(up3, att3) merge3 = C.conv_block(merge3, 4 * n_filters, 1) up2 = C.desconv(merge3, 2 * n_filters) att2 = attention(up2, skip2, 2 * n_filters) merge2 = C.add(up2, att2) merge2 = C.conv_block(merge2, 2 * n_filters, 1) up1 = C.desconv(merge2, n_filters) att1 = attention(up1, skip1, n_filters) net = C.add(up1, att1) net = C.conv_block(net, n_filters, 1) net = C.desconv(net, 1) #merge1 = C.conv_block(merge1, n_filters) if one_hot_label: net = Conv2D(2, 1, 1, activation='relu', border_mode='same')(net) net = Reshape((2, input_shape[0] * input_shape[1]))(net) net = Permute((2, 1))(net) net = Activation('softmax')(net) else: net = Conv2D(1, 1, 1, activation='sigmoid')(net) model = Model(inputs=[ inputs, inputs_waves1, inputs_waves2, inputs_waves3, inputs_waves4 ], outputs=net) return model