def ConfusionMatrix(logits, labels, dataset, filename, text_file): C = np.zeros((len(dataset.tacticName), len(dataset.tacticName))) CM = np.zeros((len(dataset.tacticName) + 1, len(dataset.tacticName) + 1)) ''' convert tactic from network order to dataset order ''' reorder = np.concatenate(dataset.C5k_CLASS).ravel().tolist() orderTactic = [dataset.tacticName[i] for i in reorder] #orderTactic = ['F23','EV','HK','PD','RB','WS','SP','WW','PT','WV'] tacticNum = len(orderTactic) for bagIdx in range(len(labels)): gt = np.argmax(labels[bagIdx]) pred = np.argmax(logits[bagIdx]) #pred = logits[bagIdx] C[gt, pred] = C[gt, pred] + 1 gtC = np.sum(C, axis=1) predC = np.sum(C, axis=0) TP = np.diag(C) precision = TP / predC recall = TP / gtC CM[0:tacticNum, 0:tacticNum] = C CM[tacticNum, 0:tacticNum] = precision CM[0:tacticNum, tacticNum] = recall.T CM[tacticNum, tacticNum] = np.sum(TP) / np.sum(C) * 100 rowIdx = orderTactic + ['Precision'] colIdx = orderTactic + ['Recall'] df = pd.DataFrame(CM, index=rowIdx, columns=colIdx) utils.printLog(text_file, df.round(3).to_string()) #print(df.round(3)) #text_file.write(df.round(3).to_string()) #print(C) #text_file.write(np.array2string(C)) if filename is not None: df.round(3).to_csv(filename, na_rep='NaN')
def run(self): utils.printLog('SERVER', ('listening on port %s' % self.port)) addr = usocket.getaddrinfo('0.0.0.0', self.port)[0][-1] socket = usocket.socket() socket.bind(addr) socket.listen(1) while True: cl, addr = socket.accept() cl.settimeout(2.5) self.process(cl, addr)
def runBlast(fastaFile, blastFile): """Run (multithreaded when possible) BLAST on the fastaFile and output the result with option 6 format as the blastFile""" i0 = time.clock() nbCPU = multiprocessing.cpu_count() n = nbCPU - 2 tag = random.randint(100000, 1000000) dbFile = fastaFile + "_" + str(tag) + ".db" dbCmd = """makeblastdb -in %s -input_type "fasta" -dbtype prot -out %s -hash_index""" % ( fastaFile, dbFile) proc1 = Popen(args=[dbCmd], shell=True, executable="/bin/bash") proc1.communicate() blastCmd = """%s -i %s -out %s -db %s -th %d -evalue 1e-5""" % ( blastAll, fastaFile, blastFile, dbFile, n) try: BA.main(inFile=fastaFile, out=blastFile, db=dbFile, th=n) except IOError as e: printLog("Error in %s: %s\nExiting." % (blastCmd, e), handle) return () cleanCmd = """rm %s*""" % dbFile proc3 = Popen(args=[cleanCmd], shell=True, executable="/bin/bash") proc3.communicate() myTimer(i0, "Completed runBlast")
def runDescription(annotFile, radical=None, ID=None, keyList=None, handle=sys.stderr, config=None): """Compute the result of Description.py on the trailFile hierarchy with basis the annotFile.""" if not ID: ID = "UniqID" if keyList: keyString = ",".join(keyList) descFile = radical + ".desc" xmlFile = radical + ".xml_desc" configFile = radical + ".config" trailFile = radical + ".trail" edgeFile = radical + ".edges" compFile = radical + ".twin_comp" #cmd8 = """%s -i %s -o %s -X %s -a -D -c %s -k %s -H %s %s %s""" % (Description,ID,descFile,configFile,compFile,keyString,trailFile,edgeFile,annotFile) cmd8 = """%s -i %s -o %s -X %s -a -D -c %s -H %s %s %s""" % ( Description, ID, descFile, configFile, compFile, trailFile, edgeFile, annotFile) printLog( "--------------------------------------------------\nConfiguring %s" % cmd8, handle) try: if config: D.Main(edgeFile, annotFile, nodeID=ID, outFile=descFile, X=configFile, restrAnnot=True, display=False, comp=compFile, hist=trailFile, keyList=keyString, log=handle) else: D.Main(edgeFile, annotFile, nodeID=ID, outFile=descFile, X=configFile, restrAnnot=True, display=True, comp=compFile, hist=trailFile, keyList=keyString, log=handle) except IOError as e: printLog("Error in %s: %s\nExiting." % (cmd8, e), handle) return () #time.sleep(15) if config: xmlform.main(xmlFile=configFile) cmd8bis = """%s -i %s -o %s -O %s -x %s -a -H %s %s %s""" % ( Description, ID, descFile, xmlFile, configFile, trailFile, edgeFile, annotFile) printLog( "--------------------------------------------------\nRunning %s" % cmd8bis, handle) try: D.Main(edgeFile, annotFile, nodeID=ID, outFile=descFile, Xout=xmlFile, x=configFile, restrAnnot=True, display=True, hist=trailFile, keyList=keyList, log=handle) except IOError as e: printLog("Error in %s: %s\nExiting." % (cmd8bis, e), handle) return ()
import machine import sonoff_server_controller import server import status_server_controller import utils import network utils.printLog("NODEMCU", "sonoff") statusPin = machine.Pin(13, machine.Pin.OUT) def timeout1seconds(): isconnected = network.WLAN(network.STA_IF).isconnected() statusPin.value(1 if not isconnected else 0) def timeout10minutes(): utils.syncDatetime() tim0 = machine.Timer(0) tim0.init(period=1000, mode=machine.Timer.PERIODIC, callback=lambda t: timeout1seconds()) tim1 = machine.Timer(1) tim1.init(period=600000, mode=machine.Timer.PERIODIC, callback=lambda t: timeout10minutes()) timeout10minutes() sonoffServerController = sonoff_server_controller.getInstance() controllers = [sonoffServerController] statusController = status_server_controller.StatusServerController('Sonoff', controllers) _server = server.Server(33455, controllers + [statusController]) _server.run()
import config import temperature_sensor import display import machine import server import status_server_controller import utils import utime utils.printLog("NODEMCU", "thermometer boot up") i2c = machine.I2C(scl=machine.Pin(config.D2), sda=machine.Pin(config.D1), freq=400000) _temperature_sensor = temperature_sensor.TemperatureSensor( machine.Pin(config.D3)) _display = display.Display(i2c, _temperature_sensor) def timeout1second(timer): _temperature_sensor.update() hour = utime.localtime(utime.time())[3] _display.setBacklight(7 <= hour and hour <= 22) _display.update() def timeout1minute(timer): _temperature_sensor.upload()
import config import esp import machine import utils import ubinascii utils.printLog('ANIMATOR', 'fast set leds') _config = utils.readJson('animator.data') if _config and _config['powered_on'] and _config['use_color']: pin = machine.Pin(config.D4, machine.Pin.OUT) bytes = ubinascii.unhexlify(_config['color']) color = [bytes[1], bytes[0], bytes[2]] esp.neopixel_write(pin, bytearray(color * _config['leds']), 1) utils.printLog('ANIMATOR', 'finish')
if __name__ == '__main__': MethodString = """Clustering method (-m METHOD): 'fg': community_fastgreedy(self, weights=None) \tCommunity structure based on the greedy optimization of modularity. 'im': community_infomap(self, edge_weights=None, vertex_weights=None, trials=10)F \tFinds the community structure of the network according to the Infomap method of Martin Rosvall and Carl T. 'le': community_leading_eigenvector(clusters=None, weights=None, arpack_options=None) \tNewman's leading eigenvector method for detecting community structure. 'lp': community_label_propagation(weights=None, initial=None, fixed=None) \tFinds the community structure of the graph according to the label propagation method of Raghavan et al. 'ml': community_multilevel(self, weights=None, return_levels=False) \tCommunity structure based on the multilevel algorithm of Blondel et al. 'om': community_optimal_modularity(self, *args, **kwds) \tCalculates the optimal modularity score of the graph and the corresponding community structure. 'eb': community_edge_betweenness(self, clusters=None, directed=True, weights=None) \tCommunity structure based on the betweenness of the edges in the network. 'sg': community_spinglass(weights=None, spins=25, parupdate=False, start_temp=1, stop_temp=0.01, cool_fact=0.99, update_rule='config', gamma=1, implementation='orig', lambda_=1) \tFinds the community structure of the graph according to the spinglass community detection method of Reichardt & Bornholdt. 'wt': community_walktrap(self, weights=None, steps=4) \tCommunity detection algorithm of Latapy & Pons, based on random walks. """ prog = sys.argv[0].split("/")[-1] parser = processArgs() args = parser.parse_args() header = " ".join(sys.argv) printLog(header,args.l,mode="w") print(vars(args)) Main(edgeFile=args.edgeFile,outFile=args.o,method=args.m,log=args.l)
def Main(blastFile=None, genome2sequence=None, sep=None, thr=None, cov=None, in_network=None, fasta=None, aln=None, clust=None, annot=None, key=None, keyList=None, log=None, directory=None, config=None): """ Main program """ ### try: startWD = os.path.abspath(os.path.dirname(blastFile)) except: startWD = os.path.abspath(os.getcwd()) os.chdir(startWD) if directory: rootDir = os.path.abspath(directory) if not os.path.exists(rootDir): os.makedirs(rootDir) else: rootDir = os.getcwd() if log != sys.stderr: log = os.path.join(rootDir, log) ### Argument processing ============================================================================================================= if not blastFile or not genome2sequence: sys.exit("Required files %s and %s" % ("blastFile", "genome2sequence")) blastFile = os.path.abspath(blastFile) genome2sequence = os.path.abspath(genome2sequence) ThresholdList = list(map(int, thr.strip().split(","))) cover = float(cov) print("Starting directory: %s" % startWD) print("Root directory: %s" % rootDir) if fasta: if aln == "b": runBlast(fasta, blastFile) elif aln == "d": runDiamond(fasta, blastFile) else: sys.exit( "Wrong sequence comparison option -- use (b) for BLAST - (d) for DIAMOND" ) UniqID = key ## Filename definitions ============================================================================================================= if in_network: geneNetwork = os.path.abspath(in_network) else: geneNetwork = blastFile + ".cleanNetwork" if annot: annot = os.path.abspath(os.path.join(startWD, annot)) if keyList: keyList = keyList.split(",") else: with open(annot, 'r') as ANNOT: keyList = ANNOT.readline().strip().split(sep)[1:] else: annot = None keyList = None ## Corps du programme =========================================== inext = time.clock() os.chdir(rootDir) ## A) from the blast output to the sequence families # a) filter self-hits and keep only best hit if not in_network: cmd1 = "%s -n 1 -i %s" % ( cleanblast, blastFile ) # the output are three files named blastFile".cleanNetwork", blastFile".cleanNetwork.dico" and blastFile".cleanNetwork.genes" printLog( "--------------------------------------------------\nRunning %s" % cmd1, log) proc1 = Popen(args=[cmd1], shell=True, stdout=PIPE, executable="/bin/bash") out = proc1.communicate()[0] printLog(out.decode('utf-8'), log) # b) perform complete analysis for each threshold for n in ThresholdList: STR = """--------------------------------------------------\nSimilarity threshold %d%%""" % n printLog(STR, log) completeAnalysis(geneNetwork, genome2sequence, n, cover, a=annot, clustType=clust, UniqID=key, sep=sep, keyList=keyList, handle=log, config=config) os.chdir(rootDir) ## Fin ====================================================== prog = myModule() if prog == "__main__.py": prog = sys.argv[0].split("/")[-1] ## Sortie ====================================================== return ()
def keyPlayerResult(Y_pred, Y_label, y_pred, dataset, phase, text_file, log_file, info_file): if phase == 'train': selectIndex = dataset.trainIdx else: selectIndex = dataset.testIdx NAME = [] for i in range(len(selectIndex)): hit = [(selectIndex[i] in dataset.videoIndex[r]) for r in range(len(dataset.videoIndex))] loc = hit.index(True) tacticName = dataset.tacticName[loc] string = tacticName + "-" + selectIndex[i].astype(str) + ":" NAME.append(string) #nv_reorder = [0,1,2,3,8,4,6,5,9,5,7] #[[0,1,2,3,5,7],[6,9],[4,8]] reorder = np.concatenate(dataset.C5k_CLASS).ravel().tolist() inv_reorder = [reorder.index(i) for i in range(len(dataset.tacticName))] PRED_NAME = [] for i in range(len(selectIndex)): pred_loc = Y_pred[i].tolist().index(1) tacticName = dataset.tacticName[inv_reorder[pred_loc]] PRED_NAME.append(tacticName) rolePlayerMap = dataset.gtRoleOrder[selectIndex] Y_correct_map = Y_pred * Y_label Y_correct = 2 * np.sum(Y_correct_map, axis=-1, keepdims=True) - 1 #correct=1, error=-1 NUM_PLAYER = rolePlayerMap.shape[1] keyPlayers = y_pred * (rolePlayerMap + 1) * np.tile( Y_correct, [1, NUM_PLAYER]) DAT = np.column_stack((PRED_NAME, keyPlayers.astype(int))) df = pd.DataFrame(DAT, index=NAME, columns=['Y_pred', 'p1', 'p2', 'p3', 'p4', 'p5']) if text_file is not None: #np.savetxt(text_file, DAT, delimiter=" ", fmt="%s") df.to_csv(text_file, na_rep='NaN') #np.savetxt(text_file,keyPlayers,fmt='%d', delimiter=' ') #with open(text_file,'w') as file: #file.write(np.array2string(keyPlayers)) rolePlayerAccumMap = np.zeros((len(dataset.tacticName), dataset.numPlayer)) for i in range(len(selectIndex)): hit = [(selectIndex[i] in dataset.videoIndex[r]) for r in range(len(dataset.videoIndex))] loc = hit.index(True) for p in range(dataset.numPlayer): if keyPlayers[i][p] > 0: roleIndex = int(keyPlayers[i][p]) - 1 rolePlayerAccumMap[ loc, roleIndex] = rolePlayerAccumMap[loc, roleIndex] + 1 num_k = np.zeros(len(dataset.tacticName), dtype=np.int8) k = 0 boundary = len(dataset.C5k_CLASS[0]) for idx in range(len(dataset.tacticName)): if idx >= boundary: k = k + 1 boundary = boundary + len(dataset.C5k_CLASS[k]) num_k[idx] = dataset.k[k] reorderMapTemp = rolePlayerAccumMap[reorder] reorderMapSum = np.sum(reorderMapTemp, axis=1, keepdims=True) num_vid = reorderMapSum / np.expand_dims(num_k, axis=1) reorderMap = np.concatenate((reorderMapTemp, num_vid), axis=1) orderTactic = [dataset.tacticName[i] for i in reorder] orderTacticInfo = [ orderTactic[t] + '(' + num_k[t].astype(str) + ')' for t in range(len(dataset.tacticName)) ] role = ['r1', 'r2', 'r3', 'r4', 'r5', 'num_vid'] df = pd.DataFrame(reorderMap, index=orderTacticInfo, columns=role).astype(int) #print(df) utils.printLog(log_file, " ") utils.printLog(log_file, df.to_string()) if info_file is not None: df.to_csv(info_file, na_rep='NaN')
import animator_server_controller import config import machine import server import status_server_controller import utils utils.printLog("NODEMCU", "animator boot up") animatorServerController = animator_server_controller.AnimatorServerController( machine.Pin(config.D4, machine.Pin.OUT)) statusController = status_server_controller.StatusServerController( 'Animator', [animatorServerController]) def timeoutTick(timer): animatorServerController.tick() def timeout10minutes(timer): utils.syncDatetime() timeout10minutes(None) tim1 = machine.Timer(0) tim1.init(period=1, mode=machine.Timer.PERIODIC, callback=timeoutTick) tim3 = machine.Timer(2) tim3.init(period=600000, mode=machine.Timer.PERIODIC, callback=timeout10minutes)
import config import machine import utils import vl53l0x utils.printLog("NODEMCU", "radar boot up") i2c = machine.I2C(scl=machine.Pin(config.D2), sda=machine.Pin(config.D1), freq=400000) distanceSensor = vl53l0x.VL53L0X(i2c, 0x29) def timeout100ms(timer): utils.printLog("RADAR", distanceSensor.read()) tim1 = machine.Timer(0) tim1.init(period=100, mode=machine.Timer.PERIODIC, callback=timeout100ms)
def timeout100ms(timer): utils.printLog("RADAR", distanceSensor.read())
def __init__(self, port): self.socket = socket.socket() self.port = int(port) self.socket.bind(('', self.port)) self.socket.listen(5) printLog('Starting server at {}, Ctrl+C to stop'.format('')) self.channels = {} self.clients = {} self.SOCKET_LIST = [self.socket] try: while True: reading, writing, exceptioning = select.select( self.SOCKET_LIST, [], []) # omit timeout to prevent high CPU utilization for sock in reading: if sock == self.socket: # accept connections from outside # new socket is created for server to communicate with client # this frees up server to listen for more connections (new_socket, address) = self.socket.accept() self.SOCKET_LIST.append(new_socket) self.clients[new_socket.fileno()] = { 'name': '', 'channel': '', 'buffer': [] } else: try: data = sock.recv(MESSAGE_LENGTH).decode() if data: msg_length = len(data) output_str = None if self.clients[sock.fileno()].get( 'buffer', []): cached_msg = self.clients[ sock.fileno()].get('buffer', []).pop() cached_len = len(cached_msg) if cached_len + msg_length > MESSAGE_LENGTH: output_str = cached_msg + data[:MESSAGE_LENGTH - cached_len] self.clients[sock.fileno()].get( 'buffer', []).append(data[MESSAGE_LENGTH - cached_len:]) elif cached_len + msg_length == MESSAGE_LENGTH: output_str = cached_msg + data else: self.clients[sock.fileno()].get( 'buffer', []).append(cached_msg + data) else: if msg_length < MESSAGE_LENGTH: self.clients[sock.fileno()].get( 'buffer', []).append(data) else: output_str = data if output_str: output_str = output_str.rstrip() if not self.clients[sock.fileno()].get( 'name', ''): self.clients[ sock.fileno()]['name'] = output_str else: name = self.clients.get( sock.fileno(), {}).get('name') msg_lst = output_str.split(' ') # print(msg_lst) if msg_lst[0].startswith('/'): if msg_lst[0].startswith('/join'): self.join_channel( msg_lst, sock) elif msg_lst[0].startswith( '/create'): self.create_channel( msg_lst, sock) elif msg_lst[0].startswith( '/list'): self.list_channel(sock) else: self.send( SERVER_INVALID_CONTROL_MESSAGE .format(output_str), sock) else: channel = self.clients.get( sock.fileno(), {}).get('channel') if channel: self.broadcast( "[{}] {}".format( name, output_str), sock, self.channels.get( channel, []), channel) else: self.send( SERVER_CLIENT_NOT_IN_CHANNEL, sock) else: self.remove_socket(sock) except Exception as e: self.remove_socket(sock) printError(e) self.socket.close() except KeyboardInterrupt: printLog("Caught interrupt, stopping server", True) # self.socket.shutdown(socket.SHUT_RDWR) self.socket.close() sys.exit()
import config import temperature_sensor import machine import pin_scheduler import server import status_server_controller import utils utils.printLog("NODEMCU", "water can boot up") _temperature_sensor = temperature_sensor.TemperatureSensor() pinScheduler = pin_scheduler.PinScheduler( machine.Pin(config.D5, machine.Pin.OUT), [((19, 0, 0), 36)]) statusController = status_server_controller.StatusServerController( 'Pin Scheduler', []) _server = server.Server(config.SERVER_PORT, [statusController]) def timeout1second(timer): _temperature_sensor.update() pinScheduler.update() def timeout1minute(timer): if not pinScheduler.isTimeNearScheduler(): _temperature_sensor.upload() def timeout10minutes(timer): pass
def Main(edgeFile=None,annotFile=None,sep=None,outFile=None,Xout=None,restrAnnot=None,nodeList=None,NodeType=None,nodeID=None,unilat=None,track=None,empty=None,\ x=None,X=None,K=None,hist=None,display=None,trail=None,comp=None,keyList=None,log=None): """ Main program """ ### Argument/options listing startWD = os.getcwd() if log != sys.stderr: try: log = os.path.join(startWD, log) except TypeError: log = sys.stderr ### Argument processing ======================================== ## Filename definitions ====================================== i0 = time.clock() inext = i0 if not outFile: inRad = edgeFile.split(".")[0] outFile = inRad + ".desc" if empty: track = None ## File reading options ======================================== # Step 1) Store the node type (top(1)/bottom(2) in the bipartite graph), adapted for the k-partite case if not os.stat(edgeFile).st_size: if myModule() == "__main__.py": sys.exit("Error: Empty file %s" % edgeFile) else: raise IOError("Empty file %s" % edgeFile) nodeType = ut.readNodeType( edgeFile, Type=NodeType ) # this step is not REALLY necessary, in the sense that only the values of the nodeType file are used here try: nodeTypes = list(set(nodeType.values( ))) # likely an overkill, but does not seem to be time-consuming except AttributeError: # this is for the unipartite case (or is it?) nodeTypes = [1] inext = myTimer(inext, "Reading nodeType", handle=log) ## Step 2) Read XML configuration file or generate and exit. ++++++++++++++++++++++++++++++++++++++++++++++++++ # a) set variables. if keyList: keyDict = ut.processOptions(keyList, nodeTypes) else: selectedKeys = list(ut.getHeader(annotFile).keys()) selectedKeys.remove(nodeID) keyDict = dict() for n in nodeTypes: keyDict[n] = selectedKeys trailObjects = [] compObject = None root = os.getcwd() if comp: compObject = ut.myMod(fileName=comp, attDict={0: "Module"}) if hist: # added option to generate complete trailHistory in the XML file : options.H is the main trailFile (from rootDir to cwDir) history = ut.trailTrack(hist) root = ut.trailHist(hist)['root'] k = 1 for trailName in history: trailKeyDict = dict([(i, "NodeType" + str(i)) for i in nodeTypes]) Trail = ut.myTrail(fileName=trailName, rank=k, attDict=trailKeyDict) trailObjects.append(Trail) k += 1 if x: # this option gives the name of the config file, and proceeds with the description procedure configFile = x if X: # options.X is the name of the configurationFile that will be generated (default = "config.xml") if x == X: configFile = ut.generateXML(nodeTypes, trailObjects=trailObjects, compObject=compObject, attDict=keyDict, outFile=X, display=display, root=root) else: sys.exit("Conflicting fields -x and -X. Check and run again.") if K: ret = xmlform.main(xmlFile=configFile) if ret == "Cancel": sys.exit(0) trailObjects, compObject, keyDict, selectedKeys, XML = ut.readConfigFile( configFile) ut.printDescription(trailObjects, compObject, keyDict, selectedKeys, handle=sys.stderr) else: # this block will generate the config file and stop: we start with this part. if X: # options.X is the name of the configurationFile that will be generated (default = "config.xml") outConf = X else: outConf = "config.xml" ## selectedKeys are obtained as header of the annotFile configFile = ut.generateXML(nodeTypes, trailObjects=trailObjects, compObject=compObject, attDict=keyDict, outFile=outConf, display=display, root=root) #configFile = generateXML(nodeTypes,trailObjects=trailObjects,compObject=compObject,attDict=keyDict,outFile=X,display=display) if myModule() == "__main__.py": printLog( "Configured file %s: please check options, and pass it with -x option" % outConf, log) return () ## Step 3) Define nodeLists of currentID and UniqID. +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ if NodeType == '2': nodes = nodeType.keys() elif NodeType == '1': nodes = ut.readNodes(edgeFile, sep=sep) nodeType = ut.initDict(nodes, value=1) else: nodes = ut.readNodes(edgeFile, sep=sep) if nodeList: # if we explicitly give a file with the currentID to restrict to. nodeFile = options.n nodes = ut.file2set( nodeFile) # nodes is actually a list (but without repetitions)! inext = myTimer(inext, "Reading nodeFile", handle=log) if unilat: nTypes = set(unilat.strip().split(",")) nodes = (node for node in nodes if nodeType[node] in nTypes) printLog("""Loaded %d nodes""" % len(nodes), log) # Selected UniqIDs: ======== if trailObjects: trailObjects[-1].getDict( ) # here the dictionaries of the main trail file are loaded. current2UniqID = trailObjects[-1].dict_inv myEntries = ut.unList(map(lambda x: current2UniqID[x], nodes)) else: myEntries = nodes current2UniqID = None printLog("""Found %d entries""" % len(myEntries), log) inext = myTimer(inext, "Reading allEntries", handle=log) # Annotation file processing: ========== if restrAnnot: annotationDict, fields = ut.restrictAnnot(annotFile, mainKey=str(nodeID), valueKeyList=selectedKeys) else: annotationDict, fields = ut.myLoadAnnotations( annotFile, mainKey=str(nodeID), valueKeyList=selectedKeys, counter=0) inext = myTimer(inext, "Reading annotations", handle=log) ## Step 4) Construct the actual description. +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ OutFile = Xout ut.xmlDescription(annot=annotationDict, nodeDict=current2UniqID, entries=myEntries, compObject=compObject, trails=trailObjects, nodeType=nodeType, keyDict=keyDict, xmlOutFile=OutFile, outFile=outFile, track=track, X=XML, handle=log) if Xout: printLog("XML output written to %s" % OutFile, log) else: printLog("Description written to %s" % outFile, log) ## Output and exit ====================================================== prog = myModule() if prog == "__main__.py": prog = sys.argv[0].split("/")[-1] inext = myTimer(i0, "Total computing time for %s" % prog, handle=log) return
xmlOutFile=OutFile, outFile=outFile, track=track, X=XML, handle=log) if Xout: printLog("XML output written to %s" % OutFile, log) else: printLog("Description written to %s" % outFile, log) ## Output and exit ====================================================== prog = myModule() if prog == "__main__.py": prog = sys.argv[0].split("/")[-1] inext = myTimer(i0, "Total computing time for %s" % prog, handle=log) return #========= Main program if __name__ == '__main__': prog = sys.argv[0].split("/")[-1] parser = processArgs() args = parser.parse_args() CMD = " ".join(sys.argv) printLog(CMD, args.l, mode="w") print(vars(args)) if not args.G: Main(edgeFile=args.i,annotFile=args.a,sep=args.s,outFile=args.o,Xout=args.O,restrAnnot=args.A,nodeList=args.n,NodeType=args.N,\ nodeID=args.I,unilat=args.u,track=args.T,empty=args.E,x=args.x,X=args.X,K=args.K,hist=args.H,display=args.D,trail=args.t,comp=args.c,keyList=args.k,log=args.l) else: ds_launcher.main(prog, args)
def Main(edgeFile=None,outFile=None,sep=None,unilat=None,twin_supp=None,Twin_Supp=None,min_supp=None,min_size=None,nodeType=None,comp=None,debug=None,log=None): """ Main program """ i0 = time.clock() inext = i0 ## File reading options ======================================== if not outFile: outFile = edgeFile+".twins" thr = min_supp try: k_part = int(nodeType) except (TypeError,ValueError): k_part = nodeType ## File reading ======================================== if not os.stat(edgeFile).st_size: if myModule() == "__main__.py": sys.exit("Error: Empty file %s" % edgeFile) else: raise IOError("Empty file %s" % edgeFile) g = ut.myReadGraph(edgeFile) print(g.summary()) id2name = {} name2id = {} for n in g.vs(): name = n['name'] ind = n.index id2name[ind] = name name2id[name] = ind inext = myTimer(i0,"Loading graph",handle=log) ## Program body =========================================== # Adjacency list computation ------------------------------ getName = lambda x:id2name[x] nodes = None if unilat: typeSet = set(map(lambda x:int(x),unilat.strip().split(","))) typeDict = defaultdict(int) if k_part == 2 or not k_part: typeDict.update(ut.rawNodeType(edgeFile)) elif k_part != 1: typeDict.update(ut.loadNodeType(k_part)) nodes = (n.index for n in g.vs() if typeDict[n['name']] in typeSet) ADJ = ut.getAdjlist(g,nodes=nodes) inext = myTimer(inext,"Computation of adjacency list",handle=log) # Twin computation ---------------------------------------- support,twins = ut.detectRepeated(ADJ,k_init=0,debug=debug) # support: groupID -> common_list_of_neighbours; twins: node -> groupID_of_its_twin_class inext = myTimer(inext,"Computation of twins",handle=log) new_support = dict([(gid,tuple(map(getName,support[gid]))) for gid in support]) new_twins = dict([(id2name[node],twins[node]) for node in twins]) support = new_support twins = new_twins inext = myTimer(inext,"Renumbering of twins",handle=log) sniwt = ut.InvertMap(twins) # groupID -> list_of_twin_nodes inext = myTimer(inext,"Computation of twin support",handle=log) # Computation of components (twins + support) if comp: with open(comp,'w') as h: for key,val in iter(twins.items()): outString = str(key)+sep+str(val)+"\n" h.write(outString) inext = myTimer(inext,"Writing twins file",handle=log) for val,nodes in iter(support.items()): for node in nodes: outString = str(node)+sep+str(val)+"\n" h.write(outString) inext = myTimer(inext,"Writing twins component file",handle=log) # Computation of twinSupport (twinID twinNb twinSupport) if twin_supp: with open(twin_supp,'w') as g: for i,nodeList in iter(sniwt.items()): supp = support[i] if len(supp) >= thr and len(nodeList) >= min_size: # Threshold for trivial twins (new option 15/12/15) vals = [str(i),str(len(nodeList)),str(len(supp))] vals.extend(list(map(lambda x:str(x),supp))) g.write("\t".join(vals)+"\n") inext = myTimer(inext,"Writing twins support file",handle=log) # Computation of TwinSupport (twinID twinNodes twinSupport) if Twin_Supp: with open(Twin_Supp,'w') as g: for i,nodeList in iter(sniwt.items()): supp = support[i] if len(supp) >= thr and len(nodeList) >= min_size: # Threshold for trivial twins (new option 15/12/15) myTwins = ",".join(map(lambda x:str(x),nodeList)) mySupport = ",".join(map(lambda x:str(x),supp)) vals = [str(i)] vals.extend(list(map(lambda x:str(x),supp))) g.write("\t".join(vals)+"\n") inext = myTimer(inext,"Writing Twins Support file",handle=log) ut.outputDict(twins,outFile,sep=sep) allTwins = len(sniwt.keys()) t = len([i for (i,v) in iter(sniwt.items()) if len(v) == 1]) try: tp = 100*float(t)/float(allTwins) except: tp = 0 nt = allTwins - t try: ntp = 100*float(nt)/float(allTwins) except: ntp = 0 printLog("""Found %s twins, %s trivial twins (%.2f%%) and %s non-trivial twins (%.2f%%)""" % (allTwins,t,tp,nt,ntp),log) ## Ending ====================================================== prog = myModule() if prog == "__main__.py": prog = sys.argv[0].split("/")[-1] inext = myTimer(i0,"Total computing time for %s" % prog,handle=log) return
import config import machine import pin_server_controller import server import status_server_controller import temperature_sensor import utils utils.printLog("NODEMCU", "rstrip boot up") _temperature_sensor = temperature_sensor.TemperatureSensor( machine.Pin(config.D6)) def timeout1minute(timer): _temperature_sensor.update() _temperature_sensor.upload() def timeout10minutes(timer): utils.syncDatetime() tim0 = machine.Timer(0) tim0.init(period=60000, mode=machine.Timer.PERIODIC, callback=timeout1minute) tim1 = machine.Timer(1) tim1.init(period=600000, mode=machine.Timer.PERIODIC, callback=lambda t: timeout10minutes()) timeout1minute(None) timeout10minutes(None)
def completeAnalysis(geneNetwork, genome2sequence, n, c, a=None, clustType=None, UniqID=None, sep=None, keyList=None, handle=sys.stderr, config=None): """Perform complete bipartite and twin analysis at a given identity threshold n""" directory = "graphs" + str(n) try: os.mkdir(directory) except OSError: pass # Names and file definitions if clustType == 'cc': seqCompFile = "CC.nodes" # compFile for sequences eFile = "CC.edges" iFile = "CC.info" elif clustType == 'families': seqCompFile = "family.nodes" # compFile for sequences eFile = "family.edges" iFile = "family.info" else: sys.exit("Bad clustering type -- see -C option") edgeFile = "graph.edges" # edgeFile trailFile = "graph.trail" # trailFile geneNetworkDico = geneNetwork + ".dico" geneNetworkGenes = geneNetwork + ".genes" ## ============================== # c) assemble sequence families by computing the connected components cmd2 = """%s -i %s -d %s -n %s -m %s -p %d""" % ( familydetector, geneNetwork, directory, geneNetworkGenes, clustType, n) printLog( "--------------------------------------------------\nRunning %s" % cmd2, handle) proc2 = Popen(args=[cmd2], shell=True, stdout=PIPE, executable="/bin/bash") out = proc2.communicate()[0] printLog(out.decode('utf-8'), handle) mySeqCompFile = os.path.join(directory, seqCompFile) myiFile = os.path.join(directory, iFile) myeFile = os.path.join(directory, eFile) # renumber back families through geneNetworkDico dic1 = ut.loadMapping(geneNetworkDico) dic2 = ut.node2communityFasta(mySeqCompFile, sep=sep) compDict = ut.composeDict(dic1, dic2) ut.outputDict(compDict, mySeqCompFile, sep=sep) cleanCmd = """rm %s %s""" % (myiFile, myeFile) procClean = Popen(args=[cleanCmd], shell=True, executable="/bin/bash") procClean.communicate() ## B) from the sequence families to the bipartite graph # a) Cluster sequence families and quotient the graph cmd3 = """%s -c %s -k %s -d %s %s %s %s""" % ( FactorGraph, mySeqCompFile, UniqID, directory, genome2sequence, edgeFile, trailFile) printLog( "--------------------------------------------------\nRunning %s" % cmd3, handle) FG.Main(edgeFile=genome2sequence, outEdgeFile=edgeFile, outTrailFile=trailFile, direct=directory, community=mySeqCompFile, comm_id=UniqID, sep=sep, log=handle, header=cmd3) os.chdir(directory) printLog( "--------------------------------------------------\ncd %s" % directory, handle) ## rad = "graph0" # b) Remove the degree one nodes from the sequence side edges = rad + ".edges" cmd4 = """%s -d 1 -u 2 %s %s""" % (shaveGraph, edgeFile, edges) printLog( "--------------------------------------------------\nRunning %s" % cmd4, handle) SG.Main(edgeFile=edgeFile, outEdgeFile=edges, degree=1, nodeType=2, sep=sep, log=handle) # d) Compute twins and twin supports of the bipartite graph twins = rad + ".twins" twinComp = rad + ".twin_comp" cmd6 = """%s -o %s -u 2 -c %s %s """ % (DetectTwins, twins, twinComp, edges) printLog( "--------------------------------------------------\nRunning %s" % cmd6, handle) try: DT.Main(edgeFile=edges, outFile=twins, sep=sep, unilat='2', comp=twinComp, log=handle) except IOError as e: printLog("Error in %s: %s\nExiting." % (cmd6, e), handle) return () ## C) from the bipartite graph to the twins and articulation points # a) twin quotienting twinDir = "TwinQuotient" try: os.mkdir(twinDir) except OSError: pass rad = "graph1" newEdges = rad + ".edges" newTrail = rad + ".trail" cmd7 = """%s -c %s -k %s -d %s -t %s %s %s %s""" % ( FactorGraph, twins, UniqID, twinDir, trailFile, edges, newEdges, newTrail) printLog( "--------------------------------------------------\nRunning %s" % cmd7, handle) try: FG.Main(edgeFile=edges, outEdgeFile=newEdges, outTrailFile=newTrail, direct=twinDir, community=twins, comm_id=UniqID, in_trail=trailFile, sep=sep, log=handle, header=cmd7) except IOError as e: printLog("Error in %s: %s\nExiting." % (cmd7, e), handle) return () os.chdir(twinDir) printLog( "--------------------------------------------------\ncd %s" % twinDir, handle) # b) Computing articulation points and biconnected components ART, BIC = getArticulationPoints(newEdges) artPoints = rad + ".art" aP = open(artPoints, "w") printLog( "--------------------------------------------------\nPrinting %d articulation points in %s" % (len(ART), artPoints), handle) for node in ART: outString = """%s\t%s\n""" % (node, ",".join( [str(ID) for ID in BIC[node]])) aP.write(outString) aP.close() bcNb = 0 bicFile = rad + ".bic_comp" bC = open(bicFile, "w") for node in BIC: for ID in BIC[node]: bcNb = max(bcNb, ID) bC.write("""%s\t%d\n""" % (node, ID)) bC.close() printLog( "--------------------------------------------------\nPrinting %d biconnected components in %s" % (bcNb + 1, bicFile), handle) ## D) annotations and twin component analysis if a: edges = rad + ".edges" twins = rad + ".twins" twinComp = rad + ".twin_comp" cmd9 = """%s -o %s -u 2 -c %s %s """ % (DetectTwins, twins, twinComp, edges) printLog( "--------------------------------------------------\nRunning %s" % cmd9, handle) try: DT.Main(edgeFile=edges, outFile=twins, sep=sep, unilat='2', comp=twinComp, log=handle) except IOError as e: printLog("Error in %s: %s\nExiting." % (cmd9, e), handle) return () runDescription(a, radical=rad, ID=UniqID, keyList=keyList, handle=handle, config=config)