def main(): t0 = time.time() print "Preparing Inputs..." pi.prepareInputs() ndsm = io.imread("C:\\bertud_temp\\ndsm.tif") classified = io.imread("C:\\bertud_temp\\classified.tif") classified = classified[0:len(ndsm),0:len(ndsm[0])] slope = io.imread("C:\\bertud_temp\\slope.tif") numret = io.imread("C:\\bertud_temp\\numret.tif") print "Generating Initial Mask..." initialMask = ma.generateInitialMask(ndsm,classified,slope,numret) io.imsave("C:\\bertud_temp\\initialMask.tif",initialMask) pieces = br.performBoundaryRegularizationV2(initialMask,numProcesses=3) finalMask = ma.buildFinalMask(pieces,initialMask) io.imsave("C:\\bertud_temp\\finalMask.tif",finalMask) # pickle.dump(pieces,open("E:/BertudV2/pieces.pickle","wb")) t1 = time.time() print "Finished everything in ",round(t1-t0,2),"s."
def main(): t0 = time.time() print "Preparing Inputs..." pi.prepareInputs() ndsm = io.imread("C:\\bertud_temp\\ndsm.tif") classified = io.imread("C:\\bertud_temp\\classified.tif") classified = classified[0:len(ndsm), 0:len(ndsm[0])] slope = io.imread("C:\\bertud_temp\\slope.tif") numret = io.imread("C:\\bertud_temp\\numret.tif") print "Generating Initial Mask..." initialMask = ma.generateInitialMask(ndsm, classified, slope, numret) io.imsave("C:\\bertud_temp\\initialMask.tif", initialMask) pieces = br.performBoundaryRegularizationV2(initialMask, numProcesses=3) finalMask = ma.buildFinalMask(pieces, initialMask) io.imsave("C:\\bertud_temp\\finalMask.tif", finalMask) # pickle.dump(pieces,open("E:/BertudV2/pieces.pickle","wb")) t1 = time.time() print "Finished everything in ", round(t1 - t0, 2), "s."
def NtuplesProductionMain(input_options): parser = argparse.ArgumentParser() for option in ["doVars","doMergeVars","doInputs","doMergeInputs","prepareNorm","doHistos","doPlotImages","isTest"]: parser.add_argument('--'+option,action='store_true',dest=option, help="Boolean option: simply type --"+option) parser.add_argument('--year', action='store', dest="year", default="2017") parser.add_argument('--Samples', action='append', dest="Samples", default=[]) parser.add_argument('--SubSamples', action='store', dest="SubSamples", default="") parser.add_argument('--Channel', action='store', dest="Channel", default="lepton") parser.add_argument('--Collection', action='store', dest="Collection", default="Puppi") parser.add_argument('--ptmin', action='store', dest="ptmin", default="") parser.add_argument('--ptmax', action='store', dest="ptmax", default="") parser.add_argument('--firstfile', action='store', dest="firstfile", default="None") parser.add_argument('--lastfile', action='store', dest="lastfile", default="None") parser.add_argument('--others', action='store', dest="others", default="") args = parser.parse_args() print "\n****************************************" for arg in args.__dict__: print arg, " "*(15-len(arg)), args.__dict__[arg] print "****************************************\n" MRB = ModuleRunnerBase() if not args.Samples: args.Samples = MRB.Samples_matching print "Samples", args.Samples, args.SubSamples pts = [[args.ptmin,args.ptmax]] if args.ptmin!=args.ptmax else [["200", "4000"]] plotting_pts = pts+[["200", "300"],["300", "500"],["500", "4000"],["4000", "10000"]] print "pts", pts if args.doVars: NH = NtuplesHandler(Channel=args.Channel, Collection=args.Collection, Samples=args.Samples, year=args.year, extraText="", isTest=args.isTest) for Sample in args.Samples: print Sample for SubSample in MRB.Samples_dict[Sample]: if SubSample!=args.SubSamples and args.SubSamples!="": continue print "CreateVars:", SubSample NH.NHBDict[Sample][SubSample].ConvertRoot2Numpy(Sample=SubSample, firstfile=eval(args.firstfile),lastfile=eval(args.lastfile)) if args.doMergeVars: NH = NtuplesHandler(Channel=args.Channel, Collection=args.Collection, Samples=args.Samples, year=args.year, extraText="", isTest=args.isTest) NH.MergeVars(others=args.others) if args.doInputs: PreIn = PrepareInputs(Channel=args.Channel, Collection=args.Collection, Samples=args.Samples, year=args.year, pt_min=int(args.ptmin),pt_max=int(args.ptmax), extraText="", isTest=args.isTest) PreIn.Preprocessing(firstfile=eval(args.firstfile),lastfile=eval(args.lastfile), others=args.others) if args.doMergeInputs: print args.Samples for Sample in args.Samples: for pt in pts: print Sample, pt PreIn = PrepareInputs(Channel=args.Channel, Collection=args.Collection, Samples=[Sample], year=args.year, pt_min=int(pt[0]),pt_max=int(pt[1]), extraText="", isTest=args.isTest) PreIn.MergeInputs(others=args.others) if args.doHistos: print "doHistos" for Sample in args.Samples: for pt in pts: print Sample, pt PreIn = PrepareInputs(Channel=args.Channel, Collection=args.Collection, Samples=[Sample], year=args.year, pt_min=int(pt[0]),pt_max=int(pt[1]), extraText="", isTest=args.isTest) for pt_cuts in plotting_pts: PreIn.FileToHist(pt_cuts=pt_cuts) if args.doPlotImages: for pt in pts: print "doPlotImages", pt PreIn = PrepareInputs(Channel=args.Channel, Collection=args.Collection, Samples=args.Samples, year=args.year, pt_min=int(pt[0]),pt_max=int(pt[1]), extraText="", isTest=args.isTest) for pt_cuts in plotting_pts: PreIn.PlotInputVars(pt_cuts=pt_cuts) if args.prepareNorm: PreIn = PrepareInputs(Channel=args.Channel, Collection=args.Collection, Samples=args.Samples, extraText="",year=args.year, controls=[]) PreIn.FitNorm()
def main(): # instantiate application with open("config/slave_config.json", "r") as f: configfile = f.read() config = json.loads(configfile) if not os.path.exists(config["tempFolder"]): os.makedirs(config["tempFolder"]) if not os.path.exists(config["buildingPickleFolder"]): os.makedirs(config["buildingPickleFolder"]) recommendedCores = int(config["maxAllowableCore"]) WORKERID = str(config["workerID"]) print config["pythonPath"] # Run worker_usage.py worker_usage_process = subprocess.Popen([config["pythonPath"], "worker_usage.py"], shell=True) # Attach atexit event atexit.register(exit_handler) # os.killpg(os.getpgid(pro.pid), signal.SIGTERM) app = wx.PySimpleApp() taskbar = BertudTaskBarIcon() # print("This is worker %s" % WORKERNAME) # print "TESSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSST",config["dispatcherNameServer"] # make connection to dispatcher server dispatcher = Pyro4.core.Proxy("PYRONAME:" + config["dispatcherNameServer"] + "@" + config["dispatcherIP"]) gotItem = False # Loop for getting work while True: # Check for work in dispatcher try: time.sleep(0.5) cpu = psutil.cpu_percent() print cpu, EXTREME_CPU_USAGE if cpu < EXTREME_CPU_USAGE: item, laz = dispatcher.getWork(WORKERID) gotItem = True print "GOT ITEM!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" else: print "EXTREEEEEEME!" gotItem = False dispatcher.updateWorkerStatus(WORKERID, "busy") # If there are no work available except ValueError: print ("no work available yet.") dispatcher.updateWorkerStatus(WORKERID, "1") # No connection to dispatcher except: print "PUMASOK!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" # Set the taskbar's icon to gray - means no connection taskbar.set_icon(TRAY_ICON_GRAY) # Loop for reconnecting to dispatcher while True: # Try to reconnect to dispatcher try: print ("Dispatcher not found. Reconnecting...") dispatcher._pyroReconnect() # Can't connect -> Sleep then retry again except Exception: time.sleep(1) # Reconnecting succesful else: # Set the taskbar;s icon to green -> available taskbar.set_icon(TRAY_ICON_GREEN) taskbar.balloon_running() print ("Connected to dispatcher. Getting work now.") dispatcher.updateWorkerStatus(WORKERID, "1") break # Processing work from dispatcher else: if gotItem: gotItem = False # Set taskbar's icon to red -> working taskbar.set_icon(TRAY_ICON_RED) taskbar.balloon_work() print ("Got some work...") # print item print "Changed worker status" dispatcher.updateWorkerStatus(WORKERID, "2") print dispatcher.getUpdates()[0] # Use the data collected from the dispatcher # ndsm = item.data["ndsm"] # classified= item.data["classified"] # slope= item.data["slope"] # slopeslope= item.data["slopeslope"] item.start_time = time.time() print "Clearing temp folder..." # http://stackoverflow.com/questions/185936/delete-folder-contents-in-python for the_file in os.listdir(config["tempFolder"]): file_path = os.path.join(config["tempFolder"], the_file) try: if os.path.isfile(file_path): os.unlink(file_path) elif os.path.isdir(file_path): shutil.rmtree(file_path) except Exception, e: print e with open(config["tempFolder"] + "/pointcloud.laz", "wb") as file: file.write(laz) # Process the data try: t0 = time.time() sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Preparing Inputs")]) print "Preparing Inputs..." pi.prepareInputs(config["tempFolder"], config["lastoolsPath"]) ndsm = io.imread(config["tempFolder"] + "/ndsm.tif") classified = io.imread(config["tempFolder"] + "/classified.tif") classified = classified[0 : len(ndsm), 0 : len(ndsm[0])] slope = io.imread(config["tempFolder"] + "/slope.tif") numret = io.imread(config["tempFolder"] + "/numret.tif") sendLogs( dispatcher, taskbar, WORKERID, [("INFO", "FINISHED Preparing Inputs - " + str(round(time.time() - t0, 2)))], ) t0 = time.time() sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Generating Initial Mask")]) print "Generating Initial Mask..." initialMask = ma.generateInitialMask(ndsm, classified, slope, numret) sendLogs( dispatcher, taskbar, WORKERID, [("INFO", "FINISHED Generating Initial Mask - " + str(round(time.time() - t0, 2)))], ) # external.tifffile.imsave("initialMask.tif",initialMask) except: e = sys.exc_info()[0] error_msg = "Preparing Inputs / Generating Initial Mask - %s" % e item.error_msg = error_msg item.end_time = time.time() try: dispatcher.saveError(item, error_msg) dispatcher.updateWorkerStatus(WORKERID, "1") except: while True: # Try to reconnect to dispatcher try: print ("Dispatcher not found. Reconnecting...") dispatcher._pyroReconnect() # Can't connect -> Sleep then retry again except Exception: time.sleep(1) # Reconnecting succesful else: dispatcher.saveError(item, error_msg) dispatcher.updateWorkerStatus(WORKERID, "1") print ("Connected to dispatcher.") break else: # set the item's worker t0 = time.time() sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Building Regularization")]) print "Performing basic boundary regularization..." pieces, pbr_logs = br.performBoundaryRegularizationV2( initialMask, item.path, config["buildingPickleFolder"], numProcesses=getRecCores(maxCores=recommendedCores), ) if len(pbr_logs) > 0: # pbr_logs.append(("ERROR", "")) sendLogs(dispatcher, taskbar, WORKERID, pbr_logs) sendLogs( dispatcher, taskbar, WORKERID, [("INFO", "FINISHED Building Regularization - " + str(round(time.time() - t0, 2)))], ) t0 = time.time() sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Building Final Mask")]) finalMask = ma.buildFinalMask(pieces, initialMask) sendLogs( dispatcher, taskbar, WORKERID, [("INFO", "FINISHED Building Final Mask - " + str(round(time.time() - t0, 2)))], ) external.tifffile.imsave("finalmask.tif", finalMask) item.result = finalMask item.end_time = time.time() # KAGEYAMA # return the result to the dispatcher try: with open(config["tempFolder"] + "/dsm.tif", "rb") as f_dsm: dsm = f_dsm.read() with open(config["tempFolder"] + "/ndsm.tif", "rb") as f_ndsm: ndsm = f_ndsm.read() dispatcher.putResult(item, finalMask, dsm, ndsm) dispatcher.updateWorkerStatus(WORKERID, "1") except: while True: # Try to reconnect to dispatcher try: print ("Dispatcher not found. Reconnecting...") dispatcher._pyroReconnect() # Can't connect -> Sleep then retry again except Exception: time.sleep(1) # Reconnecting succesful else: dispatcher.putResult(item, finalMask, dsm, ndsm) dispatcher.updateWorkerStatus(WORKERID, "1") print ("Connected to dispatcher.") break # dispatcher.putResult(item) # set taskbar's icon to green -> available taskbar.set_icon(TRAY_ICON_GREEN) taskbar.balloon_free()
def main(): #instantiate application with open("config/slave_config.json", "r") as f: configfile = f.read() config = json.loads(configfile) if not os.path.exists(config["tempFolder"]): os.makedirs(config["tempFolder"]) if not os.path.exists(config["buildingPickleFolder"]): os.makedirs(config["buildingPickleFolder"]) recommendedCores = int(config["maxAllowableCore"]) WORKERID = str(config["workerID"]) print config["pythonPath"] # Run worker_usage.py worker_usage_process = subprocess.Popen( [config["pythonPath"], "worker_usage.py"], shell=True) # Attach atexit event atexit.register(exit_handler) # os.killpg(os.getpgid(pro.pid), signal.SIGTERM) app = wx.PySimpleApp() taskbar = BertudTaskBarIcon() # print("This is worker %s" % WORKERNAME) # print "TESSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSST",config["dispatcherNameServer"] #make connection to dispatcher server dispatcher = Pyro4.core.Proxy("PYRONAME:" + config["dispatcherNameServer"] + "@" + config["dispatcherIP"]) gotItem = False #Loop for getting work while True: #Check for work in dispatcher try: time.sleep(0.5) cpu = psutil.cpu_percent() print cpu, EXTREME_CPU_USAGE if cpu < EXTREME_CPU_USAGE: item, laz = dispatcher.getWork(WORKERID) gotItem = True print "GOT ITEM!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" else: print "EXTREEEEEEME!" gotItem = False dispatcher.updateWorkerStatus(WORKERID, 'busy') #If there are no work available except ValueError: print("no work available yet.") dispatcher.updateWorkerStatus(WORKERID, '1') #No connection to dispatcher except: print "PUMASOK!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" #Set the taskbar's icon to gray - means no connection taskbar.set_icon(TRAY_ICON_GRAY) #Loop for reconnecting to dispatcher while True: #Try to reconnect to dispatcher try: print("Dispatcher not found. Reconnecting...") dispatcher._pyroReconnect() #Can't connect -> Sleep then retry again except Exception: time.sleep(1) #Reconnecting succesful else: #Set the taskbar;s icon to green -> available taskbar.set_icon(TRAY_ICON_GREEN) taskbar.balloon_running() print("Connected to dispatcher. Getting work now.") dispatcher.updateWorkerStatus(WORKERID, '1') break #Processing work from dispatcher else: if gotItem: gotItem = False #Set taskbar's icon to red -> working taskbar.set_icon(TRAY_ICON_RED) taskbar.balloon_work() print("Got some work...") # print item print "Changed worker status" dispatcher.updateWorkerStatus(WORKERID, '2') print dispatcher.getUpdates()[0] # Use the data collected from the dispatcher # ndsm = item.data["ndsm"] # classified= item.data["classified"] # slope= item.data["slope"] # slopeslope= item.data["slopeslope"] item.start_time = time.time() print "Clearing temp folder..." # http://stackoverflow.com/questions/185936/delete-folder-contents-in-python for the_file in os.listdir(config["tempFolder"]): file_path = os.path.join(config["tempFolder"], the_file) try: if os.path.isfile(file_path): os.unlink(file_path) elif os.path.isdir(file_path): shutil.rmtree(file_path) except Exception, e: print e with open(config["tempFolder"] + "/pointcloud.laz", "wb") as file: file.write(laz) #Process the data try: t0 = time.time() sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Preparing Inputs")]) print "Preparing Inputs..." pi.prepareInputs(config["tempFolder"], config["lastoolsPath"]) ndsm = io.imread(config["tempFolder"] + "/ndsm.tif") classified = io.imread(config["tempFolder"] + "/classified.tif") classified = classified[0:len(ndsm), 0:len(ndsm[0])] slope = io.imread(config["tempFolder"] + "/slope.tif") numret = io.imread(config["tempFolder"] + "/numret.tif") sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "FINISHED Preparing Inputs - " + str(round(time.time() - t0, 2)))]) t0 = time.time() sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Generating Initial Mask")]) print "Generating Initial Mask..." initialMask = ma.generateInitialMask( ndsm, classified, slope, numret) sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "FINISHED Generating Initial Mask - " + str(round(time.time() - t0, 2)))]) # external.tifffile.imsave("initialMask.tif",initialMask) except: e = sys.exc_info()[0] error_msg = "Preparing Inputs / Generating Initial Mask - %s" % e item.error_msg = error_msg item.end_time = time.time() try: dispatcher.saveError(item, error_msg) dispatcher.updateWorkerStatus(WORKERID, '1') except: while True: #Try to reconnect to dispatcher try: print("Dispatcher not found. Reconnecting...") dispatcher._pyroReconnect() #Can't connect -> Sleep then retry again except Exception: time.sleep(1) #Reconnecting succesful else: dispatcher.saveError(item, error_msg) dispatcher.updateWorkerStatus(WORKERID, '1') print("Connected to dispatcher.") break else: #set the item's worker t0 = time.time() sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Building Regularization")]) print "Performing basic boundary regularization..." pieces, pbr_logs = br.performBoundaryRegularizationV2( initialMask, item.path, config["buildingPickleFolder"], numProcesses=getRecCores(maxCores=recommendedCores)) if len(pbr_logs) > 0: # pbr_logs.append(("ERROR", "")) sendLogs(dispatcher, taskbar, WORKERID, pbr_logs) sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "FINISHED Building Regularization - " + str(round(time.time() - t0, 2)))]) t0 = time.time() sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Building Final Mask")]) finalMask = ma.buildFinalMask(pieces, initialMask) sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "FINISHED Building Final Mask - " + str(round(time.time() - t0, 2)))]) external.tifffile.imsave("finalmask.tif", finalMask) item.result = finalMask item.end_time = time.time() #KAGEYAMA #return the result to the dispatcher try: with open(config["tempFolder"] + "/dsm.tif", "rb") as f_dsm: dsm = f_dsm.read() with open(config["tempFolder"] + "/ndsm.tif", "rb") as f_ndsm: ndsm = f_ndsm.read() dispatcher.putResult(item, finalMask, dsm, ndsm) dispatcher.updateWorkerStatus(WORKERID, '1') except: while True: #Try to reconnect to dispatcher try: print("Dispatcher not found. Reconnecting...") dispatcher._pyroReconnect() #Can't connect -> Sleep then retry again except Exception: time.sleep(1) #Reconnecting succesful else: dispatcher.putResult(item, finalMask, dsm, ndsm) dispatcher.updateWorkerStatus(WORKERID, '1') print("Connected to dispatcher.") break # dispatcher.putResult(item) #set taskbar's icon to green -> available taskbar.set_icon(TRAY_ICON_GREEN) taskbar.balloon_free()