def main():
t0 = time.time()
print "Preparing Inputs..."
pi.prepareInputs()
ndsm = io.imread("C:\\bertud_temp\\ndsm.tif")
classified = io.imread("C:\\bertud_temp\\classified.tif")
classified = classified[0:len(ndsm),0:len(ndsm[0])]
slope = io.imread("C:\\bertud_temp\\slope.tif")
numret = io.imread("C:\\bertud_temp\\numret.tif")
print "Generating Initial Mask..."
initialMask = ma.generateInitialMask(ndsm,classified,slope,numret)
io.imsave("C:\\bertud_temp\\initialMask.tif",initialMask)
pieces = br.performBoundaryRegularizationV2(initialMask,numProcesses=3)
finalMask = ma.buildFinalMask(pieces,initialMask)
io.imsave("C:\\bertud_temp\\finalMask.tif",finalMask)
# pickle.dump(pieces,open("E:/BertudV2/pieces.pickle","wb"))
t1 = time.time()
print "Finished everything in ",round(t1-t0,2),"s."
def main():
t0 = time.time()
print "Preparing Inputs..."
pi.prepareInputs()
ndsm = io.imread("C:\\bertud_temp\\ndsm.tif")
classified = io.imread("C:\\bertud_temp\\classified.tif")
classified = classified[0:len(ndsm), 0:len(ndsm[0])]
slope = io.imread("C:\\bertud_temp\\slope.tif")
numret = io.imread("C:\\bertud_temp\\numret.tif")
print "Generating Initial Mask..."
initialMask = ma.generateInitialMask(ndsm, classified, slope, numret)
io.imsave("C:\\bertud_temp\\initialMask.tif", initialMask)
pieces = br.performBoundaryRegularizationV2(initialMask, numProcesses=3)
finalMask = ma.buildFinalMask(pieces, initialMask)
io.imsave("C:\\bertud_temp\\finalMask.tif", finalMask)
# pickle.dump(pieces,open("E:/BertudV2/pieces.pickle","wb"))
t1 = time.time()
print "Finished everything in ", round(t1 - t0, 2), "s."
def main():
# instantiate application
with open("config/slave_config.json", "r") as f:
configfile = f.read()
config = json.loads(configfile)
if not os.path.exists(config["tempFolder"]):
os.makedirs(config["tempFolder"])
if not os.path.exists(config["buildingPickleFolder"]):
os.makedirs(config["buildingPickleFolder"])
recommendedCores = int(config["maxAllowableCore"])
WORKERID = str(config["workerID"])
print config["pythonPath"]
# Run worker_usage.py
worker_usage_process = subprocess.Popen([config["pythonPath"], "worker_usage.py"], shell=True)
# Attach atexit event
atexit.register(exit_handler)
# os.killpg(os.getpgid(pro.pid), signal.SIGTERM)
app = wx.PySimpleApp()
taskbar = BertudTaskBarIcon()
# print("This is worker %s" % WORKERNAME)
# print "TESSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSST",config["dispatcherNameServer"]
# make connection to dispatcher server
dispatcher = Pyro4.core.Proxy("PYRONAME:" + config["dispatcherNameServer"] + "@" + config["dispatcherIP"])
gotItem = False
# Loop for getting work
while True:
# Check for work in dispatcher
try:
time.sleep(0.5)
cpu = psutil.cpu_percent()
print cpu, EXTREME_CPU_USAGE
if cpu < EXTREME_CPU_USAGE:
item, laz = dispatcher.getWork(WORKERID)
gotItem = True
print "GOT ITEM!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
else:
print "EXTREEEEEEME!"
gotItem = False
dispatcher.updateWorkerStatus(WORKERID, "busy")
# If there are no work available
except ValueError:
print ("no work available yet.")
dispatcher.updateWorkerStatus(WORKERID, "1")
# No connection to dispatcher
except:
print "PUMASOK!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
# Set the taskbar's icon to gray - means no connection
taskbar.set_icon(TRAY_ICON_GRAY)
# Loop for reconnecting to dispatcher
while True:
# Try to reconnect to dispatcher
try:
print ("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
# Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
# Reconnecting succesful
else:
# Set the taskbar;s icon to green -> available
taskbar.set_icon(TRAY_ICON_GREEN)
taskbar.balloon_running()
print ("Connected to dispatcher. Getting work now.")
dispatcher.updateWorkerStatus(WORKERID, "1")
break
# Processing work from dispatcher
else:
if gotItem:
gotItem = False
# Set taskbar's icon to red -> working
taskbar.set_icon(TRAY_ICON_RED)
taskbar.balloon_work()
print ("Got some work...")
# print item
print "Changed worker status"
dispatcher.updateWorkerStatus(WORKERID, "2")
print dispatcher.getUpdates()[0]
# Use the data collected from the dispatcher
# ndsm = item.data["ndsm"]
# classified= item.data["classified"]
# slope= item.data["slope"]
# slopeslope= item.data["slopeslope"]
item.start_time = time.time()
print "Clearing temp folder..."
# http://stackoverflow.com/questions/185936/delete-folder-contents-in-python
for the_file in os.listdir(config["tempFolder"]):
file_path = os.path.join(config["tempFolder"], the_file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception, e:
print e
with open(config["tempFolder"] + "/pointcloud.laz", "wb") as file:
file.write(laz)
# Process the data
try:
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Preparing Inputs")])
print "Preparing Inputs..."
pi.prepareInputs(config["tempFolder"], config["lastoolsPath"])
ndsm = io.imread(config["tempFolder"] + "/ndsm.tif")
classified = io.imread(config["tempFolder"] + "/classified.tif")
classified = classified[0 : len(ndsm), 0 : len(ndsm[0])]
slope = io.imread(config["tempFolder"] + "/slope.tif")
numret = io.imread(config["tempFolder"] + "/numret.tif")
sendLogs(
dispatcher,
taskbar,
WORKERID,
[("INFO", "FINISHED Preparing Inputs - " + str(round(time.time() - t0, 2)))],
)
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Generating Initial Mask")])
print "Generating Initial Mask..."
initialMask = ma.generateInitialMask(ndsm, classified, slope, numret)
sendLogs(
dispatcher,
taskbar,
WORKERID,
[("INFO", "FINISHED Generating Initial Mask - " + str(round(time.time() - t0, 2)))],
)
# external.tifffile.imsave("initialMask.tif",initialMask)
except:
e = sys.exc_info()[0]
error_msg = "Preparing Inputs / Generating Initial Mask - %s" % e
item.error_msg = error_msg
item.end_time = time.time()
try:
dispatcher.saveError(item, error_msg)
dispatcher.updateWorkerStatus(WORKERID, "1")
except:
while True:
# Try to reconnect to dispatcher
try:
print ("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
# Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
# Reconnecting succesful
else:
dispatcher.saveError(item, error_msg)
dispatcher.updateWorkerStatus(WORKERID, "1")
print ("Connected to dispatcher.")
break
else:
# set the item's worker
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Building Regularization")])
print "Performing basic boundary regularization..."
pieces, pbr_logs = br.performBoundaryRegularizationV2(
initialMask,
item.path,
config["buildingPickleFolder"],
numProcesses=getRecCores(maxCores=recommendedCores),
)
if len(pbr_logs) > 0:
# pbr_logs.append(("ERROR", ""))
sendLogs(dispatcher, taskbar, WORKERID, pbr_logs)
sendLogs(
dispatcher,
taskbar,
WORKERID,
[("INFO", "FINISHED Building Regularization - " + str(round(time.time() - t0, 2)))],
)
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Building Final Mask")])
finalMask = ma.buildFinalMask(pieces, initialMask)
sendLogs(
dispatcher,
taskbar,
WORKERID,
[("INFO", "FINISHED Building Final Mask - " + str(round(time.time() - t0, 2)))],
)
external.tifffile.imsave("finalmask.tif", finalMask)
item.result = finalMask
item.end_time = time.time()
# KAGEYAMA
# return the result to the dispatcher
try:
with open(config["tempFolder"] + "/dsm.tif", "rb") as f_dsm:
dsm = f_dsm.read()
with open(config["tempFolder"] + "/ndsm.tif", "rb") as f_ndsm:
ndsm = f_ndsm.read()
dispatcher.putResult(item, finalMask, dsm, ndsm)
dispatcher.updateWorkerStatus(WORKERID, "1")
except:
while True:
# Try to reconnect to dispatcher
try:
print ("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
# Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
# Reconnecting succesful
else:
dispatcher.putResult(item, finalMask, dsm, ndsm)
dispatcher.updateWorkerStatus(WORKERID, "1")
print ("Connected to dispatcher.")
break
# dispatcher.putResult(item)
# set taskbar's icon to green -> available
taskbar.set_icon(TRAY_ICON_GREEN)
taskbar.balloon_free()
def main():
#instantiate application
with open("config/slave_config.json", "r") as f:
configfile = f.read()
config = json.loads(configfile)
if not os.path.exists(config["tempFolder"]):
os.makedirs(config["tempFolder"])
if not os.path.exists(config["buildingPickleFolder"]):
os.makedirs(config["buildingPickleFolder"])
recommendedCores = int(config["maxAllowableCore"])
WORKERID = str(config["workerID"])
print config["pythonPath"]
# Run worker_usage.py
worker_usage_process = subprocess.Popen(
[config["pythonPath"], "worker_usage.py"], shell=True)
# Attach atexit event
atexit.register(exit_handler)
# os.killpg(os.getpgid(pro.pid), signal.SIGTERM)
app = wx.PySimpleApp()
taskbar = BertudTaskBarIcon()
# print("This is worker %s" % WORKERNAME)
# print "TESSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSST",config["dispatcherNameServer"]
#make connection to dispatcher server
dispatcher = Pyro4.core.Proxy("PYRONAME:" +
config["dispatcherNameServer"] + "@" +
config["dispatcherIP"])
gotItem = False
#Loop for getting work
while True:
#Check for work in dispatcher
try:
time.sleep(0.5)
cpu = psutil.cpu_percent()
print cpu, EXTREME_CPU_USAGE
if cpu < EXTREME_CPU_USAGE:
item, laz = dispatcher.getWork(WORKERID)
gotItem = True
print "GOT ITEM!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
else:
print "EXTREEEEEEME!"
gotItem = False
dispatcher.updateWorkerStatus(WORKERID, 'busy')
#If there are no work available
except ValueError:
print("no work available yet.")
dispatcher.updateWorkerStatus(WORKERID, '1')
#No connection to dispatcher
except:
print "PUMASOK!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
#Set the taskbar's icon to gray - means no connection
taskbar.set_icon(TRAY_ICON_GRAY)
#Loop for reconnecting to dispatcher
while True:
#Try to reconnect to dispatcher
try:
print("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
#Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
#Reconnecting succesful
else:
#Set the taskbar;s icon to green -> available
taskbar.set_icon(TRAY_ICON_GREEN)
taskbar.balloon_running()
print("Connected to dispatcher. Getting work now.")
dispatcher.updateWorkerStatus(WORKERID, '1')
break
#Processing work from dispatcher
else:
if gotItem:
gotItem = False
#Set taskbar's icon to red -> working
taskbar.set_icon(TRAY_ICON_RED)
taskbar.balloon_work()
print("Got some work...")
# print item
print "Changed worker status"
dispatcher.updateWorkerStatus(WORKERID, '2')
print dispatcher.getUpdates()[0]
# Use the data collected from the dispatcher
# ndsm = item.data["ndsm"]
# classified= item.data["classified"]
# slope= item.data["slope"]
# slopeslope= item.data["slopeslope"]
item.start_time = time.time()
print "Clearing temp folder..."
# http://stackoverflow.com/questions/185936/delete-folder-contents-in-python
for the_file in os.listdir(config["tempFolder"]):
file_path = os.path.join(config["tempFolder"], the_file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception, e:
print e
with open(config["tempFolder"] + "/pointcloud.laz",
"wb") as file:
file.write(laz)
#Process the data
try:
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "STARTED Preparing Inputs")])
print "Preparing Inputs..."
pi.prepareInputs(config["tempFolder"],
config["lastoolsPath"])
ndsm = io.imread(config["tempFolder"] + "/ndsm.tif")
classified = io.imread(config["tempFolder"] +
"/classified.tif")
classified = classified[0:len(ndsm), 0:len(ndsm[0])]
slope = io.imread(config["tempFolder"] + "/slope.tif")
numret = io.imread(config["tempFolder"] + "/numret.tif")
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "FINISHED Preparing Inputs - " +
str(round(time.time() - t0, 2)))])
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "STARTED Generating Initial Mask")])
print "Generating Initial Mask..."
initialMask = ma.generateInitialMask(
ndsm, classified, slope, numret)
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "FINISHED Generating Initial Mask - " +
str(round(time.time() - t0, 2)))])
# external.tifffile.imsave("initialMask.tif",initialMask)
except:
e = sys.exc_info()[0]
error_msg = "Preparing Inputs / Generating Initial Mask - %s" % e
item.error_msg = error_msg
item.end_time = time.time()
try:
dispatcher.saveError(item, error_msg)
dispatcher.updateWorkerStatus(WORKERID, '1')
except:
while True:
#Try to reconnect to dispatcher
try:
print("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
#Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
#Reconnecting succesful
else:
dispatcher.saveError(item, error_msg)
dispatcher.updateWorkerStatus(WORKERID, '1')
print("Connected to dispatcher.")
break
else:
#set the item's worker
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "STARTED Building Regularization")])
print "Performing basic boundary regularization..."
pieces, pbr_logs = br.performBoundaryRegularizationV2(
initialMask,
item.path,
config["buildingPickleFolder"],
numProcesses=getRecCores(maxCores=recommendedCores))
if len(pbr_logs) > 0:
# pbr_logs.append(("ERROR", ""))
sendLogs(dispatcher, taskbar, WORKERID, pbr_logs)
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "FINISHED Building Regularization - " +
str(round(time.time() - t0, 2)))])
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "STARTED Building Final Mask")])
finalMask = ma.buildFinalMask(pieces, initialMask)
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "FINISHED Building Final Mask - " +
str(round(time.time() - t0, 2)))])
external.tifffile.imsave("finalmask.tif", finalMask)
item.result = finalMask
item.end_time = time.time()
#KAGEYAMA
#return the result to the dispatcher
try:
with open(config["tempFolder"] + "/dsm.tif",
"rb") as f_dsm:
dsm = f_dsm.read()
with open(config["tempFolder"] + "/ndsm.tif",
"rb") as f_ndsm:
ndsm = f_ndsm.read()
dispatcher.putResult(item, finalMask, dsm, ndsm)
dispatcher.updateWorkerStatus(WORKERID, '1')
except:
while True:
#Try to reconnect to dispatcher
try:
print("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
#Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
#Reconnecting succesful
else:
dispatcher.putResult(item, finalMask, dsm,
ndsm)
dispatcher.updateWorkerStatus(WORKERID, '1')
print("Connected to dispatcher.")
break
# dispatcher.putResult(item)
#set taskbar's icon to green -> available
taskbar.set_icon(TRAY_ICON_GREEN)
taskbar.balloon_free()
