def main():
t0 = time.time()
print "Preparing Inputs..."
pi.prepareInputs()
ndsm = io.imread("C:\\bertud_temp\\ndsm.tif")
classified = io.imread("C:\\bertud_temp\\classified.tif")
classified = classified[0:len(ndsm),0:len(ndsm[0])]
slope = io.imread("C:\\bertud_temp\\slope.tif")
numret = io.imread("C:\\bertud_temp\\numret.tif")
print "Generating Initial Mask..."
initialMask = ma.generateInitialMask(ndsm,classified,slope,numret)
io.imsave("C:\\bertud_temp\\initialMask.tif",initialMask)
pieces = br.performBoundaryRegularizationV2(initialMask,numProcesses=3)
finalMask = ma.buildFinalMask(pieces,initialMask)
io.imsave("C:\\bertud_temp\\finalMask.tif",finalMask)
# pickle.dump(pieces,open("E:/BertudV2/pieces.pickle","wb"))
t1 = time.time()
print "Finished everything in ",round(t1-t0,2),"s."
def main():
t0 = time.time()
print "Preparing Inputs..."
pi.prepareInputs()
ndsm = io.imread("C:\\bertud_temp\\ndsm.tif")
classified = io.imread("C:\\bertud_temp\\classified.tif")
classified = classified[0:len(ndsm), 0:len(ndsm[0])]
slope = io.imread("C:\\bertud_temp\\slope.tif")
numret = io.imread("C:\\bertud_temp\\numret.tif")
print "Generating Initial Mask..."
initialMask = ma.generateInitialMask(ndsm, classified, slope, numret)
io.imsave("C:\\bertud_temp\\initialMask.tif", initialMask)
pieces = br.performBoundaryRegularizationV2(initialMask, numProcesses=3)
finalMask = ma.buildFinalMask(pieces, initialMask)
io.imsave("C:\\bertud_temp\\finalMask.tif", finalMask)
# pickle.dump(pieces,open("E:/BertudV2/pieces.pickle","wb"))
t1 = time.time()
print "Finished everything in ", round(t1 - t0, 2), "s."
def main():
# instantiate application
with open("config/slave_config.json", "r") as f:
configfile = f.read()
config = json.loads(configfile)
if not os.path.exists(config["tempFolder"]):
os.makedirs(config["tempFolder"])
if not os.path.exists(config["buildingPickleFolder"]):
os.makedirs(config["buildingPickleFolder"])
recommendedCores = int(config["maxAllowableCore"])
WORKERID = str(config["workerID"])
print config["pythonPath"]
# Run worker_usage.py
worker_usage_process = subprocess.Popen([config["pythonPath"], "worker_usage.py"], shell=True)
# Attach atexit event
atexit.register(exit_handler)
# os.killpg(os.getpgid(pro.pid), signal.SIGTERM)
app = wx.PySimpleApp()
taskbar = BertudTaskBarIcon()
# print("This is worker %s" % WORKERNAME)
# print "TESSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSST",config["dispatcherNameServer"]
# make connection to dispatcher server
dispatcher = Pyro4.core.Proxy("PYRONAME:" + config["dispatcherNameServer"] + "@" + config["dispatcherIP"])
gotItem = False
# Loop for getting work
while True:
# Check for work in dispatcher
try:
time.sleep(0.5)
cpu = psutil.cpu_percent()
print cpu, EXTREME_CPU_USAGE
if cpu < EXTREME_CPU_USAGE:
item, laz = dispatcher.getWork(WORKERID)
gotItem = True
print "GOT ITEM!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
else:
print "EXTREEEEEEME!"
gotItem = False
dispatcher.updateWorkerStatus(WORKERID, "busy")
# If there are no work available
except ValueError:
print ("no work available yet.")
dispatcher.updateWorkerStatus(WORKERID, "1")
# No connection to dispatcher
except:
print "PUMASOK!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
# Set the taskbar's icon to gray - means no connection
taskbar.set_icon(TRAY_ICON_GRAY)
# Loop for reconnecting to dispatcher
while True:
# Try to reconnect to dispatcher
try:
print ("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
# Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
# Reconnecting succesful
else:
# Set the taskbar;s icon to green -> available
taskbar.set_icon(TRAY_ICON_GREEN)
taskbar.balloon_running()
print ("Connected to dispatcher. Getting work now.")
dispatcher.updateWorkerStatus(WORKERID, "1")
break
# Processing work from dispatcher
else:
if gotItem:
gotItem = False
# Set taskbar's icon to red -> working
taskbar.set_icon(TRAY_ICON_RED)
taskbar.balloon_work()
print ("Got some work...")
# print item
print "Changed worker status"
dispatcher.updateWorkerStatus(WORKERID, "2")
print dispatcher.getUpdates()[0]
# Use the data collected from the dispatcher
# ndsm = item.data["ndsm"]
# classified= item.data["classified"]
# slope= item.data["slope"]
# slopeslope= item.data["slopeslope"]
item.start_time = time.time()
print "Clearing temp folder..."
# http://stackoverflow.com/questions/185936/delete-folder-contents-in-python
for the_file in os.listdir(config["tempFolder"]):
file_path = os.path.join(config["tempFolder"], the_file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception, e:
print e
with open(config["tempFolder"] + "/pointcloud.laz", "wb") as file:
file.write(laz)
# Process the data
try:
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Preparing Inputs")])
print "Preparing Inputs..."
pi.prepareInputs(config["tempFolder"], config["lastoolsPath"])
ndsm = io.imread(config["tempFolder"] + "/ndsm.tif")
classified = io.imread(config["tempFolder"] + "/classified.tif")
classified = classified[0 : len(ndsm), 0 : len(ndsm[0])]
slope = io.imread(config["tempFolder"] + "/slope.tif")
numret = io.imread(config["tempFolder"] + "/numret.tif")
sendLogs(
dispatcher,
taskbar,
WORKERID,
[("INFO", "FINISHED Preparing Inputs - " + str(round(time.time() - t0, 2)))],
)
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Generating Initial Mask")])
print "Generating Initial Mask..."
initialMask = ma.generateInitialMask(ndsm, classified, slope, numret)
sendLogs(
dispatcher,
taskbar,
WORKERID,
[("INFO", "FINISHED Generating Initial Mask - " + str(round(time.time() - t0, 2)))],
)
# external.tifffile.imsave("initialMask.tif",initialMask)
except:
e = sys.exc_info()[0]
error_msg = "Preparing Inputs / Generating Initial Mask - %s" % e
item.error_msg = error_msg
item.end_time = time.time()
try:
dispatcher.saveError(item, error_msg)
dispatcher.updateWorkerStatus(WORKERID, "1")
except:
while True:
# Try to reconnect to dispatcher
try:
print ("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
# Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
# Reconnecting succesful
else:
dispatcher.saveError(item, error_msg)
dispatcher.updateWorkerStatus(WORKERID, "1")
print ("Connected to dispatcher.")
break
else:
# set the item's worker
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Building Regularization")])
print "Performing basic boundary regularization..."
pieces, pbr_logs = br.performBoundaryRegularizationV2(
initialMask,
item.path,
config["buildingPickleFolder"],
numProcesses=getRecCores(maxCores=recommendedCores),
)
if len(pbr_logs) > 0:
# pbr_logs.append(("ERROR", ""))
sendLogs(dispatcher, taskbar, WORKERID, pbr_logs)
sendLogs(
dispatcher,
taskbar,
WORKERID,
[("INFO", "FINISHED Building Regularization - " + str(round(time.time() - t0, 2)))],
)
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID, [("INFO", "STARTED Building Final Mask")])
finalMask = ma.buildFinalMask(pieces, initialMask)
sendLogs(
dispatcher,
taskbar,
WORKERID,
[("INFO", "FINISHED Building Final Mask - " + str(round(time.time() - t0, 2)))],
)
external.tifffile.imsave("finalmask.tif", finalMask)
item.result = finalMask
item.end_time = time.time()
# KAGEYAMA
# return the result to the dispatcher
try:
with open(config["tempFolder"] + "/dsm.tif", "rb") as f_dsm:
dsm = f_dsm.read()
with open(config["tempFolder"] + "/ndsm.tif", "rb") as f_ndsm:
ndsm = f_ndsm.read()
dispatcher.putResult(item, finalMask, dsm, ndsm)
dispatcher.updateWorkerStatus(WORKERID, "1")
except:
while True:
# Try to reconnect to dispatcher
try:
print ("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
# Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
# Reconnecting succesful
else:
dispatcher.putResult(item, finalMask, dsm, ndsm)
dispatcher.updateWorkerStatus(WORKERID, "1")
print ("Connected to dispatcher.")
break
# dispatcher.putResult(item)
# set taskbar's icon to green -> available
taskbar.set_icon(TRAY_ICON_GREEN)
taskbar.balloon_free()
def prepocess(self,
img,
rootCrown,
scale=1.0,
nrExRoot=1,
marker=True,
stemCorrection=False):
print 'starting to segment'
rIdx = -1
self.__io.setServerPath('./')
circleIdx = circleRatio = circleWidth = circleHeight = imgCircle = 0
Failed = False
orig = img.copy()
mask = Masking.Masking(scale=scale)
imgGrey = img.astype(np.uint8)
print 'make mask'
imgBinary = mask.calculateMask(imgGrey)
print 'saving binary mask'
scipy.misc.imsave(
self.__io.getHomePath() + '/Mask/' + self.__io.getFileName() +
'.png', imgBinary)
pathold = os.getcwd()
os.chdir(self.__io.getHomePath())
self.__io.writeServerFile(
'dirt_out.csv',
self.__io.getHomePath() + '/Mask/' + self.__io.getFileName() +
'.png,' + str(self.__io.getID()) + ',0')
os.chdir(pathold)
imgLabel = self.calculateLabelHist(imgBinary)
if marker == True:
print 'Marker is True'
circleIdx, circleRatio, circleWidth, circleHeight, imgCircle = self.findCircle(
imgLabel.copy())
else:
print 'Marker is False'
circleIdx, circleRatio, circleWidth, circleHeight, imgCircle = -1, 1, 1, 1, None
rectIdx, _, _, _, imgTag, tagText = self.findTag(imgLabel.copy(),
imgBinary,
orig,
rect_ratio=5.)
if rectIdx >= 0:
print 'tagIdx' + str(rectIdx)
try:
self.__labelHist[rectIdx] = 0
except:
pass
if circleIdx >= 0 and marker == True:
try:
self.__labelHist[circleIdx] = 0
except:
pass
'''
These two functions belong together and have to be called right after each other. I know, that is bad.
'''
if rootCrown == True:
rIdx, rIdxList, crownMin, crownMax, crownBottom, crownTop = self.findRoot(
imgLabel.copy())
if stemCorrection == True:
print 'Stem reconstruction is active '
imgRoot = self.correctForStem(imgLabel.copy(),
[circleIdx, rectIdx, rIdx],
crownMin, crownMax, crownBottom,
crownTop, rIdx, rIdxList)
else:
print 'No stem reconstruction active'
imgReturn = np.zeros_like(imgLabel)
imgReturn[rIdxList] = 1
imgRoot = imgReturn[crownMax:crownMin, crownBottom:crownTop]
if nrExRoot > 1 and rootCrown == True:
for i in range(nrExRoot):
exRIdx, imgExRoot, centerPtx, centerPty = self.findExcisedRoot(
imgLabel.copy(), [circleIdx, rectIdx, rIdx], crownMin,
crownMax)
if exRIdx != -1:
print 'found excised root ' + str(i)
try:
scipy.misc.imsave(
self.__io.getHomePath() + '/Lateral/' +
self.__io.getFileName() + '_' + str(centerPtx) +
'_' + str(centerPty) + '.png', imgExRoot)
print 'excised root ' + str(i) + 'saved'
except:
print 'NOT SAVED !!!'
raise
try:
pathold = os.getcwd()
os.chdir(self.__io.getHomePath())
self.__io.writeServerFile(
'dirt_out.csv',
self.__io.getHomePath() + '/Lateral/' +
self.__io.getFileName() + '_' + str(centerPtx) +
'_' + str(centerPty) + '.png,' +
str(self.__io.getID()) + ',0')
print 'excised root ' + str(i) + 'saved Server'
os.chdir(pathold)
except:
print 'NOT SAVED !!!!'
raise
elif nrExRoot == 1 and rootCrown == True:
exRIdx, imgExRoot, centerPtx, centerPty = self.findExcisedRoot(
imgLabel.copy(), [circleIdx, rectIdx, rIdx], crownMin,
crownMax)
if exRIdx != -1:
print 'found the excised root '
try:
pathold = os.getcwd()
os.chdir(self.__io.getHomePath())
scipy.misc.imsave(
self.__io.getHomePath() + '/Lateral/' +
self.__io.getFileName() + '_' + str(centerPtx) + '_' +
str(centerPty) + '.png', imgExRoot)
print 'excised root saved'
self.__io.writeServerFile(
'dirt_out.csv',
self.__io.getHomePath() + '/Lateral/' +
self.__io.getFileName() + '_' + str(centerPtx) + '_' +
str(centerPty) + '.png,' + str(self.__io.getID()) +
',0')
print 'excised root saved Server'
os.chdir(pathold)
except:
print 'NOT SAVED !!!!'
elif nrExRoot == 1 and rootCrown == False:
exRIdx, imgExRoot, centerPtx, centerPty = self.findExcisedRoot(
imgLabel.copy(), [circleIdx, rectIdx], 0, 1)
if exRIdx != -1:
print 'found the excised root '
rIdx = -1
try:
pathold = os.getcwd()
os.chdir(self.__io.getHomePath())
scipy.misc.imsave(
self.__io.getHomePath() + '/Lateral/' +
self.__io.getFileName() + '_' + str(centerPtx) + '_' +
str(centerPty) + '.png', imgExRoot)
print 'excised root saved'
self.__io.writeServerFile(
'dirt_out.csv',
self.__io.getHomePath() + '/Lateral/' +
self.__io.getFileName() + '_' + str(centerPtx) + '_' +
str(centerPty) + '.png,' + str(self.__io.getID()) +
',0')
print 'excised root saved Server'
os.chdir(pathold)
except:
print 'NOT SAVED !!!!'
if marker == True:
scipy.misc.imsave(
self.__io.getHomePath() + '/Mask/' + self.__io.getFileName() +
'Circle.png', imgCircle)
scipy.misc.imsave(
self.__io.getHomePath() + '/Mask/' + self.__io.getFileName() +
'Tag.png', imgTag)
#pathold=os.getcwd()
#os.chdir(self.__io.getHomePath())
if marker == True:
self.__io.writeServerFile(
'dirt_out.csv',
self.__io.getHomePath() + '/Mask/' + self.__io.getFileName() +
'Circle.png,' + str(self.__io.getID()) + ',0')
#os.chdir(pathold)
if rIdx != -1:
'''
If image is usable, then it gets segmented and copied. Otherwise we ignore it
'''
try:
print 'root image to be saved'
scipy.misc.imsave(
self.__io.getHomePath() + '/Crown/' +
self.__io.getFileName() + '.png', imgRoot)
except:
print 'CROWN NOT SAVED'
raise
try:
pathold = os.getcwd()
os.chdir(self.__io.getHomePath())
self.__io.writeServerFile(
'dirt_out.csv',
self.__io.getHomePath() + '/Crown/' +
self.__io.getFileName() + '.png,' +
str(self.__io.getID()) + ',0')
os.chdir(pathold)
except:
print 'MASK NOT WRITTEN TO SERVER FILE'
elif rIdx == -1 and exRIdx != -1:
print "Only excised roots computed"
else:
Failed = True
print "old path: " + pathold
return Failed, tagText, circleRatio, circleWidth, circleHeight
def main():
#instantiate application
with open("config/slave_config.json", "r") as f:
configfile = f.read()
config = json.loads(configfile)
if not os.path.exists(config["tempFolder"]):
os.makedirs(config["tempFolder"])
if not os.path.exists(config["buildingPickleFolder"]):
os.makedirs(config["buildingPickleFolder"])
recommendedCores = int(config["maxAllowableCore"])
WORKERID = str(config["workerID"])
print config["pythonPath"]
# Run worker_usage.py
worker_usage_process = subprocess.Popen(
[config["pythonPath"], "worker_usage.py"], shell=True)
# Attach atexit event
atexit.register(exit_handler)
# os.killpg(os.getpgid(pro.pid), signal.SIGTERM)
app = wx.PySimpleApp()
taskbar = BertudTaskBarIcon()
# print("This is worker %s" % WORKERNAME)
# print "TESSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSST",config["dispatcherNameServer"]
#make connection to dispatcher server
dispatcher = Pyro4.core.Proxy("PYRONAME:" +
config["dispatcherNameServer"] + "@" +
config["dispatcherIP"])
gotItem = False
#Loop for getting work
while True:
#Check for work in dispatcher
try:
time.sleep(0.5)
cpu = psutil.cpu_percent()
print cpu, EXTREME_CPU_USAGE
if cpu < EXTREME_CPU_USAGE:
item, laz = dispatcher.getWork(WORKERID)
gotItem = True
print "GOT ITEM!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
else:
print "EXTREEEEEEME!"
gotItem = False
dispatcher.updateWorkerStatus(WORKERID, 'busy')
#If there are no work available
except ValueError:
print("no work available yet.")
dispatcher.updateWorkerStatus(WORKERID, '1')
#No connection to dispatcher
except:
print "PUMASOK!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
#Set the taskbar's icon to gray - means no connection
taskbar.set_icon(TRAY_ICON_GRAY)
#Loop for reconnecting to dispatcher
while True:
#Try to reconnect to dispatcher
try:
print("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
#Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
#Reconnecting succesful
else:
#Set the taskbar;s icon to green -> available
taskbar.set_icon(TRAY_ICON_GREEN)
taskbar.balloon_running()
print("Connected to dispatcher. Getting work now.")
dispatcher.updateWorkerStatus(WORKERID, '1')
break
#Processing work from dispatcher
else:
if gotItem:
gotItem = False
#Set taskbar's icon to red -> working
taskbar.set_icon(TRAY_ICON_RED)
taskbar.balloon_work()
print("Got some work...")
# print item
print "Changed worker status"
dispatcher.updateWorkerStatus(WORKERID, '2')
print dispatcher.getUpdates()[0]
# Use the data collected from the dispatcher
# ndsm = item.data["ndsm"]
# classified= item.data["classified"]
# slope= item.data["slope"]
# slopeslope= item.data["slopeslope"]
item.start_time = time.time()
print "Clearing temp folder..."
# http://stackoverflow.com/questions/185936/delete-folder-contents-in-python
for the_file in os.listdir(config["tempFolder"]):
file_path = os.path.join(config["tempFolder"], the_file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception, e:
print e
with open(config["tempFolder"] + "/pointcloud.laz",
"wb") as file:
file.write(laz)
#Process the data
try:
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "STARTED Preparing Inputs")])
print "Preparing Inputs..."
pi.prepareInputs(config["tempFolder"],
config["lastoolsPath"])
ndsm = io.imread(config["tempFolder"] + "/ndsm.tif")
classified = io.imread(config["tempFolder"] +
"/classified.tif")
classified = classified[0:len(ndsm), 0:len(ndsm[0])]
slope = io.imread(config["tempFolder"] + "/slope.tif")
numret = io.imread(config["tempFolder"] + "/numret.tif")
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "FINISHED Preparing Inputs - " +
str(round(time.time() - t0, 2)))])
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "STARTED Generating Initial Mask")])
print "Generating Initial Mask..."
initialMask = ma.generateInitialMask(
ndsm, classified, slope, numret)
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "FINISHED Generating Initial Mask - " +
str(round(time.time() - t0, 2)))])
# external.tifffile.imsave("initialMask.tif",initialMask)
except:
e = sys.exc_info()[0]
error_msg = "Preparing Inputs / Generating Initial Mask - %s" % e
item.error_msg = error_msg
item.end_time = time.time()
try:
dispatcher.saveError(item, error_msg)
dispatcher.updateWorkerStatus(WORKERID, '1')
except:
while True:
#Try to reconnect to dispatcher
try:
print("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
#Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
#Reconnecting succesful
else:
dispatcher.saveError(item, error_msg)
dispatcher.updateWorkerStatus(WORKERID, '1')
print("Connected to dispatcher.")
break
else:
#set the item's worker
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "STARTED Building Regularization")])
print "Performing basic boundary regularization..."
pieces, pbr_logs = br.performBoundaryRegularizationV2(
initialMask,
item.path,
config["buildingPickleFolder"],
numProcesses=getRecCores(maxCores=recommendedCores))
if len(pbr_logs) > 0:
# pbr_logs.append(("ERROR", ""))
sendLogs(dispatcher, taskbar, WORKERID, pbr_logs)
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "FINISHED Building Regularization - " +
str(round(time.time() - t0, 2)))])
t0 = time.time()
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "STARTED Building Final Mask")])
finalMask = ma.buildFinalMask(pieces, initialMask)
sendLogs(dispatcher, taskbar, WORKERID,
[("INFO", "FINISHED Building Final Mask - " +
str(round(time.time() - t0, 2)))])
external.tifffile.imsave("finalmask.tif", finalMask)
item.result = finalMask
item.end_time = time.time()
#KAGEYAMA
#return the result to the dispatcher
try:
with open(config["tempFolder"] + "/dsm.tif",
"rb") as f_dsm:
dsm = f_dsm.read()
with open(config["tempFolder"] + "/ndsm.tif",
"rb") as f_ndsm:
ndsm = f_ndsm.read()
dispatcher.putResult(item, finalMask, dsm, ndsm)
dispatcher.updateWorkerStatus(WORKERID, '1')
except:
while True:
#Try to reconnect to dispatcher
try:
print("Dispatcher not found. Reconnecting...")
dispatcher._pyroReconnect()
#Can't connect -> Sleep then retry again
except Exception:
time.sleep(1)
#Reconnecting succesful
else:
dispatcher.putResult(item, finalMask, dsm,
ndsm)
dispatcher.updateWorkerStatus(WORKERID, '1')
print("Connected to dispatcher.")
break
# dispatcher.putResult(item)
#set taskbar's icon to green -> available
taskbar.set_icon(TRAY_ICON_GREEN)
taskbar.balloon_free()
