def main():
sys.setrecursionlimit(5000)
CEGraph = nx.DiGraph()
prunedGraph = nx.DiGraph()
MCCList = []
MCSList = []
MCSMCCNodesList = []
allMCSsList = []
allCETRMMList = []
# For GrADs
subprocess.call('export DISPLAY=:0.0', shell=True)
# For first time working with the raw MERG zipped files
# rawMERG = "/directory/to/the/raw/MERGfiles"
# utils.preprocessing_merg(rawMERG)
# ---------------------------------------------------------------------------------
# ---------------------------------- user inputs --------------------------------------
userVariables = variables.UserVariables(useJSON=False)
graphVariables = variables.define_graph_variables()
# ---------------------------------- end user inputs --------------------------------------
# Create main directory and file structure for storing intel
userVariables.DIRS['mainDirStr'] = iomethods.create_main_directory(userVariables.DIRS['mainDirStr'])
TRMMCEdirName = userVariables.DIRS['mainDirStr']+'/TRMMnetcdfCEs'
CEdirName = userVariables.DIRS['mainDirStr']+'/MERGnetcdfCEs'
# -------------------------------------------------------------------------------------------------
# Getting started. Make it so number one!
print ("-"*80)
print "\t\t Starting the MCCSearch Analysis "
print ("-"*80)
print "\n -------------- Reading MERG Data ----------"
mergImgs, timeList, LAT, LON, userVariables = iomethods.read_data('ch4', 'latitude', 'longitude', userVariables)
print "\n -------------- findCloudElements ----------"
CEGraph, _ = mccSearch.find_cloud_elements(mergImgs, timeList, LAT, LON, userVariables, graphVariables, userVariables.DIRS['TRMMdirName'])
# theList = CEGraph.successors(node)
# If the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
# CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
# allCETRMMList=mccSearch.findPrecipRate(DIRS['TRMMdirName'],timeList)
# ----------------------------------------------------------------------------------------------
print "\n -------------- findCloudClusters ----------"
prunedGraph = mccSearch.find_cloud_clusters(CEGraph, userVariables, graphVariables)
print "\n -------------- findMCCs ----------"
MCCList, MCSList = mccSearch.find_MCC(prunedGraph, userVariables, graphVariables)
# Now ready to perform various calculations/metrics
print ("-"*80)
print "\n -------------- METRICS ----------"
print ("-"*80)
# Some calculations/metrics that work that work
print "creating the MCC userfile ", metrics.create_text_file(MCCList, 1, userVariables, graphVariables)
print "creating the MCS userfile ", metrics.create_text_file(MCSList, 2, userVariables, graphVariables)
plot_menu(MCCList, MCSList, userVariables.DIRS)
# Let's get outta here! Engage!
print ("-"*80)
def read_vars(userVariables):
# for GrADs
subprocess.call('export DISPLAY=:0.0', shell=True)
graphVariables = variables.define_graph_variables()
# ---------------------------------- end user inputs --------------------------------------
# Checks that inputs are ok
try:
if not os.path.exists(userVariables.DIRS['TRMMdirName']):
print "Error: TRMM invalid path!"
userVariables.DIRS['TRMMdirName'] = raw_input(
"> Please enter the location to the raw TRMM netCDF files: \n")
except:
pass
try:
if not os.path.exists(userVariables.DIRS['CEoriDirName']):
print "Error! MERG invalid path!"
userVariables.DIRS['CEoriDirName'] = raw_input(
"> Please enter the directory to the MERG netCDF files: \n")
except:
print "..."
# Check validity of time
while utils.valid_date(userVariables.startDateTime) != True:
print "Invalid time entered for startDateTime!"
while utils.valid_date(userVariables.endDateTime) != True:
print "Invalid time entered for endDateTime!"
# Check if all the files exisits in the MERG and TRMM directories entered
test, _ = check_for_files(userVariables.DIRS['TRMMdirName'],
userVariables.startDateTime,
userVariables.endDateTime, 3, 'hour')
if test is False:
print "Error with files in the TRMM directory entered. Please check your files before restarting. "
return
test, userVariables.filelist = check_for_files(
userVariables.DIRS['CEoriDirName'], userVariables.startDateTime,
userVariables.endDateTime, 1, 'hour')
if test is False:
print "Error with files in the original MERG directory entered. Please check your files before restarting. "
return
# Create main directory and file structure for storing intel
userVariables.DIRS['mainDirStr'] = create_main_directory(
userVariables.DIRS['mainDirStr'])
TRMMCEdirName = userVariables.DIRS['mainDirStr'] + '/TRMMnetcdfCEs'
CEdirName = userVariables.DIRS['mainDirStr'] + '/MERGnetcdfCEs'
return graphVariables
def read_vars(userVariables):
# for GrADs
subprocess.call('export DISPLAY=:0.0', shell=True)
graphVariables = variables.define_graph_variables()
# ---------------------------------- end user inputs --------------------------------------
# Checks that inputs are ok
try:
if not os.path.exists(userVariables.DIRS['TRMMdirName']):
print "Error: TRMM invalid path!"
userVariables.DIRS['TRMMdirName'] = raw_input("> Please enter the location to the raw TRMM netCDF files: \n")
except:
pass
try:
if not os.path.exists(userVariables.DIRS['CEoriDirName']):
print "Error! MERG invalid path!"
userVariables.DIRS['CEoriDirName'] = raw_input("> Please enter the directory to the MERG netCDF files: \n")
except:
print "..."
# Check validity of time
while utils.valid_date(userVariables.startDateTime) != True:
print "Invalid time entered for startDateTime!"
while utils.valid_date(userVariables.endDateTime) != True:
print "Invalid time entered for endDateTime!"
# Check if all the files exisits in the MERG and TRMM directories entered
test, _ = check_for_files(userVariables.DIRS['TRMMdirName'], userVariables.startDateTime, userVariables.endDateTime, 3, 'hour')
if test is False:
print "Error with files in the TRMM directory entered. Please check your files before restarting. "
return
test, userVariables.filelist = check_for_files(userVariables.DIRS['CEoriDirName'], userVariables.startDateTime, userVariables.endDateTime, 1, 'hour')
if test is False:
print "Error with files in the original MERG directory entered. Please check your files before restarting. "
return
# Create main directory and file structure for storing intel
userVariables.DIRS['mainDirStr'] = create_main_directory(userVariables.DIRS['mainDirStr'])
TRMMCEdirName = userVariables.DIRS['mainDirStr']+'/TRMMnetcdfCEs'
CEdirName = userVariables.DIRS['mainDirStr']+'/MERGnetcdfCEs'
return graphVariables
def main():
sys.setrecursionlimit(5000)
CEGraph = nx.DiGraph()
prunedGraph = nx.DiGraph()
MCCList = []
MCSList = []
MCSMCCNodesList = []
allMCSsList = []
allCETRMMList = []
DIRS = {}
# For GrADs
subprocess.call('export DISPLAY=:0.0', shell=True)
userVariables = variables.define_user_variables(useJSON=True)
graphVariables = variables.define_graph_variables()
# Create main directory and file structure for storing intel
userVariables.DIRS['mainDirStr'] = iomethods.create_main_directory(userVariables.DIRS['mainDirStr'])
TRMMCEdirName = userVariables.DIRS['mainDirStr']+'/TRMMnetcdfCEs'
CEdirName = userVariables.DIRS['mainDirStr']+'/MERGnetcdfCEs'
# Let's go!
# ----------------------------------CORE GTG STEPS------------------------------------------------
print "\n -------------- Read MERG Data ----------"
mergImgs, timeList, LAT, LON, userVariables = iomethods.read_data('ch4', 'latitude', 'longitude', userVariables)
print ("-"*80)
print "\n -------------- TESTING findCloudElements ----------"
CEGraph, _ = mccSearch.find_cloud_elements(mergImgs, timeList, LAT, LON, userVariables, graphVariables, userVariables.DIRS['TRMMdirName'])
# #********* OR *******
# CEGraph = mccSearch.find_cloud_elements(mergImgs,timeList,LAT, LON, userVariables, graphVariables)
# allCETRMMList = mccSearch.find_precip_rate(userVariables.DIRS['TRMMdirName'],timeList)
# #********************
print ("-"*80)
print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
print ("-"*80)
print "\n -------------- TESTING findCloudClusters ----------"
prunedGraph = mccSearch.find_cloud_clusters(CEGraph, userVariables, graphVariables)
print ("-"*80)
print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
print ("-"*80)
print "\n -------------- TESTING findMCCs ----------"
MCCList, MCSList = mccSearch.find_MCC(prunedGraph, userVariables, graphVariables)
print ("-"*80)
print "MCC List has been acquired ", len(MCCList)
print "MCS List has been acquired ", len(MCSList)
print ("-"*80)
# ---------------------------------END CORE GTG STEPS----------------------------------------------
# Now ready to perform various calculations/metrics
print "\n -------------- TESTING METRICS ----------"
# Some calculations/metrics that work that work
# print "creating the MCC userfile ", metrics.createTextFile(MCCList,1)
# print "creating the MCS userfile ", metrics.createTextFile(MCSList,2)
# MCCTimes, tdelta = metrics.temporalAndAreaInfoMetric(MCCList)
# print "number of MCCs is: ", metrics.numberOfFeatures(MCCList)
# print "longest duration is: ", metrics.longestDuration(MCCTimes), "hrs"
# print "shortest duration is: ", metrics.shortestDuration(MCCTimes), "hrs"
# print "Average duration is: ", metrics.averageDuration(MCCTimes), "hrs"
# print "Average size is: ", metrics.averageFeatureSize(MCCList), "km^2"
# Some plots that work
# plotting.plotAccTRMM(MCCList)
# plotting.displayPrecip(MCCList)
# plotting.plotAccuInTimeRange('yyyy-mm-dd_hh:mm:ss', 'yyyy-mm-dd_hh:mm:ss')
# plotting.displaySize(MCCList)
# plotting.displayPrecip(MCCList)
# plotting.plotHistogram(MCCList)
#
print ("-"*80)
def main():
sys.setrecursionlimit(5000)
CEGraph = nx.DiGraph()
prunedGraph = nx.DiGraph()
MCCList = []
MCSList = []
MCSMCCNodesList = []
allMCSsList = []
allCETRMMList = []
DIRS = {}
# For GrADs
subprocess.call('export DISPLAY=:0.0', shell=True)
# For first time working with the raw MERG zipped files
# rawMERG = "/directory/to/the/raw/MERGfiles"
# utils.preprocessing_merg(rawMERG)
# ---------------------------------------------------------------------------------
# ---------------------------------- user inputs --------------------------------------
userVariables = variables.UserVariables(useJSON=False)
graphVariables = variables.define_graph_variables()
# ---------------------------------- end user inputs --------------------------------------
# Create main directory and file structure for storing intel
userVariables.DIRS['mainDirStr'] = iomethods.create_main_directory(userVariables.DIRS['mainDirStr'])
TRMMCEdirName = userVariables.DIRS['mainDirStr']+'/TRMMnetcdfCEs'
CEdirName = userVariables.DIRS['mainDirStr']+'/MERGnetcdfCEs'
unittestFile = open(userVariables.DIRS['mainDirStr']+'/textFiles/unittestResults.txt', 'wb')
unittestFile.write("\n Timing results for "+userVariables.startDateTime+" to "+userVariables.endDateTime)
# Let's go!
# Time how long it takes to complete reading in the data
print "\n Start the timer "
startTime = time.time()
print "\n -------------- Read MERG Data ----------"
print "\n Start the timer for the data ingest process"
readMergStart = time.time()
mergImgs, timeList, LAT, LON, userVariables = iomethods.read_data('ch4', 'latitude', 'longitude', userVariables)
readMergEnd = time.time()
print "\n End the timer for the data ingest process"
print "\n Total time to complete data ingest is %g seconds" % (readMergEnd - readMergStart)
unittestFile.write("\n 1. Total time to complete data ingest is %g seconds" % (readMergEnd - readMergStart))
print ("-"*80)
print "\n -------------- TESTING findCloudElements ----------"
print "\n Start the timer for findCloudElements process"
findCEsStart = time.time()
print "\n Using both MERG and TRMM simultaneously "
# CEGraph = mccSearch.find_cloud_elements(mergImgs,timeList,userVariables.DIRS['mainDirStr'], LAT,LON,userVariables, graphVariables, userVariables.DIRS['TRMMdirName'])
CEGraph, _ = mccSearch.find_cloud_elements(mergImgs, timeList, LAT, LON, userVariables, graphVariables, userVariables. DIRS['TRMMdirName'])
findCEsEnd = time.time()
print "\n Number of cloud elements found is: ", CEGraph.number_of_nodes()
print "\n End the timer for findCloudElements process"
print "\n Total time to complete finding cloud elements is %g seconds" % (findCEsEnd - findCEsStart)
unittestFile.write("\n 2. Total time to complete finding cloud elements is %g seconds" % (findCEsEnd - findCEsStart))
# #********* OR *******
# # #timing each separately
# CEGraph = mccSearch.find_cloud_elements(mergImgs, timeList, DIRS['mainDirStr'], LAT, LON)
# findCEsEnd = time.time()
# print "\n End the timer for findCloudElements process using MERG only"
# print "\n Total time to complete finding cloud elements in MERG only is %g seconds" % (findCEsEnd - findCEsStart)
# unittestFile.write("\n Total time to complete finding cloud elements in MERG only is %g seconds" % (findCEsEnd - findCEsStart))
# # *** TRMM DATA SET TIMINGS ***
# print "\n Start the timer for findCloudElements process using TRMM only"
# findCETRMMStart = time.time()
# allCETRMMList = mccSearch.find_precip_rate(DIRS['TRMMdirName'],timeList)
# findCETRMMEnd = time.time()
# print "\n End the timer for findCloudElements process using TRMM only"
# print "\n Total time to complete finding cloud elements in TRMM only is %g seconds"%(findCETRMMEnd - findCETRMMStart)
# unittestFile.write("\n Total time to complete finding cloud elements in TRMM only is %g seconds"%(findCETRMMEnd - findCETRMMStart))
# print "\n Number of cloud elements found is: ", CEGraph.number_of_nodes()
# print "\n Total time to complete finding cloud elements is %g seconds"%(findCETRMMEnd - findCEsStart)
# unittestFile.write("\n Total time to complete finding cloud elements is %g seconds" % (findCETRMMEnd - findCEsStart))
# print ("-"*80)
print "\n -------------- TESTING findCloudClusters ----------"
print "\n Start the timer for findCloudClusters process"
findCloudClustersStart = time.time()
prunedGraph = mccSearch.find_cloud_clusters(CEGraph, userVariables, graphVariables)
print "The number of nodes in the prunedGraph is: ", prunedGraph.number_of_nodes()
findCloudClustersEnd = time.time()
print "\n End the timer for the findCloudClusters process"
print "\n Total time to complete finding the cloud clusters is %g seconds" % (findCloudClustersEnd - findCloudClustersStart)
unittestFile.write("\n 3. Total time to complete finding the cloud clusters is %g seconds" % (findCloudClustersEnd - findCloudClustersStart))
print "\n The CEGraph nodes are: %s " % CEGraph.nodes()
unittestFile.write("\n The CEGraph nodes are: %s " % CEGraph.nodes())
print "\n The pruned graph nodes are: %s " % prunedGraph.nodes()
unittestFile.write("\n\n The pruned graph nodes are: %s " % prunedGraph.nodes())
print ("-"*80)
print "\n -------------- TESTING findMCCs ----------"
print "\n Start the timer for the findMCCs process"
findMCCStart = time.time()
MCCList, MCSList = mccSearch.find_MCC(prunedGraph, userVariables, graphVariables)
print "\n MCC List has been acquired ", len(MCCList)
print "\n MCS List has been acquired ", len(MCSList)
findMCCEnd = time.time()
print "\n End the timer for the findMCCs process"
print "\n Total time to complete finding the MCCs is %g seconds" % (findMCCEnd - findMCCStart)
unittestFile.write("\n 4. Total time to complete finding the MCCs is %g seconds" % (findMCCEnd - findMCCStart))
print ("-"*80)
# End the timer
endTime = time.time()
print ("*"*80)
print "\n The entire evaluation took %g seconds to complete" % (endTime - startTime)
unittestFile.write("\n The entire evaluation took %g seconds to complete" % (endTime - startTime))
unittestFile.write("\n ----------------------------------------------------------------")
unittestFile.write("\n Number of cloud elements found is: %d" % CEGraph.number_of_nodes())
unittestFile.write("\n Number of edges (with the cloud elements) found is: %d" % CEGraph.number_of_edges())
unittestFile.write("\n The number of nodes in the prunedGraph is: %d" % prunedGraph.number_of_nodes())
unittestFile.write("\n The number of edges (with nodes) in the prunedGraph is: %d" % prunedGraph.number_of_edges())
unittestFile.write("\n MCC List has been acquired %d" % len(MCCList))
unittestFile.write("\n MCS List has been acquired %d" % len(MCSList))
unittestFile.write("\n The CEGraph nodes are: %s" % CEGraph.nodes())
unittestFile.write("\n The prunedGraph nodes are: %s" % prunedGraph.nodes())
print ("-"*80)
# TODO: report the domain
unittestFile.close()
def main():
sys.setrecursionlimit(5000)
CEGraph = nx.DiGraph()
prunedGraph = nx.DiGraph()
MCCList =[]
MCSList=[]
MCSMCCNodesList =[]
allMCSsList =[]
allCETRMMList =[]
DIRS={}
#for GrADs
subprocess.call('export DISPLAY=:0.0', shell=True)
#for first time working with the raw MERG zipped files
# rawMERG = "/directory/to/the/raw/MERGfiles"
# utils.preprocessing_merg(rawMERG)
# ---------------------------------------------------------------------------------
# ---------------------------------- user inputs --------------------------------------
# DIRS['mainDirStr'] = "/Users/youssefbiaz/Documents/USC/,2015-3Fall'15/5CSCI401/grab-tag-graph/baselineTimings/output/baselineTimings"#"/directory/to/where/to/store/outputs"
#DIRS['TRMMdirName'] = "/Users/youssefbiaz/Documents/USC/,2015-3Fall'15/5CSCI401/grab-tag-graph/baselineTimings/datadir/TRMM"#"/directory/to/the/TRMM/netCDF/files"
#DIRS['CEoriDirName'] = "/Users/youssefbiaz/Documents/USC/,2015-3Fall'15/5CSCI401/grab-tag-graph/baselineTimings/datadir/MERG"#"/directory/to/the/MERG/netCDF/files"
#get the dates for analysis
#startDateTime = "200908310000" #"yyyymmddhrmm"
#endDateTime = "200908312100"
userVariables = variables.define_user_variables()
graphVariables = variables.define_graph_variables()
# ---------------------------------- end user inputs --------------------------------------
# Checks that inputs are ok
try:
if not os.path.exists(userVariables.DIRS['CEoriDirName']):
print "Error! MERG invalid path!"
userVariables.DIRS['CEoriDirName'] = raw_input("> Please enter the directory to the MERG netCDF files: \n")
except:
print "..."
try:
if not os.path.exists(userVariables.DIRS['TRMMdirName']):
print "Error: TRMM invalid path!"
userVariables.DIRS['TRMMdirName'] = raw_input("> Please enter the location to the raw TRMM netCDF files: \n")
except:
pass
try:
if not os.path.exists(userVariables.DIRS['CEoriDirName']):
print "Error! MERG invalid path!"
userVariables.DIRS['CEoriDirName'] = raw_input("> Please enter the directory to the MERG netCDF files: \n")
except:
print "..."
#check validity of time
while utils.valid_date(userVariables.startDateTime) != True:
print "Invalid time entered for startDateTime!"
while utils.valid_date(userVariables.endDateTime) != True:
print "Invalid time entered for endDateTime!"
#check if all the files exisits in the MERG and TRMM directories entered
test,_ = iomethods.check_for_files(userVariables.DIRS['TRMMdirName'], userVariables.startDateTime, userVariables.endDateTime, 3, 'hour')
if test == False:
print "Error with files in the TRMM directory entered. Please check your files before restarting. "
return
#test,filelist = iomethods.check_for_files(startDateTime, endDateTime, DIRS['CEoriDirName'],1)
test,filelist = iomethods.check_for_files(userVariables.DIRS['CEoriDirName'], userVariables.startDateTime, userVariables.endDateTime, 1, 'hour')
if test == False:
print "Error with files in the original MERG directory entered. Please check your files before restarting. "
return
# end checks
# create main directory and file structure for storing intel
userVariables.DIRS['mainDirStr'] = iomethods.create_main_directory(userVariables.DIRS['mainDirStr'])
TRMMCEdirName = userVariables.DIRS['mainDirStr']+'/TRMMnetcdfCEs'
CEdirName = userVariables.DIRS['mainDirStr']+'/MERGnetcdfCEs'
unittestFile = open(userVariables.DIRS['mainDirStr']+'/unittestResults.txt','wb')
unittestFile.write("\n Timing results for "+userVariables.startDateTime+" to "+userVariables.endDateTime)
#let's go!
# time how long it takes to complete reading in the data
print "\n Start the timer "
starttime = time.time()
print "\n -------------- Read MERG Data ----------"
print "\n Start the timer for the data ingest process"
readMergStart = time.time()
mergImgs, timeList, LAT, LON = iomethods.read_data(userVariables.DIRS['CEoriDirName'],'ch4','latitude','longitude', userVariables,filelist)
readMergEnd = time.time()
print "\n End the timer for the data ingest process"
print "\n Total time to complete data ingest is %g seconds"%(readMergEnd - readMergStart)
unittestFile.write("\n 1. Total time to complete data ingest is %g seconds"%(readMergEnd - readMergStart))
print ("-"*80)
print "\n -------------- TESTING findCloudElements ----------"
print "\n Start the timer for findCloudElements process"
findCEsStart = time.time()
print "\n Using both MERG and TRMM simultaneously "
CEGraph = mccSearch.find_cloud_elements(mergImgs,timeList,DIRS['mainDirStr'], LAT,LON,DIRS['TRMMdirName'])
findCEsEnd = time.time()
print "\n End the timer for findCloudElements process using MERG only"
print "\n Total time to complete finding cloud elements in MERG only is %g seconds"%(findCEsEnd - findCEsStart)
unittestFile.write("\n Total time to complete finding cloud elements in MERG only is %g seconds"%(findCEsEnd - findCEsStart))
print "\n Number of cloud elements found is: ", CEGraph.number_of_nodes()
print "\n End the timer for findCloudElements process"
print "\n Total time to complete finding cloud elements is %g seconds"%(findCEsEnd - findCEsStart)
unittestFile.write("\n 2. Total time to complete finding cloud elements is %g seconds"%(findCEsEnd - findCEsStart))
print "\n The CEGraph nodes are: %s "%CEGraph.nodes()
unittestFile.write("\n The CEGraph nodes are: %s "%CEGraph.nodes())
print ("-"*80)
# #********* OR *******
# #timing each separately
# CEGraph = mccSearch.find_cloud_elements(mergImgs,timeList,DIRS['mainDirStr'], LAT,LON)
# findCEsEnd = time.time()
# print "\n End the timer for findCloudElements process using MERG only"
# print "\n Total time to complete finding cloud elements in MERG only is %g seconds"%(findCEsEnd - findCEsStart)
# unittestFile.write("\n Total time to complete finding cloud elements in MERG only is %g seconds"%(findCEsEnd - findCEsStart))
# print "\n Start the timer for findCloudElements process using TRMM only"
# findCETRMMStart = time.time()
# allCETRMMList = mccSearch.find_precip_rate(DIRS['TRMMdirName'],timeList)
# findCETRMMEnd = time.time()
# print "\n End the timer for findCloudElements process using TRMM only"
# print "\n Total time to complete finding cloud elements in TRMM only is %g seconds"%(findCEsEnd - findCEsStart)
# unittestFile.write("\n Total time to complete finding cloud elements in TRMM only is %g seconds"%(findCEsEnd - findCEsStart))
# print "\n Number of cloud elements found is: ", CEGraph.number_of_nodes()
# print "\n Total time to complete finding cloud elements is %g seconds"%(findCETRMMEnd - findCEsStart)
# unittestFile.write("\n Total time to complete finding cloud elements is %g seconds"%(findCETRMMEnd - findCEsStart))
# print "\n The CEGraph nodes are: %s "%CEGraph.nodes()
# unittestFile.write("\n The CEGraph nodes are: %s "%CEGraph.nodes())
# print ("-"*80)
print "\n -------------- TESTING findCloudClusters ----------"
print "\n Start the timer for findCloudClusters process"
findCloudClustersStart = time.time()
prunedGraph = mccSearch.find_cloud_clusters(CEGraph,userVariables,graphVariables)
print "The number of nodes in the prunedGraph is: ", prunedGraph.number_of_nodes()
findCloudClustersEnd = time.time()
print "\n End the timer for the findCloudClusters process"
print "\n Total time to complete finding the cloud clusters is %g seconds"%(findCloudClustersEnd - findCloudClustersStart)
unittestFile.write("\n 3. Total time to complete finding the cloud clusters is %g seconds"%(findCloudClustersEnd - findCloudClustersStart))
unittestFile.write("\n The prunedGraph nodes are: %s "%prunedGraph.nodes())
print "\n The prunedGraph nodes are: %s "%prunedGraph.nodes()
print ("-"*80)
print "\n -------------- TESTING findMCCs ----------"
print "\n Start the timer for the findMCCs process"
findMCCStart = time.time()
MCCList,MCSList = mccSearch.find_MCC(prunedGraph,userVariables,graphVariables)
print "\n MCC List has been acquired ", len(MCCList)
print "\n MCS List has been acquired ", len(MCSList)
findMCCEnd = time.time()
print "\n End the timer for the findMCCs process"
print "\n Total time to complete finding the MCCs is %g seconds"%(findMCCEnd - findMCCStart)
unittestFile.write("\n 4. Total time to complete finding the MCCs is %g seconds"%(findMCCEnd - findMCCStart))
print ("-"*80)
#end the timer
endtime = time.time()
print ("*"*80)
print "\n The entire evaluation took %g seconds to complete" %(endtime - starttime)
unittestFile.write("\n The entire evaluation took %g seconds to complete" %(endtime - starttime))
unittestFile.write("\n ----------------------------------------------------------------")
unittestFile.write("\n Number of cloud elements found is: %d"%CEGraph.number_of_nodes())
unittestFile.write("\n Number of edges (with the cloud elements) found is: %d"%CEGraph.number_of_edges())
unittestFile.write("\n The number of nodes in the prunedGraph is: %d" %prunedGraph.number_of_nodes())
unittestFile.write("\n The number of edges (with nodes) in the prunedGraph is: %d" %prunedGraph.number_of_edges())
unittestFile.write("\n MCC List has been acquired %d" %len(MCCList))
unittestFile.write("\n MCS List has been acquired %d" %len(MCSList))
# TODO: report the domain
unittestFile.close()
def main():
sys.setrecursionlimit(5000)
CEGraph = nx.DiGraph()
prunedGraph = nx.DiGraph()
MCCList = []
MCSList = []
MCSMCCNodesList = []
allMCSsList = []
allCETRMMList = []
DIRS = {}
# For GrADs
subprocess.call('export DISPLAY=:0.0', shell=True)
userVariables = variables.define_user_variables(useJSON=True)
graphVariables = variables.define_graph_variables()
# Create main directory and file structure for storing intel
userVariables.DIRS['mainDirStr'] = iomethods.create_main_directory(
userVariables.DIRS['mainDirStr'])
TRMMCEdirName = userVariables.DIRS['mainDirStr'] + '/TRMMnetcdfCEs'
CEdirName = userVariables.DIRS['mainDirStr'] + '/MERGnetcdfCEs'
# Let's go!
# ----------------------------------CORE GTG STEPS------------------------------------------------
print "\n -------------- Read MERG Data ----------"
mergImgs, timeList, LAT, LON, userVariables = iomethods.read_data(
'ch4', 'latitude', 'longitude', userVariables)
print("-" * 80)
print "\n -------------- TESTING findCloudElements ----------"
CEGraph, _ = mccSearch.find_cloud_elements(
mergImgs, timeList, LAT, LON, userVariables, graphVariables,
userVariables.DIRS['TRMMdirName'])
# #********* OR *******
# CEGraph = mccSearch.find_cloud_elements(mergImgs,timeList,LAT, LON, userVariables, graphVariables)
# allCETRMMList = mccSearch.find_precip_rate(userVariables.DIRS['TRMMdirName'],timeList)
# #********************
print("-" * 80)
print "number of nodes in CEGraph is: ", CEGraph.number_of_nodes()
print("-" * 80)
print "\n -------------- TESTING findCloudClusters ----------"
prunedGraph = mccSearch.find_cloud_clusters(CEGraph, userVariables,
graphVariables)
print("-" * 80)
print "number of nodes in prunedGraph is: ", prunedGraph.number_of_nodes()
print("-" * 80)
print "\n -------------- TESTING findMCCs ----------"
MCCList, MCSList = mccSearch.find_MCC(prunedGraph, userVariables,
graphVariables)
print("-" * 80)
print "MCC List has been acquired ", len(MCCList)
print "MCS List has been acquired ", len(MCSList)
print("-" * 80)
# ---------------------------------END CORE GTG STEPS----------------------------------------------
# Now ready to perform various calculations/metrics
print "\n -------------- TESTING METRICS ----------"
# Some calculations/metrics that work that work
# print "creating the MCC userfile ", metrics.createTextFile(MCCList,1)
# print "creating the MCS userfile ", metrics.createTextFile(MCSList,2)
# MCCTimes, tdelta = metrics.temporalAndAreaInfoMetric(MCCList)
# print "number of MCCs is: ", metrics.numberOfFeatures(MCCList)
# print "longest duration is: ", metrics.longestDuration(MCCTimes), "hrs"
# print "shortest duration is: ", metrics.shortestDuration(MCCTimes), "hrs"
# print "Average duration is: ", metrics.averageDuration(MCCTimes), "hrs"
# print "Average size is: ", metrics.averageFeatureSize(MCCList), "km^2"
# Some plots that work
# plotting.plotAccTRMM(MCCList)
# plotting.displayPrecip(MCCList)
# plotting.plotAccuInTimeRange('yyyy-mm-dd_hh:mm:ss', 'yyyy-mm-dd_hh:mm:ss')
# plotting.displaySize(MCCList)
# plotting.displayPrecip(MCCList)
# plotting.plotHistogram(MCCList)
#
print("-" * 80)
def main():
sys.setrecursionlimit(5000)
CEGraph = nx.DiGraph()
prunedGraph = nx.DiGraph()
MCCList = []
MCSList = []
MCSMCCNodesList = []
allMCSsList = []
allCETRMMList = []
# For GrADs
subprocess.call('export DISPLAY=:0.0', shell=True)
# For first time working with the raw MERG zipped files
# rawMERG = "/directory/to/the/raw/MERGfiles"
# utils.preprocessing_merg(rawMERG)
# ---------------------------------------------------------------------------------
# ---------------------------------- user inputs --------------------------------------
userVariables = variables.UserVariables(useJSON=False)
graphVariables = variables.define_graph_variables()
# ---------------------------------- end user inputs --------------------------------------
# Create main directory and file structure for storing intel
userVariables.DIRS['mainDirStr'] = iomethods.create_main_directory(
userVariables.DIRS['mainDirStr'])
TRMMCEdirName = userVariables.DIRS['mainDirStr'] + '/TRMMnetcdfCEs'
CEdirName = userVariables.DIRS['mainDirStr'] + '/MERGnetcdfCEs'
# -------------------------------------------------------------------------------------------------
# Getting started. Make it so number one!
print("-" * 80)
print "\t\t Starting the MCCSearch Analysis "
print("-" * 80)
print "\n -------------- Reading MERG Data ----------"
mergImgs, timeList, LAT, LON, userVariables = iomethods.read_data(
'ch4', 'latitude', 'longitude', userVariables)
print "\n -------------- findCloudElements ----------"
CEGraph, _ = mccSearch.find_cloud_elements(
mergImgs, timeList, LAT, LON, userVariables, graphVariables,
userVariables.DIRS['TRMMdirName'])
# theList = CEGraph.successors(node)
# If the TRMMdirName wasnt entered for whatever reason, you can still get the TRMM data this way
# CEGraph = mccSearch.findCloudElements(mergImgs,timeList)
# allCETRMMList=mccSearch.findPrecipRate(DIRS['TRMMdirName'],timeList)
# ----------------------------------------------------------------------------------------------
print "\n -------------- findCloudClusters ----------"
prunedGraph = mccSearch.find_cloud_clusters(CEGraph, userVariables,
graphVariables)
print "\n -------------- findMCCs ----------"
MCCList, MCSList = mccSearch.find_MCC(prunedGraph, userVariables,
graphVariables)
# Now ready to perform various calculations/metrics
print("-" * 80)
print "\n -------------- METRICS ----------"
print("-" * 80)
# Some calculations/metrics that work that work
print "creating the MCC userfile ", metrics.create_text_file(
MCCList, 1, userVariables, graphVariables)
print "creating the MCS userfile ", metrics.create_text_file(
MCSList, 2, userVariables, graphVariables)
plot_menu(MCCList, MCSList, userVariables.DIRS)
# Let's get outta here! Engage!
print("-" * 80)
def main():
sys.setrecursionlimit(5000)
CEGraph = nx.DiGraph()
prunedGraph = nx.DiGraph()
MCCList = []
MCSList = []
MCSMCCNodesList = []
allMCSsList = []
allCETRMMList = []
DIRS = {}
# For GrADs
subprocess.call('export DISPLAY=:0.0', shell=True)
# For first time working with the raw MERG zipped files
# rawMERG = "/directory/to/the/raw/MERGfiles"
# utils.preprocessing_merg(rawMERG)
# ---------------------------------------------------------------------------------
# ---------------------------------- user inputs --------------------------------------
userVariables = variables.UserVariables(useJSON=False)
graphVariables = variables.define_graph_variables()
# ---------------------------------- end user inputs --------------------------------------
# Create main directory and file structure for storing intel
userVariables.DIRS['mainDirStr'] = iomethods.create_main_directory(
userVariables.DIRS['mainDirStr'])
TRMMCEdirName = userVariables.DIRS['mainDirStr'] + '/TRMMnetcdfCEs'
CEdirName = userVariables.DIRS['mainDirStr'] + '/MERGnetcdfCEs'
unittestFile = open(
userVariables.DIRS['mainDirStr'] + '/textFiles/unittestResults.txt',
'wb')
unittestFile.write("\n Timing results for " + userVariables.startDateTime +
" to " + userVariables.endDateTime)
# Let's go!
# Time how long it takes to complete reading in the data
print "\n Start the timer "
startTime = time.time()
print "\n -------------- Read MERG Data ----------"
print "\n Start the timer for the data ingest process"
readMergStart = time.time()
mergImgs, timeList, LAT, LON, userVariables = iomethods.read_data(
'ch4', 'latitude', 'longitude', userVariables)
readMergEnd = time.time()
print "\n End the timer for the data ingest process"
print "\n Total time to complete data ingest is %g seconds" % (
readMergEnd - readMergStart)
unittestFile.write(
"\n 1. Total time to complete data ingest is %g seconds" %
(readMergEnd - readMergStart))
print("-" * 80)
print "\n -------------- TESTING findCloudElements ----------"
print "\n Start the timer for findCloudElements process"
findCEsStart = time.time()
print "\n Using both MERG and TRMM simultaneously "
# CEGraph = mccSearch.find_cloud_elements(mergImgs,timeList,userVariables.DIRS['mainDirStr'], LAT,LON,userVariables, graphVariables, userVariables.DIRS['TRMMdirName'])
CEGraph, _ = mccSearch.find_cloud_elements(
mergImgs, timeList, LAT, LON, userVariables, graphVariables,
userVariables.DIRS['TRMMdirName'])
findCEsEnd = time.time()
print "\n Number of cloud elements found is: ", CEGraph.number_of_nodes()
print "\n End the timer for findCloudElements process"
print "\n Total time to complete finding cloud elements is %g seconds" % (
findCEsEnd - findCEsStart)
unittestFile.write(
"\n 2. Total time to complete finding cloud elements is %g seconds" %
(findCEsEnd - findCEsStart))
# #********* OR *******
# # #timing each separately
# CEGraph = mccSearch.find_cloud_elements(mergImgs, timeList, DIRS['mainDirStr'], LAT, LON)
# findCEsEnd = time.time()
# print "\n End the timer for findCloudElements process using MERG only"
# print "\n Total time to complete finding cloud elements in MERG only is %g seconds" % (findCEsEnd - findCEsStart)
# unittestFile.write("\n Total time to complete finding cloud elements in MERG only is %g seconds" % (findCEsEnd - findCEsStart))
# # *** TRMM DATA SET TIMINGS ***
# print "\n Start the timer for findCloudElements process using TRMM only"
# findCETRMMStart = time.time()
# allCETRMMList = mccSearch.find_precip_rate(DIRS['TRMMdirName'],timeList)
# findCETRMMEnd = time.time()
# print "\n End the timer for findCloudElements process using TRMM only"
# print "\n Total time to complete finding cloud elements in TRMM only is %g seconds"%(findCETRMMEnd - findCETRMMStart)
# unittestFile.write("\n Total time to complete finding cloud elements in TRMM only is %g seconds"%(findCETRMMEnd - findCETRMMStart))
# print "\n Number of cloud elements found is: ", CEGraph.number_of_nodes()
# print "\n Total time to complete finding cloud elements is %g seconds"%(findCETRMMEnd - findCEsStart)
# unittestFile.write("\n Total time to complete finding cloud elements is %g seconds" % (findCETRMMEnd - findCEsStart))
# print ("-"*80)
print "\n -------------- TESTING findCloudClusters ----------"
print "\n Start the timer for findCloudClusters process"
findCloudClustersStart = time.time()
prunedGraph = mccSearch.find_cloud_clusters(CEGraph, userVariables,
graphVariables)
print "The number of nodes in the prunedGraph is: ", prunedGraph.number_of_nodes(
)
findCloudClustersEnd = time.time()
print "\n End the timer for the findCloudClusters process"
print "\n Total time to complete finding the cloud clusters is %g seconds" % (
findCloudClustersEnd - findCloudClustersStart)
unittestFile.write(
"\n 3. Total time to complete finding the cloud clusters is %g seconds"
% (findCloudClustersEnd - findCloudClustersStart))
print "\n The CEGraph nodes are: %s " % CEGraph.nodes()
unittestFile.write("\n The CEGraph nodes are: %s " % CEGraph.nodes())
print "\n The pruned graph nodes are: %s " % prunedGraph.nodes()
unittestFile.write("\n\n The pruned graph nodes are: %s " %
prunedGraph.nodes())
print("-" * 80)
print "\n -------------- TESTING findMCCs ----------"
print "\n Start the timer for the findMCCs process"
findMCCStart = time.time()
MCCList, MCSList = mccSearch.find_MCC(prunedGraph, userVariables,
graphVariables)
print "\n MCC List has been acquired ", len(MCCList)
print "\n MCS List has been acquired ", len(MCSList)
findMCCEnd = time.time()
print "\n End the timer for the findMCCs process"
print "\n Total time to complete finding the MCCs is %g seconds" % (
findMCCEnd - findMCCStart)
unittestFile.write(
"\n 4. Total time to complete finding the MCCs is %g seconds" %
(findMCCEnd - findMCCStart))
print("-" * 80)
# End the timer
endTime = time.time()
print("*" * 80)
print "\n The entire evaluation took %g seconds to complete" % (endTime -
startTime)
unittestFile.write("\n The entire evaluation took %g seconds to complete" %
(endTime - startTime))
unittestFile.write(
"\n ----------------------------------------------------------------")
unittestFile.write("\n Number of cloud elements found is: %d" %
CEGraph.number_of_nodes())
unittestFile.write(
"\n Number of edges (with the cloud elements) found is: %d" %
CEGraph.number_of_edges())
unittestFile.write("\n The number of nodes in the prunedGraph is: %d" %
prunedGraph.number_of_nodes())
unittestFile.write(
"\n The number of edges (with nodes) in the prunedGraph is: %d" %
prunedGraph.number_of_edges())
unittestFile.write("\n MCC List has been acquired %d" % len(MCCList))
unittestFile.write("\n MCS List has been acquired %d" % len(MCSList))
unittestFile.write("\n The CEGraph nodes are: %s" % CEGraph.nodes())
unittestFile.write("\n The prunedGraph nodes are: %s" %
prunedGraph.nodes())
print("-" * 80)
# TODO: report the domain
unittestFile.close()