def main(RootDir, CubesPath, CubesRootNames, CubeType, CubeDataType, Seasons,
TrajectoriesPath, ReceptorType, Grid, OilWeatheringType, OutputTimes,
NumLEs):
# create the dir for all the cubes:
FullCubesPath = os.path.join(RootDir, CubesPath)
if not os.path.isdir(FullCubesPath):
os.mkdir(FullCubesPath)
# Build list of Trajectory dirs and cube names:
CubesList = []
for ((Season, Months), CubeRootName) in zip(Seasons, CubesRootNames):
# build the directory structure
SeasonTrajDir = os.path.join(RootDir, TrajectoriesPath, Season) #DDR
# SeasonTrajDir = os.path.join(setup.RootDir, setup.TrajectoriesPath)
SeasonCubesDir = os.path.join(RootDir, CubesPath, Season)
if not os.path.isdir(SeasonCubesDir):
print "Creating directory: ", SeasonCubesDir
os.mkdir(SeasonCubesDir)
DirList = os.listdir(SeasonTrajDir)
DirList.sort()
for d in DirList:
try:
int(d[4:7])
GoOn = True
except ValueError:
GoOn = False
if GoOn:
TrajName = os.path.join(RootDir, TrajectoriesPath, Season, d)
CubeName = os.path.join(
RootDir, CubesPath, Season,
"%s%s%s" % (CubeRootName, d[4:7].zfill(4), ".bin"))
CubesList.append((TrajName, CubeName, Months))
print len(CubesList)
if ReceptorType == "Grid":
Receptors = tap_mod.Grid(Grid.min_long, Grid.max_long, Grid.min_lat,
Grid.max_lat, Grid.num_lat, Grid.num_long)
for TrajFilesDir, CubeName, Months in CubesList:
print "processing Cube::", CubeName
if not os.path.isdir(TrajFilesDir):
raise Exception(TrajFilesDir + " is not a valid directory")
## if the file exists and it is non-empty it will be skipped
CubeExists = os.path.isfile(CubeName)
if CubeExists and (os.path.getsize(CubeName) > 0):
print "Cube: %s Exists...skipping" % (CubeName, )
elif CubeExists and ((
(time() - os.path.getmtime(CubeName)) / 60) < 30):
print "EmptyCube: %s Exists less than 30min old...skipping" % (
CubeName, )
else:
print "Compute Cube: %s \n from trajectory files in:\n %s" % (
CubeName, TrajFilesDir)
# TrajFiles = [os.path.join(TrajFilesDir, "time%03i.nc"%(i+1)) for i in range(setup.NumStarts)]
TrajFiles = [
os.path.join(TrajFilesDir, i)
for i in os.listdir(TrajFilesDir)
]
# filter TrajFiles list
# if setup.CubeStartFilter:
tmnth = [int(i[-9:-7]) for i in TrajFiles]
nmnth = np.asarray(tmnth)
tt = np.where(np.in1d(nmnth, Months))
tlist = []
for i in tt[0]:
tlist.append(TrajFiles[i])
TrajFiles = tlist
# kludge fix to pad length of seasonal traj files to total NumStarts
# if len(TrajFiles < setup.NumStarts):
if len(TrajFiles) < 500:
tlist = TrajFiles[0:250]
for i in TrajFiles[0:250]:
tlist.append(i)
TrajFiles = tlist
# make sure they exist
## fixme: should I check file size, too.
for f in TrajFiles:
if not os.path.exists(f):
raise Exception("trajectory file missing: %s" % f)
# print "trajectory files are all there..."
print "# of files %s ::" % (str(len(TrajFiles)))
print "there are %i trajectory files" % len(TrajFiles)
print "The first 5 are:"
for name in TrajFiles[:5]:
print name
start = time()
if not os.path.isdir(os.path.split(CubeName)[0]):
os.mkdir(os.path.split(CubeName)[0])
if CubeType == "Cumulative":
if OilWeatheringType is not None:
raise NotImplimentedError(
"weathering not implimented for cumulative cubes")
print "computing cumulative cubes"
Cube = tap_mod.CompTAPIICube(
TrajFiles, OutputTimes, Receptors,
oil_weathering.OilTypes[OilWeatheringType])
elif CubeType == "Volume":
print "computing volume cubes"
Cube = tap_mod.CompThicknessCube(
TrajFiles, OutputTimes, Receptors,
oil_weathering.OilTypes[OilWeatheringType])
types = {
'float32': np.float32,
'uint8': np.uint8,
'uint16': np.uint16,
}
try:
Cube = tap_mod.transform(Cube,
NumLEs,
n=1.5,
dtype=types[CubeDataType])
except KeyError:
raise ValueError("Unsupported Cube Data Type: %s" %
CubeDataType)
#print " The Cube Shape is: %s"%(Cube.shape,)
#print "The cube is %i elements, each one %i bytes"%(np.product(Cube.shape), Cube.itemsize())
print "Computing the whole cube took %f seconds (%f minutes)" % (
(time() - start), (time() - start) / 60)
# write it out to a file:
print "writing out cube:", CubeName
#CubeFile = open(CubeName,'wb')
#CubeFile.write(Cube.tostring())
#CubeFile.close()
Cube.tofile(CubeName)
else:
print "Receptortype", ReceptorType, " isn't implimented yet"
tlist= TrajFiles
TrajFiles = []
for tt in tlist[0:setup.NumStarts]:
TrajFiles.append(tt)
print "there are %i trajectory files"%len(TrajFiles)
if not os.path.isdir(os.path.split(CubeName)[0]):
os.mkdir(os.path.split(CubeName)[0])
if setup.CubeType == "Cumulative":
if setup.OilWeatheringType is not None:
raise NotImplimentedError("weathering not implimented for cumulative cubes")
print "computing cumulative cubes"
Cube = tap_mod.CompTAPIICube(TrajFiles,
setup.OutputTimes,
Receptors,
oil_weathering.OilTypes[setup.OilWeatheringType])
elif setup.CubeType == "Volume":
print "computing volume cubes"
Cube = tap_mod.CompThicknessCube(TrajFiles,
setup.OutputTimes,
Receptors,
oil_weathering.OilTypes[setup.OilWeatheringType])
types = {'float32': np.float32,
'uint8' : np.uint8,
'uint16' : np.uint16,
}
try:
Cube = tap_mod.transform(Cube, setup.NumLEs, n = 1.5, dtype=types[setup.CubeDataType])
except KeyError:
raise ValueError("Unsupported Cube Data Type: %s"%CubeDataType)
def main(RunSite, RootDir, CubesPath, CubesRootNames, CubeType, CubeDataType,
Seasons, TrajectoriesPath, ReceptorType, Grid, OilWeatheringType,
OutputTimes, NumLEs, NumStarts):
# create the dir for all the cubes:
FullCubesPath = os.path.join(RootDir, CubesPath)
CubesPath_mf = CubesPath + '_mf'
FullCubesPath_mf = os.path.join(RootDir, CubesPath_mf)
if not os.path.isdir(FullCubesPath):
os.mkdir(FullCubesPath)
if not os.path.isdir(FullCubesPath_mf):
os.mkdir(FullCubesPath_mf)
# Build list of Trajectory dirs and cube names:
CubesList = []
for ((Season, Months), CubeRootName) in zip(Seasons, CubesRootNames):
# build the directory structure
SeasonCubesDir = os.path.join(RootDir, CubesPath, Season)
if not os.path.isdir(SeasonCubesDir):
print "Creating directory: ", SeasonCubesDir
os.mkdir(SeasonCubesDir)
SeasonCubesDir_mf = os.path.join(RootDir, CubesPath_mf, Season)
if not os.path.isdir(SeasonCubesDir_mf):
print "Creating directory: ", SeasonCubesDir_mf
os.mkdir(SeasonCubesDir_mf)
d = 'pos_%03i' % (RunSite + 1)
TrajName = os.path.join(RootDir, TrajectoriesPath, Season, d)
CubeName = os.path.join(
RootDir, CubesPath, Season,
"%s%s%s" % (CubeRootName, d[4:7].zfill(4), ".bin"))
CubeName_mf = os.path.join(
RootDir, CubesPath_mf, Season,
"%s%s%s" % (CubeRootName, d[4:7].zfill(4), ".bin"))
CubesList.append((TrajName, CubeName, CubeName_mf, Months))
print len(CubesList)
if ReceptorType == "Grid":
Receptors = tap_mod.Grid(Grid.min_long, Grid.max_long, Grid.min_lat,
Grid.max_lat, Grid.num_lat, Grid.num_long)
for TrajFilesDir, CubeName, CubeName_mf, Months in CubesList:
print "processing Cube::", CubeName
if not os.path.isdir(TrajFilesDir):
raise Exception(TrajFilesDir + " is not a valid directory")
## if the file exists and it is non-empty it will be skipped
CubeExists = os.path.isfile(CubeName)
if CubeExists and (os.path.getsize(CubeName) > 0):
print "Cube: %s Exists...skipping" % (CubeName, )
elif CubeExists and ((
(time() - os.path.getmtime(CubeName)) / 60) < 30):
print "EmptyCube: %s Exists less than 30min old...skipping" % (
CubeName, )
else:
print "Compute Cube: %s \n from trajectory files in:\n %s" % (
CubeName, TrajFilesDir)
# TrajFiles = [os.path.join(TrajFilesDir, "time%03i.nc"%(i+1)) for i in range(setup.NumStarts)]
Files = os.listdir(TrajFilesDir)
print Files
while len(Files) < NumStarts:
print 'TrajFiles not yet visible in folders, take a small nap'
time.sleep(30)
TrajFiles = [os.path.join(TrajFilesDir, i) for i in Files]
# make sure they exist
## fixme: should I check file size, too.
for f in TrajFiles:
if not os.path.exists(f):
raise Exception("trajectory file missing: %s" % f)
# print "trajectory files are all there..."
print "# of files %s ::" % (str(len(TrajFiles)))
print "there are %i trajectory files" % len(TrajFiles)
print "The first 5 are:"
for name in TrajFiles[:5]:
print name
start = time()
if not os.path.isdir(os.path.split(CubeName)[0]):
os.mkdir(os.path.split(CubeName)[0])
if not os.path.isdir(os.path.split(CubeName_mf)[0]):
os.mkdir(os.path.split(CubeName_mf)[0])
if CubeType == "Cumulative":
if OilWeatheringType is not None:
raise NotImplimentedError(
"weathering not implimented for cumulative cubes")
print "computing cumulative cubes"
Cube = tap_mod.CompTAPIICube(
TrajFiles, OutputTimes, Receptors,
oil_weathering.OilTypes[OilWeatheringType])
elif CubeType == "Volume":
print "computing volume cubes"
Cube, Cube_mf = CompThicknessCube_mf(
TrajFiles, OutputTimes, Receptors,
oil_weathering.OilTypes[OilWeatheringType])
types = {
'float32': np.float32,
'uint8': np.uint8,
'uint16': np.uint16,
}
try:
Cube = tap_mod.transform(Cube,
NumLEs,
n=1.5,
dtype=types[CubeDataType])
Cube_mf = tap_mod.transform(Cube_mf,
NumLEs,
n=1.5,
dtype=types[CubeDataType])
except KeyError:
raise ValueError("Unsupported Cube Data Type: %s" %
CubeDataType)
# print " The Cube Shape is: %s"%(Cube.shape,)
# print "The cube is %i elements, each one %i bytes"%(np.product(Cube.shape), Cube.itemsize())
print "Computing the whole cube took %f seconds (%f minutes)" % (
(time() - start), (time() - start) / 60)
# write it out to a file:
print "writing out cube:", CubeName
# CubeFile = open(CubeName,'wb')
# CubeFile.write(Cube.tostring())
# CubeFile.close()
Cube.tofile(CubeName)
Cube_mf.tofile(CubeName_mf)
else:
print "Receptortype", ReceptorType, " isn't implimented yet"