def test_one_particle():
from batch_gnome import cy_tap_comp_volume
# set up a grid:
grid = tap_mod.Grid(min_long=-10,
max_long=10,
min_lat=-5,
max_lat=5,
num_lat=5,
num_long=10)
LE_positions = np.array(((1.5, 2.5), ), np.float32)
Vol1 = tap_mod.CompThicknessCubeTimestepOld(grid, LE_positions)
# reshape:
print Vol1
assert Vol1[5, 3] == 1 # should be full mass in the one grid box
assert Vol1.sum() == 1 # mass conserved
NumLEs = len(LE_positions)
LE_mass = np.ones((NumLEs, ), dtype=np.float32)
flags = flags = np.zeros((NumLEs), dtype=np.uint8)
Vol2 = cy_tap_comp_volume.comp_volume(LE_positions, LE_mass, flags, grid)
# reshape:
print Vol2
assert Vol2[5, 3] == 1 # should be full mass in the one grid box
assert Vol2.sum() == 1 # mass conserved
assert np.array_equal(Vol1, Vol2)
def main(RootDir, MapName, MapFileName, MapFileType, NumStarts, Seasons,
StartSites, OutputTimes, OutputUserStrings, PresetLOCS,
PresetSpillAmounts, ReceptorType, Grid):
print "starting"
StartTimeFiles = [(os.path.join(RootDir, s[0] + 'Starts.txt'), s[0])
for s in Seasons]
CubesRootNames = ["Arc_" for i in StartTimeFiles
] # built to match the start time files
file = open(os.path.join(RootDir, "site.txt"), 'w')
file.write('"%s" //Map Name for User\n' % MapName)
file.write(
'"%s" "%s" // "map file name" "map file type" -- either "BNA" or "PICT" \n'
% (MapFileName, MapFileType))
file.write(
'%i SPILLS // number of spills per site and time in every cube \n' %
NumStarts)
file.write('%i SEASONS\n' % len(Seasons))
for i in range(len(Seasons)):
Season = Seasons[i][0]
file.write(
'"%s" "%s" "%s" // Name for menu, prefix of cube names, name of directory\n'
% (Season, CubesRootNames[i], Season))
file.write('%i TIMES \n' % (len(OutputTimes)))
#reverse output times:
OutputTimes.reverse()
OutputUserStrings.reverse()
for OutputTime, OutputUserString in zip(OutputTimes, OutputUserStrings):
file.write(
'%i "%s" // numHours "user string" blank user string means not in menu \n'
% (OutputTime, OutputUserString))
file.write('%i AMOUNTS // amounts of oil preset for the user\n' %
len(PresetSpillAmounts))
for s in PresetSpillAmounts:
file.write('%s \n' % s)
file.write('%i LOCS // levels of concern (LOC) preset for the user \n' %
len(PresetLOCS))
for s in PresetLOCS:
file.write('%s \n' % s)
# now write the BNA of the receptor sites:
if ReceptorType == "Grid":
#file.write("This is where the receptors go\n")
from batch_gnome import tap_mod
Receptors = tap_mod.Grid(Grid.min_long, Grid.max_long, Grid.min_lat,
Grid.max_lat, Grid.num_lat, Grid.num_long)
file.write("%i SITES // number of sites\n" % Receptors.num_cells)
Receptors.WriteBNA(file)
# write cube locations:
file.write("%i CUBES\n" % len(StartSites))
print "Writing the start sites"
for site in StartSites:
file.write(site + "\n")
file.close()
#nc_data_file = "micro_test.nc"
#tap_data_file = "micro_test.bin"
#nc_data_file = "mini_test_1000LE.nc"
#tap_data_file = "mini_test_1000LE.bin"
nc_data_file = "small_test_10000LE.nc"
tap_data_file = "small_test_10000LE.bin"
OutputTimes = [4, 8, 12, 16, 20, 24]
# a grid to match test data:
num_lat = 8
num_long = 10
grid = tap_mod.Grid(min_long=-120,
max_long=-118.8,
min_lat=47.4,
max_lat=47.8,
num_lat=num_lat,
num_long=num_long)
def test_compare_with_old_way():
OutputTimes = [8, 16, 24]
#OutputTimes = [8]
cube1 = tap_mod.CompThicknessCubeOld([tap_data_file],
OutputTimes,
grid,
Weather=None)
cube2 = tap_mod.CompThicknessCube([nc_data_file],
OutputTimes,
grid,
except ValueError:
GoOn = False
if GoOn:
TrajName = os.path.join(setup.RootDir,setup.TrajectoriesPath,Season,d)
CubeName = os.path.join(setup.RootDir, setup.CubesPath, Season, "%s%s%s"%(CubeRootName, d[4:7].zfill(4),".bin") )
CubesList.append((TrajName, CubeName, Months))
print len(CubesList)
if setup.ReceptorType == "Grid":
Grid = setup.Grid
Receptors = tap_mod.Grid(Grid.min_long,
Grid.max_long,
Grid.min_lat,
Grid.max_lat,
Grid.num_lat,
Grid.num_long)
for TrajFilesDir, CubeName, Months in CubesList:
print
print "processing Cube::", CubeName
if not os.path.isdir(TrajFilesDir):
raise Exception(TrajFilesDir + " is not a valid directory")
## if the file exists and it is non-empty it will be skipped
CubeExists = os.path.isfile(CubeName)
if CubeExists and ( os.path.getsize(CubeName) > 0 ):
print "Cube: %s Exists...skipping"%(CubeName,)
elif CubeExists and (((time() - os.path.getmtime(CubeName)) / 60) < 30 ):
print "EmptyCube: %s Exists less than 30min old...skipping"%(CubeName,)
def main(RootDir, CubesPath, CubesRootNames, CubeType, CubeDataType, Seasons,
TrajectoriesPath, ReceptorType, Grid, OilWeatheringType, OutputTimes,
NumLEs):
# create the dir for all the cubes:
FullCubesPath = os.path.join(RootDir, CubesPath)
if not os.path.isdir(FullCubesPath):
os.mkdir(FullCubesPath)
# Build list of Trajectory dirs and cube names:
CubesList = []
for ((Season, Months), CubeRootName) in zip(Seasons, CubesRootNames):
# build the directory structure
SeasonTrajDir = os.path.join(RootDir, TrajectoriesPath, Season) #DDR
# SeasonTrajDir = os.path.join(setup.RootDir, setup.TrajectoriesPath)
SeasonCubesDir = os.path.join(RootDir, CubesPath, Season)
if not os.path.isdir(SeasonCubesDir):
print "Creating directory: ", SeasonCubesDir
os.mkdir(SeasonCubesDir)
DirList = os.listdir(SeasonTrajDir)
DirList.sort()
for d in DirList:
try:
int(d[4:7])
GoOn = True
except ValueError:
GoOn = False
if GoOn:
TrajName = os.path.join(RootDir, TrajectoriesPath, Season, d)
CubeName = os.path.join(
RootDir, CubesPath, Season,
"%s%s%s" % (CubeRootName, d[4:7].zfill(4), ".bin"))
CubesList.append((TrajName, CubeName, Months))
print len(CubesList)
if ReceptorType == "Grid":
Receptors = tap_mod.Grid(Grid.min_long, Grid.max_long, Grid.min_lat,
Grid.max_lat, Grid.num_lat, Grid.num_long)
for TrajFilesDir, CubeName, Months in CubesList:
print "processing Cube::", CubeName
if not os.path.isdir(TrajFilesDir):
raise Exception(TrajFilesDir + " is not a valid directory")
## if the file exists and it is non-empty it will be skipped
CubeExists = os.path.isfile(CubeName)
if CubeExists and (os.path.getsize(CubeName) > 0):
print "Cube: %s Exists...skipping" % (CubeName, )
elif CubeExists and ((
(time() - os.path.getmtime(CubeName)) / 60) < 30):
print "EmptyCube: %s Exists less than 30min old...skipping" % (
CubeName, )
else:
print "Compute Cube: %s \n from trajectory files in:\n %s" % (
CubeName, TrajFilesDir)
# TrajFiles = [os.path.join(TrajFilesDir, "time%03i.nc"%(i+1)) for i in range(setup.NumStarts)]
TrajFiles = [
os.path.join(TrajFilesDir, i)
for i in os.listdir(TrajFilesDir)
]
# filter TrajFiles list
# if setup.CubeStartFilter:
tmnth = [int(i[-9:-7]) for i in TrajFiles]
nmnth = np.asarray(tmnth)
tt = np.where(np.in1d(nmnth, Months))
tlist = []
for i in tt[0]:
tlist.append(TrajFiles[i])
TrajFiles = tlist
# kludge fix to pad length of seasonal traj files to total NumStarts
# if len(TrajFiles < setup.NumStarts):
if len(TrajFiles) < 500:
tlist = TrajFiles[0:250]
for i in TrajFiles[0:250]:
tlist.append(i)
TrajFiles = tlist
# make sure they exist
## fixme: should I check file size, too.
for f in TrajFiles:
if not os.path.exists(f):
raise Exception("trajectory file missing: %s" % f)
# print "trajectory files are all there..."
print "# of files %s ::" % (str(len(TrajFiles)))
print "there are %i trajectory files" % len(TrajFiles)
print "The first 5 are:"
for name in TrajFiles[:5]:
print name
start = time()
if not os.path.isdir(os.path.split(CubeName)[0]):
os.mkdir(os.path.split(CubeName)[0])
if CubeType == "Cumulative":
if OilWeatheringType is not None:
raise NotImplimentedError(
"weathering not implimented for cumulative cubes")
print "computing cumulative cubes"
Cube = tap_mod.CompTAPIICube(
TrajFiles, OutputTimes, Receptors,
oil_weathering.OilTypes[OilWeatheringType])
elif CubeType == "Volume":
print "computing volume cubes"
Cube = tap_mod.CompThicknessCube(
TrajFiles, OutputTimes, Receptors,
oil_weathering.OilTypes[OilWeatheringType])
types = {
'float32': np.float32,
'uint8': np.uint8,
'uint16': np.uint16,
}
try:
Cube = tap_mod.transform(Cube,
NumLEs,
n=1.5,
dtype=types[CubeDataType])
except KeyError:
raise ValueError("Unsupported Cube Data Type: %s" %
CubeDataType)
#print " The Cube Shape is: %s"%(Cube.shape,)
#print "The cube is %i elements, each one %i bytes"%(np.product(Cube.shape), Cube.itemsize())
print "Computing the whole cube took %f seconds (%f minutes)" % (
(time() - start), (time() - start) / 60)
# write it out to a file:
print "writing out cube:", CubeName
#CubeFile = open(CubeName,'wb')
#CubeFile.write(Cube.tostring())
#CubeFile.close()
Cube.tofile(CubeName)
else:
print "Receptortype", ReceptorType, " isn't implimented yet"
import os, sys
import numpy as np
from batch_gnome import tap_mod, tap_comp_volume
# some test data
infilename = "micro_test.bin"
(Trajectory, (NumTimesteps, NumLEs), HeaderData,
Flags) = tap_mod.ReadTrajectory(infilename)
print Flags
print Flags.dtype
max = Trajectory.max(axis=0).max(axis=0)
min = Trajectory.min(axis=0).min(axis=0)
# set up a grid:
grid = tap_mod.Grid(min[0], max[0], min[1], max[1], 10, 10)
print grid
# loop through the time steps:
for t_ind in xrange(NumTimesteps):
#for t_ind in [10]:
print "timestep :", t_ind
Positions = Trajectory[t_ind]
flags = Flags[t_ind]
masses = np.ones_like(flags, dtype=np.float32)
Volumes = tap_comp_volume.comp_volume(Positions, masses, flags, grid)
print Volumes
cy_weathered = cy_MediumCrude.weather(mass, age)
cy_time = time.time() - start
print "cython version took %s seconds" % cy_time
print "speedup ratio:", py_time / cy_time
if np.array_equal(weathered, cy_weathered):
print "They are the same"
else:
print "No match!!! somethign is wrong!!!"
# set up a grid:
grid = tap_mod.Grid(min_long=-10,
max_long=10,
min_lat=-5,
max_lat=5,
num_lat=5,
num_long=10)
def test_multiple_LEs():
positions = (np.random.random(
(10000, 2)) * ((grid.max_long - grid.min_long),
(grid.max_long - grid.min_long)) +
(grid.min_long, grid.min_lat)).astype(np.float32)
mass = np.ones((positions.shape[0], ), np.float32)
flags = np.zeros_like(mass, dtype=np.uint8)
# flags = np.array( (16, 0) , dtype=np.uint8) # 16 is the notOnSurface: flag
start = time.time()
mass_grid1 = comp_volume(positions, mass, flags, grid)
def main(RunSite, RootDir, CubesPath, CubesRootNames, CubeType, CubeDataType,
Seasons, TrajectoriesPath, ReceptorType, Grid, OilWeatheringType,
OutputTimes, NumLEs, NumStarts):
# create the dir for all the cubes:
FullCubesPath = os.path.join(RootDir, CubesPath)
CubesPath_mf = CubesPath + '_mf'
FullCubesPath_mf = os.path.join(RootDir, CubesPath_mf)
if not os.path.isdir(FullCubesPath):
os.mkdir(FullCubesPath)
if not os.path.isdir(FullCubesPath_mf):
os.mkdir(FullCubesPath_mf)
# Build list of Trajectory dirs and cube names:
CubesList = []
for ((Season, Months), CubeRootName) in zip(Seasons, CubesRootNames):
# build the directory structure
SeasonCubesDir = os.path.join(RootDir, CubesPath, Season)
if not os.path.isdir(SeasonCubesDir):
print "Creating directory: ", SeasonCubesDir
os.mkdir(SeasonCubesDir)
SeasonCubesDir_mf = os.path.join(RootDir, CubesPath_mf, Season)
if not os.path.isdir(SeasonCubesDir_mf):
print "Creating directory: ", SeasonCubesDir_mf
os.mkdir(SeasonCubesDir_mf)
d = 'pos_%03i' % (RunSite + 1)
TrajName = os.path.join(RootDir, TrajectoriesPath, Season, d)
CubeName = os.path.join(
RootDir, CubesPath, Season,
"%s%s%s" % (CubeRootName, d[4:7].zfill(4), ".bin"))
CubeName_mf = os.path.join(
RootDir, CubesPath_mf, Season,
"%s%s%s" % (CubeRootName, d[4:7].zfill(4), ".bin"))
CubesList.append((TrajName, CubeName, CubeName_mf, Months))
print len(CubesList)
if ReceptorType == "Grid":
Receptors = tap_mod.Grid(Grid.min_long, Grid.max_long, Grid.min_lat,
Grid.max_lat, Grid.num_lat, Grid.num_long)
for TrajFilesDir, CubeName, CubeName_mf, Months in CubesList:
print "processing Cube::", CubeName
if not os.path.isdir(TrajFilesDir):
raise Exception(TrajFilesDir + " is not a valid directory")
## if the file exists and it is non-empty it will be skipped
CubeExists = os.path.isfile(CubeName)
if CubeExists and (os.path.getsize(CubeName) > 0):
print "Cube: %s Exists...skipping" % (CubeName, )
elif CubeExists and ((
(time() - os.path.getmtime(CubeName)) / 60) < 30):
print "EmptyCube: %s Exists less than 30min old...skipping" % (
CubeName, )
else:
print "Compute Cube: %s \n from trajectory files in:\n %s" % (
CubeName, TrajFilesDir)
# TrajFiles = [os.path.join(TrajFilesDir, "time%03i.nc"%(i+1)) for i in range(setup.NumStarts)]
Files = os.listdir(TrajFilesDir)
print Files
while len(Files) < NumStarts:
print 'TrajFiles not yet visible in folders, take a small nap'
time.sleep(30)
TrajFiles = [os.path.join(TrajFilesDir, i) for i in Files]
# make sure they exist
## fixme: should I check file size, too.
for f in TrajFiles:
if not os.path.exists(f):
raise Exception("trajectory file missing: %s" % f)
# print "trajectory files are all there..."
print "# of files %s ::" % (str(len(TrajFiles)))
print "there are %i trajectory files" % len(TrajFiles)
print "The first 5 are:"
for name in TrajFiles[:5]:
print name
start = time()
if not os.path.isdir(os.path.split(CubeName)[0]):
os.mkdir(os.path.split(CubeName)[0])
if not os.path.isdir(os.path.split(CubeName_mf)[0]):
os.mkdir(os.path.split(CubeName_mf)[0])
if CubeType == "Cumulative":
if OilWeatheringType is not None:
raise NotImplimentedError(
"weathering not implimented for cumulative cubes")
print "computing cumulative cubes"
Cube = tap_mod.CompTAPIICube(
TrajFiles, OutputTimes, Receptors,
oil_weathering.OilTypes[OilWeatheringType])
elif CubeType == "Volume":
print "computing volume cubes"
Cube, Cube_mf = CompThicknessCube_mf(
TrajFiles, OutputTimes, Receptors,
oil_weathering.OilTypes[OilWeatheringType])
types = {
'float32': np.float32,
'uint8': np.uint8,
'uint16': np.uint16,
}
try:
Cube = tap_mod.transform(Cube,
NumLEs,
n=1.5,
dtype=types[CubeDataType])
Cube_mf = tap_mod.transform(Cube_mf,
NumLEs,
n=1.5,
dtype=types[CubeDataType])
except KeyError:
raise ValueError("Unsupported Cube Data Type: %s" %
CubeDataType)
# print " The Cube Shape is: %s"%(Cube.shape,)
# print "The cube is %i elements, each one %i bytes"%(np.product(Cube.shape), Cube.itemsize())
print "Computing the whole cube took %f seconds (%f minutes)" % (
(time() - start), (time() - start) / 60)
# write it out to a file:
print "writing out cube:", CubeName
# CubeFile = open(CubeName,'wb')
# CubeFile.write(Cube.tostring())
# CubeFile.close()
Cube.tofile(CubeName)
Cube_mf.tofile(CubeName_mf)
else:
print "Receptortype", ReceptorType, " isn't implimented yet"