def loadMissions(baseUrl,
fileList,
activityName,
campaignName,
pName,
pColor,
pTypeName,
aTypeName,
parmList,
dbName,
stride=1):
'''Load missions from OPeNDAP url from either a list of files from a base or a single URL with a given activityName '''
if fileList:
for (aName, file) in zip([a + ' (stride=%d)' % stride for a in files],
fileList):
url = baseUrl + file
print "loadMissions(): Calling runLoader() with parmList = %s" % parmList
DAPloaders.runTrajectoryLoader(url, campaignName, aName, pName,
pColor, pTypeName, aTypeName,
parmList, dbName, stride)
elif activityName:
url = baseUrl
DAPloaders.runTrajectoryLoader(url, campaignName, activityName, pName,
pColor, pTypeName, aTypeName, parmList,
dbName, stride)
else:
print "loadMissions(): Must specify either a fileList or an activityName"
def loadM1(self, stride=None):
'''
Mooring M1 specific load functions
'''
platformName = 'M1_Mooring'
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.m1_files], self.m1_files):
url = os.path.join(self.m1_base, file)
dataStartDatetime = None
if self.args.append:
# Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(self.dbAlias).filter(activity__name=aName).aggregate(Max('timevalue'))['timevalue__max']
if dataStartDatetime:
# Subract an hour to fill in missing_values at end from previous load
dataStartDatetime = dataStartDatetime - timedelta(seconds=3600)
DAPloaders.runMooringLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['m1'], 'mooring', 'Mooring Deployment',
self.m1_parms, self.dbAlias, stride, self.m1_startDatetime, self.m1_endDatetime, dataStartDatetime)
# For timeseriesProfile data we need to pass the nominaldepth of the plaform
# so that the model is put at the correct depth in the Spatial -> 3D view.
try:
self.addPlatformResources('http://stoqs.mbari.org/x3d/m1_assembly/m1_assembly_scene.x3d',
platformName, nominaldepth=self.m1_nominaldepth)
except AttributeError:
self.addPlatformResources('http://stoqs.mbari.org/x3d/m1_assembly/m1_assembly_scene.x3d',
platformName)
def loadTethys(self, stride=None):
'''
Tethys specific load functions
'''
pName = 'tethys'
stride = stride or self.stride
for (aName,
f) in zip([a + getStrideText(stride) for a in self.tethys_files],
self.tethys_files):
url = self.tethys_base + f
# shorten the activity names
aName = aName.rsplit('/', 1)[-1]
try:
DAPloaders.runLrauvLoader(url,
self.campaignName,
self.campaignDescription,
aName,
pName,
self.colors['tethys'],
'auv',
'AUV mission',
self.tethys_parms,
self.dbAlias,
stride,
grdTerrain=self.grdTerrain,
command_line_args=self.args,
timezone='America/New_York')
except DAPloaders.NoValidData:
self.logger.info("No valid data in %s" % url)
self.addPlatformResources(
'http://stoqs.mbari.org/x3d/lrauv/lrauv_tethys.x3d', pName)
def load_glider_ctd(self, stride=None):
'''
Glider load functions. Requires apriori knowledge of glider file names so we can extract platform and color name
To be used with gliders that follow the same naming convention, i.e. nemesis_ctd.nc, ucsc260_ctd.nc
and that load the exact same parameters, i.e. TEMP, PSAL or TEMP, PSAL, FLU2 or TEMP, FLU2, OPBS etc
'''
stride = stride or self.stride
for (aName, file) in zip(
[a + getStrideText(stride) for a in self.glider_ctd_files],
self.glider_ctd_files):
url = self.glider_ctd_base + file
pName = aName.split('/')[-1].split('.')[0]
p = re.compile('-\d+T\d+_Time')
pName = p.sub('', pName)
if pName.find('-') != -1:
logger.warn(
"Replacing '-' characters in platform name %s with '_'s",
pName)
pName = pName.replace('-', '_')
logger.info("Executing runGliderLoader with url = %s", url)
DAPloaders.runGliderLoader(url, self.campaignName, aName, pName,
'FFBA26', 'glider', 'Glider Mission',
self.glider_ctd_parms, self.dbAlias,
stride, self.glider_ctd_startDatetime,
self.glider_ctd_endDatetime)
def loadTethys(self, stride=None):
'''
Tethys specific load functions
'''
pName = 'tethys'
stride = stride or self.stride
for (aName, f) in zip([ a + getStrideText(stride) for a in self.tethys_files], self.tethys_files):
url = self.tethys_base + f
dataStartDatetime = None
startDatetime = self.tethys_startDatetime
endDatetime = self.tethys_endDatetime
if self.args.append:
# Return datetime of last timevalue - if data are loaded from multiple
# activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(self.dbAlias).filter(
activity__name=aName).aggregate(
Max('timevalue'))['timevalue__max']
try:
DAPloaders.runLrauvLoader(url, self.campaignName, self.campaignDescription, aName,
pName, self.colors['tethys'], 'auv', 'AUV mission',
self.tethys_parms, self.dbAlias, stride,
grdTerrain=self.grdTerrain, dataStartDatetime=dataStartDatetime,
endDatetime=endDatetime, startDatetime=startDatetime, timezone='America/New_York')
except DAPloaders.NoValidData:
self.logger.info("No valid data in %s" % url)
self.addPlatformResources('http://stoqs.mbari.org/x3d/lrauv/lrauv_tethys.x3d', pName)
def loadBEDS(self, stride=None, featureType='trajectory'):
'''
BEDS specific load functions; featureType can be 'trajectory' or 'timeSeries'.
Use 'trajectory' for events that we've fudged into a trajectory netCDF file
using the canyon's thalweg. Use 'timeSeries' for events for which the BED
does not significantly translate.
'''
stride = stride or self.stride
for (aName, pName, file, plotTimeSeriesDepth, fg) in zip(
[ a.split('/')[-1] + ' (stride=%d)' % stride for a in self.bed_files],
self.bed_platforms, self.bed_files, self.bed_depths, self.bed_framegrabs):
url = os.path.join(self.bed_base, file)
try:
if featureType.lower() == 'trajectory':
# To get timeSeries plotting for trajectories (in the Parameter tab of the UI)
# assign a plotTimeSeriesDepth value of the starting depth in meters.
DAPloaders.runBEDTrajectoryLoader(url, self.campaignName, self.campaignDescription,
aName, pName, self.colors[pName.lower()], 'bed',
'deployment', self.bed_parms, self.dbAlias, stride,
plotTimeSeriesDepth=plotTimeSeriesDepth,
grdTerrain=self.grdTerrain, framegrab=fg)
elif featureType.lower() == 'timeseries':
DAPloaders.runTimeSeriesLoader(url, self.campaignName, self.campaignDescription,
aName, pName, self.colors[pName.lower()], 'bed',
'deployment', self.bed_parms, self.dbAlias, stride)
self.addPlatformResources('http://stoqs.mbari.org/x3d/beds/beds_housing_with_axes_src_scene.x3d',
pName, scalefactor=10)
except (DAPloaders.OpendapError, DAPloaders.InvalidSliceRequest):
pass
def loadCastaway(self):
'''
Sparus specific load functions
'''
for (aName, file) in zip([ a + ' (stride=%d)' % self.stride for a in self.castaway_files], self.castaway_files):
url = self.castaway_base + file
DAPloaders.runTrajectoryLoader(url, self.campaignName, aName, 'castaway', self.colors['castaway'], 'mooring', 'CTD Profile',
self.castaway_parms, self.dbAlias, self.stride)
def loadSparus(self):
'''
Sparus specific load functions
'''
for (aName, file) in zip([ a + ' (stride=%d)' % self.stride for a in self.sparus_files], self.sparus_files):
url = self.sparus_base + file
DAPloaders.runTrajectoryLoader(url, self.campaignName, aName, 'sparus', self.colors['sparus'], 'auv', 'AUV mission',
self.sparus_parms, self.dbAlias, self.stride)
def loadM1met(self, stride=None):
'''
Mooring M1met specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.m1met_files], self.m1met_files):
url = self.m1met_base + file
DAPloaders.runMooringLoader(url, self.campaignName, self.campaignDescription, aName, 'M1_Mooring', self.colors['m1'], 'mooring', 'Mooring Deployment',
self.m1met_parms, self.dbAlias, stride, self.m1met_startDatetime, self.m1met_endDatetime)
def loadJMuctd(self, stride=None):
'''
Martin specific underway load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.JMuctd_files], self.JMuctd_files):
url = self.JMuctd_base + file
DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, 'John_Martin_UCTD', self.colors['martin'], 'ship', 'cruise',
self.JMuctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
def loadCarmen(self, stride=None):
'''
Glider specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.carmen_files], self.carmen_files):
url = self.carmen_base + file
DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, 'carmen', self.colors['carmen'], 'glider', 'Glider Mission',
self.carmen_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
def loadESPbruce(self, stride=None):
'''
ESPbruce specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.espbruce_files], self.espbruce_files):
url = self.espbruce_base + file
DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, 'espbruce', self.colors['espbruce'], 'espbruce', 'ESP bruce Mission',
self.espbruce_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
def loadRCuctd(self, stride=None, platformName='RachelCarson_UCTD', activitytypeName='Rachel Carson Underway CTD Data'):
'''
RC uctd specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.rcuctd_files], self.rcuctd_files):
url = self.rcuctd_base + file
DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['carson'], 'ship', activitytypeName,
self.rcuctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
def loadFulmar(self, stride=None):
'''
Fulmar specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.fulmar_files], self.fulmar_files):
url = self.fulmar_base + file
DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, 'fulmar', self.colors['fulmar'], 'ship', 'cruise',
self.fulmar_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
def loadOA2met(self, stride=None):
'''
Mooring OA specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.OA2met_files], self.OA2met_files):
url = os.path.join(self.OA2met_base, file)
DAPloaders.runMooringLoader(url, self.campaignName, self.campaignDescription, aName, 'OA2_Mooring', self.colors['oa2'], 'mooring', 'Mooring Deployment',
self.OA2met_parms, self.dbAlias, stride, self.OA2met_startDatetime, self.OA2met_endDatetime)
def load_slocum_260(self, stride=None):
'''
Glider specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.slocum_260_files], self.slocum_260_files):
url = self.slocum_260_base + file
DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, 'Slocum_260', self.colors['slocum_260'], 'glider', 'Glider Mission',
self.slocum_260_parms, self.dbAlias, stride, self.slocum_260_startDatetime, self.slocum_260_endDatetime, grdTerrain=self.grdTerrain)
def loadESPmack(self, stride=None):
'''
ESPmack specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.espmack_files], self.espmack_files):
url = self.espmack_base + file
print "url = %s" % url
DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, 'ESP_Mack_Drifter', self.colors['espmack'], 'espmack', 'ESP mack Mission',
self.espmack_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
def loadDorado(self, stride=None):
'''
Dorado specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.dorado_files], self.dorado_files):
url = self.dorado_base + file
DAPloaders.runDoradoLoader(url, self.campaignName, self.campaignDescription, aName, 'Dorado', self.colors['dorado'], 'auv', 'AUV mission',
self.dorado_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
load_gulps(aName, file, self.dbAlias)
def load_NPS34(self, stride=None):
'''
Glider specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.nps34_files], self.nps34_files):
url = self.nps34_base + file
print "url = %s" % url
DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, 'NPS_Glider_34', self.colors['nps34'], 'glider', 'Glider Mission',
self.nps34_parms, self.dbAlias, stride, self.nps34_startDatetime, self.nps34_endDatetime, grdTerrain=self.grdTerrain)
def load_wg_tex_met(self, stride=None):
'''
Glider specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.wg_tex_met_files], self.wg_tex_met_files):
url = self.wg_tex_met_base + file
DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, 'wg_Tex_Glider', self.colors['wg_tex'], 'waveglider', 'Glider Mission',
self.wg_tex_met_parms, self.dbAlias, stride, self.wg_tex_met_startDatetime, self.wg_tex_met_endDatetime,
grdTerrain=self.grdTerrain)
def loadStella(self, stride=None):
'''
Stella drift specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.stella_files], self.stella_files):
url = self.stella_base + file
dname='Stella' + aName[6:9]
DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, dname, self.colors[dname], 'drifter', 'Stella drifter Mission',
self.stella_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
def loadRusalka(self, stride=None):
'''
Glider specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.rusalka_files], self.rusalka_files):
url = self.rusalka_base + file
print "url = %s" % url
DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, 'rusalka', self.colors['rusalka'], 'glider', 'Glider Mission',
self.rusalka_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
def load(self, job):
data_start = None
print('====>Processing decimated nc file {}'.format(job.activity_name))
(url_dest, start, end) = self.process_decimated(job)
print('====>Loading {}'.format(job.activity_name))
if self.args.append:
core_name = job.activity_name.split('_')[0]
# Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
data_start = \
InstantPoint.objects.using(self.args.database).filter(activity__name__contains=core_name).aggregate(
Max('timevalue'))['timevalue__max']
coord = {}
for p in self.args.plotparms:
coord[p] = {'time': p + '_time', 'latitude': p + '_latitude', 'longitude': p + '_longitude',
'depth': p + '_depth'}
try:
if not self.args.debug:
logger.info("Instantiating Lrauv_Loader for url = {}".format(url_dest))
DAPloaders.runLrauvLoader(cName = self.args.campaign,
cDesc = None,
aName = job.activity_name,
aTypeName = 'LRAUV mission',
pName = self.vehicle,
pTypeName = 'auv',
pColor = self.cl.colors[self.vehicle],
url = url_dest,
parmList = self.args.parms,
dbAlias = self.args.database,
stride = int(self.args.stride),
startDatetime = start,
dataStartDatetime = data_start,
endDatetime = end,
contourUrl = self.args.contourUrl,
auxCoords = coord,
timezone = 'America/Los_Angeles',
command_line_args = self.args)
except DAPloaders.NoValidData:
logger.info("No measurements in this log set. Activity was not created as there was nothing to load.")
except pydap.exceptions.ServerError as e:
logger.warning(e)
except DAPloaders.ParameterNotFound as e:
logger.warning(e)
except DAPloaders.InvalidSliceRequest as e:
logger.warning(e)
except Exception as e:
logger.warning(e)
return url_dest, start, end
def loadWFuctd(self, stride=None, platformName='WesternFlyer_UCTD', activitytypeName='Western Flyer Underway CTD Data'):
'''
WF uctd specific load functions. Override defaults for @platformName and activitytypeName if it's desired
to consider uctd and pctd coming from the same platform. You may want to do this to use the data
visualization capabilities in STOQS.
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.wfuctd_files], self.wfuctd_files):
url = self.wfuctd_base + file
print "url = %s" % url
DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['flyer'], 'ship', activitytypeName,
self.wfuctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
def loadMackMoor(self, stride=None):
'''
Mooring Mack specific load functions
'''
stride = stride or self.stride
pName = 'ESP_Mack_Mooring'
for (aName, file) in zip([ a + getStrideText(stride) for a in self.mack_moor_files], self.mack_moor_files):
url = os.path.join(self.mack_moor_base, file)
DAPloaders.runMooringLoader(url, self.campaignName, self.campaignDescription, aName, pName, self.colors['espmack'], 'mooring', 'Mooring Deployment',
self.mack_moor_parms, self.dbAlias, stride, self.mack_moor_startDatetime, self.mack_moor_endDatetime)
# Let browser code use {{STATIC_URL}} to fill in the /stoqs/static path
self.addPlatformResources('x3d/ESPMooring/esp_base_scene.x3d', pName)
def loadMissions(baseUrl, fileList, activityName, campaignName, pName, pColor, pTypeName, aTypeName, parmList, dbName, stride = 1):
'''Load missions from OPeNDAP url from either a list of files from a base or a single URL with a given activityName '''
c_desc = 'Mar Menor In Situ (AUV and Castaway) data loads'
if fileList:
for (aName, f) in zip([ a + ' (stride=%d)' % stride for a in files], fileList):
url = baseUrl + f
print "loadMissions(): Calling runLoader() with parmList = %s" % parmList
DAPloaders.runTrajectoryLoader(url, campaignName, c_desc, aName, pName, pColor, pTypeName, aTypeName, parmList, dbName, stride)
elif activityName:
url = baseUrl
DAPloaders.runTrajectoryLoader(url, campaignName, c_desc, activityName, pName, pColor, pTypeName, aTypeName, parmList, dbName, stride)
else:
print "loadMissions(): Must specify either a fileList or an activityName"
def loadCastaway(self):
'''
Sparus specific load functions
'''
for (aName, file) in zip(
[a + ' (stride=%d)' % self.stride for a in self.castaway_files],
self.castaway_files):
url = self.castaway_base + file
DAPloaders.runTrajectoryLoader(url, self.campaignName, aName,
'castaway', self.colors['castaway'],
'mooring', 'CTD Profile',
self.castaway_parms, self.dbAlias,
self.stride)
def load_glider_met(self, stride=None):
'''
Glider load functions. Requires apriori knowledge of glider file names so we can extract platform and color name
To be used with gliders that follow the same naming convention, i.e. nemesis_met.nc, ucsc260_met.nc
and that load the exact same parameters, i.e. meanu,meanv or windspeed, winddirection etc.
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.glider_met_files], self.glider_met_files):
url = self.glider_met_base + file
gplatform=aName.split('_')[0].upper() + '_Glider'
gname=aName.split('_')[0].lower()
DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, gplatform, self.colors[gname], 'glider', 'Glider Mission',
self.glider_met_parms, self.dbAlias, stride, self.glider_met_startDatetime, self.glider_met_endDatetime, grdTerrain=self.grdTerrain)
def loadSparus(self):
'''
Sparus specific load functions
'''
for (aName, file) in zip(
[a + ' (stride=%d)' % self.stride for a in self.sparus_files],
self.sparus_files):
url = self.sparus_base + file
DAPloaders.runTrajectoryLoader(url, self.campaignName, aName,
'sparus', self.colors['sparus'],
'auv', 'AUV mission',
self.sparus_parms, self.dbAlias,
self.stride)
def loadDorado(self, stride=None):
'''
Dorado specific load functions
'''
pName = 'dorado'
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.dorado_files], self.dorado_files):
url = self.dorado_base + file
DAPloaders.runDoradoLoader(url, self.campaignName, self.campaignDescription, aName, pName, self.colors['dorado'], 'auv', 'AUV mission',
self.dorado_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
load_gulps(aName, file, self.dbAlias)
self.addPlatformResources('http://stoqs.mbari.org/x3d/dorado/simpleDorado389.x3d', pName)
def loadTethys(self, stride=None):
'''
Tethys specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.tethys_files], self.tethys_files):
url = self.tethys_base + file
dataStartDatetime = None
if self.args.append:
# Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(self.dbAlias).filter(activity__name=aName).aggregate(Max('timevalue'))['timevalue__max']
DAPloaders.runLrauvLoader(url, self.campaignName, self.campaignDescription, aName, 'Tethys', self.colors['tethys'], 'auv', 'AUV mission',
self.tethys_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain, dataStartDatetime=dataStartDatetime)
def load_glider_ctd(self, stride=None):
'''
Glider load functions. Requires apriori knowledge of glider file names so we can extract platform and color name
To be used with gliders that follow the same naming convention, i.e. nemesis_ctd.nc, ucsc260_ctd.nc
and that load the exact same parameters, i.e. TEMP, PSAL or TEMP, PSAL, FLU2 or TEMP, FLU2, OPBS etc
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.glider_ctd_files], self.glider_ctd_files):
url = self.glider_ctd_base + file
gplatform=aName.split('_')[0].upper() + '_Glider'
gname=aName.split('_')[0].lower()
print "url = %s" % url
print "platform = %s" % gplatform
DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, gplatform, self.colors[gname], 'glider', 'Glider Mission',
self.glider_ctd_parms, self.dbAlias, stride, self.glider_ctd_startDatetime, self.glider_ctd_endDatetime, grdTerrain=self.grdTerrain)
def loadJMpctd(self, stride=None, platformName='John_Martin_PCTD', activitytypeName='John Martin Profile CTD Data'):
'''
Martin specific underway load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.JMpctd_files], self.JMpctd_files):
url = self.JMpctd_base + file
DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['martin'], 'ship', activitytypeName,
self.JMpctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
# load all the bottles
sl = SeabirdLoader(aName[:5], platformName, dbAlias=self.dbAlias, campaignName=self.campaignName, platformColor=self.colors['martin'], platformTypeName='ship', dodsBase=self.JMpctd_base)
if self.args.verbose:
sl.logger.setLevel(logging.DEBUG)
sl.tdsBase= self.tdsBase
sl.process_btl_files(self.JMpctd_files)
def loadWFpctd(self, stride=None, platformName='WesternFlyer_PCTD', activitytypeName='Western Flyer Profile CTD Data'):
'''
WF pctd specific load functions. Override defaults for @platformName and activitytypeName if it's desired
to consider uctd and pctd coming from the same platform. You may want to do this to use the data
visualization capabilities in STOQS.
'''
stride = stride or self.stride
for (aName, file) in zip([ a.split('.')[0] + getStrideText(stride) for a in self.wfpctd_files], self.wfpctd_files):
url = self.wfpctd_base + file
DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['flyer'], 'ship', activitytypeName,
self.wfpctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
# Now load all the bottles
sl = SeabirdLoader('activity name', platformName, dbAlias=self.dbAlias, campaignName=self.campaignName, platformColor=self.colors['flyer'], dodsBase=self.wfpctd_base)
if self.args.verbose:
sl.logger.setLevel(logging.DEBUG)
sl.tdsBase= self.tdsBase
sl.process_btl_files(self.wfpctd_files)
def loadCCEBIN(self, stride=None):
'''
Mooring CCEBIN specific load functions
'''
platformName = 'CCEBIN'
stride = stride or self.stride
for (aName,
f) in zip([a + getStrideText(stride) for a in self.ccebin_files],
self.ccebin_files):
url = os.path.join(self.ccebin_base, f)
dataStartDatetime = None
if self.args.append:
# Return datetime of last timevalue - if data are loaded from multiple
# activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(
self.dbAlias).filter(activity__name=aName).aggregate(
Max('timevalue'))['timevalue__max']
if dataStartDatetime:
# Subract an hour to fill in missing_values at end from previous load
dataStartDatetime = dataStartDatetime - timedelta(
seconds=3600)
if 'adcp' in f:
Mooring_Loader.getFeatureType = lambda self: 'timeseriesprofile'
else:
Mooring_Loader.getFeatureType = lambda self: 'timeseries'
DAPloaders.runMooringLoader(
url, self.campaignName, self.campaignDescription, aName,
platformName, self.colors['ccebin'], 'mooring',
'Mooring Deployment', self.ccebin_parms, self.dbAlias, stride,
self.ccebin_startDatetime, self.ccebin_endDatetime,
dataStartDatetime)
# For timeseriesProfile data we need to pass the nominaldepth of the plaform
# so that the model is put at the correct depth in the Spatial -> 3D view.
try:
self.addPlatformResources(
'http://stoqs.mbari.org/x3d/cce_bin_assem/cce_bin_assem_src_scene.x3d',
platformName,
nominaldepth=self.ccebin_nominaldepth)
except AttributeError:
self.addPlatformResources(
'http://stoqs.mbari.org/x3d/cce_bin_assem/cce_bin_assem_src_scene.x3d',
platformName)
def loadBEDS(self, stride=None, featureType='trajectory'):
'''
BEDS specific load functions; featureType can be 'trajectory' or 'timeSeries'.
Use 'trajectory' for events that we've fudged into a trajectory netCDF file
using the canyon's thalweg. Use 'timeSeries' for events for which the BED
does not significantly translate.
'''
stride = stride or self.stride
for (aName, pName, file, plotTimeSeriesDepth,
fg) in zip([
a.split('/')[-1] + ' (stride=%d)' % stride
for a in self.bed_files
], self.bed_platforms, self.bed_files, self.bed_depths,
self.bed_framegrabs):
url = os.path.join(self.bed_base, file)
try:
if featureType.lower() == 'trajectory':
# To get timeSeries plotting for trajectories (in the Parameter tab of the UI)
# assign a plotTimeSeriesDepth value of the starting depth in meters.
DAPloaders.runBEDTrajectoryLoader(
url,
self.campaignName,
self.campaignDescription,
aName,
pName,
self.colors[pName.lower()],
'bed',
'deployment',
self.bed_parms,
self.dbAlias,
stride,
plotTimeSeriesDepth=plotTimeSeriesDepth,
grdTerrain=self.grdTerrain,
framegrab=fg)
elif featureType.lower() == 'timeseries':
DAPloaders.runTimeSeriesLoader(
url, self.campaignName, self.campaignDescription,
aName, pName, self.colors[pName.lower()], 'bed',
'deployment', self.bed_parms, self.dbAlias, stride)
self.addPlatformResources(
'http://stoqs.mbari.org/x3d/beds/beds_housing_with_axes_src_scene.x3d',
pName,
scalefactor=10)
except (DAPloaders.OpendapError, DAPloaders.InvalidSliceRequest,
webob.exc.HTTPError):
pass
def loadRCpctd(self, stride=None, platformName='RachelCarson_PCTD', activitytypeName='Rachel Carson Profile CTD Data'):
'''
RC pctd specific load functions
'''
stride = stride or self.stride
#platformName = 'rc_pctd'
for (aName, file) in zip([ a.split('.')[0] + getStrideText(stride) for a in self.rcpctd_files], self.rcpctd_files):
url = self.rcpctd_base + file
DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['carson'], 'ship', activitytypeName,
self.rcpctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
# load all the bottles
sl = SeabirdLoader(aName[:5], platformName, dbAlias=self.dbAlias, campaignName=self.campaignName, platformColor=self.colors['carson'], platformTypeName='ship', dodsBase=self.rcpctd_base)
if self.args.verbose:
sl.logger.setLevel(logging.DEBUG)
sl.tdsBase= self.tdsBase
sl.process_btl_files(self.rcpctd_files)
def loadM1(self, stride=None):
'''
Mooring M1 specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.m1_files], self.m1_files):
url = os.path.join(self.m1_base, file)
dataStartDatetime = None
if self.args.append:
# Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(self.dbAlias).filter(activity__name=aName).aggregate(Max('timevalue'))['timevalue__max']
if dataStartDatetime:
# Subract an hour to fill in missing_values at end from previous load
dataStartDatetime = dataStartDatetime - timedelta(seconds=3600)
DAPloaders.runMooringLoader(url, self.campaignName, self.campaignDescription, aName, 'M1_Mooring', self.colors['m1'], 'mooring', 'Mooring Deployment',
self.m1_parms, self.dbAlias, stride, self.m1_startDatetime, self.m1_endDatetime, dataStartDatetime)
def loadBEDS(self, stride=None, featureType='trajectory'):
'''
BEDS specific load functions; featureType can be 'trajectory' or 'timeSeries'. Use 'trajectory' for events that we've fudged
into a trajectory netCDF file using the canyon's thalweg. Use 'timeSeries' for events for which the BED does not significantly translate.
'''
stride = stride or self.stride
for (aName, pName, file, plotTimeSeriesDepth) in zip(
[ a + ' (stride=%d)' % stride for a in self.bed_files],
self.bed_platforms, self.bed_files, self.bed_depths):
url = self.bed_base + file
if featureType == 'trajectory':
# To get timeSeries plotting for trajectories (in the Parameter tab of the UI) assign a plotTimeSeriesDepth value of the starting depth in meters.
DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, pName, self.colors[pName.lower()], 'bed', 'deployment',
self.bed_parms, self.dbAlias, stride, plotTimeSeriesDepth=plotTimeSeriesDepth, grdTerrain=self.grdTerrain)
elif featureType == 'timeSeries':
DAPloaders.runTimeSeriesLoader(url, self.campaignName, self.campaignDescription, aName, pName, self.colors[pName.lower()], 'bed', 'deployment',
self.bed_parms, self.dbAlias, stride)
# Leave commented out to indicate how this would be used (X3DOM can't handle old style timestamp routing that we used to do in VRML)
##self.addPlaybackResources(x3dplaybackurl, aName)
self.addPlatformResources('http://dods.mbari.org/data/beds/x3d/beds_housing_with_axes.x3d', pName)
Max('timevalue'))['timevalue__max']
try:
if not args.debug:
logger.info("Instantiating Lrauv_Loader for url = %s",
url_src)
lrauvLoad = DAPloaders.runLrauvLoader(
cName=args.campaign,
cDesc=None,
aName=aName,
aTypeName='LRAUV mission',
pName=platformName,
pTypeName='auv',
pColor=cl.colors[platformName],
url=url_src,
parmList=args.plotparms,
dbAlias=args.database,
stride=int(args.stride),
startDatetime=startDatetime,
dataStartDatetime=dataStartDatetime,
endDatetime=endDatetime,
contourUrl=args.contourUrl,
auxCoords=coord,
timezone='America/Los_Angeles',
command_line_args=args)
endDatetimeUTC = pytz.utc.localize(endDatetime)
endDatetimeLocal = endDatetimeUTC.astimezone(
pytz.timezone('America/Los_Angeles'))
startDatetimeUTC = pytz.utc.localize(startDatetime)
startDatetimeLocal = startDatetimeUTC.astimezone(
# We have a new folderName, set the end time for the previous Activity and "close" that log set
mod.Activity.objects.filter(name=lastAName).update(
enddate=lastAEndTime)
logger.info(
"Set endDatetime = %s for previous Activity.id = %s",
a.enddate, a.id)
# Tethys loads from TDS on beach - create with Null end time, we don't know the end until we have the next folder
aName = activityBaseName + folderName
##newTethysDatetime = newTethysDatetime - timedelta(hours = 4)
try:
lrauvLoad = DAPloaders.Lrauv_Loader(
activityName=aName,
url=newTethysURL,
startDatetime=folderStart,
endDatetime=None,
dataStartDatetime=lastTethysDatetime,
platformName='tethys',
stride=1)
except DAPloaders.NoValidData:
# Make sure we don't visit this startFolder again - add 1 second to it
startFolderName = (
datetime(*time.strptime(folderName, '%Y%m%dT%H%M%S')[:6]) +
timedelta(seconds=1)).strftime('%Y%m%dT%H%M%S')
logger.info(
"No measurements in this log set. Activity was not created as there was nothing to load."
)
if not previousFolderName:
logger.info(
"previousFolderName = None, indicating that we are looking for valid data in the last folder"
)
# Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(
args.database).filter(activity__name=aName).aggregate(
Max('timevalue'))['timevalue__max']
try:
logger.debug("Instantiating Lrauv_Loader for url = %s",
url_src)
lrauvLoad = DAPloaders.runLrauvLoader(
cName=args.campaign,
cDesc=None,
aName=aName,
aTypeName='LRAUV mission',
pName=platformName,
pTypeName='auv',
pColor=colors[platformName],
url=url_src,
parmList=parms,
dbAlias=args.database,
stride=args.stride,
startDatetime=startDatetime,
dataStartDatetime=dataStartDatetime,
endDatetime=endDatetime)
except DAPloaders.NoValidData:
logger.info(
"No measurements in this log set. Activity was not created as there was nothing to load."
)
except pydap.exceptions.ServerError as e:
logger.warn(e)
__doc__ = '''
Use DAPloaders.py to load data from realtime Tethys shore.nc files into the
stoqs_realtime database.
Mike McCann
MBARI 17 May 2011
@var __date__: Date of last svn commit
@undocumented: __doc__ parser
@status: production
@license: GPL
'''
import sys
sys.path.insert(0, "/home/stoqsadm/dev/stoqs/src/loaders")
import DAPloaders
from datetime import datetime
# Tethys loads from TDS on beach
lrauvLoad = DAPloaders.Lrauv_Loader(
'Tethys realtime - May 2011 (testing 1)',
url='http://beach.mbari.org:8080/thredds/dodsC/agg/tethys_ctd',
startDatetime=datetime(2011, 5, 8),
endDatetime=datetime(2011, 5, 16),
platformName='tethys',
stride=1)
lrauvLoad.process_data()
