def loadTethys(self, stride=None):
'''
Tethys specific load functions
'''
pName = 'tethys'
stride = stride or self.stride
for (aName, f) in zip([ a + getStrideText(stride) for a in self.tethys_files], self.tethys_files):
url = self.tethys_base + f
dataStartDatetime = None
startDatetime = self.tethys_startDatetime
endDatetime = self.tethys_endDatetime
if self.args.append:
# Return datetime of last timevalue - if data are loaded from multiple
# activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(self.dbAlias).filter(
activity__name=aName).aggregate(
Max('timevalue'))['timevalue__max']
try:
DAPloaders.runLrauvLoader(url, self.campaignName, self.campaignDescription, aName,
pName, self.colors['tethys'], 'auv', 'AUV mission',
self.tethys_parms, self.dbAlias, stride,
grdTerrain=self.grdTerrain, dataStartDatetime=dataStartDatetime,
endDatetime=endDatetime, startDatetime=startDatetime, timezone='America/New_York')
except DAPloaders.NoValidData:
self.logger.info("No valid data in %s" % url)
self.addPlatformResources('http://stoqs.mbari.org/x3d/lrauv/lrauv_tethys.x3d', pName)
def loadTethys(self, stride=None):
'''
Tethys specific load functions
'''
pName = 'tethys'
stride = stride or self.stride
for (aName,
f) in zip([a + getStrideText(stride) for a in self.tethys_files],
self.tethys_files):
url = self.tethys_base + f
# shorten the activity names
aName = aName.rsplit('/', 1)[-1]
try:
DAPloaders.runLrauvLoader(url,
self.campaignName,
self.campaignDescription,
aName,
pName,
self.colors['tethys'],
'auv',
'AUV mission',
self.tethys_parms,
self.dbAlias,
stride,
grdTerrain=self.grdTerrain,
command_line_args=self.args,
timezone='America/New_York')
except DAPloaders.NoValidData:
self.logger.info("No valid data in %s" % url)
self.addPlatformResources(
'http://stoqs.mbari.org/x3d/lrauv/lrauv_tethys.x3d', pName)
def load(self, job):
data_start = None
print('====>Processing decimated nc file {}'.format(job.activity_name))
(url_dest, start, end) = self.process_decimated(job)
print('====>Loading {}'.format(job.activity_name))
if self.args.append:
core_name = job.activity_name.split('_')[0]
# Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
data_start = \
InstantPoint.objects.using(self.args.database).filter(activity__name__contains=core_name).aggregate(
Max('timevalue'))['timevalue__max']
coord = {}
for p in self.args.plotparms:
coord[p] = {'time': p + '_time', 'latitude': p + '_latitude', 'longitude': p + '_longitude',
'depth': p + '_depth'}
try:
if not self.args.debug:
logger.info("Instantiating Lrauv_Loader for url = {}".format(url_dest))
DAPloaders.runLrauvLoader(cName = self.args.campaign,
cDesc = None,
aName = job.activity_name,
aTypeName = 'LRAUV mission',
pName = self.vehicle,
pTypeName = 'auv',
pColor = self.cl.colors[self.vehicle],
url = url_dest,
parmList = self.args.parms,
dbAlias = self.args.database,
stride = int(self.args.stride),
startDatetime = start,
dataStartDatetime = data_start,
endDatetime = end,
contourUrl = self.args.contourUrl,
auxCoords = coord,
timezone = 'America/Los_Angeles',
command_line_args = self.args)
except DAPloaders.NoValidData:
logger.info("No measurements in this log set. Activity was not created as there was nothing to load.")
except pydap.exceptions.ServerError as e:
logger.warning(e)
except DAPloaders.ParameterNotFound as e:
logger.warning(e)
except DAPloaders.InvalidSliceRequest as e:
logger.warning(e)
except Exception as e:
logger.warning(e)
return url_dest, start, end
def loadTethys(self, stride=None):
'''
Tethys specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.tethys_files], self.tethys_files):
url = self.tethys_base + file
dataStartDatetime = None
if self.args.append:
# Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(self.dbAlias).filter(activity__name=aName).aggregate(Max('timevalue'))['timevalue__max']
DAPloaders.runLrauvLoader(url, self.campaignName, self.campaignDescription, aName, 'Tethys', self.colors['tethys'], 'auv', 'AUV mission',
self.tethys_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain, dataStartDatetime=dataStartDatetime)
def loadMakai(self, stride=None):
'''
Makai specific load functions
'''
stride = stride or self.stride
for (aName, file) in zip([ a + getStrideText(stride) for a in self.makai_files], self.makai_files):
url = self.makai_base + file
dataStartDatetime = None
startDatetime = self.makai_startDatetime
endDatetime = self.makai_endDatetime
if self.args.append:
# Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(self.dbAlias).filter(activity__name=aName).aggregate(Max('timevalue'))['timevalue__max']
try:
# Set stride to 1 for telemetered data
DAPloaders.runLrauvLoader(url, self.campaignName, self.campaignDescription, aName, 'Makai', self.colors['makai'], 'auv', 'AUV mission',
self.makai_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain, dataStartDatetime=dataStartDatetime,
endDatetime=endDatetime, startDatetime=startDatetime)
except DAPloaders.NoValidData as e:
self.logger.info("No valid data in %s" % url)
def loadTethys(self, stride=None):
'''
Tethys specific load functions
'''
pName = 'tethys'
stride = stride or self.stride
for (aName, f) in zip([ a + getStrideText(stride) for a in self.tethys_files], self.tethys_files):
url = self.tethys_base + f
# shorten the activity names
aName = aName.rsplit('/', 1)[-1]
try:
DAPloaders.runLrauvLoader(url, self.campaignName, self.campaignDescription, aName,
pName, self.colors['tethys'], 'auv', 'AUV mission',
self.tethys_parms, self.dbAlias, stride,
grdTerrain=self.grdTerrain, command_line_args=self.args,
timezone='America/New_York')
except DAPloaders.NoValidData:
self.logger.info("No valid data in %s" % url)
self.addPlatformResources('http://stoqs.mbari.org/x3d/lrauv/lrauv_tethys.x3d', pName)
Max('timevalue'))['timevalue__max']
try:
if not args.debug:
logger.info("Instantiating Lrauv_Loader for url = %s",
url_src)
lrauvLoad = DAPloaders.runLrauvLoader(
cName=args.campaign,
cDesc=None,
aName=aName,
aTypeName='LRAUV mission',
pName=platformName,
pTypeName='auv',
pColor=cl.colors[platformName],
url=url_src,
parmList=args.plotparms,
dbAlias=args.database,
stride=int(args.stride),
startDatetime=startDatetime,
dataStartDatetime=dataStartDatetime,
endDatetime=endDatetime,
contourUrl=args.contourUrl,
auxCoords=coord,
timezone='America/Los_Angeles',
command_line_args=args)
endDatetimeUTC = pytz.utc.localize(endDatetime)
endDatetimeLocal = endDatetimeUTC.astimezone(
pytz.timezone('America/Los_Angeles'))
startDatetimeUTC = pytz.utc.localize(startDatetime)
startDatetimeLocal = startDatetimeUTC.astimezone(
if args.append:
# Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(args.database).filter(activity__name=aName).aggregate(Max('timevalue'))['timevalue__max']
try:
if not args.debug:
logger.info("Instantiating Lrauv_Loader for url = %s", url_src)
lrauvLoad = DAPloaders.runLrauvLoader(cName = args.campaign,
cDesc = None,
aName = aName,
aTypeName = 'LRAUV mission',
pName = platformName,
pTypeName = 'auv',
pColor = colors[platformName],
url = url_src,
parmList = parm_list,
dbAlias = args.database,
stride = int(args.stride),
startDatetime = startDatetime,
dataStartDatetime = dataStartDatetime,
endDatetime = endDatetime,
contourUrl = args.contourUrl,
auxCoords = coord)
endDatetimeUTC = pytz.utc.localize(endDatetime)
endDatetimeLocal = endDatetimeUTC.astimezone(pytz.timezone('America/Los_Angeles'))
startDatetimeUTC = pytz.utc.localize(startDatetime)
startDatetimeLocal = startDatetimeUTC.astimezone(pytz.timezone('America/Los_Angeles'))
# format contour output file name replacing file extension with .png
if args.outDir.startswith('/tmp'):
core_aName = aName.split('_')[0]
# Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(args.database).filter(activity__name__contains=core_aName).aggregate(Max('timevalue'))['timevalue__max']
try:
if not args.debug:
logger.info("Instantiating Lrauv_Loader for url = %s", url_src)
lrauvLoad = DAPloaders.runLrauvLoader(cName = args.campaign,
cDesc = None,
aName = aName,
aTypeName = 'LRAUV mission',
pName = platformName,
pTypeName = 'auv',
pColor = cl.colors[platformName],
url = url_src,
parmList = args.plotparms,
dbAlias = args.database,
stride = int(args.stride),
startDatetime = startDatetime,
dataStartDatetime = dataStartDatetime,
endDatetime = endDatetime,
contourUrl = args.contourUrl,
auxCoords = coord,
timezone = 'America/Los_Angeles',
command_line_args = args)
endDatetimeUTC = pytz.utc.localize(endDatetime)
endDatetimeLocal = endDatetimeUTC.astimezone(pytz.timezone('America/Los_Angeles'))
startDatetimeUTC = pytz.utc.localize(startDatetime)
startDatetimeLocal = startDatetimeUTC.astimezone(pytz.timezone('America/Los_Angeles'))
# format contour output file name replacing file extension with .png
# Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
dataStartDatetime = InstantPoint.objects.using(
args.database).filter(activity__name=aName).aggregate(
Max('timevalue'))['timevalue__max']
try:
logger.debug("Instantiating Lrauv_Loader for url = %s",
url_src)
lrauvLoad = DAPloaders.runLrauvLoader(
cName=args.campaign,
cDesc=None,
aName=aName,
aTypeName='LRAUV mission',
pName=platformName,
pTypeName='auv',
pColor=colors[platformName],
url=url_src,
parmList=parms,
dbAlias=args.database,
stride=args.stride,
startDatetime=startDatetime,
dataStartDatetime=dataStartDatetime,
endDatetime=endDatetime)
except DAPloaders.NoValidData:
logger.info(
"No measurements in this log set. Activity was not created as there was nothing to load."
)
except pydap.exceptions.ServerError as e:
logger.warn(e)
dataStartDatetime = (
InstantPoint.objects.using(args.database)
.filter(activity__name=aName)
.aggregate(Max("timevalue"))["timevalue__max"]
)
try:
logger.debug("Instantiating Lrauv_Loader for url = %s", url_src)
lrauvLoad = DAPloaders.runLrauvLoader(
cName=args.campaign,
cDesc=None,
aName=aName,
aTypeName="LRAUV mission",
pName=platformName,
pTypeName="auv",
pColor=colors[platformName],
url=url_src,
parmList=parms,
dbAlias=args.database,
stride=args.stride,
startDatetime=startDatetime,
dataStartDatetime=dataStartDatetime,
endDatetime=endDatetime,
)
except DAPloaders.NoValidData:
logger.info("No measurements in this log set. Activity was not created as there was nothing to load.")
except pydap.exceptions.ServerError as e:
logger.warn(e)
except DAPloaders.ParameterNotFound as e:
