Exemplo n.º 1
0
def loadMissions(baseUrl,
                 fileList,
                 activityName,
                 campaignName,
                 pName,
                 pColor,
                 pTypeName,
                 aTypeName,
                 parmList,
                 dbName,
                 stride=1):
    '''Load missions from OPeNDAP url from either a list of files from a base or a single URL with a given activityName '''

    if fileList:
        for (aName, file) in zip([a + ' (stride=%d)' % stride for a in files],
                                 fileList):
            url = baseUrl + file
            print "loadMissions(): Calling runLoader() with parmList = %s" % parmList
            DAPloaders.runTrajectoryLoader(url, campaignName, aName, pName,
                                           pColor, pTypeName, aTypeName,
                                           parmList, dbName, stride)
    elif activityName:
        url = baseUrl
        DAPloaders.runTrajectoryLoader(url, campaignName, activityName, pName,
                                       pColor, pTypeName, aTypeName, parmList,
                                       dbName, stride)
    else:
        print "loadMissions(): Must specify either a fileList or an activityName"
Exemplo n.º 2
0
    def loadM1(self, stride=None):
        '''
        Mooring M1 specific load functions
        '''
        platformName = 'M1_Mooring'
        stride = stride or self.stride
        for (aName, file) in zip([ a + getStrideText(stride) for a in self.m1_files], self.m1_files):
            url = os.path.join(self.m1_base, file)
            
            dataStartDatetime = None
            if self.args.append:
                # Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
                dataStartDatetime = InstantPoint.objects.using(self.dbAlias).filter(activity__name=aName).aggregate(Max('timevalue'))['timevalue__max']
                if dataStartDatetime:
                    # Subract an hour to fill in missing_values at end from previous load
                    dataStartDatetime = dataStartDatetime - timedelta(seconds=3600)

            DAPloaders.runMooringLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['m1'], 'mooring', 'Mooring Deployment', 
                                        self.m1_parms, self.dbAlias, stride, self.m1_startDatetime, self.m1_endDatetime, dataStartDatetime)
    
        # For timeseriesProfile data we need to pass the nominaldepth of the plaform
        # so that the model is put at the correct depth in the Spatial -> 3D view.
        try:
            self.addPlatformResources('http://stoqs.mbari.org/x3d/m1_assembly/m1_assembly_scene.x3d', 
                                      platformName, nominaldepth=self.m1_nominaldepth)
        except AttributeError:
            self.addPlatformResources('http://stoqs.mbari.org/x3d/m1_assembly/m1_assembly_scene.x3d', 
                                      platformName)
Exemplo n.º 3
0
    def loadTethys(self, stride=None):
        '''
        Tethys specific load functions
        '''
        pName = 'tethys'
        stride = stride or self.stride
        for (aName,
             f) in zip([a + getStrideText(stride) for a in self.tethys_files],
                       self.tethys_files):
            url = self.tethys_base + f
            # shorten the activity names
            aName = aName.rsplit('/', 1)[-1]

            try:
                DAPloaders.runLrauvLoader(url,
                                          self.campaignName,
                                          self.campaignDescription,
                                          aName,
                                          pName,
                                          self.colors['tethys'],
                                          'auv',
                                          'AUV mission',
                                          self.tethys_parms,
                                          self.dbAlias,
                                          stride,
                                          grdTerrain=self.grdTerrain,
                                          command_line_args=self.args,
                                          timezone='America/New_York')
            except DAPloaders.NoValidData:
                self.logger.info("No valid data in %s" % url)

        self.addPlatformResources(
            'http://stoqs.mbari.org/x3d/lrauv/lrauv_tethys.x3d', pName)
Exemplo n.º 4
0
    def load_glider_ctd(self, stride=None):
        '''
        Glider load functions.  Requires apriori knowledge of glider file names so we can extract platform and color name
        To be used with gliders that follow the same naming convention, i.e. nemesis_ctd.nc, ucsc260_ctd.nc
        and that load the exact same parameters, i.e. TEMP, PSAL or TEMP, PSAL, FLU2 or TEMP, FLU2, OPBS etc
        '''
        stride = stride or self.stride
        for (aName, file) in zip(
            [a + getStrideText(stride) for a in self.glider_ctd_files],
                self.glider_ctd_files):
            url = self.glider_ctd_base + file
            pName = aName.split('/')[-1].split('.')[0]
            p = re.compile('-\d+T\d+_Time')
            pName = p.sub('', pName)
            if pName.find('-') != -1:
                logger.warn(
                    "Replacing '-' characters in platform name %s with '_'s",
                    pName)
                pName = pName.replace('-', '_')

            logger.info("Executing runGliderLoader with url = %s", url)
            DAPloaders.runGliderLoader(url, self.campaignName, aName, pName,
                                       'FFBA26', 'glider', 'Glider Mission',
                                       self.glider_ctd_parms, self.dbAlias,
                                       stride, self.glider_ctd_startDatetime,
                                       self.glider_ctd_endDatetime)
Exemplo n.º 5
0
    def loadTethys(self, stride=None):
        '''
        Tethys specific load functions
        '''
        pName = 'tethys'
        stride = stride or self.stride
        for (aName, f) in zip([ a + getStrideText(stride) for a in self.tethys_files], self.tethys_files):
            url = self.tethys_base + f
            dataStartDatetime = None
            startDatetime = self.tethys_startDatetime
            endDatetime = self.tethys_endDatetime
            if self.args.append:
                # Return datetime of last timevalue - if data are loaded from multiple
                # activities return the earliest last datetime value
                dataStartDatetime = InstantPoint.objects.using(self.dbAlias).filter(
                                            activity__name=aName).aggregate(
                                            Max('timevalue'))['timevalue__max']

            try:
                DAPloaders.runLrauvLoader(url, self.campaignName, self.campaignDescription, aName,
                                          pName, self.colors['tethys'], 'auv', 'AUV mission',
                                          self.tethys_parms, self.dbAlias, stride,
                                          grdTerrain=self.grdTerrain, dataStartDatetime=dataStartDatetime,
                                          endDatetime=endDatetime, startDatetime=startDatetime, timezone='America/New_York')
            except DAPloaders.NoValidData:
                self.logger.info("No valid data in %s" % url)

        self.addPlatformResources('http://stoqs.mbari.org/x3d/lrauv/lrauv_tethys.x3d', pName)
Exemplo n.º 6
0
 def loadBEDS(self, stride=None, featureType='trajectory'):
     '''
     BEDS specific load functions; featureType can be 'trajectory' or 'timeSeries'.
     Use 'trajectory' for events that we've fudged into a trajectory netCDF file
     using the canyon's thalweg.  Use 'timeSeries' for events for which the BED
     does not significantly translate.
     '''
     stride = stride or self.stride
     for (aName, pName, file, plotTimeSeriesDepth, fg) in zip(
                         [ a.split('/')[-1] + ' (stride=%d)' % stride for a in self.bed_files], 
                         self.bed_platforms, self.bed_files, self.bed_depths, self.bed_framegrabs):
         url = os.path.join(self.bed_base, file)
         try:
             if featureType.lower() == 'trajectory':
                 # To get timeSeries plotting for trajectories (in the Parameter tab of the UI) 
                 # assign a plotTimeSeriesDepth value of the starting depth in meters.
                 DAPloaders.runBEDTrajectoryLoader(url, self.campaignName, self.campaignDescription,
                                                   aName, pName, self.colors[pName.lower()], 'bed',
                                                   'deployment', self.bed_parms, self.dbAlias, stride,
                                                   plotTimeSeriesDepth=plotTimeSeriesDepth,
                                                   grdTerrain=self.grdTerrain, framegrab=fg)
             elif featureType.lower() == 'timeseries':
                 DAPloaders.runTimeSeriesLoader(url, self.campaignName, self.campaignDescription,
                                                aName, pName, self.colors[pName.lower()], 'bed', 
                                                'deployment', self.bed_parms, self.dbAlias, stride)
             self.addPlatformResources('http://stoqs.mbari.org/x3d/beds/beds_housing_with_axes_src_scene.x3d',
                                       pName, scalefactor=10)
         except (DAPloaders.OpendapError, DAPloaders.InvalidSliceRequest):
             pass
Exemplo n.º 7
0
 def loadCastaway(self):
     '''
     Sparus specific load functions
     '''
     for (aName, file) in zip([ a + ' (stride=%d)' % self.stride for a in self.castaway_files], self.castaway_files):
         url = self.castaway_base + file
         DAPloaders.runTrajectoryLoader(url, self.campaignName, aName, 'castaway', self.colors['castaway'], 'mooring', 'CTD Profile', 
                                     self.castaway_parms, self.dbAlias, self.stride)
Exemplo n.º 8
0
 def loadSparus(self):
     '''
     Sparus specific load functions
     '''
     for (aName, file) in zip([ a + ' (stride=%d)' % self.stride for a in self.sparus_files], self.sparus_files):
         url = self.sparus_base + file
         DAPloaders.runTrajectoryLoader(url, self.campaignName, aName, 'sparus', self.colors['sparus'], 'auv', 'AUV mission', 
                                     self.sparus_parms, self.dbAlias, self.stride)
Exemplo n.º 9
0
 def loadM1met(self, stride=None):
     '''
     Mooring M1met specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.m1met_files], self.m1met_files):
         url = self.m1met_base + file
         DAPloaders.runMooringLoader(url, self.campaignName, self.campaignDescription, aName, 'M1_Mooring', self.colors['m1'], 'mooring', 'Mooring Deployment', 
                                     self.m1met_parms, self.dbAlias, stride, self.m1met_startDatetime, self.m1met_endDatetime)
Exemplo n.º 10
0
 def loadJMuctd(self, stride=None):
     '''
     Martin specific underway load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.JMuctd_files], self.JMuctd_files):
         url = self.JMuctd_base + file
         DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, 'John_Martin_UCTD', self.colors['martin'], 'ship', 'cruise', 
                                     self.JMuctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
Exemplo n.º 11
0
 def loadCarmen(self, stride=None):
     '''
     Glider specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.carmen_files], self.carmen_files):
         url = self.carmen_base + file
         DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, 'carmen', self.colors['carmen'], 'glider', 'Glider Mission', 
                                     self.carmen_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
Exemplo n.º 12
0
 def loadESPbruce(self, stride=None):
     '''
     ESPbruce specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.espbruce_files], self.espbruce_files):
         url = self.espbruce_base + file
         DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, 'espbruce', self.colors['espbruce'], 'espbruce', 'ESP bruce Mission', 
                                     self.espbruce_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
Exemplo n.º 13
0
 def loadRCuctd(self, stride=None, platformName='RachelCarson_UCTD', activitytypeName='Rachel Carson Underway CTD Data'):
     '''
     RC uctd specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.rcuctd_files], self.rcuctd_files):
         url = self.rcuctd_base + file
         DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['carson'], 'ship', activitytypeName, 
                                     self.rcuctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
Exemplo n.º 14
0
 def loadFulmar(self, stride=None):
     '''
     Fulmar specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.fulmar_files], self.fulmar_files):
         url = self.fulmar_base + file
         DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, 'fulmar', self.colors['fulmar'], 'ship', 'cruise', 
                                     self.fulmar_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
Exemplo n.º 15
0
 def loadOA2met(self, stride=None):
     '''
     Mooring OA specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.OA2met_files], self.OA2met_files):
         url = os.path.join(self.OA2met_base, file)
         DAPloaders.runMooringLoader(url, self.campaignName, self.campaignDescription, aName, 'OA2_Mooring', self.colors['oa2'], 'mooring', 'Mooring Deployment', 
                                     self.OA2met_parms, self.dbAlias, stride, self.OA2met_startDatetime, self.OA2met_endDatetime)
Exemplo n.º 16
0
 def load_slocum_260(self, stride=None):
     '''
     Glider specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.slocum_260_files], self.slocum_260_files):
         url = self.slocum_260_base + file
         DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, 'Slocum_260', self.colors['slocum_260'], 'glider', 'Glider Mission', 
                                     self.slocum_260_parms, self.dbAlias, stride, self.slocum_260_startDatetime, self.slocum_260_endDatetime, grdTerrain=self.grdTerrain)
Exemplo n.º 17
0
 def loadESPmack(self, stride=None):
     '''
     ESPmack specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.espmack_files], self.espmack_files):
         url = self.espmack_base + file
         print "url = %s" % url
         DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, 'ESP_Mack_Drifter', self.colors['espmack'], 'espmack', 'ESP mack Mission', 
                                     self.espmack_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
Exemplo n.º 18
0
 def loadDorado(self, stride=None):
     '''
     Dorado specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.dorado_files], self.dorado_files):
         url = self.dorado_base + file
         DAPloaders.runDoradoLoader(url, self.campaignName, self.campaignDescription, aName, 'Dorado', self.colors['dorado'], 'auv', 'AUV mission', 
                                     self.dorado_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
         load_gulps(aName, file, self.dbAlias)
Exemplo n.º 19
0
 def load_NPS34(self, stride=None):
     '''
     Glider specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.nps34_files], self.nps34_files):
         url = self.nps34_base + file
         print "url = %s" % url
         DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, 'NPS_Glider_34', self.colors['nps34'], 'glider', 'Glider Mission', 
                                     self.nps34_parms, self.dbAlias, stride, self.nps34_startDatetime, self.nps34_endDatetime, grdTerrain=self.grdTerrain)
Exemplo n.º 20
0
 def load_wg_tex_met(self, stride=None):
     '''
     Glider specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.wg_tex_met_files], self.wg_tex_met_files):
         url = self.wg_tex_met_base + file
         DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, 'wg_Tex_Glider', self.colors['wg_tex'], 'waveglider', 'Glider Mission', 
                                     self.wg_tex_met_parms, self.dbAlias, stride, self.wg_tex_met_startDatetime, self.wg_tex_met_endDatetime,
                                     grdTerrain=self.grdTerrain)
Exemplo n.º 21
0
 def loadStella(self, stride=None):
     '''
     Stella drift specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.stella_files], self.stella_files):
         url = self.stella_base + file
         dname='Stella' + aName[6:9]
         DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, dname, self.colors[dname], 'drifter', 'Stella drifter Mission', 
                                     self.stella_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
Exemplo n.º 22
0
 def loadRusalka(self, stride=None):
     '''
     Glider specific load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.rusalka_files], self.rusalka_files):
         url = self.rusalka_base + file
         print "url = %s" % url
         DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, 'rusalka', self.colors['rusalka'], 'glider', 'Glider Mission', 
                                     self.rusalka_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
Exemplo n.º 23
0
    def load(self, job):
        data_start = None
        print('====>Processing decimated nc file {}'.format(job.activity_name))
        (url_dest, start, end) = self.process_decimated(job)

        print('====>Loading {}'.format(job.activity_name))
        if self.args.append:
            core_name = job.activity_name.split('_')[0]
            # Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
            data_start = \
            InstantPoint.objects.using(self.args.database).filter(activity__name__contains=core_name).aggregate(
                Max('timevalue'))['timevalue__max']

        coord = {} 
        for p in self.args.plotparms:
            coord[p] = {'time': p + '_time', 'latitude': p + '_latitude', 'longitude': p + '_longitude',
                        'depth': p + '_depth'}
        try:
            if not self.args.debug:
                logger.info("Instantiating Lrauv_Loader for url = {}".format(url_dest))
                DAPloaders.runLrauvLoader(cName = self.args.campaign,
                                                  cDesc = None,
                                                  aName = job.activity_name,
                                                  aTypeName = 'LRAUV mission',
                                                  pName = self.vehicle,
                                                  pTypeName = 'auv',
                                                  pColor = self.cl.colors[self.vehicle],
                                                  url = url_dest,
                                                  parmList = self.args.parms,
                                                  dbAlias = self.args.database,
                                                  stride = int(self.args.stride),
                                                  startDatetime = start,
                                                  dataStartDatetime = data_start,
                                                  endDatetime = end,
                                                  contourUrl = self.args.contourUrl,
                                                  auxCoords = coord,
                                                  timezone = 'America/Los_Angeles',
                                                  command_line_args = self.args)
        except DAPloaders.NoValidData:
            logger.info("No measurements in this log set. Activity was not created as there was nothing to load.")

        except pydap.exceptions.ServerError as e:
            logger.warning(e)

        except DAPloaders.ParameterNotFound as e:
            logger.warning(e)

        except DAPloaders.InvalidSliceRequest as e:
            logger.warning(e)

        except Exception as e:
            logger.warning(e)
            
        return url_dest, start, end
Exemplo n.º 24
0
 def loadWFuctd(self, stride=None, platformName='WesternFlyer_UCTD', activitytypeName='Western Flyer Underway CTD Data'):
     '''
     WF uctd specific load functions.  Override defaults for @platformName and activitytypeName if it's desired
     to consider uctd and pctd coming from the same platform.  You may want to do this to use the data 
     visualization capabilities in STOQS.
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.wfuctd_files], self.wfuctd_files):
         url = self.wfuctd_base + file
         print "url = %s" % url
         DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['flyer'], 'ship', activitytypeName,
                                     self.wfuctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
Exemplo n.º 25
0
    def loadMackMoor(self, stride=None):
        '''
        Mooring Mack specific load functions
        '''
        stride = stride or self.stride
        pName = 'ESP_Mack_Mooring'
        for (aName, file) in zip([ a + getStrideText(stride) for a in self.mack_moor_files], self.mack_moor_files):
            url = os.path.join(self.mack_moor_base, file)
            DAPloaders.runMooringLoader(url, self.campaignName, self.campaignDescription, aName, pName, self.colors['espmack'], 'mooring', 'Mooring Deployment',
                                       self.mack_moor_parms, self.dbAlias, stride, self.mack_moor_startDatetime, self.mack_moor_endDatetime)

        # Let browser code use {{STATIC_URL}} to fill in the /stoqs/static path
        self.addPlatformResources('x3d/ESPMooring/esp_base_scene.x3d', pName)
Exemplo n.º 26
0
def loadMissions(baseUrl, fileList, activityName, campaignName, pName, pColor, pTypeName, aTypeName, parmList, dbName, stride = 1):
    '''Load missions from OPeNDAP url from either a list of files from a base or a single URL with a given activityName '''
    c_desc = 'Mar Menor In Situ (AUV and Castaway) data loads'
    if fileList: 
        for (aName, f) in zip([ a + ' (stride=%d)' % stride for a in files], fileList):
            url = baseUrl + f
            print "loadMissions(): Calling runLoader() with parmList = %s" % parmList
            DAPloaders.runTrajectoryLoader(url, campaignName, c_desc, aName, pName, pColor, pTypeName, aTypeName, parmList, dbName, stride)
    elif activityName:
        url = baseUrl
        DAPloaders.runTrajectoryLoader(url, campaignName, c_desc, activityName, pName, pColor, pTypeName, aTypeName, parmList, dbName, stride)
    else:
        print "loadMissions(): Must specify either a fileList or an activityName"
Exemplo n.º 27
0
 def loadCastaway(self):
     '''
     Sparus specific load functions
     '''
     for (aName, file) in zip(
         [a + ' (stride=%d)' % self.stride for a in self.castaway_files],
             self.castaway_files):
         url = self.castaway_base + file
         DAPloaders.runTrajectoryLoader(url, self.campaignName, aName,
                                        'castaway', self.colors['castaway'],
                                        'mooring', 'CTD Profile',
                                        self.castaway_parms, self.dbAlias,
                                        self.stride)
Exemplo n.º 28
0
 def load_glider_met(self, stride=None):
     '''
     Glider load functions.  Requires apriori knowledge of glider file names so we can extract platform and color name
     To be used with gliders that follow the same naming convention, i.e. nemesis_met.nc, ucsc260_met.nc
     and that load the exact same parameters, i.e. meanu,meanv or windspeed, winddirection etc.
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.glider_met_files], self.glider_met_files):
         url = self.glider_met_base + file
         gplatform=aName.split('_')[0].upper() + '_Glider'
         gname=aName.split('_')[0].lower()
         DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, gplatform, self.colors[gname], 'glider', 'Glider Mission', 
                                     self.glider_met_parms, self.dbAlias, stride, self.glider_met_startDatetime, self.glider_met_endDatetime, grdTerrain=self.grdTerrain)
Exemplo n.º 29
0
 def loadSparus(self):
     '''
     Sparus specific load functions
     '''
     for (aName, file) in zip(
         [a + ' (stride=%d)' % self.stride for a in self.sparus_files],
             self.sparus_files):
         url = self.sparus_base + file
         DAPloaders.runTrajectoryLoader(url, self.campaignName, aName,
                                        'sparus', self.colors['sparus'],
                                        'auv', 'AUV mission',
                                        self.sparus_parms, self.dbAlias,
                                        self.stride)
Exemplo n.º 30
0
    def loadDorado(self, stride=None):
        '''
        Dorado specific load functions
        '''
        pName = 'dorado'
        stride = stride or self.stride
        for (aName, file) in zip([ a + getStrideText(stride) for a in self.dorado_files], self.dorado_files):
            url = self.dorado_base + file
            DAPloaders.runDoradoLoader(url, self.campaignName, self.campaignDescription, aName, pName, self.colors['dorado'], 'auv', 'AUV mission', 
                                        self.dorado_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
            load_gulps(aName, file, self.dbAlias)

        self.addPlatformResources('http://stoqs.mbari.org/x3d/dorado/simpleDorado389.x3d', pName)
Exemplo n.º 31
0
    def loadTethys(self, stride=None):
        '''
        Tethys specific load functions
        '''
        stride = stride or self.stride
        for (aName, file) in zip([ a + getStrideText(stride) for a in self.tethys_files], self.tethys_files):
            url = self.tethys_base + file
            dataStartDatetime = None
            if self.args.append:
                # Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
                dataStartDatetime = InstantPoint.objects.using(self.dbAlias).filter(activity__name=aName).aggregate(Max('timevalue'))['timevalue__max']

            DAPloaders.runLrauvLoader(url, self.campaignName, self.campaignDescription, aName, 'Tethys', self.colors['tethys'], 'auv', 'AUV mission', 
                                        self.tethys_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain, dataStartDatetime=dataStartDatetime)
Exemplo n.º 32
0
 def load_glider_ctd(self, stride=None):
     '''
     Glider load functions.  Requires apriori knowledge of glider file names so we can extract platform and color name
     To be used with gliders that follow the same naming convention, i.e. nemesis_ctd.nc, ucsc260_ctd.nc
     and that load the exact same parameters, i.e. TEMP, PSAL or TEMP, PSAL, FLU2 or TEMP, FLU2, OPBS etc
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.glider_ctd_files], self.glider_ctd_files):
         url = self.glider_ctd_base + file
         gplatform=aName.split('_')[0].upper() + '_Glider'
         gname=aName.split('_')[0].lower()
         print "url = %s" % url
         print "platform = %s" % gplatform 
         DAPloaders.runGliderLoader(url, self.campaignName, self.campaignDescription, aName, gplatform, self.colors[gname], 'glider', 'Glider Mission', 
                                     self.glider_ctd_parms, self.dbAlias, stride, self.glider_ctd_startDatetime, self.glider_ctd_endDatetime, grdTerrain=self.grdTerrain)
Exemplo n.º 33
0
 def loadJMpctd(self, stride=None, platformName='John_Martin_PCTD', activitytypeName='John Martin Profile CTD Data'):
     '''
     Martin specific underway load functions
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a + getStrideText(stride) for a in self.JMpctd_files], self.JMpctd_files):
         url = self.JMpctd_base + file
         DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['martin'], 'ship', activitytypeName,
                                     self.JMpctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
     # load all the bottles           
     sl = SeabirdLoader(aName[:5], platformName, dbAlias=self.dbAlias, campaignName=self.campaignName, platformColor=self.colors['martin'], platformTypeName='ship', dodsBase=self.JMpctd_base)
     if self.args.verbose:
         sl.logger.setLevel(logging.DEBUG)
     sl.tdsBase= self.tdsBase
     sl.process_btl_files(self.JMpctd_files)
Exemplo n.º 34
0
 def loadWFpctd(self, stride=None, platformName='WesternFlyer_PCTD', activitytypeName='Western Flyer Profile CTD Data'):
     '''
     WF pctd specific load functions. Override defaults for @platformName and activitytypeName if it's desired
     to consider uctd and pctd coming from the same platform.  You may want to do this to use the data 
     visualization capabilities in STOQS.
     '''
     stride = stride or self.stride
     for (aName, file) in zip([ a.split('.')[0] + getStrideText(stride) for a in self.wfpctd_files], self.wfpctd_files):
         url = self.wfpctd_base + file
         DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['flyer'], 'ship', activitytypeName, 
                                     self.wfpctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
     # Now load all the bottles           
     sl = SeabirdLoader('activity name', platformName, dbAlias=self.dbAlias, campaignName=self.campaignName, platformColor=self.colors['flyer'], dodsBase=self.wfpctd_base)
     if self.args.verbose:
         sl.logger.setLevel(logging.DEBUG)
     sl.tdsBase= self.tdsBase
     sl.process_btl_files(self.wfpctd_files)
Exemplo n.º 35
0
    def loadCCEBIN(self, stride=None):
        '''
        Mooring CCEBIN specific load functions
        '''
        platformName = 'CCEBIN'
        stride = stride or self.stride
        for (aName,
             f) in zip([a + getStrideText(stride) for a in self.ccebin_files],
                       self.ccebin_files):
            url = os.path.join(self.ccebin_base, f)

            dataStartDatetime = None
            if self.args.append:
                # Return datetime of last timevalue - if data are loaded from multiple
                # activities return the earliest last datetime value
                dataStartDatetime = InstantPoint.objects.using(
                    self.dbAlias).filter(activity__name=aName).aggregate(
                        Max('timevalue'))['timevalue__max']
                if dataStartDatetime:
                    # Subract an hour to fill in missing_values at end from previous load
                    dataStartDatetime = dataStartDatetime - timedelta(
                        seconds=3600)

            if 'adcp' in f:
                Mooring_Loader.getFeatureType = lambda self: 'timeseriesprofile'
            else:
                Mooring_Loader.getFeatureType = lambda self: 'timeseries'

            DAPloaders.runMooringLoader(
                url, self.campaignName, self.campaignDescription, aName,
                platformName, self.colors['ccebin'], 'mooring',
                'Mooring Deployment', self.ccebin_parms, self.dbAlias, stride,
                self.ccebin_startDatetime, self.ccebin_endDatetime,
                dataStartDatetime)

        # For timeseriesProfile data we need to pass the nominaldepth of the plaform
        # so that the model is put at the correct depth in the Spatial -> 3D view.
        try:
            self.addPlatformResources(
                'http://stoqs.mbari.org/x3d/cce_bin_assem/cce_bin_assem_src_scene.x3d',
                platformName,
                nominaldepth=self.ccebin_nominaldepth)
        except AttributeError:
            self.addPlatformResources(
                'http://stoqs.mbari.org/x3d/cce_bin_assem/cce_bin_assem_src_scene.x3d',
                platformName)
Exemplo n.º 36
0
 def loadBEDS(self, stride=None, featureType='trajectory'):
     '''
     BEDS specific load functions; featureType can be 'trajectory' or 'timeSeries'.
     Use 'trajectory' for events that we've fudged into a trajectory netCDF file
     using the canyon's thalweg.  Use 'timeSeries' for events for which the BED
     does not significantly translate.
     '''
     stride = stride or self.stride
     for (aName, pName, file, plotTimeSeriesDepth,
          fg) in zip([
              a.split('/')[-1] + ' (stride=%d)' % stride
              for a in self.bed_files
          ], self.bed_platforms, self.bed_files, self.bed_depths,
                     self.bed_framegrabs):
         url = os.path.join(self.bed_base, file)
         try:
             if featureType.lower() == 'trajectory':
                 # To get timeSeries plotting for trajectories (in the Parameter tab of the UI)
                 # assign a plotTimeSeriesDepth value of the starting depth in meters.
                 DAPloaders.runBEDTrajectoryLoader(
                     url,
                     self.campaignName,
                     self.campaignDescription,
                     aName,
                     pName,
                     self.colors[pName.lower()],
                     'bed',
                     'deployment',
                     self.bed_parms,
                     self.dbAlias,
                     stride,
                     plotTimeSeriesDepth=plotTimeSeriesDepth,
                     grdTerrain=self.grdTerrain,
                     framegrab=fg)
             elif featureType.lower() == 'timeseries':
                 DAPloaders.runTimeSeriesLoader(
                     url, self.campaignName, self.campaignDescription,
                     aName, pName, self.colors[pName.lower()], 'bed',
                     'deployment', self.bed_parms, self.dbAlias, stride)
             self.addPlatformResources(
                 'http://stoqs.mbari.org/x3d/beds/beds_housing_with_axes_src_scene.x3d',
                 pName,
                 scalefactor=10)
         except (DAPloaders.OpendapError, DAPloaders.InvalidSliceRequest,
                 webob.exc.HTTPError):
             pass
Exemplo n.º 37
0
    def loadRCpctd(self, stride=None, platformName='RachelCarson_PCTD', activitytypeName='Rachel Carson Profile CTD Data'):
        '''
        RC pctd specific load functions
        '''
        stride = stride or self.stride
        #platformName = 'rc_pctd'
        for (aName, file) in zip([ a.split('.')[0] + getStrideText(stride) for a in self.rcpctd_files], self.rcpctd_files):
            url = self.rcpctd_base + file
            DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, platformName, self.colors['carson'], 'ship', activitytypeName, 
                                        self.rcpctd_parms, self.dbAlias, stride, grdTerrain=self.grdTerrain)
        # load all the bottles           

        sl = SeabirdLoader(aName[:5], platformName, dbAlias=self.dbAlias, campaignName=self.campaignName, platformColor=self.colors['carson'], platformTypeName='ship', dodsBase=self.rcpctd_base)
        if self.args.verbose:
            sl.logger.setLevel(logging.DEBUG)
        sl.tdsBase= self.tdsBase
        sl.process_btl_files(self.rcpctd_files)
Exemplo n.º 38
0
    def loadM1(self, stride=None):
        '''
        Mooring M1 specific load functions
        '''
        stride = stride or self.stride
        for (aName, file) in zip([ a + getStrideText(stride) for a in self.m1_files], self.m1_files):
            url = os.path.join(self.m1_base, file)
            
            dataStartDatetime = None
            if self.args.append:
                # Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
                dataStartDatetime = InstantPoint.objects.using(self.dbAlias).filter(activity__name=aName).aggregate(Max('timevalue'))['timevalue__max']
                if dataStartDatetime:
                    # Subract an hour to fill in missing_values at end from previous load
                    dataStartDatetime = dataStartDatetime - timedelta(seconds=3600)

            DAPloaders.runMooringLoader(url, self.campaignName, self.campaignDescription, aName, 'M1_Mooring', self.colors['m1'], 'mooring', 'Mooring Deployment', 
                                        self.m1_parms, self.dbAlias, stride, self.m1_startDatetime, self.m1_endDatetime, dataStartDatetime)
Exemplo n.º 39
0
    def loadBEDS(self, stride=None, featureType='trajectory'):
        '''
        BEDS specific load functions; featureType can be 'trajectory' or 'timeSeries'.  Use 'trajectory' for events that we've fudged
        into a trajectory netCDF file using the canyon's thalweg.  Use 'timeSeries' for events for which the BED does not significantly translate.
        '''
        stride = stride or self.stride
        for (aName, pName, file, plotTimeSeriesDepth) in zip(
                            [ a + ' (stride=%d)' % stride for a in self.bed_files], 
                            self.bed_platforms, self.bed_files, self.bed_depths):
            url = self.bed_base + file
            if featureType == 'trajectory':
                # To get timeSeries plotting for trajectories (in the Parameter tab of the UI) assign a plotTimeSeriesDepth value of the starting depth in meters.
                DAPloaders.runTrajectoryLoader(url, self.campaignName, self.campaignDescription, aName, pName, self.colors[pName.lower()], 'bed', 'deployment', 
                                        self.bed_parms, self.dbAlias, stride, plotTimeSeriesDepth=plotTimeSeriesDepth, grdTerrain=self.grdTerrain)
            elif featureType == 'timeSeries':
                DAPloaders.runTimeSeriesLoader(url, self.campaignName, self.campaignDescription, aName, pName, self.colors[pName.lower()], 'bed', 'deployment', 
                                        self.bed_parms, self.dbAlias, stride)

            # Leave commented out to indicate how this would be used (X3DOM can't handle old style timestamp routing that we used to do in VRML)
            ##self.addPlaybackResources(x3dplaybackurl, aName)

            self.addPlatformResources('http://dods.mbari.org/data/beds/x3d/beds_housing_with_axes.x3d', pName)
Exemplo n.º 40
0
                            Max('timevalue'))['timevalue__max']

            try:
                if not args.debug:
                    logger.info("Instantiating Lrauv_Loader for url = %s",
                                url_src)
                    lrauvLoad = DAPloaders.runLrauvLoader(
                        cName=args.campaign,
                        cDesc=None,
                        aName=aName,
                        aTypeName='LRAUV mission',
                        pName=platformName,
                        pTypeName='auv',
                        pColor=cl.colors[platformName],
                        url=url_src,
                        parmList=args.plotparms,
                        dbAlias=args.database,
                        stride=int(args.stride),
                        startDatetime=startDatetime,
                        dataStartDatetime=dataStartDatetime,
                        endDatetime=endDatetime,
                        contourUrl=args.contourUrl,
                        auxCoords=coord,
                        timezone='America/Los_Angeles',
                        command_line_args=args)

                endDatetimeUTC = pytz.utc.localize(endDatetime)
                endDatetimeLocal = endDatetimeUTC.astimezone(
                    pytz.timezone('America/Los_Angeles'))
                startDatetimeUTC = pytz.utc.localize(startDatetime)
                startDatetimeLocal = startDatetimeUTC.astimezone(
Exemplo n.º 41
0
                # We have a new folderName, set the end time for the previous Activity and "close" that log set
                mod.Activity.objects.filter(name=lastAName).update(
                    enddate=lastAEndTime)
                logger.info(
                    "Set endDatetime = %s for previous Activity.id = %s",
                    a.enddate, a.id)

        # Tethys loads from TDS on beach - create with Null end time, we don't know the end until we have the next folder
        aName = activityBaseName + folderName
        ##newTethysDatetime = newTethysDatetime - timedelta(hours = 4)
        try:
            lrauvLoad = DAPloaders.Lrauv_Loader(
                activityName=aName,
                url=newTethysURL,
                startDatetime=folderStart,
                endDatetime=None,
                dataStartDatetime=lastTethysDatetime,
                platformName='tethys',
                stride=1)
        except DAPloaders.NoValidData:
            # Make sure we don't visit this startFolder again - add 1 second to it
            startFolderName = (
                datetime(*time.strptime(folderName, '%Y%m%dT%H%M%S')[:6]) +
                timedelta(seconds=1)).strftime('%Y%m%dT%H%M%S')
            logger.info(
                "No measurements in this log set. Activity was not created as there was nothing to load."
            )
            if not previousFolderName:
                logger.info(
                    "previousFolderName = None, indicating that we are looking for valid data in the last folder"
                )
Exemplo n.º 42
0
                # Return datetime of last timevalue - if data are loaded from multiple activities return the earliest last datetime value
                dataStartDatetime = InstantPoint.objects.using(
                    args.database).filter(activity__name=aName).aggregate(
                        Max('timevalue'))['timevalue__max']

            try:
                logger.debug("Instantiating Lrauv_Loader for url = %s",
                             url_src)
                lrauvLoad = DAPloaders.runLrauvLoader(
                    cName=args.campaign,
                    cDesc=None,
                    aName=aName,
                    aTypeName='LRAUV mission',
                    pName=platformName,
                    pTypeName='auv',
                    pColor=colors[platformName],
                    url=url_src,
                    parmList=parms,
                    dbAlias=args.database,
                    stride=args.stride,
                    startDatetime=startDatetime,
                    dataStartDatetime=dataStartDatetime,
                    endDatetime=endDatetime)

            except DAPloaders.NoValidData:
                logger.info(
                    "No measurements in this log set. Activity was not created as there was nothing to load."
                )

            except pydap.exceptions.ServerError as e:
                logger.warn(e)
Exemplo n.º 43
0
__doc__ = '''

Use DAPloaders.py to load data from realtime Tethys shore.nc files into the
stoqs_realtime database.

Mike McCann
MBARI 17 May 2011

@var __date__: Date of last svn commit
@undocumented: __doc__ parser
@status: production
@license: GPL
'''

import sys
sys.path.insert(0, "/home/stoqsadm/dev/stoqs/src/loaders")

import DAPloaders
from datetime import datetime

# Tethys loads from TDS on beach
lrauvLoad = DAPloaders.Lrauv_Loader(
    'Tethys realtime - May 2011 (testing 1)',
    url='http://beach.mbari.org:8080/thredds/dodsC/agg/tethys_ctd',
    startDatetime=datetime(2011, 5, 8),
    endDatetime=datetime(2011, 5, 16),
    platformName='tethys',
    stride=1)

lrauvLoad.process_data()