def buildParmDict(self): ''' Build parameter dictionary akin to that returned by pydap. The parameters from the .btl file must match the parameters read from the .nc file. See comments for mapping copied from pctdToNetCDF.py. ''' # Match the mapping done in pctdToNetCDF.py: # self.pr_list.append(float(r['PrDM'])) # self.depth = self.ncFile.createVariable('depth', 'float64', ('time',)) # self.depth.long_name = 'DEPTH' # self.depth.standard_name = 'depth' # self.depth.units = 'm' # self.depth[:] = csiro.depth(self.pr_list, self.lat_list) # Convert pressure to depth # self.t1_list.append(r['T190C']) # temp = self.ncFile.createVariable('TEMP', 'float64', ('time',)) # temp.long_name = 'Temperature, 2 [ITS-90, deg C]' # temp.standard_name = 'sea_water_temperature' # temp.units = 'Celsius' # self.sal_list.append(r['Sal00']) # sal = self.ncFile.createVariable('PSAL', 'float64', ('time',)) # sal.long_name = 'Salinity, Practical [PSU]' # sal.standard_name = 'sea_water_salinity' # self.xmiss_list.append(r['Xmiss']) # xmiss = self.ncFile.createVariable('xmiss', 'float64', ('time',)) # xmiss.long_name = 'Beam Transmission, Chelsea/Seatech' # xmiss.units = '%' # self.ecofl_list.append(r['FlECO-AFL']) # ecofl = self.ncFile.createVariable('ecofl', 'float64', ('time',)) # ecofl.long_name = 'Fluorescence, WET Labs ECO-AFL/FL' # ecofl.units = 'mg/m^3' # self.oxygen_list.append(r['Sbeox0ML/L']) # oxygen = self.ncFile.createVariable('oxygen', 'float64', ('time',)) # oxygen.long_name = 'Oxygen, SBE 43' # oxygen.units = 'ml/l' parmDict = {} pr = BaseType('nameless') pr.attributes = {'colname': 'PrDM', 'units': 'm' , 'long_name': 'DEPTH', 'standard_name': 'depth'} temp = BaseType('nameless') temp.attributes = {'colname': 'T190C', 'units': 'ITS-90, deg C', 'long_name': 'temperature', 'standard_name': 'sea_water_temperature'} sal = BaseType('nameless') sal.attributes = {'colname': 'Sal00', 'units': '1' , 'long_name': 'salinity', 'standard_name': 'sea_water_salinity'} xmiss = BaseType('nameless') xmiss.attributes = {'colname': 'Xmiss', 'units': '%', 'long_name': 'Beam Transmission, Chelsea/Seatech'} ecofl = BaseType('nameless') ecofl.attributes = {'colname': 'FlECO-AFL', 'units': 'mg/m^3', 'long_name': 'Fluorescence, WET Labs ECO-AFL/FL'} wetstar = BaseType('nameless') wetstar.attributes = {'colname': 'WetStar', 'units': 'mg/m^3', 'long_name': 'Fluorescence, WET Labs WETstar'} oxygen = BaseType('nameless') oxygen.attributes = {'colname': 'Sbeox0ML/L', 'units': 'ml/l', 'long_name': 'Oxygen, SBE 43'} # The colname attribute must be the keys that DictReader returns - the keys of this dictionary will be the Parameter names in stoqs parmDict = {'pressure': pr, 'TEMP': temp, 'PSAL': sal, 'xmiss': xmiss, 'ecofl': ecofl, 'oxygen': oxygen, 'wetstar': wetstar} return parmDict
def process_csv_file(self, fh): ''' Iterate through lines of iterator to csv file and pull out data for loading into STOQS ''' ds = {} DA = BaseType('nameless') DA.attributes = { 'units': 'ng ml-1 ', 'long_name': 'Domoic Acid', 'standard_name': 'domoic_acid', 'type': 'float', 'description': 'Domoic acid', 'origin': 'www.sccoos.org' } PD = BaseType('nameless') PD.attributes = { 'units': 'cells l-1', 'long_name': 'Pseudo-nitzschia delicatissima group', 'standard_name': 'pseudo_nitzschia_delicatissima', 'name': 'pseudo_nitzschia_delicatissima', 'type': 'float', 'description': 'Pseudo-nitzschia delicatissima group (cells/L)', 'origin': 'www.sccoos.org' } PA = BaseType('nameless') PA.attributes = { 'units': 'cells l-1', 'long_name': 'Pseudo-nitzschia seriata group', 'standard_name': 'pseudo_nitzschia_seriata', 'name': 'pseudo_nitzschia_seriata', 'type': 'float', 'description': 'Pseudo-nitzschia seriata group (cells/L)', 'origin': 'www.sccoos.org' } alexandrium = BaseType('nameless') alexandrium.attributes = { 'units': 'cells l-1', 'long_name': 'Alexandrium', 'standard_name': 'alexandrium', 'name': 'alexandrium', 'type': 'float', 'description': 'Alexandrium spp. (cells/L)', 'origin': 'www.sccoos.org' } phosphate = BaseType('nameless') phosphate.attributes = { 'units': 'm-3 mol l-1', 'long_name': 'Phosphate', 'standard_name': 'phosphate_dissolved_in_seawater', 'name': 'Phosphate', 'type': 'float', 'description': 'Phosphate (uM)', 'origin': 'www.sccoos.org' } ammonia = BaseType('nameless') ammonia.attributes = { 'units': 'm-3 mol l-1', 'long_name': 'Ammonia', 'standard_name': 'ammonia_dissolved_in_seawater', 'name': 'ammonia_dissolved_in_sewater', 'type': 'float', 'description': 'Ammonia (uM)', 'origin': 'www.sccoos.org' } silicate = BaseType('nameless') silicate.attributes = { 'units': 'm-3 mol l-1', 'long_name': 'Silicate', 'standard_name': 'silicate_dissolved_in_seawater', 'name': 'silicate_dissolved_in_seawater', 'type': 'float', 'description': 'Silicate (uM)', 'origin': 'www.sccoos.org' } chlorophyll = BaseType('nameless') chlorophyll.attributes = { 'units': 'kg m-3', 'long_name': 'Chlorophyll', 'standard_name': 'mass_concentration_of_chlorophyll_in_sea_water', 'name': 'mass_concentration_of_chlorophyll_in_sea_water', 'type': 'float', 'description': 'Chlorophyll (kg/m3)', 'origin': 'www.sccoos.org' } prorocentrum = BaseType('nameless') prorocentrum.attributes = { 'units': 'cells l-1', 'long_name': 'Prorocentrum', 'standard_name': 'mass_concentration_of_prorocentrum_in_sea_water', 'name': 'mass_concentration_of_prorocentrum_in_sea_water', 'type': 'float', 'description': 'Prorocentrum spp. (cells/L)', 'origin': 'www.sccoos.org' } self.ds = { 'Domoic Acid (ng/mL)': DA, 'Pseudo-nitzschia seriata group (cells/L)': PA, 'Pseudo-nitzschia delicatissima group (cells/L)': PD, 'Phosphate (uM)': phosphate, 'Silicate (uM)': silicate, 'Ammonia (uM)': ammonia, 'Chlorophyll (mg/m3)': chlorophyll, 'Chlorophyll 1 (mg/m3)': chlorophyll, 'Chlorophyll 2 (mg/m3)': chlorophyll, 'Alexandrium spp. (cells/L)': alexandrium } self.include_names = [ 'Pseudo-nitzschia seriata group (cells/L)', 'Pseudo-nitzschia delicatissima group (cells/L)', 'Domoic Acid (ng/mL)', 'Chlorophyll (mg/m3)', 'Chlorophyll 1 (mg/m3)', 'Chlorophyll 2 (mg/m3)', 'Prorocentrum spp. (cells/L)', 'Silicate (uM)', 'Ammonia (uM)', 'Nitrate (uM)', 'Phosphate (uM)', 'Alexandrium spp. (cells/L)' ] self.initDB() for pn in self.include_names: self.parmCount[pn] = 0 reader = csv.reader(fh) for line in fh: # Skip all lines that don't begin with '"' nor ' ' then open that with csv.DictReader if not line.startswith('"') and not line.startswith(' '): titles = next(reader) reader = csv.DictReader(fh, titles) for r in reader: year = int(r['year']) month = int(r['month']) day = int(r['day']) time = r['time'] lat = float(r['latitude']) lon = float(r['longitude']) depth = float(r['depth (m)']) location = r['location'] hours = int(time.split(':')[0]) mins = int(time.split(':')[1]) secs = int(time.split(':')[2]) parmNameValues = [] for name in list(self.ds.keys()): if name.startswith('Chlorophyll'): parmNameValues.append( (name, 1e-5 * float(r[name]))) else: parmNameValues.append((name, float(r[name]))) # Check to make sure all data from this file are from the same location. # The program could be modified to read data in one file from multiple locations by reading data into a hash keyed by location name # and then stepping through each key of the hash saving the data for each location into it's own activity. For now just require # each data file to have data from just one location. try: if lat != lastlat or lon != lastlon: logger.error( "lat and lon are not the same for location = %s and lastlocation = %s. The input data should have just one location." % (location, lastlocation)) sys.exit(-1) except NameError as e: # Expected first time through when lastlon & lastlat don't yet exist pass # Load data dt = datetime(year, month, day, hours, mins, secs) self.load_measurement(lon, lat, depth, dt, parmNameValues) # Load sample bName = dt.isoformat() self.load_sample(lon, lat, depth, dt, bName) lastlat = lat lastlon = lon lastlocation = location logger.info("Data load complete, %d records loaded.", self.loaded) fh.close() # Update the Activity with information we now have following the load # Careful with the structure of this comment. It is parsed in views.py to give some useful links in showActivities() newComment = "%d MeasuredParameters loaded. Loaded on %sZ" % ( self.loaded, datetime.utcnow()) logger.info( "runHABLoader(): Updating its comment with newComment = %s", newComment) aName = location num_updated = m.Activity.objects.using(self.dbAlias).filter( id=self.activity.id).update(name=aName, comment=newComment, maptrack=None, mappoint=Point(lon, lat), mindepth=self.mindepth, maxdepth=self.maxdepth, num_measuredparameters=self.loaded, loaded_date=datetime.utcnow()) self.updateActivityParameterStats(self.parameterCount) self.updateCampaignStartEnd()
def buildParmDict(self): ''' Build parameter dictionary akin to that returned by pydap. The parameters from the .btl file must match the parameters read from the .nc file. See comments for mapping copied from pctdToNetCDF.py. ''' # Match the mapping done in pctdToNetCDF.py: # self.pr_list.append(float(r['PrDM'])) # self.depth = self.ncFile.createVariable('depth', 'float64', ('time',)) # self.depth.long_name = 'DEPTH' # self.depth.standard_name = 'depth' # self.depth.units = 'm' # self.depth[:] = csiro.depth(self.pr_list, self.lat_list) # Convert pressure to depth # self.t1_list.append(r['T190C']) # temp = self.ncFile.createVariable('TEMP', 'float64', ('time',)) # temp.long_name = 'Temperature, 2 [ITS-90, deg C]' # temp.standard_name = 'sea_water_temperature' # temp.units = 'Celsius' # self.sal_list.append(r['Sal00']) # sal = self.ncFile.createVariable('PSAL', 'float64', ('time',)) # sal.long_name = 'Salinity, Practical [PSU]' # sal.standard_name = 'sea_water_salinity' # self.xmiss_list.append(r['Xmiss']) # xmiss = self.ncFile.createVariable('xmiss', 'float64', ('time',)) # xmiss.long_name = 'Beam Transmission, Chelsea/Seatech' # xmiss.units = '%' # self.ecofl_list.append(r['FlECO-AFL']) # ecofl = self.ncFile.createVariable('ecofl', 'float64', ('time',)) # ecofl.long_name = 'Fluorescence, WET Labs ECO-AFL/FL' # ecofl.units = 'mg/m^3' # self.oxygen_list.append(r['Sbeox0ML/L']) # oxygen = self.ncFile.createVariable('oxygen', 'float64', ('time',)) # oxygen.long_name = 'Oxygen, SBE 43' # oxygen.units = 'ml/l' parmDict = {} pr = BaseType('nameless') pr.attributes = { 'colname': 'PrDM', 'units': 'm', 'long_name': 'DEPTH', 'standard_name': 'depth' } temp = BaseType('nameless') temp.attributes = { 'colname': 'T190C', 'units': 'ITS-90, deg C', 'long_name': 'temperature', 'standard_name': 'sea_water_temperature' } sal = BaseType('nameless') sal.attributes = { 'colname': 'Sal00', 'units': '1', 'long_name': 'salinity', 'standard_name': 'sea_water_salinity' } xmiss = BaseType('nameless') xmiss.attributes = { 'colname': 'Xmiss', 'units': '%', 'long_name': 'Beam Transmission, Chelsea/Seatech' } ecofl = BaseType('nameless') ecofl.attributes = { 'colname': 'FlECO-AFL', 'units': 'mg/m^3', 'long_name': 'Fluorescence, WET Labs ECO-AFL/FL' } wetstar = BaseType('nameless') wetstar.attributes = { 'colname': 'WetStar', 'units': 'mg/m^3', 'long_name': 'Fluorescence, WET Labs WETstar' } oxygen = BaseType('nameless') oxygen.attributes = { 'colname': 'Sbeox0ML/L', 'units': 'ml/l', 'long_name': 'Oxygen, SBE 43' } # The colname attribute must be the keys that DictReader returns - the keys of this dictionary will be the Parameter names in stoqs parmDict = { 'pressure': pr, 'TEMP': temp, 'PSAL': sal, 'xmiss': xmiss, 'ecofl': ecofl, 'oxygen': oxygen, 'wetstar': wetstar } return parmDict
def process_csv_file(self, fh): ''' Iterate through lines of iterator to csv file and pull out data for loading into STOQS ''' ds = {} DA = BaseType() DA.attributes = {'units': 'ng ml-1 ' , 'long_name': 'Domoic Acid', 'standard_name': 'domoic_acid', 'type': 'float', 'description': 'Domoic acid' , 'origin': 'www.sccoos.org' } PD = BaseType() PD.attributes = {'units': 'cells l-1', 'long_name': 'Pseudo-nitzschia delicatissima group', 'standard_name': 'pseudo_nitzschia_delicatissima', 'name': 'pseudo_nitzschia_delicatissima' , 'type': 'float' , 'description': 'Pseudo-nitzschia delicatissima group (cells/L)' , 'origin': 'www.sccoos.org' } PA = BaseType() PA.attributes = {'units': 'cells l-1', 'long_name': 'Pseudo-nitzschia seriata group', 'standard_name': 'pseudo_nitzschia_seriata', 'name': 'pseudo_nitzschia_seriata' , 'type': 'float' , 'description': 'Pseudo-nitzschia seriata group (cells/L)' , 'origin': 'www.sccoos.org' } alexandrium = BaseType() alexandrium.attributes = {'units': 'cells l-1', 'long_name': 'Alexandrium', 'standard_name': 'alexandrium', 'name': 'alexandrium' , 'type': 'float' , 'description': 'Alexandrium spp. (cells/L)' , 'origin': 'www.sccoos.org' } phosphate = BaseType() phosphate.attributes = {'units': 'm-3 mol l-1', 'long_name': 'Phosphate', 'standard_name': 'phosphate_dissolved_in_seawater', 'name': 'Phosphate' , 'type': 'float' , 'description': 'Phosphate (uM)' , 'origin': 'www.sccoos.org' } ammonia = BaseType() ammonia.attributes = {'units': 'm-3 mol l-1', 'long_name': 'Ammonia', 'standard_name': 'ammonia_dissolved_in_seawater', 'name': 'ammonia_dissolved_in_sewater' , 'type': 'float' , 'description': 'Ammonia (uM)' , 'origin': 'www.sccoos.org' } silicate = BaseType() silicate.attributes = {'units': 'm-3 mol l-1', 'long_name': 'Silicate', 'standard_name': 'silicate_dissolved_in_seawater', 'name': 'silicate_dissolved_in_seawater' , 'type': 'float' , 'description': 'Silicate (uM)' , 'origin': 'www.sccoos.org' } chlorophyll = BaseType() chlorophyll.attributes = {'units': 'kg m-3', 'long_name': 'Chlorophyll', 'standard_name': 'mass_concentration_of_chlorophyll_in_sea_water', 'name': 'mass_concentration_of_chlorophyll_in_sea_water' , 'type': 'float' , 'description': 'Chlorophyll (kg/m3)' , 'origin': 'www.sccoos.org' } prorocentrum = BaseType() prorocentrum.attributes = {'units': 'cells l-1', 'long_name': 'Prorocentrum', 'standard_name': 'mass_concentration_of_prorocentrum_in_sea_water', 'name': 'mass_concentration_of_prorocentrum_in_sea_water' , 'type': 'float' , 'description': 'Prorocentrum spp. (cells/L)' , 'origin': 'www.sccoos.org' } self.ds = { 'Domoic Acid (ng/mL)': DA, 'Pseudo-nitzschia seriata group (cells/L)': PA, 'Pseudo-nitzschia delicatissima group (cells/L)': PD, 'Phosphate (uM)': phosphate, 'Silicate (uM)': silicate, 'Ammonia (uM)': ammonia, 'Chlorophyll (mg/m3)': chlorophyll, 'Chlorophyll 1 (mg/m3)': chlorophyll, 'Chlorophyll 2 (mg/m3)': chlorophyll , 'Alexandrium spp. (cells/L)': alexandrium } self.include_names = ['Pseudo-nitzschia seriata group (cells/L)', 'Pseudo-nitzschia delicatissima group (cells/L)', 'Domoic Acid (ng/mL)', 'Chlorophyll (mg/m3)', 'Chlorophyll 1 (mg/m3)', 'Chlorophyll 2 (mg/m3)', 'Prorocentrum spp. (cells/L)', 'Silicate (uM)', 'Ammonia (uM)', 'Nitrate (uM)', 'Phosphate (uM)', 'Alexandrium spp. (cells/L)'] self.initDB() for pn in self.include_names: self.parmCount[pn] = 0 reader = csv.reader(fh) for line in fh: # Skip all lines that don't begin with '"' nor ' ' then open that with csv.DictReader if not line.startswith('"') and not line.startswith(' '): titles = reader.next() reader = csv.DictReader(fh, titles) for r in reader: year = int(r['year']) month = int(r['month']) day = int(r['day']) time = r['time'] lat = float(r['latitude']) lon = float(r['longitude']) depth = float(r['depth (m)']) location = r['location'] hours = int(time.split(':')[0]) mins = int(time.split(':')[1]) secs = int(time.split(':')[2]) parmNameValues = [] for name in self.ds.keys(): if name.startswith('Chlorophyll'): parmNameValues.append((name, 1e-5*float(r[name]))) else: parmNameValues.append((name, float(r[name]))) # Check to make sure all data from this file are from the same location. # The program could be modified to read data in one file from multiple locations by reading data into a hash keyed by location name # and then stepping through each key of the hash saving the data for each location into it's own activity. For now just require # each data file to have data from just one location. try: if lat != lastlat or lon != lastlon: logger.error("lat and lon are not the same for location = %s and lastlocation = %s. The input data should have just one location." % (location, lastlocation)) sys.exit(-1) except NameError, e: # Expected first time through when lastlon & lastlat don't yet exist pass # Load data dt = datetime(year, month, day, hours, mins, secs) self.load_measurement(lon, lat, depth, dt, parmNameValues) # Load sample bName = dt.isoformat() self.load_sample(lon, lat, depth, dt, bName) lastlat = lat lastlon = lon lastlocation = location
def process_csv_file(self, fh): ''' Iterate through lines of iterator to csv file and pull out data for loading into STOQS ''' ds = {} DA = BaseType('nameless') DA.attributes = {'units': 'ng ml-1 ' , 'long_name': 'Domoic Acid', 'standard_name': 'domoic_acid', 'type': 'float', 'description': 'Domoic acid' , 'origin': 'www.sccoos.org' } PD = BaseType('nameless') PD.attributes = {'units': 'cells l-1', 'long_name': 'Pseudo-nitzschia delicatissima group', 'standard_name': 'pseudo_nitzschia_delicatissima', 'name': 'pseudo_nitzschia_delicatissima' , 'type': 'float' , 'description': 'Pseudo-nitzschia delicatissima group (cells/L)' , 'origin': 'www.sccoos.org' } PA = BaseType('nameless') PA.attributes = {'units': 'cells l-1', 'long_name': 'Pseudo-nitzschia seriata group', 'standard_name': 'pseudo_nitzschia_seriata', 'name': 'pseudo_nitzschia_seriata' , 'type': 'float' , 'description': 'Pseudo-nitzschia seriata group (cells/L)' , 'origin': 'www.sccoos.org' } alexandrium = BaseType('nameless') alexandrium.attributes = {'units': 'cells l-1', 'long_name': 'Alexandrium', 'standard_name': 'alexandrium', 'name': 'alexandrium' , 'type': 'float' , 'description': 'Alexandrium spp. (cells/L)' , 'origin': 'www.sccoos.org' } phosphate = BaseType('nameless') phosphate.attributes = {'units': 'm-3 mol l-1', 'long_name': 'Phosphate', 'standard_name': 'phosphate_dissolved_in_seawater', 'name': 'Phosphate' , 'type': 'float' , 'description': 'Phosphate (uM)' , 'origin': 'www.sccoos.org' } ammonia = BaseType('nameless') ammonia.attributes = {'units': 'm-3 mol l-1', 'long_name': 'Ammonia', 'standard_name': 'ammonia_dissolved_in_seawater', 'name': 'ammonia_dissolved_in_sewater' , 'type': 'float' , 'description': 'Ammonia (uM)' , 'origin': 'www.sccoos.org' } silicate = BaseType('nameless') silicate.attributes = {'units': 'm-3 mol l-1', 'long_name': 'Silicate', 'standard_name': 'silicate_dissolved_in_seawater', 'name': 'silicate_dissolved_in_seawater' , 'type': 'float' , 'description': 'Silicate (uM)' , 'origin': 'www.sccoos.org' } chlorophyll = BaseType('nameless') chlorophyll.attributes = {'units': 'kg m-3', 'long_name': 'Chlorophyll', 'standard_name': 'mass_concentration_of_chlorophyll_in_sea_water', 'name': 'mass_concentration_of_chlorophyll_in_sea_water' , 'type': 'float' , 'description': 'Chlorophyll (kg/m3)' , 'origin': 'www.sccoos.org' } prorocentrum = BaseType('nameless') prorocentrum.attributes = {'units': 'cells l-1', 'long_name': 'Prorocentrum', 'standard_name': 'mass_concentration_of_prorocentrum_in_sea_water', 'name': 'mass_concentration_of_prorocentrum_in_sea_water' , 'type': 'float' , 'description': 'Prorocentrum spp. (cells/L)' , 'origin': 'www.sccoos.org' } self.ds = { 'Domoic Acid (ng/mL)': DA, 'Pseudo-nitzschia seriata group (cells/L)': PA, 'Pseudo-nitzschia delicatissima group (cells/L)': PD, 'Phosphate (uM)': phosphate, 'Silicate (uM)': silicate, 'Ammonia (uM)': ammonia, 'Chlorophyll (mg/m3)': chlorophyll, 'Chlorophyll 1 (mg/m3)': chlorophyll, 'Chlorophyll 2 (mg/m3)': chlorophyll , 'Alexandrium spp. (cells/L)': alexandrium } self.include_names = ['Pseudo-nitzschia seriata group (cells/L)', 'Pseudo-nitzschia delicatissima group (cells/L)', 'Domoic Acid (ng/mL)', 'Chlorophyll (mg/m3)', 'Chlorophyll 1 (mg/m3)', 'Chlorophyll 2 (mg/m3)', 'Prorocentrum spp. (cells/L)', 'Silicate (uM)', 'Ammonia (uM)', 'Nitrate (uM)', 'Phosphate (uM)', 'Alexandrium spp. (cells/L)'] self.initDB() for pn in self.include_names: self.parmCount[pn] = 0 reader = csv.reader(fh) for line in fh: # Skip all lines that don't begin with '"' nor ' ' then open that with csv.DictReader if not line.startswith('"') and not line.startswith(' '): titles = next(reader) reader = csv.DictReader(fh, titles) for r in reader: year = int(r['year']) month = int(r['month']) day = int(r['day']) time = r['time'] lat = float(r['latitude']) lon = float(r['longitude']) depth = float(r['depth (m)']) location = r['location'] hours = int(time.split(':')[0]) mins = int(time.split(':')[1]) secs = int(time.split(':')[2]) parmNameValues = [] for name in list(self.ds.keys()): if name.startswith('Chlorophyll'): parmNameValues.append((name, 1e-5*float(r[name]))) else: parmNameValues.append((name, float(r[name]))) # Check to make sure all data from this file are from the same location. # The program could be modified to read data in one file from multiple locations by reading data into a hash keyed by location name # and then stepping through each key of the hash saving the data for each location into it's own activity. For now just require # each data file to have data from just one location. try: if lat != lastlat or lon != lastlon: logger.error("lat and lon are not the same for location = %s and lastlocation = %s. The input data should have just one location." % (location, lastlocation)) sys.exit(-1) except NameError as e: # Expected first time through when lastlon & lastlat don't yet exist pass # Load data dt = datetime(year, month, day, hours, mins, secs) self.load_measurement(lon, lat, depth, dt, parmNameValues) # Load sample bName = dt.isoformat() self.load_sample(lon, lat, depth, dt, bName) lastlat = lat lastlon = lon lastlocation = location logger.info("Data load complete, %d records loaded.", self.loaded) fh.close() # Update the Activity with information we now have following the load # Careful with the structure of this comment. It is parsed in views.py to give some useful links in showActivities() newComment = "%d MeasuredParameters loaded. Loaded on %sZ" % (self.loaded, datetime.utcnow()) logger.info("runHABLoader(): Updating its comment with newComment = %s", newComment) aName = location num_updated = m.Activity.objects.using(self.dbAlias).filter(id = self.activity.id).update( name = aName, comment = newComment, maptrack = None, mappoint = 'POINT(%s %s)' % (lon, lat), mindepth = self.mindepth, maxdepth = self.maxdepth, num_measuredparameters = self.loaded, loaded_date = datetime.utcnow()) self.updateActivityParameterStats(self.parameterCount) self.updateCampaignStartEnd()
def buildParmDict(self): """ Build parameter dictionary akin to that returned by pydap. The parameters from the .btl file must match the parameters read from the .nc file. See comments for mapping copied from pctdToNetCDF.py. """ # Match the mapping done in pctdToNetCDF.py: # self.pr_list.append(float(r['PrDM'])) # self.depth = self.ncFile.createVariable('depth', 'float64', ('time',)) # self.depth.long_name = 'DEPTH' # self.depth.standard_name = 'depth' # self.depth.units = 'm' # self.depth[:] = csiro.depth(self.pr_list, self.lat_list) # Convert pressure to depth # self.t1_list.append(r['T190C']) # temp = self.ncFile.createVariable('TEMP', 'float64', ('time',)) # temp.long_name = 'Temperature, 2 [ITS-90, deg C]' # temp.standard_name = 'sea_water_temperature' # temp.units = 'Celsius' # self.sal_list.append(r['Sal00']) # sal = self.ncFile.createVariable('PSAL', 'float64', ('time',)) # sal.long_name = 'Salinity, Practical [PSU]' # sal.standard_name = 'sea_water_salinity' # self.xmiss_list.append(r['Xmiss']) # xmiss = self.ncFile.createVariable('xmiss', 'float64', ('time',)) # xmiss.long_name = 'Beam Transmission, Chelsea/Seatech' # xmiss.units = '%' # self.ecofl_list.append(r['FlECO-AFL']) # ecofl = self.ncFile.createVariable('ecofl', 'float64', ('time',)) # ecofl.long_name = 'Fluorescence, WET Labs ECO-AFL/FL' # ecofl.units = 'mg/m^3' # self.oxygen_list.append(r['Sbeox0ML/L']) # oxygen = self.ncFile.createVariable('oxygen', 'float64', ('time',)) # oxygen.long_name = 'Oxygen, SBE 43' # oxygen.units = 'ml/l' parmDict = {} pr = BaseType() pr.attributes = {"colname": "PrDM", "units": m, "long_name": "DEPTH", "standard_name": "depth"} temp = BaseType() temp.attributes = { "colname": "T190C", "units": "ITS-90, deg C", "long_name": "temperature", "standard_name": "sea_water_temperature", } sal = BaseType() sal.attributes = { "colname": "Sal00", "units": "1", "long_name": "salinity", "standard_name": "sea_water_salinity", } xmiss = BaseType() xmiss.attributes = {"colname": "Xmiss", "units": "%", "long_name": "Beam Transmission, Chelsea/Seatech"} ecofl = BaseType() ecofl.attributes = {"colname": "FlECO-AFL", "units": "mg/m^3", "long_name": "Fluorescence, WET Labs ECO-AFL/FL"} wetstar = BaseType() wetstar.attributes = {"colname": "WetStar", "units": "mg/m^3", "long_name": "Fluorescence, WET Labs WETstar"} oxygen = BaseType() oxygen.attributes = {"colname": "Sbeox0ML/L", "units": "ml/l", "long_name": "Oxygen, SBE 43"} # The colname attribute must be the keys that DictReader returns - the keys of this dictionary will be the Parameter names in stoqs parmDict = { "pressure": pr, "TEMP": temp, "PSAL": sal, "xmiss": xmiss, "ecofl": ecofl, "oxygen": oxygen, "wetstar": wetstar, } return parmDict