def buildParmDict(self):
'''
Build parameter dictionary akin to that returned by pydap. The parameters from the .btl file must
match the parameters read from the .nc file. See comments for mapping copied from pctdToNetCDF.py.
'''
# Match the mapping done in pctdToNetCDF.py:
# self.pr_list.append(float(r['PrDM']))
# self.depth = self.ncFile.createVariable('depth', 'float64', ('time',))
# self.depth.long_name = 'DEPTH'
# self.depth.standard_name = 'depth'
# self.depth.units = 'm'
# self.depth[:] = csiro.depth(self.pr_list, self.lat_list) # Convert pressure to depth
# self.t1_list.append(r['T190C'])
# temp = self.ncFile.createVariable('TEMP', 'float64', ('time',))
# temp.long_name = 'Temperature, 2 [ITS-90, deg C]'
# temp.standard_name = 'sea_water_temperature'
# temp.units = 'Celsius'
# self.sal_list.append(r['Sal00'])
# sal = self.ncFile.createVariable('PSAL', 'float64', ('time',))
# sal.long_name = 'Salinity, Practical [PSU]'
# sal.standard_name = 'sea_water_salinity'
# self.xmiss_list.append(r['Xmiss'])
# xmiss = self.ncFile.createVariable('xmiss', 'float64', ('time',))
# xmiss.long_name = 'Beam Transmission, Chelsea/Seatech'
# xmiss.units = '%'
# self.ecofl_list.append(r['FlECO-AFL'])
# ecofl = self.ncFile.createVariable('ecofl', 'float64', ('time',))
# ecofl.long_name = 'Fluorescence, WET Labs ECO-AFL/FL'
# ecofl.units = 'mg/m^3'
# self.oxygen_list.append(r['Sbeox0ML/L'])
# oxygen = self.ncFile.createVariable('oxygen', 'float64', ('time',))
# oxygen.long_name = 'Oxygen, SBE 43'
# oxygen.units = 'ml/l'
parmDict = {}
pr = BaseType('nameless')
pr.attributes = {'colname': 'PrDM', 'units': 'm' , 'long_name': 'DEPTH', 'standard_name': 'depth'}
temp = BaseType('nameless')
temp.attributes = {'colname': 'T190C', 'units': 'ITS-90, deg C', 'long_name': 'temperature', 'standard_name': 'sea_water_temperature'}
sal = BaseType('nameless')
sal.attributes = {'colname': 'Sal00', 'units': '1' , 'long_name': 'salinity', 'standard_name': 'sea_water_salinity'}
xmiss = BaseType('nameless')
xmiss.attributes = {'colname': 'Xmiss', 'units': '%', 'long_name': 'Beam Transmission, Chelsea/Seatech'}
ecofl = BaseType('nameless')
ecofl.attributes = {'colname': 'FlECO-AFL', 'units': 'mg/m^3', 'long_name': 'Fluorescence, WET Labs ECO-AFL/FL'}
wetstar = BaseType('nameless')
wetstar.attributes = {'colname': 'WetStar', 'units': 'mg/m^3', 'long_name': 'Fluorescence, WET Labs WETstar'}
oxygen = BaseType('nameless')
oxygen.attributes = {'colname': 'Sbeox0ML/L', 'units': 'ml/l', 'long_name': 'Oxygen, SBE 43'}
# The colname attribute must be the keys that DictReader returns - the keys of this dictionary will be the Parameter names in stoqs
parmDict = {'pressure': pr, 'TEMP': temp, 'PSAL': sal, 'xmiss': xmiss, 'ecofl': ecofl, 'oxygen': oxygen, 'wetstar': wetstar}
return parmDict
def process_csv_file(self, fh):
'''
Iterate through lines of iterator to csv file and pull out data for loading into STOQS
'''
ds = {}
DA = BaseType('nameless')
DA.attributes = {
'units': 'ng ml-1 ',
'long_name': 'Domoic Acid',
'standard_name': 'domoic_acid',
'type': 'float',
'description': 'Domoic acid',
'origin': 'www.sccoos.org'
}
PD = BaseType('nameless')
PD.attributes = {
'units': 'cells l-1',
'long_name': 'Pseudo-nitzschia delicatissima group',
'standard_name': 'pseudo_nitzschia_delicatissima',
'name': 'pseudo_nitzschia_delicatissima',
'type': 'float',
'description': 'Pseudo-nitzschia delicatissima group (cells/L)',
'origin': 'www.sccoos.org'
}
PA = BaseType('nameless')
PA.attributes = {
'units': 'cells l-1',
'long_name': 'Pseudo-nitzschia seriata group',
'standard_name': 'pseudo_nitzschia_seriata',
'name': 'pseudo_nitzschia_seriata',
'type': 'float',
'description': 'Pseudo-nitzschia seriata group (cells/L)',
'origin': 'www.sccoos.org'
}
alexandrium = BaseType('nameless')
alexandrium.attributes = {
'units': 'cells l-1',
'long_name': 'Alexandrium',
'standard_name': 'alexandrium',
'name': 'alexandrium',
'type': 'float',
'description': 'Alexandrium spp. (cells/L)',
'origin': 'www.sccoos.org'
}
phosphate = BaseType('nameless')
phosphate.attributes = {
'units': 'm-3 mol l-1',
'long_name': 'Phosphate',
'standard_name': 'phosphate_dissolved_in_seawater',
'name': 'Phosphate',
'type': 'float',
'description': 'Phosphate (uM)',
'origin': 'www.sccoos.org'
}
ammonia = BaseType('nameless')
ammonia.attributes = {
'units': 'm-3 mol l-1',
'long_name': 'Ammonia',
'standard_name': 'ammonia_dissolved_in_seawater',
'name': 'ammonia_dissolved_in_sewater',
'type': 'float',
'description': 'Ammonia (uM)',
'origin': 'www.sccoos.org'
}
silicate = BaseType('nameless')
silicate.attributes = {
'units': 'm-3 mol l-1',
'long_name': 'Silicate',
'standard_name': 'silicate_dissolved_in_seawater',
'name': 'silicate_dissolved_in_seawater',
'type': 'float',
'description': 'Silicate (uM)',
'origin': 'www.sccoos.org'
}
chlorophyll = BaseType('nameless')
chlorophyll.attributes = {
'units': 'kg m-3',
'long_name': 'Chlorophyll',
'standard_name': 'mass_concentration_of_chlorophyll_in_sea_water',
'name': 'mass_concentration_of_chlorophyll_in_sea_water',
'type': 'float',
'description': 'Chlorophyll (kg/m3)',
'origin': 'www.sccoos.org'
}
prorocentrum = BaseType('nameless')
prorocentrum.attributes = {
'units': 'cells l-1',
'long_name': 'Prorocentrum',
'standard_name': 'mass_concentration_of_prorocentrum_in_sea_water',
'name': 'mass_concentration_of_prorocentrum_in_sea_water',
'type': 'float',
'description': 'Prorocentrum spp. (cells/L)',
'origin': 'www.sccoos.org'
}
self.ds = {
'Domoic Acid (ng/mL)': DA,
'Pseudo-nitzschia seriata group (cells/L)': PA,
'Pseudo-nitzschia delicatissima group (cells/L)': PD,
'Phosphate (uM)': phosphate,
'Silicate (uM)': silicate,
'Ammonia (uM)': ammonia,
'Chlorophyll (mg/m3)': chlorophyll,
'Chlorophyll 1 (mg/m3)': chlorophyll,
'Chlorophyll 2 (mg/m3)': chlorophyll,
'Alexandrium spp. (cells/L)': alexandrium
}
self.include_names = [
'Pseudo-nitzschia seriata group (cells/L)',
'Pseudo-nitzschia delicatissima group (cells/L)',
'Domoic Acid (ng/mL)', 'Chlorophyll (mg/m3)',
'Chlorophyll 1 (mg/m3)', 'Chlorophyll 2 (mg/m3)',
'Prorocentrum spp. (cells/L)', 'Silicate (uM)', 'Ammonia (uM)',
'Nitrate (uM)', 'Phosphate (uM)', 'Alexandrium spp. (cells/L)'
]
self.initDB()
for pn in self.include_names:
self.parmCount[pn] = 0
reader = csv.reader(fh)
for line in fh:
# Skip all lines that don't begin with '"' nor ' ' then open that with csv.DictReader
if not line.startswith('"') and not line.startswith(' '):
titles = next(reader)
reader = csv.DictReader(fh, titles)
for r in reader:
year = int(r['year'])
month = int(r['month'])
day = int(r['day'])
time = r['time']
lat = float(r['latitude'])
lon = float(r['longitude'])
depth = float(r['depth (m)'])
location = r['location']
hours = int(time.split(':')[0])
mins = int(time.split(':')[1])
secs = int(time.split(':')[2])
parmNameValues = []
for name in list(self.ds.keys()):
if name.startswith('Chlorophyll'):
parmNameValues.append(
(name, 1e-5 * float(r[name])))
else:
parmNameValues.append((name, float(r[name])))
# Check to make sure all data from this file are from the same location.
# The program could be modified to read data in one file from multiple locations by reading data into a hash keyed by location name
# and then stepping through each key of the hash saving the data for each location into it's own activity. For now just require
# each data file to have data from just one location.
try:
if lat != lastlat or lon != lastlon:
logger.error(
"lat and lon are not the same for location = %s and lastlocation = %s. The input data should have just one location."
% (location, lastlocation))
sys.exit(-1)
except NameError as e:
# Expected first time through when lastlon & lastlat don't yet exist
pass
# Load data
dt = datetime(year, month, day, hours, mins, secs)
self.load_measurement(lon, lat, depth, dt, parmNameValues)
# Load sample
bName = dt.isoformat()
self.load_sample(lon, lat, depth, dt, bName)
lastlat = lat
lastlon = lon
lastlocation = location
logger.info("Data load complete, %d records loaded.", self.loaded)
fh.close()
# Update the Activity with information we now have following the load
# Careful with the structure of this comment. It is parsed in views.py to give some useful links in showActivities()
newComment = "%d MeasuredParameters loaded. Loaded on %sZ" % (
self.loaded, datetime.utcnow())
logger.info(
"runHABLoader(): Updating its comment with newComment = %s",
newComment)
aName = location
num_updated = m.Activity.objects.using(self.dbAlias).filter(
id=self.activity.id).update(name=aName,
comment=newComment,
maptrack=None,
mappoint=Point(lon, lat),
mindepth=self.mindepth,
maxdepth=self.maxdepth,
num_measuredparameters=self.loaded,
loaded_date=datetime.utcnow())
self.updateActivityParameterStats(self.parameterCount)
self.updateCampaignStartEnd()
def buildParmDict(self):
'''
Build parameter dictionary akin to that returned by pydap. The parameters from the .btl file must
match the parameters read from the .nc file. See comments for mapping copied from pctdToNetCDF.py.
'''
# Match the mapping done in pctdToNetCDF.py:
# self.pr_list.append(float(r['PrDM']))
# self.depth = self.ncFile.createVariable('depth', 'float64', ('time',))
# self.depth.long_name = 'DEPTH'
# self.depth.standard_name = 'depth'
# self.depth.units = 'm'
# self.depth[:] = csiro.depth(self.pr_list, self.lat_list) # Convert pressure to depth
# self.t1_list.append(r['T190C'])
# temp = self.ncFile.createVariable('TEMP', 'float64', ('time',))
# temp.long_name = 'Temperature, 2 [ITS-90, deg C]'
# temp.standard_name = 'sea_water_temperature'
# temp.units = 'Celsius'
# self.sal_list.append(r['Sal00'])
# sal = self.ncFile.createVariable('PSAL', 'float64', ('time',))
# sal.long_name = 'Salinity, Practical [PSU]'
# sal.standard_name = 'sea_water_salinity'
# self.xmiss_list.append(r['Xmiss'])
# xmiss = self.ncFile.createVariable('xmiss', 'float64', ('time',))
# xmiss.long_name = 'Beam Transmission, Chelsea/Seatech'
# xmiss.units = '%'
# self.ecofl_list.append(r['FlECO-AFL'])
# ecofl = self.ncFile.createVariable('ecofl', 'float64', ('time',))
# ecofl.long_name = 'Fluorescence, WET Labs ECO-AFL/FL'
# ecofl.units = 'mg/m^3'
# self.oxygen_list.append(r['Sbeox0ML/L'])
# oxygen = self.ncFile.createVariable('oxygen', 'float64', ('time',))
# oxygen.long_name = 'Oxygen, SBE 43'
# oxygen.units = 'ml/l'
parmDict = {}
pr = BaseType('nameless')
pr.attributes = {
'colname': 'PrDM',
'units': 'm',
'long_name': 'DEPTH',
'standard_name': 'depth'
}
temp = BaseType('nameless')
temp.attributes = {
'colname': 'T190C',
'units': 'ITS-90, deg C',
'long_name': 'temperature',
'standard_name': 'sea_water_temperature'
}
sal = BaseType('nameless')
sal.attributes = {
'colname': 'Sal00',
'units': '1',
'long_name': 'salinity',
'standard_name': 'sea_water_salinity'
}
xmiss = BaseType('nameless')
xmiss.attributes = {
'colname': 'Xmiss',
'units': '%',
'long_name': 'Beam Transmission, Chelsea/Seatech'
}
ecofl = BaseType('nameless')
ecofl.attributes = {
'colname': 'FlECO-AFL',
'units': 'mg/m^3',
'long_name': 'Fluorescence, WET Labs ECO-AFL/FL'
}
wetstar = BaseType('nameless')
wetstar.attributes = {
'colname': 'WetStar',
'units': 'mg/m^3',
'long_name': 'Fluorescence, WET Labs WETstar'
}
oxygen = BaseType('nameless')
oxygen.attributes = {
'colname': 'Sbeox0ML/L',
'units': 'ml/l',
'long_name': 'Oxygen, SBE 43'
}
# The colname attribute must be the keys that DictReader returns - the keys of this dictionary will be the Parameter names in stoqs
parmDict = {
'pressure': pr,
'TEMP': temp,
'PSAL': sal,
'xmiss': xmiss,
'ecofl': ecofl,
'oxygen': oxygen,
'wetstar': wetstar
}
return parmDict
def process_csv_file(self, fh):
'''
Iterate through lines of iterator to csv file and pull out data for loading into STOQS
'''
ds = {}
DA = BaseType()
DA.attributes = {'units': 'ng ml-1 ' ,
'long_name': 'Domoic Acid',
'standard_name': 'domoic_acid',
'type': 'float',
'description': 'Domoic acid' ,
'origin': 'www.sccoos.org' }
PD = BaseType()
PD.attributes = {'units': 'cells l-1',
'long_name': 'Pseudo-nitzschia delicatissima group',
'standard_name': 'pseudo_nitzschia_delicatissima',
'name': 'pseudo_nitzschia_delicatissima' ,
'type': 'float' ,
'description': 'Pseudo-nitzschia delicatissima group (cells/L)' ,
'origin': 'www.sccoos.org'
}
PA = BaseType()
PA.attributes = {'units': 'cells l-1',
'long_name': 'Pseudo-nitzschia seriata group',
'standard_name': 'pseudo_nitzschia_seriata',
'name': 'pseudo_nitzschia_seriata' ,
'type': 'float' ,
'description': 'Pseudo-nitzschia seriata group (cells/L)' ,
'origin': 'www.sccoos.org'
}
alexandrium = BaseType()
alexandrium.attributes = {'units': 'cells l-1',
'long_name': 'Alexandrium',
'standard_name': 'alexandrium',
'name': 'alexandrium' ,
'type': 'float' ,
'description': 'Alexandrium spp. (cells/L)' ,
'origin': 'www.sccoos.org'
}
phosphate = BaseType()
phosphate.attributes = {'units': 'm-3 mol l-1',
'long_name': 'Phosphate',
'standard_name': 'phosphate_dissolved_in_seawater',
'name': 'Phosphate' ,
'type': 'float' ,
'description': 'Phosphate (uM)' ,
'origin': 'www.sccoos.org'
}
ammonia = BaseType()
ammonia.attributes = {'units': 'm-3 mol l-1',
'long_name': 'Ammonia',
'standard_name': 'ammonia_dissolved_in_seawater',
'name': 'ammonia_dissolved_in_sewater' ,
'type': 'float' ,
'description': 'Ammonia (uM)' ,
'origin': 'www.sccoos.org'
}
silicate = BaseType()
silicate.attributes = {'units': 'm-3 mol l-1',
'long_name': 'Silicate',
'standard_name': 'silicate_dissolved_in_seawater',
'name': 'silicate_dissolved_in_seawater' ,
'type': 'float' ,
'description': 'Silicate (uM)' ,
'origin': 'www.sccoos.org'
}
chlorophyll = BaseType()
chlorophyll.attributes = {'units': 'kg m-3',
'long_name': 'Chlorophyll',
'standard_name': 'mass_concentration_of_chlorophyll_in_sea_water',
'name': 'mass_concentration_of_chlorophyll_in_sea_water' ,
'type': 'float' ,
'description': 'Chlorophyll (kg/m3)' ,
'origin': 'www.sccoos.org'
}
prorocentrum = BaseType()
prorocentrum.attributes = {'units': 'cells l-1',
'long_name': 'Prorocentrum',
'standard_name': 'mass_concentration_of_prorocentrum_in_sea_water',
'name': 'mass_concentration_of_prorocentrum_in_sea_water' ,
'type': 'float' ,
'description': 'Prorocentrum spp. (cells/L)' ,
'origin': 'www.sccoos.org'
}
self.ds = { 'Domoic Acid (ng/mL)': DA, 'Pseudo-nitzschia seriata group (cells/L)': PA,
'Pseudo-nitzschia delicatissima group (cells/L)': PD,
'Phosphate (uM)': phosphate,
'Silicate (uM)': silicate, 'Ammonia (uM)': ammonia,
'Chlorophyll (mg/m3)': chlorophyll, 'Chlorophyll 1 (mg/m3)': chlorophyll,
'Chlorophyll 2 (mg/m3)': chlorophyll ,
'Alexandrium spp. (cells/L)': alexandrium
}
self.include_names = ['Pseudo-nitzschia seriata group (cells/L)',
'Pseudo-nitzschia delicatissima group (cells/L)',
'Domoic Acid (ng/mL)',
'Chlorophyll (mg/m3)', 'Chlorophyll 1 (mg/m3)', 'Chlorophyll 2 (mg/m3)',
'Prorocentrum spp. (cells/L)', 'Silicate (uM)', 'Ammonia (uM)',
'Nitrate (uM)', 'Phosphate (uM)',
'Alexandrium spp. (cells/L)']
self.initDB()
for pn in self.include_names:
self.parmCount[pn] = 0
reader = csv.reader(fh)
for line in fh:
# Skip all lines that don't begin with '"' nor ' ' then open that with csv.DictReader
if not line.startswith('"') and not line.startswith(' '):
titles = reader.next()
reader = csv.DictReader(fh, titles)
for r in reader:
year = int(r['year'])
month = int(r['month'])
day = int(r['day'])
time = r['time']
lat = float(r['latitude'])
lon = float(r['longitude'])
depth = float(r['depth (m)'])
location = r['location']
hours = int(time.split(':')[0])
mins = int(time.split(':')[1])
secs = int(time.split(':')[2])
parmNameValues = []
for name in self.ds.keys():
if name.startswith('Chlorophyll'):
parmNameValues.append((name, 1e-5*float(r[name])))
else:
parmNameValues.append((name, float(r[name])))
# Check to make sure all data from this file are from the same location.
# The program could be modified to read data in one file from multiple locations by reading data into a hash keyed by location name
# and then stepping through each key of the hash saving the data for each location into it's own activity. For now just require
# each data file to have data from just one location.
try:
if lat != lastlat or lon != lastlon:
logger.error("lat and lon are not the same for location = %s and lastlocation = %s. The input data should have just one location." % (location, lastlocation))
sys.exit(-1)
except NameError, e:
# Expected first time through when lastlon & lastlat don't yet exist
pass
# Load data
dt = datetime(year, month, day, hours, mins, secs)
self.load_measurement(lon, lat, depth, dt, parmNameValues)
# Load sample
bName = dt.isoformat()
self.load_sample(lon, lat, depth, dt, bName)
lastlat = lat
lastlon = lon
lastlocation = location
def process_csv_file(self, fh):
'''
Iterate through lines of iterator to csv file and pull out data for loading into STOQS
'''
ds = {}
DA = BaseType('nameless')
DA.attributes = {'units': 'ng ml-1 ' ,
'long_name': 'Domoic Acid',
'standard_name': 'domoic_acid',
'type': 'float',
'description': 'Domoic acid' ,
'origin': 'www.sccoos.org' }
PD = BaseType('nameless')
PD.attributes = {'units': 'cells l-1',
'long_name': 'Pseudo-nitzschia delicatissima group',
'standard_name': 'pseudo_nitzschia_delicatissima',
'name': 'pseudo_nitzschia_delicatissima' ,
'type': 'float' ,
'description': 'Pseudo-nitzschia delicatissima group (cells/L)' ,
'origin': 'www.sccoos.org'
}
PA = BaseType('nameless')
PA.attributes = {'units': 'cells l-1',
'long_name': 'Pseudo-nitzschia seriata group',
'standard_name': 'pseudo_nitzschia_seriata',
'name': 'pseudo_nitzschia_seriata' ,
'type': 'float' ,
'description': 'Pseudo-nitzschia seriata group (cells/L)' ,
'origin': 'www.sccoos.org'
}
alexandrium = BaseType('nameless')
alexandrium.attributes = {'units': 'cells l-1',
'long_name': 'Alexandrium',
'standard_name': 'alexandrium',
'name': 'alexandrium' ,
'type': 'float' ,
'description': 'Alexandrium spp. (cells/L)' ,
'origin': 'www.sccoos.org'
}
phosphate = BaseType('nameless')
phosphate.attributes = {'units': 'm-3 mol l-1',
'long_name': 'Phosphate',
'standard_name': 'phosphate_dissolved_in_seawater',
'name': 'Phosphate' ,
'type': 'float' ,
'description': 'Phosphate (uM)' ,
'origin': 'www.sccoos.org'
}
ammonia = BaseType('nameless')
ammonia.attributes = {'units': 'm-3 mol l-1',
'long_name': 'Ammonia',
'standard_name': 'ammonia_dissolved_in_seawater',
'name': 'ammonia_dissolved_in_sewater' ,
'type': 'float' ,
'description': 'Ammonia (uM)' ,
'origin': 'www.sccoos.org'
}
silicate = BaseType('nameless')
silicate.attributes = {'units': 'm-3 mol l-1',
'long_name': 'Silicate',
'standard_name': 'silicate_dissolved_in_seawater',
'name': 'silicate_dissolved_in_seawater' ,
'type': 'float' ,
'description': 'Silicate (uM)' ,
'origin': 'www.sccoos.org'
}
chlorophyll = BaseType('nameless')
chlorophyll.attributes = {'units': 'kg m-3',
'long_name': 'Chlorophyll',
'standard_name': 'mass_concentration_of_chlorophyll_in_sea_water',
'name': 'mass_concentration_of_chlorophyll_in_sea_water' ,
'type': 'float' ,
'description': 'Chlorophyll (kg/m3)' ,
'origin': 'www.sccoos.org'
}
prorocentrum = BaseType('nameless')
prorocentrum.attributes = {'units': 'cells l-1',
'long_name': 'Prorocentrum',
'standard_name': 'mass_concentration_of_prorocentrum_in_sea_water',
'name': 'mass_concentration_of_prorocentrum_in_sea_water' ,
'type': 'float' ,
'description': 'Prorocentrum spp. (cells/L)' ,
'origin': 'www.sccoos.org'
}
self.ds = { 'Domoic Acid (ng/mL)': DA, 'Pseudo-nitzschia seriata group (cells/L)': PA,
'Pseudo-nitzschia delicatissima group (cells/L)': PD,
'Phosphate (uM)': phosphate,
'Silicate (uM)': silicate, 'Ammonia (uM)': ammonia,
'Chlorophyll (mg/m3)': chlorophyll, 'Chlorophyll 1 (mg/m3)': chlorophyll,
'Chlorophyll 2 (mg/m3)': chlorophyll ,
'Alexandrium spp. (cells/L)': alexandrium
}
self.include_names = ['Pseudo-nitzschia seriata group (cells/L)',
'Pseudo-nitzschia delicatissima group (cells/L)',
'Domoic Acid (ng/mL)',
'Chlorophyll (mg/m3)', 'Chlorophyll 1 (mg/m3)', 'Chlorophyll 2 (mg/m3)',
'Prorocentrum spp. (cells/L)', 'Silicate (uM)', 'Ammonia (uM)',
'Nitrate (uM)', 'Phosphate (uM)',
'Alexandrium spp. (cells/L)']
self.initDB()
for pn in self.include_names:
self.parmCount[pn] = 0
reader = csv.reader(fh)
for line in fh:
# Skip all lines that don't begin with '"' nor ' ' then open that with csv.DictReader
if not line.startswith('"') and not line.startswith(' '):
titles = next(reader)
reader = csv.DictReader(fh, titles)
for r in reader:
year = int(r['year'])
month = int(r['month'])
day = int(r['day'])
time = r['time']
lat = float(r['latitude'])
lon = float(r['longitude'])
depth = float(r['depth (m)'])
location = r['location']
hours = int(time.split(':')[0])
mins = int(time.split(':')[1])
secs = int(time.split(':')[2])
parmNameValues = []
for name in list(self.ds.keys()):
if name.startswith('Chlorophyll'):
parmNameValues.append((name, 1e-5*float(r[name])))
else:
parmNameValues.append((name, float(r[name])))
# Check to make sure all data from this file are from the same location.
# The program could be modified to read data in one file from multiple locations by reading data into a hash keyed by location name
# and then stepping through each key of the hash saving the data for each location into it's own activity. For now just require
# each data file to have data from just one location.
try:
if lat != lastlat or lon != lastlon:
logger.error("lat and lon are not the same for location = %s and lastlocation = %s. The input data should have just one location." % (location, lastlocation))
sys.exit(-1)
except NameError as e:
# Expected first time through when lastlon & lastlat don't yet exist
pass
# Load data
dt = datetime(year, month, day, hours, mins, secs)
self.load_measurement(lon, lat, depth, dt, parmNameValues)
# Load sample
bName = dt.isoformat()
self.load_sample(lon, lat, depth, dt, bName)
lastlat = lat
lastlon = lon
lastlocation = location
logger.info("Data load complete, %d records loaded.", self.loaded)
fh.close()
# Update the Activity with information we now have following the load
# Careful with the structure of this comment. It is parsed in views.py to give some useful links in showActivities()
newComment = "%d MeasuredParameters loaded. Loaded on %sZ" % (self.loaded, datetime.utcnow())
logger.info("runHABLoader(): Updating its comment with newComment = %s", newComment)
aName = location
num_updated = m.Activity.objects.using(self.dbAlias).filter(id = self.activity.id).update(
name = aName,
comment = newComment,
maptrack = None,
mappoint = 'POINT(%s %s)' % (lon, lat),
mindepth = self.mindepth,
maxdepth = self.maxdepth,
num_measuredparameters = self.loaded,
loaded_date = datetime.utcnow())
self.updateActivityParameterStats(self.parameterCount)
self.updateCampaignStartEnd()
def buildParmDict(self):
"""
Build parameter dictionary akin to that returned by pydap. The parameters from the .btl file must
match the parameters read from the .nc file. See comments for mapping copied from pctdToNetCDF.py.
"""
# Match the mapping done in pctdToNetCDF.py:
# self.pr_list.append(float(r['PrDM']))
# self.depth = self.ncFile.createVariable('depth', 'float64', ('time',))
# self.depth.long_name = 'DEPTH'
# self.depth.standard_name = 'depth'
# self.depth.units = 'm'
# self.depth[:] = csiro.depth(self.pr_list, self.lat_list) # Convert pressure to depth
# self.t1_list.append(r['T190C'])
# temp = self.ncFile.createVariable('TEMP', 'float64', ('time',))
# temp.long_name = 'Temperature, 2 [ITS-90, deg C]'
# temp.standard_name = 'sea_water_temperature'
# temp.units = 'Celsius'
# self.sal_list.append(r['Sal00'])
# sal = self.ncFile.createVariable('PSAL', 'float64', ('time',))
# sal.long_name = 'Salinity, Practical [PSU]'
# sal.standard_name = 'sea_water_salinity'
# self.xmiss_list.append(r['Xmiss'])
# xmiss = self.ncFile.createVariable('xmiss', 'float64', ('time',))
# xmiss.long_name = 'Beam Transmission, Chelsea/Seatech'
# xmiss.units = '%'
# self.ecofl_list.append(r['FlECO-AFL'])
# ecofl = self.ncFile.createVariable('ecofl', 'float64', ('time',))
# ecofl.long_name = 'Fluorescence, WET Labs ECO-AFL/FL'
# ecofl.units = 'mg/m^3'
# self.oxygen_list.append(r['Sbeox0ML/L'])
# oxygen = self.ncFile.createVariable('oxygen', 'float64', ('time',))
# oxygen.long_name = 'Oxygen, SBE 43'
# oxygen.units = 'ml/l'
parmDict = {}
pr = BaseType()
pr.attributes = {"colname": "PrDM", "units": m, "long_name": "DEPTH", "standard_name": "depth"}
temp = BaseType()
temp.attributes = {
"colname": "T190C",
"units": "ITS-90, deg C",
"long_name": "temperature",
"standard_name": "sea_water_temperature",
}
sal = BaseType()
sal.attributes = {
"colname": "Sal00",
"units": "1",
"long_name": "salinity",
"standard_name": "sea_water_salinity",
}
xmiss = BaseType()
xmiss.attributes = {"colname": "Xmiss", "units": "%", "long_name": "Beam Transmission, Chelsea/Seatech"}
ecofl = BaseType()
ecofl.attributes = {"colname": "FlECO-AFL", "units": "mg/m^3", "long_name": "Fluorescence, WET Labs ECO-AFL/FL"}
wetstar = BaseType()
wetstar.attributes = {"colname": "WetStar", "units": "mg/m^3", "long_name": "Fluorescence, WET Labs WETstar"}
oxygen = BaseType()
oxygen.attributes = {"colname": "Sbeox0ML/L", "units": "ml/l", "long_name": "Oxygen, SBE 43"}
# The colname attribute must be the keys that DictReader returns - the keys of this dictionary will be the Parameter names in stoqs
parmDict = {
"pressure": pr,
"TEMP": temp,
"PSAL": sal,
"xmiss": xmiss,
"ecofl": ecofl,
"oxygen": oxygen,
"wetstar": wetstar,
}
return parmDict
