def score_imported_oils(settings):
'''
Here is where we score the quality of the oil records that we have
imported from the flatfile.
'''
adios_id = settings['adios_id'] if 'adios_id' in settings else None
with transaction.manager:
session = _get_db_session()
if adios_id is not None:
sys.stderr.write('scoring the imported oil record {0}...\n'
.format(adios_id))
try:
imp_obj = (session.query(ImportedRecord)
.filter(ImportedRecord.adios_oil_id == adios_id)
.one())
except:
print 'Could not find imported record {0}'.format(adios_id)
raise
to_xls = settings['to_xls'] if 'to_xls' in settings else None
if to_xls is not None and xlwt_available:
export_oil_score_to_xls(imp_obj)
else:
print '{0}\t{1}\t{2}'.format(imp_obj.adios_oil_id,
imp_obj.oil_name,
score_imported_oil(imp_obj))
else:
sys.stderr.write('scoring the imported oil records in database...')
for o in session.query(ImportedRecord):
print '{0}\t{1}\t{2}'.format(o.adios_oil_id,
o.oil_name,
score_imported_oil(o))
def get_oils(request):
session = _get_db_session()
obj_id = obj_id_from_url(request)
if not obj_id:
# Return all oils in JSON format. We only return the searchable
# columns.
oils = session.query(Oil)
return [{
'adios_oil_id': o.imported.adios_oil_id,
'name': o.name,
'location': o.imported.location,
'field_name': o.imported.field_name,
'product_type': o.imported.product_type,
'oil_class': o.imported.oil_class,
'api': o.api,
'pour_point': get_pour_point(o),
'viscosity': get_oil_viscosity(o),
'categories': get_category_paths(o),
'categories_str': get_category_paths_str(o),
'synonyms': get_synonyms(o)
} for o in oils]
else:
try:
oil = (session.query(Oil).join(ImportedRecord).filter(
ImportedRecord.adios_oil_id == obj_id).one())
return prune_oil_json(oil.tojson())
except NoResultFound:
raise HTTPNotFound()
def score_imported_oils(settings):
'''
Here is where we score the quality of the oil records that we have
imported from the flatfile.
'''
adios_id = settings['adios_id'] if 'adios_id' in settings else None
with transaction.manager:
session = _get_db_session()
if adios_id is not None:
sys.stderr.write('scoring the imported oil record {0}...\n'
.format(adios_id))
try:
oil_obj = (session.query(ImportedRecord)
.filter(ImportedRecord.adios_oil_id == adios_id)
.one())
to_xls = settings['to_xls'] if 'to_xls' in settings else None
if to_xls is not None and xlwt_available:
export_oil_score_to_xls(oil_obj)
else:
print '{0}\t{1}\t{2}'.format(oil_obj.adios_oil_id,
oil_obj.oil_name,
score_imported_oil(oil_obj))
except:
print 'Could not find record {0}'.format(adios_id)
raise
else:
sys.stderr.write('scoring the imported oil records in database...')
for o in session.query(ImportedRecord):
print '{0}\t{1}\t{2}'.format(o.adios_oil_id,
o.oil_name,
score_imported_oil(o))
def audit_database(settings):
'''
Just a quick check of the data values that we loaded
when we initialized the database.
'''
with transaction.manager:
session = _get_db_session()
sys.stderr.write('Auditing the records in database...')
for o in session.query(ImportedRecord):
if 1 and o.synonyms:
print
print [s.name for s in o.synonyms]
if 1 and o.densities:
print
print [d.kg_m_3 for d in o.densities]
print [d.ref_temp_k for d in o.densities]
print [d.weathering for d in o.densities]
if 1 and o.kvis:
print
print [k.m_2_s for k in o.kvis]
print [k.ref_temp_k for k in o.kvis]
print [k.weathering for k in o.kvis]
if 1 and o.dvis:
print
print [d.kg_ms for d in o.dvis]
print [d.ref_temp_k for d in o.dvis]
print [d.weathering for d in o.dvis]
if 1 and o.cuts:
print
print [c.vapor_temp_k for c in o.cuts]
print [c.liquid_temp_k for c in o.cuts]
print [c.fraction for c in o.cuts]
if 1:
tox = [t for t in o.toxicities if t.tox_type == 'EC']
if tox:
print
print [t.species for t in tox]
print [t.after_24h for t in tox]
print [t.after_48h for t in tox]
print [t.after_96h for t in tox]
if 1:
tox = [t for t in o.toxicities if t.tox_type == 'LC']
if tox:
print
print [t.species for t in tox]
print [t.after_24h for t in tox]
print [t.after_48h for t in tox]
print [t.after_96h for t in tox]
print 'finished!!!'
def set_oiltype(self, oiltype):
self.oil_name = oiltype
self.set_config('seed:oil_type', oiltype)
if self.oil_weathering_model == 'noaa':
try:
from oil_library import get_oil_props, _get_db_session
from oil_library.models import Oil as ADIOS_Oil
from oil_library.oil_props import OilProps
session = _get_db_session()
oils = session.query(ADIOS_Oil).filter(
ADIOS_Oil.name == oiltype)
ADIOS_ids = [oil.adios_oil_id for oil in oils]
if len(ADIOS_ids) == 0:
raise ValueError(
'Oil type "%s" not found in NOAA database' % oiltype)
elif len(ADIOS_ids) == 1:
self.oiltype = get_oil_props(oiltype)
else:
logging.warning(
'Two oils found with name %s (ADIOS IDs %s and %s). Using the first.'
% (oiltype, ADIOS_ids[0], ADIOS_ids[1]))
self.oiltype = OilProps(oils[0])
except Exception as e:
logging.warning(e)
return
if oiltype not in self.oiltypes:
raise ValueError(
'Oiltype %s is unknown. The following oiltypes are available: %s'
% (oiltype, str(self.oiltypes)))
indx = self.oiltypes.index(oiltype)
linenumber = self.oiltypes_linenumbers[indx]
oilfile = open(self.oiltype_file, 'r')
for i in range(linenumber + 1):
oilfile.readline()
ref = oilfile.readline().split()
self.oil_data = {}
self.oil_data['reference_thickness'] = np.float(ref[0])
self.oil_data['reference_wind'] = np.float(ref[1])
tref = []
fref = []
wmax = []
while True:
line = oilfile.readline()
if not line[0].isdigit():
break
line = line.split()
tref.append(line[0])
fref.append(line[1])
wmax.append(line[3])
oilfile.close()
self.oil_data['tref'] = np.array(tref, dtype='float') * 3600.
self.oil_data['fref'] = np.array(fref, dtype='float') * .01
self.oil_data['wmax'] = np.array(wmax, dtype='float')
self.oil_data['oiltype'] = oiltype # Store name of oil type
def __init__(self, weathering_model='default', *args, **kwargs):
if weathering_model == 'noaa':
try:
from oil_library import _get_db_session
from oil_library.models import Oil, ImportedRecord
except:
raise ImportError('NOAA oil library must be installed from: '
'https://github.com/NOAA-ORR-ERD/OilLibrary')
# Get list of all oiltypes in NOAA database
session = _get_db_session()
if 'location' in kwargs:
self.oiltypes = session.query(
Oil.name).join(ImportedRecord).filter(
ImportedRecord.location == kwargs['location']).all()
del kwargs['location']
all_oiltypes = session.query(Oil.name).all()
generic_oiltypes = [
o for o in all_oiltypes if o[0][0:2] == '*G'
]
self.oiltypes.extend(generic_oiltypes)
else:
self.oiltypes = session.query(Oil.name).all()
self.oiltypes = sorted([o[0] for o in self.oiltypes])
self.oiltypes = [
ot for ot in self.oiltypes if ot not in self.duplicate_oils
]
else:
# Read oil properties from file
self.oiltype_file = os.path.dirname(os.path.realpath(__file__)) + \
'/oilprop.dat'
oiltypes = []
linenumbers = []
with open(self.oiltype_file) as f:
for i, line in enumerate(f):
if line[0].isalpha():
oiltype = line.strip()[:-2].strip()
oiltypes.append(oiltype)
linenumbers.append(i)
oiltypes, linenumbers = zip(*sorted(zip(oiltypes, linenumbers)))
self.oiltypes = oiltypes
self.oiltypes_linenumbers = linenumbers
self.oil_weathering_model = weathering_model
# Update config with oiltypes
oiltypes = [str(a) for a in self.oiltypes]
self._add_config('seed:oil_type', oiltypes, 'Oil type', overwrite=True)
# Calling general constructor of parent class
super(OpenOil, self).__init__(*args, **kwargs)
# Overriding with specific configspec
self._add_configstring(self.configspec)
def export_database(settings):
'''
Just a quick check of the data values that we loaded
when we initialized the database.
'''
with transaction.manager:
session = _get_db_session()
sys.stderr.write('Exporting the records in database...')
for o in session.query(Oil):
print o
print o.tojson()
def export_database(settings):
'''
Just a quick check of the data values that we loaded
when we initialized the database.
'''
with transaction.manager:
session = _get_db_session()
sys.stderr.write('Exporting the records in database...')
for o in session.query(Oil):
print o
print o.tojson()
def __init__(self, weathering_model='default', *args, **kwargs):
self._add_configstring(self.configspec)
if weathering_model == 'noaa':
try:
from oil_library import _get_db_session
from oil_library.models import Oil, ImportedRecord
except:
raise ImportError('NOAA oil library must be installed from: '
'https://github.com/NOAA-ORR-ERD/OilLibrary')
# Get list of all oiltypes in NOAA database
session = _get_db_session()
if 'location' in kwargs:
self.oiltypes = session.query(
Oil.name).join(ImportedRecord).filter(
ImportedRecord.location == kwargs['location']).all()
del kwargs['location']
else:
self.oiltypes = session.query(Oil.name).all()
self.oiltypes = sorted([o[0] for o in self.oiltypes])
else:
# Read oil properties from file
self.oiltype_file = os.path.dirname(os.path.realpath(__file__)) + \
'/oilprop.dat'
oilprop = open(self.oiltype_file)
oiltypes = []
linenumbers = []
for i, line in enumerate(oilprop.readlines()):
if line[0].isalpha():
oiltype = line.strip()[:-2].strip()
oiltypes.append(oiltype)
linenumbers.append(i)
oiltypes, linenumbers = zip(*sorted(zip(oiltypes, linenumbers)))
self.oiltypes = oiltypes
self.oiltypes_linenumbers = linenumbers
self.oil_weathering_model = weathering_model
# Update config with oiltypes
# TODO: add unicode support
oiltypes = [a.encode('ascii', 'ignore') for a in self.oiltypes]
oiltypes = 'option(%s, default=\'%s\')' % (str(oiltypes)[1:-1],
oiltypes[0])
self._add_config('input:spill:oil_type',
oiltypes,
'Oil type',
overwrite=True)
# Calling general constructor of parent class
super(OpenOil, self).__init__(*args, **kwargs)
def __init__(self, weathering_model='default', *args, **kwargs):
self._add_configstring(self.configspec)
if weathering_model == 'noaa':
try:
from oil_library import _get_db_session
from oil_library.models import Oil, ImportedRecord
except:
raise ImportError(
'NOAA oil library must be installed from: '
'https://github.com/NOAA-ORR-ERD/OilLibrary')
# Get list of all oiltypes in NOAA database
session = _get_db_session()
if 'location' in kwargs:
self.oiltypes = session.query(Oil.name).join(
ImportedRecord).filter(ImportedRecord.
location==kwargs['location']).all()
del kwargs['location']
else:
self.oiltypes = session.query(Oil.name).all()
self.oiltypes = sorted([o[0] for o in self.oiltypes])
else:
# Read oil properties from file
self.oiltype_file = os.path.dirname(os.path.realpath(__file__)) + \
'/oilprop.dat'
oilprop = open(self.oiltype_file)
oiltypes = []
linenumbers = []
for i, line in enumerate(oilprop.readlines()):
if line[0].isalpha():
oiltype = line.strip()[:-2].strip()
oiltypes.append(oiltype)
linenumbers.append(i)
oiltypes, linenumbers = zip(*sorted(zip(oiltypes, linenumbers)))
self.oiltypes = oiltypes
self.oiltypes_linenumbers = linenumbers
self.oil_weathering_model = weathering_model
# Update config with oiltypes
# TODO: add unicode support
oiltypes = [a.encode('ascii','ignore') for a in self.oiltypes]
oiltypes = 'option(%s, default=\'%s\')' % (
str(oiltypes)[1:-1], oiltypes[0])
self._add_config('input:spill:oil_type', oiltypes,
'Oil type', overwrite=True)
# Calling general constructor of parent class
super(OpenOil, self).__init__(*args, **kwargs)
def get_oils(request):
session = _get_db_session()
obj_id = obj_id_from_url(request)
if not obj_id:
# Return all oils in JSON format. We only return the searchable
# columns.
return [get_oil_searchable_fields(o) for o in session.query(Oil)]
else:
try:
oil = (session.query(Oil).join(ImportedRecord)
.filter(ImportedRecord.adios_oil_id == obj_id).one())
return prune_oil_json(oil.tojson())
except NoResultFound:
raise HTTPNotFound()
def audit_database(settings):
'''
Just a quick check of the data values that we loaded
when we initialized the database.
'''
with transaction.manager:
session = _get_db_session()
sys.stderr.write('Auditing the records in database...')
for o in session.query(ImportedRecord):
if 1 and o.synonyms:
print
print [s.name for s in o.synonyms]
if 1 and o.densities:
print
print [d.kg_m_3 for d in o.densities]
print [d.ref_temp_k for d in o.densities]
def get_distinct(request):
'''Returns all oils in JSON format'''
session = _get_db_session()
res = []
attrs = ('location',
'field_name')
for a in attrs:
values = [r[0] for r in (session.query(getattr(ImportedRecord, a))
.distinct().all())]
res.append(dict(column=a, values=values))
categories = dict([(c.name, [child.name for child in c.children])
for c in (session.query(Category)
.filter(Category.parent == None))
])
res.append(dict(column='product_type', values=categories))
return res
def get_distinct(request):
'''Returns all oils in JSON format'''
session = _get_db_session()
res = []
attrs = ('location', 'field_name')
for a in attrs:
values = [
r[0] for r in (
session.query(getattr(ImportedRecord, a)).distinct().all())
]
res.append(dict(column=a, values=values))
categories = dict([
(c.name, [child.name for child in c.children])
for c in (session.query(Category).filter(Category.parent == None))
])
res.append(dict(column='product_type', values=categories))
return res
'''
This is just a scratchpad script I use inside ipython
'''
import oil_library
from oil_library import get_oil, _get_db_session
from oil_library.models import Oil
from oil_library.oil_props import OilProps
from pprint import PrettyPrinter
pp = PrettyPrinter(indent=2)
session = _get_db_session()
print 'Our session:', session.connection().engine
# oil_obj = get_oil('LUCKENBACH FUEL OIL')
oil_obj = get_oil('BAHIA')
props_obj = OilProps(oil_obj)
pp.pprint([f for f in props_obj.component_mass_fractions()])
oil_obj = session.query(Oil).filter(Oil.adios_oil_id == 'AD00584').one()
my_str = oil_obj.imported.comments
print my_str
benz = get_oil(oil_library.sample_oils.benzene.json_data)
print benz
def plot_oil_viscosities(settings):
with transaction.manager:
# -- Our loading routine --
session = _get_db_session()
if "adios_id" not in settings:
raise ValueError("adios_id setting is required.")
adios_id = settings["adios_id"]
print "our session: %s" % (session)
try:
oilobj = session.query(Oil).join(ImportedRecord).filter(ImportedRecord.adios_oil_id == adios_id).one()
except NoResultFound:
raise NoResultFound("No Oil was found matching adios_id {0}".format(adios_id))
if oilobj:
print "Our oil object: %s" % (oilobj)
oil_props = OilProps(oilobj)
print "\nOilProps:", oil_props
print oil_props.get_viscosity()
print "\nOur viscosities:"
print [v for v in oilobj.kvis]
print "\nOur unweathered viscosities (m^2/s, Kdegrees):"
vis = [v for v in oilobj.kvis if v.weathering <= 0.0]
print vis
for i in [(v.m_2_s, v.ref_temp_k, v.weathering) for v in vis]:
print i
x = np.array([v.ref_temp_k for v in vis]) - 273.15
y = np.array([v.m_2_s for v in vis])
xmin = x.min()
xmax = x.max()
xpadding = 0.5 if xmax == xmin else (xmax - xmin) * 0.3
ymin = y.min()
ymax = y.max()
ypadding = (ymax / 2) if ymax == ymin else (ymax - ymin) * 0.3
plt.plot(x, y, "ro")
plt.xlabel(r"Temperature ($^\circ$C)")
plt.ylabel("Unweathered Kinematic Viscosity (m$^2$/s)")
plt.yscale("log", subsy=[2, 3, 4, 5, 6, 7, 8, 9])
plt.grid(True)
plt.axis([xmin - xpadding, xmax + xpadding, 0, ymax + ypadding])
# now we add the annotations
for xx, yy in np.vstack((x, y)).transpose():
print (xx, yy)
if xx > x.mean():
xalign = -xpadding / 3
else:
xalign = xpadding / 3
yalign = ypadding / 3
plt.annotate(
"(%s$^\circ$C, %s m$^2$/s)" % (xx, yy),
xy=(xx + (xalign / 10), yy + (yalign / 10)),
xytext=(xx + xalign, yy + yalign),
arrowprops=dict(facecolor="black", shrink=0.01),
fontsize=9,
)
plt.show()
def plot_oil_viscosities(settings):
with transaction.manager:
# -- Our loading routine --
session = _get_db_session()
if 'adios_id' not in settings:
raise ValueError('adios_id setting is required.')
adios_id = settings['adios_id']
print 'our session: %s' % (session)
try:
oilobj = (session.query(Oil).join(ImportedRecord)
.filter(ImportedRecord.adios_oil_id == adios_id)
.one())
except NoResultFound:
raise NoResultFound('No Oil was found matching adios_id {0}'
.format(adios_id))
if oilobj:
print 'Our oil object: %s' % (oilobj)
oil_props = OilProps(oilobj)
print '\nOilProps:', oil_props
print oil_props.kvis_at_temp()
print '\nOur viscosities:'
print [v for v in oilobj.kvis]
print '\nOur unweathered viscosities (m^2/s, Kdegrees):'
vis = [v for v in oilobj.kvis if v.weathering <= 0.0]
print vis
for i in [(v.m_2_s, v.ref_temp_k, v.weathering)
for v in vis]:
print i
x = np.array([v.ref_temp_k for v in vis]) - 273.15
y = np.array([v.m_2_s for v in vis])
xmin = x.min()
xmax = x.max()
xpadding = .5 if xmax == xmin else (xmax - xmin) * .3
ymin = y.min()
ymax = y.max()
ypadding = (ymax / 2) if ymax == ymin else (ymax - ymin) * .3
plt.plot(x, y, 'ro')
plt.xlabel(r'Temperature ($^\circ$C)')
plt.ylabel('Unweathered Kinematic Viscosity (m$^2$/s)')
plt.yscale('log', subsy=[2, 3, 4, 5, 6, 7, 8, 9])
plt.grid(True)
plt.axis([xmin - xpadding, xmax + xpadding, 0, ymax + ypadding])
# now we add the annotations
for xx, yy in np.vstack((x, y)).transpose():
print (xx, yy)
if xx > x.mean():
xalign = -xpadding / 3
else:
xalign = xpadding / 3
yalign = ypadding / 3
plt.annotate('(%s$^\circ$C, %s m$^2$/s)' % (xx, yy),
xy=(xx + (xalign / 10),
yy + (yalign / 10)),
xytext=(xx + xalign, yy + yalign),
arrowprops=dict(facecolor='black',
shrink=0.01),
fontsize=9)
plt.show()
def audit_distillation_cuts(settings):
'''
Just a quick check of the data values that we loaded
when we initialized the database.
'''
with transaction.manager:
session = _get_db_session()
sys.stderr.write('Auditing the records in database...')
num_cuts = []
liquid_temp = []
vapor_temp = []
fraction = []
for o in session.query(Oil):
num_cuts.append(len(o.cuts))
for i, cut in enumerate(o.cuts):
if len(liquid_temp) > i:
if cut.liquid_temp_k:
liquid_temp[i].append(cut.liquid_temp_k)
if cut.vapor_temp_k:
vapor_temp[i].append(cut.vapor_temp_k)
if cut.fraction:
fraction[i].append(cut.fraction)
else:
liquid_temp.append([])
vapor_temp.append([])
fraction.append([])
print ('\nNumber of cut entries (min, max, avg): ({0}, {1}, {2})'
.format(min(num_cuts),
max(num_cuts),
float(sum(num_cuts)) / len(num_cuts))
)
print '\nLiquid Temperature:'
for i in range(len(liquid_temp)):
if len(liquid_temp[i]) > 0:
temp = liquid_temp[i]
avg = float(sum(temp)) / len(temp)
print ('\tCut #{0} (set-size, min, max, avg): '
'({1}, {2}, {3}, {4})'
.format(i, len(temp), min(temp), max(temp), avg))
else:
temp = liquid_temp[i]
print ('\tCut #{0} (set-size, min, max, avg): '
'({1}, {2}, {3}, {4})'
.format(i, len(temp), None, None, None))
print '\nVapor Temperature:'
for i in range(len(vapor_temp)):
print ('\tCut #{0} (set-size, min, max, avg): '
'({1}, {2}, {3}, {4})'
.format(i,
len(vapor_temp[i]),
min(vapor_temp[i]),
max(vapor_temp[i]),
float(sum(vapor_temp[i])) / len(vapor_temp[i]))
)
print '\nFraction:'
for i in range(len(fraction)):
print ('\tCut #{0} (set-size, min, max, avg): '
'({1}, {2}, {3}, {4})'
.format(i,
len(fraction[i]),
min(fraction[i]),
max(fraction[i]),
float(sum(fraction[i])) / len(fraction[i]))
)
print 'finished!!!'
def audit_distillation_cuts(settings):
'''
Just a quick check of the data values that we loaded
when we initialized the database.
'''
with transaction.manager:
session = _get_db_session()
sys.stderr.write('Auditing the records in database...')
num_cuts = []
liquid_temp = []
vapor_temp = []
fraction = []
for o in session.query(Oil):
num_cuts.append(len(o.cuts))
for i, cut in enumerate(o.cuts):
if len(liquid_temp) > i:
if cut.liquid_temp_k:
liquid_temp[i].append(cut.liquid_temp_k)
if cut.vapor_temp_k:
vapor_temp[i].append(cut.vapor_temp_k)
if cut.fraction:
fraction[i].append(cut.fraction)
else:
liquid_temp.append([])
vapor_temp.append([])
fraction.append([])
print ('\nNumber of cut entries (min, max, avg): ({0}, {1}, {2})'
.format(min(num_cuts),
max(num_cuts),
float(sum(num_cuts)) / len(num_cuts))
)
print '\nLiquid Temperature:'
for i in range(len(liquid_temp)):
if len(liquid_temp[i]) > 0:
temp = liquid_temp[i]
avg = float(sum(temp)) / len(temp)
print ('\tCut #{0} (set-size, min, max, avg): '
'({1}, {2}, {3}, {4})'
.format(i, len(temp), min(temp), max(temp), avg))
else:
temp = liquid_temp[i]
print ('\tCut #{0} (set-size, min, max, avg): '
'({1}, {2}, {3}, {4})'
.format(i, len(temp), None, None, None))
print '\nVapor Temperature:'
for i in range(len(vapor_temp)):
print ('\tCut #{0} (set-size, min, max, avg): '
'({1}, {2}, {3}, {4})'
.format(i,
len(vapor_temp[i]),
min(vapor_temp[i]),
max(vapor_temp[i]),
float(sum(vapor_temp[i])) / len(vapor_temp[i]))
)
print '\nFraction:'
for i in range(len(fraction)):
print ('\tCut #{0} (set-size, min, max, avg): '
'({1}, {2}, {3}, {4})'
.format(i,
len(fraction[i]),
min(fraction[i]),
max(fraction[i]),
float(sum(fraction[i])) / len(fraction[i]))
)
print 'finished!!!'