def restore(self, fname, overwrite=False):
# Determine the id of the outputfile
if self.quantity == 'radiance':
mech = ("mechanism = '" +
", ".join(sorted([m for m in self.mechanism])) + "'")
wave_ = sorted([w.value for w in self.wavelength])
wave = ("wavelength = '" + ", ".join([str(w)
for w in wave_]) + "'")
else:
mech = 'mechanism is NULL'
wave = 'wavelength is NULL'
with database_connect() as con:
idnum_ = pd.read_sql(
f'''SELECT idnum
FROM outputfile
WHERE filename='{fname}' ''', con)
oid = idnum_.idnum[0]
result = pd.read_sql(
f'''SELECT filename FROM modelimages
WHERE out_idnum = {oid} and
quantity = '{self.quantity}' and
origin = '{self.origin.object}' and
dims[1] = {self.dims[0]} and
dims[2] = {self.dims[1]} and
center[1] = {self.center[0].value} and
center[2] = {self.center[1].value} and
width[1] = {self.width[0].value} and
width[2] = {self.width[1].value} and
subobslongitude = {self.subobslongitude.value} and
subobslatitude = {self.subobslatitude.value} and
{mech} and
{wave}''', con)
if (len(result) == 1) and overwrite:
if os.path.exists(result.filename[0]):
os.remove(result.filename[0])
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''DELETE FROM modelimages
WHERE filename = %s''', (result.filename[0], ))
image, packets = None, None
elif len(result) == 1:
image, packets = pickle.load(open(result.filename[0], 'rb'))
elif len(result) == 0:
image, packets = None, None
else:
raise RuntimeError('ModelImage.restore',
'Should not be able to get here.')
return image, packets
def insert(self):
ids = self.search()
if ids is None:
if self.type == 'uniform':
long = [l.value for l in self.longitude]
lat = [l.value for l in self.latitude]
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO spatdist_uniform (
exobase, longitude, latitude) VALUES (
%s, %s::DOUBLE PRECISION[2],
%s::DOUBLE PRECISION[2])''',
(self.exobase, long, lat))
elif self.type == 'surface map':
with database_connect() as con:
cur = con.cursor()
sslon = (None if self.subsolarlon is None else
self.subsolarlon.value)
cur.execute(
'''INSERT INTO spatdist_surfmap (
exobase, mapfile, subsolarlon,
coordinate_system) VALUES (
%s, %s, %s, %s)''',
(self.exobase, self.mapfile, sslon,
self.coordinate_system))
elif self.type == 'surface spot':
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO spatdist_spot (
exobase, longitude, latitude, sigma) VALUES (
%s, %s, %s, %s)''',
(self.exobase, self.longitude.value,
self.latitude.value, self.sigma.value))
elif self.type == 'fitted output':
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO spatdist_fittedoutput (
unfit_outid, query)
VALUES (%s, %s)''',
(self.unfit_outid, self.query))
else:
raise InputError(
'SpatialDist.search()',
f'SpatialDist.type = {self.type} not allowed.')
ids = self.search()
assert ids is not None
else:
pass
return ids
def insert(self):
# check to see if it is already there
ids = self.search()
if ids is None:
if self.type == 'geometry with starttime':
if self.objects is None:
objs = None
else:
objs = [o.object for o in self.objects]
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO geometry_with_time (
planet, startpoint, objects,
starttime VALUES (%s, %s, %s, %s)''',
(self.planet.object, self.startpoint, objs, self.time))
elif self.type == 'geometry without starttime':
if self.objects is None:
objs = None
else:
objs = [o.object for o in self.objects]
subspt = [s.value for s in self.subsolarpoint]
if self.phi is None:
phi = None
else:
phi = [p.value for p in self.phi]
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO geometry_without_time (
planet, startpoint, objects, phi,
subsolarpt, taa) VALUES (
%s, %s, %s::SSObject[], %s,
%s::DOUBLE PRECISION[2], %s)''',
(self.planet.object, self.startpoint, objs, phi,
subspt, self.taa.value))
else:
raise InputError('Geometry.search()',
f'geometry.type = {self.type} not allowed.')
ids = self.search()
assert ids is not None
else:
pass
return ids
def search(self):
params = [
self.endtime.value, self.species, self.lifetime.value,
self.outeredge, self.step_size, self.fitted
]
if self.resolution is None:
resol = 'resolution is NULL'
else:
resol = 'resolution = %s'
params.append(self.resolution)
query = f'''SELECT idnum
FROM options
WHERE endtime = %s and
species = %s and
lifetime = %s and
outer_edge = %s and
step_size = %s and
{resol} and
fitted = %s'''
with database_connect() as con:
cur = con.cursor()
cur.execute(query, tuple(params))
if cur.rowcount == 0:
return None
elif cur.rowcount == 1:
return cur.fetchone()[0]
else:
raise RuntimeError('Options.search()',
'Duplicates in options table')
def search(self):
if self.type == 'uniform':
params = [
self.exobase,
[self.longitude[0].value, self.longitude[1].value],
[self.latitude[0].value, self.latitude[1].value]
]
query = '''SELECT idnum
FROM spatdist_uniform
WHERE exobase = %s and
longitude = %s::DOUBLE PRECISION[2] and
latitude = %s::DOUBLE PRECISION[2]'''
elif self.type == 'surface map':
params = [self.exobase, self.mapfile]
if self.subsolarlon is None:
sslon = 'subsolarlon is NULL'
else:
sslon = 'subsolarlon = %s'
params.append(self.subsolarlon.value)
params.append(self.coordinate_system)
query = f'''SELECT idnum
FROM spatdist_surfmap
WHERE exobase = %s and
mapfile = %s and
{sslon} and
coordinate_system = %s'''
elif self.type == 'surface spot':
params = [
self.exobase, self.longitude.value, self.latitude.value,
self.sigma.value
]
query = '''SELECT idnum
FROM spatdist_spot
WHERE exobase = %s and
longitude = %s and
latitude = %s and
sigma = %s'''
elif self.type == 'fitted output':
params = [self.unfit_outid, self.query]
query = '''SELECT idnum
FROM spatdist_fittedoutput
WHERE unfit_outid = %s and
query = %s'''
else:
raise InputError('SpatialDist.__init__',
f'SpatialDist.type = {self.type} not defined.')
with database_connect() as con:
cur = con.cursor()
# print(cur.mogrify(query, tuple(params)))
cur.execute(query, tuple(params))
if cur.rowcount == 0:
return None
elif cur.rowcount == 1:
return cur.fetchone()[0]
else:
raise RuntimeError('SpatialDist.search()',
'Duplicates in spatial distribution table')
def insert(self):
ids = self.search()
if ids is None:
if self.type == 'radial':
assert 0, 'Should not be able to get here.'
elif self.type == 'isotropic':
alt = [a.value for a in self.altitude]
az = [a.value for a in self.azimuth]
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO angdist_isotropic (
altitude, azimuth) VALUES (
%s::DOUBLE PRECISION[2],
%s::DOUBLE PRECISION[2])''', (alt, az))
else:
raise InputError(
'AngularDist.search()',
f'angulardist.type = {self.type} not allowed.')
ids = self.search()
assert ids is not None
else:
pass
return ids
def search(self):
if self.type == 'radial':
return 0
elif self.type == 'isotropic':
params = [[a.value for a in self.altitude],
[a.value for a in self.azimuth]]
query = '''SELECT idnum
FROM angdist_isotropic
WHERE altitude=%s::DOUBLE PRECISION[2] and
azimuth=%s::DOUBLE PRECISION[2]'''
else:
raise InputError('AngularDist.__init__',
f'AngularDist.type = {self.type} not defined.')
with database_connect() as con:
cur = con.cursor()
cur.execute(query, tuple(params))
if cur.rowcount == 0:
return None
elif cur.rowcount == 1:
return cur.fetchone()[0]
else:
raise RuntimeError('AngularDist.search()',
'Duplicates in angular distribution table')
def search(self):
if self.type == 'gaussian':
params = [self.vprob.value, self.sigma.value]
query = '''SELECT idnum
FROM speeddist_gaussian
WHERE vprob = %s and
sigma = %s'''
elif self.type == 'sputtering':
params = [self.alpha, self.beta, self.U.value]
query = '''SELECT idnum
FROM speeddist_sputtering
WHERE alpha = %s and
beta = %s and
U = %s'''
elif self.type == 'maxwellian':
params = [self.temperature.value]
query = '''SELECT idnum
FROM speeddist_maxwellian
WHERE temperature = %s'''
elif self.type == 'flat':
params = [self.vprob.value, self.delv.value]
query = '''SELECT idnum
FROM speeddist_flat
WHERE vprob = %s and
delv = %s'''
elif self.type == 'user defined':
params = [self.vdistfile]
query = '''SELECT idnum
FROM speeddist_user
WHERE vdistfile = %s'''
elif self.type == 'fitted output':
params = [self.unfit_outid, self.query]
query = '''SELECT idnum
FROM speeddist_fittedoutput
WHERE unfit_outid = %s and
query = %s'''
else:
raise InputError('SpeedDist.__init__',
f'SpeedDist.type = {self.type} not defined.')
with database_connect() as con:
cur = con.cursor()
cur.execute(query, tuple(params))
if cur.rowcount == 0:
return None
elif cur.rowcount == 1:
return cur.fetchone()[0]
else:
raise RuntimeError('SpeedDist.search()',
'Duplicates in speed distribution table')
def insert(self):
ids = self.search()
if ids is None:
if self.sticktype == 'constant':
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO surface_int_constant (
stickcoef, accomfactor) VALUES (
%s, %s)''',
(self.stickcoef, self.accomfactor))
elif self.sticktype == 'surface map':
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO surface_int_map (
mapfile, accomfactor) VALUES (
%s, %s)''',
(self.stick_mapfile, self.accomfactor))
elif self.sticktype == 'temperature dependent':
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO surface_int_tempdependent (
accomfactor, a) VALUES
(%s, %s::DOUBLE PRECISION[3])''',
(self.accomfactor, self.A))
else:
raise InputError(
'SurfaceInteraction.search()',
f'surfaceinteraction.sticktype = {self.sticktype} not allowed.'
)
ids = self.search()
assert ids is not None
else:
pass
return ids
def save(self, iteration_result):
'''
Insert the result of a LOS iteration into the database
:param iteration_result: LOS result from a single outputfile
:return: name of saved file
'''
if self.quantity == 'radiance':
mech = ', '.join(sorted([m for m in self.mechanism]))
wave_ = sorted([w.value for w in self.wavelength])
wave = ', '.join([str(w) for w in wave_])
else:
mech = None
wave = None
tempname = f'temp_{str(random.randint(0, 1000000))}'
if isinstance(iteration_result, FittedIterationResult):
ufit_id = iteration_result.unfit_outid
else:
ufit_id = None
with database_connect() as con:
cur = con.cursor()
cur.execute(f'''INSERT into uvvsmodels (out_idnum, unfit_idnum,
quantity, query, dphi, mechanism, wavelength,
fitted, filename)
values (%s, %s, %s, %s, %s, %s, %s, %s, %s)''',
(iteration_result.out_idnum, ufit_id, self.quantity,
self.query, self.dphi, mech, wave, self.fitted,
tempname))
# Determine the savefile name
idnum_ = pd.read_sql(f'''SELECT idnum
FROM uvvsmodels
WHERE filename='{tempname}';''', con)
assert len(idnum_) == 1
idnum = int(idnum_.idnum[0])
savefile = os.path.join(os.path.dirname(iteration_result.outputfile),
f'model.{idnum}.pkl')
print(f'Saving model result {savefile}')
cur.execute(f'''UPDATE uvvsmodels
SET filename=%s
WHERE idnum=%s''', (savefile, idnum))
with open(savefile, 'wb') as f:
pickle.dump(iteration_result, f)
return savefile
def save(self, data_packets, surface):
# Insert the model into the database
# Save is on an outputfile basis
if self.quantity == 'radiance':
mech = ', '.join(sorted([m for m in self.mechanism]))
wave_ = sorted([w.value for w in self.wavelength])
wave = ', '.join([str(w) for w in wave_])
else:
mech = None
wave = None
# Save query with all white space removed and lowercase
tempname = f'temp_{str(random.randint(0, 1000000))}'
with database_connect() as con:
cur = con.cursor()
cur.execute(
f'''INSERT into uvvsmodels (out_idnum, quantity,
query, dphi, mechanism, wavelength,
fitted, filename)
values (%s, %s, %s, %s, %s, %s, %s, %s)''',
(data_packets['out_idnum'], self.quantity, self.scdata.query,
self.dphi, mech, wave, self.fitted, tempname))
# Determine the savefile name
idnum_ = pd.read_sql(
f'''SELECT idnum
FROM uvvsmodels
WHERE filename='{tempname}';''', con)
assert len(idnum_) == 1
idnum = int(idnum_.idnum[0])
savefile = os.path.join(
os.path.dirname(data_packets['outputfile']),
f'model.{idnum}.pkl')
cur.execute(
f'''UPDATE uvvsmodels
SET filename=%s
WHERE idnum=%s''', (savefile, idnum))
with open(savefile, 'wb') as f:
pickle.dump(data_packets, f)
if surface is None:
savefile_surf = None
else:
savefile_surf = self.surfacefile_name(savefile)
surface.to_parquet(savefile_surf)
return savefile, savefile_surf
def insert(self):
ids = self.search()
if ids is None:
if self.type == 'gaussian':
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO speeddist_gaussian (
vprob, sigma) VALUES (%s, %s)''',
(self.vprob.value, self.sigma.value))
elif self.type == 'sputtering':
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO speeddist_sputtering (
alpha, beta, U) VALUES (%s, %s, %s)''',
(self.alpha, self.beta, self.U.value))
elif self.type == 'maxwellian':
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO speeddist_maxwellian (
temperature) VALUES (%s)''',
(self.temperature.value, ))
elif self.type == 'flat':
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO speeddist_flat (
vprob, delv) VALUES (%s, %s)''',
(self.vprob.value, self.delv.value))
elif self.type == 'user defined':
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO speeddist_user (
vdistfile) VALUES (%s)''',
(self.vdistfile, ))
elif self.type == 'fitted output':
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO speeddist_fittedoutput (
unfit_outid, query)
VALUES (%s, %s)''',
(self.unfit_outid, self.query))
else:
raise InputError('SpeedDist.search()',
f'speeddist.type = {self.type} not allowed.')
ids = self.search()
assert ids is not None
else:
pass
return ids
def insert(self):
ids = self.search()
if ids is None:
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT INTO forces (
gravity, radpres) VALUES (%s, %s)''',
(self.gravity, self.radpres))
ids = self.search()
assert ids is not None
else:
pass
return ids
def search(self):
query = f'''SELECT idnum FROM forces
WHERE gravity = %s and
radpres = %s'''
with database_connect() as con:
cur = con.cursor()
cur.execute(query, (self.gravity, self.radpres))
if cur.rowcount == 0:
return None
elif cur.rowcount == 1:
return cur.fetchone()[0]
else:
raise RuntimeError('Forces.search()',
'Duplicates in forces table')
def search(self):
"""
:return: dictionary containing search results:
{outputfilename: (modelfile_id, modelfile_name)}
"""
search_results = {}
for outputfile in self.outputfiles:
with database_connect() as con:
# Determine the id of the outputfile
idnum_ = pd.read_sql(
f'''SELECT idnum
FROM outputfile
WHERE filename='{outputfile}' ''', con)
oid = idnum_.idnum[0]
if self.quantity == 'radiance':
mech = ("mechanism = '" +
", ".join(sorted([m
for m in self.mechanism])) + "'")
wave_ = sorted([w.value for w in self.wavelength])
wave = ("wavelength = '" +
", ".join([str(w) for w in wave_]) + "'")
else:
mech = 'mechanism is NULL'
wave = 'wavelength is NULL'
result = pd.read_sql(
f'''SELECT idnum, filename FROM uvvsmodels
WHERE out_idnum={oid} and
quantity = '{self.quantity}' and
query = '{self.scdata.query}' and
dphi = {self.dphi} and
{mech} and
{wave} and
fitted = {self.fitted}''', con)
# Should only have one match per outputfile
assert len(result) <= 1
if len(result) == 0:
search_results[outputfile] = None
else:
surffile = self.surfacefile_name(result.iloc[0, 1])
search_results[outputfile] = (result.iloc[0, 0],
result.iloc[0, 1], surffile)
return search_results
def search(self):
""" Search the database for previous model runs with the same inputs.
See :doc:`searchtolerances` for tolerances used in searches.
**Parameters**
No parameters.
**Returns**
* A list of filenames corresponding to the inputs.
* Number of packets contained in those saved outputs.
* Total modeled source rate.
"""
geo_id = self.geometry.search()
sint_id = self.surfaceinteraction.search()
for_id = self.forces.search()
spat_id = self.spatialdist.search()
spd_id = self.speeddist.search()
ang_id = self.angulardist.search()
opt_id = self.options.search()
if None in [geo_id, sint_id, for_id, spat_id, spd_id, ang_id, opt_id]:
return [], [], 0., 0.
else:
query = f'''SELECT idnum, filename, npackets, totalsource
FROM outputfile
WHERE geo_type = '{self.geometry.type}' and
geo_id = {geo_id} and
sint_type = '{self.surfaceinteraction.sticktype}' and
sint_id = {sint_id} and
force_id = {for_id} and
spatdist_type = '{self.spatialdist.type}' and
spatdist_id = {spat_id} and
spddist_type = '{self.speeddist.type}' and
spddist_id = {spd_id} and
angdist_type = '{self.angulardist.type}' and
angdist_id = {ang_id} and
opt_id = {opt_id}'''
with database_connect() as con:
result = pd.read_sql(query, con)
return (result.idnum.to_list(), result.filename.to_list(),
result.npackets.sum(), result.totalsource.sum())
def save(self, fname, image, packets):
# Determine the id of the outputfile
idnum = int(os.path.basename(fname).split('.')[0])
# Insert the image into the database
if self.quantity == 'radiance':
mech = ', '.join(sorted([m for m in self.mechanism]))
wave_ = sorted([w.value for w in self.wavelength])
wave = ', '.join([str(w) for w in wave_])
else:
mech = None
wave = None
width = [w.value for w in self.width]
center = [c.value for c in self.center]
with database_connect() as con:
cur = con.cursor()
cur.execute(
f'''INSERT into modelimages (out_idnum, quantity,
origin, dims, center, width, subobslongitude,
subobslatitude, mechanism, wavelength,
filename)
VALUES (%s, %s, %s, %s::INT[2],
%s::DOUBLE PRECISION[2],
%s::DOUBLE PRECISION[2],
%s, %s, %s, %s, 'temp')''',
(idnum, self.quantity, self.origin.object, self.dims, center,
width, self.subobslongitude.value, self.subobslatitude.value,
mech, wave))
idnum_ = pd.read_sql(
f'''SELECT idnum
FROM modelimages
WHERE filename = 'temp';''', con)
assert len(idnum_) == 1
idnum = int(idnum_.idnum[0])
savefile = os.path.join(os.path.dirname(fname), f'image.{idnum}.pkl')
with open(savefile, 'wb') as f:
pickle.dump((image, packets), f)
cur.execute(
f'''UPDATE modelimages
SET filename=%s
WHERE idnum = %s''', (savefile, idnum))
con.close()
def insert(self):
ids = self.search()
if ids is None:
with database_connect() as con:
cur = con.cursor()
cur.execute(
'''INSERT into options (endtime, species, lifetime,
outer_edge, step_size, resolution, fitted) VALUES (
%s, %s, %s, %s, %s, %s, %s)''',
(self.endtime.value, self.species, self.lifetime.value,
self.outeredge, self.step_size, self.resolution,
self.fitted))
ids = self.search()
assert ids is not None
else:
pass
return ids
def search(self):
if self.sticktype == 'constant':
params = [self.stickcoef]
if self.accomfactor is None:
afactor = 'accomfactor is NULL'
else:
afactor = 'accomfactor = %s'
params.append(self.accomfactor)
query = f"""SELECT idnum
FROM surface_int_constant
WHERE stickcoef = %s and
{afactor}"""
elif self.sticktype == 'temperature dependent':
params = [self.accomfactor, self.A]
query = f"""SELECT idnum
FROM surface_int_tempdependent
WHERE accomfactor = %s and
a = %s::DOUBLE PRECISION[3]"""
elif self.sticktype == 'surface map':
params = [self.stick_mapfile, self.accomfactor]
query = f"""SELECT idnum
FROM surface_int_map
WHERE mapfile = %s and
accomfactor = %s"""
else:
raise InputError(
'SurfaceInteraction.search()',
f'surfaceinteraction.sticktype = {self.sticktype} not allowed.'
)
with database_connect() as con:
cur = con.cursor()
cur.execute(query, tuple(params))
if cur.rowcount == 0:
return None
elif cur.rowcount == 1:
return cur.fetchone()[0]
else:
raise RuntimeError('SurfaceInteraction.search()',
'Duplicates in surface interaction table')
def delete_models(self):
"""Deletes any LOSResult models associated with this data and input
This may never actually do anything. Overwrite=True will also
erase the outputfiles (which erases any models that depend on them).
Unless I put separate outputfile and modelfile delete switches,
This shouldn't do anything"""
search_results = self.search()
if len(search_results) != 0:
print('Warning: LOSResult.delete_models found something to delete')
for _, search_result in search_results.items():
if search_result is not None:
idnum, modelfile = search_result
with database_connect() as con:
cur = con.cursor()
cur.execute(
f'''DELETE from uvvsmodels
WHERE idnum = %s''', (idnum, ))
if os.path.exists(modelfile):
os.remove(modelfile)
else:
pass
def test_database_connect():
# Test database connection
with database_connect() as con:
assert con.autocommit
def _spectrum_process(self, spectrum, packets, tree, dist, i, ofile):
x_sc = spectrum[xcols].values.astype(float)
bore = spectrum[borecols].values.astype(float)
dd = 30 # Furthest distance we need to look
x_far = x_sc + bore * dd
while np.linalg.norm(x_far) > self.oedge:
dd -= 0.1
x_far = x_sc + bore * dd
t = [0.05]
while t[-1] < dd:
t.append(t[-1] + t[-1] * np.sin(self.dphi))
t = np.array(t)
Xbore = x_sc[np.newaxis, :] + bore[np.newaxis, :] * t[:, np.newaxis]
wid = t * np.sin(self.dphi)
ind = np.concatenate(tree.query_radius(Xbore, wid))
ilocs = np.unique(ind).astype(int)
indicies = packets.iloc[ilocs].index
subset = packets.loc[indicies]
xpr = subset[xcols] - x_sc[np.newaxis, :]
rpr = np.sqrt(xpr['x'] * xpr['x'] + xpr['y'] * xpr['y'] +
xpr['z'] * xpr['z'])
losrad = np.sum(xpr * bore[np.newaxis, :], axis=1)
inview = rpr < dist
if np.any(inview):
used_packets = subset[inview].index.to_list()
used_packets0 = subset.loc[inview, 'Index']
Apix = np.pi * (rpr[inview] *
np.sin(self.dphi))**2 * (self.unit.to(u.cm))**2
wtemp = subset.loc[inview, 'weight'] / Apix
if self.quantity == 'radiance':
# Determine if any packets are in shadow
# Projection of packet onto LOS
# Point along LOS the packet represents
losrad_ = losrad[inview].values
hit = (x_sc[np.newaxis, :] +
bore[np.newaxis, :] * losrad_[:, np.newaxis])
rhohit = np.linalg.norm(hit[:, [0, 2]], axis=1)
out_of_shadow = (rhohit > 1) | (hit[:, 1] < 0)
wtemp *= out_of_shadow
rad = wtemp.sum()
# Save the weight information
rat = spectrum.radiance / rad if rad > 0 else 0.
params = []
for U, w, f0, u0 in zip(
used_packets, wtemp.values,
packets.loc[used_packets,
'frac0'].values, used_packets0.values):
params.append((self.scdata.query, ofile, int(i), int(U), w,
float(f0), int(u0), rat))
# params = list(zip(used_packets, wtemp.values,
# packets.loc[used_packets, 'frac0'].values.astype('float'),
# used_packets0.values.astype('float')))
statement = '''INSERT into savedpackets (
query, outputfile, specind, oint,
weight, frac0, index0, ratio) VALUES (
%s, %s, %s, %s, %s::DOUBLE PRECISION,
%s::DOUBLE PRECISION, %s,
%s::DOUBLE PRECISION)'''
with database_connect() as con:
cur = con.cursor()
execute_batch(cur, statement, params)
del params
else:
assert False, 'Other quantities not set up.'
else:
pass
def determine_source_from_data(self, modnum, weight_method='scaling'):
modkey = f'model{modnum:02d}'
maskkey = f'mask{modnum:02d}'
mask = self.scdata.data[maskkey]
self.outputfiles = self.scdata.model_info[modkey]['outputfiles']
surface_source = pd.DataFrame(columns=[
'outputfile', 'Index', 'longitude', 'latitude', 'velocity'
])
nsteps, endtime = 0, 0
index0 = {}
for outputfile in self.outputfiles:
# Restore the unfit output file
output = Output.restore(outputfile)
self.totalsource += output.totalsource
endtime = output.inputs.options.endtime
nsteps = output.nsteps
with database_connect() as con:
ind0 = pd.read_sql(
f'''SELECT DISTINCT index0 FROM savedpackets
WHERE query='{self.scdata.query}' and
outputfile='{outputfile}';''', con)
index0[outputfile] = ind0.index0.to_list()
del output
# vel_ = np.sqrt(output.X0.vx**2 + output.X0.vy**2 +
# output.X0.vz**2) * output.inputs.geometry.planet.radius
# surface_source = surface_source.append(pd.DataFrame(
# {'outputfile': [outputfile for _ in range(len(ind0))],
# 'Index': ind0,
# 'longitude': output.X0.loc[ind0, 'longitude'].values,
# 'latitude': output.X0.loc[ind0, 'latitude'].values,
# 'velocity': vel_.loc[ind0]}), ignore_index = True)
if weight_method == 'scaling':
# Determine the proper weightings
outputfile_str = self.outputfiles.__str__().replace('[',
'(').replace(
']', ')')
statement = f'''UPDATE savedpackets
SET scale_factor=%s::DOUBLE PRECISION
WHERE query='{self.scdata.query}' and
outputfile=%s and
index0=%s'''
with database_connect() as con:
cur = con.cursor()
scale_factor = pd.read_sql(
f'''SELECT outputfile, index0, AVG(ratio) from savedpackets
WHERE query='{self.scdata.query}' and
outputfile in {outputfile_str}
GROUP BY outputfile, index0''', con)
scale_factor_ind = scale_factor.set_index(
['outputfile', 'index0'])
params = list(
zip(scale_factor_ind['avg'].values,
[ind[0] for ind in scale_factor_ind.index],
[ind[1] for ind in scale_factor_ind.index]))
execute_batch(cur, statement, params)
# for ind, scfactor in scale_factor_ind.iterrows():
# cur.execute(f'''UPDATE savedpackets
# SET scale_factor=%s::DOUBLE PRECISION
# WHERE query='{self.scdata.query}' and
# outputfile=%s and
# index0=%s''',
# (scfactor, ind[0], ind[1]))
weight = pd.read_sql(
f'''SELECT outputfile, oint, index0, weight*scale_factor fit_weight
FROM savedpackets
WHERE query='{self.scdata.query}' and
outputfile in {outputfile_str};''', con)
from IPython import embed
embed()
import sys
sys.exit()
# ind = pd.MultiIndex.from_arrays([weight.outputfile.to_list(),
# weight.index0.to_list()])
# scale_factor_ = scale_factor_ind.loc[ind, 'avg'].reset_index()
# fit_weight = weight.weight * scale_factor_['avg']
# Determine surface weighting
# surface_ind = surface_source.set_index(['outputfile', 'Index'])
# surface_ind.loc[scale_factor_.index, 'weight'] = scale_factor_
# surface_ind.dropna(inplace=True)
# surface_source = surface_ind.reset_index()
elif weight_method == 'leastsq':
# This method produces too many scale_factors < 0
assert 0
# Solve linear equations to get best fit
temp = self.data_packets.data.set_index('specind')
temp = temp.loc[mask]
temp.reset_index(inplace=True)
W = np.zeros((len(data), len(self.modelfiles),
self.data_packets.data.Index0.max() + 1))
spec_ind = [data.index.get_loc(i) for i in temp.specind.values]
pack_ind = temp.index.to_list()
outputfiles = {
ofile: i
for i, ofile in enumerate(self.modelfiles.keys())
}
oo = [outputfiles[ofile] for ofile in temp.outputfile]
W[spec_ind, oo, temp.Index0.to_list()] = temp.weight
W2 = W.squeeze(axis=1) / W.mean()
# assert np.all(W == W2.reshape(W.shape))
assert W.shape[1] == 1
scale_factor_, r, rank, s = lin.lstsq(
W2, data.loc[:, 'radiance'].values)
scale_factor_ /= scale_factor_[scale_factor_ > 0].mean()
self.data_packets.data['scale_factor'] = (
scale_factor_[self.data_packets.data.Index0.to_list()])
self.data_packets.data['fit_weight'] = (
self.data_packets.data.weight *
self.data_packets.data.scale_factor)
surface_source['weight'] = scale_factor_[
surface_source.Index.to_list()]
scaled = pd.read_pickle('surface_source_scaled.pkl')
from IPython import embed
embed()
import sys
sys.exit()
self.data_packets.data['scale_factor'] = (
scale_factor_[self.data_packets.data['Index0'].to_list()])
# Determine surface weighting
surface_source.dropna(inplace=True)
# surface_source = surface_source[surface_source.weight > 0]
else:
raise InputError('LOSResult.determine_source_from_data: '
'Not a valid weighting method')
initial = set(
zip(self.data_packets.data.outputfile.to_list(),
self.data_packets.data.Index0.to_list(),
self.data_packets.data.frac0.to_list(),
self.data_packets.data.scale_factor.to_list()))
frac0 = sum(x[2] * x[3] for x in initial)
self.totalsource = frac0 * nsteps
mod_rate = self.totalsource / endtime.value
atoms_per_packet = 1e23 / mod_rate
spec_group = self.data_packets.data.groupby('specind')
fitrad_ = spec_group['fit_weight'].sum() * atoms_per_packet
self.radiance.loc[fitrad_.index] = fitrad_
self.radiance *= u.R
# Create a new sourcemap
source, xx, yy = np.histogram2d(surface_source['longitude'],
surface_source['latitude'],
weights=surface_source['weight'],
range=[[0, 2 * np.pi],
[-np.pi / 2, np.pi / 2]],
bins=(NLONBINS, NLATBINS))
source = source / np.cos(yy + (yy[1] - yy[0]) / 2.)[np.newaxis, :-1]
source[:, [0, -1]] = 0
v_source, v = np.histogram(surface_source['velocity'],
bins=NVELBINS,
range=[0, surface_source['velocity'].max()],
weights=surface_source['weight'])
v_source /= np.max(v_source)
# packets available
packets, _, _ = np.histogram2d(surface_source['longitude'],
surface_source['latitude'],
range=[[0, 2 * np.pi],
[-np.pi / 2, np.pi / 2]],
bins=(NLONBINS, NLATBINS))
packets = packets / np.cos(yy + (yy[1] - yy[0]) / 2.)[np.newaxis, :-1]
packets[:, [0, -1]] = 0
v_packets, _ = np.histogram(
surface_source['velocity'],
bins=NVELBINS,
range=[0, surface_source['velocity'].max()])
v_packets = v_packets / np.max(v_packets)
sourcemap = {
'longitude': xx * u.rad,
'latitude': yy * u.rad,
'abundance': source,
'p_available': packets,
'velocity': v * u.km / u.s,
'vdist': v_source,
'v_available': v_packets,
'coordinate_system': 'solar-fixed'
}
self.sourcemap = sourcemap
def simulate_data_from_inputs(self,
inputs_,
npackets,
overwrite=False,
packs_per_it=None):
"""Given a set of inputs, determine what the spacecraft should see.
Parameters
==========
inputs_
A nexoclom Input object or the name of an inputs file
"""
if isinstance(inputs_, str):
self.inputs = Input(inputs_)
elif isinstance(inputs_, Input):
self.inputs = copy.deepcopy(inputs_)
else:
raise InputError('nexoclom.LOSResult', 'Problem with the inputs.')
# TAA needs to match the data
self.inputs.geometry.taa = self.scdata.taa
self.unit = u.def_unit('R_' + self.inputs.geometry.planet.object,
self.inputs.geometry.planet.radius)
# If using a planet-fixed source map, need to set subsolarlon
if ((self.inputs.spatialdist.type == 'surface map') and
(self.inputs.spatialdist.coordinate_system == 'planet-fixed')):
self.inputs.spatialdist.subsolarlon = self.scdata.subslong.median(
) * u.rad
else:
pass
# Run the model
self.fitted = False
self.inputs.run(npackets,
packs_per_it=packs_per_it,
overwrite=overwrite)
self.search_for_outputs()
dist_from_plan = self._data_setup()
data = self.scdata.data
for outputfile in self.outputfiles:
with database_connect() as con:
ct = pd.read_sql(
f'''SELECT count(*) FROM savedpackets
WHERE query='{self.scdata.query}' and
outputfile='{outputfile}';''', con)
output = Output.restore(outputfile)
self.totalsource += output.totalsource
if ct.loc[0, 'count'] == 0:
packets = copy.deepcopy(output.X)
packets['frac0'] = output.X0.loc[packets['Index'],
'frac'].values
packets['radvel_sun'] = (
packets['vy'] + output.vrplanet.to(self.unit / u.s).value)
self.oedge = output.inputs.options.outeredge * 2
# Will base shadow on line of sight, not the packets
out_of_shadow = np.ones(len(packets))
self.packet_weighting(packets, out_of_shadow, output.aplanet)
# This sets limits on regions where packets might be
tree = self._tree(packets[xcols].values)
print(f'{data.shape[0]} spectra taken.')
for i, spectrum in data.iterrows():
self._spectrum_process(spectrum, packets, tree,
dist_from_plan[i], i, outputfile)
ind = data.index.get_loc(i)
if (ind % (len(data) // 10)) == 0:
print(f'Completed {ind+1} spectra')
else:
pass
del output
mod_rate = self.totalsource / self.inputs.options.endtime.value
atoms_per_packet = 1e23 / mod_rate
outputfile_str = self.outputfiles.__str__().replace('[', '(').replace(
']', ')')
with database_connect() as con:
rad_ = pd.read_sql(
f'''SELECT specind, sum(weight) from savedpackets
WHERE query='{self.scdata.query}' and
outputfile in {outputfile_str}
GROUP BY specind''', con)
self.radiance.loc[
rad_.specind.to_list()] = rad_['sum'].values * atoms_per_packet
self.radiance *= u.R
def delete_files(self):
"""Delete output files and remove them from the database.
**Parameters**
filelist
List of files to remove. This can be found with Inputs.search()
**Returns**
No outputs.
"""
idnum, filelist, _, _ = self.search()
with database_connect() as con:
cur = con.cursor()
for i, f in zip(idnum, filelist):
# Remove from database and delete file
print(f'Deleting outputfile {os.path.basename(f)}')
cur.execute(
'''DELETE FROM outputfile
WHERE idnum = %s''', (i, ))
if os.path.exists(f):
os.remove(f)
# Delete any model images that depend on this output
cur.execute(
'''SELECT idnum, filename FROM modelimages
WHERE out_idnum = %s''', (i, ))
for mid, mfile in cur.fetchall():
print(f'Deleting model image {os.path.basename(mfile)}')
cur.execute(
'''DELETE from modelimages
WHERE idnum = %s''', (mid, ))
if os.path.exists(mfile):
os.remove(mfile)
# Delete any uvvsmodels that depend on this output
cur.execute(
'''SELECT idnum, filename FROM uvvsmodels
WHERE out_idnum = %s''', (i, ))
for mid, mfile in cur.fetchall():
print(
f'Deleting uvvsmodel result {os.path.basename(mfile)}')
cur.execute(
'''DELETE from uvvsmodels
WHERE idnum = %s''', (mid, ))
if os.path.exists(mfile):
os.remove(mfile)
# Remove the fitted uvvsmodels that depend on this output
uvvsmods = pd.read_sql(
f'''SELECT filename FROM uvvsmodels
WHERE unfit_idnum={i}''', con)
for _, uvvsmod in uvvsmods.iterrows():
print(f'Deleting fitted uvvsmodel result ' +
os.path.basename(uvvsmod.filename))
cur.execute(f'''DELETE FROM uvvsmodels
WHERE filename='{uvvsmod.filename}';''')
if os.path.exists(uvvsmod.filename):
os.remove(uvvsmod.filename)
# Delete any fitted outputs that depend on this output
spatdist_id = pd.read_sql(
f'''SELECT idnum FROM spatdist_fittedoutput
WHERE unfit_outid = {i}''', con)
for num in spatdist_id.idnum.to_list():
# Remove the outputfile
fitoutfile = pd.read_sql(
f'''SELECT idnum, filename FROM outputfile
WHERE spatdist_type = 'fitted output' and
spatdist_id = {num}''', con)
for _, row in fitoutfile.iterrows():
print(f'Deleting fitted output ' +
os.path.basename(row.filename))
cur.execute(f'''DELETE FROM outputfile
WHERE filename='{row.filename}';''')
if os.path.exists(row.filename):
os.remove(row.filename)
# Remove the modelimages
modims = pd.read_sql(
f'''SELECT filename FROM modelimages
WHERE out_idnum={row.idnum}''', con)
for _, modim in modims.iterrows():
print(f'Deleting model image ' +
os.path.basename(modim.filename))
cur.execute(f'''DELETE FROM modelimages
WHERE filename='{modim.filename}';'''
)
if os.path.exists(modim.filename):
os.remove(modim.filename)
def search(self):
if self.type == 'geometry with starttime':
params = [self.planet, self.startpoint]
if self.objects is None:
objects = 'objects is NULL'
else:
objects = 'objects = %s'
params.append([o.object for o in self.objects])
params.append(self.time)
query = f'''
SELECT idnum
FROM geometry_with_time
WHERE planet = %s and
startpoint = %s and
{objects} and
starttime = %s'''
elif self.type == 'geometry without starttime':
params = [self.planet.object, self.startpoint]
if self.objects is None:
objects = 'objects is NULL'
else:
objects = 'objects = %s::SSObject[]'
params.append([o.object for o in self.objects])
if self.phi is None:
phi = 'phi is NULL'
else:
phi = 'phi is %s'
params.append([p.value for p in self.phi])
params.append([s.value for s in self.subsolarpoint])
params.append(self.taa.value - dtaa / 2.)
params.append(self.taa.value + dtaa / 2.)
query = f"""
SELECT idnum, taa
FROM geometry_without_time
WHERE planet = %s and
startpoint = %s and
{objects} and
{phi} and
subsolarpt = %s::DOUBLE PRECISION[2] and
taa > %s and
taa < %s"""
else:
raise InputError('geometry.search()',
f'geometry.type = {self.type} not allowed.')
with database_connect() as con:
cur = con.cursor()
cur.execute(query, tuple(params))
if cur.rowcount == 0:
return None
elif cur.rowcount == 1:
return cur.fetchone()[0]
else:
results = cur.fetchall()
diff = [np.abs(row[1] - self.taa.value) for row in results]
result = [
row[0] for row in results
if np.abs(row[1] - self.taa.value) == min(diff)
]
if len(result) == 1:
return result[0]
else:
raise RuntimeError('geometry.search()',
'Duplicates in geometry table')
