def upload_data(args, molecule, isos, d_refs):
"""
Upload the new transitions and states to the database. Only do this for
real if args.dry_run = False.
Arguments:
args: the processed command line arguments with the names of files to
use in uploading the data, the list of HITRAN molecule and isotopologue
IDs to resolve the global_iso_id to etc...
molecule: the Molecule object for the molecule whose transitions and
states are to be uploaded.
isos: a list of Iso objects, ordered by their local isotopologue ID
cases_list: a list of Case objects, where the index is the case_id (ie
cases_list[0] is None, cases_list[1] represents the dcs case etc...
d_refs: a dictionary of RefsMap objects, keyed by HITRAN-style refID,
e.g. 'O2-gamma_self-2'.
"""
if args.dry_run:
vprint('[DRY RUN] Uploading to database...')
else:
vprint('Uploading to database...')
# first, expire old lines
expire_old_transitions(args)
# get the all the molecular state description 'cases' in a list indexed
# by caseID
cases = Case.objects.all()
cases_list = [None,] # caseIDs start at 1, so case_list[0]=None
for case in cases:
cases_list.append(case)
# find out the ID at which we can start adding states
try:
first_stateID = State.objects.all().order_by('-id')[0].id + 1
except IndexError:
# no states in the database yet, so we start at 1
first_stateID = 1
vprint('new states will be added with ids starting at %d' % first_stateID)
# upload the new states
states = upload_states(args, isos, cases_list)
# get the Source objects we'll need to attach to the parameters
sources = get_sources(d_refs)
# this is the default Source for when we can't find anything better:
hitran86_source = Source.objects.all().filter(pk=HITRAN1986_SOURCEID).get()
# now read in and upload the transitions
if args.dry_run:
vprint('[DRY RUN] Uploading transitions ...')
else:
cursor = connection.cursor()
vprint('Uploading transitions ...')
start_time = time.time()
ntrans = 0
for line in open(args.trans_file_upload, 'r'):
line = line.rstrip() # strip the EOL because the last field is par_line
trans = HITRANTransition()
for prm_name in trans_prms:
# create and attach the HITRANParam objects
setattr(trans, prm_name, HITRANParam(None))
fields = line.split(',')
for i, output_field in enumerate(trans_fields):
# set the transition attributes
trans.set_param(output_field.name, fields[i], output_field.fmt)
# attach the upper state to the transition
if trans.stateIDp < first_stateID:
# this state is already in the database: find it
trans.statep = State.objects.all().get(pk=trans.stateIDp)
else:
# new upper state: get it from the states list
trans.statep = states[trans.stateIDp-first_stateID]
# attach the lower state to the transition
if trans.stateIDpp < first_stateID:
# this state is already in the database: find it
trans.statepp = State.objects.all().get(pk=trans.stateIDpp)
else:
# new lower state: get it from the states list
trans.statepp = states[trans.stateIDpp-first_stateID]
# attach the case_module for this transition's states' quantum numbers
trans.case_module = hitran_meta.get_case_module(trans.molec_id,
trans.local_iso_id)
# fetch the right Iso object
iso = isos[trans.local_iso_id-1]
# this_trans is a hitranmeta.Trans object for the MySQL database
this_trans = Trans(iso=iso, statep=trans.statep, statepp=trans.statepp,
nu=trans.nu.val, Sw=trans.Sw.val, A=trans.A.val,
multipole=trans.multipole, Elower=trans.Elower, gp=trans.gp,
gpp=trans.gpp, valid_from=args.s_mod_date,
par_line=trans.par_line)
ntrans += 1
if not args.dry_run:
# if we're really uploading, save the transition to the database
this_trans.save()
# create and execute the INSERT strings for the transition's parameters
for prm_name in trans_prms:
val = trans.get_param_attr(prm_name, 'val')
if val is None:
# no value for this parameter - move on to the next one
continue
# fetch the Source object for this parameter
source_id = trans.get_param_attr(prm_name, 'source_id')
if source_id is None:
# if we can't identify source_id, it's missing from the
# hitranmeta_refs_map and/or hitransmeta_source tables:
# this is fatal and we must exit with an error
print 'Error! no reference specified for', prm_name
sys.exit(1)
# create the parameter object for this parameter
prm_fields = ['trans_id', 'val']
prm_entries = [str(this_trans.id), str(val)]
err = trans.get_param_attr(prm_name, 'err')
if err is not None:
prm_fields.append('err')
prm_entries.append(str(err))
prm_fields.append('ierr')
prm_entries.append(str(trans.get_param_attr(prm_name, 'ierr')))
prm_fields.append('source_id')
prm_entries.append(str(source_id))
s_insert = 'INSERT INTO prm_%s (%s) VALUES (%s)'\
% (prm_name.lower(), ', '.join(prm_fields),
', '.join(prm_entries))
if not args.dry_run:
# if we're really uploading, save the prm to the database
cursor.execute(s_insert)
end_time = time.time()
vprint('%d transitions read in (%s)' % (ntrans,
timed_at(end_time - start_time)))
def upload_states(args, isos, cases_list):
"""
Read in, store, and upload the states to enter the database from the
.states file.
Arguments:
args: the processed command line arguments with the names of files to
use in uploading the data, the list of HITRAN molecule and isotopologue
IDs to resolve the global_iso_id to etc...
isos: a list of Iso objects, ordered by their local isotopologue ID
cases_list: a list of Case objects, where the index is the case_id (ie
cases_list[0] is None, cases_list[1] represents the dcs case etc...
Returns:
a list of the State objects uploaded.
"""
if args.dry_run:
vprint('[DRY RUN] Uploading states...')
else:
vprint('Uploading states...')
# the uploaded states will be stored in this list:
states = []
start_time = time.time()
for line in open(args.states_file, 'r'):
global_iso_id = int(line[:4])
# state energy
try:
E = float(line[5:15])
except (TypeError, ValueError):
# undefined energy for this state
E = None
# state degeneracy
try:
g = int(line[16:21])
except (TypeError, ValueError):
# undefined degeneracy for this state
g = None
# state quantum numbers as a string of ';' separated name=value pairs
s_qns = line[22:].strip()
if not s_qns:
# no quantum numbers resolved for this state
s_qns == None
# the native HITRAN IDs for molecule and isotopologue:
molec_id, local_iso_id = args.hitran_ids[global_iso_id]
# get the right Class to use to describe this state
CaseClass = hitran_meta.get_case_class(molec_id, local_iso_id)
# state is one of the hitran_case states (e.g. an HDcs object)
state = CaseClass(molec_id=molec_id, local_iso_id=local_iso_id,
global_iso_id=global_iso_id, E=E, g=g, s_qns=s_qns)
# retrieve the correct Iso object for this isotopologue
iso = isos[local_iso_id-1]
# this_state is a hitranlbl.State object for the MySQL database
this_state = State(iso=iso, energy=state.E, g=state.g,
s_qns=state.s_qns, qns_xml=state.get_qns_xml())
if not args.dry_run:
# if we're doing it for real, save the State just created
this_state.save()
states.append(this_state)
# now create the quantum numbers entries for this state
case = cases_list[state.__class__.caseID]
# loop over all the possible quantum number names for this case
# XXX this will fail to include all of the quantum numbers if
# the case does not have the right quantum numbers in its
# ordered_qn_list (e.g. asymcs currently only goes to v12...)
for qn_name in state.__class__.ordered_qn_list:
# get the value of this quantum number
qn_val = state.get(qn_name)
if qn_val is None:
# if the quantum number isn't defined, move to the next one
continue
# get any attribute metadata for this quantum number
qn_attr = state.serialize_qn_attrs(qn_name)
if qn_attr:
# strip the initial '#'
qn_attr = qn_attr[1:]
else:
qn_attr = None
# get the XML for this quantum number
xml = state.get_qn_xml(qn_name)
if not xml:
xml = None
# create the quantum number object ...
qn = Qns(case=case, state=this_state, qn_name=qn_name,
qn_val=str(qn_val), qn_attr=qn_attr, xml=xml)
if not args.dry_run:
# ... and save it to the database if we're not on a dry run
qn.save()
end_time = time.time()
vprint('%d states read in (%s)' % (len(states),
timed_at(end_time - start_time)))
return states
#print this_trans.stateIDp, this_trans.stateIDpp
this_trans.statep = get_state(this_trans.stateIDp)
this_trans.statepp = get_state(this_trans.stateIDpp)
this_trans.case_module = hitran_meta.get_case_module(this_trans.molec_id,
this_trans.local_iso_id)
# OH (A-X) system is a special case:
if this_trans.statep.global_iso_id == 48 and s_qns[15]=='A':
this_trans.case_module = hcase_OHAX
if not this_trans.validate_as_par():
this_trans.old_par_line = this_trans.par_line
this_trans.par_line = correct_par(this_trans)
if this_trans.validate_as_par():
print >>co, '%d-%s' % (line_no, this_trans.old_par_line)
print >>co, '%d+%s' % (line_no, this_trans.par_line)
ncorrections += 1
else:
print this_trans.par_line,'\nfailed to validate! I produced:'
print this_trans.get_par_str()
sys.exit(1)
trans.append(this_trans)
co.close()
print '%d transitions read in.' % (len(trans))
end_time = time.time()
print '%d transitions and %d states in %s' % (len(trans), len(states),
xn_utils.timed_at(end_time - start_time))
print '%d corrections made to original .par file saved as %s'\
% (ncorrections, corrections_file)
