def resolve(id, ra, dec, catalog, table): Session, engine = load_connection(SETTINGS['CONNECTION_STRING'], echo=False) session = Session() already_found = [result.file_id for result in session.query(table).filter(table.file_id == id)] if len(already_found): print("Resolving {} {} {}: Already Resolved".format(id, ra, dec)) else: lines = catalog.query_region(coord.SkyCoord(ra=ra, dec=dec, unit=(u.deg, u.deg), frame='icrs'), radius="1s") #-- index into first (only) table if len(lines) > 1: lines = lines[0] if len(lines) >= 1: print("Resolving {} {} {}, {} Matches Found".format(id, ra, dec, len(lines))) #strip off everything but the first: data = {key:clean(lines[0][key][0].item()) for key in lines[0].keys() if not key in ['_RAJ2000', '_DEJ2000']} data['file_id'] = id print data session.add(table(**data)) session.commit() else: print("Resolving {} {} {}: No Data found".format(id, ra, dec)) session.close() engine.dispose()
def rm_name(name): Session, engine = load_connection(SETTINGS['CONNECTION_STRING'], echo=False) session = Session() session.query(Sed).filter(Sed.designation==name).delete() session.commit() session.close() engine.dispose()
def parse_initial(row): Session, engine = load_connection(SETTINGS['CONNECTION_STRING'], echo=False) session = Session() row_data = {key:clean(value) for key,value in zip(row.columns, row.data)} session.add(Sed(**row_data)) session.commit() session.close() engine.dispose()
def parse_classifications(txtline): Session, engine = load_connection(SETTINGS['CONNECTION_STRING'], echo=False) session = Session() data = json.loads(txtline) ddid = data['subject_ids'][0]['$oid'] for line in data['annotations']: if 'classified_as' in line: to_insert = {'ddid': ddid, 'classified_as': line['classified_as']} session.add(Classifications(**to_insert)) session.commit() session.close() engine.dispose()
def parse_subjects(txtline): Session, engine = load_connection(SETTINGS['CONNECTION_STRING'], echo=False) session = Session() data = json.loads(txtline) to_insert = {'ddid': data['_id']['$oid'], 'wise_id': data['metadata'].get('wise_id', None), 'state': data['state'], 'im_2massj': data['location'].get('2massj', None), 'im_2massk': data['location'].get('2massk', None)} session.add(Subjects(**to_insert)) session.commit() session.close() engine.dispose()
import astropy.units as u import numpy as np import multiprocessing as mp import yaml import json from db_tables import load_connection, open_settings from db_tables import Sed, Urat, Hip, Ais, Hip_main, Iphas from db_tables import Base from db_tables import Subjects, Classifications SETTINGS = yaml.load(open(os.path.join(os.environ['HOME'], 'dd_configure.yaml'))) print SETTINGS Session, engine = load_connection(SETTINGS['CONNECTION_STRING'], echo=True) #------------------------------------------------------------------------------- def clean(value): if isinstance(value, float): if np.isnan(value): return None return round(value, 7) elif isinstance(value, str): if value == 'nan': return None if value == 'masked': return None if str(value) == '--': return None