class CoRoTId(object): ''' classdocs ''' def __init__(self): ''' Constructor ''' self.data = [] from datasource import DataSource self.wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') def fromfile(self, filename=None): import pyfits if filename is None: filename = '/work2/jwe/NGC2236/NGC2236corotid.fits' hdulist = pyfits.open(filename) self.data = hdulist[1].data hdulist.close() def todatabase(self): for d in self.data: print d['starid'], d['corotid_2'] self.wifsip.execute("""UPDATE ngc2236 SET corotid = %d WHERE starid = '%s';""" % (d['corotid_2'], d['starid']), commit=False) self.wifsip.commit()
class TwoMass(object): ''' classdocs ''' def __init__(self): ''' Constructor ''' self.data = [] from datasource import DataSource self.wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') def fromfile(self, filename=None): import pyfits if filename is None: filename = '/work2/jwe/m48/data/2MASS.fits' hdulist = pyfits.open(filename) self.data = hdulist[1].data hdulist.close() def todatabase(self): for d in self.data: print d['starid'], d['ra_cone'], d['dec_cone'] self.wifsip.execute("""UPDATE m48stars SET ra = %f, dec = %f WHERE starid = '%s';""" % (d['ra_cone'], d['dec_cone'], d['starid']), commit=False) self.wifsip.commit()
class Yadav(object): ''' classdocs ''' def __init__(self): ''' Constructor ''' self.data = [] from datasource import DataSource self.wifsip = DataSource(database=config.dbname, user=config.dbuser, host=config.dbhost) def fromfile(self, filename=None): hdulist = pyfits.open(filename) self.data = hdulist[1].data self.keys = [] for col in hdulist[1].columns: self.keys.append(col.name) hdulist.close() def todatabase(self): for d in self.data: print(d['seq']) record = {} for key in self.keys: record[key] = d[key] if np.isnan(record[key]): record[key] = 'NULL' query = """INSERT INTO m67 (seq, ra, dec, vmag, bmag, icmag, pmra, pmdec, pmb, hrv) VALUES (%(Seq)d, %(RAJ2000)f, %(DEJ2000)f, %(Vmag)s, %(Bmag)s, %(Icmag)s, %(pmRA)f, %(pmDE)f, %(Pmb)f, %(HRV)s);""" % record self.wifsip.execute(query, commit=False) self.wifsip.commit()
def make_cmd(): import pylab as plt import numpy as np from datasource import DataSource wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') query = "UPDATE m48stars SET bv = (bmag-vmag)+(0.761837-0.060177);" wifsip.execute(query) query = "SELECT vmag+0.060177, bv FROM m48stars WHERE NOT bv is NULL and nv>1 and nb>1;" data = wifsip.query(query) wifsip.close() vmag = np.array([d[0] for d in data]) bv = np.array([d[1] for d in data]) plt.scatter(bv, vmag, edgecolor='none', alpha=0.75, s=4, c='k') plt.ylim(22.0, 7.0) plt.xlim(-0.2, 2.0) plt.xlabel('B - V') plt.ylabel('V [mag]') plt.title('M 48') plt.grid() plt.savefig('/work2/jwe/m48/m48cmd.pdf') #plt.show() plt.close()
class StackedPhot(object): ''' classdocs ''' def __init__(self): ''' Constructor ''' self.data = [] from datasource import DataSource self.wifsip = DataSource(database=config.dbname, user=config.dbuser, host=config.dbhost) def fromfile(self, filename=None): import pyfits hdulist = pyfits.open(filename) self.data = hdulist[1].data self.keys = [] for col in hdulist[1].columns: self.keys.append(col.name) hdulist.close() def todatabase(self): for d in self.data: print d['starid'] record = {} for key in self.keys: record[key] = d[key] query = """UPDATE ngc2236 SET vmag = %(VMAG)f, bmag = %(BMAG)f, bv = %(BMAG)f-%(VMAG)f, nv = %(V_FLAGS)d, nb = %(B_FLAGS)d, vmag_err = %(VMAG_ERR)f, bmag_err = %(BMAG_ERR)f, member=TRUE WHERE starid = '%(starid)s';""" % record self.wifsip.execute(query, commit=False) self.wifsip.commit()
def calibrate(self): # B: m=0.9777689; c=1.7573974; r=0.9991517 # V: m=0.9545878; c=1.004512; r=0.9996585 from datasource import DataSource wifsip = DataSource(host='pina', database='wifsip', user='******') if filter == 'B': query = """UPDATE ngc1647stars SET Bmag = Bmag*0.9777689+1.7573974;""" elif filter == 'V': query = """UPDATE ngc1647stars SET Vmag = Vmag*0.9545878+1.004512;""" wifsip.execute(query) wifsip.execute('UPDATE ngc1647stars SET bv = Bmag-Vmag;') wifsip.close()
def store(self, star, period=None, theta=None): from datasource import DataSource wifsip = DataSource(host='pina', database='wifsip', user='******') query = "UPDATE ngc1647stars" if not period is None: query += " SET period=%f" % period if not theta is None: query += ", theta=%f" % theta query += " WHERE id like '%s';" % star try: wifsip.execute(query) finally: wifsip.close()
def update_db(self): from datasource import DataSource wifsip = DataSource(host='pina', database='wifsip', user='******') query = """INSERT INTO ngc2281stars SELECT distinct matched.id FROM frames, phot, matched WHERE frames.object like 'NGC 2281 BVI %%' AND frames.filter like '%s' AND frames.objid = phot.objid AND (matched.objid,matched.star) = (phot.objid,phot.star) AND matched.id NOT IN (SELECT id FROM ngc2281stars);""" % self.filter wifsip.execute(query) wifsip.close()
def store(self, star, period=None, theta=None): from datasource import DataSource from numpy import isnan wifsip = DataSource(host='pina', database='wifsip', user='******') query = "UPDATE ngc2281stars" if not period is None: query += " SET period=%f" % period if not ((theta is None) or isnan(theta)): query += ", theta=%f" % theta query += " WHERE id like '%s';" % star try: wifsip.execute(query) wifsip.commit() finally: wifsip.close()
def create_db(self): query = """DROP TABLE IF EXISTS turner; CREATE TABLE turner( star varchar(12), rem varchar(2), vmag real, bv real, ub real, n integer, code integer, ra float, dec float, coord point, PRIMARY KEY (star,rem));""" from datasource import DataSource wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') wifsip.execute(query) wifsip.close()
class PhotDatabase(object): ''' classdocs ''' def __init__(self, filtercol): ''' Constructor ''' self.data = [] from datasource import DataSource self.wifsip = DataSource(database=config.dbname, user=config.dbuser, host=config.dbhost, dictcursor=True) if not filtercol in ['u', 'v', 'b', 'y', 'hbn', 'hbw', 'I']: raise (ValueError) self.__filtercol = filtercol print self.__filtercol def fromfile(self, filename=None): import pyfits hdulist = pyfits.open(filename) self.data = hdulist[1].data self.keys = [] for col in hdulist[1].columns: self.keys.append(col.name) hdulist.close() def todatabase(self): for d in self.data: record = {} for key in self.keys: record[key] = d[key] record['filter'] = self.__filtercol if self.__filtercol == 'I': record['filter'] = 'imag' query = """UPDATE ngc2236 SET %(filter)s = %(MAG_ISOCOR)f, %(filter)s_err = %(MAGERR_ISO)f WHERE circle(coord,0) <@ circle(point(%(ALPHAWIN_J2000).11f,%(DELTAWIN_J2000).11f), 0.6/3600.0);""" % record self.wifsip.execute(query, commit=False) self.wifsip.commit()
def update(self, filename): '''update the V and B-V according to Pesch 1961''' from datasource import DataSource f = open(filename, 'r') lines = f.readlines() f.close() lines = lines[1:] wifsip = DataSource(host = 'pina', database = 'wifsip', user = '******') for l in lines: ls = l.split() star = ls[2] v = ls[3] bv = ls[4] ub = ls[5] n = ls[6] code = ls[7] print star,v,bv,ub,n,code query = "UPDATE ngc2281ref SET vmag=%s, bv=%s WHERE id like '%s'" % (v,bv,star) wifsip.execute(query) wifsip.close()
def create_table(self): ''' makes the database table ''' from datasource import DataSource stellads = DataSource(database=config.dbname, user=config.dbuser, host=config.dbhost) query = """CREATE TABLE ngc6633gab ( id integer, ra_j2000 double precision, dec_j2000 double precision, bmag real, s_bmag real, vmag real, s_vmag real, imag real, s_imag real, coord point, PRIMARY KEY (id) ); CREATE INDEX idx_ngc6633gab_coord ON ngc6633 USING GIST (circle(coord,0));""" stellads.execute(query) stellads.close()
def main(argv): target_url = '' COMMAND = '' ALL = False AUTH = False DATASOURCE = False TERM = False COURSE = False USER = False MEMBERSHIP = False CLEANUP = False datasource_PK1 = None datasource_session = None usageStr = "\nrestdemo.py -t|--target <target root URL> -c|--command <command>\n" usageStr += "e.g restdemo.py -t www.myschool.edu -c create_course\n" usageStr += "command: <command>_<object> where <command> is one of the following:\n" usageStr += "\tcreate, read, read_all_, read_all_course_memberships, read_all_user_memberships, update, delete\n" usageStr += "and <object> is one of the following:\n" usageStr += "\tdatasource, term, course, user\n" usageStr += "-t is required; No -c args will run demo in predetermined order.\n" usageStr += "'-c authorize' demomonstrates the authorization process and does not create objects." usageStr += "-c commands require a valid datasource PK1 - \n" usageStr += "\ta datasource get will be run in these cases, defaulting to create\n" usageStr += "\tif the demo datasource does not exist." if len(sys.argv) > 1: #there are command line arguments try: opts, args = getopt.getopt(argv, "ht:c:", ["target=", "command="]) except getopt.GetoptError: print(usageStr) sys.exit(2) for opt, arg in opts: if opt == '-h': print(usageStr) sys.exit() elif opt == '-d': print("Deleting at end of run.") CLEANUP = True elif opt in ("-t", "--target"): target_url = arg.lstrip() elif opt in ("-c", "--command"): COMMAND = arg else: COMMAND = "Run All" print('[main] Target is:', target_url) print('[main] Command is:', COMMAND) else: print(usageStr) sys.exit(2) #Set up some booleans for processing flags and order of processing if "course" in COMMAND: print("[main] Run course command") COURSE = True elif "membership" in COMMAND: print("[main] Run membership command") MEMBERSHIP = True elif "user" in COMMAND: print("[main] Run user command") USER = True elif "term" in COMMAND: print("[main] Run term command") TERM = True elif "datasource" in COMMAND: print("[main] Run datasource command") DATASOURCE = True elif "authorize" in COMMAND: print("[main] Run authorization command") AUTH = True else: print("[main] Empty Command: Run All\n") ALL = True print('\n[main] Acquiring auth token...\n') authorized_session = AuthToken(target_url) authorized_session.setToken() print('\n[main] Returned token: ' + authorized_session.getToken() + '\n') if not AUTH: #run commands in required order if running ALL if DATASOURCE or ALL: #process Datasource command print("\n[main] Run datasource command: " + ('ALL' if ALL else COMMAND) + '...') datasource_session = DataSource(target_url, authorized_session.getToken()) if 'datasource' in COMMAND: datasource_session.execute(COMMAND, authorized_session.getToken()) else: if not datasource_PK1 or datasource_PK1 is None: datasource_session.createDataSource( authorized_session.getToken()) datasource_session.getDataSource( authorized_session.getToken()) datasource_session.getDataSources( authorized_session.getToken()) datasource_session.updateDataSource( authorized_session.getToken()) if TERM or ALL: term_session = Term(target_url, authorized_session.getToken()) #process term command print("\n[main] Run term command: " + ('ALL' if ALL else COMMAND) + '...') if 'term' in COMMAND: if (('delete' in COMMAND) or ('read' in COMMAND)): print( "[main] Deleting or getting does not require a datasource." ) else: if not datasource_PK1: print("[main] confirm datasource.") datasource_session = DataSource( target_url, authorized_session.getToken()) datasource_session.checkDataSource( authorized_session.getToken()) term_session.execute(COMMAND, "externalId:" + TERMEXTERNALID, authorized_session.getToken()) else: term_session.getTerms(authorized_session.getToken()) term_session.createTerm("externalId:" + DSKEXTERNALID, authorized_session.getToken()) term_session.getTerm(authorized_session.getToken()) term_session.updateTerm("externalId:" + DSKEXTERNALID, authorized_session.getToken()) if COURSE or ALL: course_session = Course(target_url, authorized_session.getToken()) #process course command print("\n[main] Run course command: " + ('ALL' if ALL else COMMAND) + '...') if 'course' in COMMAND: if (('delete' in COMMAND) or ('read' in COMMAND)): print( "[main] Deleting or getting does not require a datasource." ) else: if not datasource_PK1: print("[main] confirm datasource.") datasource_session = DataSource( target_url, authorized_session.getToken()) datasource_session.checkDataSource( authorized_session.getToken()) course_session.execute(COMMAND, "externalId:" + DSKEXTERNALID, authorized_session.getToken()) else: course_session.getCourses(authorized_session.getToken()) course_session.createCourse("externalId:" + DSKEXTERNALID, authorized_session.getToken()) course_session.getCourse(authorized_session.getToken()) course_session.updateCourse("externalId:" + DSKEXTERNALID, authorized_session.getToken()) if USER or ALL: user_session = User(target_url, authorized_session.getToken()) #process user command print("\n[main] Run user command: " + ('ALL' if ALL else COMMAND) + '...') if 'user' in COMMAND: if (('delete' in COMMAND) or ('read' in COMMAND)): print( "[main] Deleting or getting does not require a datasource." ) else: if not datasource_PK1: print("[main] confirm datasource.") datasource_session = DataSource( target_url, authorized_session.getToken()) datasource_session.checkDataSource( authorized_session.getToken()) user_session.execute(COMMAND, "externalId:" + DSKEXTERNALID, authorized_session.getToken()) else: user_session.getUsers(authorized_session.getToken()) user_session.createUser("externalId:" + DSKEXTERNALID, authorized_session.getToken()) user_session.getUser(authorized_session.getToken()) user_session.updateUser("externalId:" + DSKEXTERNALID, authorized_session.getToken()) if MEMBERSHIP or ALL: membership_session = Membership(target_url, authorized_session.getToken()) #process membership command print("\n[main] Run membership command: " + ('ALL' if ALL else COMMAND) + '...') if 'membership' in COMMAND: if (('delete' in COMMAND) or ('read' in COMMAND)): print( "[main] Deleting or getting does not require a datasource." ) else: if not datasource_PK1: print("[main] confirm datasource.") datasource_session = DataSource( target_url, authorized_session.getToken()) datasource_session.checkDataSource( authorized_session.getToken()) membership_session.execute(COMMAND, "externalId:" + DSKEXTERNALID, authorized_session.getToken()) else: membership_session.getCourseMemberships( authorized_session.getToken()) membership_session.createMembership( "externalId:" + DSKEXTERNALID, authorized_session.getToken()) membership_session.getUserMemberships( authorized_session.getToken()) membership_session.updateMembership( "externalId:" + DSKEXTERNALID, authorized_session.getToken()) membership_session.readUserMemberships( authorized_session.getToken()) #clean up if not using individual commands if ALL: print('\n[main] Completing Demo and deleting created objects...') print("[main] Deleting membership") membership_session.deleteMembership(authorized_session.getToken()) print("[main] Deleting Course") user_session.deleteUser(authorized_session.getToken()) print("[main] Deleting Course") course_session.deleteCourse(authorized_session.getToken()) print("[main] Deleting Term") term_session.deleteTerm(authorized_session.getToken()) print("[main] Deleting DataSource") datasource_session.deleteDataSource(authorized_session.getToken()) else: print("\nRemember to delete created demo objects!") print("[main] Processing Complete")
class Ngc2236(object): def __init__(self): """Constructor""" from cluster import Cluster from astronomy import mag_distance from datasource import DataSource self.wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') self.stars = [] c = Cluster('NGC 2236') self.age = 10**c['logage']/1e6 # in Myr self.ebv = c['ebv'] self.dm = mag_distance(c['d'])- 4.83 def clearperiods(self): """ reset the periods in the database table """ if not raw_input('press Y to erase the periods in table ')=='Y': return query="""UPDATE ngc2236 SET period=NULL, period_err=NULL, amp=NULL, amp_err=NULL WHERE period>0; """ logger.info('resetting periods ...') self.wifsip.execute(query) def getstars(self): """ build up a list of stars, where we do not have periods yet """ query = """SELECT starid FROM ngc2236 WHERE NOT corotid IS NULL;""" logger.info('fetching stars ...') result = self.wifsip.query(query) logger.info('... %d stars found' % len(result)) print '... %d stars found' % len(result) self.stars = [s[0] for s in result] def analysis(self, show=False): """perform a PDM analysis on each lightcurve""" from matplotlib import rcParams from functions import sigma_clip, phase print 'Analysis' fig_width = 18.3/2.54 # width in inches, was 7.48in fig_height = 23.3/2.54 # height in inches, was 25.5 fig_size = [fig_width,fig_height] #set plot attributes params = {'backend': 'Agg', 'axes.labelsize': 12, 'axes.titlesize': 12, 'font.size': 12, 'xtick.labelsize': 12, 'ytick.labelsize': 12, 'figure.figsize': fig_size, 'savefig.dpi' : 300, 'font.family': 'sans-serif', 'axes.linewidth' : 0.5, 'xtick.major.size' : 2, 'ytick.major.size' : 2, } rcParams.update(params) for starid in self.stars: star = NGC2236Star(starid) print '%-24s '% starid, lc = star.lightcurve() meanflux = np.nanmean(lc.flux) lc.flux /= meanflux lc.flux *= meanflux lc.rebin(0.0125) lc.interpolate() lc.normalize() raw_time, raw_flux = (lc.time, lc.flux) lc.medfilter() lc.jmpflt() lc.detrend() time, flux = (lc.time, lc.flux) time -= min(time) # convert to magnitudes mag = -2.512*np.log10(meanflux*flux) + 24 # remove the mean so we are around 0.0 magnitudes mag -= np.mean(mag) # perform a 3sigma clipping time, mag = sigma_clip(time, mag) # calculate fourier spectrum with zero padding n = len(mag) n2 = 8*n ft = np.fft.rfft(mag, n2) #amp = abs(ft)/n power = abs(ft)**2/n freq = np.fft.fftfreq(n2, d=lc.dt()) # apply a bessel filter to eliminate runlength artifacts from scipy import signal b, a = signal.butter(4, 2./max(time), 'high', analog=True) _, h = signal.freqs(b, a, worN=freq[1:n]) filt_pwr = abs(ft[1:n]*h)**2/n i = np.argmax(filt_pwr)+1 maxfreq = freq[i] period = 1./maxfreq from pdm import pdm #refine period using pdm pdm_periods, pdm_thetas = pdm(time, mag, period/1.5, period*1.5, 0.0125) i = np.argmin(pdm_thetas) period = pdm_periods[i] print 'P = %5.2f' % period, star['period'] = period periods = 1./freq[1:n] periods = periods[::-1] filt_pwr = filt_pwr[::-1] norm = np.mean(filt_pwr) print '%.1f' % (max(filt_pwr)/norm) power[1:n] /= norm filt_pwr /= norm star['amp'] = max(filt_pwr)/norm ph_time, ph_mag = phase(time, mag, period) num = len(ph_time)/round(max(time)/period) rph_mag, _ = signal.resample(ph_mag, num, t = ph_time) rph_time = np.linspace(min(ph_time),max(ph_time),num) bv = star['bv'] if bv is None: bv=-99 plt.subplot(411) ################################################## plt.title('%s (%d) B-V=%.2f' % (starid, star['corotid'], bv)) plt.scatter(raw_time-min(raw_time), raw_flux, edgecolor='none', alpha=0.5, s=3, color='k') plt.ylim(min(raw_flux),max(raw_flux)) plt.xlim(0,max(time)) plt.subplot(412) ################################################## plt.plot(time, -mag, 'k') plt.ylim(min(-mag),max(-mag)) plt.xlim(0,max(time)) plt.subplot(413) ################################################## plt.plot(1./freq[1:n], power[1:n], 'k') plt.plot(1./freq[1:n], filt_pwr, 'g') #plt.plot(1./w, 20 * np.log10(abs(h))) plt.axvline(period) #plt.axhline(np.mean(filt_pwr[1:n])) plt.xlim(0.1, max(time)) plt.subplot(414) ################################################## plt.plot(rph_time, -rph_mag,color='k') plt.plot(rph_time+period, -rph_mag,color='k') #plt.plot(phased_time, phased_mag, 'g') #plt.plot(phased_time+period, phased_mag, 'g') plt.axvline(period, linestyle='--') plt.xlabel('P = %.2f' % period) plt.xlim(0., period*2) #plt.grid() if show: plt.show() else: plt.savefig(config.plotpath+'%s(%d).pdf' % (starid,star['corotid'])) plt.close() def set_tab_column(self): self.wifsip.execute('UPDATE ngc2236 set tab=NULL;') query = """SELECT starid FROM ngc2236 WHERE NOT bv IS NULL ORDER BY vmag;""" result = self.wifsip.query(query) starids = [r[0] for r in result] for starid in starids: tab = starids.index(starid)+1 print '%4d %s' % (tab,starid) query = "UPDATE ngc2236 set tab=%d WHERE starid='%s';" % (tab,starid) self.wifsip.execute(query) def update_coordinates(self): from datasource import DataSource corot = DataSource(database='corot', user='******', host='pina.aip.de') query = """SELECT corotid FROM ngc2236 WHERE NOT corotid IS NULL ORDER BY vmag;""" corotids = [c[0] for c in self.wifsip.query(query)] for corotid in corotids: query = """SELECT alpha, delta FROM corot WHERE corotid = %d;""" % corotid ra, dec = corot.query(query)[0] print corotid,ra,dec record = dict(zip(['corotid','ra','dec'], [corotid,ra,dec])) query = """UPDATE ngc2236 SET ra = %(ra)f, dec = %(dec)f, coord = point(%(ra)f,%(dec)f) WHERE corotid = %(corotid)d;""" % record self.wifsip.execute(query, commit=False) self.wifsip.commit()
class Stetson(object): ''' classdocs ''' def __init__(self): ''' Constructor ''' self.data = [] from datasource import DataSource self.wifsip = DataSource(database='stella', user='******', host='pera.aip.de') def fromfile(self, filename=None): import pyfits hdulist = pyfits.open(filename) self.data = hdulist[1].data self.keys = [] for col in hdulist[1].columns: self.keys.append(col.name) hdulist.close() print self.keys def todatabase(self): for d in self.data: print d['ID_1'] query = """INSERT INTO ngc6633ref (starid, ra, dec, dx, dy, x, y, bmag, bsigma, nb, vmag, vsigma, nv, imag, isigma, ni, coord) VALUES ('%(ID_1)s', %(RA)f, %(DEC)f, %(dX)f, %(dY)f, %(X)f, %(Y)f, %(B)f,%(sigmaB)f,%(NB)f,%(V)f,%(sigmaV)f,%(NV)f,%(I)f,%(sigmaI)f,%(NI)f, point(%(RA)f,%(DEC)f));""" % d self.wifsip.execute(query, commit=False) self.wifsip.commit() def readpos(self, filename='NGC6633.pos'): posfile = open(filename, 'rt') data = np.genfromtxt(posfile, dtype=None, names=('RA', 'DEC', 'HH', 'MM', 'SS', 'DD', 'DM', 'DS', 'dX', 'dY', 'X', 'Y', 'ID')) for d in data: #print ">%(ID)s<" % d query = """UPDATE ngc6633ref SET ra=%(RA).11f,dec=%(DEC).11f, coord=point(%(RA).11f,%(DEC).11f) WHERE starid='%(ID)s';""" % d print query self.wifsip.execute(query) posfile.close() def setucac4(self, filename='Stetson_UCAC4.fit'): import pyfits hdulist = pyfits.open(filename) ucac = hdulist[1].data hdulist.close() print ucac.columns print ucac['ucacid'] #clear existing coordinates self.wifsip.execute('UPDATE ngc6633ref SET coord=NULL;') ra = ucac['raj2000'] dec = ucac['dej2000'] x = ucac['X'] y = ucac['Y'] A = np.vstack([x, y, np.ones(len(x))]).T wcsra = np.linalg.lstsq(A, ra)[0] wcsdec = np.linalg.lstsq(A, dec)[0] print wcsra print wcsdec dx1, dy1, cx = wcsra dx2, dy2, cy = wcsdec param = { 'dx1': dx1, 'dy1': dy1, 'cx': cx, 'dx2': dx2, 'dy2': dy2, 'cy': cy } query = """UPDATE ngc6633ref SET coord=point(X*%(dx1).11g+Y*%(dy1).11g+%(cx).11f, X*%(dx2).11g+Y*%(dy2).11g+%(cy).11f);""" % param print query self.wifsip.execute(query) def calibrate(self): from tools import log ffield = {'V': 'vmag', 'B': 'bmag', 'I': 'imag'} for filtercol in ['V', 'B', 'I']: query = """SELECT objid FROM frames WHERE object LIKE 'NGC 6633 BVI %%' AND filter = '%s';""" % filtercol frames = self.wifsip.query(query) for frame in frames: params = {'objid': frame[0], 'filterfield': ffield[filtercol]} query = """SELECT starid, mag_isocor, magerr_isocor, %(filterfield)s, mag_isocor - %(filterfield)s FROM phot, ngc6633ref WHERE objid = '%(objid)s' AND circle(phot.coord,0) <@ circle(ngc6633ref.coord, 1.0/3600.0) AND NOT %(filterfield)s IS NULL AND flags<4;""" % params result = self.wifsip.query(query) if len(result) > 0: ocs = np.array([r[4] for r in result]) errs = np.array([r[2] for r in result]) weights = 1 / errs std = np.std(ocs) i = np.where(abs(ocs - np.mean(ocs)) < std)[0] #for r in result: # log(config.logfile, '%-11s %.4f %.4f %.3f %6.3f' % r) corr = 0.0 if len(i) > 4: corr = np.average(ocs[i], weights=weights[i]) std = np.std(ocs[i]) log(config.logfile, '%-19s %1s %-7.4f %.3f %3d %3d' % \ (frame[0], filtercol, corr, std, len(i), len(ocs))) params['corr'] = corr params['good'] = 'NULL' if std < 0.05: params['good'] = 'TRUE' else: params['good'] = 'FALSE' query = """UPDATE frames SET corr=%(corr)f, good=%(good)s WHERE objid ='%(objid)s';""" % params else: query = """UPDATE frames SET corr=NULL, good=FALSE WHERE objid ='%(objid)s';""" % params self.wifsip.execute(query) def calibratebv(self): '''drop view ngc6633match; create view ngc6633match as SELECT ngc6633ref.starid, ngc6633.vmag, ngc6633.bmag, ngc6633ref.vmag "vmag_ref", ngc6633ref.bmag "bmag_ref" FROM ngc6633, ngc6633ref WHERE circle(ngc6633.coord,0) <@ circle(ngc6633ref.coord, 1.0/3600.0);''' from dbtable import DBTable match = DBTable( self.wifsip, 'ngc6633match', condition= 'NOT vmag_ref IS NULL AND NOT bmag_ref IS NULL AND vmag>12') vmag = match['vmag'] bmag = match['bmag'] bv = bmag - vmag vmagref = match['vmag_ref'] bmagref = match['bmag_ref'] bvref = bmagref - vmagref A = np.vstack([vmag, bv, np.ones(len(vmag))]).T vcorr = np.linalg.lstsq(A, vmagref)[0] B = np.vstack([bmag, bv, np.ones(len(bmag))]).T bcorr = np.linalg.lstsq(B, bmagref)[0] print vcorr print bcorr
class Photometry(object): ''' classdocs ''' def __init__(self, filtercol='V'): ''' Constructor ''' from datasource import DataSource self.wifsip = DataSource(database='wifsip', host='pina', user='******') if filtercol in ('B', 'V'): self.filtercol = filtercol else: raise (ValueError) self.frames = [] print 'filter %s' % self.filtercol def createtable(self): ''' create table for the photometry ''' if not raw_input('press Y to erase m48stars') == 'Y': return query = """ DROP TABLE IF EXISTS m48stars; CREATE TABLE m48stars( starid varchar(25), bv real, vmag real default 0, vmag_err real, bmag real default 0, bmag_err real, period real, period_err real, theta real, amp real, amp_err real, nv integer default 0, nb integer default 0, ra double precision, dec double precision, coord point, PRIMARY KEY (starid)); GRANT SELECT ON m48stars TO public; CREATE INDEX idx_m48stars_coords ON m48stars USING GIST (circle(coord,1.0/3600.)); """ self.wifsip.execute(query) print "table 'm48stars' created" def cleartable(self): if not raw_input('press Y to clear m48stars') == 'Y': return query = """ UPDATE m48stars SET vmag = 0, vmag_err = NULL, bmag = 0, bmag_err =NULL, nv = 0, nb = 0, bv = NULL; """ self.wifsip.execute(query) print "table 'm48stars' cleared" def getframes(self): for field in ['C', 'NW', 'NE', 'SW', 'SE']: query = """SELECT object, objid, abs(corr) FROM frames WHERE object LIKE 'M 48 BVI %s' AND filter LIKE '%s' AND NOT corr IS NULL ORDER BY abs(corr) limit 5;""" % (field, self.filtercol) result = self.wifsip.query(query) if len(result) == 0: print 'no frames found!' for r in result: print '%s\t%s\t%.3f: ' % r objid = r[1] self.filltable(objid) self.frames.append(objid) self.wifsip.dropview('phot1') self.update_magnitudes() #print '\n'.join(self.frames) def filltable(self, objid): #get the stars from the phot table ... query = """ SELECT phot.objid ||'#'|| star, mag_auto-corr, alphawin_j2000 , deltawin_j2000 FROM phot,frames WHERE frames.objid='%s' AND phot.objid=frames.objid AND flags<8;""" % (objid) result = self.wifsip.query(query) #... and inject them into the m48stars stars = len(result) if stars > 400: print '%5d stars: ' % stars, oldstars = 0 newstars = 0 for r in result: ostars = self.addstar(r[0], r[1], r[2], r[3]) if ostars == 0: newstars += 1 else: oldstars += 1 print '%5d old , %5d new' % (oldstars, newstars) self.wifsip.commit() else: print 'not enough stars (%d)' % stars #value = '%s\t%f\t%f\t%f' % r #print value def addstar(self, starid, mag, ra, dec): # identify by coordinates, if the star is already in table query = """SELECT starid FROM m48stars WHERE circle(point(%f,%f),0)<@circle(coord,1.0/3600.) ORDER BY point(%f,%f)<->coord LIMIT 1;""" % (ra, dec, ra, dec) result = self.wifsip.query(query) oldstar = 0 # if not: append new star if len(result) == 0: oldstar = 0 if self.filtercol == 'B': query = """INSERT INTO m48stars (starid, bmag, nb, ra, dec, coord) VALUES ('%s', %f, 1, %f, %f, point(%f,%f))""" % ( starid, mag, ra, dec, ra, dec) elif self.filtercol == 'V': query = """INSERT INTO m48stars (starid, vmag, nv, ra, dec, coord) VALUES ('%s', %f, 1, %f, %f, point(%f,%f))""" % ( starid, mag, ra, dec, ra, dec) # if star exists: add up magnitudes, increase counter else: oldstar = 1 oldid = result[0][0] if self.filtercol == 'B': query = """UPDATE m48stars SET bmag = bmag + %f, nb = nb + 1 WHERE starid = '%s'; """ % (mag, oldid) elif self.filtercol == 'V': query = """UPDATE m48stars SET vmag = vmag + %f, nv = nv + 1 WHERE starid = '%s'; """ % (mag, oldid) self.wifsip.execute(query) return oldstar def update_magnitudes(self): if self.filtercol == 'B': query = """UPDATE m48stars SET bmag=bmag/nb WHERE nb>1; UPDATE m48stars SET bmag=NULL WHERE nb=0;""" elif self.filtercol == 'V': query = """ UPDATE m48stars SET vmag=vmag/nv WHERE nv>1; UPDATE m48stars SET vmag=NULL WHERE nv=0;""" self.wifsip.execute(query) def update_sigmas(self): import numpy as np if self.filtercol == 'V': field = 'vmag' elif self.filtercol == 'B': field = 'bmag' query = """SELECT starid, coord FROM m48stars WHERE (NOT bv IS NULL) AND (%s_err IS NULL);""" % (field) starlist = self.wifsip.query(query) for star in starlist: print '%5d ' % starlist.index(star), print '%-24s: %-25s' % star, query = """SELECT phot.objid, mag_auto-corr FROM phot, frames WHERE object like 'M 48 BVI %%' AND phot.objid=frames.objid AND filter='%s' AND flags<8 AND point%s <@ circle(phot.coord,1./3600.) ORDER BY abs(corr) LIMIT 5;""" % (self.filtercol, star[1]) result = self.wifsip.query(query) mags = np.array([r[1] for r in result]) try: err = np.std(mags) print mags, print '%.3f %.4f' % (np.mean(mags), err) if np.isfinite(err): query = "UPDATE m48stars SET %s_err=%f WHERE starid='%s';" % ( field, err, star[0]) self.wifsip.execute(query) except TypeError: print 'no data' def update_bv(self): query = "UPDATE m48stars SET bv = bmag-vmag;" self.wifsip.execute(query)
class Clusters(object): ''' classdocs ''' def __init__(self): ''' Constructor ''' self.table = DataSource(database=_config.dbname, user=_config.dbuser, host=_config.dbhost) def create_table(self): query = """DROP TABLE clusters; CREATE TABLE clusters ( name varchar(18), ra double precision, dec double precision, class varchar(3), diam real, d int, ebv real, logage real, pmra real, epmra real, pmdec real, epmdec real, nc int, ref1 varchar(3), rv real, erv real, n int, ref2 varchar(4), me real, eme real, nme real, trtyp varchar(8), coord point, PRIMARY KEY (name));""" self.table.execute(query) def fromfile(self, filename='/work2/jwe/SOCS/clusters.txt'): f = open(filename, 'rt') lines = f.readlines() f.close() def nfloat(s): from numpy import nan try: result = float(s) except ValueError: return nan return result def nint(s): from numpy import nan try: result = int(s) except ValueError: return nan return result cur = self.table.cursor values = '' for l in lines: coords = l[18:37] c = SkyCoord(coords, 'icrs', unit=(u.hourangle, u.deg)) # @UndefinedVariable record = {'name': l[0:18].rstrip(), 'ra': c.ra.degree, 'dec': c.dec.degree, 'class': l[40:43].rstrip(), 'diam': nfloat(l[45:52]), 'd': nint(l[55:60]), 'ebv': nfloat(l[65:70]), 'logage': nfloat(l[73:79]), 'pmra': nfloat(l[84:90]), 'epmra': nfloat(l[92:96]), 'pmdec': nfloat(l[100:106]), 'epmdec': nfloat(l[108:112]), 'nc': nint(l[113:118]), 'ref1': l[119:123].rstrip(), 'rv': nfloat(l[127:134]), 'erv': nfloat(l[138:143]), 'n': nint(l[147:150]), 'ref2': nint(l[155:159]), 'me': nfloat(l[162:168]), 'eme': nfloat(l[171:176]), 'nme': nint(l[177:180]), 'trtyp': l[183:191].rstrip(), 'coord': '(%f,%f)' % (c.ra.degree, c.dec.degree)} # print record def nstr(s): if len(str(s).strip()) == 0: return '\\N' elif type(s) is str: return str(s) else: return str(s) valline = '\t'.join([nstr(v) for v in record.values()]) valline = valline.replace('nan', '\\N') print(valline) values += valline + '\n' columns = record.keys() f = StringIO.StringIO(values) try: cur.copy_from(f, 'clusters', columns=columns) finally: self.table.commit() def query(self): query = """SELECT name, ra, dec, ebv, diam from clusters WHERE (name like 'NGC %' or name like 'IC %') AND diam>10 AND diam<60 AND d<1500 AND dec>-15.0 AND ebv<0.3 AND logage>8.0 AND logage<=9.5 AND abs(rv)>2.0;""" result = self.table.query(query) names = [r[0] for r in result] ra = np.array([r[1] for r in result]) dec = np.array([r[2] for r in result]) ebv = np.array([r[3] for r in result]) diam = np.array([r[4] for r in result]) mycmap = plt.cm.get_cmap('Reds') # mycmap.set_under('w')fig, ax_f = plt.subplots() _, ax1 = plt.subplots(figsize=(10, 7)) plt.scatter(ra / 15., dec, s=diam * 4, cmap=mycmap, c=ebv) for rai, deci, iname in zip(ra, dec, names): if iname in ['IC 4756', 'NGC 2682', 'NGC 2319', 'NGC 2374', 'NGC 7209', 'NGC 7243', 'NGC 7082', 'NGC 225']: horizontalalignment = 'right' withdash = None dashlength = None elif iname in ['NGC 2413']: horizontalalignment = 'right' withdash = True dashlength = 20.0 else: horizontalalignment = 'left' withdash = None dashlength = None if withdash: plt.text(rai / 15., deci, iname, withdash=withdash, dashlength=dashlength, horizontalalignment=horizontalalignment) else: plt.text(rai / 15., deci, iname, horizontalalignment=horizontalalignment) # plt.draw() ax2 = ax1.twiny() print(np.arange(13, 0, -1)) ax1.set_xlabel('right ascension') ax1.set_ylabel('declination') ax2.set_xlabel('culmination month') ax1.set_xticks(np.arange(24)) ax1.set_yticks(np.arange(-15, 65, 5)) ax2.set_xticks(np.arange(0, 13) + 0.4333) ax2.set_xticklabels(['9', '8', '7', '6', '5', '4', '3', '2', '1', '12', '11', '10']) ax1.set_xlim(24, 0) # ax2.set_xlim(12, 0) ax1.set_ylim(-15, 65) ax1.grid() plt.minorticks_on() cbar = plt.colorbar() # cbar.ax.set_yticklabels(['0','1','2','>3']) cbar.set_label('E(B - V)', rotation=270) plt.savefig('/work2/jwe/SOCS/plots/cluster_query.pdf') plt.close() print(len(names)) print('\n'.join(names))
class CDSTable(object): ''' classdocs ''' def __init__(self): ''' Constructor ''' self.datastruct = [] from datasource import DataSource self.wifsip = DataSource(database=config.dbname, user=config.dbuser, host=config.dbhost, dictcursor=True) def savetable2(self): query = """SELECT tab, vmag, bv, p_fin, e_pfin, amp, amp_err, member, simbad, provisional FROM m48stars WHERE good ORDER BY vmag ;""" self.data = self.wifsip.query(query) #Id Vmag B-V P P_er amp amp_e M P Simbad # 1 2 3 4 #1234567890123456789012345678901234567890123456789012 # 287 13.597 0.56 2.88 0.10 0.018 0.004 M p BJG 3157 f = open(config.resultpath + 'table2.dat', 'wt') for d in self.data: #i = data.index(d)+1 simbad = '' if type(d['simbad']) is str and d['simbad'].find( 'Cl* NGC 2548 ') == 0: simbad = d['simbad'][13:] if str('simbad') == 'None': simbad = '' memstr = '-' if d['member']: memstr = 'M' elif d['member'] == False: memstr = 'N' prostr = '-' if d['provisional']: prostr = 'p' elif not d['provisional']: prostr = 'c' params = d.copy() params['simbad'] = simbad params['memstr'] = memstr params['prostr'] = prostr s = '%(tab)4d %(vmag)6.3f %(bv)4.2f %(p_fin)5.2f %(e_pfin).2f %(amp).3f %(amp_err).3f %(memstr)s %(prostr)s %(simbad)s\n' % params print s, f.write(s) f.close() def saveappendix(self): from numpy import sqrt from astropy import units as u from astropy.coordinates import SkyCoord #TODO: omit simbad column query = """SELECT tab, vmag, vmag_err, bv, bmag_err, coord[0] "ra", coord[1] "dec", member FROM m48stars WHERE not bv IS NULL ORDER BY vmag;""" data = self.wifsip.query(query) f = open(config.resultpath + 'table_a1.dat', 'wt') for d in data: memstr = '-' if d['member']: memstr = 'M' elif d['member'] == False: memstr = 'N' try: bv_err = sqrt(d['vmag_err']**2 + d['bmag_err']**2) except TypeError: bv_err = 0.0 c = SkyCoord(ra=d['ra'] * u.deg, dec=d['dec'] * u.deg, frame='icrs') # @UndefinedVariable ra_str = c.ra.to_string(unit=u.hourangle, sep=' ', pad=True, precision=2) # @UndefinedVariable dec_str = c.dec.to_string(sep=' ', precision=2, pad=True) posstr = c.to_string(precision=5) #if bmag_err is None: bmag_err=0.0 params = d.copy() params['memstr'] = memstr params['bv_err'] = bv_err params['ra_str'] = ra_str params['dec_str'] = dec_str params['posstr'] = posstr if d['vmag_err'] is None: params['vmag_err'] = 0.0 try: s = '%(tab)4d %(posstr)s %(vmag)6.3f %(vmag_err)5.3f %(bv)6.3f %(bv_err)5.3f %(ra_str)s %(dec_str)s %(memstr)s\n' % params print s, f.write(s) except TypeError: print d, len(s) f.close() def update_coords(self, filtercol='V'): import numpy as np from tools import log query = """SELECT starid, ra, dec FROM m48stars WHERE NOT bv IS NULL AND coord IS NULL ORDER BY tab;""" result = self.wifsip.query(query) stars = [] for starid, ra, dec in result: stars.append({'starid': starid, 'ra': ra, 'dec': dec}) for star in stars: #print star['NUMBER'],star['MAG_ISO'], params = { 'ra': star['ra'], 'dec': star['dec'], 'filtercol': filtercol } query = """SELECT alphawin_j2000, deltawin_j2000 FROM phot, frames WHERE object like 'M 48 %%' AND phot.objid = frames.objid AND filter = '%(filtercol)s' AND circle(phot.coord,0) <@ circle(point(%(ra).11f,%(dec).11f), 0.3/3600.0) AND flags<5;""" % params result = self.wifsip.query(query) #print result if len(result) == 0: print '%-25s: no data' % star['starid'] query = """UPDATE m48stars SET coord = NULL WHERE starid = '%s';""" % star['starid'] self.wifsip.execute(query) continue elif len(result) == 1: print '%-25s: not enough data' % star['starid'] query = """UPDATE m48stars SET coord = NULL WHERE starid = '%s';""" % star['starid'] self.wifsip.execute(query) continue elif len(result) > 1: ra = np.array([r[0] for r in result]) dec = np.array([r[1] for r in result]) mean_ra = np.mean(ra) mean_dec = np.mean(dec) std_ra = np.std(ra) std_dec = np.std(dec) # remove outliers std = np.sqrt(std_ra**2 + std_dec**2) dist = np.sqrt((ra - mean_ra)**2 + (dec - mean_dec)**2) i = np.where(dist < 2 * std) mean_ra = np.mean(ra[i]) mean_dec = np.mean(dec[i]) std_ra = np.std(ra[i]) std_dec = np.std(dec[i]) n = len(ra[i]) params['starid'] = star['starid'] params['ra'] = mean_ra params['dec'] = mean_dec params['n'] = n params['sra'] = std_ra * 3600.0 params['sdec'] = std_dec * 3600.0 query = """UPDATE m48stars SET coord = point(%(ra).11f,%(dec).11f) WHERE starid = '%(starid)s';""" % params #query = query.replace('nan', 'NULL') self.wifsip.execute(query) log( config.projectpath + 'coords1.log', '%(starid)-25s, %(ra).6f, %(dec).6f, %(sra).2f, %(sdec).2f, %(n)3d' % params) self.wifsip.commit()
class WHydra(object): ''' classdocs ''' def __init__(self, field_name = 'M67field'): ''' Constructor ''' self.field_name = field_name self.wifsip = DataSource(database=config.dbname, user=config.dbuser, host=config.dbhost) kpno = ephem.Observer() #31.958036,-111.600578 kpno.lon, kpno.lat = '-111.600578', '31.958036' kpno.horizon = '-0:34' kpno.elevation = 2096 tzi = pytz.timezone('MST') #fmt = '%Y-%m-%d %H:%M:%S %Z%z' obsdate = datetime.datetime(2015,2,10,4,0,0, tzinfo=tzi) kpno.date = obsdate + datetime.timedelta(7*ephem.hour) d = kpno.date.datetime() self.input_epoch = 2000 self.current_epoch = d.year + d.timetuple().tm_yday/365. # LST at mid-exposure in decimal hours lst = kpno.sidereal_time() print('local siderial time: ', lst) self.siderial_time = ast.hms2hh(str(lst)) self.exposure_length = 40.0/60.0 # in hours self.wavelength = 5125 # in Angstroem self.cable = 'BLUE' self.weighting = 'STRONG' self.guidewave = 6000 self.center = self._get_center() c = Cluster('NGC 2682') self.ebv = c['ebv'] self.dm = ast.distance_modulus(c['d']) target = {} self.table = [] target['id'] = 6000 target['name'] = 'M67center' target['ra'] = self.center[0] target['dec'] = self.center[1] target['class'] = 'C' self.table.append(target) self.targeted = [] # number of sky fibers self.skies = 6 # number of field orientation probes self.fops = 6 self.tweakcenter = False self.tweakangle = False print('center', self.center) print('E(B-V)', self.ebv) print('DM', self.dm) def _get_center(self): data = self.wifsip.query("""select avg(ra),avg(dec) from m67;""") return (data[0][0],data[0][1]) def priorities(self, verbose = False): """updates the priorities in the m67 table""" def makeplot(bv, v, p, filename=None): """plot the priorities""" import matplotlib matplotlib.use('Agg') from matplotlib import pyplot pyplot.scatter(bv, v, c=p, edgecolor='none', alpha=0.75) pyplot.xlim(0.4,1.0) pyplot.ylim(16.5,12.0) pyplot.title('M 67') pyplot.xlabel('B - V') pyplot.ylabel('V mag') if filename is None: pyplot.show() else: pyplot.savefig(filename, dpi=300) pyplot.close() def plotradec(ra, dec, p, filename=None): """plot the priorities""" import matplotlib matplotlib.use('Agg') from matplotlib import pyplot pyplot.scatter(ra, dec, c=p, edgecolor='none', alpha=0.75) pyplot.title('M 67') pyplot.xlabel('R.A.') pyplot.ylabel('Dec') if filename is None: pyplot.show() else: pyplot.savefig(filename, dpi=300) pyplot.close() print('calculate priorities ...') self.wifsip.execute("UPDATE m67 SET priority=NULL, pointing=NULL;") self.wifsip.execute("""UPDATE m67 SET priority=1.0 WHERE bmag-vmag>0.6 and bmag-vmag<1.0 AND vmag>13.5 and vmag<16 and pmb>0;""") data = self.wifsip.query("""SELECT seq, vmag, bmag-vmag, pmb FROM m67 WHERE bmag-vmag>0.6 AND bmag-vmag<0.75 AND vmag>13.5 and vmag<16 ORDER BY pmb DESC;""") seq = [d[0] for d in data] v = np.array([d[1] for d in data]) bv = np.array([d[2] for d in data]) pmb = np.array([d[3] for d in data]) p1 = scaleto(pmb, [0.0, 1.0]) print(len(seq), 'stars brighter V<16') for i in range(len(seq)): if verbose: print('%4d: %.3f --> %.3f' % (seq[i], v[i], p1[i])) self.wifsip.execute("""UPDATE m67 SET priority = priority * %f WHERE seq = %d;""" % (p1[i], seq[i]), commit=False) self.wifsip.commit() data = self.wifsip.query("""SELECT bmag-vmag, vmag, priority FROM m67 WHERE priority > 0.0 ORDER BY seq;""") bv = np.array([d[0] for d in data]) vmag = np.array([d[1] for d in data]) p = np.array([d[2] for d in data]) makeplot(bv,vmag, p, filename=config.plotpath+'priorities.pdf') def setpointing(self, pointing): """set pointings according to hydrasim output""" #TODO: to be redone! targets = ",".join([str(t) for t in self.targeted]) print(targets) query = """UPDATE m67 SET pointing=%d WHERE seq in (%s)""" % (pointing,targets) self.wifsip.execute(query) def from_database(self, maglimit=16.5): """ load targets from database that have not priority set yet and are within the maglimit """ data = self.wifsip.query("""SELECT seq, ra, dec, vmag FROM m67 WHERE priority > 0.0 AND vmag<%f AND pointing IS NULL ORDER BY priority DESC;""" % maglimit) for d in data: target = {} target['id'] = int(d[0]) target['name'] = 'Star%04dm%.2f' % (int(d[0]),float(d[3])) target['ra'] = float(d[1]) target['dec'] = float(d[2]) target['class'] = 'O' self.table.append(target) """ hydrawiynmanual.pdf p.41: The stars selected for use by the FOPs should fall in the magnitude range 10<V<14. If possible, keep the range in magnitude of your FOPs sample in each field to 3 magnitudes or less so that the intensity of each star falls within the dynamic range of the FOPs TV camera. Including the FOPs star magnitudes in the configuration file may also be useful later when setting up the field at the telescope. """ query = """SELECT vmag, ra, dec, seq FROM m67 WHERE vmag>10 and vmag<13 ORDER BY vmag;""" fops = self.wifsip.query(query) print(len(fops), 'FOPs stars') for f in fops: target = {} target['id'] = 6001 + int(f[3]) target['name'] = 'FOP%dm%.2f' % (int(f[3]),float(f[0])) target['ra'] = float(f[1]) target['dec'] = float(f[2]) target['class'] = 'F' self.table.append(target) #extra targets query="""select vmag, ra, dec, seq FROM m67 WHERE NOT bmag-vmag IS NULL ORDER BY vmag LIMIT 999;""" extra = self.wifsip.query(query) print(len(extra), 'extra stars') for d in extra: target = {} target['id'] = 7000 + int(d[3]) target['name'] = str(d[0]) target['ra'] = float(d[1]) target['dec'] = float(d[2]) target['class'] = 'E' self.table.append(target) def skyfile(self, filename='/work2/jwe/M67/skyfile.txt'): """ load the sky positions """ from astropy import units as u from astropy.coordinates import SkyCoord ras = [132.686, 132.703, 132.820, 132.965] decs = [11.591, 11.745, 11.849, 11.869, 12.091] skypos = [] for ra in ras: for de in decs: skypos.append((ra,de)) print(len(skypos), 'sky positions') for radec in skypos: target = {} target['id'] = 9000 + skypos.index(radec) target['name'] = 'Sky'+str(target['id']) target['ra'] = radec[0] target['dec'] = radec[1] target['class'] = 'S' self.table.append(target) def tofile(self, filename = None, verbose=False): """ writes the .ast file that finally goes into whydra for processing """ import astronomy as ast if filename is None: filename = hydrapath+self.field_name+'.ast' f = open(filename,'w') f.write('FIELD NAME: %s\n' % self.field_name) f.write('INPUT EPOCH: %.2f\n' % self.input_epoch) f.write('CURRENT EPOCH: %.2f\n' % self.current_epoch) f.write('SIDERIAL TIME: %.3f\n' % self.siderial_time) f.write('EXPOSURE LENGTH: %.2f\n' % self.exposure_length) f.write('WAVELENGTH: %.0f.\n' % self.wavelength) f.write('CABLE: %s\n' % self.cable) f.write('WEIGHTING: %s\n' % self.weighting) f.write('GUIDEWAVE: %.0f.\n' % self.guidewave) f.write('#0000000011111111112222222222333333333344444444445555\n') f.write('#2345678901234567890123456789012345678901234567890123\n') for t in self.table: ra = ast.dd2hms(t['ra']) dec = ast.dd2dms(t['dec']) s = '%04d %-20s %02d %02d %06.3f %+02.2d %02d %05.2f %1s\n' % \ (t['id'],t['name'],ra[0],ra[1],ra[2], dec[0], dec[1], dec[2], t['class']) if verbose: print(s.rstrip('\n')) f.write(s) f.close() #s = 'pointing\tR.A.\tDec\tmag\tcomment' def fromfile(self, filename = None): if filename is None: filename = hydrapath+self.field_name+'.hydra' f = open(filename,'r') lines = f.readlines() f.close() for l in lines: if l[0]=='#': l = '' #if l.find('FIELD NAME:')>=0: # self.field_name = l[l.find(':')+1:].strip() if l.find('INPUT EPOCH:')>=0: self.input_epoch = float(l[l.find(':')+1:]) if l.find('CURRENT EPOCH:')>=0: self.current_epoch = float(l[l.find(':')+1:]) if l.find('SIDERIAL TIME:')>=0: self.siderial_time = float(l[l.find(':')+1:]) if l.find('EXPOSURE LENGTH:')>=0: self.exposure_length = float(l[l.find(':')+1:]) if l.find('WAVELENGTH:')>=0: self.wavelength = float(l[l.find(':')+1:]) if l.find('CABLE:')>=0: self.cable = l[l.find(':')+1:].strip() if l.find('WEIGHTING:')>=0: self.weighting = l[l.find(':')+1:].strip() if l.find('GUIDEWAVE:')>=0: self.guidewave = float(l[l.find(':')+1:]) if len(l.strip())>=53: target = { 'id': int(l[0:4]), 'name': l[5:25], 'ra': ast.hms2dd(l[26:38]), 'dec': ast.dms2dd(l[39:51]), 'class': l[52]} if l.find('STATUS=')>=55: status=l[55:66].strip() if target['class'] in ['O','E']: if (status=='STATUS=OK' or status=='STATUS=EDGE'): self.table.append(target) else: self.targeted.append(target['id']) elif target['class'] in ['C','F','S']: self.table.append(target) def make_plot(self, filename): plt.figure(figsize=[18/2.54,18/2.54]) fov = plt.Circle((self.center[0], self.center[1]), 0.5, facecolor='none', edgecolor='g') fig = plt.gcf() fig.gca().add_artist(fov) # plt.scatter(ra,dec, marker=',', c='k', s=0.1) for t in self.table: if t['class']=='O': plt.scatter(t['ra'], t['dec'], marker='o', c='g', edgecolor='none') if t['class']=='E': plt.scatter(t['ra'], t['dec'], marker=',', c='gray', edgecolor='none') if t['class']=='F': plt.scatter(t['ra'], t['dec'], marker='^', c='r', edgecolor='none') if t['class']=='S': plt.scatter(t['ra'], t['dec'], marker='h', c='b', edgecolor='none') if t['class']=='C': plt.scatter(t['ra'], t['dec'], marker='+', c='k', edgecolor='none') plt.xlim([self.center[0]+0.55,self.center[0]-0.55]) plt.ylim([self.center[1]-0.55,self.center[1]+0.55]) plt.xlabel('R.A.') plt.ylabel('Dec') plt.grid() plt.savefig(filename, dpi=300) plt.close() def dohydra(self): """ write the commands file and execute the shell script """ f = open('/home/jwe/bin/hydra_simulator/cmds.%s' % self.field_name,'wt') f.write('%s.ast\n' % self.field_name) f.write('%s\n' % self.field_name) f.write('%d\n' % self.fops) f.write('%d\n' % self.skies) if self.tweakangle: f.write('y\n') else: f.write('n\n') if self.tweakcenter: f.write('y\n') else: f.write('n\n') f.close() subprocess.call(['/home/jwe/bin/hydra_simulator/dowhydra.sh',self.field_name]) def getconcentricities(self): """ fetches the current concentricities file from the WIYN web page """ response = urlopen('http://www.wiyn.org/concentricities') html = response.read() f = open(hydrapath+'concentricities','wt') f.write(html) f.close()
class Calibrate(object): ''' classdocs ''' def __init__(self, obj='', filtercol='V'): ''' Constructor: Initialize database set filtercolor ''' from datasource import DataSource self.wifsip = DataSource(database='wifsip', host='pina', user='******') self.obj = obj if filtercol in ('B', 'V', 'R', 'I'): self.filtercol = filtercol else: raise (ValueError) logging.basicConfig(filename=config.projectpath + 'calibration.log', format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO) logging.info('Object: %s', self.obj) logging.info('Setting filtercolor=%s', self.filtercol) def getframes(self, refframe): logging.info('getframes ...') query = "SELECT object FROM frames WHERE objid='%s';" % refframe obj = self.wifsip.query(query)[0][0] print(refframe, '-->', obj) logging.info('%s --> %s' % (refframe, obj)) query = """SELECT objid FROM frames WHERE object LIKE '%s' AND filter like '%s';""" % (obj, self.filtercol) result = self.wifsip.query(query) self.frames = [r[0] for r in result] logging.info('%d frames' % len(self.frames)) def getrefframes(self): ''' choose the frames with the highest number of matched stars for each field and filter ''' logging.info('getrefframes ...') query = """SELECT object,max(matched) FROM frames WHERE object LIKE '%s' AND filter = '%s' GROUP BY object ORDER BY object;""" % (self.obj, self.filtercol) result = self.wifsip.query(query) logging.info('%d frames' % len(result)) for r in result: print(r[0], '\t', r[1]) maxmatched = [{'object': r[0], 'matched': r[1]} for r in result] self.ref = [] for m in maxmatched: query = """SELECT objid FROM frames WHERE object LIKE '%s' AND filter like '%s' AND matched = %d;""" % \ (m['object'], self.filtercol, m['matched']) self.ref.append(self.wifsip.query(query)[0][0]) logging.info('%d frames' % len(self.ref)) def corrframes(self, refframe=''): logging.info('create view phot1') try: self.wifsip.execute('DROP VIEW phot1;') except psycopg2.ProgrammingError: logging.info('drop view phot1 failed') query = """CREATE VIEW phot1 AS SELECT * FROM phot WHERE objid='%s' AND phot.mag_auto>12 and phot.mag_auto<16 AND flags=0 ORDER BY phot.mag_auto;""" % (refframe) self.wifsip.execute(query) for frame in self.frames: query = """SELECT phot.mag_auto, phot1.mag_auto FROM phot, phot1 WHERE phot.objid='%s' AND circle(phot.coord,0) <@ circle(phot1.coord,0.15/3600.); """ % frame result = self.wifsip.query(query) if len(result) == 0: logging.warn('no data for frame %s' % frame) omag = np.array([r[0] for r in result]) cmag = np.array([r[1] for r in result]) mstd = np.std(omag - cmag) corr = np.mean(omag - cmag) s = '%s %6.3f %.3f %3d' % (frame, corr, mstd, len(omag)) logging.info(s) if len(omag) < 100 or mstd > 0.015: corr = np.nan # print '#' else: print(s) self.updateframe(frame, corr) logging.info('drop view phot1') self.wifsip.dropview('phot1') def updateframe(self, frame, corr): if np.isnan(corr): query = """UPDATE frames SET corr = NULL WHERE objid = '%s';""" % frame else: query = """UPDATE frames SET corr = %f WHERE objid = '%s';""" % (corr, frame) self.wifsip.execute(query) def resetframes(self): logging.info('reset frames') query = """UPDATE frames SET corr=NULL WHERE object LIKE '%s' AND filter like '%s';""" % (self.obj, self.filtercol) self.wifsip.execute(query)
class NGC2236Star(dict): ''' class that interfaces the ngc2236 table on wifsip database ''' def __init__(self, starid): """ Constructor: connects to the database """ from datasource import DataSource self.starid = starid self.wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') if '%' in self.starid: self.starid = self['starid'] def keys(self): """ returns the keys from the dict """ query = """SELECT column_name, data_type, character_maximum_length FROM INFORMATION_SCHEMA.COLUMNS WHERE table_name = 'ngc2236';""" result = self.wifsip.query(query) keys = [r[0] for r in result] return keys def values(self): """ returns the values from the dict """ if '%' in self.starid: query = """SELECT * from ngc2236 where starid like '%s'""" % self.starid else: query = """SELECT * from ngc2236 where starid = '%s'""" % self.starid result = self.wifsip.query(query) values = [r for r in result[0]] if '%' in self.starid: self.starid = values[0] return values def __setitem__(self, key, value): """ setter method: sets value based on key in database table """ if value is None: query = """UPDATE ngc2236 SET %s=NULL WHERE starid='%s';""" % (key, self.starid) else: if type(value) is str: value = "'%s'" % value query = """UPDATE ngc2236 SET %s=%s WHERE starid='%s';""" % (key, str(value), self.starid) self.wifsip.execute(query) def __getitem__(self, key): """ getter method: fetches value based on key from database table """ result = self.wifsip.query("""SELECT %s FROM ngc2236 WHERE starid = '%s';""" % (key, self.starid)) return result[0][0] def lightcurve(self): """ return the lightcurve object """ return NGCLightCurve(self['corotid'])
class WHydra(object): ''' classdocs ''' def __init__(self, field_name = 'M48field'): ''' Constructor ''' import ephem import datetime import pytz import astronomy as ast from datasource import DataSource self.field_name = field_name self.wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') kpno = ephem.Observer() #31.958036,-111.600578 kpno.lon, kpno.lat = '-111.600578', '31.958036' kpno.horizon = '-0:34' kpno.elevation = 2096 tzi = pytz.timezone('MST') #fmt = '%Y-%m-%d %H:%M:%S %Z%z' obsdate = datetime.datetime(2015,2,9,23,0,0, tzinfo=tzi) kpno.date = obsdate + datetime.timedelta(7*ephem.hour) d = kpno.date.datetime() self.input_epoch = 2000 self.current_epoch = d.year + d.timetuple().tm_yday/365. # LST at mid-exposure in decimal hours lst = kpno.sidereal_time() print 'local siderial time: ',lst self.siderial_time = ast.hms2hh(str(lst)) self.exposure_length = 40.0/60.0 # in hours self.wavelength = 6000 # in Angstroem self.cable = 'BLUE' self.weighting = 'STRONG' self.guidewave = 6000 self.center = self._get_center() self.ebv = 0.031 # from Webda self.dm = 9.53 # from Webda target = {} self.table = [] target['id'] = 6000 target['name'] = 'M48center' target['ra'] = self.center[0] target['dec'] = self.center[1] target['class'] = 'C' self.table.append(target) self.targeted = [] # number of sky fibers self.skies = 6 # number of field orientation probes self.fops = 6 self.tweakcenter = False self.tweakangle = False def _get_center(self): data = self.wifsip.query("""select avg(ra),avg(dec) from m48stars;""") return (data[0][0],data[0][1]) def priorities(self, verbose = False): """updates the priorieties in the m48stars table""" import numpy as np from functions import scaleto print 'calculate priorities ...' self.wifsip.execute("UPDATE m48stars SET priority=NULL;") self.wifsip.execute("""UPDATE m48stars SET priority=1.0 WHERE vmag<16.5 AND NOT tab IS NULL;""") data = self.wifsip.query("""SELECT tab, vmag FROM m48stars WHERE not bv is null AND vmag<16.5 ORDER BY tab;""") tab = [d[0] for d in data] v = np.array([d[1] for d in data]) p1 = scaleto(v,[1.0, 0.8]) print len(tab),'stars brighter V<16.5' for i in range(len(tab)): if verbose: print '%4d: %.3f --> %.3f' % (tab[i], v[i],p1[i]) self.wifsip.execute("""UPDATE m48stars SET priority = priority * %f WHERE tab = %d;""" % (p1[i], tab[i]), commit=False) self.wifsip.commit() lengood = self.wifsip.query("select count(starid) from m48stars where good;") print lengood[0],'stars with periods' self.wifsip.execute("""UPDATE m48stars SET priority = priority * 0.5 WHERE NOT good;""") self.wifsip.commit() iso = IsoChrone('/work2/jwe/m48/data/output256520738433.dat') x = iso['V'] + self.dm y = iso['B-V'] + self.ebv data = self.wifsip.query("""SELECT tab, vmag, bv FROM m48stars WHERE not bv is null AND vmag<16.5 ORDER BY tab;""") tab = [d[0] for d in data] v= np.array([d[1] for d in data]) bv = np.array([d[2] for d in data]) p = np.zeros(len(tab)) print len(tab),'stars for isochrone priority' for i in range(len(tab)): p[i] = np.min(abs(x-v[i])+abs(y-bv[i])) p[p > 0.4] = 0.4 p = scaleto(p, [1.0, 0.0]) for t in tab: i = tab.index(t) if verbose: print '%d: V=%.3f B-V=%.3f p=%.3f' % (t, v[i], bv[i], p[i]) self.wifsip.execute("""UPDATE m48stars SET priority = priority * %f WHERE tab = %d;""" % (p[i], t), commit=False) self.wifsip.commit() data = self.wifsip.query("""SELECT tab, ra, dec FROM m48stars WHERE not ra is NULL AND not dec is NULL AND priority > 0.0 ORDER BY TAB;""") tab = [d[0] for d in data] print len(tab),'stars for distance priority' ra = np.array([d[1] for d in data]) dec = np.array([d[2] for d in data]) dist = np.sqrt((ra-self.center[0])**2+(dec-self.center[1])**2) # only rank stars where distance greater than 0.5 degrees i = np.where(dist > 0.5) p = scaleto(dist, [1.0, 0.0]) for t in tab: i = tab.index(t) try: if verbose: print '%d: d=%.3f p=%.3f' % (t,dist[i],p[i]) self.wifsip.execute("""UPDATE m48stars SET priority = priority * %f WHERE tab = %d;""" % (p[i], t), commit=False) except TypeError: print t self.wifsip.commit() def setpointing(self, pointing): """set pointings according to hydrasim output""" #TODO: to be redone! targets = ",".join([str(t) for t in self.targeted]) print targets query = """UPDATE m48stars SET pointing=%d WHERE tab in (%s)""" % (pointing,targets) self.wifsip.execute(query) def from_database(self, maglimit=16.5): """ load targets from database that have not priority set yet and are within the maglimit """ data = self.wifsip.query("""SELECT tab, starid, ra, dec, vmag FROM m48stars WHERE priority > 0.0 AND vmag<%f AND pointing IS NULL ORDER BY priority DESC;""" % maglimit) for d in data: target = {} target['id'] = int(d[0]) target['name'] = 'Star%04dm%.2f' % (int(d[0]),float(d[4])) target['ra'] = float(d[2]) target['dec'] = float(d[3]) target['class'] = 'O' self.table.append(target) """ hydrawiynmanual.pdf p.41: The stars selected for use by the FOPs should fall in the magnitude range 10<V<14. If possible, keep the range in magnitude of your FOPs sample in each field to 3 magnitudes or less so that the intensity of each star falls within the dynamic range of the FOPs TV camera. Including the FOPs star magnitudes in the configuration file may also be useful later when setting up the field at the telescope. """ #TODO: import at most 2000 twomass coordinates fops = self.wifsip.query("""SELECT tab, ra, dec,vmag FROM m48stars WHERE vmag>10 AND vmag<13 AND (priority < 0.3 OR priority IS NULL) AND tab is NOT NULL ORDER BY vmag LIMIT 2000;""") print len(fops),'FOPs stars' for f in fops: target = {} target['id'] = 6001 + int(f[0]) target['name'] = 'FOP%dm%.2f' % (int(f[0]),float(f[3])) target['ra'] = float(f[1]) target['dec'] = float(f[2]) target['class'] = 'F' self.table.append(target) #extra targets extra = self.wifsip.query("""SELECT starid, ra, dec FROM m48phot WHERE vmag<6.5*bv+10.4 and vmag>5*bv+10 AND vmag<%f ORDER BY vmag;""" % maglimit) print len(extra),'extra stars' for d in extra: target = {} target['id'] = 7000+extra.index(d) target['name'] = str(d[0]) target['ra'] = float(d[1]) target['dec'] = float(d[2]) target['class'] = 'E' self.table.append(target) def skyfile(self, filename='/work2/jwe/m48/data/sky_positions.txt'): """ load the sky positions """ from astropy import units as u from astropy.coordinates import SkyCoord f = open(filename,'r') lines = f.readlines() f.close() print len(lines),'sky positions' for l in lines: try: c = SkyCoord(l, 'icrs', unit=(u.hourangle, u.deg)) # @UndefinedVariable except ValueError: print 'empty line' else: target = {} target['id'] = 9000 + lines.index(l) target['name'] = 'Sky'+str(target['id']) target['ra'] = c.ra.degree target['dec'] = c.dec.degree target['class'] = 'S' self.table.append(target) def tofile(self, filename = None, verbose=False): """ writes the .ast file that finally goes into whydra for processing """ import astronomy as ast if filename is None: filename = hydrapath+self.field_name+'.ast' f = open(filename,'w') f.write('FIELD NAME: %s\n' % self.field_name) f.write('INPUT EPOCH: %.2f\n' % self.input_epoch) f.write('CURRENT EPOCH: %.2f\n' % self.current_epoch) f.write('SIDERIAL TIME: %.3f\n' % self.siderial_time) f.write('EXPOSURE LENGTH: %.2f\n' % self.exposure_length) f.write('WAVELENGTH: %.0f.\n' % self.wavelength) f.write('CABLE: %s\n' % self.cable) f.write('WEIGHTING: %s\n' % self.weighting) f.write('GUIDEWAVE: %.0f.\n' % self.guidewave) f.write('#0000000011111111112222222222333333333344444444445555\n') f.write('#2345678901234567890123456789012345678901234567890123\n') for t in self.table: ra = ast.dd2hms(t['ra']) dec = ast.dd2dms(t['dec']) s = '%04d %-20s %02d %02d %06.3f %+02.2d %02d %05.2f %1s\n' % \ (t['id'],t['name'],ra[0],ra[1],ra[2], dec[0], dec[1], dec[2], t['class']) if verbose: print s.rstrip('\n') f.write(s) f.close() #s = 'pointing\tR.A.\tDec\tmag\tcomment' def fromfile(self, filename = None): import astronomy as ast if filename is None: filename = hydrapath+self.field_name+'.hydra' f = open(filename,'r') lines = f.readlines() f.close() for l in lines: if l[0]=='#': l = '' #if l.find('FIELD NAME:')>=0: # self.field_name = l[l.find(':')+1:].strip() if l.find('INPUT EPOCH:')>=0: self.input_epoch = float(l[l.find(':')+1:]) if l.find('CURRENT EPOCH:')>=0: self.current_epoch = float(l[l.find(':')+1:]) if l.find('SIDERIAL TIME:')>=0: self.siderial_time = float(l[l.find(':')+1:]) if l.find('EXPOSURE LENGTH:')>=0: self.exposure_length = float(l[l.find(':')+1:]) if l.find('WAVELENGTH:')>=0: self.wavelength = float(l[l.find(':')+1:]) if l.find('CABLE:')>=0: self.cable = l[l.find(':')+1:].strip() if l.find('WEIGHTING:')>=0: self.weighting = l[l.find(':')+1:].strip() if l.find('GUIDEWAVE:')>=0: self.guidewave = float(l[l.find(':')+1:]) if len(l.strip())>=53: target = { 'id': int(l[0:4]), 'name': l[5:25], 'ra': ast.hms2dd(l[26:38]), 'dec': ast.dms2dd(l[39:51]), 'class': l[52]} if l.find('STATUS=')>=55: status=l[55:66].strip() if target['class'] in ['O','E']: if (status=='STATUS=OK' or status=='STATUS=EDGE'): self.table.append(target) else: self.targeted.append(target['id']) elif target['class'] in ['C','F','S']: self.table.append(target) def make_plot(self, filename): import matplotlib.pyplot as plt plt.figure(figsize=[18/2.54,18/2.54]) fov = plt.Circle((self.center[0], self.center[1]), 0.5, facecolor='none', edgecolor='g') fig = plt.gcf() fig.gca().add_artist(fov) # plt.scatter(ra,dec, marker=',', c='k', s=0.1) for t in self.table: if t['class']=='O': plt.scatter(t['ra'], t['dec'], marker='o', c='g') if t['class']=='E': plt.scatter(t['ra'], t['dec'], marker=',', c='gray') if t['class']=='F': plt.scatter(t['ra'], t['dec'], marker='^', c='r') if t['class']=='S': plt.scatter(t['ra'], t['dec'], marker='h', c='b') if t['class']=='C': plt.scatter(t['ra'], t['dec'], marker='+', c='k') plt.xlim([self.center[0]+0.55,self.center[0]-0.55]) plt.ylim([self.center[1]-0.55,self.center[1]+0.55]) plt.xlabel('R.A.') plt.ylabel('Dec') plt.grid() #plt.show() plt.savefig(filename, dpi=300) plt.close() #data = self.wifsip.query("""SELECT vmag, bv, priority # FROM m48stars # WHERE priority>0.1;""") #v= np.array([d[0] for d in data]) #bv = np.array([d[1] for d in data]) #p = [d[2] for d in data] #from matplotlib import pyplot #pyplot.scatter(bv, v, c=p) #pyplot.xlim(-0.1,1.6) #pyplot.ylim(16.5,7.5) #pyplot.show() #plt.savefig(filename, dpi=300) #plt.close() def dohydra(self): """ write the commands file and execute the shell script """ import subprocess f = open('/home/jwe/bin/hydra_simulator/cmds.%s' % self.field_name,'wt') f.write('%s.ast\n' % self.field_name) f.write('%s\n' % self.field_name) f.write('%d\n' % self.fops) f.write('%d\n' % self.skies) if self.tweakangle: f.write('y\n') else: f.write('n\n') if self.tweakcenter: f.write('y\n') else: f.write('n\n') f.close() subprocess.call(['/home/jwe/bin/hydra_simulator/dowhydra.sh',self.field_name]) def getconcentricities(self): """ fetches the current concentricities file from the WIYN web page """ import urllib2 response = urllib2.urlopen('http://www.wiyn.org/concentricities') html = response.read() f = open(hydrapath+'concentricities','wt') f.write(html) f.close()
class Analysis(object): def __init__(self): ''' Constructor ''' from datasource import DataSource self.wifsip = DataSource(database=config.dbname, user=config.dbuser, host=config.dbhost) self.stars = [] query = "SELECT starid, bv ,vmag ,vmag_err from ngc6633;" self._stars = self.wifsip.query(query) columns = self.wifsip.columns('ngc6633') data_types = self.wifsip.data_types('ngc6633') for c, d in zip(columns, data_types): print c, d arraydata = [] for star in self._stars: arraydata.append(tuple(star)) columns = ['starid', 'bv', 'vmag', 'vmag_err'] data_types = ['S25', np.float16, np.float16, np.float16] self.stars = np.array(arraydata, dtype=zip(columns, data_types)) # self.age = 10**8.557/1e6 # in Myr from Webda # self.ebv = 0.031 # from Webda # self.dm = 9.53 # from Webda def getstars(self): """ build up a list of stars, where we do not have periods yet """ query = "SELECT starid, vmag, bv" \ " FROM ngc6633 " \ " WHERE vmag<18 " \ " AND vmag < 10 + bv*5.4 " \ " AND vmag > 7.5 + bv*5.4" \ # " AND bv>0.4 " \ " AND NOT good IS NULL" \ " ORDER BY vmag LIMIT 100;" result = self.wifsip.query(query) print '... %d stars found' % len(result) self.stars = [s[0] for s in result] self.vmag = [s[1] for s in result] self.bv = [s[2] for s in result] def setperiod(self, starid, period, theta=1.0): self.starid = starid params = {'starid': starid, 'period': period, 'theta': theta} if np.isfinite(period): query = """UPDATE ngc6633 SET period = %(period)f, theta = %(theta)f WHERE starid='%(starid)s';""" % params else: query = """UPDATE ngc6633 SET period = NULL, theta=NULL WHERE starid='%(starid)s';""" % params self.wifsip.execute(query) def setamp(self, starid, amplitude): self.starid = starid params = {'starid': starid, 'amp': amplitude} if np.isfinite(amplitude): query = """UPDATE ngc6633 SET amp = %(amp)f WHERE starid='%(starid)s';""" % params else: query = """UPDATE ngc6633 SET amp = NULL WHERE starid='%(starid)s';""" % params self.wifsip.execute(query) def setbad(self, starid): self.starid = starid query = """UPDATE ngc6633 SET good = False WHERE starid='%s';""" % starid self.wifsip.execute(query) #def __setattr__(self, name, value): # params = {'starid': self.starid, 'name': name, 'value': str(value)} # # query = """UPDATE ngc6633 SET %(name)s = %(value)s WHERE starid='%(starid)s';""" % params # self.wifsip.execute(query) #def __getattribute__(self, name): # params = {'starid': self.starid, 'name': name} # query = "SELECT %(name)s from ngc6633 WHERE starid='%(starid)s';" % params # result = self.wifsip.query(query) # return result[0] def plot_lightcurve(self): """ plot the lightcurve for a given star """ mean = np.mean(self.mag) plt.hlines(mean, min(self.hjd), max(self.hjd), linestyle='--') plt.xlim(min(self.hjd), max(self.hjd)) plt.grid() plt.errorbar(self.hjd, self.mag, yerr=self.err * 0.5, fmt='o') ylim = plt.ylim() plt.ylim(ylim[1], ylim[0]) def plot_clean(self, periods, amplitudes): plt.plot(periods, amplitudes, 'k') i = np.argmax(amplitudes) period = periods[i] plt.axvline(x=period, color='red', alpha=0.5) plt.axhline(np.mean(amplitudes), color='b', ls='--') plt.axhline(5. * np.mean(amplitudes), color='g', ls='--') plt.xlim(min(periods), max(periods)) plt.minorticks_on() def plot_pdm(self, periods, thetas): #plt.plot(periods, thetas, 'k') from scipy import signal kernel = signal.gaussian(101, 2) n = len(thetas) / 2 padded_thetas = np.lib.pad(thetas, n, mode='constant', constant_values=(1.0, 1.0)) smoothed = signal.fftconvolve(padded_thetas, kernel, mode='same')[n:-n] / 5 plt.plot(periods, smoothed, 'k') i = np.argmin(smoothed) period = periods[i] plt.axvline(x=period, color='red', alpha=0.5) plt.axhline(0.8, color='b', ls='--') plt.xlim(min(periods), max(periods)) plt.ylim(0.0, 1.0) plt.minorticks_on() def plot_phase(self, period): from functions import phase tp, yp = phase(self.hjd, self.mag, period) s1 = np.sin(2 * np.pi * tp / period) c1 = np.cos(2 * np.pi * tp / period) s2 = np.sin(4 * np.pi * tp / period) c2 = np.cos(4 * np.pi * tp / period) A = np.column_stack((np.ones(tp.size), s1, c1, s2, c2)) c, _, _, _ = np.linalg.lstsq(A, yp) tp1 = np.linspace(0.0, 2 * period, 100) s1 = np.sin(2 * np.pi * tp1 / period) c1 = np.cos(2 * np.pi * tp1 / period) s2 = np.sin(4 * np.pi * tp1 / period) c2 = np.cos(4 * np.pi * tp1 / period) plt.scatter(tp, yp, edgecolor='none', alpha=0.75) plt.scatter(tp + period, yp, edgecolor='none', alpha=0.75) plt.plot(tp1, c[1] * s1 + c[2] * c1 + c[3] * s2 + c[4] * c2, 'k', linestyle='--', linewidth=2) plt.xlim(0.0, 2 * period) plt.ylim(max(yp - np.mean(yp)), min(yp - np.mean(yp))) plt.xlabel('P = %.4f' % period) def analysis(self, show=False): """perform a PDM analysis on each lightcurve""" from matplotlib import rcParams print 'Analysis' fig_width = 18.3 / 2.54 # width in inches, was 7.48in fig_height = 23.3 / 2.54 # height in inches, was 25.5 fig_size = [fig_width, fig_height] #set plot attributes params = { 'backend': 'Agg', 'axes.labelsize': 12, 'axes.titlesize': 12, 'font.size': 12, 'xtick.labelsize': 12, 'ytick.labelsize': 12, 'figure.figsize': fig_size, 'savefig.dpi': 300, 'font.family': 'sans-serif', 'axes.linewidth': 0.5, 'xtick.major.size': 2, 'ytick.major.size': 2, } rcParams.update(params) minperiod = 1.3 maxperiod = 15 for starid, vmag, bv in zip(self.stars, self.vmag, self.bv): print '%-24s ' % starid, try: lc = LightCurve(starid) except IOError: logger.error("Can't load lightcurve %s" % starid) print 'no lightcurve' self.setbad(starid) continue if len(lc) < 50: logger.warn("%s: not enough datapoints" % starid) print 'not enough datapoints' continue # perform a 3sigma clipping i = np.where(lc.hjd > 2456762)[0] lc.mag = lc.mag[i] lc.hjd = lc.hjd[i] lc.normalize() lc.clip() lc.detrend() lc.sigma_clip() lc.normalize() self.hjd = lc.hjd self.mag = lc.mag self.err = lc.err clean_periods, clean_amplitudes = lc.clean(minperiod, maxperiod) pdm_periods, pdm_thetas = lc.pdm(minperiod, maxperiod) #period = clean_periods[np.argmax(clean_amplitudes)] from scipy import interpolate i = np.argsort(clean_periods) c_periods = clean_periods[i] c_amps = clean_amplitudes[i] c_periods = np.insert(c_periods, 0, 0.0) c_amps = np.insert(c_amps, 0, 0.0) c_periods = np.insert(c_periods, -1, maxperiod) c_amps = np.insert(c_amps, -1, 0.0) c_int = interpolate.interp1d(c_periods, c_amps) # use interpolation function returned by `interp1d` sum_amp = c_int(pdm_periods) * (1. - pdm_thetas) sum_amp /= max(sum_amp) i = np.argmax(sum_amp) period = pdm_periods[i] theta = pdm_thetas[i] import functions as fx ci = np.argmax(c_amps) try: pgf_amp, pgf_mean, pgf_sigma = fx.gauss_fit( pdm_periods, 1. - pdm_thetas, pdm_thetas[i], period, 1.0) except: pgf_sigma = 0.0 try: cgf_amp, cgf_mean, cgf_sigma = fx.gauss_fit( c_periods, c_amps, c_amps[ci], c_periods[ci], 1.0) except: cfg_sigma = 0.0 sigma_limit = 5.3 theta_limit = 0.8 try: ci = np.argmax(c_amps) params = { 'period': period, 'period_err': pgf_sigma, 'clean_period': c_periods[ci], 'clean_amp': c_amps[ci], 'clean_sigma': cgf_sigma, 'theta': theta, 'freq': period, } keys = ', '.join(params.keys()) values = ', '.join([str(v) for v in params.values()]) query = """UPDATE ngc6633 SET (%s) = (%s) WHERE starid='%s';""" % ( keys, values, starid) self.wifsip.execute(query) except: print 'Cannot store params for starid %s' % starid mamp = np.mean(clean_amplitudes) if max(clean_amplitudes ) > sigma_limit * mamp and pdm_thetas[i] < theta_limit: print "%.2f %.1f" % (period, max(clean_amplitudes) / mamp) self.setperiod(starid, period) else: print '< %.1f sigmas or theta %.2f > %.2f' % ( sigma_limit, pdm_thetas[i], theta_limit) self.setperiod(starid, np.nan) continue amp, _ = lc.phased(period) self.setamp(starid, amp) star = {'tab': 0, 'bv': bv} plt.subplot( 411) ################################################## plt.title('%s (%d) V = %.2f B-V=%.2f' % (starid, star['tab'], vmag, bv)) self.plot_lightcurve() plt.subplot( 412) ################################################## self.plot_clean(clean_periods, clean_amplitudes) plt.subplot( 413) ################################################## plt.plot(pdm_periods, sum_amp, 'g') plt.axvline(period, color='b') self.plot_pdm(pdm_periods, pdm_thetas) plt.subplot( 414) ################################################## self.plot_phase(period) # plt.scatter(tp, yp-np.mean(yp), edgecolor='none', alpha=0.75) # plt.plot(tp1,c[1]*s1+c[2]*c1+c[3]*s2+c[4]*c2, 'k', # linestyle='--', linewidth=2) # plt.xlim(0.0,period) # plt.ylim(max(yp-np.mean(yp)),min(yp-np.mean(yp))) # plt.xlabel('P = %.4f' % period) # plt.grid() # #plt.show() # period_err = 0.0 # comment = 'P=%6.3f+-%.3f a=%.3f+-%.4f %.2f' % \ # (period, period_err, amp, amp_err, theta) # plt.savefig(config.plotpath+'%s(%d).pdf' % (starid,star['tab'])) # plt.close() # # logger.info( comment) # print comment if show: plt.show() else: plt.savefig(config.plotpath + '%s(%d).pdf' % (starid, star['tab'])) plt.close()
def main(argv): target_url = '' COMMAND = '' ALL = False AUTH = False DATASOURCE = False TERM = False COURSE = False USER = False MEMBERSHIP = False CLEANUP = False datasource_PK1 = None usageStr = "\nrestdemo.py -t|--target <target root URL> -c|--command <command>\n" usageStr += "e.g restdemo.py -t www.myschool.edu -c create_course\n" usageStr += "command: <command>_<object> where <command> is one of the following:\n" usageStr += "\tcreate, read, read_all, update, delete\n" usageStr += "and <object> is one of the following:\n" usageStr += "\tdatasource, term, course, user, membership\n" usageStr += "-t is required; No -c args will run demo in predetermined order.\n" usageStr += "'-c authorize' demomonstrates the authorization process and does not create objects." usageStr += "-c commands require a valid datasource PK1 - \n" usageStr += "\ta datasource get will be run in these cases, defaulting to create\n" usageStr += "\tif the demo datasource does not exist." if len(sys.argv) > 1: #there are command line arguments try: opts, args = getopt.getopt(argv,"ht:c:",["target=","command="]) except getopt.GetoptError: print (usageStr) sys.exit(2) for opt, arg in opts: if opt == '-h': print (usageStr) sys.exit() elif opt == '-d': print ("Deleting at end of run.") CLEANUP = True elif opt in ("-t", "--target"): target_url = arg.lstrip() elif opt in ("-c", "--command"): COMMAND = arg else: COMMAND = "Run All" print ('[main] Target is:', target_url) print ('[main] Command is:', COMMAND) else: print(usageStr) sys.exit(2) #Set up some booleans for processing flags and order of processing if "course" in COMMAND: print("[main] Run course command") COURSE = True elif "user" in COMMAND: print("[main] Run user command") USER = True elif "membership" in COMMAND: print("[main] Run membership command") MEMBERSHIP = True elif "term" in COMMAND: print("[main] Run term command") TERM = True elif "datasource" in COMMAND: print("[main] Run datasource command") DATASOURCE = True elif "authorize" in COMMAND: print("[main] Run authorization command") AUTH = True else: print("[main] Empty Command: Run All\n") ALL = True print ('\n[main] Acquiring auth token...\n') authorized_session = AuthToken(target_url) authorized_session.setToken() print ('\n[main] Returned token: ' + authorized_session.getToken() + '\n') if not AUTH: #run commands in required order if running ALL if DATASOURCE or ALL: #process Datasource command print("\n[main] Run datasource command: " + ('ALL' if ALL else COMMAND) + '...') datasource_session = DataSource(target_url, authorized_session.getToken()) if 'datasource' in COMMAND: datasource_session.execute(COMMAND, authorized_session.getToken()) else: #datasource_session.getDataSources(authorized_session.getToken()) datasource_session.createDataSource(authorized_session.getToken()) datasource_PK1 = datasource_session.datasource_PK1 print("[main] datasource_PK1: " + datasource_PK1) datasource_session.getDataSource(authorized_session.getToken()) datasource_session.updateDataSource(authorized_session.getToken()) if TERM or ALL: term_session = Term(target_url, authorized_session.getToken()) #process term command print("\n[main] Run term command: " + ('ALL' if ALL else COMMAND) + '...') if 'term' in COMMAND: if (('delete' in COMMAND) or ('read' in COMMAND)): print ("[main] Deleting or getting does not require a datasource.") else: print("[main] datasource_PK1: not known... searching...") datasource_session = DataSource(target_url, authorized_session.getToken()) datasource_session.getDataSource(authorized_session.getToken()) datasource_PK1 = datasource_session.datasource_PK1 print("[main] datasource_PK1: " + datasource_PK1) if (datasource_PK1 is None): print ("[main] data source not found, creating for demo...") datasource_session.createDataSource(authorized_session.getToken()) datasource_PK1 = datasource_session.datasource_PK1 term_session.execute(COMMAND, datasource_PK1, authorized_session.getToken()) else: #term_session.getTerms(authorized_session.getToken()) term_session.createTerm(datasource_PK1, authorized_session.getToken()) term_session.getTerm(authorized_session.getToken()) term_session.updateTerm(datasource_PK1, authorized_session.getToken()) if COURSE or ALL: course_session = Course(target_url, authorized_session.getToken()) #process course command print("\n[main] Run course command: " + ('ALL' if ALL else COMMAND) + '...') if 'course' in COMMAND: if (('delete' in COMMAND) or ('read' in COMMAND)): print ("[main] Deleting or getting does not require a datasource.") else: print("[main] datasource_PK1: not known... searching...") datasource_session = DataSource(target_url, authorized_session.getToken()) datasource_session.getDataSource(authorized_session.getToken()) datasource_PK1 = datasource_session.datasource_PK1 print("[main] datasource_PK1: " + datasource_PK1) if (datasource_PK1 is None): print ("[main] data source not found, creating for demo...") datasource_session.createDataSource(authorized_session.getToken()) datasource_PK1 = datasource_session.datasource_PK1 course_session.execute(COMMAND, datasource_PK1, authorized_session.getToken()) else: #course_session.getCourses(authorized_session.getToken()) course_session.createCourse(datasource_PK1, authorized_session.getToken()) course_session.getCourse(authorized_session.getToken()) course_session.updateCourse(datasource_PK1, authorized_session.getToken()) if USER or ALL: user_session = User(target_url, authorized_session.getToken()) #process user command print("\n[main] Run user command: " + ('ALL' if ALL else COMMAND) + '...') if 'user' in COMMAND: if (('delete' in COMMAND) or ('read' in COMMAND)): print ("[main] Deleting or getting does not require a datasource.") else: print("[main] datasource_PK1: not known... searching...") datasource_session = DataSource(target_url, authorized_session.getToken()) datasource_session.getDataSource(authorized_session.getToken()) datasource_PK1 = datasource_session.datasource_PK1 print("[main] datasource_PK1: " + datasource_PK1) if (datasource_PK1 is None): print ("[main] data source not found, creating for demo...") datasource_session.createDataSource(authorized_session.getToken()) datasource_PK1 = datasource_session.datasource_PK1 user_session.execute(COMMAND, datasource_PK1, authorized_session.getToken()) else: #user_session.getUsers(authorized_session.getToken()) user_session.createUser(datasource_PK1, authorized_session.getToken()) user_session.getUser(authorized_session.getToken()) user_session.updateUser(datasource_PK1, authorized_session.getToken()) if MEMBERSHIP or ALL: membership_session = Membership(target_url, authorized_session.getToken()) #process membership command print("\n[main] Run membership command: " + ('ALL' if ALL else COMMAND) + '...') if 'membership' in COMMAND: if (('delete' in COMMAND) or ('read' in COMMAND)): print ("[main] Deleting or getting does not require a datasource.") else: print("[main] datasource_PK1: not known... searching...") datasource_session = DataSource(target_url, authorized_session.getToken()) datasource_session.getDataSource(authorized_session.getToken()) datasource_PK1 = datasource_session.datasource_PK1 print("[main] datasource_PK1: " + datasource_PK1) if (datasource_PK1 is None): print ("[main] data source not found, creating for demo...") datasource_session.createDataSource(authorized_session.getToken()) datasource_PK1 = datasource_session.datasource_PK1 membership_session.execute(COMMAND, datasource_PK1, authorized_session.getToken()) else: #membership_session.getMemberships(authorized_session.getToken()) membership_session.createMembership(datasource_PK1, authorized_session.getToken()) membership_session.getMembership(authorized_session.getToken()) membership_session.updateMembership(datasource_PK1, authorized_session.getToken()) membership_session.readUserMemberships(authorized_session.getToken()) #clean up if not using individual commands if ALL: print('\n[main] Completing Demo and deleting created objects...') print ("[main] Deleting membership") membership_session.deleteMembership(authorized_session.getToken()) print ("[main] Deleting Course") user_session.deleteUser(authorized_session.getToken()) print ("[main] Deleting Course") course_session.deleteCourse(authorized_session.getToken()) print ("[main] Deleting Term") term_session.deleteTerm(authorized_session.getToken()) print ("[main] Deleting DataSource") datasource_session.deleteDataSource(authorized_session.getToken()) else: print("Remember to delete created demo objects!") print("[main] Processing Complete")
class M48Phot(object): ''' Import photometry from David James ''' def __init__(self): ''' Constructor ''' from datasource import DataSource self.data = [] self.table = DataSource(database='wifsip', user='******', host='pina.aip.de') def fromfile(self, filename): f = open(filename) lines = f.readlines() f.close p = filename.find('M48-') field = filename[p + 4:p + 6] for l in lines: sl = l.split() record = { 'starid': '%s-%s-%3.3d' % (sl[0], field, int(sl[1])), 'ccdx': float(sl[2]), 'ccdy': float(sl[3]), 'ra': float(sl[10]), 'dec': float(sl[11]), 'vmag': float(sl[12]), 'vmag_err': float(sl[13]), 'bv': float(sl[14]), 'bv_err': float(sl[15]), 'vi': float(sl[16]), 'vi_err': float(sl[17]) } if record['bv'] > 9: record['bv'] = None if record['bv_err'] > 9: record['bv_err'] = None if record['vi'] > 9: record['vi'] = None if record['vi_err'] > 9: record['vi_err'] = None self.data.append(record) print record def createtable(self): ''' create table for the photometry ''' if not raw_input('press Y to erase m48phot') == 'Y': return query = """ DROP TABLE IF EXISTS m48phot; CREATE TABLE m48phot( starid varchar(10), ccdx real, ccdy real, ra double precision, dec double precision, vmag real default 0, vmag_err real, bv real, bv_err real, vi real, vi_err real, coord point, PRIMARY KEY (starid)); GRANT SELECT ON m48phot TO public; CREATE INDEX idx_m48phot_coords ON m48phot USING GIST (circle(coord,0)); """ self.table.execute(query) print "table 'm48phot' created" def todatabase(self): import StringIO cur = self.table.cursor def nstr(s): if s is None: return '\N' elif type(s) is str: return str(s) else: return str(s) values = '' for record in self.data: valline = '\t'.join([nstr(v) for v in record.values()]) print valline values += valline + '\n' columns = record.keys() f = StringIO.StringIO(values) try: cur.copy_from(f, 'm48phot', columns=columns) finally: self.table.commit()
class LightcurveRMS(object): ''' Class to implement all methods for analyzing and plotting ''' def __init__(self): ''' Constructor ''' from datasource import DataSource self.wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') def load_lightcurve(self, tab=1): query = """SELECT mag_auto+corr FROM phot, frames, m48stars WHERE circle(phot.coord,0) <@ circle(m48stars.coord,1.0/3600) AND tab=%d AND filter='V' AND frames.objid=phot.objid AND phot.flags<8 AND NOT corr IS NULL ORDER BY phot.objid;""" % tab result = self.wifsip.query(query) mags = np.array([r[0] for r in result]) if len(mags) >= 3: print '%4d' % tab, valid = abs(mags - np.mean(mags)) < 3.0 * np.std(mags) magsc = np.compress(valid, mags) print 'mean1 = %6.3f' % np.mean(magsc), print 'std1 = %.3f' % np.std(magsc), print ' %.3f' % (np.std(magsc) / np.sqrt(len(magsc))), print 'n = %3d' % len(magsc) query = """INSERT INTO m48_lightcurves (tab, vmean, vstd, nv) VALUES (%d, %f, %f, %d);""" % (tab, np.mean(magsc), np.std(magsc), len(magsc)) #query = """UPDATE m48_lightcurves SET bmean = %f, bstd = %f, nb = %d #WHERE tab=%d;""" % (np.mean(magsc), np.std(magsc), len(magsc), tab) query = query.replace('nan', 'NULL') self.wifsip.execute(query) else: print tab, 'no data' if tab % 50 == 0: self.plot_cmd() def plot_cmd(self): query = """SELECT bv, vmean, vstd, good, member FROM m48stars, m48_lightcurves WHERE m48stars.tab=m48_lightcurves.tab;""" result = self.wifsip.query(query) bv = np.array([r[0] for r in result]) vmag = np.array([r[1] for r in result]) vstd = np.array([r[2] for r in result]) good = [r[3] for r in result] member = [r[4] for r in result] plt.xlim(0.0, 1.8) plt.ylim(18.0, 8.0) plt.xlabel('B - V') plt.ylabel('V mag') plt.grid() s = ((vstd - np.min(vstd)) / (np.max(vstd) - np.min(vstd))) * 200.0 plt.scatter(bv, vmag, s=s, color='r', alpha=0.75, label='star') members = np.where(member) goods = np.where(good) #print member,members,bv[members] plt.scatter(bv[members], vmag[members], s=s[members], color='b', alpha=0.75, label='member') plt.scatter(bv[goods], vmag[goods], s=s[goods], color='g', alpha=0.75, label='good') plt.savefig(config.plotpath + 'M48_activity_cmd.pdf') #plt.legend() plt.close() @property def maxtab(self): result = self.wifsip.query('SELECT max(tab) from m48_lightcurves;') return result[0][0] + 1
class Photometry(object): ''' Photometry class has the following tasks: * create the db table * optionally clear the values for recalculation * build up a list of stars * collect B and V magnitudes for each star ''' def __init__(self, objname=None, filtercol='V', dbname=None): ''' Constructor ''' self.wifsip = DataSource(database='wifsip', host='pina', user='******') print(objname) if objname is None: # 'M48 BVI' raise (ValueError, 'objname not set') else: self.objname = objname if filtercol in ('B', 'V', 'R', 'I', 'u', 'v', 'b', 'y', 'hbn', 'hbw'): self.filtercol = filtercol else: raise (ValueError, 'unknown filter color') self.frames = [] if dbname is None: # 'M48stars' raise (ValueError, 'tablename not set') else: self.dbname = dbname print('filter %s' % self.filtercol) def createtable(self): ''' create table for the photometry ''' if not input('press Y to erase ' + self.dbname) == 'Y': return query = """ DROP TABLE IF EXISTS %(dbname)s; CREATE TABLE %(dbname)s( starid varchar(25), bv real, vmag real default 0, vmag_err real, bmag real default 0, bmag_err real, period real, period_err real, amp real, amp_err real, nv integer default 0, nb integer default 0, ra double precision, dec double precision, coord point, PRIMARY KEY (starid)); GRANT SELECT ON %(dbname)s TO public; CREATE INDEX idx_%(dbname)s_coords ON %(dbname)s USING GIST (circle(coord,1.0/3600.)); """ % { 'dbname': self.dbname } self.wifsip.execute(query) print("table '%s' created" % self.dbname) def cleartable(self): """ clears photometric values from the table for recalculation """ if not input('press Y to clear ' + self.dbname) == 'Y': return query = """ UPDATE %s SET vmag = 0, vmag_err = NULL, bmag = 0, bmag_err = NULL, nv = 0, nb = 0, bv = NULL; """ % self.dbname self.wifsip.execute(query) print("table '%s' cleared" % self.dbname) def getframes(self, fields=['C', 'NW', 'NE', 'SW', 'SE']): for field in fields: if len(field) > 0: objname = self.objname + ' ' + field else: objname = self.objname query = """SELECT object, objid, abs(corr) FROM frames WHERE object LIKE '%s' AND filter LIKE '%s' AND NOT corr IS NULL ORDER BY abs(corr) limit 5;""" % (objname, self.filtercol) result = self.wifsip.query(query) if len(result) == 0: print('no frames found!') for r in result: print('%s\t%s\t%.3f: ' % r) objid = r[1] self.filltable(objid) self.frames.append(objid) self.wifsip.dropview('phot1') self.update_magnitudes() # print '\n'.join(self.frames) def filltable(self, objid): # get the stars from the phot table ... query = """ SELECT phot.objid ||'#'|| star, mag_auto-corr, alphawin_j2000 , deltawin_j2000 FROM phot,frames WHERE frames.objid='%s' AND phot.objid=frames.objid AND flags<8;""" % (objid) result = self.wifsip.query(query) # ... and inject them into the m48stars stars = len(result) if stars > 400: print('%5d stars: ' % stars, end=' ') oldstars = 0 newstars = 0 for r in result: ostars = self.addstar(r[0], r[1], r[2], r[3]) if ostars == 0: newstars += 1 else: oldstars += 1 print('%5d old , %5d new' % (oldstars, newstars)) self.wifsip.commit() else: print('not enough stars (%d)' % stars) def addstar(self, starid, mag, ra, dec): # identify by coordinates, if the star is already in table query = """SELECT starid FROM %(dbname)s WHERE circle(point(%(ra)f,%(dec)f),0)<@circle(coord,1.0/3600.) ORDER BY point(%(ra)f,%(dec)f)<->coord LIMIT 1;""" % { 'dbname': self.dbname, 'ra': ra, 'dec': dec } result = self.wifsip.query(query) oldstar = 0 # if not: append new star if self.filtercol == 'B': mname, nname = ('bmag', 'nb') elif self.filtercol == 'V': mname, nname = ('vmag', 'nv') if len(result) == 0: oldstar = 0 query = """INSERT INTO %s (starid, %s, %s, ra, dec, coord) VALUES ('%s', %f, 1, %f, %f, point(%f,%f))""" % \ (self.dbname, mname, nname, starid, mag, ra, dec, ra, dec) # if star exists: add up magnitudes, increase counter else: oldstar = 1 oldid = result[0][0] query = """UPDATE %(dbname)s SET %(mname)s = %(mname)s + %(mag)f, %(nname)s = %(nname)s + 1 WHERE starid = '%(oldid)s'; """ % { 'dbname': self.dbname, 'mname': mname, 'nname': nname, 'mag': mag, 'oldid': oldid } self.wifsip.execute(query) return oldstar def update_magnitudes(self): if self.filtercol == 'B': magfield, nfield = 'bmag', 'nb' elif self.filtercol == 'V': magfield, nfield = 'vmag', 'nv' query = """UPDATE %(dbname)s SET %(magfield)s=%(magfield)s/%(nfield)s WHERE %(nfield)s>1; UPDATE %(dbname)s SET %(magfield)s=NULL WHERE %(nfield)s=0;""" % \ {'dbname': self.dbname, 'magfield': magfield, 'nfield': nfield} self.wifsip.execute(query) def update_sigmas(self): """ update the photometric erros after the magnitude has been calculated """ import numpy as np if self.filtercol == 'V': field = 'vmag' elif self.filtercol == 'B': field = 'bmag' query = """SELECT starid, coord FROM %s WHERE (NOT bv IS NULL) AND (%s_err IS NULL);""" % (self.dbname, field) starlist = self.wifsip.query(query) for star in starlist: print('%5d ' % starlist.index(star), end=' ') print('%-24s: %-25s' % star, end=' ') query = """SELECT phot.objid, mag_auto-corr FROM phot, frames WHERE object like '%s %%' AND phot.objid=frames.objid AND filter='%s' AND flags<8 AND point%s <@ circle(phot.coord,1./3600.) ORDER BY abs(corr) LIMIT 5;""" % (self.objname, self.filtercol, star[1]) result = self.wifsip.query(query) mags = np.array([r[1] for r in result]) try: err = np.std(mags) print(mags, end=' ') print('%.3f %.4f' % (np.mean(mags), err)) if np.isfinite(err): query = "UPDATE %s SET %s_err=%f WHERE starid='%s';" % \ (self.dbname, field, err, star[0]) self.wifsip.execute(query) except TypeError: print('no data') def update_bv(self): """ just calculate the B-V """ query = "UPDATE %s SET bv = bmag-vmag;" % self.dbname self.wifsip.execute(query)
def calibrate(self): from datasource import DataSource wifsip = DataSource(host='pina', database='wifsip', user='******') wifsip.execute('UPDATE ngc2281stars SET bv = Bmag-Vmag;') wifsip.close()
class CorrFrames(object): def __init__(self): ''' Constructor ''' from datasource import DataSource self.frames = [] self.ref = {} self.std = {} self.corr = {} self.wifsip = DataSource(host = 'pina', database = 'wifsip', user = '******') def load_frames(self): query = """select objid from frames where filter like 'rp' and object like 'NGC 1647 field 1' and expt = 62.5 and good order by objid""" self.frames = self.wifsip.query(query) def load_ref(self, filename): f = open(filename, 'rt') lines = f.readlines() f.close() for l in lines: key, mag, std = l.split('\t') self.ref[key] = float(mag) self.std[key] = float(std) def load_stars(self, objid): query = """SELECT id, mag_isocor FROM matched, phot WHERE phot.objid like '%s' AND phot.objid = matched.objid AND phot.star = matched.star;""" % objid data = self.wifsip.query(query) stars = {} for d in data: stars[d[0]] = d[1] return stars def update_frame(self, objid, corr): query = """update frames SET corr = %f WHERE objid like '%s';""" % (corr,objid) self.wifsip.execute(query) def correct(self): from numpy import array, average, mean, isfinite n = len(self.frames) for objid in self.frames: i = self.frames.index(objid) mag = self.load_stars(objid) omag = array([mag[key] for key in set(self.ref.keys()) & set(mag.keys())]) cmag = array([self.ref[key] for key in set(self.ref.keys()) & set(mag.keys())]) wts = 1./array([self.std[key] for key in set(self.std.keys()) & set(mag.keys())]) try: corr = average(omag-cmag, weights=wts) except ZeroDivisionError: corr = mean(omag-cmag) if isfinite(corr): print i, n, objid[0], corr self.corr['key'] = corr self.update_frame(objid[0], corr)
class FrameCal(object): ''' classdocs ''' def __init__(self, filter='V'): ''' Constructor ''' from datasource import DataSource self.filter = filter self.frames = [] self.slopes = {} self.intercepts = {} self.r_values = {} self.p_values = {} self.std_errs = {} self.wifsip = DataSource(host = 'pina', database = 'wifsip', user = '******') #self.frameslist() #self.calframes() def frameslist(self): ''' Get the list of frames in a certain filter ''' query = """SELECT objid, stars FROM frames WHERE object LIKE 'NGC 2281 BVI %%' AND filter LIKE '%s';""" % self.filter result = self.wifsip.query(query) self.frames = [r[0] for r in result] def calframe(self, frame): if frame in self.slopes: return (self.slopes[frame], self.intercepts[frame], self.r_values[frame], self.p_values[frame], self.std_errs[frame]) else: raise KeyError('objid %s not found' % frame) return (1.0, 0.0, 0.0, 0.0, 0.0) def updateframe(self, objid): print objid query = "UPDATE frames SET good=TRUE WHERE objid = '%s';" % objid self.wifsip.execute(query) def calframes(self, silent=True): import numpy as np from scipy import stats if self.filter == 'V': mag = 'vmag' if self.filter == 'B': mag = 'bmag' for frame in self.frames: query = """SELECT phot.mag_auto, %s FROM frames, phot, ngc2281ref WHERE frames.objid like '%s' AND frames.objid=phot.objid AND frames.filter='%s' AND circle(ngc2281ref.coord,3./3600.) @> circle(phot.coord,.0) AND bv>0.0;""" % (mag, frame, self.filter) result = self.wifsip.query(query) omags = np.array([s[0] for s in result]) cmags = np.array([s[1] for s in result]) slope, intercept, r_value, p_value, std_err = stats.linregress(omags, cmags) self.slopes[frame] = slope self.intercepts[frame] = intercept self.r_values[frame] = r_value self.p_values[frame] = p_value self.std_errs[frame] = std_err if std_err < 0.02: self.updateframe(frame) if not silent: print '%s\t%.3f\t%6.3f\t%.3f\t%.4f\t%.3f\t%3d' % \ (frame, slope, intercept, r_value, p_value, std_err, len(omags)) print len(self.frames), ' frames calibrated'