def getphot(self, ra, dec, filtercol): from datasource import DataSource wifsip = DataSource(database=config.dbname, user=config.dbuser, host=config.dbhost, dictcursor=True) params = {'filtercol': filtercol, 'ra': ra, 'dec': dec} #'IC 4756 v2 %% uvby' query = """SELECT mag_auto, zeropnt, expt, flux_auto FROM phot, frames WHERE object like 'M 67%%' AND phot.objid = frames.objid AND filter='%(filtercol)s' AND circle(phot.coord,0.0)<@circle(point(%(ra).11f,%(dec).11f),0.6/3600.0) AND flags=0""" % params data = wifsip.query(query) zeropnt = np.array([d['zeropnt'] for d in data]) mag_isocor = np.array([d['mag_auto'] for d in data]) expt = np.array([d['expt'] for d in data]) flux_auto = np.array([d['flux_auto'] for d in data]) if len(mag_isocor) == 0: return np.nan, np.nan, np.nan #mags = mag_isocor-zeropnt mags = -2.5 * np.log10(flux_auto / expt) valid = abs(mags - np.mean(mags)) < 2.0 * np.std(mags) mags = np.compress(valid, mags) mag = np.mean(mags) return mag, np.std(mags), len(mags)
def __init__(self, param): ''' Constructor: initialize the database and query it according the parameter, if a string is given, then we assume, it is the name, if it is a tuple, then we take them as coordinates ''' from datasource import DataSource self.corot = DataSource(database='corot', user='******', host='pina.aip.de') if type(param) is str: try: values = self._byname(param)[0] except IndexError: values = [None,None,None,None,None,None,None,None,None,None] if type(param) is tuple: try: values = self._bycoord(param)[0] except IndexError: values = [None,None,None,None,None,None,None,None,None,None] keys = ['twomass', 'raj2000', 'dej2000', 'jmag', 'e_jmag' ,'hmag' , 'e_hmag', 'kmag' , 'e_kmag', 'coord'] for key, value in zip(keys,values): if key=='coord' and not value is None: # we want to return a tuple vals = value.strip('()').split(',') self[key] = (float(vals[0]),float(vals[1])) else: self[key] = value
def plot_cpd(self): ''' create plot for Heraeus Talk ''' from datasource import DataSource wifsip = DataSource(database='wifsip', user='******', host='oldpina.aip.de') query = """SELECT bv, period FROM m48stars WHERE good;""" data = wifsip.query(query) bv = np.array([d[0] for d in data]) period = np.array([d[1] for d in data]) import gyroage from functions import logspace bv360 = logspace(0.5, 2.0, num=100) #P = gyroage.gyroperiod(bv360, 360.0, version=2007) P, pc = gyroage.gyroperiod(bv360, 360.0, version=2003) plt.plot(bv360, pc, color='b', linestyle='--') plt.plot(bv360, P, color='r') plt.scatter(bv-self.ebv, period, s=1, edgecolor='none', c='k') plt.xlabel('(B - V)$_0$') plt.ylabel('period [days]') plt.ylim(0.0, 20.0) plt.xlim(0.0, 2.0) plt.grid() plt.savefig('/home/jwe/Documents/Talks/Heraeus2015/m48cpd.eps') plt.savefig('/home/jwe/Documents/Talks/Heraeus2015/m48cpd.pdf') plt.close()
def lightcurve_fromdb(self, starid): """ extract a single lightcurve from the database and return epoch (hjd), magnitude and error """ from datasource import DataSource import numpy as np import log wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') query = """SELECT frames.hjd, phot.mag_auto, phot.magerr_auto, phot.flags FROM frames, matched, phot WHERE id LIKE '%s' AND filter LIKE 'rp' AND frames.objid = matched.objid AND (phot.objid,phot.star) = (matched.objid,matched.star) ORDER BY hjd;""" % (starid) # AND hjd>2455473.5 AND hjd<2455477 AND frames.good log.log('/work1/jwe/NGC1647/analysis.log', 'fetching star %s' % starid) # AND frames.good # AND hjd>2455470 AND hjd<2455510 data = wifsip.query(query) wifsip.close() hjd = np.array([d[0] for d in data]) mag = np.array([d[1] for d in data]) err = np.array([d[2] for d in data]) return (hjd, mag, err)
def _fromdatabase(self): """ import the table from a database """ from datasource import DataSource from astropy.coordinates import SkyCoord # @UnresolvedImport from astropy import units as u self.wifsip = DataSource(database=config.dbname, user=config.dbuser, host=config.dbhost) self.stars = [] columns = self.wifsip.columns('ngc6633') params = {'columns': ', '.join(columns), 'key': columns[0]} query = "SELECT %(columns)s FROM ngc6633 ORDER BY %(key)s;" % params self._stars = self.wifsip.query(query) columns = self.wifsip.columns('ngc6633') data_types = self.wifsip.data_types('ngc6633') arraydata = [] c = columns.index('coord') for star in self._stars: starlist = list(star) coord = star[c] ra, dec = coord.strip('()').split(',') newcoord = SkyCoord(ra=float(ra)*u.deg, dec=float(dec)*u.deg) starlist[c] = newcoord arraydata.append(tuple(starlist)) dtype = zip(columns, data_types) #print dtype self.stars = np.array(arraydata, dtype = dtype)
def __init__(self): self.stock_cache = {} self.datasource = DataSource() # We will use this to pass data within the class as data can be huge # and we do not want data passing overhead self.stockdata = None
def make_cmd(self): import pylab as plt import numpy as np from datasource import DataSource wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') query = "SELECT vmag, bv FROM ngc2281stars where vmag>0.0 and bv>0.0;" data = wifsip.query(query) wifsip.close() vmag = np.array([d[0] for d in data]) bv = np.array([d[1] for d in data]) plt.scatter(bv - self.ebv, vmag, edgecolor='none', alpha=0.75) k = (21.0 - 10.5) / (1.8 - 0.0) d = 10.5 x = np.linspace(0.0, 2.5, 10) y = k * x + d plt.plot(x, y, linestyle='dashed', color='k') plt.ylim(21.0, 10.0) plt.xlim(0.0, 2.2) plt.title('NGC 2281') plt.xlabel('(B - V)$_0$') plt.ylabel('V [mag]') plt.grid() plt.savefig('/work1/jwe/Dropbox/NGC2281/plots/ngc2281cmd.pdf') #plt.show() plt.close()
def __init__(self, refframe): self.ref = {} self.refframe = refframe self.wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') self.offset = {}
def __init__(self): self.__on = True self.__actions = [ self.__add_person, self.__get_all_persons, self.__add_animal, self.__get_all_animals, self.__quit ] self.__datasource = DataSource()
def lightcurve_fromdb(self): """ extract a single lightcurve from the database and return epoch (hjd), magnitude and error """ from datasource import DataSource import numpy as np wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') query = """SELECT frames.hjd, phot.mag_isocor, phot.magerr_isocor FROM frames, matched, phot WHERE id LIKE '%s' AND filter LIKE 'rp' AND frames.objid = matched.objid AND (phot.objid,phot.star) = (matched.objid,matched.star) ORDER BY hjd;""" % (self.id) # AND hjd>2455473.5 AND hjd<2455477 AND frames.good # AND frames.good # AND hjd>2455470 AND hjd<2455510 data = wifsip.query(query) wifsip.close() self.hjd = np.array([d[0] for d in data]) self.mag = np.array([d[1] for d in data]) self.err = np.array([d[2] for d in data]) self.hjd -= self.hjd[0]
class JobState(object): url_lst = [] years = [2017] datasource = DataSource() def url_job(self): for year in self.years: self.cursor.execute("SELECT code FROM `sel_trade_port`") port_codes = self.cursor.fetchall() for port_code in port_codes: self.cursor.execute( "SELECT code FROM `goods_code` where code_length = 2 and year_id = %s" % year) goods_codes = self.cursor.fetchall() for good_code in goods_codes: ie_types = [0, 1] for ie_type in ie_types: self.do_process(ie_type, year, port_code[0], good_code[0]) self.datasource.insertUrlsBatch('port', self.url_lst, year) self.url_lst = [] def do_process(self, ie_type, year, port_code, goods_code): url = 'pageNum=1&pageSize=9999&iEType' \ '=%s¤cyType=rmb&year=%s&startMonth=1&endMonth=12&monthFlag=1&codeTsFlag=true' \ '&codeLength=2&outerField1=CODE_TS&outerField2=&outerField3' \ '=TRADE_CO_PORT&outerField4=&outerValue1=%s&outerValue2=&outerValue3=%s&outerValue4' \ '=&orderType=PRICE+DESC&historyTable=true' % (ie_type, year, goods_code, port_code) item = (url, ie_type) self.url_lst.append(item)
def __init__(self): ''' Constructor ''' from datasource import DataSource table = DataSource(database='stella', user='******', host='pera.aip.de', dictcursor=True) columns = ['starid', 'ra', 'dec', '"b-y"', 'm1', 'c1', 'beta'] query = "SELECT %s FROM referencestars order by starid;" % ', '.join( columns) data = table.query(query) columns[3] = 'b-y' d0 = data[0] #for c in columns: print c dtypes = ['|S11', 'f4', 'f4', 'f4', 'f4', 'f4', 'f4'] for c in columns: dtypes.append(type(d0[c])) arraydata = [] for star in data: arraydata.append(tuple(star)) self.stars = np.array(arraydata, dtype=zip(columns, dtypes))
def make_cmd(): import pylab as plt import numpy as np from datasource import DataSource wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') query = "UPDATE m48stars SET bv = (bmag-vmag)+(0.761837-0.060177);" wifsip.execute(query) query = "SELECT vmag+0.060177, bv FROM m48stars WHERE NOT bv is NULL and nv>1 and nb>1;" data = wifsip.query(query) wifsip.close() vmag = np.array([d[0] for d in data]) bv = np.array([d[1] for d in data]) plt.scatter(bv, vmag, edgecolor='none', alpha=0.75, s=4, c='k') plt.ylim(22.0, 7.0) plt.xlim(-0.2, 2.0) plt.xlabel('B - V') plt.ylabel('V [mag]') plt.title('M 48') plt.grid() plt.savefig('/work2/jwe/m48/m48cmd.pdf') #plt.show() plt.close()
def make_cmd(show=False): import numpy as np import matplotlib.pyplot as plt from datasource import DataSource wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') query = "SELECT vmag, bmag FROM ngc6940 WHERE vmag>0.0 and bmag>0.0;" data = wifsip.query(query) vmag = np.array([d[0] for d in data]) * 1.0042005546775856 + 0.24536565071778343 bmag = np.array([d[1] for d in data]) * 1.0017849466111253 + 1.3145083952754286 bv = bmag - vmag plt.scatter(bv - 0.214, vmag, edgecolor='none', alpha=0.5, s=2, c='k') plt.axvline(0.653, linestyle='--', color='y') plt.title('NGC6940 Color Magnitude Diagram E(B-V)=0.214') plt.ylim(21.0, 10.0) plt.xlim(0.0, 1.6) plt.xlabel('(B - V)$_0$') plt.ylabel('V [mag]') plt.grid() plt.minorticks_on() if show: plt.show() else: plt.savefig(config.resultpath + 'ngc6940cmd.eps') plt.savefig(config.resultpath + 'ngc6940cmd.pdf') plt.close()
def __init__(self, field_name = 'M67field'): ''' Constructor ''' self.field_name = field_name self.wifsip = DataSource(database=config.dbname, user=config.dbuser, host=config.dbhost) kpno = ephem.Observer() #31.958036,-111.600578 kpno.lon, kpno.lat = '-111.600578', '31.958036' kpno.horizon = '-0:34' kpno.elevation = 2096 tzi = pytz.timezone('MST') #fmt = '%Y-%m-%d %H:%M:%S %Z%z' obsdate = datetime.datetime(2015,2,10,4,0,0, tzinfo=tzi) kpno.date = obsdate + datetime.timedelta(7*ephem.hour) d = kpno.date.datetime() self.input_epoch = 2000 self.current_epoch = d.year + d.timetuple().tm_yday/365. # LST at mid-exposure in decimal hours lst = kpno.sidereal_time() print('local siderial time: ', lst) self.siderial_time = ast.hms2hh(str(lst)) self.exposure_length = 40.0/60.0 # in hours self.wavelength = 5125 # in Angstroem self.cable = 'BLUE' self.weighting = 'STRONG' self.guidewave = 6000 self.center = self._get_center() c = Cluster('NGC 2682') self.ebv = c['ebv'] self.dm = ast.distance_modulus(c['d']) target = {} self.table = [] target['id'] = 6000 target['name'] = 'M67center' target['ra'] = self.center[0] target['dec'] = self.center[1] target['class'] = 'C' self.table.append(target) self.targeted = [] # number of sky fibers self.skies = 6 # number of field orientation probes self.fops = 6 self.tweakcenter = False self.tweakangle = False print('center', self.center) print('E(B-V)', self.ebv) print('DM', self.dm)
def __init__(self, day=today('%Y_%m_%d'), settings=__import__('settings', level=0), mapping=__import__('mapping', level=0), output=Queue.Queue()): threading.Thread.__init__(self) self.day = day self.settings = settings self.mapping = mapping self.output = output self.datasource = DataSource(settings) self.config = Config('reader-%s.cfg' % day) self.deadline = str2timestamp(day_offset(day_str=self.day, offset=7, format='%Y_%m_%d'), format='%Y_%m_%d') self.stopped = False self.current_index = int(self.config.get('current_index', 0)) self.now = self.config.get('now') logging.info("Reader-%s init done!" % self.day)
def analysisresults(): if request.method == 'POST': ds = DataSource('hayesrichn', 'orange227blue') dataset1 = request.form.get('dataset1') datatype1 = request.form.get('datatype1') dataset2 = request.form.get('dataset2') datatype2 = request.form.get('datatype2') regression_data = regression(dataset1, dataset2, datatype1, datatype2, ds) returnhtml = "<h2>ANALYSIS RESULT</h2>" returnhtml = flask.Markup(returnhtml) return render_template('analysis.html', result=regression_data[0].decode('utf8'), predicted_value=regression_data[1], predictor_value=regression_data[2], datatype1=datatype1, dataset1=dataset1, datatype2=datatype2, dataset2=dataset2, returndata=reversed(regression_data[3]), returnhtml=returnhtml) else: returnhtml = "<h2>Query Failed</h2>" returnhtml = flask.Markup(returnhtml) return render_template('results.html', returnhtml=returnhtml)
def __init__(self, objname=None, filtercol='V', dbname=None): ''' Constructor ''' self.wifsip = DataSource(database='wifsip', host='pina', user='******') print(objname) if objname is None: # 'M48 BVI' raise (ValueError, 'objname not set') else: self.objname = objname if filtercol in ('B', 'V', 'R', 'I', 'u', 'v', 'b', 'y', 'hbn', 'hbw'): self.filtercol = filtercol else: raise (ValueError, 'unknown filter color') self.frames = [] if dbname is None: # 'M48stars' raise (ValueError, 'tablename not set') else: self.dbname = dbname print('filter %s' % self.filtercol)
def __init__(self, field, datapath, lightcurvepath, plotpath, filtercol='V', flaglimit=4, starslimit=10000, dettemp=-110.0): ''' Constructor ''' from datasource import DataSource self.wifsip = DataSource(database='stella', host='pera', user='******') self.coords = {} self.objids = [] self.field = field self.filtercol = filtercol self.flaglimit = flaglimit self.starslimit = starslimit self.dettemp = dettemp self.filename = datapath + self.field self.lightcurvepath = lightcurvepath self.plotpath = plotpath print(self.field) print('filter: %s' % self.filtercol) print('maxstars: %d' % self.starslimit) print('dettemp: <%.1f' % self.dettemp)
def test_initialize_column_rename(self): df = pd.read_csv('sample.csv', parse_dates=['timestamp']) df.columns = ['TS', 'TRADINGSYMBOL', 'OPEN', 'HIGH', 'LOW', 'CLOSE', 'VOLUME', 'PREVCLOSE'] self.ds = DataSource(data=df, timestamp='TS', symbol='TRADINGSYMBOL') self.assertEqual(self.ds.data.columns[0], 'timestamp') self.assertEqual(self.ds.data.columns[1], 'symbol')
def doimport(self): '''import the file and shove it into the database''' from datasource import DataSource from astronomy import hms2dd, dms2dd from StringIO import StringIO f = open(self.filename, 'r') lines = f.readlines() f.close() # skip header information and compendium lines = lines[71:247] print lines[0] values = '' for l in lines: id = l[0:9].strip() ra = hms2dd(l[9:19].strip()) dec = dms2dd(l[20:31].strip()) s = l[32:33] try: ucac2 = int(l[34:42]) except ValueError: ucac2 = None tyc2 = l[43:54] v = float(l[56:61]) remarks = l[62:].rstrip() print id,ra,dec,s,ucac2,tyc2,v,remarks values += '%s\t%f\t%f\t%s\t%s\t%s\t%f\t%s\n' % (id,ra,dec,s,ucac2,tyc2,v,remarks) values = values.replace('None', '\N') f = StringIO(values) print values wifsip = DataSource(host = 'pina', database = 'wifsip', user = '******') wifsip.cursor.copy_from(f, 'ngc2281ref', columns=('id','ra','dec','s','ucac2','tyc2','vmag', 'remarks')) wifsip.commit() wifsip.close()
def make_cmd(self): import pylab as plt import numpy as np from datasource import DataSource wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') query = "SELECT vmag, bv FROM ngc1647stars;" data = wifsip.query(query) wifsip.close() vmag = np.array([d[0] for d in data]) bv = np.array([d[1] for d in data]) plt.scatter(bv, vmag, edgecolor='none', alpha=0.75) k = 8 / (2.0 - 0.32) d = 20 - 2.0 * k x = np.linspace(0.0, 2.5, 10) y = k * x + d plt.plot(x, y, linestyle='dashed', color='k') plt.ylim(21.0, 8.0) plt.xlim(0.0, 2.2) plt.xlabel('B - V') plt.ylabel('V [mag]') plt.grid() plt.savefig('/work1/jwe/Dropbox/NGC1647/plots/ngc1647cmd.pdf') #plt.show() plt.close()
def __init__(self): ''' Constructor ''' from datasource import DataSource self.landolt = DataSource(database='stella', host='pera', user='******')
def __init__(self): ''' Constructor ''' from datasource import DataSource self.table = DataSource(database='stella', user='******', host='pera.aip.de') self.data = []
def __init__(self, field_name = 'M48field'): ''' Constructor ''' import ephem import datetime import pytz import astronomy as ast from datasource import DataSource self.field_name = field_name self.wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de') kpno = ephem.Observer() #31.958036,-111.600578 kpno.lon, kpno.lat = '-111.600578', '31.958036' kpno.horizon = '-0:34' kpno.elevation = 2096 tzi = pytz.timezone('MST') #fmt = '%Y-%m-%d %H:%M:%S %Z%z' obsdate = datetime.datetime(2015,2,9,23,0,0, tzinfo=tzi) kpno.date = obsdate + datetime.timedelta(7*ephem.hour) d = kpno.date.datetime() self.input_epoch = 2000 self.current_epoch = d.year + d.timetuple().tm_yday/365. # LST at mid-exposure in decimal hours lst = kpno.sidereal_time() print 'local siderial time: ',lst self.siderial_time = ast.hms2hh(str(lst)) self.exposure_length = 40.0/60.0 # in hours self.wavelength = 6000 # in Angstroem self.cable = 'BLUE' self.weighting = 'STRONG' self.guidewave = 6000 self.center = self._get_center() self.ebv = 0.031 # from Webda self.dm = 9.53 # from Webda target = {} self.table = [] target['id'] = 6000 target['name'] = 'M48center' target['ra'] = self.center[0] target['dec'] = self.center[1] target['class'] = 'C' self.table.append(target) self.targeted = [] # number of sky fibers self.skies = 6 # number of field orientation probes self.fops = 6 self.tweakcenter = False self.tweakangle = False
def buildtable(self): """ builds the table of stars """ import numpy as np epochs = len(self.objids) stars = len(self.stars) if fileexists('/work2/jwe/NGC2281/' + self.filter + 'array.npy'): m = np.load('/work2/jwe/NGC2281/' + self.filter + 'array.npy') else: from datasource import DataSource from framecal import FrameCal fc = FrameCal(self.filter) m = np.zeros([epochs, stars]) # objid is specific to a filter so we only need to query the objid wifsip = DataSource(host='pina', database='wifsip', user='******') for objid in self.objids: k = self.objids.index(objid) print k, epochs, objid, query = """SELECT matched.id, phot.mag_auto, phot.mag_errauto FROM phot, matched WHERE phot.objid like '%s' AND (matched.objid,matched.star) = (phot.objid,phot.star) AND phot.flags = 0;""" % objid result = wifsip.query(query) starids = [s[0] for s in result] mags = [s[1] for s in result] err = [s[2] for s in result] slope, intercept, _, _, _ = fc.calframe(objid) print len(mags) for starid in starids: i = self.stars.index(starid) m[k, i] = mags[starids.index(starid)] * slope + intercept np.save('/work2/jwe/NGC2281/' + self.filter + 'array.npy', m) wifsip.close() i = np.where(m == 0.0) m[i] = np.nan from scipy import stats # calculate the observed average for the stars avg = stats.nanmean(m, axis=0) for k in range(epochs): print k, epochs, self.objids[k] # calculate the mean of offsets off = stats.nanmedian(m[k, :] - avg) # correct epoch for mean of offsets m[k, :] += off # calculate new corrected means avg = stats.nanmean(m, axis=0) std = stats.nanstd(m, axis=0) for i in range(len(self.stars)): print self.stars[i], avg[i], std[i]
def __init__(self, util=None, standalone=False): """ Initializer :param util: utility object :param standalone: True - standalone version, False - part of Peppy player """ ScreensaverMeter.__init__(self) if util: self.util = util else: self.util = MeterUtil() use_vu_meter = getattr(self.util, USE_VU_METER, None) self.name = "peppymeter" base_path = "." if __package__: pkg_parts = __package__.split(".") if len(pkg_parts) > 0: base_path = os.path.join(os.getcwd(), "screensaver", self.name) parser = ConfigFileParser(base_path) self.util.meter_config = parser.meter_config self.outputs = {} if standalone: if self.util.meter_config[USE_LOGGING]: logging.basicConfig(level=logging.NOTSET) else: logging.disable(logging.CRITICAL) # no VU Meter support for Windows if "win" in sys.platform and self.util.meter_config[DATA_SOURCE][ TYPE] == SOURCE_PIPE: self.util.meter_config[DATA_SOURCE][TYPE] = SOURCE_NOISE self.data_source = DataSource(self.util) if self.util.meter_config[DATA_SOURCE][ TYPE] == SOURCE_PIPE or use_vu_meter == True: self.data_source.start_data_source() if self.util.meter_config[OUTPUT_DISPLAY]: self.meter = self.output_display(self.data_source) if self.util.meter_config[OUTPUT_SERIAL]: self.outputs[OUTPUT_SERIAL] = SerialInterface( self.util.meter_config, self.data_source) if self.util.meter_config[OUTPUT_I2C]: self.outputs[OUTPUT_I2C] = I2CInterface(self.util.meter_config, self.data_source) if self.util.meter_config[OUTPUT_PWM]: self.outputs[OUTPUT_PWM] = PWMInterface(self.util.meter_config, self.data_source) self.start_interface_outputs()
def __init__(self): ''' Constructor ''' from datasource import DataSource self.wifsip = DataSource(database='wifsip', user='******', host='pina.aip.de')
def __init__(self): ''' Constructor ''' self.data = [] from datasource import DataSource self.wifsip = DataSource(database=config.dbname, user=config.dbuser, host=config.dbhost)
def __init__(self): ''' Constructor ''' from datasource import DataSource self.frames = [] self.ref = {} self.std = {} self.corr = {} self.wifsip = DataSource(host = 'pina', database = 'wifsip', user = '******')