def setUp(self): """ create an in memory DB and open a connection to it """ filename = os.path.dirname(__file__) + '/../configs/test.cfg' self.dbi = DataBaseInterface(test=True, configfile=filename) self.session = self.dbi.Session() self.jd = 2456892.20012000 self.pol = 'xx' self.filename = '/data0/zen.2456785.123456.uv' self.host = 'pot0' self.length = 10.16639 / 60. / 24
def setUp(self): """ create an in memory DB and open a connection to it """ filename=os.path.dirname(__file__)+'/../configs/test.cfg' self.dbi = DataBaseInterface(test=True,configfile=filename) self.session = self.dbi.Session() self.jd = 2456892.20012000 self.pol = 'xx' self.filename='/data0/zen.2456785.123456.uv' self.host = 'pot0' self.length = 10.16639/60./24
o.set_usage('summarize_night.py obsnum') o.set_description(__doc__) #o.add_option('--length',type=float, # help='length of the input observations in minutes [default=average difference between filenames]') o.add_option('-v', action='store_true', help='set log level to debug') o.add_option('--status', default='UV_POT', help='set the observation to this status [default=UV_POT]') opts, args = o.parse_args(sys.argv[1:]) #connect to the database if opts.v: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.WARNING) logger = logging.getLogger('summarize_still') dbi = DataBaseInterface() s = dbi.Session() print "summarizing Distiller" OBSs = s.query(Observation) JDs = [OBS.julian_date for OBS in OBSs] nights = n.sort(list(set(map(int, JDs)))) print "number of nights ingested:", len(nights) Nobs = s.query(Observation).count() Nprogress = s.query(Observation).filter( Observation.status != 'NEW', Observation.status != 'UV_POT', Observation.status != 'NEW', Observation.status != 'COMPLETE').count() Ncomplete = s.query(Observation).filter( Observation.status == 'COMPLETE').count() print "Total observations in still:", Nobs print "Number complete:", Ncomplete
class TestDBI(unittest.TestCase): def setUp(self): """ create an in memory DB and open a connection to it """ filename=os.path.dirname(__file__)+'/../configs/test.cfg' self.dbi = DataBaseInterface(test=True,configfile=filename) self.session = self.dbi.Session() self.jd = 2456892.20012000 self.pol = 'xx' self.filename='/data0/zen.2456785.123456.uv' self.host = 'pot0' self.length = 10.16639/60./24 def test_add_log(self): obsnum = self.dbi.add_observation( self.jd,self.pol,self.filename,self.host,length=self.length) self.dbi.add_log(obsnum,'UV','test123',0) logtext = self.dbi.get_logs(obsnum) self.assertEqual(logtext.strip(),'test123') LOG = self.session.query(Log).filter(Log.obsnum==obsnum).one() def test_update_log(self): obsnum = self.dbi.add_observation( self.jd,self.pol,self.filename,self.host,length=self.length) self.dbi.add_log(obsnum,'UV','test123',None) logtext = self.dbi.get_logs(obsnum) self.assertEqual(logtext.strip(),'test123') LOG = self.session.query(Log).filter(Log.obsnum==obsnum).one() self.dbi.update_log(obsnum,exit_status=0) newsession = self.dbi.Session() LOG = newsession.query(Log).filter(Log.obsnum==obsnum).one() self.assertEqual(LOG.exit_status,0) def test_configparsing(self): logger.info('Note: did you remember to do "cp configs/test.cfg ~/.paperstill/db.cfg" ? ') self.assertEqual(self.dbi.dbinfo['hostip'],'memory') def test_obsnum_increment(self): dt = self.length jds = n.arange(0,10)*dt+self.jd obsnums=[] for jd in jds: obsnums.append(jdpol2obsnum(jd,self.pol,dt)) delta = n.diff(obsnums) for d in delta: self.assertEqual(d,1) obsnum = self.dbi.add_observation(self.jd,self.pol,self.filename,self.host,length=self.length) self.assertEqual(obsnum,jdpol2obsnum(self.jd,self.pol,self.length)) def test_add_observation(self): """ use the dbi to create a record. basically tests the same as test_Observation_and_file but with the dbi wrapper """ obsnum = self.dbi.add_observation( self.jd,self.pol,self.filename,self.host,length=self.length) OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one() self.assertEqual(float(OBS.julian_date),self.jd) self.assertEqual(OBS.obsnum,jdpol2obsnum(self.jd,self.pol,self.length)) def test_add_file(self): """ todo update from code """ #first add the observation obsnum = self.dbi.add_observation(self.jd,self.pol,self.filename,self.host) #then add a file to it filenum = self.dbi.add_file(obsnum,self.host,self.filename+'cRRE') #then grab the file record back FILE = self.session.query(File).filter(File.filenum==filenum).one() #and check that its right self.assertEqual(FILE.filename,self.filename+'cRRE') def test_add_files(self): #first add the observation obsnum = self.dbi.add_observation(self.jd,self.pol,self.filename,self.host,self.length) #add two subsequent products files = ['/data0/zen.2456785.123456.uvc','/data0/zen.2456785.323456.uvcR'] for filename in files: filenum = self.dbi.add_file(obsnum,self.host,filename) #how I get all the files for a given obs OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one() self.assertEqual(len(OBS.files),3)#check that we got three files def test_add_observations(self): #form up the observation list obslist =[] jds = n.arange(0,10)*self.length+2456446.1234 pols = ['xx','yy','xy','yx'] for pol in pols: for jdi in xrange(len(jds)): obslist.append({'julian_date':jds[jdi], 'pol':pol, 'host':self.host, 'filename':self.filename, 'length':self.length}) if jdi!=0: obslist[-1]['neighbor_low'] = jds[jdi-1] if jdi<len(jds[:-1]): obslist[-1]['neighbor_high'] = jds[jdi+1] obsnums = self.dbi.add_observations(obslist) nobs = self.session.query(func.count(Observation.obsnum)).scalar() self.assertEqual(len(obslist),nobs) #did we add observations? #did they get the proper neighbor assignments for obsnum in obsnums: OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one() #find the original record we put into add_observations and check that the neighbors match for obs in obslist: if obs['julian_date']==OBS.julian_date: if obs.has_key('neighbor_low'): self.assertEqual(OBS.low_neighbors[0].julian_date, obs['neighbor_low']) if obs.has_key('neighbor_high'): self.assertEqual(OBS.high_neighbors[0].julian_date, obs['neighbor_high']) break def test_list_observations(self): #form up the observation list obslist =[] jds = n.arange(0,10)*self.length+2456446.1234 pols = ['xx','yy','xy','yx'] for pol in pols: for jdi in xrange(len(jds)): obslist.append({'julian_date':jds[jdi], 'pol':pol, 'host':self.host, 'filename':self.filename, 'length':self.length}) if jdi!=0: obslist[-1]['neighbor_low'] = jds[jdi-1] if jdi<len(jds[:-1]): obslist[-1]['neighbor_high'] = jds[jdi+1] obsnums = self.dbi.add_observations(obslist) tic = time.time() observations = self.dbi.list_observations() #print "time to execute list_observations",time.time()-tic,'s' self.assertEqual(n.sum(n.array(observations)-n.array(obsnums)),0) def test_get_neighbors(self): """ First set up a likely triplet of observations """ #form up the observation list obslist =[] jds = n.arange(0,10)*self.length+2456446.1234 pols = ['xx','yy','xy','yx'] for pol in pols: for jdi in xrange(len(jds)): obslist.append({'julian_date':jds[jdi], 'pol':pol, 'host':self.host, 'filename':self.filename, 'length':self.length}) if jdi!=0: obslist[-1]['neighbor_low'] = jds[jdi-1] if jdi!=(len(jds)-1): obslist[-1]['neighbor_high'] = jds[jdi+1] obsnums = self.dbi.add_observations(obslist) obsnums.sort() i = 5# I have ten time stamps. this guys should have plenty of neighbors mytestobsnum = obsnums[i] #choose a middle obs tic = time.time() neighbors = self.dbi.get_neighbors(mytestobsnum) #print "time to execute get_neighbors",time.time()-tic,'s' self.assertEqual(len(neighbors),2) self.assertEqual(neighbors[0],obsnums[i-1])#low self.assertEqual(neighbors[1],obsnums[i+1])#high def test_set_obs_status(self): """ set the status with the dbi function then check it with under the hood stuff """ #first create an observation in the first place obsnum = self.dbi.add_observation( self.jd,self.pol,self.filename,self.host) # then set the status to something else self.dbi.set_obs_status(obsnum,'UV') # get the status back out OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one() self.assertEqual(OBS.status,'UV') def test_get_obs_status(self): """ set the status with the dbi function (cause I tested it with more basic tests already) """ #first create an observation in the first place obsnum = self.dbi.add_observation( self.jd,self.pol,self.filename,self.host) # then set the status to something else self.dbi.set_obs_status(obsnum,'UV') #then get the status back tic = time.time() status = self.dbi.get_obs_status(obsnum) #print "time to execute get_obs_status",time.time()-tic,'s' self.assertEqual(status,'UV') def test_time_transaction(self): """ """ #first create an observation in the first place obsnum = self.dbi.add_observation( self.jd,self.pol,self.filename,self.host) # then set the status to something else tic = time.time() self.dbi.set_obs_status(obsnum,'UV') print "time to execute set_obs_status",time.time() - tic,'s' def test_get_input_file(self): """ create an observation add the initial file get the pot host and path """ #first add the observation obsnum = self.dbi.add_observation( self.jd,self.pol,self.filename,self.host) #then add a file to it host,path,filename = self.dbi.get_input_file(obsnum) self.assertEqual(host,self.host) self.assertEqual(path,os.path.dirname(self.filename)) self.assertEqual(filename,os.path.basename(self.filename)) def test_get_output_location(self): """ create an observation add the initial file get the pot host and path """ #first add the observation obsnum = self.dbi.add_observation( self.jd,self.pol,self.filename,self.host) host,path = self.dbi.get_output_location(obsnum) self.assertEqual(host,self.host) self.assertEqual(path,os.path.dirname(self.filename)) def test_still_path(self): """ """ obsnum = self.dbi.add_observation( self.jd,self.pol,self.filename,self.host) self.dbi.set_obs_still_path(obsnum,'/data/') still_path = self.dbi.get_obs_still_path(obsnum) self.assertEqual(still_path,'/data/') def test_obs_pid(self): """ """ obsnum = self.dbi.add_observation( self.jd,self.pol,self.filename,self.host) self.dbi.set_obs_pid(obsnum,9999) pid = self.dbi.get_obs_pid(obsnum) self.assertEqual(pid,9999)
o.set_usage('reset_observations.py *.uv') o.set_description(__doc__) #o.add_option('--length',type=float, # help='length of the input observations in minutes [default=average difference between filenames]') o.add_option('-v',action='store_true', help='set log level to debug') o.add_option('--status',default='UV_POT', help='set the observation to this status [default=UV_POT]') opts, args = o.parse_args(sys.argv[1:]) #connect to the database if opts.v: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(lebel=logging.INFO) logger = logging.getLogger('reset_observations') dbi = DataBaseInterface() # for each file get the obsnum, then reset the status to UV_POT obsnums = [] for filename in args: logger.debug("looking for file {filename}".format(filename=filename)) s = dbi.Session() FILE = s.query(File).filter(File.filename==filename).one()#XXX note assumes we are not noting that this file is copied. obsnum = FILE.obsnum logger.debug("found obsnum {obsnum}".format(obsnum=obsnum)) s.close() logger.debug("setting status to {status}".format(status=opts.status)) dbi.set_obs_status(obsnum,opts.status) dbi.set_obs_pid(obsnum,None) dbi.set_obs_still_host(obsnum,None) dbi.add_log(obsnum,'NEW',"issuing a reset_observations",0)
def file2jd(zenuv): return re.findall(r'\d+\.\d+', zenuv)[0] def file2pol(zenuv): return re.findall(r'\.(.{2})\.',zenuv)[0] o = optparse.OptionParser() o.set_usage('add_observations.py *.uv') o.set_description(__doc__) o.add_option('--length',type=float, help='length of the input observations in minutes [default=average difference between filenames]') o.add_option('-t',action='store_true', help='Test. Only print, do not touch db') o.add_option('--overwrite',action='store_true', help='Default action is to skip obsrvations already in the db. Setting this option overrides this safety feature and attempts anyway') opts, args = o.parse_args(sys.argv[1:]) #connect to the database dbi = DataBaseInterface() #check that all files exist for filename in args: assert(filename.startswith('/')) assert(os.path.exists(filename)) #now run through all the files and build the relevant information for the db # get the pols pols = [] jds = [] for filename in args: pols.append(file2pol(filename)) jds.append(float(file2jd(filename))) jds = n.array(jds) nights = list(set(jds.astype(n.int))) if not opts.length is None:
o = optparse.OptionParser() o.set_usage('print_logs.py obsnum') o.set_description(__doc__) #o.add_option('--length',type=float, # help='length of the input observations in minutes [default=average difference between filenames]') o.add_option('-v', action='store_true', help='set log level to debug') o.add_option('--tail', action='store_true', help='Monitor the log for changes') o.add_option('--logtxt', action='store_true', help='Print the full log text.') opts, args = o.parse_args(sys.argv[1:]) #connect to the database if opts.v: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(lebel=logging.INFO) logger = logging.getLogger('monitor_log') dbi = DataBaseInterface() # for each file get the obsnum, then reset the status to UV_POT obsnums = [] for arg in args: if arg[0] == '/': #we have a file! logger.debug("looking for file {filename}".format(filename=arg)) s = dbi.Session() File = s.query(File).filter(File.filename == arg).one( ) #XXX note assumes we are not noting that this file is copied. obsnum = File.obsnum logger.debug("found obsnum {obsnum}".format(obsnum=obsnum)) s.close() obsnums.append(obsnum) else: obsnums.append(arg) #it must be an obsnum
o.set_usage('summarize_night.py obsnum') o.set_description(__doc__) #o.add_option('--length',type=float, # help='length of the input observations in minutes [default=average difference between filenames]') o.add_option('-v', action='store_true', help='set log level to debug') o.add_option('--status', default='UV_POT', help='set the observation to this status [default=UV_POT]') opts, args = o.parse_args(sys.argv[1:]) #connect to the database if opts.v: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) logger = logging.getLogger('reset_observations') dbi = DataBaseInterface() s = dbi.Session() OBSs = s.query(Observation).filter( Observation.julian_date.between(float(args[0]), float(args[0]) + 0.9999)).order_by( Observation.obsnum.desc()).all() obsnums = [OBS.obsnum for OBS in OBSs] still_times = {} log_time_range = [datetime(3001, 1, 1), datetime(1, 1, 1)] for i, OBS in enumerate(OBSs): obsnum = OBS.obsnum if OBS.stillhost is None: continue print '\t'.join( map(str, [obsnum, OBS.stillhost,
type=float, help= 'length of the input observations in minutes [default=average difference between filenames]' ) o.add_option('-t', action='store_true', help='Test. Only print, do not touch db') o.add_option( '--overwrite', action='store_true', help= 'Default action is to skip obsrvations already in the db. Setting this option overrides this safety feature and attempts anyway' ) opts, args = o.parse_args(sys.argv[1:]) #connect to the database dbi = DataBaseInterface() #check that all files exist for filename in args: assert (filename.startswith('/')) assert (os.path.exists(filename)) #now run through all the files and build the relevant information for the db # get the pols pols = [] jds = [] for filename in args: pols.append(file2pol(filename)) jds.append(float(file2jd(filename))) jds = n.array(jds) nights = list(set(jds.astype(n.int))) if not opts.length is None:
o.set_usage('print_logs.py obsnum') o.set_description(__doc__) #o.add_option('--length',type=float, # help='length of the input observations in minutes [default=average difference between filenames]') o.add_option('-v',action='store_true', help='set log level to debug') o.add_option('--status',default='UV_POT', help='set the observation to this status [default=UV_POT]') opts, args = o.parse_args(sys.argv[1:]) #connect to the database if opts.v: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(lebel=logging.INFO) logger = logging.getLogger('reset_observations') dbi = DataBaseInterface() # for each file get the obsnum, then reset the status to UV_POT obsnums = [] for arg in args: if arg[0]=='/': #we have a file! logger.debug("looking for file {filename}".format(filename=arg)) s = dbi.Session() File = s.query(File).filter(File.filename==arg).one()#XXX note assumes we are not noting that this file is copied. obsnum = File.obsnum logger.debug("found obsnum {obsnum}".format(obsnum=obsnum)) s.close() obsnums.append(obsnum) else: obsnums.append(arg)#it must be an obsnum logger.debug("found %d obsnums"%len(obsnums))
def __init__(self, nobs, npols, test=True): DataBaseInterface.__init__(self, test=test) self.length = 10 / 60. / 24 self.host = 'pot0' self.defaultstatus = 'UV_POT' self.Add_Fake_Observations(nobs, npols)
from ddr_compress.dbi import DataBaseInterface,gethostname import optparse,os,sys,re,numpy as n def file2jd(zenuv): return re.findall(r'\d+\.\d+', zenuv)[0] def file2pol(zenuv): return re.findall(r'\.(.{2})\.',zenuv)[0] o = optparse.OptionParser() o.set_usage('add_observations.py *.uv') o.set_description(__doc__) o.add_option('--length',type=float, help='length of the input observations in minutes [default=average difference between filenames]') opts, args = o.parse_args(sys.argv[1:]) #connect to the database dbi = DataBaseInterface() #check that all files exist for filename in args: assert(filename.startswith('/')) assert(os.path.exists(filename)) #now run through all the files and build the relevant information for the db # get the pols pols = [] jds = [] for filename in args: pols.append(file2pol(filename)) jds.append(float(file2jd(filename))) jds = n.array(jds) if not opts.length is None: djd = opts.length/60./24
def __init__(self,nobs,npols,test=True): DataBaseInterface.__init__(self,test=test) self.length = 10/60./24 self.host = 'pot0' self.defaultstatus='UV_POT' self.Add_Fake_Observations(nobs,npols)
#! /usr/bin/env python import optparse,sys,os from ddr_compress.dbi import DataBaseInterface #open a connection to the db dbi = DataBaseInterface() #check that the tables
o.set_usage('print_logs.py obsnum') o.set_description(__doc__) #o.add_option('--length',type=float, # help='length of the input observations in minutes [default=average difference between filenames]') o.add_option('-v', action='store_true', help='set log level to debug') o.add_option('--status', default='UV_POT', help='set the observation to this status [default=UV_POT]') opts, args = o.parse_args(sys.argv[1:]) #connect to the database if opts.v: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(lebel=logging.INFO) logger = logging.getLogger('reset_observations') dbi = DataBaseInterface() # for each file get the obsnum, then reset the status to UV_POT obsnums = [] for arg in args: if arg[0] == '/': #we have a file! logger.debug("looking for file {filename}".format(filename=arg)) s = dbi.Session() File = s.query(File).filter(File.filename == arg).one( ) #XXX note assumes we are not noting that this file is copied. obsnum = File.obsnum logger.debug("found obsnum {obsnum}".format(obsnum=obsnum)) s.close() obsnums.append(obsnum) else: obsnums.append(arg) #it must be an obsnum
#! /usr/bin/env python from ddr_compress.dbi import DataBaseInterface,Observation,File from sqlalchemy import func import curses,time,os,sys import numpy as n #setup my curses stuff following # https://docs.python.org/2/howto/curses.html stdscr = curses.initscr() curses.noecho() curses.cbreak() stdscr.keypad(1) stdscr.nodelay(1) #setup my db connection dbi = DataBaseInterface() stdscr.addstr("PAPER Distiller Status Board. Monitoring : {dbname}".format(dbname=dbi.dbinfo['dbname'])) stdscr.addstr(1,0,"Press 'q' to exit") statheight = 50 statusscr = curses.newwin(statheight,400,5,0) statusscr.keypad(1) statusscr.nodelay(1) curline = 2 colwidth = 50 obslines = 20 stat = ['\\','|','/','-','.'] i =0 try: while(1): #get the screen dimensions
#! /usr/bin/env python from ddr_compress.dbi import DataBaseInterface,Observation,File from sqlalchemy import func import curses,time,os #setup my curses stuff following # https://docs.python.org/2/howto/curses.html stdscr = curses.initscr() curses.noecho() curses.cbreak() stdscr.keypad(1) stdscr.nodelay(1) #setup my db connection dbi = DataBaseInterface() stdscr.addstr("PAPER Distiller Status Board. Monitoring: {dbname}".format(dbname=dbi.dbinfo['dbname'])) stdscr.addstr(1,0,"Press 'q' to exit") statheight = 50 statusscr = curses.newwin(statheight,400,5,0) statusscr.keypad(1) statusscr.nodelay(1) curline = 2 colwidth = 40 obslines = 20 i=0 stat = ['\\','|','/','-','.'] try: while(1): #get the screen dimensions
class TestDBI(unittest.TestCase): def setUp(self): """ create an in memory DB and open a connection to it """ filename = os.path.dirname(__file__) + '/../configs/test.cfg' self.dbi = DataBaseInterface(test=True, configfile=filename) self.session = self.dbi.Session() self.jd = 2456892.20012000 self.pol = 'xx' self.filename = '/data0/zen.2456785.123456.uv' self.host = 'pot0' self.length = 10.16639 / 60. / 24 def test_logging(self): obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host, length=self.length) self.dbi.add_log(obsnum, 'UV', 'test123', 0) logtext = self.dbi.get_logs(obsnum) self.assertEqual(logtext.strip(), 'test123') LOG = self.session.query(Log).filter(Log.obsnum == obsnum).one() print LOG.timestamp def test_configparsing(self): logger.info( 'Note: did you remember to do "cp configs/test.cfg ~/.paperstill/db.cfg" ? ' ) self.assertEqual(self.dbi.dbinfo['hostip'], 'memory') def test_obsnum_increment(self): dt = self.length jds = n.arange(0, 10) * dt + self.jd obsnums = [] for jd in jds: obsnums.append(jdpol2obsnum(jd, self.pol, dt)) delta = n.diff(obsnums) for d in delta: self.assertEqual(d, 1) obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host, length=self.length) self.assertEqual(obsnum, jdpol2obsnum(self.jd, self.pol, self.length)) def test_add_observation(self): """ use the dbi to create a record. basically tests the same as test_Observation_and_file but with the dbi wrapper """ obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host, length=self.length) OBS = self.session.query(Observation).filter( Observation.obsnum == obsnum).one() self.assertEqual(float(OBS.julian_date), self.jd) self.assertEqual(OBS.obsnum, jdpol2obsnum(self.jd, self.pol, self.length)) def test_add_file(self): """ todo update from code """ #first add the observation obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host) #then add a file to it filenum = self.dbi.add_file(obsnum, self.host, self.filename + 'cRRE') #then grab the file record back FILE = self.session.query(File).filter(File.filenum == filenum).one() #and check that its right self.assertEqual(FILE.filename, self.filename + 'cRRE') def test_add_files(self): #first add the observation obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host, self.length) #add two subsequent products files = [ '/data0/zen.2456785.123456.uvc', '/data0/zen.2456785.323456.uvcR' ] for filename in files: filenum = self.dbi.add_file(obsnum, self.host, filename) #how I get all the files for a given obs OBS = self.session.query(Observation).filter( Observation.obsnum == obsnum).one() self.assertEqual(len(OBS.files), 3) #check that we got three files def test_add_observations(self): #form up the observation list obslist = [] jds = n.arange(0, 10) * self.length + 2456446.1234 pols = ['xx', 'yy', 'xy', 'yx'] for pol in pols: for jdi in xrange(len(jds)): obslist.append({ 'julian_date': jds[jdi], 'pol': pol, 'host': self.host, 'filename': self.filename, 'length': self.length }) if jdi != 0: obslist[-1]['neighbor_low'] = jds[jdi - 1] if jdi < len(jds[:-1]): obslist[-1]['neighbor_high'] = jds[jdi + 1] obsnums = self.dbi.add_observations(obslist) nobs = self.session.query(func.count(Observation.obsnum)).scalar() self.assertEqual(len(obslist), nobs) #did we add observations? #did they get the proper neighbor assignments for obsnum in obsnums: OBS = self.session.query(Observation).filter( Observation.obsnum == obsnum).one() #find the original record we put into add_observations and check that the neighbors match for obs in obslist: if obs['julian_date'] == OBS.julian_date: if obs.has_key('neighbor_low'): self.assertEqual(OBS.low_neighbors[0].julian_date, obs['neighbor_low']) if obs.has_key('neighbor_high'): self.assertEqual(OBS.high_neighbors[0].julian_date, obs['neighbor_high']) break def test_list_observations(self): #form up the observation list obslist = [] jds = n.arange(0, 10) * self.length + 2456446.1234 pols = ['xx', 'yy', 'xy', 'yx'] for pol in pols: for jdi in xrange(len(jds)): obslist.append({ 'julian_date': jds[jdi], 'pol': pol, 'host': self.host, 'filename': self.filename, 'length': self.length }) if jdi != 0: obslist[-1]['neighbor_low'] = jds[jdi - 1] if jdi < len(jds[:-1]): obslist[-1]['neighbor_high'] = jds[jdi + 1] obsnums = self.dbi.add_observations(obslist) tic = time.time() observations = self.dbi.list_observations() #print "time to execute list_observations",time.time()-tic,'s' self.assertEqual(n.sum(n.array(observations) - n.array(obsnums)), 0) def test_get_neighbors(self): """ First set up a likely triplet of observations """ #form up the observation list obslist = [] jds = n.arange(0, 10) * self.length + 2456446.1234 pols = ['xx', 'yy', 'xy', 'yx'] for pol in pols: for jdi in xrange(len(jds)): obslist.append({ 'julian_date': jds[jdi], 'pol': pol, 'host': self.host, 'filename': self.filename, 'length': self.length }) if jdi != 0: obslist[-1]['neighbor_low'] = jds[jdi - 1] if jdi != (len(jds) - 1): obslist[-1]['neighbor_high'] = jds[jdi + 1] obsnums = self.dbi.add_observations(obslist) obsnums.sort() i = 5 # I have ten time stamps. this guys should have plenty of neighbors mytestobsnum = obsnums[i] #choose a middle obs tic = time.time() neighbors = self.dbi.get_neighbors(mytestobsnum) #print "time to execute get_neighbors",time.time()-tic,'s' self.assertEqual(len(neighbors), 2) self.assertEqual(neighbors[0], obsnums[i - 1]) #low self.assertEqual(neighbors[1], obsnums[i + 1]) #high def test_set_obs_status(self): """ set the status with the dbi function then check it with under the hood stuff """ #first create an observation in the first place obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host) # then set the status to something else self.dbi.set_obs_status(obsnum, 'UV') # get the status back out OBS = self.session.query(Observation).filter( Observation.obsnum == obsnum).one() self.assertEqual(OBS.status, 'UV') def test_get_obs_status(self): """ set the status with the dbi function (cause I tested it with more basic tests already) """ #first create an observation in the first place obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host) # then set the status to something else self.dbi.set_obs_status(obsnum, 'UV') #then get the status back tic = time.time() status = self.dbi.get_obs_status(obsnum) #print "time to execute get_obs_status",time.time()-tic,'s' self.assertEqual(status, 'UV') def test_time_transaction(self): """ """ #first create an observation in the first place obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host) # then set the status to something else tic = time.time() self.dbi.set_obs_status(obsnum, 'UV') print "time to execute set_obs_status", time.time() - tic, 's' def test_get_input_file(self): """ create an observation add the initial file get the pot host and path """ #first add the observation obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host) #then add a file to it host, path, filename = self.dbi.get_input_file(obsnum) self.assertEqual(host, self.host) self.assertEqual(path, os.path.dirname(self.filename)) self.assertEqual(filename, os.path.basename(self.filename)) def test_get_output_location(self): """ create an observation add the initial file get the pot host and path """ #first add the observation obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host) host, path = self.dbi.get_output_location(obsnum) self.assertEqual(host, self.host) self.assertEqual(path, os.path.dirname(self.filename)) def test_still_path(self): """ """ obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host) self.dbi.set_obs_still_path(obsnum, '/data/') still_path = self.dbi.get_obs_still_path(obsnum) self.assertEqual(still_path, '/data/') def test_obs_pid(self): """ """ obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename, self.host) self.dbi.set_obs_pid(obsnum, 9999) pid = self.dbi.get_obs_pid(obsnum) self.assertEqual(pid, 9999)
def file2pol(zenuv): return re.findall(r'\.(.{2})\.', zenuv)[0] o = optparse.OptionParser() o.set_usage('add_observations.py *.uv') o.set_description(__doc__) o.add_option( '--length', type=float, help= 'length of the input observations in minutes [default=average difference between filenames]' ) opts, args = o.parse_args(sys.argv[1:]) #connect to the database dbi = DataBaseInterface() #check that all files exist for filename in args: assert (filename.startswith('/')) assert (os.path.exists(filename)) #now run through all the files and build the relevant information for the db # get the pols pols = [] jds = [] for filename in args: pols.append(file2pol(filename)) jds.append(float(file2jd(filename))) jds = n.array(jds) if not opts.length is None: djd = opts.length / 60. / 24
#o.add_option('--length',type=float, # help='length of the input observations in minutes [default=average difference between filenames]') o.add_option('-v',action='store_true', help='set log level to debug') o.add_option('--tail',action='store_true', help='Monitor the log for changes') o.add_option('--logtxt',action='store_true', help='Print the full log text.') opts, args = o.parse_args(sys.argv[1:]) #connect to the database if opts.v: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(lebel=logging.INFO) logger = logging.getLogger('monitor_log') dbi = DataBaseInterface() # for each file get the obsnum, then reset the status to UV_POT obsnums = [] for arg in args: if arg[0]=='/': #we have a file! logger.debug("looking for file {filename}".format(filename=arg)) s = dbi.Session() File = s.query(File).filter(File.filename==arg).one()#XXX note assumes we are not noting that this file is copied. obsnum = File.obsnum logger.debug("found obsnum {obsnum}".format(obsnum=obsnum)) s.close() obsnums.append(obsnum) else: obsnums.append(arg)#it must be an obsnum logger.debug("found %d obsnums"%len(obsnums))
o.set_usage('summarize_night.py obsnum') o.set_description(__doc__) #o.add_option('--length',type=float, # help='length of the input observations in minutes [default=average difference between filenames]') o.add_option('-v',action='store_true', help='set log level to debug') o.add_option('--status',default='UV_POT', help='set the observation to this status [default=UV_POT]') opts, args = o.parse_args(sys.argv[1:]) #connect to the database if opts.v: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) logger = logging.getLogger('reset_observations') dbi = DataBaseInterface() s = dbi.Session() OBSs = s.query(Observation).filter(Observation.julian_date.between(float(args[0]),float(args[0])+0.9999)).order_by(Observation.obsnum.desc()).all() obsnums = [OBS.obsnum for OBS in OBSs] still_times ={} log_time_range = [datetime(3001,1,1),datetime(1,1,1)] for i,OBS in enumerate(OBSs): obsnum = OBS.obsnum if OBS.stillhost is None:continue print '\t'.join(map(str,[obsnum,OBS.stillhost,dbi.get_input_file(obsnum)[2],OBS.status])), LOGs = s.query(Log).filter(Log.obsnum==obsnum).order_by(Log.timestamp.desc()).all() if len(LOGs)==0:print 'NO LOGS';continue stoptime=LOGs[0].timestamp computation_start=0 if opts.v: