def setUp(self):
"""
create an in memory DB and open a connection to it
"""
filename = os.path.dirname(__file__) + '/../configs/test.cfg'
self.dbi = DataBaseInterface(test=True, configfile=filename)
self.session = self.dbi.Session()
self.jd = 2456892.20012000
self.pol = 'xx'
self.filename = '/data0/zen.2456785.123456.uv'
self.host = 'pot0'
self.length = 10.16639 / 60. / 24
def setUp(self):
"""
create an in memory DB and open a connection to it
"""
filename=os.path.dirname(__file__)+'/../configs/test.cfg'
self.dbi = DataBaseInterface(test=True,configfile=filename)
self.session = self.dbi.Session()
self.jd = 2456892.20012000
self.pol = 'xx'
self.filename='/data0/zen.2456785.123456.uv'
self.host = 'pot0'
self.length = 10.16639/60./24
o.set_usage('summarize_night.py obsnum')
o.set_description(__doc__)
#o.add_option('--length',type=float,
# help='length of the input observations in minutes [default=average difference between filenames]')
o.add_option('-v', action='store_true', help='set log level to debug')
o.add_option('--status',
default='UV_POT',
help='set the observation to this status [default=UV_POT]')
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
if opts.v:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARNING)
logger = logging.getLogger('summarize_still')
dbi = DataBaseInterface()
s = dbi.Session()
print "summarizing Distiller"
OBSs = s.query(Observation)
JDs = [OBS.julian_date for OBS in OBSs]
nights = n.sort(list(set(map(int, JDs))))
print "number of nights ingested:", len(nights)
Nobs = s.query(Observation).count()
Nprogress = s.query(Observation).filter(
Observation.status != 'NEW', Observation.status != 'UV_POT',
Observation.status != 'NEW', Observation.status != 'COMPLETE').count()
Ncomplete = s.query(Observation).filter(
Observation.status == 'COMPLETE').count()
print "Total observations in still:", Nobs
print "Number complete:", Ncomplete
class TestDBI(unittest.TestCase):
def setUp(self):
"""
create an in memory DB and open a connection to it
"""
filename=os.path.dirname(__file__)+'/../configs/test.cfg'
self.dbi = DataBaseInterface(test=True,configfile=filename)
self.session = self.dbi.Session()
self.jd = 2456892.20012000
self.pol = 'xx'
self.filename='/data0/zen.2456785.123456.uv'
self.host = 'pot0'
self.length = 10.16639/60./24
def test_add_log(self):
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host,length=self.length)
self.dbi.add_log(obsnum,'UV','test123',0)
logtext = self.dbi.get_logs(obsnum)
self.assertEqual(logtext.strip(),'test123')
LOG = self.session.query(Log).filter(Log.obsnum==obsnum).one()
def test_update_log(self):
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host,length=self.length)
self.dbi.add_log(obsnum,'UV','test123',None)
logtext = self.dbi.get_logs(obsnum)
self.assertEqual(logtext.strip(),'test123')
LOG = self.session.query(Log).filter(Log.obsnum==obsnum).one()
self.dbi.update_log(obsnum,exit_status=0)
newsession = self.dbi.Session()
LOG = newsession.query(Log).filter(Log.obsnum==obsnum).one()
self.assertEqual(LOG.exit_status,0)
def test_configparsing(self):
logger.info('Note: did you remember to do "cp configs/test.cfg ~/.paperstill/db.cfg" ? ')
self.assertEqual(self.dbi.dbinfo['hostip'],'memory')
def test_obsnum_increment(self):
dt = self.length
jds = n.arange(0,10)*dt+self.jd
obsnums=[]
for jd in jds:
obsnums.append(jdpol2obsnum(jd,self.pol,dt))
delta = n.diff(obsnums)
for d in delta:
self.assertEqual(d,1)
obsnum = self.dbi.add_observation(self.jd,self.pol,self.filename,self.host,length=self.length)
self.assertEqual(obsnum,jdpol2obsnum(self.jd,self.pol,self.length))
def test_add_observation(self):
"""
use the dbi to create a record.
basically tests the same as test_Observation_and_file
but with the dbi wrapper
"""
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host,length=self.length)
OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one()
self.assertEqual(float(OBS.julian_date),self.jd)
self.assertEqual(OBS.obsnum,jdpol2obsnum(self.jd,self.pol,self.length))
def test_add_file(self):
"""
todo update from code
"""
#first add the observation
obsnum = self.dbi.add_observation(self.jd,self.pol,self.filename,self.host)
#then add a file to it
filenum = self.dbi.add_file(obsnum,self.host,self.filename+'cRRE')
#then grab the file record back
FILE = self.session.query(File).filter(File.filenum==filenum).one()
#and check that its right
self.assertEqual(FILE.filename,self.filename+'cRRE')
def test_add_files(self):
#first add the observation
obsnum = self.dbi.add_observation(self.jd,self.pol,self.filename,self.host,self.length)
#add two subsequent products
files = ['/data0/zen.2456785.123456.uvc','/data0/zen.2456785.323456.uvcR']
for filename in files:
filenum = self.dbi.add_file(obsnum,self.host,filename)
#how I get all the files for a given obs
OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one()
self.assertEqual(len(OBS.files),3)#check that we got three files
def test_add_observations(self):
#form up the observation list
obslist =[]
jds = n.arange(0,10)*self.length+2456446.1234
pols = ['xx','yy','xy','yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({'julian_date':jds[jdi],
'pol':pol,
'host':self.host,
'filename':self.filename,
'length':self.length})
if jdi!=0:
obslist[-1]['neighbor_low'] = jds[jdi-1]
if jdi<len(jds[:-1]):
obslist[-1]['neighbor_high'] = jds[jdi+1]
obsnums = self.dbi.add_observations(obslist)
nobs = self.session.query(func.count(Observation.obsnum)).scalar()
self.assertEqual(len(obslist),nobs) #did we add observations?
#did they get the proper neighbor assignments
for obsnum in obsnums:
OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one()
#find the original record we put into add_observations and check that the neighbors match
for obs in obslist:
if obs['julian_date']==OBS.julian_date:
if obs.has_key('neighbor_low'):
self.assertEqual(OBS.low_neighbors[0].julian_date,
obs['neighbor_low'])
if obs.has_key('neighbor_high'):
self.assertEqual(OBS.high_neighbors[0].julian_date,
obs['neighbor_high'])
break
def test_list_observations(self):
#form up the observation list
obslist =[]
jds = n.arange(0,10)*self.length+2456446.1234
pols = ['xx','yy','xy','yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({'julian_date':jds[jdi],
'pol':pol,
'host':self.host,
'filename':self.filename,
'length':self.length})
if jdi!=0:
obslist[-1]['neighbor_low'] = jds[jdi-1]
if jdi<len(jds[:-1]):
obslist[-1]['neighbor_high'] = jds[jdi+1]
obsnums = self.dbi.add_observations(obslist)
tic = time.time()
observations = self.dbi.list_observations()
#print "time to execute list_observations",time.time()-tic,'s'
self.assertEqual(n.sum(n.array(observations)-n.array(obsnums)),0)
def test_get_neighbors(self):
"""
First set up a likely triplet of observations
"""
#form up the observation list
obslist =[]
jds = n.arange(0,10)*self.length+2456446.1234
pols = ['xx','yy','xy','yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({'julian_date':jds[jdi],
'pol':pol,
'host':self.host,
'filename':self.filename,
'length':self.length})
if jdi!=0:
obslist[-1]['neighbor_low'] = jds[jdi-1]
if jdi!=(len(jds)-1):
obslist[-1]['neighbor_high'] = jds[jdi+1]
obsnums = self.dbi.add_observations(obslist)
obsnums.sort()
i = 5# I have ten time stamps. this guys should have plenty of neighbors
mytestobsnum = obsnums[i] #choose a middle obs
tic = time.time()
neighbors = self.dbi.get_neighbors(mytestobsnum)
#print "time to execute get_neighbors",time.time()-tic,'s'
self.assertEqual(len(neighbors),2)
self.assertEqual(neighbors[0],obsnums[i-1])#low
self.assertEqual(neighbors[1],obsnums[i+1])#high
def test_set_obs_status(self):
"""
set the status with the dbi function then check it with
under the hood stuff
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
# then set the status to something else
self.dbi.set_obs_status(obsnum,'UV')
# get the status back out
OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one()
self.assertEqual(OBS.status,'UV')
def test_get_obs_status(self):
"""
set the status with the dbi function (cause I tested it with more basic tests already)
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
# then set the status to something else
self.dbi.set_obs_status(obsnum,'UV')
#then get the status back
tic = time.time()
status = self.dbi.get_obs_status(obsnum)
#print "time to execute get_obs_status",time.time()-tic,'s'
self.assertEqual(status,'UV')
def test_time_transaction(self):
"""
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
# then set the status to something else
tic = time.time()
self.dbi.set_obs_status(obsnum,'UV')
print "time to execute set_obs_status",time.time() - tic,'s'
def test_get_input_file(self):
"""
create an observation
add the initial file
get the pot host and path
"""
#first add the observation
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
#then add a file to it
host,path,filename = self.dbi.get_input_file(obsnum)
self.assertEqual(host,self.host)
self.assertEqual(path,os.path.dirname(self.filename))
self.assertEqual(filename,os.path.basename(self.filename))
def test_get_output_location(self):
"""
create an observation
add the initial file
get the pot host and path
"""
#first add the observation
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
host,path = self.dbi.get_output_location(obsnum)
self.assertEqual(host,self.host)
self.assertEqual(path,os.path.dirname(self.filename))
def test_still_path(self):
"""
"""
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
self.dbi.set_obs_still_path(obsnum,'/data/')
still_path = self.dbi.get_obs_still_path(obsnum)
self.assertEqual(still_path,'/data/')
def test_obs_pid(self):
"""
"""
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
self.dbi.set_obs_pid(obsnum,9999)
pid = self.dbi.get_obs_pid(obsnum)
self.assertEqual(pid,9999)
o.set_usage('reset_observations.py *.uv')
o.set_description(__doc__)
#o.add_option('--length',type=float,
# help='length of the input observations in minutes [default=average difference between filenames]')
o.add_option('-v',action='store_true',
help='set log level to debug')
o.add_option('--status',default='UV_POT',
help='set the observation to this status [default=UV_POT]')
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
if opts.v:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(lebel=logging.INFO)
logger = logging.getLogger('reset_observations')
dbi = DataBaseInterface()
# for each file get the obsnum, then reset the status to UV_POT
obsnums = []
for filename in args:
logger.debug("looking for file {filename}".format(filename=filename))
s = dbi.Session()
FILE = s.query(File).filter(File.filename==filename).one()#XXX note assumes we are not noting that this file is copied.
obsnum = FILE.obsnum
logger.debug("found obsnum {obsnum}".format(obsnum=obsnum))
s.close()
logger.debug("setting status to {status}".format(status=opts.status))
dbi.set_obs_status(obsnum,opts.status)
dbi.set_obs_pid(obsnum,None)
dbi.set_obs_still_host(obsnum,None)
dbi.add_log(obsnum,'NEW',"issuing a reset_observations",0)
def file2jd(zenuv):
return re.findall(r'\d+\.\d+', zenuv)[0]
def file2pol(zenuv):
return re.findall(r'\.(.{2})\.',zenuv)[0]
o = optparse.OptionParser()
o.set_usage('add_observations.py *.uv')
o.set_description(__doc__)
o.add_option('--length',type=float,
help='length of the input observations in minutes [default=average difference between filenames]')
o.add_option('-t',action='store_true',
help='Test. Only print, do not touch db')
o.add_option('--overwrite',action='store_true',
help='Default action is to skip obsrvations already in the db. Setting this option overrides this safety feature and attempts anyway')
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
dbi = DataBaseInterface()
#check that all files exist
for filename in args:
assert(filename.startswith('/'))
assert(os.path.exists(filename))
#now run through all the files and build the relevant information for the db
# get the pols
pols = []
jds = []
for filename in args:
pols.append(file2pol(filename))
jds.append(float(file2jd(filename)))
jds = n.array(jds)
nights = list(set(jds.astype(n.int)))
if not opts.length is None:
o = optparse.OptionParser()
o.set_usage('print_logs.py obsnum')
o.set_description(__doc__)
#o.add_option('--length',type=float,
# help='length of the input observations in minutes [default=average difference between filenames]')
o.add_option('-v', action='store_true', help='set log level to debug')
o.add_option('--tail', action='store_true', help='Monitor the log for changes')
o.add_option('--logtxt', action='store_true', help='Print the full log text.')
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
if opts.v:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(lebel=logging.INFO)
logger = logging.getLogger('monitor_log')
dbi = DataBaseInterface()
# for each file get the obsnum, then reset the status to UV_POT
obsnums = []
for arg in args:
if arg[0] == '/': #we have a file!
logger.debug("looking for file {filename}".format(filename=arg))
s = dbi.Session()
File = s.query(File).filter(File.filename == arg).one(
) #XXX note assumes we are not noting that this file is copied.
obsnum = File.obsnum
logger.debug("found obsnum {obsnum}".format(obsnum=obsnum))
s.close()
obsnums.append(obsnum)
else:
obsnums.append(arg) #it must be an obsnum
o.set_usage('summarize_night.py obsnum')
o.set_description(__doc__)
#o.add_option('--length',type=float,
# help='length of the input observations in minutes [default=average difference between filenames]')
o.add_option('-v', action='store_true', help='set log level to debug')
o.add_option('--status',
default='UV_POT',
help='set the observation to this status [default=UV_POT]')
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
if opts.v:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('reset_observations')
dbi = DataBaseInterface()
s = dbi.Session()
OBSs = s.query(Observation).filter(
Observation.julian_date.between(float(args[0]),
float(args[0]) + 0.9999)).order_by(
Observation.obsnum.desc()).all()
obsnums = [OBS.obsnum for OBS in OBSs]
still_times = {}
log_time_range = [datetime(3001, 1, 1), datetime(1, 1, 1)]
for i, OBS in enumerate(OBSs):
obsnum = OBS.obsnum
if OBS.stillhost is None: continue
print '\t'.join(
map(str,
[obsnum, OBS.stillhost,
type=float,
help=
'length of the input observations in minutes [default=average difference between filenames]'
)
o.add_option('-t',
action='store_true',
help='Test. Only print, do not touch db')
o.add_option(
'--overwrite',
action='store_true',
help=
'Default action is to skip obsrvations already in the db. Setting this option overrides this safety feature and attempts anyway'
)
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
dbi = DataBaseInterface()
#check that all files exist
for filename in args:
assert (filename.startswith('/'))
assert (os.path.exists(filename))
#now run through all the files and build the relevant information for the db
# get the pols
pols = []
jds = []
for filename in args:
pols.append(file2pol(filename))
jds.append(float(file2jd(filename)))
jds = n.array(jds)
nights = list(set(jds.astype(n.int)))
if not opts.length is None:
o.set_usage('print_logs.py obsnum')
o.set_description(__doc__)
#o.add_option('--length',type=float,
# help='length of the input observations in minutes [default=average difference between filenames]')
o.add_option('-v',action='store_true',
help='set log level to debug')
o.add_option('--status',default='UV_POT',
help='set the observation to this status [default=UV_POT]')
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
if opts.v:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(lebel=logging.INFO)
logger = logging.getLogger('reset_observations')
dbi = DataBaseInterface()
# for each file get the obsnum, then reset the status to UV_POT
obsnums = []
for arg in args:
if arg[0]=='/': #we have a file!
logger.debug("looking for file {filename}".format(filename=arg))
s = dbi.Session()
File = s.query(File).filter(File.filename==arg).one()#XXX note assumes we are not noting that this file is copied.
obsnum = File.obsnum
logger.debug("found obsnum {obsnum}".format(obsnum=obsnum))
s.close()
obsnums.append(obsnum)
else:
obsnums.append(arg)#it must be an obsnum
logger.debug("found %d obsnums"%len(obsnums))
def __init__(self, nobs, npols, test=True):
DataBaseInterface.__init__(self, test=test)
self.length = 10 / 60. / 24
self.host = 'pot0'
self.defaultstatus = 'UV_POT'
self.Add_Fake_Observations(nobs, npols)
from ddr_compress.dbi import DataBaseInterface,gethostname
import optparse,os,sys,re,numpy as n
def file2jd(zenuv):
return re.findall(r'\d+\.\d+', zenuv)[0]
def file2pol(zenuv):
return re.findall(r'\.(.{2})\.',zenuv)[0]
o = optparse.OptionParser()
o.set_usage('add_observations.py *.uv')
o.set_description(__doc__)
o.add_option('--length',type=float,
help='length of the input observations in minutes [default=average difference between filenames]')
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
dbi = DataBaseInterface()
#check that all files exist
for filename in args:
assert(filename.startswith('/'))
assert(os.path.exists(filename))
#now run through all the files and build the relevant information for the db
# get the pols
pols = []
jds = []
for filename in args:
pols.append(file2pol(filename))
jds.append(float(file2jd(filename)))
jds = n.array(jds)
if not opts.length is None:
djd = opts.length/60./24
def __init__(self,nobs,npols,test=True):
DataBaseInterface.__init__(self,test=test)
self.length = 10/60./24
self.host = 'pot0'
self.defaultstatus='UV_POT'
self.Add_Fake_Observations(nobs,npols)
#! /usr/bin/env python import optparse,sys,os from ddr_compress.dbi import DataBaseInterface #open a connection to the db dbi = DataBaseInterface() #check that the tables
o.set_usage('print_logs.py obsnum')
o.set_description(__doc__)
#o.add_option('--length',type=float,
# help='length of the input observations in minutes [default=average difference between filenames]')
o.add_option('-v', action='store_true', help='set log level to debug')
o.add_option('--status',
default='UV_POT',
help='set the observation to this status [default=UV_POT]')
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
if opts.v:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(lebel=logging.INFO)
logger = logging.getLogger('reset_observations')
dbi = DataBaseInterface()
# for each file get the obsnum, then reset the status to UV_POT
obsnums = []
for arg in args:
if arg[0] == '/': #we have a file!
logger.debug("looking for file {filename}".format(filename=arg))
s = dbi.Session()
File = s.query(File).filter(File.filename == arg).one(
) #XXX note assumes we are not noting that this file is copied.
obsnum = File.obsnum
logger.debug("found obsnum {obsnum}".format(obsnum=obsnum))
s.close()
obsnums.append(obsnum)
else:
obsnums.append(arg) #it must be an obsnum
#! /usr/bin/env python
from ddr_compress.dbi import DataBaseInterface,Observation,File
from sqlalchemy import func
import curses,time,os,sys
import numpy as n
#setup my curses stuff following
# https://docs.python.org/2/howto/curses.html
stdscr = curses.initscr()
curses.noecho()
curses.cbreak()
stdscr.keypad(1)
stdscr.nodelay(1)
#setup my db connection
dbi = DataBaseInterface()
stdscr.addstr("PAPER Distiller Status Board. Monitoring : {dbname}".format(dbname=dbi.dbinfo['dbname']))
stdscr.addstr(1,0,"Press 'q' to exit")
statheight = 50
statusscr = curses.newwin(statheight,400,5,0)
statusscr.keypad(1)
statusscr.nodelay(1)
curline = 2
colwidth = 50
obslines = 20
stat = ['\\','|','/','-','.']
i =0
try:
while(1):
#get the screen dimensions
#! /usr/bin/env python
from ddr_compress.dbi import DataBaseInterface,Observation,File
from sqlalchemy import func
import curses,time,os
#setup my curses stuff following
# https://docs.python.org/2/howto/curses.html
stdscr = curses.initscr()
curses.noecho()
curses.cbreak()
stdscr.keypad(1)
stdscr.nodelay(1)
#setup my db connection
dbi = DataBaseInterface()
stdscr.addstr("PAPER Distiller Status Board. Monitoring: {dbname}".format(dbname=dbi.dbinfo['dbname']))
stdscr.addstr(1,0,"Press 'q' to exit")
statheight = 50
statusscr = curses.newwin(statheight,400,5,0)
statusscr.keypad(1)
statusscr.nodelay(1)
curline = 2
colwidth = 40
obslines = 20
i=0
stat = ['\\','|','/','-','.']
try:
while(1):
#get the screen dimensions
class TestDBI(unittest.TestCase):
def setUp(self):
"""
create an in memory DB and open a connection to it
"""
filename = os.path.dirname(__file__) + '/../configs/test.cfg'
self.dbi = DataBaseInterface(test=True, configfile=filename)
self.session = self.dbi.Session()
self.jd = 2456892.20012000
self.pol = 'xx'
self.filename = '/data0/zen.2456785.123456.uv'
self.host = 'pot0'
self.length = 10.16639 / 60. / 24
def test_logging(self):
obsnum = self.dbi.add_observation(self.jd,
self.pol,
self.filename,
self.host,
length=self.length)
self.dbi.add_log(obsnum, 'UV', 'test123', 0)
logtext = self.dbi.get_logs(obsnum)
self.assertEqual(logtext.strip(), 'test123')
LOG = self.session.query(Log).filter(Log.obsnum == obsnum).one()
print LOG.timestamp
def test_configparsing(self):
logger.info(
'Note: did you remember to do "cp configs/test.cfg ~/.paperstill/db.cfg" ? '
)
self.assertEqual(self.dbi.dbinfo['hostip'], 'memory')
def test_obsnum_increment(self):
dt = self.length
jds = n.arange(0, 10) * dt + self.jd
obsnums = []
for jd in jds:
obsnums.append(jdpol2obsnum(jd, self.pol, dt))
delta = n.diff(obsnums)
for d in delta:
self.assertEqual(d, 1)
obsnum = self.dbi.add_observation(self.jd,
self.pol,
self.filename,
self.host,
length=self.length)
self.assertEqual(obsnum, jdpol2obsnum(self.jd, self.pol, self.length))
def test_add_observation(self):
"""
use the dbi to create a record.
basically tests the same as test_Observation_and_file
but with the dbi wrapper
"""
obsnum = self.dbi.add_observation(self.jd,
self.pol,
self.filename,
self.host,
length=self.length)
OBS = self.session.query(Observation).filter(
Observation.obsnum == obsnum).one()
self.assertEqual(float(OBS.julian_date), self.jd)
self.assertEqual(OBS.obsnum,
jdpol2obsnum(self.jd, self.pol, self.length))
def test_add_file(self):
"""
todo update from code
"""
#first add the observation
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
#then add a file to it
filenum = self.dbi.add_file(obsnum, self.host, self.filename + 'cRRE')
#then grab the file record back
FILE = self.session.query(File).filter(File.filenum == filenum).one()
#and check that its right
self.assertEqual(FILE.filename, self.filename + 'cRRE')
def test_add_files(self):
#first add the observation
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host, self.length)
#add two subsequent products
files = [
'/data0/zen.2456785.123456.uvc', '/data0/zen.2456785.323456.uvcR'
]
for filename in files:
filenum = self.dbi.add_file(obsnum, self.host, filename)
#how I get all the files for a given obs
OBS = self.session.query(Observation).filter(
Observation.obsnum == obsnum).one()
self.assertEqual(len(OBS.files), 3) #check that we got three files
def test_add_observations(self):
#form up the observation list
obslist = []
jds = n.arange(0, 10) * self.length + 2456446.1234
pols = ['xx', 'yy', 'xy', 'yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({
'julian_date': jds[jdi],
'pol': pol,
'host': self.host,
'filename': self.filename,
'length': self.length
})
if jdi != 0:
obslist[-1]['neighbor_low'] = jds[jdi - 1]
if jdi < len(jds[:-1]):
obslist[-1]['neighbor_high'] = jds[jdi + 1]
obsnums = self.dbi.add_observations(obslist)
nobs = self.session.query(func.count(Observation.obsnum)).scalar()
self.assertEqual(len(obslist), nobs) #did we add observations?
#did they get the proper neighbor assignments
for obsnum in obsnums:
OBS = self.session.query(Observation).filter(
Observation.obsnum == obsnum).one()
#find the original record we put into add_observations and check that the neighbors match
for obs in obslist:
if obs['julian_date'] == OBS.julian_date:
if obs.has_key('neighbor_low'):
self.assertEqual(OBS.low_neighbors[0].julian_date,
obs['neighbor_low'])
if obs.has_key('neighbor_high'):
self.assertEqual(OBS.high_neighbors[0].julian_date,
obs['neighbor_high'])
break
def test_list_observations(self):
#form up the observation list
obslist = []
jds = n.arange(0, 10) * self.length + 2456446.1234
pols = ['xx', 'yy', 'xy', 'yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({
'julian_date': jds[jdi],
'pol': pol,
'host': self.host,
'filename': self.filename,
'length': self.length
})
if jdi != 0:
obslist[-1]['neighbor_low'] = jds[jdi - 1]
if jdi < len(jds[:-1]):
obslist[-1]['neighbor_high'] = jds[jdi + 1]
obsnums = self.dbi.add_observations(obslist)
tic = time.time()
observations = self.dbi.list_observations()
#print "time to execute list_observations",time.time()-tic,'s'
self.assertEqual(n.sum(n.array(observations) - n.array(obsnums)), 0)
def test_get_neighbors(self):
"""
First set up a likely triplet of observations
"""
#form up the observation list
obslist = []
jds = n.arange(0, 10) * self.length + 2456446.1234
pols = ['xx', 'yy', 'xy', 'yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({
'julian_date': jds[jdi],
'pol': pol,
'host': self.host,
'filename': self.filename,
'length': self.length
})
if jdi != 0:
obslist[-1]['neighbor_low'] = jds[jdi - 1]
if jdi != (len(jds) - 1):
obslist[-1]['neighbor_high'] = jds[jdi + 1]
obsnums = self.dbi.add_observations(obslist)
obsnums.sort()
i = 5 # I have ten time stamps. this guys should have plenty of neighbors
mytestobsnum = obsnums[i] #choose a middle obs
tic = time.time()
neighbors = self.dbi.get_neighbors(mytestobsnum)
#print "time to execute get_neighbors",time.time()-tic,'s'
self.assertEqual(len(neighbors), 2)
self.assertEqual(neighbors[0], obsnums[i - 1]) #low
self.assertEqual(neighbors[1], obsnums[i + 1]) #high
def test_set_obs_status(self):
"""
set the status with the dbi function then check it with
under the hood stuff
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
# then set the status to something else
self.dbi.set_obs_status(obsnum, 'UV')
# get the status back out
OBS = self.session.query(Observation).filter(
Observation.obsnum == obsnum).one()
self.assertEqual(OBS.status, 'UV')
def test_get_obs_status(self):
"""
set the status with the dbi function (cause I tested it with more basic tests already)
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
# then set the status to something else
self.dbi.set_obs_status(obsnum, 'UV')
#then get the status back
tic = time.time()
status = self.dbi.get_obs_status(obsnum)
#print "time to execute get_obs_status",time.time()-tic,'s'
self.assertEqual(status, 'UV')
def test_time_transaction(self):
"""
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
# then set the status to something else
tic = time.time()
self.dbi.set_obs_status(obsnum, 'UV')
print "time to execute set_obs_status", time.time() - tic, 's'
def test_get_input_file(self):
"""
create an observation
add the initial file
get the pot host and path
"""
#first add the observation
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
#then add a file to it
host, path, filename = self.dbi.get_input_file(obsnum)
self.assertEqual(host, self.host)
self.assertEqual(path, os.path.dirname(self.filename))
self.assertEqual(filename, os.path.basename(self.filename))
def test_get_output_location(self):
"""
create an observation
add the initial file
get the pot host and path
"""
#first add the observation
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
host, path = self.dbi.get_output_location(obsnum)
self.assertEqual(host, self.host)
self.assertEqual(path, os.path.dirname(self.filename))
def test_still_path(self):
"""
"""
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
self.dbi.set_obs_still_path(obsnum, '/data/')
still_path = self.dbi.get_obs_still_path(obsnum)
self.assertEqual(still_path, '/data/')
def test_obs_pid(self):
"""
"""
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
self.dbi.set_obs_pid(obsnum, 9999)
pid = self.dbi.get_obs_pid(obsnum)
self.assertEqual(pid, 9999)
def file2pol(zenuv):
return re.findall(r'\.(.{2})\.', zenuv)[0]
o = optparse.OptionParser()
o.set_usage('add_observations.py *.uv')
o.set_description(__doc__)
o.add_option(
'--length',
type=float,
help=
'length of the input observations in minutes [default=average difference between filenames]'
)
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
dbi = DataBaseInterface()
#check that all files exist
for filename in args:
assert (filename.startswith('/'))
assert (os.path.exists(filename))
#now run through all the files and build the relevant information for the db
# get the pols
pols = []
jds = []
for filename in args:
pols.append(file2pol(filename))
jds.append(float(file2jd(filename)))
jds = n.array(jds)
if not opts.length is None:
djd = opts.length / 60. / 24
#o.add_option('--length',type=float,
# help='length of the input observations in minutes [default=average difference between filenames]')
o.add_option('-v',action='store_true',
help='set log level to debug')
o.add_option('--tail',action='store_true',
help='Monitor the log for changes')
o.add_option('--logtxt',action='store_true',
help='Print the full log text.')
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
if opts.v:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(lebel=logging.INFO)
logger = logging.getLogger('monitor_log')
dbi = DataBaseInterface()
# for each file get the obsnum, then reset the status to UV_POT
obsnums = []
for arg in args:
if arg[0]=='/': #we have a file!
logger.debug("looking for file {filename}".format(filename=arg))
s = dbi.Session()
File = s.query(File).filter(File.filename==arg).one()#XXX note assumes we are not noting that this file is copied.
obsnum = File.obsnum
logger.debug("found obsnum {obsnum}".format(obsnum=obsnum))
s.close()
obsnums.append(obsnum)
else:
obsnums.append(arg)#it must be an obsnum
logger.debug("found %d obsnums"%len(obsnums))
o.set_usage('summarize_night.py obsnum')
o.set_description(__doc__)
#o.add_option('--length',type=float,
# help='length of the input observations in minutes [default=average difference between filenames]')
o.add_option('-v',action='store_true',
help='set log level to debug')
o.add_option('--status',default='UV_POT',
help='set the observation to this status [default=UV_POT]')
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
if opts.v:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('reset_observations')
dbi = DataBaseInterface()
s = dbi.Session()
OBSs = s.query(Observation).filter(Observation.julian_date.between(float(args[0]),float(args[0])+0.9999)).order_by(Observation.obsnum.desc()).all()
obsnums = [OBS.obsnum for OBS in OBSs]
still_times ={}
log_time_range = [datetime(3001,1,1),datetime(1,1,1)]
for i,OBS in enumerate(OBSs):
obsnum = OBS.obsnum
if OBS.stillhost is None:continue
print '\t'.join(map(str,[obsnum,OBS.stillhost,dbi.get_input_file(obsnum)[2],OBS.status])),
LOGs = s.query(Log).filter(Log.obsnum==obsnum).order_by(Log.timestamp.desc()).all()
if len(LOGs)==0:print 'NO LOGS';continue
stoptime=LOGs[0].timestamp
computation_start=0
if opts.v:
