class TestDBI(unittest.TestCase):
def setUp(self):
"""
create an in memory DB and open a connection to it
"""
filename=os.path.dirname(__file__)+'/../configs/test.cfg'
self.dbi = DataBaseInterface(test=True,configfile=filename)
self.session = self.dbi.Session()
self.jd = 2456892.20012000
self.pol = 'xx'
self.filename='/data0/zen.2456785.123456.uv'
self.host = 'pot0'
self.length = 10.16639/60./24
def test_add_log(self):
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host,length=self.length)
self.dbi.add_log(obsnum,'UV','test123',0)
logtext = self.dbi.get_logs(obsnum)
self.assertEqual(logtext.strip(),'test123')
LOG = self.session.query(Log).filter(Log.obsnum==obsnum).one()
def test_update_log(self):
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host,length=self.length)
self.dbi.add_log(obsnum,'UV','test123',None)
logtext = self.dbi.get_logs(obsnum)
self.assertEqual(logtext.strip(),'test123')
LOG = self.session.query(Log).filter(Log.obsnum==obsnum).one()
self.dbi.update_log(obsnum,exit_status=0)
newsession = self.dbi.Session()
LOG = newsession.query(Log).filter(Log.obsnum==obsnum).one()
self.assertEqual(LOG.exit_status,0)
def test_configparsing(self):
logger.info('Note: did you remember to do "cp configs/test.cfg ~/.paperstill/db.cfg" ? ')
self.assertEqual(self.dbi.dbinfo['hostip'],'memory')
def test_obsnum_increment(self):
dt = self.length
jds = n.arange(0,10)*dt+self.jd
obsnums=[]
for jd in jds:
obsnums.append(jdpol2obsnum(jd,self.pol,dt))
delta = n.diff(obsnums)
for d in delta:
self.assertEqual(d,1)
obsnum = self.dbi.add_observation(self.jd,self.pol,self.filename,self.host,length=self.length)
self.assertEqual(obsnum,jdpol2obsnum(self.jd,self.pol,self.length))
def test_add_observation(self):
"""
use the dbi to create a record.
basically tests the same as test_Observation_and_file
but with the dbi wrapper
"""
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host,length=self.length)
OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one()
self.assertEqual(float(OBS.julian_date),self.jd)
self.assertEqual(OBS.obsnum,jdpol2obsnum(self.jd,self.pol,self.length))
def test_add_file(self):
"""
todo update from code
"""
#first add the observation
obsnum = self.dbi.add_observation(self.jd,self.pol,self.filename,self.host)
#then add a file to it
filenum = self.dbi.add_file(obsnum,self.host,self.filename+'cRRE')
#then grab the file record back
FILE = self.session.query(File).filter(File.filenum==filenum).one()
#and check that its right
self.assertEqual(FILE.filename,self.filename+'cRRE')
def test_add_files(self):
#first add the observation
obsnum = self.dbi.add_observation(self.jd,self.pol,self.filename,self.host,self.length)
#add two subsequent products
files = ['/data0/zen.2456785.123456.uvc','/data0/zen.2456785.323456.uvcR']
for filename in files:
filenum = self.dbi.add_file(obsnum,self.host,filename)
#how I get all the files for a given obs
OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one()
self.assertEqual(len(OBS.files),3)#check that we got three files
def test_add_observations(self):
#form up the observation list
obslist =[]
jds = n.arange(0,10)*self.length+2456446.1234
pols = ['xx','yy','xy','yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({'julian_date':jds[jdi],
'pol':pol,
'host':self.host,
'filename':self.filename,
'length':self.length})
if jdi!=0:
obslist[-1]['neighbor_low'] = jds[jdi-1]
if jdi<len(jds[:-1]):
obslist[-1]['neighbor_high'] = jds[jdi+1]
obsnums = self.dbi.add_observations(obslist)
nobs = self.session.query(func.count(Observation.obsnum)).scalar()
self.assertEqual(len(obslist),nobs) #did we add observations?
#did they get the proper neighbor assignments
for obsnum in obsnums:
OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one()
#find the original record we put into add_observations and check that the neighbors match
for obs in obslist:
if obs['julian_date']==OBS.julian_date:
if obs.has_key('neighbor_low'):
self.assertEqual(OBS.low_neighbors[0].julian_date,
obs['neighbor_low'])
if obs.has_key('neighbor_high'):
self.assertEqual(OBS.high_neighbors[0].julian_date,
obs['neighbor_high'])
break
def test_list_observations(self):
#form up the observation list
obslist =[]
jds = n.arange(0,10)*self.length+2456446.1234
pols = ['xx','yy','xy','yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({'julian_date':jds[jdi],
'pol':pol,
'host':self.host,
'filename':self.filename,
'length':self.length})
if jdi!=0:
obslist[-1]['neighbor_low'] = jds[jdi-1]
if jdi<len(jds[:-1]):
obslist[-1]['neighbor_high'] = jds[jdi+1]
obsnums = self.dbi.add_observations(obslist)
tic = time.time()
observations = self.dbi.list_observations()
#print "time to execute list_observations",time.time()-tic,'s'
self.assertEqual(n.sum(n.array(observations)-n.array(obsnums)),0)
def test_get_neighbors(self):
"""
First set up a likely triplet of observations
"""
#form up the observation list
obslist =[]
jds = n.arange(0,10)*self.length+2456446.1234
pols = ['xx','yy','xy','yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({'julian_date':jds[jdi],
'pol':pol,
'host':self.host,
'filename':self.filename,
'length':self.length})
if jdi!=0:
obslist[-1]['neighbor_low'] = jds[jdi-1]
if jdi!=(len(jds)-1):
obslist[-1]['neighbor_high'] = jds[jdi+1]
obsnums = self.dbi.add_observations(obslist)
obsnums.sort()
i = 5# I have ten time stamps. this guys should have plenty of neighbors
mytestobsnum = obsnums[i] #choose a middle obs
tic = time.time()
neighbors = self.dbi.get_neighbors(mytestobsnum)
#print "time to execute get_neighbors",time.time()-tic,'s'
self.assertEqual(len(neighbors),2)
self.assertEqual(neighbors[0],obsnums[i-1])#low
self.assertEqual(neighbors[1],obsnums[i+1])#high
def test_set_obs_status(self):
"""
set the status with the dbi function then check it with
under the hood stuff
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
# then set the status to something else
self.dbi.set_obs_status(obsnum,'UV')
# get the status back out
OBS = self.session.query(Observation).filter(Observation.obsnum==obsnum).one()
self.assertEqual(OBS.status,'UV')
def test_get_obs_status(self):
"""
set the status with the dbi function (cause I tested it with more basic tests already)
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
# then set the status to something else
self.dbi.set_obs_status(obsnum,'UV')
#then get the status back
tic = time.time()
status = self.dbi.get_obs_status(obsnum)
#print "time to execute get_obs_status",time.time()-tic,'s'
self.assertEqual(status,'UV')
def test_time_transaction(self):
"""
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
# then set the status to something else
tic = time.time()
self.dbi.set_obs_status(obsnum,'UV')
print "time to execute set_obs_status",time.time() - tic,'s'
def test_get_input_file(self):
"""
create an observation
add the initial file
get the pot host and path
"""
#first add the observation
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
#then add a file to it
host,path,filename = self.dbi.get_input_file(obsnum)
self.assertEqual(host,self.host)
self.assertEqual(path,os.path.dirname(self.filename))
self.assertEqual(filename,os.path.basename(self.filename))
def test_get_output_location(self):
"""
create an observation
add the initial file
get the pot host and path
"""
#first add the observation
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
host,path = self.dbi.get_output_location(obsnum)
self.assertEqual(host,self.host)
self.assertEqual(path,os.path.dirname(self.filename))
def test_still_path(self):
"""
"""
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
self.dbi.set_obs_still_path(obsnum,'/data/')
still_path = self.dbi.get_obs_still_path(obsnum)
self.assertEqual(still_path,'/data/')
def test_obs_pid(self):
"""
"""
obsnum = self.dbi.add_observation(
self.jd,self.pol,self.filename,self.host)
self.dbi.set_obs_pid(obsnum,9999)
pid = self.dbi.get_obs_pid(obsnum)
self.assertEqual(pid,9999)
class TestDBI(unittest.TestCase):
def setUp(self):
"""
create an in memory DB and open a connection to it
"""
filename = os.path.dirname(__file__) + '/../configs/test.cfg'
self.dbi = DataBaseInterface(test=True, configfile=filename)
self.session = self.dbi.Session()
self.jd = 2456892.20012000
self.pol = 'xx'
self.filename = '/data0/zen.2456785.123456.uv'
self.host = 'pot0'
self.length = 10.16639 / 60. / 24
def test_logging(self):
obsnum = self.dbi.add_observation(self.jd,
self.pol,
self.filename,
self.host,
length=self.length)
self.dbi.add_log(obsnum, 'UV', 'test123', 0)
logtext = self.dbi.get_logs(obsnum)
self.assertEqual(logtext.strip(), 'test123')
LOG = self.session.query(Log).filter(Log.obsnum == obsnum).one()
print LOG.timestamp
def test_configparsing(self):
logger.info(
'Note: did you remember to do "cp configs/test.cfg ~/.paperstill/db.cfg" ? '
)
self.assertEqual(self.dbi.dbinfo['hostip'], 'memory')
def test_obsnum_increment(self):
dt = self.length
jds = n.arange(0, 10) * dt + self.jd
obsnums = []
for jd in jds:
obsnums.append(jdpol2obsnum(jd, self.pol, dt))
delta = n.diff(obsnums)
for d in delta:
self.assertEqual(d, 1)
obsnum = self.dbi.add_observation(self.jd,
self.pol,
self.filename,
self.host,
length=self.length)
self.assertEqual(obsnum, jdpol2obsnum(self.jd, self.pol, self.length))
def test_add_observation(self):
"""
use the dbi to create a record.
basically tests the same as test_Observation_and_file
but with the dbi wrapper
"""
obsnum = self.dbi.add_observation(self.jd,
self.pol,
self.filename,
self.host,
length=self.length)
OBS = self.session.query(Observation).filter(
Observation.obsnum == obsnum).one()
self.assertEqual(float(OBS.julian_date), self.jd)
self.assertEqual(OBS.obsnum,
jdpol2obsnum(self.jd, self.pol, self.length))
def test_add_file(self):
"""
todo update from code
"""
#first add the observation
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
#then add a file to it
filenum = self.dbi.add_file(obsnum, self.host, self.filename + 'cRRE')
#then grab the file record back
FILE = self.session.query(File).filter(File.filenum == filenum).one()
#and check that its right
self.assertEqual(FILE.filename, self.filename + 'cRRE')
def test_add_files(self):
#first add the observation
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host, self.length)
#add two subsequent products
files = [
'/data0/zen.2456785.123456.uvc', '/data0/zen.2456785.323456.uvcR'
]
for filename in files:
filenum = self.dbi.add_file(obsnum, self.host, filename)
#how I get all the files for a given obs
OBS = self.session.query(Observation).filter(
Observation.obsnum == obsnum).one()
self.assertEqual(len(OBS.files), 3) #check that we got three files
def test_add_observations(self):
#form up the observation list
obslist = []
jds = n.arange(0, 10) * self.length + 2456446.1234
pols = ['xx', 'yy', 'xy', 'yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({
'julian_date': jds[jdi],
'pol': pol,
'host': self.host,
'filename': self.filename,
'length': self.length
})
if jdi != 0:
obslist[-1]['neighbor_low'] = jds[jdi - 1]
if jdi < len(jds[:-1]):
obslist[-1]['neighbor_high'] = jds[jdi + 1]
obsnums = self.dbi.add_observations(obslist)
nobs = self.session.query(func.count(Observation.obsnum)).scalar()
self.assertEqual(len(obslist), nobs) #did we add observations?
#did they get the proper neighbor assignments
for obsnum in obsnums:
OBS = self.session.query(Observation).filter(
Observation.obsnum == obsnum).one()
#find the original record we put into add_observations and check that the neighbors match
for obs in obslist:
if obs['julian_date'] == OBS.julian_date:
if obs.has_key('neighbor_low'):
self.assertEqual(OBS.low_neighbors[0].julian_date,
obs['neighbor_low'])
if obs.has_key('neighbor_high'):
self.assertEqual(OBS.high_neighbors[0].julian_date,
obs['neighbor_high'])
break
def test_list_observations(self):
#form up the observation list
obslist = []
jds = n.arange(0, 10) * self.length + 2456446.1234
pols = ['xx', 'yy', 'xy', 'yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({
'julian_date': jds[jdi],
'pol': pol,
'host': self.host,
'filename': self.filename,
'length': self.length
})
if jdi != 0:
obslist[-1]['neighbor_low'] = jds[jdi - 1]
if jdi < len(jds[:-1]):
obslist[-1]['neighbor_high'] = jds[jdi + 1]
obsnums = self.dbi.add_observations(obslist)
tic = time.time()
observations = self.dbi.list_observations()
#print "time to execute list_observations",time.time()-tic,'s'
self.assertEqual(n.sum(n.array(observations) - n.array(obsnums)), 0)
def test_get_neighbors(self):
"""
First set up a likely triplet of observations
"""
#form up the observation list
obslist = []
jds = n.arange(0, 10) * self.length + 2456446.1234
pols = ['xx', 'yy', 'xy', 'yx']
for pol in pols:
for jdi in xrange(len(jds)):
obslist.append({
'julian_date': jds[jdi],
'pol': pol,
'host': self.host,
'filename': self.filename,
'length': self.length
})
if jdi != 0:
obslist[-1]['neighbor_low'] = jds[jdi - 1]
if jdi != (len(jds) - 1):
obslist[-1]['neighbor_high'] = jds[jdi + 1]
obsnums = self.dbi.add_observations(obslist)
obsnums.sort()
i = 5 # I have ten time stamps. this guys should have plenty of neighbors
mytestobsnum = obsnums[i] #choose a middle obs
tic = time.time()
neighbors = self.dbi.get_neighbors(mytestobsnum)
#print "time to execute get_neighbors",time.time()-tic,'s'
self.assertEqual(len(neighbors), 2)
self.assertEqual(neighbors[0], obsnums[i - 1]) #low
self.assertEqual(neighbors[1], obsnums[i + 1]) #high
def test_set_obs_status(self):
"""
set the status with the dbi function then check it with
under the hood stuff
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
# then set the status to something else
self.dbi.set_obs_status(obsnum, 'UV')
# get the status back out
OBS = self.session.query(Observation).filter(
Observation.obsnum == obsnum).one()
self.assertEqual(OBS.status, 'UV')
def test_get_obs_status(self):
"""
set the status with the dbi function (cause I tested it with more basic tests already)
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
# then set the status to something else
self.dbi.set_obs_status(obsnum, 'UV')
#then get the status back
tic = time.time()
status = self.dbi.get_obs_status(obsnum)
#print "time to execute get_obs_status",time.time()-tic,'s'
self.assertEqual(status, 'UV')
def test_time_transaction(self):
"""
"""
#first create an observation in the first place
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
# then set the status to something else
tic = time.time()
self.dbi.set_obs_status(obsnum, 'UV')
print "time to execute set_obs_status", time.time() - tic, 's'
def test_get_input_file(self):
"""
create an observation
add the initial file
get the pot host and path
"""
#first add the observation
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
#then add a file to it
host, path, filename = self.dbi.get_input_file(obsnum)
self.assertEqual(host, self.host)
self.assertEqual(path, os.path.dirname(self.filename))
self.assertEqual(filename, os.path.basename(self.filename))
def test_get_output_location(self):
"""
create an observation
add the initial file
get the pot host and path
"""
#first add the observation
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
host, path = self.dbi.get_output_location(obsnum)
self.assertEqual(host, self.host)
self.assertEqual(path, os.path.dirname(self.filename))
def test_still_path(self):
"""
"""
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
self.dbi.set_obs_still_path(obsnum, '/data/')
still_path = self.dbi.get_obs_still_path(obsnum)
self.assertEqual(still_path, '/data/')
def test_obs_pid(self):
"""
"""
obsnum = self.dbi.add_observation(self.jd, self.pol, self.filename,
self.host)
self.dbi.set_obs_pid(obsnum, 9999)
pid = self.dbi.get_obs_pid(obsnum)
self.assertEqual(pid, 9999)
help='set log level to debug')
o.add_option('--status',default='UV_POT',
help='set the observation to this status [default=UV_POT]')
opts, args = o.parse_args(sys.argv[1:])
#connect to the database
if opts.v:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(lebel=logging.INFO)
logger = logging.getLogger('reset_observations')
dbi = DataBaseInterface()
# for each file get the obsnum, then reset the status to UV_POT
obsnums = []
for filename in args:
logger.debug("looking for file {filename}".format(filename=filename))
s = dbi.Session()
FILE = s.query(File).filter(File.filename==filename).one()#XXX note assumes we are not noting that this file is copied.
obsnum = FILE.obsnum
logger.debug("found obsnum {obsnum}".format(obsnum=obsnum))
s.close()
logger.debug("setting status to {status}".format(status=opts.status))
dbi.set_obs_status(obsnum,opts.status)
dbi.set_obs_pid(obsnum,None)
dbi.set_obs_still_host(obsnum,None)
dbi.add_log(obsnum,'NEW',"issuing a reset_observations",0)
