def main(options, args):
#logger = ssdlog.make_logger('checkstat', options)
ro.init()
st = ro.remoteObjectProxy(options.statussvc)
insdata = INSdata()
statusDict = {}
lookupDict = {}
# Get all the names of the 'ALIVE' status tables for the instruments
for insname in insdata.getNames():
if insname == 'VGW':
continue
inscode = insdata.getCodeByName(insname)
tblname = '%3.3sS0001' % inscode
alias = 'GEN2.STATUS.TBLTIME.%s' % tblname
statusDict[alias] = 0
lookupDict[alias] = insname
# Additional tables to check
for tblname in ('TSCS', 'TSCL', 'TSCV', 'VGWD', 'VGWQ'):
alias = 'GEN2.STATUS.TBLTIME.%s' % tblname
statusDict[alias] = 0
lookupDict[alias] = tblname
fetchDict = st.fetch(statusDict)
if options.sorttime:
times = fetchDict.items()
times.sort(_timecmp)
keys = [(alias, lookupDict[alias]) for alias in \
map(lambda x: x[0], times)]
else:
keys = lookupDict.items()
keys.sort(lambda x, y: cmp(x[1], y[1]))
#print keys
for alias, name in keys:
timeval = fetchDict[alias]
if timeval == '##NODATA##':
time_s = 'No record'
elif isinstance(timeval, float):
time_s = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(timeval))
else:
time_s = 'ERROR: %s' % str(timeval)
print "%-8.8s %s" % (name, time_s)
class MockFrameSource(object):
def __init__(self):
self.count = 1
self.insconfig = INSconfig()
def get(self, *args):
if len(args) == 3:
(insname, frametype, count) = args
count = int(count)
inscode = self.insconfig.getCodeByName(insname)
frameid = ('%-3.3s%1.1s%08.8d:%04.4d' %
(inscode, frametype, self.count, count))
elif len(args) == 2:
(insname, frametype) = args
inscode = self.insconfig.getCodeByName(insname)
frameid = ('%-3.3s%1.1s%08.8d' % (inscode, frametype, self.count))
count = 1
else:
raise DecodeError("Bad arguments to get_f_no: %s" % str(args))
self.count += count
return frameid
class Frame(object):
def __init__(self, summit_dir, base_dir, error=None, logger=None):
self.error = error
self.logger = logger
self.stars = None # frames in STARS
self.summit = {} # all frames @ summit
self.base = {} # all frames @ base
self.summit_a = {} # A-frames @ summit
self.base_a = {} # A-frames @ base
self.summit_q = {} # Q-frames @ summit
self.base_q = {} # Q-frames @ base
self.intersect = None # union of summit and base frames
self.summit_dir = summit_dir # frame dir @summit
self.base_dir = base_dir # frame dir @base
self.obcp = INSdata()
self.starsdb = STARSdb(logger)
def get_base(self):
self.logger.debug('getting base frames...')
sync_base = {}
try:
frames = set(self.summit.keys()).difference(self.base.keys())
for f in frames:
sync_base[f] = self.summit[f]
return sync_base
except Exception as e:
self.logger.error('error: getting base frames... %s' % e)
if self.error:
enditer = self.error.get_end_iter()
self.error.insert(enditer,
'error: getting base frames... %s\n' % e)
def get_summit(self):
self.logger.debug('getting summit frames...')
sync_summit = {}
try:
frames = set(self.base.keys()).difference(self.summit.keys())
for f in frames:
sync_summit[f] = self.base[f]
return sync_summit
except Exception as e:
self.logger.error('error: getting summit frames... %s' % e)
if self.error:
enditer = self.error.get_end_iter()
self.error.insert(enditer,
'error: getting summit frames... %s\n' % e)
def get_stars(self):
self.logger.debug('getting stars frames...')
frames = list(set(self.intersect).difference(self.stars))
try:
stars = {}
for frame in frames:
stars[frame] = self.base[frame]
return stars
except Exception as e:
self.logger.error('error: getting stars frames... %s' % e)
if self.error:
enditer = self.error.get_end_iter()
self.error.insert(enditer,
'error: getting stars frames... %s\n' % e)
def get_delete(self):
self.logger.debug('getting delete frames...')
try:
frames = list(set(self.intersect).intersection(self.stars))
delete = {}
for frame in frames:
delete[frame] = self.base[frame]
return delete
except Exception as e:
self.logger.error('error: getting summit frames... %s' % e)
if self.error:
enditer = self.error.get_end_iter()
self.error.insert(enditer,
'error: getting delete frames... %s\n' % e)
def __search_stars(self, frameid):
self.logger.debug('querying stars...')
# note: STARS somehow does not let me query both A/Q frames in parallel.
# need to query one after other
stars_q = self.__search_Q_frames(frameid)
stars_a = self.__search_A_frames(frameid)
self.stars = stars_a + stars_q
def __search_Q_frames(self, frameid):
intersect_q = list(
set(self.base_q.keys()).intersection(self.summit_q.keys()))
stars_q = []
if not len(intersect_q):
return stars_q
intersect_q.sort()
sframe, e = os.path.splitext(intersect_q[0])
eframe, e = os.path.splitext(intersect_q[-1])
self.logger.debug('ins=%s Q between %s and %s' %
(frameid, sframe, eframe))
stars = self.starsdb.are_frames_in_stars(frameid, sframe, eframe)
stars_q.extend(map(lambda x: '%s.fits' % x, stars))
self.logger.debug('Q query done num=%d' % len(stars_q))
return stars_q
def __search_A_frames(self, frameid):
intersect_a = list(
set(self.base_a.keys()).intersection(self.summit_a.keys()))
stars_a = []
if not len(intersect_a):
return stars_a
intersect_a.sort()
sframe, e = os.path.splitext(intersect_a[0])
eframe, e = os.path.splitext(intersect_a[-1])
self.logger.debug('ins=%s A between %s and %s' %
(frameid, sframe, eframe))
stars = self.starsdb.are_frames_in_stars(frameid, sframe, eframe)
stars_a.extend(map(lambda x: '%s.fits' % x, stars))
self.logger.debug('A query done num=%d' % len(stars_a))
return stars_a
def collect_frames(self, ins):
today = datetime.datetime.today()
frameid = self.obcp.getCodeByName(ins)
try:
self.base_a, self.base_q = self.__collect_frames(
self.base_dir, ins, frameid, today)
self.summit_a, self.summit_q = self.__collect_frames(
self.summit_dir, ins, frameid, today)
self.summit.clear()
self.base.clear()
self.summit.update(self.summit_a)
self.summit.update(self.summit_q)
self.base.update(self.base_a)
self.base.update(self.base_q)
self.intersect = list(
set(self.base.keys()).intersection(self.summit.keys()))
self.logger.debug(
'base_a=%d base_q=%d base=%d' %
(len(self.base_a), len(self.base_q), len(self.base)))
self.logger.debug(
'summit_a=%d summit_q=%d summit=%d' %
(len(self.summit_a), len(self.summit_q), len(self.summit)))
self.logger.debug('intersect=%d' % (len(self.intersect)))
#
self.__search_stars(frameid)
except Exception as e:
self.logger.error('error: collecting frames. %s ' % e)
if self.error:
enditer = self.error.get_end_iter()
self.error.insert(enditer,
'error: collecting frames.. %s\n' % e)
def __collect_frames(self, path, ins, frameid, today):
#print 'collecting base frame....'
Afits = '%sA*.fits' % frameid
Qfits = '%sQ*.fits' % frameid
ins_dir = os.path.join(path, ins)
os.chdir(ins_dir)
Aframes = glob.glob(Afits)
Qframes = glob.glob(Qfits)
Aframes = self.__get_frame_info(Aframes, today)
Qframes = self.__get_frame_info(Qframes, today)
return (Aframes, Qframes)
def __get_frame_info(self, frames, today):
""" get time stamp, age, size of a file """
stat = os.stat
fromtimestamp = datetime.datetime.fromtimestamp
frame_info = {} #; append=frame_info.append
for frame in frames:
info = stat(frame)
m_time = fromtimestamp(info.st_mtime)
age = today - m_time
frame_info[frame] = (m_time.strftime("%Y-%m-%d %H:%M:%S"),
age.days, info.st_size)
return frame_info
class DbTool(object):
"""Simple class to manipulate the SOSS database. Uses the SOSSdb module
access functions.
"""
def __init__(self, logger, host):
"""Constructor.
"""
self.logger = logger
# Create SOSS DB object
self.db = SOSSdb.SOSSdb(self.logger, host=host)
# For looking up instruments
self.ins_config = INSconfig()
super(DbTool, self).__init__()
def check_options(self, options, optionlist):
"""Check an options object (can be a Bunch, generic object, or as e.g. returned
by optparse module) for a set of required attributes that must be NOT None.
If any of the options are missing, an error message is generated to prompt for
the missing ones.
If the option name and the options attribute have different names, then they can
be distinguished by a colon in the optionlist.
Example:
optionlist: ['starttime', 'endtime', 'inst:instruments', 'obsgrp:division']
The options object must contain the attributes starttime, endtime, instruments
and division. Supposed endtime and division are None. Then the following
error string is logged and an Error raised:
'Missing options: --endtime, --obsgrp
"""
missing = []
for varname in optionlist:
option = varname
if ':' in varname:
(option, varname) = varname.split(':')
try:
val = getattr(options, varname)
if val == None:
missing.append('--%s' % option)
except AttributeError:
missing.append('--%s' % option)
if len(missing) > 0:
raise Error("Missing options: %s" % ', '.join(missing))
def set_inst(self, bnch, instruments):
"""Check a list of instrument names for correctness and set the instruments
attribute in the bunch (bnch is a record from the SOSS usradmintbl).
If an instrument using AO is specified, and AO is not present, then add AO
to the list.
"""
for insname in instruments:
insname = insname.upper()
try:
self.ins_config.getCodeByName(insname)
except KeyError:
errmsg = "'%s' is not a valid instrument name" % insname
raise Error(errmsg)
# Append AO if CIAO specified, but not AO as well
if ('CIAO' in instruments) and not ('AO' in instruments):
instruments.append('AO')
bnch.instruments = instruments
def _fillRecord(self, bnch, options):
"""Update a record (bnch) from the SOSS usradmintbl. bnch is just a Bunch
object with fields set to python translated values from the table. Options
is another Bunch or any object with matching attributes. If any of them
are present and NOT None, then they are copied with the approriate checks
and type conversions to the record and then the record is written to the
database. May raise an Error.
"""
# Update start and end dates
# TODO: parse variety of date formats, not just twin style
if options.starttime:
twin_starttime = options.starttime
if not ':' in twin_starttime:
twin_starttime += '-00:00:00'
if not re.match(r'^\d{4}\-\d{2}\-\d{2}\-\d{2}:\d{2}:\d{2}$',
twin_starttime):
raise Error("Please specify dates in the form YYYY-MM-DD[-HH:MM:SS]")
bnch.starttime = SOSSdb.datetime_fromSOSS2(twin_starttime)
if options.endtime:
twin_endtime = options.endtime
if not ':' in twin_endtime:
twin_endtime += '-00:00:00'
if not re.match(r'^\d{4}\-\d{2}\-\d{2}\-\d{2}:\d{2}:\d{2}$',
twin_endtime):
raise Error("Please specify dates in the form YYYY-MM-DD[-HH:MM:SS]")
bnch.endtime = SOSSdb.datetime_fromSOSS2(twin_endtime)
# Update division/observation group (to 'purpose' field in db)
if options.division:
#bnch.purpose = ObsProposalToSOSSobsGrp(options.division)
bnch.purpose = options.division
# Update instrument list. options value is a comma-separated string of
# isntrument names. If it begins with a + or - then those instruments
# are added or removed to the set already in bnch.instruments
if options.instruments:
if options.instruments.startswith('+'):
instruments = list(bnch.instruments)
add_instruments = options.instruments[1:].split(',')
for insname in add_instruments:
insname = insname.upper()
if not insname in instruments:
instruments.append(insname)
elif options.instruments.startswith('-'):
instruments = list(bnch.instruments)
sub_instruments = options.instruments[1:].split(',')
for insname in sub_instruments:
insname = insname.upper()
if insname in instruments:
instruments.remove(insname)
else:
instruments = options.instruments.split(',')
self.set_inst(bnch, instruments)
# Update "oplevel". This field is best left to default, unless you really
# know what you are doing.
if options.oplevel == 'default':
bnch.oplevel = g_oplevel['default'].split(',')
elif options.oplevel == 'advanced':
bnch.oplevel = g_oplevel['advanced'].split(',')
elif options.oplevel:
bnch.oplevel = options.oplevel.split(',')
self.logger.debug("Updated record is: %s" % str(bnch))
def getProp(self, propid):
"""Method to get a record from the usradmintbl.
"""
try:
return self.db.getProp(propid)
except SOSSdb.SOSSdbError, e:
raise Error("Couldn't access proposal: %s" % str(e))
class frameSvc(object):
"""
Implements a frame number service compatible with the one provided
by SOSS.
"""
def __init__(self, svcname, logger, monitor=None, monchannels=[]):
self.logger = logger
self.monitor = monitor
self.monchannels = monchannels
# TODO: parameterize tag template (from svcname?)
self.tag_template = 'mon.frame.%s.frameSvc'
self.insconfig = INSconfig()
# For updating status system with session info
# TODO: parameterize object
self.status = ro.remoteObjectProxy('status')
# For mutual exclusion
self.lock = threading.RLock()
# Need this because we don't inherit from ro.remoteObjectServer,
# but are delegated to from it
def ro_echo(self, arg):
return arg
def _set_necessary_status(self, insname, frameid, frametype):
instcode = self.insconfig.getCodeByName(insname)
statusDict = {}
if frametype == 'A':
statusDict['FITS.%s.FRAMEID' % instcode] = frameid
elif frametype == 'Q':
statusDict['FITS.%s.FRAMEIDQ' % instcode] = frameid
self.logger.debug("Updating status: %s" % str(statusDict))
self.status.store(statusDict)
def getFrameMap(self, insname, frametype):
self.logger.info('getting frame map....')
with self.lock:
insname = insname.upper()
frametype = frametype.upper()
if len(frametype) > 1:
prefix = frametype[1]
frametype = frametype[0]
else:
prefix = '0'
try:
frameid = framedb.getFrameMap(insname, frametype,
prefix=prefix)
return (ro.OK, frameid)
except Exception, e:
raise frameError("Error obtaining framemap info: %s" % (str(e)))