def __init__(self):
super(proc_daemon, self).__init__()
## Constructor time
self._creation_ts = time.time()
## API for proc_daemon is ds_master to interact with ProcessTable
self._api = ds_master(pubdb_conn_info.admin_info(),
logger=self._logger)
## Array of proc_action instance (one per project)
self._project_v = {}
## Array of execution timestamp (one per project)
self._exe_time_v = {}
## Array of execution counter (one per project)
self._exe_ctr_v = {}
## Network node name
self._server = pub_env.kSERVER_NAME
## Server configuration ... instantiated within load_daemon
self._config = None
## Log ... instantiated within load_daemon
self._log = None
## Next execution timestamp ... for processing
self._next_exec_time = time.time()
## SIGINT marker
self._exit_routine = False
## Log function
self._logger_func = None
## Server status handler
self._server_handler = None
def __init__(self):
super(proc_daemon,self).__init__()
## Constructor time
self._creation_ts = time.time()
## API for proc_daemon is ds_master to interact with ProcessTable
self._api = ds_master(pubdb_conn_info.admin_info(),logger=self._logger)
## Array of proc_action instance (one per project)
self._project_v = {}
## Array of execution timestamp (one per project)
self._exe_time_v = {}
## Array of execution counter (one per project)
self._exe_ctr_v = {}
## Network node name
self._server = pub_env.kSERVER_NAME
## Server configuration ... instantiated within load_daemon
self._config = None
## Log ... instantiated within load_daemon
self._log = None
## Next execution timestamp ... for processing
self._next_exec_time = time.time()
## SIGINT marker
self._exit_routine = False
## Log function
self._logger_func = None
## Server status handler
self._server_handler = None
#!/usr/bin/env python
import sys
from dstream.ds_api import ds_master
from pub_dbi import pubdb_conn_info
from pub_util import pub_logger
import time
if not len(sys.argv) == 2:
print
print 'Usage: %s PROJECT_NAME' % sys.argv[0]
print
sys.exit(1)
logger = pub_logger.get_logger('ds_master')
k = ds_master(pubdb_conn_info.admin_info(), logger)
if not k.connect():
sys.exit(1)
k.project_version_update(sys.argv[1])
sys.exit(0)
if conn.project_exist(p._project):
status = status and conn.project_version_update(p)
else:
status = status and conn.define_project(p)
return status
if __name__ == '__main__':
logger = pub_logger.get_logger('register_project')
# DB interface for altering ProcessTable
conn=ds_master(pubdb_conn_info.admin_info(),logger)
# Connect to DB
conn.connect()
if len(sys.argv)<2:
print 'Usage:',sys.argv[0],'$CONFIG_FILE'
sys.exit(1)
c = open(sys.argv[1],'r').read()
projects = register_project.parse(conn,logger,c)
register(conn,logger,projects)
#!/usr/bin/env python
import argparse, sys
from dstream.ds_api import death_star
from pub_dbi import pubdb_conn_info
from pub_util import pub_logger
myparser = argparse.ArgumentParser(description='Remove a run table.')
if not len(sys.argv) == 2:
print 'Usage: %s TABLE_NAME' % sys.argv[0]
sys.exit(1)
tname = sys.argv[1]
logger = pub_logger.get_logger('death_star')
k=death_star( pubdb_conn_info.admin_info(),
logger )
if not k.connect():
sys.exit(1)
k.end_of_galaxy(tname)
myparser = argparse.ArgumentParser(description='Filling a run table w/ new run/subrun')
myparser.add_argument('--name', dest='name', action='store',
default='TestRun', type=str,
help='Name of a run table to create/alter')
myparser.add_argument('--run', dest='run', action='store',
default=0, type=int,
help='Run number to be added')
myparser.add_argument('--nsubruns',dest='nsubruns',action='store',
default=0, type=int,
help='Number of sub-runs to be added')
args = myparser.parse_args()
logger = pub_logger.get_logger('death_star')
k=death_star( pubdb_conn_info.admin_info(),
logger )
if not k.connect():
sys.exit(1)
# fake time stamp
ts = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime( time.time() ) )
for subrun in xrange(args.nsubruns):
k.insert_into_death_star(args.name,args.run,subrun,ts,ts)
def process_newruns(self):
# Attempt to connect DB. If failure, abort
if not self.connect():
self.error('Cannot connect to DB! Aborting...')
return
# loop through all directories in which data is to be found
for path_num in xrange(len(self._data_dir)):
data_path = self._data_dir[path_num]
self.info('Start access in data directory %s' % data_path)
run_lim = None
# if we are not yet at the last directory (for which there should be no run limit)
if (path_num < len(self._run_bound)):
run_lim = self._run_bound[path_num]
self.info('Run limit for this directory: %i' % run_lim)
# get ALL closed data files in DATADIR
if (os.path.isdir(data_path) == False):
self.error('DATA DIR %s does not exist' % data_path)
return
self.info('Looking for data files in: %s' % data_path)
dircontents = os.listdir(data_path)
# create a dictionary to keep track of
# - file name ----- NAME
# - run number ---- RUN
# - subrun number - SUBRUN
# - time-create --- TIMEC
# - time-modify --- TIMEM
# : dictionary key: ------ (RUN,SUBRUN)
# : dictionary content: -- (NAME,TIMEC,TIMEM)
file_info = {}
for f in dircontents:
filepath = data_path + '/' + f
# check that this is a file
if (os.path.isfile(filepath) == False):
continue
try:
time_create = os.path.getctime(filepath)
time_modify = os.path.getmtime(filepath)
# file format:
# NoiseRun-YYYY_M_DD_HH_MM_SS-RUN-SUBRUN.ubdaq
run = int(f.replace('.ubdaq', '').split('-')[-2])
subrun = int(f.replace('.ubdaq', '').split('-')[-1])
#self.info('found run (%i, %i)'%(run,subrun))
file_info[tuple(
(run, subrun))] = [f, time_create, time_modify]
except:
# if file-name is .ubdaq then we have a problem
# were not able to read run-subrun info from file
if (f.find('.ubdaq')):
self.info(
'Could not read RUN/SUBRUN info for file %s' % f)
# sort the dictionary
# we want to ignore the largest run/subrun information
# this will prevent us from potentially logging info
# for a file that has not yet been closed
sorted_file_info = sorted(file_info)
# get tuple with largest run/subrun info found in files
max_file_info = sorted_file_info[-1]
# fetch from database the last run/subrun number recorded
logger = pub_logger.get_logger('register_new_run')
reader = ds_api.ds_reader(pubdb_conn_info.reader_info(), logger)
last_recorded_info = reader.get_last_run_subrun(self._runtable)
# log which (run,subrun) pair was added last
self.info('last recorded (run,subrun) is (%d,%d)' %
(int(last_recorded_info[0]), int(last_recorded_info[1])))
self.info(
'No run with (run,subrun) smaller than this will be added to the RunTable'
)
# if we made it this far the file info needs to be
# recorded to the database
# DANGER *** DANGER *** DANGER *** DANGER
# we will now invoke the death_start
# this API will access the RUN table end edit
# informaton, which is exactly what we need to do
# however, this is dangerous and you should not
# copy this code and re-use it somewhere
# if you do, the Granduca's wrath will be upon you
# lucikly for you, the Granduca was the first to
# abolish the death penalty on November 30th 1786
# http://en.wikipedia.org/wiki/Grand_Duchy_of_Tuscany#Reform
# However, the imperial army may be less mercyful.
# DANGER *** DANGER *** DANGER *** DANGER
logger = pub_logger.get_logger('death_star')
rundbWriter = ds_api.death_star(pubdb_conn_info.admin_info(),
logger)
# loop through dictionary keys and write to DB info
# for runs/subruns not yet stored
for info in sorted_file_info:
# this key needs to be larger than the last logged value
# but less than the last element in the dictionary
if (info >= max_file_info):
continue
if (run_lim):
if (info[0] > run_lim):
continue
if (info <= last_recorded_info):
continue
self.info('Trying to add to RunTable (run,subrun) = (%d,%d)' %
(int(info[0]), int(info[1])))
try:
# info is key (run,subrun)
# dictionary value @ key is array
# [file name, time_crate, time_modify]
run = info[0]
subrun = info[1]
run_info = file_info[info]
file_creation = time.gmtime(int(run_info[1]))
file_closing = time.gmtime(int(run_info[2]))
file_creation = time.strftime('%Y-%m-%d %H:%M:%S',
file_creation)
file_closing = time.strftime('%Y-%m-%d %H:%M:%S',
file_closing)
self.info('filling death star...')
# insert into the death start
rundbWriter.insert_into_death_star(self._runtable, info[0],
info[1], file_creation,
file_closing)
# Report starting
self.info(
'recording info for new run: run=%d, subrun=%d ...' %
(int(run), int(subrun)))
except:
# we did not succeed in adding this (run,subrun)
self.info('FAILED to add run=%d, subrun=%d to RunTable' %
(int(run), int(subrun)))
def process_newruns(self):
# Attempt to connect DB. If failure, abort
if not self.connect():
self.error('Cannot connect to DB! Aborting...')
return
# loop through all directories in which data is to be found
for path_num in xrange(len(self._data_dir)):
data_path = self._data_dir[path_num]
self.info('Start access in data directory %s'%data_path)
run_lim = None
# if we are not yet at the last directory (for which there should be no run limit)
if ( path_num < len(self._run_bound) ):
run_lim = self._run_bound[path_num]
self.info('Run limit for this directory: %i'%run_lim)
# get ALL closed data files in DATADIR
if (os.path.isdir(data_path) == False):
self.error('DATA DIR %s does not exist'%data_path)
return
self.info('Looking for data files in: %s'%data_path)
dircontents = os.listdir(data_path)
# create a dictionary to keep track of
# - file name ----- NAME
# - run number ---- RUN
# - subrun number - SUBRUN
# - time-create --- TIMEC
# - time-modify --- TIMEM
# : dictionary key: ------ (RUN,SUBRUN)
# : dictionary content: -- (NAME,TIMEC,TIMEM)
file_info = {}
for f in dircontents:
filepath = data_path+'/'+f
# check that this is a file
if (os.path.isfile(filepath) == False):
continue
try:
time_create = os.path.getctime(filepath)
time_modify = os.path.getmtime(filepath)
# file format:
# NoiseRun-YYYY_M_DD_HH_MM_SS-RUN-SUBRUN.ubdaq
run = int(f.replace('.ubdaq','').split('-')[-2])
subrun = int(f.replace('.ubdaq','').split('-')[-1])
#self.info('found run (%i, %i)'%(run,subrun))
file_info[tuple((run,subrun))] = [f,time_create,time_modify]
except:
# if file-name is .ubdaq then we have a problem
# were not able to read run-subrun info from file
if (f.find('.ubdaq')):
self.info('Could not read RUN/SUBRUN info for file %s'%f)
# sort the dictionary
# we want to ignore the largest run/subrun information
# this will prevent us from potentially logging info
# for a file that has not yet been closed
sorted_file_info = sorted(file_info)
# get tuple with largest run/subrun info found in files
max_file_info = sorted_file_info[-1]
# fetch from database the last run/subrun number recorded
logger = pub_logger.get_logger('register_new_run')
reader = ds_api.ds_reader(pubdb_conn_info.reader_info(), logger)
last_recorded_info = reader.get_last_run_subrun(self._runtable)
# log which (run,subrun) pair was added last
self.info('last recorded (run,subrun) is (%d,%d)'%(int(last_recorded_info[0]),int(last_recorded_info[1])))
self.info('No run with (run,subrun) smaller than this will be added to the RunTable')
# if we made it this far the file info needs to be
# recorded to the database
# DANGER *** DANGER *** DANGER *** DANGER
# we will now invoke the death_start
# this API will access the RUN table end edit
# informaton, which is exactly what we need to do
# however, this is dangerous and you should not
# copy this code and re-use it somewhere
# if you do, the Granduca's wrath will be upon you
# lucikly for you, the Granduca was the first to
# abolish the death penalty on November 30th 1786
# http://en.wikipedia.org/wiki/Grand_Duchy_of_Tuscany#Reform
# However, the imperial army may be less mercyful.
# DANGER *** DANGER *** DANGER *** DANGER
logger = pub_logger.get_logger('death_star')
rundbWriter = ds_api.death_star(pubdb_conn_info.admin_info(),logger)
# loop through dictionary keys and write to DB info
# for runs/subruns not yet stored
for info in sorted_file_info:
# this key needs to be larger than the last logged value
# but less than the last element in the dictionary
if (info >= max_file_info):
continue;
if (run_lim):
if ( info[0] > run_lim):
continue;
if (info <= last_recorded_info):
continue;
self.info('Trying to add to RunTable (run,subrun) = (%d,%d)'%(int(info[0]),int(info[1])))
try:
# info is key (run,subrun)
# dictionary value @ key is array
# [file name, time_crate, time_modify]
run = info[0]
subrun = info[1]
run_info = file_info[info]
file_creation = time.gmtime(int(run_info[1]))
file_closing = time.gmtime(int(run_info[2]))
file_creation = time.strftime('%Y-%m-%d %H:%M:%S',file_creation)
file_closing = time.strftime('%Y-%m-%d %H:%M:%S',file_closing)
self.info('filling death star...')
# insert into the death start
rundbWriter.insert_into_death_star(self._runtable,info[0],info[1],file_creation, file_closing)
# Report starting
self.info('recording info for new run: run=%d, subrun=%d ...' % (int(run),int(subrun)))
except:
# we did not succeed in adding this (run,subrun)
self.info('FAILED to add run=%d, subrun=%d to RunTable'%(int(run),int(subrun)))