log.info(LOG_BREAK)
log.info('Starting summariser.')
# This contains all the summarising logic, contained in ApelMysqlDb() and the stored procedures.
db.summarise()
log.info('Summarising complete.')
log.info(LOG_BREAK)
if unloader_enabled:
log.info(LOG_BREAK)
log.info('Starting unloader.')
log.info('Will unload from %s.' % table_name)
interval = ccp.get('unloader', 'interval')
unloader = DbUnloader(db, unload_dir, include_vos, exclude_vos, local_jobs)
try:
if interval == 'latest':
msgs, recs = unloader.unload_latest(table_name, send_ur)
elif interval == 'gap':
start = ccp.get('unloader', 'gap_start')
end = ccp.get('unloader', 'gap_end')
msgs, recs = unloader.unload_gap(table_name, start, end, send_ur)
elif interval == 'all':
msgs, recs = unloader.unload_all(table_name, send_ur)
else:
log.warn('Unrecognised interval: %s' % interval)
log.warn('Will not start unloader.')
log.info('Unloaded %d records in %d messages.' % (recs, msgs))
withhold_dns = cp.getboolean('unloader', 'withhold_dns')
except ConfigParser.NoOptionError:
withhold_dns = False
include_vos = None
exclude_vos = None
try:
include = cp.get('unloader', 'include_vos')
include_vos = [vo.strip() for vo in include.split(',')]
except ConfigParser.NoOptionError:
# Only exclude VOs if we haven't specified the ones to include.
try:
exclude = cp.get('unloader', 'exclude_vos')
exclude_vos = [vo.strip() for vo in exclude.split(',')]
except ConfigParser.NoOptionError:
pass
unloader = DbUnloader(db, unload_dir, include_vos, exclude_vos, local_jobs,
withhold_dns)
try:
msgs, recs = unloader.unload_latest(table_name, send_ur)
log.info('%d records in %d messages unloaded from %s' %
(recs, msgs, table_name))
except KeyError:
log.error('Invalid table name: %s, omitting' % table_name)
except ApelDbException, e:
log.error('Unloading failed: %s' % e)
log.info('Unloading complete.')
log.info('=====================')
include_vos = None
exclude_vos = None
try:
include = cp.get('unloader', 'include_vos')
include_vos = [vo.strip() for vo in include.split(',')]
except ConfigParser.NoOptionError:
# Only exclude VOs if we haven't specified the ones to include.
try:
exclude = cp.get('unloader', 'exclude_vos')
exclude_vos = [vo.strip() for vo in exclude.split(',')]
except ConfigParser.NoOptionError:
pass
interval = cp.get('unloader', 'interval')
unloader = DbUnloader(db, unload_dir, include_vos, exclude_vos, local_jobs,
withhold_dns)
try:
records_per_message = int(cp.get('unloader', 'records_per_message'))
if records_per_message < RECORDS_PER_MESSAGE_MIN:
unloader.records_per_message = RECORDS_PER_MESSAGE_MIN
log.warn(
'records_per_message too small, increasing from %d to %d',
records_per_message,
RECORDS_PER_MESSAGE_MIN,
)
elif records_per_message > RECORDS_PER_MESSAGE_MAX:
unloader.records_per_message = RECORDS_PER_MESSAGE_MAX
log.warn(
'records_per_message too large, decreasing from %d to %d',
records_per_message,
RECORDS_PER_MESSAGE_MAX,
# This contains all the summarising logic, contained in ApelMysqlDb() and
# the stored procedures.
db.summarise_jobs()
log.info('Summarising complete.')
log.info(LOG_BREAK)
if unloader_enabled:
log.info(LOG_BREAK)
log.info('Starting unloader.')
log.info('Will unload from %s.', table_name)
interval = ccp.get('unloader', 'interval')
withhold_dns = ccp.getboolean('unloader', 'withhold_dns')
unloader = DbUnloader(db, unload_dir, include_vos, exclude_vos,
local_jobs, withhold_dns)
try:
if interval == 'latest':
msgs, recs = unloader.unload_latest(table_name, send_ur)
elif interval == 'gap':
start = ccp.get('unloader', 'gap_start')
end = ccp.get('unloader', 'gap_end')
msgs, recs = unloader.unload_gap(table_name, start, end,
send_ur)
elif interval == 'all':
msgs, recs = unloader.unload_all(table_name, send_ur)
else:
log.warn('Unrecognised interval: %s', interval)
log.warn('Will not start unloader.')
log.info('Unloaded %d records in %d messages.', recs, msgs)
log.info('Starting APEL dbunloader %s.%s.%s' % __version__)
unload_dir = cp.get('unloader', 'dir_location')
table_name = cp.get('unloader', 'table_name')
send_ur = cp.getboolean('unloader', 'send_ur')
local_jobs = cp.getboolean('unloader', 'local_jobs')
include_vos = None
exclude_vos = None
try:
include = cp.get('unloader', 'include_vos')
include_vos = [ vo.strip() for vo in include.split(',') ]
except ConfigParser.NoOptionError:
# Only exclude VOs if we haven't specified the ones to include.
try:
exclude = cp.get('unloader', 'exclude_vos')
exclude_vos = [ vo.strip() for vo in exclude.split(',') ]
except ConfigParser.NoOptionError:
pass
unloader = DbUnloader(db, unload_dir, include_vos, exclude_vos, local_jobs)
try:
msgs, recs = unloader.unload_latest(table_name, send_ur)
except KeyError:
log.warning('Invalid table name: %s, omitting' % table_name)
log.info('%d records in %d messages unloaded from %s' % (recs, msgs, table_name))
log.info('Unloading complete.')
log.info('=====================')
