def __init__(self, dest, qpath, cert, key, api_version):
"""Initialize a Sender."""
self._cert = cert
self._key = key
self._outq = QueueSimple(qpath)
self._dest = dest
self._api_version = api_version
def write_to_ssm(extract, config):
"""forwards usage records to SSM"""
# ensure outgoing directory existence
ssm_input_path = os.path.expanduser(config['ssm_input_path'])
if not os.access(ssm_input_path, os.F_OK):
os.makedirs(ssm_input_path, 0755)
# only write non void URs file
if len(extract) > 0:
output = config['ssm_input_header'] + "\n"
# itterate over VMs
for vmname in extract.keys():
logging.debug("generating ssm input file for VM %s" % vmname)
for item in orderedFields:
logging.debug("generating record %s: %s" %
(item, extract[vmname][item]))
output += "%s: %s\n" % (item, extract[vmname][item])
output += config['ssm_input_sep'] + "\n"
# write file
try:
dirq = QueueSimple(ssm_input_path)
dirq.add(output)
except:
logging.error('unable to push message in apel-ssm queue <%s>' %
ssm_input_path)
else:
logging.debug('no usage records, skip forwarding to SSM')
def write_to_ssm( extract, config ):
"""forwards usage records to SSM"""
# ensure outgoing directory existence
ssm_input_path = os.path.expanduser(config['ssm_input_path'])
if not os.access(ssm_input_path, os.F_OK):
os.makedirs(ssm_input_path, 0755)
# only write non void URs file
if len(extract) > 0:
output = config['ssm_input_header'] + "\n"
# itterate over VMs
for vmname in extract.keys():
logging.debug("generating ssm input file for VM %s" % vmname)
for item in orderedFields:
logging.debug("generating record %s: %s" % (item, extract[vmname][item]) )
output += "%s: %s\n" % ( item, extract[vmname][item] )
output += config['ssm_input_sep'] + "\n"
# write file
try:
dirq = QueueSimple(ssm_input_path)
dirq.add(output)
except:
logging.error('unable to push message in apel-ssm queue <%s>' % ssm_input_path)
else:
logging.debug('no usage records, skip forwarding to SSM')
def test_get_delete_messages_one(self):
qpub = QueueSimple(self.test_dir, umask=0)
qpub.add('123')
qsub = DirectoryQueueClient(self.ch)
qsub.connect()
assert qsub.get_messages() == [123]
assert qpub.count() == 0
class DirectoryQueue(MsgBase):
def __init__(self, configHolder):
super(DirectoryQueue, self).__init__(configHolder)
self.queue = QueueSimple(self.msg_queue)
def send(self, message):
self.queue.add(message)
def __init__(self, db, qpath, inc_vos=None, exc_vos=None, local=False, withhold_dns=False):
self._db = db
outpath = os.path.join(qpath, "outgoing")
self._msgq = QueueSimple(outpath)
self._inc_vos = inc_vos
self._exc_vos = exc_vos
self._local = local
self._withhold_dns = withhold_dns
def test05_add_path(self):
'QueueSimple.add_path()'
qs = QueueSimple(self.qdir)
data = 'foo0oo'
path = self.tempdir + '/foo.bar'
open(path, 'w').write(data)
elem = qs.add_path(path)
assert open(self.qdir + '/' + elem).read() == data
self.failIf(os.path.exists(path))
def test08_count(self):
'QueueSimple.count()'
qs = QueueSimple(self.qdir)
# add "normal" element
qs.add('foo')
# simply add a file (fake element) into the elements directory
fake_elem = os.listdir(self.qdir)[0] + '/' + 'foo.bar'
open(self.qdir + '/' + fake_elem, 'w').write('')
self.assertEqual(qs.count(), 1)
def test09_remove(self):
'QueueSimple.remove()'
qs = QueueSimple(self.qdir, granularity=1)
for _ in range(5):
qs.add('foo')
assert qs.count() == 5
for elem in qs:
qs.lock(elem)
qs.remove(elem)
self.assertEqual(qs.count(), 0)
def test06_lock_unlock(self):
'QueueSimple.lock()'
qs = QueueSimple(self.qdir)
data = 'foo'
elem_name = 'foo.bar'
elem_full_path = self.qdir + '/' + elem_name
open(elem_full_path, 'w').write(data)
self.assertEqual(qs.lock(elem_name), 1)
self.failUnless(os.path.exists(elem_full_path + LOCKED_SUFFIX))
qs.unlock(elem_name)
def test02_add_data(self):
'QueueSimple._add_data()'
data = 'f0o'
qs = QueueSimple(self.qdir)
_subdirName, _fullFnOrig = qs._add_data(data)
subdirs = os.listdir(self.qdir)
assert len(subdirs) == 1
assert _subdirName == subdirs[0]
subdir = self.qdir + '/' + subdirs[0]
files = os.listdir(subdir)
assert len(files) == 1
fn = subdir + '/' + files[0]
assert _fullFnOrig == fn
assert open(fn, 'r').read() == data
def test03_add_path(self):
'QueueSimple._add_path()'
data = 'abc'
qs = QueueSimple(self.qdir)
_dir = 'elems'
elems_dir = self.qdir + '/' + _dir
os.mkdir(elems_dir)
_tmpName = self.tempdir + '/elem.tmp'
open(_tmpName, 'w').write(data)
assert len(os.listdir(self.tempdir)) == 2
newName = qs._add_path(_tmpName, _dir)
assert len(os.listdir(elems_dir)) == 1
assert len(os.listdir(self.tempdir)) == 1
assert open(self.qdir + '/' + newName).read() == data
def check_dir(root):
'''
Check the directory for incoming, outgoing, reject
or accept directories. If they exist, check them for
messages.
'''
print '\nStarting message status script.'
print 'Root directory: %s\n' % root
queues = []
incoming = os.path.join(root, 'incoming')
if os.path.isdir(incoming):
queues.append(Queue(incoming, schema=QSCHEMA))
outgoing = os.path.join(root, 'outgoing')
# outgoing uses QueueSimple, not Queue
if os.path.isdir(outgoing):
queues.append(QueueSimple(outgoing))
reject = os.path.join(root, 'reject')
if os.path.isdir(reject):
queues.append(Queue(reject, schema=QSCHEMA))
accept = os.path.join(root, 'accept')
if os.path.isdir(accept):
queues.append(Queue(accept, schema=QSCHEMA))
for q in queues:
msgs, locked = check_queue(q)
#check_empty_dirs(q)
print ' Messages: %s' % msgs
print ' Locked: %s\n' % locked
if locked > 0:
question = 'Unlock %s messages?' % locked
if ask_user(question):
clear_locks(q)
def __init__(self, db, qpath, inc_vos=None, exc_vos=None, local=False):
self._db = db
outpath = os.path.join(qpath, "outgoing")
self._msgq = QueueSimple(outpath)
self._inc_vos = inc_vos
self._exc_vos = exc_vos
self._local = local
def test01_init(self):
'QueueSimple.__init__()'
path = self.tempdir + '/aaa/bbb/ccc'
granularity = 30
qs = QueueSimple(path, granularity=granularity)
assert qs.path == path
assert qs.granularity == granularity
def __init__(self, dest, qpath, cert, key, api_version):
"""Initialize a Sender."""
self._cert = cert
self._key = key
self._outq = QueueSimple(qpath)
self._dest = dest
self._api_version = api_version
def test_get_delete_messages_max(self):
num_messages_half = 5
num_messages_all = 10
qpub = QueueSimple(self.test_dir)
for i in range(num_messages_all):
qpub.add('{"message" : "%i"}' % i)
qsub = DirectoryQueueClient(self.ch)
qsub.connect()
messages = qsub.get_messages(num_messages_half)
assert len(messages) == num_messages_half
assert qpub.count() == num_messages_half
messages_read = qsub.get_messages(num_messages_half)
messages_test = [{'message': str(i)} for i in range(num_messages_half, num_messages_all)]
assert messages_read == messages_test
assert qpub.count() == 0
def submit_state_change(job, transfer, transfer_state):
"""
Writes a state change message to the dirq
"""
msg_enabled = pylons.config.get('fts3.MonitoringMessaging', False)
if not msg_enabled or msg_enabled.lower() == 'false':
return
publish_dn = pylons.config.get('fts3.MonitoringPublishDN', False)
msg_dir = pylons.config.get('fts3.MessagingDirectory', '/var/lib/fts3')
mon_dir = os.path.join(msg_dir, 'monitoring')
_user_dn = job['user_dn'] if publish_dn else ''
msg = dict(
endpnt=pylons.config['fts3.Alias'],
user_dn=_user_dn,
src_url=transfer['source_surl'],
dst_url=transfer['dest_surl'],
vo_name=job['vo_name'],
source_se=transfer['source_se'],
dest_se=transfer['dest_se'],
job_id=job['job_id'],
file_id=transfer['file_id'],
job_state=job['job_state'],
file_state=transfer_state,
retry_counter=0,
retry_max=0,
timestamp=time.time() * 1000,
job_metadata=job['job_metadata'],
file_metadata=transfer['file_metadata'],
)
tmpfile = tempfile.NamedTemporaryFile(dir=msg_dir, delete=False)
tmpfile.write("SS ")
json.dump(msg, tmpfile)
tmpfile.close()
q = QueueSimple(path=mon_dir)
q.add_path(tmpfile.name)
log.debug("Sent SUBMITTED state for %s %d" %
(job['job_id'], transfer['file_id']))
def test07_get(self):
'QueueSimple.get()'
data = 'foo'.encode()
qs = QueueSimple(self.qdir)
elem = qs.add(data)
qs.lock(elem)
self.assertEqual(qs.get(elem), data)
def new_dirq(path, _schema):
"""Create a new Directory::Queue object, optionally with schema.
"""
kwargs = {}
if opts.type == "simple":
if opts.granularity is not None:
kwargs['granularity'] = opts.granularity
return QueueSimple(path, **kwargs)
else:
if _schema:
schema = {'body': 'string', 'header': 'table?'}
kwargs['schema'] = schema
if opts.maxelts:
kwargs['maxelts'] = opts.maxelts
return queue.Queue(path, **kwargs)
def run_queue_once():
from dirq.QueueSimple import QueueSimple
dirq = QueueSimple(QDIR)
dirq.purge(30,60)
did_work = False
for name in dirq:
if not dirq.lock(name):
continue
data = dirq.get(name)
log.debug("item: %s", data)
item = json.loads(data)
signal.alarm(15)
queue_ts = item['ts']
start = time.time()
try :
block(item['ip'], item['comment'], item['duration'])
except Exception as e:
#Was this a whitelisted host or similarly invalid request
#Versus a server error?
if hasattr(e, 'response') and e.response is not None and e.response.status_code == 400 and 'non_field_errors' in e.response.json():
log.info("Ignored API Error %s", e.response.json())
else:
text = e.response.text if hasattr(e, 'response') else ''
log.exception("API Error. HTTP Response: %r", text)
raise
end = time.time()
signal.alarm(0)
dirq.remove(name)
log.info("block ip=%s queue_latency=%0.2f api_latency=%0.2f", item['ip'], start-queue_ts, end-start)
did_work = True
return did_work
class Sender(object):
"""A simple class for sending Accounting Records to a REST endpoint."""
def __init__(self, dest, qpath, cert, key, api_version):
"""Initialize a Sender."""
self._cert = cert
self._key = key
self._outq = QueueSimple(qpath)
self._dest = dest
self._api_version = api_version
def send_all(self):
"""Send all the messages in the outgoing queue via REST."""
log.info('Found %s messages.', self._outq.count())
for msgid in self._outq:
if not self._outq.lock(msgid):
log.warn('Message was locked. %s will not be sent.', msgid)
continue
text = self._outq.get(msgid)
path = '/api/%s/cloud/record' % self._api_version
data = text
self._rest_send('POST', path, data, {}, 202)
log.info("Sent %s", msgid)
time.sleep(0.1)
self._outq.remove(msgid)
log.info('Tidying message directory.')
try:
# Remove empty dirs and unlock msgs older than 5 min (default)
self._outq.purge()
except OSError, e:
log.warn('OSError raised while purging message queue: %s', e)
class Sender(object):
"""A simple class for sending Accounting Records to a REST endpoint."""
def __init__(self, dest, qpath, cert, key, api_version):
"""Initialize a Sender."""
self._cert = cert
self._key = key
self._outq = QueueSimple(qpath)
self._dest = dest
self._api_version = api_version
def send_all(self):
"""Send all the messages in the outgoing queue via REST."""
log.info('Found %s messages.', self._outq.count())
for msgid in self._outq:
if not self._outq.lock(msgid):
log.warn('Message was locked. %s will not be sent.', msgid)
continue
text = self._outq.get(msgid)
path = '/api/%s/cloud/record' % self._api_version
data = text
self._rest_send('POST', path, data, {}, 202)
log.info("Sent %s", msgid)
time.sleep(0.1)
self._outq.remove(msgid)
log.info('Tidying message directory.')
try:
# Remove empty dirs and unlock msgs older than 5 min (default)
self._outq.purge()
except OSError, e:
log.warn('OSError raised while purging message queue: %s', e)
class DbUnloader(object):
APEL_HEADERS = {JobRecord: JOB_MSG_HEADER,
SummaryRecord: SUMMARY_MSG_HEADER,
SyncRecord: SYNC_MSG_HEADER,
CloudRecord: CLOUD_MSG_HEADER,
CloudSummaryRecord: CLOUD_SUMMARY_MSG_HEADER}
RECORD_TYPES = {'VJobRecords': JobRecord,
'VSummaries': SummaryRecord,
'VSuperSummaries': SummaryRecord,
'VSyncRecords': SyncRecord,
'VCloudRecords': CloudRecord,
'VCloudSummaries': CloudSummaryRecord}
# all record types for which withholding DNs is a valid option
MAY_WITHHOLD_DNS = [JobRecord, SyncRecord, CloudRecord]
def __init__(self, db, qpath, inc_vos=None, exc_vos=None, local=False, withhold_dns=False):
self._db = db
outpath = os.path.join(qpath, "outgoing")
self._msgq = QueueSimple(outpath)
self._inc_vos = inc_vos
self._exc_vos = exc_vos
self._local = local
self._withhold_dns = withhold_dns
def _get_base_query(self, record_type):
'''
Set up a query object containing the logic which is common for
all users of this DbUnloader.
'''
query = Query()
if record_type in (JobRecord, SummaryRecord, SyncRecord):
if self._inc_vos is not None:
query.VO_in = self._inc_vos
log.info('Sending only these VOs: ')
log.info(self._inc_vos)
elif self._exc_vos is not None:
log.info('Excluding these VOs: ')
log.info(self._exc_vos)
query.VO_notin = self._exc_vos
if not self._local:
query.InfrastructureType = 'grid'
return query
def unload_all(self, table_name, car=False):
'''
Unload all records from the specified table.
'''
log.info('Unloading all records from %s.' % table_name)
record_type = self.RECORD_TYPES[table_name]
query = self._get_base_query(record_type)
msgs, records = self._write_messages(record_type, table_name, query, car)
return msgs, records
def unload_sync(self):
'''
Unload all records from the SyncRecords table or view.
'''
log.info('Writing sync messages.')
query = self._get_base_query(SyncRecord)
msgs = 0
records = 0
for batch in self._db.get_sync_records(query=query):
records += len(batch)
self._write_apel(batch)
msgs += 1
return msgs, records
def unload_gap(self, table_name, start, end, ur=False):
'''
Unload all records from the JobRecords table whose EndTime falls
within the provided dates (inclusive).
'''
record_type = self.RECORD_TYPES[table_name]
if record_type != JobRecord:
raise ApelDbException("Can only gap publish for JobRecords.")
start_tuple = [ int(x) for x in start.split('-') ]
end_tuple = [ int(x) for x in end.split('-') ]
# get the start of the start date
start_date = datetime.date(*start_tuple)
start_datetime = datetime.datetime.combine(start_date, datetime.time())
# get the end of the end date
end_date = datetime.date(*end_tuple)
end_datetime = datetime.datetime.combine(end_date, datetime.time())
end_datetime += datetime.timedelta(days=1)
log.info('Finding records with end times between:')
log.info(start_datetime)
log.info(end_datetime)
query = self._get_base_query(record_type)
query.EndTime_gt = start_datetime
query.EndTime_le = end_datetime
msgs, records = self._write_messages(record_type, table_name, query, ur)
return msgs, records
def unload_latest(self, table_name, ur=False):
'''
Unloads any records whose UpdateTime is less than the value
in the LastUpdated table.
Returns (number of files, number of records)
'''
# special case for SuperSummaries
if table_name == 'VSuperSummaries':
msgs, records = self.unload_latest_super_summaries()
else:
record_type = self.RECORD_TYPES[table_name]
query = self._get_base_query(record_type)
since = self._db.get_last_updated()
log.info('Getting records updated since: %s' % since)
if since is not None:
query.UpdateTime_gt = str(since)
msgs, records = self._write_messages(record_type, table_name, query, ur)
self._db.set_updated()
return msgs, records
def unload_latest_super_summaries(self, ur=False):
'''
Special case for the SuperSummaries table. Since it is generally
updated by the SummariseJobs() procedure, all records will
have been updated. Instead, send all records for the current
month and the preceding month.
'''
table_name = 'VSuperSummaries'
record_type = self.RECORD_TYPES[table_name]
query = self._get_base_query(record_type)
# It's actually simpler to use EarliestEndTime or LatestEndTime
# to deduce the correct records.
since = get_start_of_previous_month(datetime.datetime.now())
log.info('Getting summaries for months since: %s' % since.date())
if since is not None:
query.EarliestEndTime_gt = str(since)
msgs, records = self._write_messages(record_type, table_name, query, ur)
return msgs, records
def _write_messages(self, record_type, table_name, query, ur):
'''
Write messsages for all the records found in the specified table,
according to the logic contained in the query object.
'''
if self._withhold_dns and record_type not in self.MAY_WITHHOLD_DNS:
raise ApelDbException('Cannot withhold DNs for %s' % record_type.__name__)
msgs = 0
records = 0
for batch in self._db.get_records(record_type, table_name, query=query):
records += len(batch)
if ur:
self._write_xml(batch)
else:
self._write_apel(batch)
msgs += 1
return msgs, records
def _write_xml(self, records):
'''
Write one message in the appropriate XML format to the outgoing
message queue.
This is currently enabled only for CAR.
'''
buf = StringIO.StringIO()
if type(records[0]) == JobRecord:
XML_HEADER = '<?xml version="1.0" ?>'
UR_OPEN = '<urf:UsageRecords xmlns:urf="http://eu-emi.eu/namespaces/2012/11/computerecord" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://eu-emi.eu/namespaces/2012/11/computerecord car_v1.2.xsd">'
UR_CLOSE = '</urf:UsageRecords>'
# elif type(records[0]) == SummaryRecord:
# XML_HEADER = '<?xml version="1.0" ?>'
# UR_OPEN = '<aur:SummaryRecords xmlns:aur="http://eu-emi.eu/namespaces/2012/11/aggregatedcomputerecord" xmlns:urf="http://eu-emi.eu/namespaces/2012/11/computerecord" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://eu-emi.eu/namespaces/2012/11/aggregatedcomputerecord ">'
# UR_CLOSE = '</aur:SummaryRecords>'
else:
raise ApelDbException('Can only send URs for JobRecords.')
buf.write(XML_HEADER + '\n')
buf.write(UR_OPEN + '\n')
buf.write('\n'.join( [ record.get_ur(self._withhold_dns) for record in records ] ))
buf.write('\n' + UR_CLOSE + '\n')
self._msgq.add(buf.getvalue())
buf.close()
del buf
def _write_apel(self, records):
'''
Write one message in the APEL format to the outgoing
message queue.
'''
record_type = type(records[0])
buf = StringIO.StringIO()
buf.write(self.APEL_HEADERS[record_type] + ' \n')
buf.write('%%\n'.join( [ record.get_msg(self._withhold_dns) for record in records ] ))
buf.write('%%\n')
self._msgq.add(buf.getvalue())
buf.close()
del buf
def test04_add(self):
'QueueSimple.add()'
data = 'foo bar'
qs = QueueSimple(self.qdir)
elem = qs.add(data)
assert open(self.qdir + '/' + elem).read() == data
def queue(ip, comment, duration):
from dirq.QueueSimple import QueueSimple
dirq = QueueSimple(QDIR)
rec = dict(ip=ip, comment=comment, duration=duration, ts=time.time())
dirq.add(json.dumps(rec))
return True
def test11_purge_multDirMultElement(self):
'QueueSimple.purge() multiple directories & elements'
qs = QueueSimple(self.qdir, granularity=1)
qs.add('foo')
assert qs.count() == 1
time.sleep(2)
qs.add('bar')
assert qs.count() == 2
assert len(os.listdir(self.qdir)) == 2
qs.purge()
assert qs.count() == 2
elem = qs.first()
qs.lock(elem)
qs.remove(elem)
assert qs.count() == 1
qs.purge()
assert len(os.listdir(self.qdir)) == 1
time.sleep(2)
qs.add('baz')
assert len(os.listdir(self.qdir)) == 2
for elem in qs:
qs.lock(elem)
elem1 = qs.first()
lock_path1 = self.qdir + '/' + elem1 + LOCKED_SUFFIX
assert os.path.exists(lock_path1) is True
os.utime(lock_path1, (time.time() - 25, time.time() - 25))
qs.purge(maxlock=10)
assert os.path.exists(lock_path1) is False
elem2 = qs.next()
lock_path2 = self.qdir + '/' + elem2 + LOCKED_SUFFIX
assert os.path.exists(lock_path2) is True
class Ssm2(stomp.ConnectionListener):
'''
Minimal SSM implementation.
'''
# Schema for the dirq message queue.
QSCHEMA = {'body': 'string', 'signer': 'string', 'empaid': 'string?'}
REJECT_SCHEMA = {
'body': 'string',
'signer': 'string?',
'empaid': 'string?',
'error': 'string'
}
CONNECTION_TIMEOUT = 10
def __init__(self,
hosts_and_ports,
qpath,
cert,
key,
dest=None,
listen=None,
capath=None,
check_crls=False,
use_ssl=False,
username=None,
password=None,
enc_cert=None,
verify_enc_cert=True,
pidfile=None):
'''
Creates an SSM2 object. If a listen value is supplied,
this SSM2 will be a receiver.
'''
self._conn = None
self._last_msg = None
self._brokers = hosts_and_ports
self._cert = cert
self._key = key
self._enc_cert = enc_cert
self._capath = capath
self._check_crls = check_crls
self._user = username
self._pwd = password
self._use_ssl = use_ssl
# use pwd auth if we're supplied both user and pwd
self._use_pwd = username is not None and password is not None
self.connected = False
self._listen = listen
self._dest = dest
self._valid_dns = []
self._pidfile = pidfile
# create the filesystem queues for accepted and rejected messages
if dest is not None and listen is None:
self._outq = QueueSimple(qpath)
elif listen is not None:
inqpath = os.path.join(qpath, 'incoming')
rejectqpath = os.path.join(qpath, 'reject')
self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
else:
raise Ssm2Exception('SSM must be either producer or consumer.')
# check that the cert and key match
if not crypto.check_cert_key(self._cert, self._key):
raise Ssm2Exception('Cert and key don\'t match.')
# Check that the certificate has not expired.
if not crypto.verify_cert_date(self._cert):
raise Ssm2Exception('Certificate %s has expired.' % self._cert)
# check the server certificate provided
if enc_cert is not None:
log.info('Messages will be encrypted using %s', enc_cert)
if not os.path.isfile(self._enc_cert):
raise Ssm2Exception(
'Specified certificate file does not exist: %s.' %
self._enc_cert)
# Check that the encyption certificate has not expired.
if not crypto.verify_cert_date(enc_cert):
raise Ssm2Exception(
'Encryption certificate %s has expired. Please obtain the '
'new one from the final server receiving your messages.' %
enc_cert)
if verify_enc_cert:
if not crypto.verify_cert_path(self._enc_cert, self._capath,
self._check_crls):
raise Ssm2Exception(
'Failed to verify server certificate %s against CA path %s.'
% (self._enc_cert, self._capath))
# If the overall SSM log level is info, we want to only
# see log entries from stomp.py at the warning level and above.
if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
logging.getLogger("stomp.py").setLevel(logging.WARNING)
# If the overall SSM log level is debug, we want to only
# see log entries from stomp.py at the info level and above.
elif logging.getLogger(
"ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
logging.getLogger("stomp.py").setLevel(logging.INFO)
def set_dns(self, dn_list):
'''
Set the list of DNs which are allowed to sign incoming messages.
'''
self._valid_dns = dn_list
##########################################################################
# Methods called by stomppy
##########################################################################
def on_send(self, frame, unused_body=None):
'''
Called by stomppy when a message is sent.
unused_body is only present to have a backward compatible
method signature when using stomp.py v3.1.X
'''
try:
# Try the stomp.py v4 way first
empaid = frame.headers['empa-id']
except KeyError:
# Then you are most likely using stomp.py v4.
# on_send is now triggered on non message frames
# (such as 'CONNECT' frames) and as such without an empa-id.
empaid = 'no empa-id'
except AttributeError:
# Then you are likely using stomp.py v3
empaid = frame['empa-id']
log.debug('Sent message: %s', empaid)
def on_message(self, headers, body):
'''
Called by stomppy when a message is received.
Handle the message according to its content and headers.
'''
try:
empaid = headers['empa-id']
if empaid == 'ping': # ignore ping message
log.info('Received ping message.')
return
except KeyError:
empaid = 'noid'
log.info("Received message. ID = %s", empaid)
extracted_msg, signer, err_msg = self._handle_msg(body)
try:
# If the message is empty or the error message is not empty
# then reject the message.
if extracted_msg is None or err_msg is not None:
if signer is None: # crypto failed
signer = 'Not available.'
elif extracted_msg is not None:
# If there is a signer then it was rejected for not being
# in the DNs list, so we can use the extracted msg, which
# allows the msg to be reloaded if needed.
body = extracted_msg
log.warn("Message rejected: %s", err_msg)
name = self._rejectq.add({
'body': body,
'signer': signer,
'empaid': empaid,
'error': err_msg
})
log.info("Message saved to reject queue as %s", name)
else: # message verified ok
name = self._inq.add({
'body': extracted_msg,
'signer': signer,
'empaid': empaid
})
log.info("Message saved to incoming queue as %s", name)
except (IOError, OSError) as e:
log.error('Failed to read or write file: %s', e)
def on_error(self, unused_headers, body):
'''
Called by stomppy when an error frame is received.
'''
log.warn('Error message received: %s', body)
raise Ssm2Exception()
def on_connected(self, unused_headers, unused_body):
'''
Called by stomppy when a connection is established.
Track the connection.
'''
self.connected = True
log.info('Connected.')
def on_disconnected(self):
'''
Called by stomppy when disconnected from the broker.
'''
log.info('Disconnected from broker.')
self.connected = False
def on_receipt(self, headers, unused_body):
'''
Called by stomppy when the broker acknowledges receipt of a message.
'''
log.info('Broker received message: %s', headers['receipt-id'])
self._last_msg = headers['receipt-id']
def on_receiver_loop_completed(self, _unused_headers, _unused_body):
"""
Called by stompy when the receiver loop ends.
This is usually trigger as part of a disconnect.
"""
log.debug('on_receiver_loop_completed called.')
##########################################################################
# Message handling methods
##########################################################################
def _handle_msg(self, text):
'''
Deal with the raw message contents appropriately:
- decrypt if necessary
- verify signature
Return plain-text message, signer's DN and an error/None.
'''
if text is None or text == '':
warning = 'Empty text passed to _handle_msg.'
log.warn(warning)
return None, None, warning
# if not text.startswith('MIME-Version: 1.0'):
# raise Ssm2Exception('Not a valid message.')
# encrypted - this could be nicer
if 'application/pkcs7-mime' in text or 'application/x-pkcs7-mime' in text:
try:
text = crypto.decrypt(text, self._cert, self._key)
except crypto.CryptoException as e:
error = 'Failed to decrypt message: %s' % e
log.error(error)
return None, None, error
# always signed
try:
message, signer = crypto.verify(text, self._capath,
self._check_crls)
except crypto.CryptoException as e:
error = 'Failed to verify message: %s' % e
log.error(error)
return None, None, error
if signer not in self._valid_dns:
warning = 'Signer not in valid DNs list: %s' % signer
log.warn(warning)
return None, signer, warning
else:
log.info('Valid signer: %s', signer)
return message, signer, None
def _send_msg(self, message, msgid):
'''
Send one message using stomppy. The message will be signed using
the host cert and key. If an encryption certificate
has been supplied, the message will also be encrypted.
'''
log.info('Sending message: %s', msgid)
headers = {
'destination': self._dest,
'receipt': msgid,
'empa-id': msgid
}
if message is not None:
to_send = crypto.sign(message, self._cert, self._key)
if self._enc_cert is not None:
to_send = crypto.encrypt(to_send, self._enc_cert)
else:
to_send = ''
try:
# Try using the v4 method signiture
self._conn.send(self._dest, to_send, headers=headers)
except TypeError:
# If it fails, use the v3 metod signiture
self._conn.send(to_send, headers=headers)
def send_ping(self):
'''
If a STOMP connection is left open with no activity for an hour or
so, it stops responding. Stomppy 3.1.3 has two ways of handling
this, but stomppy 3.0.3 (EPEL 5 and 6) has neither.
To get around this, we begin and then abort a STOMP transaction to
keep the connection active.
'''
# Use time as transaction id to ensure uniqueness within each connection
transaction_id = str(time.time())
self._conn.begin({'transaction': transaction_id})
self._conn.abort({'transaction': transaction_id})
def has_msgs(self):
'''
Return True if there are any messages in the outgoing queue.
'''
return self._outq.count() > 0
def send_all(self):
'''
Send all the messages in the outgoing queue.
'''
log.info('Found %s messages.', self._outq.count())
for msgid in self._outq:
if not self._outq.lock(msgid):
log.warn('Message was locked. %s will not be sent.', msgid)
continue
text = self._outq.get(msgid)
self._send_msg(text, msgid)
log.info('Waiting for broker to accept message.')
while self._last_msg is None:
if not self.connected:
raise Ssm2Exception('Lost connection.')
time.sleep(0.1)
self._last_msg = None
self._outq.remove(msgid)
log.info('Tidying message directory.')
try:
# Remove empty dirs and unlock msgs older than 5 min (default)
self._outq.purge()
except OSError as e:
log.warn('OSError raised while purging message queue: %s', e)
############################################################################
# Connection handling methods
############################################################################
def _initialise_connection(self, host, port):
'''
Create the self._connection object with the appropriate properties,
but don't try to start the connection.
'''
log.info("Established connection to %s, port %i", host, port)
if self._use_ssl:
if ssl is None:
raise ImportError(
"SSL connection requested but the ssl module "
"wasn't found.")
log.info('Connecting using SSL...')
else:
log.warning("SSL connection not requested, your messages may be "
"intercepted.")
# _conn will use the default SSL version specified by stomp.py
self._conn = stomp.Connection([(host, port)],
use_ssl=self._use_ssl,
ssl_key_file=self._key,
ssl_cert_file=self._cert,
timeout=Ssm2.CONNECTION_TIMEOUT)
self._conn.set_listener('SSM', self)
def handle_connect(self):
'''
Assuming that the SSM has retrieved the details of the broker or
brokers it wants to connect to, connect to one.
If more than one is in the list self._network_brokers, try to
connect to each in turn until successful.
'''
for host, port in self._brokers:
self._initialise_connection(host, port)
try:
self.start_connection()
break
except ConnectFailedException as e:
# ConnectFailedException doesn't provide a message.
log.warn('Failed to connect to %s:%s.', host, port)
except Ssm2Exception as e:
log.warn('Failed to connect to %s:%s: %s', host, port, e)
if not self.connected:
raise Ssm2Exception(
'Attempts to start the SSM failed. The system will exit.')
def handle_disconnect(self):
'''
When disconnected, attempt to reconnect using the same method as used
when starting up.
'''
self.connected = False
# Shut down properly
self.close_connection()
# Sometimes the SSM will reconnect to the broker before it's properly
# shut down! This prevents that.
time.sleep(2)
# Try again according to the same logic as the initial startup
try:
self.handle_connect()
except Ssm2Exception:
self.connected = False
# If reconnection fails, admit defeat.
if not self.connected:
err_msg = 'Reconnection attempts failed and have been abandoned.'
raise Ssm2Exception(err_msg)
def start_connection(self):
'''
Once self._connection exists, attempt to start it and subscribe
to the relevant topics.
If the timeout is reached without receiving confirmation of
connection, raise an exception.
'''
if self._conn is None:
raise Ssm2Exception('Called start_connection() before a \
connection object was initialised.')
self._conn.start()
self._conn.connect(wait=True)
if self._dest is not None:
log.info('Will send messages to: %s', self._dest)
if self._listen is not None:
# Use a static ID for the subscription ID because we only ever have
# one subscription within a connection and ID is only considered
# to differentiate subscriptions within a connection.
self._conn.subscribe(destination=self._listen, id=1, ack='auto')
log.info('Subscribing to: %s', self._listen)
i = 0
while not self.connected:
time.sleep(0.1)
if i > Ssm2.CONNECTION_TIMEOUT * 10:
err = 'Timed out while waiting for connection. '
err += 'Check the connection details.'
raise Ssm2Exception(err)
i += 1
def close_connection(self):
'''
Close the connection. This is important because it runs
in a separate thread, so it can outlive the main process
if it is not ended.
'''
try:
self._conn.disconnect()
except (stomp.exception.NotConnectedException, socket.error):
self._conn = None
except AttributeError:
# AttributeError if self._connection is None already
pass
log.info('SSM connection ended.')
def startup(self):
'''
Create the pidfile then start the connection.
'''
if self._pidfile is not None:
try:
f = open(self._pidfile, 'w')
f.write(str(os.getpid()))
f.write('\n')
f.close()
except IOError as e:
log.warn('Failed to create pidfile %s: %s', self._pidfile, e)
self.handle_connect()
def shutdown(self):
'''
Close the connection then remove the pidfile.
'''
self.close_connection()
if self._pidfile is not None:
try:
if os.path.exists(self._pidfile):
os.remove(self._pidfile)
else:
log.warn('pidfile %s not found.', self._pidfile)
except IOError as e:
log.warn('Failed to remove pidfile %s: %e', self._pidfile, e)
log.warn('SSM may not start again until it is removed.')
def __init__(self,
hosts_and_ports,
qpath,
cert,
key,
dest=None,
listen=None,
capath=None,
check_crls=False,
use_ssl=False,
username=None,
password=None,
enc_cert=None,
verify_enc_cert=True,
pidfile=None,
path_type='dirq',
protocol=STOMP_MESSAGING,
project=None,
token=''):
'''
Creates an SSM2 object. If a listen value is supplied,
this SSM2 will be a receiver.
'''
self._conn = None
self._last_msg = None
self._brokers = hosts_and_ports
self._cert = cert
self._key = key
self._enc_cert = enc_cert
self._capath = capath
self._check_crls = check_crls
self._user = username
self._pwd = password
self._use_ssl = use_ssl
# use pwd auth if we're supplied both user and pwd
self._use_pwd = username is not None and password is not None
self.connected = False
self._listen = listen
self._dest = dest
self._valid_dns = []
self._pidfile = pidfile
# Used to differentiate between STOMP and AMS methods
self._protocol = protocol
# Used when interacting with an Argo Messaging Service
self._project = project
self._token = token
if self._protocol == Ssm2.AMS_MESSAGING:
self._ams = ArgoMessagingService(endpoint=self._brokers[0],
token=self._token,
cert=self._cert,
key=self._key,
project=self._project)
# create the filesystem queues for accepted and rejected messages
if dest is not None and listen is None:
# Determine what sort of outgoing structure to make
if path_type == 'dirq':
if QueueSimple is None:
raise ImportError("dirq path_type requested but the dirq "
"module wasn't found.")
self._outq = QueueSimple(qpath)
elif path_type == 'directory':
self._outq = MessageDirectory(qpath)
else:
raise Ssm2Exception('Unsupported path_type variable.')
elif listen is not None:
inqpath = os.path.join(qpath, 'incoming')
rejectqpath = os.path.join(qpath, 'reject')
# Receivers must use the dirq module, so make a quick sanity check
# that dirq is installed.
if Queue is None:
raise ImportError("Receiving SSMs must use dirq, but the dirq "
"module wasn't found.")
self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
else:
raise Ssm2Exception('SSM must be either producer or consumer.')
# check that the cert and key match
if not crypto.check_cert_key(self._cert, self._key):
raise Ssm2Exception('Cert and key don\'t match.')
# Check that the certificate has not expired.
if not crypto.verify_cert_date(self._cert):
raise Ssm2Exception('Certificate %s has expired or will expire '
'within a day.' % self._cert)
# check the server certificate provided
if enc_cert is not None:
log.info('Messages will be encrypted using %s', enc_cert)
if not os.path.isfile(self._enc_cert):
raise Ssm2Exception(
'Specified certificate file does not exist: %s.' %
self._enc_cert)
# Check that the encyption certificate has not expired.
if not crypto.verify_cert_date(enc_cert):
raise Ssm2Exception(
'Encryption certificate %s has expired or will expire '
'within a day. Please obtain the new one from the final '
'server receiving your messages.' % enc_cert)
if verify_enc_cert:
if not crypto.verify_cert_path(self._enc_cert, self._capath,
self._check_crls):
raise Ssm2Exception(
'Failed to verify server certificate %s against CA path %s.'
% (self._enc_cert, self._capath))
# If the overall SSM log level is info, we want to only
# see entries from stomp.py and connectionpool at WARNING and above.
if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
logging.getLogger("stomp.py").setLevel(logging.WARNING)
logging.getLogger(
"requests.packages.urllib3.connectionpool").setLevel(
logging.WARNING)
# If the overall SSM log level is debug, we want to only
# see entries from stomp.py and connectionpool at INFO above.
elif logging.getLogger(
"ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
logging.getLogger("stomp.py").setLevel(logging.INFO)
logging.getLogger(
"requests.packages.urllib3.connectionpool").setLevel(
logging.INFO)
def process_period(config, period):
period_start = period['instant'] + dateutil.relativedelta.relativedelta(
seconds=-period['range_sec'])
print(
f"Processing year {period['year']}, month {period['month']}, "
f"querying from {period['instant'].isoformat()} and going back {period['range_sec']} s to {period_start.isoformat()}."
)
queries = QueryLogic(queryRange=(str(period['range_sec']) + 's'))
# SSL generally not used for Prometheus access within a cluster
# Docs on instant query API: https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries
prom = PrometheusConnect(url=config.prometheus_server, disable_ssl=True)
prom_connect_params = {
'time': period['instant'].isoformat(),
'timeout': config.query_timeout
}
raw_results, results, result_lengths = {}, {}, []
# iterate over each query (cputime, starttime, endtime, cores) producing raw_results['cputime'] etc.
for query_name, query_string in vars(queries).items():
# Each of these raw_results is a list of dicts. Each dict in the list represents an individual data point, and contains:
# 'metric': a dict of one or more key-value pairs of labels, one of which is the pod name ('exported_pod').
# 'value': a list in which the 0th element is the timestamp of the value, and 1th element is the actual value we're interested in.
print(f'Executing {query_name} query: {query_string}')
t1 = timer()
raw_results[query_name] = prom.custom_query(query=query_string,
params=prom_connect_params)
t2 = timer()
results[query_name] = dict(rearrange(raw_results[query_name]))
result_lengths.append(len(results[query_name]))
t3 = timer()
print(
f'Query finished in {t2 - t1} s, processed in {t3 - t2} s. Got {len(results[query_name])} items from {len(raw_results[query_name])} results. Peak RAM usage: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}K.'
)
del raw_results[query_name]
cputime = results['cputime']
endtime = results['endtime']
starttime = results['starttime']
cores = results['cores']
# Confirm the assumption that cputime should have the fewest entries, while starttime and cores may have additional ones
# corresponding to jobs that have started but not finished yet, and endtime may have additional ones if there are pods without CPU resource requests.
# We only want the jobs for which all values are available: start time, end time, CPU request.
# Note that jobs which started last month and finished this month will be properly included and accounted in this month.
assert len(cputime) == min(
result_lengths), "cputime should be the shortest list"
# However, jobs that finished last month may show up in this month's data if they are still present on the cluster this month (in Completed state).
# Exclude them by filtering with a lambda (since you can't pass an argument to a function object AFAIK).
endtime = dict(
filter(lambda x: x[1] >= datetime.datetime.timestamp(period_start),
endtime.items()))
# Prepare to iterate over jobs which meet all criteria.
valid_jobs = cputime.keys() & endtime.keys()
# avoid sending empty records
if len(valid_jobs) == 0:
print('No records to process.')
return
sum_cputime = 0
t4 = timer()
for key in valid_jobs:
assert endtime[key] > starttime[
key], "job end time is before start time"
# double check cputime calc of this job
delta = abs(cputime[key] -
(endtime[key] - starttime[key]) * cores[key])
assert delta < 0.001, "cputime calculation is inaccurate"
sum_cputime += cputime[key]
# CPU time as calculated here means (# cores * job duration), which apparently corresponds to
# the concept of wall time in APEL accounting. It is not clear what CPU time means in APEL;
# could be the actual CPU usage % integrated over the job (# cores * job duration * usage)
# but this does not seem to be documented clearly. Some batch systems do not actually measure
# this so it is not reported consistently or accurately. Some sites have CPU efficiency
# (presumably defined as CPU time / wall time) time that is up to ~ 500% of the walltime, or
# always fixed at 100%. In Kubernetes, the actual CPU usage % is tracked by metrics server
# (not KSM), which is not meant to be used for monitoring or accounting purposes and is not
# scraped by Prometheus. So just use walltime = cputime
sum_cputime = round(sum_cputime)
sum_walltime = sum_cputime
print(f'total cputime: {sum_cputime}, total walltime: {sum_walltime}')
# Write output to the message queue on local filesystem
# https://dirq.readthedocs.io/en/latest/queuesimple.html#directory-structure
dirq = QueueSimple(str(config.output_path))
summary_output = summary_message(
config,
year=period['year'],
month=period['month'],
wall_time=sum_walltime,
cpu_time=sum_cputime,
n_jobs=len(endtime),
# this appears faster than getting min/max during the dict iteration above
first_end=round(min(endtime.values())),
last_end=round(max(endtime.values())))
sync_output = sync_message(config,
year=period['year'],
month=period['month'],
n_jobs=len(endtime))
t5 = timer()
summary_file = dirq.add(summary_output)
sync_file = dirq.add(sync_output)
print(f'Analyzed {len(endtime)} records in {t5 - t4} s.')
print(f'Writing summary record to {config.output_path}/{summary_file}:')
print('--------------------------------\n' + summary_output +
'--------------------------------')
print(f'Writing sync record to {config.output_path}/{sync_file}:')
print('--------------------------------\n' + sync_output +
'--------------------------------')
class DbUnloader(object):
APEL_HEADERS = {JobRecord: JOB_MSG_HEADER,
SummaryRecord: SUMMARY_MSG_HEADER,
SyncRecord: SYNC_MSG_HEADER,
CloudRecord: CLOUD_MSG_HEADER}
RECORD_TYPES = {'VJobRecords': JobRecord,
'VSummaries': SummaryRecord,
'VSuperSummaries': SummaryRecord,
'VSyncRecords': SyncRecord,
'VCloudRecords': CloudRecord}
def __init__(self, db, qpath, inc_vos=None, exc_vos=None, local=False):
self._db = db
outpath = os.path.join(qpath, "outgoing")
self._msgq = QueueSimple(outpath)
self._inc_vos = inc_vos
self._exc_vos = exc_vos
self._local = local
def _get_base_query(self, record_type):
'''
Set up a query object containing the logic which is common for
all users of this DbUnloader.
'''
query = Query()
if record_type in (JobRecord, SummaryRecord, SyncRecord):
if self._inc_vos is not None:
query.VO_in = self._inc_vos
log.info('Sending only these VOs: ')
log.info(self._inc_vos)
elif self._exc_vos is not None:
log.info('Excluding these VOs: ')
log.info(self._exc_vos)
query.VO_notin = self._exc_vos
if not self._local:
query.InfrastructureType = 'grid'
return query
def unload_all(self, table_name, car=False):
'''
Unload all records from the specified table.
'''
log.info('Unloading all records from %s.' % table_name)
record_type = self.RECORD_TYPES[table_name]
query = self._get_base_query(record_type)
msgs, records = self._write_messages(record_type, table_name, query, car)
return msgs, records
def unload_sync(self):
log.info('Writing sync messages.')
query = self._get_base_query(SyncRecord)
msgs = 0
records = 0
for batch in self._db.get_sync_records(query=query):
records += len(batch)
self._write_apel(batch)
msgs += 1
return msgs, records
def unload_gap(self, table_name, start, end, car=False):
'''
Unload all records from the JobRecords table whose EndTime falls
within the provided dates (inclusive).
'''
record_type = self.RECORD_TYPES[table_name]
if record_type != JobRecord:
raise ApelDbException("Can only gap publish for JobRecords.")
start_tuple = [ int(x) for x in start.split('-') ]
end_tuple = [ int(x) for x in end.split('-') ]
# get the start of the start date
start_date = datetime.date(*start_tuple)
start_datetime = datetime.datetime.combine(start_date, datetime.time())
# get the end of the end date
end_date = datetime.date(*end_tuple)
end_datetime = datetime.datetime.combine(end_date, datetime.time())
end_datetime += datetime.timedelta(days=1)
log.info('Finding records with end times between:')
log.info(start_datetime)
log.info(end_datetime)
query = self._get_base_query(record_type)
query.EndTime_gt = start_datetime
query.EndTime_le = end_datetime
msgs, records = self._write_messages(record_type, table_name, query, car)
return msgs, records
def unload_latest(self, table_name, car=False):
'''
Unloads records from database to file.
Returns the number of created files.
'''
record_type = self.RECORD_TYPES[table_name]
query = self._get_base_query(record_type)
since = self._db.get_last_updated()
log.info('Getting records updated since: %s' % since)
if since is not None:
query.UpdateTime_gt = str(since)
msgs, records = self._write_messages(record_type, table_name, query, car)
self._db.set_updated()
return msgs, records
def _write_messages(self, record_type, table_name, query, car):
'''
Write messsages for all the records found in the specified table,
according to the logic contained in the query object.
'''
msgs = 0
records = 0
for batch in self._db.get_records(record_type, table_name, query=query):
records += len(batch)
if car:
self._write_xml(batch)
else:
self._write_apel(batch)
msgs += 1
return msgs, records
def _write_xml(self, records):
'''
Write one message in the appropriate XML format to the outgoing
message queue.
This currently works only for CAR.
'''
buf = StringIO.StringIO()
if type(records[0]) == JobRecord:
XML_HEADER = '<?xml version="1.0" ?>'
UR_OPEN = '<urf:UsageRecords xmlns:urf="http://eu-emi.eu/namespaces/2012/11/computerecord" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://eu-emi.eu/namespaces/2012/11/computerecord car_v1.2.xsd">'
UR_CLOSE = '</urf:UsageRecords>'
# elif type(records[0]) == SummaryRecord:
# XML_HEADER = '<?xml version="1.0" ?>'
# UR_OPEN = '<aur:SummaryRecords xmlns:aur="http://eu-emi.eu/namespaces/2012/11/aggregatedcomputerecord" xmlns:urf="http://eu-emi.eu/namespaces/2012/11/computerecord" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://eu-emi.eu/namespaces/2012/11/aggregatedcomputerecord ">'
# UR_CLOSE = '</aur:SummaryRecords>'
else:
raise ApelDbException('Can only send URs for JobRecords.')
buf.write(XML_HEADER + '\n')
buf.write(UR_OPEN + '\n')
buf.write('\n'.join( [ record.get_ur() for record in records ] ))
buf.write('\n' + UR_CLOSE + '\n')
self._msgq.add(buf.getvalue())
buf.close()
del buf
def _write_apel(self, records):
'''
Write one message in the APEL format to the outgoing
message queue.
'''
record_type = type(records[0])
buf = StringIO.StringIO()
buf.write(self.APEL_HEADERS[record_type] + ' \n')
buf.write('%%\n'.join( [ record.get_msg() for record in records ] ))
buf.write('%%\n')
self._msgq.add(buf.getvalue())
buf.close()
del buf
from dirq.QueueSimple import QueueSimple
from random import randint
import time
path = '/tmp/test'
print("start consuming...")
dirq = QueueSimple(path)
done = 0
while True:
for name in dirq:
# print("element: %s %s" % (path, name))
if not dirq.lock(name):
#print("couldn't lock: %s" % name)
# name = dirq.next()
continue
element = dirq.get(name)
#print(element.keys())
print("Body: \"%s\"" % element)
if randint(1, 2) % 2:
done += 1
dirq.remove(name)
#print('Removed')
else:
dirq.unlock(name)
#name = dirq.next()
print("consumed %i elements" % done)
total_left = dirq.count()
print("elements left in the queue: %d" % total_left)
time.sleep(0.5)
def __init__(self,
hosts_and_ports,
qpath,
cert,
key,
dest=None,
listen=None,
capath=None,
check_crls=False,
use_ssl=False,
username=None,
password=None,
enc_cert=None,
verify_enc_cert=True,
pidfile=None):
'''
Creates an SSM2 object. If a listen value is supplied,
this SSM2 will be a receiver.
'''
self._conn = None
self._last_msg = None
self._brokers = hosts_and_ports
self._cert = cert
self._key = key
self._enc_cert = enc_cert
self._capath = capath
self._check_crls = check_crls
self._user = username
self._pwd = password
self._use_ssl = use_ssl
# use pwd auth if we're supplied both user and pwd
self._use_pwd = username is not None and password is not None
self.connected = False
self._listen = listen
self._dest = dest
self._valid_dns = []
self._pidfile = pidfile
# create the filesystem queues for accepted and rejected messages
if dest is not None and listen is None:
self._outq = QueueSimple(qpath)
elif listen is not None:
inqpath = os.path.join(qpath, 'incoming')
rejectqpath = os.path.join(qpath, 'reject')
self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
else:
raise Ssm2Exception('SSM must be either producer or consumer.')
# check that the cert and key match
if not crypto.check_cert_key(self._cert, self._key):
raise Ssm2Exception('Cert and key don\'t match.')
# Check that the certificate has not expired.
if not crypto.verify_cert_date(self._cert):
raise Ssm2Exception('Certificate %s has expired.' % self._cert)
# check the server certificate provided
if enc_cert is not None:
log.info('Messages will be encrypted using %s', enc_cert)
if not os.path.isfile(self._enc_cert):
raise Ssm2Exception(
'Specified certificate file does not exist: %s.' %
self._enc_cert)
# Check that the encyption certificate has not expired.
if not crypto.verify_cert_date(enc_cert):
raise Ssm2Exception(
'Encryption certificate %s has expired. Please obtain the '
'new one from the final server receiving your messages.' %
enc_cert)
if verify_enc_cert:
if not crypto.verify_cert_path(self._enc_cert, self._capath,
self._check_crls):
raise Ssm2Exception(
'Failed to verify server certificate %s against CA path %s.'
% (self._enc_cert, self._capath))
# If the overall SSM log level is info, we want to only
# see log entries from stomp.py at the warning level and above.
if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
logging.getLogger("stomp.py").setLevel(logging.WARNING)
# If the overall SSM log level is debug, we want to only
# see log entries from stomp.py at the info level and above.
elif logging.getLogger(
"ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
logging.getLogger("stomp.py").setLevel(logging.INFO)
from time import sleep, time
import datetime
#from cswaUpdateCSpace import processqueueelement
queue_dir = sys.argv[1]
# sample producer
# dirq = QueueSimple(queue_dir)
# for count in range(1,101):
# name = dirq.add("element %i\n" % count)
# print("# added element %i as %s" % (count, name))
# sample consumer
dirq = QueueSimple(queue_dir)
passes = 0
items = 0
elapsedtime = time()
logmessageinterval = 60 #seconds
WHEN2POST = 'queue'
while True:
for name in dirq:
if not dirq.lock(name):
continue
print("# reading element %s" % name)
data = dirq.get(name)
print data
# one could use dirq.unlock(name) to only browse the queue...
dirq.unlock(name)
import codecs
import time
import datetime
import csv
from dirq.QueueSimple import QueueSimple
from toolbox.cswaHelpers import *
from common import cspace
from common.utils import deURN
from cspace_django_site.main import cspace_django_site
MAINCONFIG = cspace_django_site.getConfig()
DIRQ = QueueSimple('/tmp/cswa')
MAXLOCATIONS = 1000
import xml.etree.ElementTree as etree
def getWhen2Post(config):
try:
when2post = config.get('info', 'when2post')
except:
# default is update immediately
when2post = 'update'
return when2post
def __init__(self,
hosts_and_ports,
qpath,
cert,
key,
dest=None,
listen=None,
capath=None,
check_crls=False,
use_ssl=False,
username=None,
password=None,
enc_cert=None,
verify_enc_cert=True,
pidfile=None):
'''
Creates an SSM2 object. If a listen value is supplied,
this SSM2 will be a receiver.
'''
self._conn = None
self._last_msg = None
self._brokers = hosts_and_ports
self._cert = cert
self._key = key
self._enc_cert = enc_cert
self._capath = capath
self._check_crls = check_crls
self._user = username
self._pwd = password
self._use_ssl = use_ssl
# use pwd auth if we're supplied both user and pwd
self._use_pwd = username is not None and password is not None
self.connected = False
self._listen = listen
self._dest = dest
self._valid_dns = []
self._pidfile = pidfile
# create the filesystem queues for accepted and rejected messages
if dest is not None and listen is None:
self._outq = QueueSimple(qpath)
elif listen is not None:
inqpath = os.path.join(qpath, 'incoming')
rejectqpath = os.path.join(qpath, 'reject')
self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
else:
raise Ssm2Exception('SSM must be either producer or consumer.')
# check that the cert and key match
if not crypto.check_cert_key(self._cert, self._key):
raise Ssm2Exception('Cert and key don\'t match.')
# check the server certificate provided
if enc_cert is not None:
log.info('Messages will be encrypted using %s' % enc_cert)
if not os.path.isfile(self._enc_cert):
raise Ssm2Exception(
'Specified certificate file does not exist: %s.' %
self._enc_cert)
if verify_enc_cert:
if not crypto.verify_cert_path(self._enc_cert, self._capath,
self._check_crls):
raise Ssm2Exception(
'Failed to verify server certificate %s against CA path %s.'
% (self._enc_cert, self._capath))
def test10_purge_oneDirOneElement(self):
'QueueSimple.purge() one directory & element'
qs = QueueSimple(self.qdir)
qs.add('foo')
self.assertEqual(qs.count(), 1)
elem = qs.first()
qs.lock(elem)
elem_path_lock = self.qdir + '/' + elem + LOCKED_SUFFIX
self.assert_(os.path.exists(elem_path_lock) is True)
time.sleep(2)
qs.purge(maxlock=1)
self.assert_(os.path.exists(elem_path_lock) is False)
self.assertEqual(qs.count(), 1)
self.assertEqual(len(os.listdir(self.qdir)), 1)
def __init__(self, configHolder):
super(DirectoryQueue, self).__init__(configHolder)
self.queue = QueueSimple(self.msg_queue)
class DbUnloader(object):
APEL_HEADERS = {
JobRecord: JOB_MSG_HEADER,
SummaryRecord: SUMMARY_MSG_HEADER,
NormalisedSummaryRecord: NORMALISED_SUMMARY_MSG_HEADER,
SyncRecord: SYNC_MSG_HEADER,
CloudRecord: CLOUD_MSG_HEADER,
CloudSummaryRecord: CLOUD_SUMMARY_MSG_HEADER
}
RECORD_TYPES = {
'VJobRecords': JobRecord,
'VSummaries': SummaryRecord,
'VSuperSummaries': SummaryRecord,
'VNormalisedSummaries': NormalisedSummaryRecord,
'VNormalisedSuperSummaries': NormalisedSummaryRecord,
'VSyncRecords': SyncRecord,
'VCloudRecords': CloudRecord,
'VCloudSummaries': CloudSummaryRecord,
'VStarRecords': StorageRecord
}
# all record types for which withholding DNs is a valid option
MAY_WITHHOLD_DNS = [JobRecord, SyncRecord, CloudRecord]
def __init__(self,
db,
qpath,
inc_vos=None,
exc_vos=None,
local=False,
withhold_dns=False):
self._db = db
outpath = os.path.join(qpath, "outgoing")
self._msgq = QueueSimple(outpath)
self._inc_vos = inc_vos
self._exc_vos = exc_vos
self._local = local
self._withhold_dns = withhold_dns
def _get_base_query(self, record_type):
'''
Set up a query object containing the logic which is common for
all users of this DbUnloader.
'''
query = Query()
if record_type in (JobRecord, SummaryRecord, NormalisedSummaryRecord,
SyncRecord):
if self._inc_vos is not None:
query.VO_in = self._inc_vos
log.info('Sending only these VOs: ')
log.info(self._inc_vos)
elif self._exc_vos is not None:
log.info('Excluding these VOs: ')
log.info(self._exc_vos)
query.VO_notin = self._exc_vos
if not self._local:
query.InfrastructureType = 'grid'
return query
def unload_all(self, table_name, car=False):
'''
Unload all records from the specified table.
'''
log.info('Unloading all records from %s.', table_name)
record_type = self.RECORD_TYPES[table_name]
query = self._get_base_query(record_type)
msgs, records = self._write_messages(record_type, table_name, query,
car)
return msgs, records
def unload_sync(self):
'''
Unload all records from the SyncRecords table or view.
'''
log.info('Writing sync messages.')
query = self._get_base_query(SyncRecord)
msgs = 0
records = 0
for batch in self._db.get_sync_records(query=query):
records += len(batch)
self._write_apel(batch)
msgs += 1
return msgs, records
def unload_gap(self, table_name, start, end, ur=False):
'''
Unload all records from the JobRecords table whose EndTime falls
within the provided dates (inclusive).
'''
record_type = self.RECORD_TYPES[table_name]
if record_type != JobRecord:
raise ApelDbException("Can only gap publish for JobRecords.")
start_tuple = [int(x) for x in start.split('-')]
end_tuple = [int(x) for x in end.split('-')]
# get the start of the start date
start_date = datetime.date(*start_tuple)
start_datetime = datetime.datetime.combine(start_date, datetime.time())
# get the end of the end date
end_date = datetime.date(*end_tuple)
end_datetime = datetime.datetime.combine(end_date, datetime.time())
end_datetime += datetime.timedelta(days=1)
log.info('Finding records with end times between:')
log.info(start_datetime)
log.info(end_datetime)
query = self._get_base_query(record_type)
query.EndTime_gt = start_datetime
query.EndTime_le = end_datetime
msgs, records = self._write_messages(record_type, table_name, query,
ur)
return msgs, records
def unload_latest(self, table_name, ur=False):
'''
Unloads any records whose UpdateTime is less than the value
in the LastUpdated table.
Returns (number of files, number of records)
'''
# Special case for [Normalised]SuperSummaries
if table_name in ('VSuperSummaries', 'VNormalisedSuperSummaries'):
msgs, records = self.unload_latest_super_summaries(table_name)
else:
record_type = self.RECORD_TYPES[table_name]
query = self._get_base_query(record_type)
since = self._db.get_last_updated()
log.info('Getting records updated since: %s', since)
if since is not None:
query.UpdateTime_gt = str(since)
msgs, records = self._write_messages(record_type, table_name,
query, ur)
self._db.set_updated()
return msgs, records
def unload_latest_super_summaries(
self,
table_name,
ur=False,
):
"""
Unload (normalised) super summaries for current and preceding month
Special case for the [Normalised]SuperSummaries table. Since it is
generally updated by the SummariseJobs() procedure, all records will
have been updated. Instead, send all records for the current
month and the preceding month.
"""
record_type = self.RECORD_TYPES[table_name]
query = self._get_base_query(record_type)
# It's actually simpler to use EarliestEndTime or LatestEndTime
# to deduce the correct records.
since = get_start_of_previous_month(datetime.datetime.now())
log.info('Getting summaries for months since: %s', since.date())
if since is not None:
query.EarliestEndTime_ge = str(since)
msgs, records = self._write_messages(record_type, table_name, query,
ur)
return msgs, records
def _write_messages(self, record_type, table_name, query, ur):
'''
Write messsages for all the records found in the specified table,
according to the logic contained in the query object.
'''
if self._withhold_dns and record_type not in self.MAY_WITHHOLD_DNS:
raise ApelDbException('Cannot withhold DNs for %s' %
record_type.__name__)
if record_type == StorageRecord and not ur:
raise ApelDbException(
'Cannot unload StorageRecords in APEL format')
msgs = 0
records = 0
for batch in self._db.get_records(record_type, table_name,
query=query):
records += len(batch)
if ur:
self._write_xml(batch)
else:
self._write_apel(batch)
msgs += 1
return msgs, records
def _write_xml(self, records):
'''
Write one message in the appropriate XML format to the outgoing
message queue.
This is currently enabled only for CAR.
'''
buf = StringIO.StringIO()
if type(records[0]) == JobRecord:
XML_HEADER = '<?xml version="1.0" ?>'
UR_OPEN = (
'<urf:UsageRecords xmlns:urf="http://eu-emi.eu/namespace'
's/2012/11/computerecord" xmlns:xsi="http://www.w3.org/2'
'001/XMLSchema-instance" xsi:schemaLocation="http://eu-e'
'mi.eu/namespaces/2012/11/computerecord car_v1.2.xsd">')
UR_CLOSE = '</urf:UsageRecords>'
# elif type(records[0]) == SummaryRecord:
# XML_HEADER = '<?xml version="1.0" ?>'
# UR_OPEN = ('<aur:SummaryRecords xmlns:aur="http://eu-emi.eu/names'
# 'paces/2012/11/aggregatedcomputerecord" xmlns:urf="htt'
# 'p://eu-emi.eu/namespaces/2012/11/computerecord" xmlns'
# ':xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:'
# 'schemaLocation="http://eu-emi.eu/namespaces/2012/11/a'
# 'ggregatedcomputerecord ">')
# UR_CLOSE = '</aur:SummaryRecords>'
elif type(records[0]) == StorageRecord:
XML_HEADER = '<?xml version="1.0" ?>'
UR_OPEN = (
'<sr:StorageUsageRecords xmlns:sr="http://eu-emi.eu/namespaces/2011/02/storagerecord">'
)
UR_CLOSE = '</sr:StorageUsageRecords>'
else:
raise ApelDbException(
'Can only send URs for JobRecords and StorageRecords.')
buf.write(XML_HEADER + '\n')
buf.write(UR_OPEN + '\n')
buf.write('\n'.join(
[record.get_ur(self._withhold_dns) for record in records]))
buf.write('\n' + UR_CLOSE + '\n')
self._msgq.add(buf.getvalue())
buf.close()
del buf
def _write_apel(self, records):
'''
Write one message in the APEL format to the outgoing
message queue.
'''
record_type = type(records[0])
buf = StringIO.StringIO()
buf.write(self.APEL_HEADERS[record_type] + ' \n')
buf.write('%%\n'.join(
[record.get_msg(self._withhold_dns) for record in records]))
buf.write('%%\n')
self._msgq.add(buf.getvalue())
buf.close()
del buf
# initialize logging
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
os.makedirs('/var/log/carpetbag', exist_ok=True)
fh = logging.FileHandler(os.path.join('/var/log/carpetbag', 'carpetbag.log'))
fh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s'))
fh.setLevel(logging.DEBUG)
logging.getLogger().addHandler(fh)
# initialize work queue
carpetbag_root = '/var/lib/carpetbag'
q_root = os.path.join(carpetbag_root, 'dirq')
UPLOADS = os.path.join(carpetbag_root, 'uploads')
QUEUE = 'package_queue'
dirq = QueueSimple(os.path.join(q_root, QUEUE))
# initialize persistent jobid
jobid_file = os.path.join(carpetbag_root, 'jobid')
jobid = 0
try:
with open(jobid_file) as f:
jobid = int(f.read())
except IOError:
pass
with open(jobid_file, 'w') as f:
f.write(str(jobid))
# initialize database
def adapt_datetime(ts):
return time.mktime(ts.timetuple())
"""Tool to convert unloaded messages to loadable ones."""
from dirq.QueueSimple import QueueSimple
from dirq.queue import Queue
inpath = '/root/iris/messages'
outpath = '/root/iris/messages/incoming'
OUTQSCHEMA = {"body": "string", "signer": "string",
"empaid": "string?", "error": "string?"}
inq = QueueSimple(inpath)
outq = Queue(outpath, schema=OUTQSCHEMA)
for name in inq:
if not inq.lock(name):
continue
data = inq.get(name)
outq.add({'body': data, 'signer': 'iris', 'empaid': 'local'})
inq.remove(name)
