def __init__(self, dest, qpath, cert, key, api_version): """Initialize a Sender.""" self._cert = cert self._key = key self._outq = QueueSimple(qpath) self._dest = dest self._api_version = api_version
def write_to_ssm(extract, config): """forwards usage records to SSM""" # ensure outgoing directory existence ssm_input_path = os.path.expanduser(config['ssm_input_path']) if not os.access(ssm_input_path, os.F_OK): os.makedirs(ssm_input_path, 0755) # only write non void URs file if len(extract) > 0: output = config['ssm_input_header'] + "\n" # itterate over VMs for vmname in extract.keys(): logging.debug("generating ssm input file for VM %s" % vmname) for item in orderedFields: logging.debug("generating record %s: %s" % (item, extract[vmname][item])) output += "%s: %s\n" % (item, extract[vmname][item]) output += config['ssm_input_sep'] + "\n" # write file try: dirq = QueueSimple(ssm_input_path) dirq.add(output) except: logging.error('unable to push message in apel-ssm queue <%s>' % ssm_input_path) else: logging.debug('no usage records, skip forwarding to SSM')
def write_to_ssm( extract, config ): """forwards usage records to SSM""" # ensure outgoing directory existence ssm_input_path = os.path.expanduser(config['ssm_input_path']) if not os.access(ssm_input_path, os.F_OK): os.makedirs(ssm_input_path, 0755) # only write non void URs file if len(extract) > 0: output = config['ssm_input_header'] + "\n" # itterate over VMs for vmname in extract.keys(): logging.debug("generating ssm input file for VM %s" % vmname) for item in orderedFields: logging.debug("generating record %s: %s" % (item, extract[vmname][item]) ) output += "%s: %s\n" % ( item, extract[vmname][item] ) output += config['ssm_input_sep'] + "\n" # write file try: dirq = QueueSimple(ssm_input_path) dirq.add(output) except: logging.error('unable to push message in apel-ssm queue <%s>' % ssm_input_path) else: logging.debug('no usage records, skip forwarding to SSM')
def test_get_delete_messages_one(self): qpub = QueueSimple(self.test_dir, umask=0) qpub.add('123') qsub = DirectoryQueueClient(self.ch) qsub.connect() assert qsub.get_messages() == [123] assert qpub.count() == 0
class DirectoryQueue(MsgBase): def __init__(self, configHolder): super(DirectoryQueue, self).__init__(configHolder) self.queue = QueueSimple(self.msg_queue) def send(self, message): self.queue.add(message)
def __init__(self, db, qpath, inc_vos=None, exc_vos=None, local=False, withhold_dns=False): self._db = db outpath = os.path.join(qpath, "outgoing") self._msgq = QueueSimple(outpath) self._inc_vos = inc_vos self._exc_vos = exc_vos self._local = local self._withhold_dns = withhold_dns
def test05_add_path(self): 'QueueSimple.add_path()' qs = QueueSimple(self.qdir) data = 'foo0oo' path = self.tempdir + '/foo.bar' open(path, 'w').write(data) elem = qs.add_path(path) assert open(self.qdir + '/' + elem).read() == data self.failIf(os.path.exists(path))
def test08_count(self): 'QueueSimple.count()' qs = QueueSimple(self.qdir) # add "normal" element qs.add('foo') # simply add a file (fake element) into the elements directory fake_elem = os.listdir(self.qdir)[0] + '/' + 'foo.bar' open(self.qdir + '/' + fake_elem, 'w').write('') self.assertEqual(qs.count(), 1)
def test09_remove(self): 'QueueSimple.remove()' qs = QueueSimple(self.qdir, granularity=1) for _ in range(5): qs.add('foo') assert qs.count() == 5 for elem in qs: qs.lock(elem) qs.remove(elem) self.assertEqual(qs.count(), 0)
def test06_lock_unlock(self): 'QueueSimple.lock()' qs = QueueSimple(self.qdir) data = 'foo' elem_name = 'foo.bar' elem_full_path = self.qdir + '/' + elem_name open(elem_full_path, 'w').write(data) self.assertEqual(qs.lock(elem_name), 1) self.failUnless(os.path.exists(elem_full_path + LOCKED_SUFFIX)) qs.unlock(elem_name)
def test02_add_data(self): 'QueueSimple._add_data()' data = 'f0o' qs = QueueSimple(self.qdir) _subdirName, _fullFnOrig = qs._add_data(data) subdirs = os.listdir(self.qdir) assert len(subdirs) == 1 assert _subdirName == subdirs[0] subdir = self.qdir + '/' + subdirs[0] files = os.listdir(subdir) assert len(files) == 1 fn = subdir + '/' + files[0] assert _fullFnOrig == fn assert open(fn, 'r').read() == data
def test03_add_path(self): 'QueueSimple._add_path()' data = 'abc' qs = QueueSimple(self.qdir) _dir = 'elems' elems_dir = self.qdir + '/' + _dir os.mkdir(elems_dir) _tmpName = self.tempdir + '/elem.tmp' open(_tmpName, 'w').write(data) assert len(os.listdir(self.tempdir)) == 2 newName = qs._add_path(_tmpName, _dir) assert len(os.listdir(elems_dir)) == 1 assert len(os.listdir(self.tempdir)) == 1 assert open(self.qdir + '/' + newName).read() == data
def check_dir(root): ''' Check the directory for incoming, outgoing, reject or accept directories. If they exist, check them for messages. ''' print '\nStarting message status script.' print 'Root directory: %s\n' % root queues = [] incoming = os.path.join(root, 'incoming') if os.path.isdir(incoming): queues.append(Queue(incoming, schema=QSCHEMA)) outgoing = os.path.join(root, 'outgoing') # outgoing uses QueueSimple, not Queue if os.path.isdir(outgoing): queues.append(QueueSimple(outgoing)) reject = os.path.join(root, 'reject') if os.path.isdir(reject): queues.append(Queue(reject, schema=QSCHEMA)) accept = os.path.join(root, 'accept') if os.path.isdir(accept): queues.append(Queue(accept, schema=QSCHEMA)) for q in queues: msgs, locked = check_queue(q) #check_empty_dirs(q) print ' Messages: %s' % msgs print ' Locked: %s\n' % locked if locked > 0: question = 'Unlock %s messages?' % locked if ask_user(question): clear_locks(q)
def __init__(self, db, qpath, inc_vos=None, exc_vos=None, local=False): self._db = db outpath = os.path.join(qpath, "outgoing") self._msgq = QueueSimple(outpath) self._inc_vos = inc_vos self._exc_vos = exc_vos self._local = local
def test01_init(self): 'QueueSimple.__init__()' path = self.tempdir + '/aaa/bbb/ccc' granularity = 30 qs = QueueSimple(path, granularity=granularity) assert qs.path == path assert qs.granularity == granularity
def test_get_delete_messages_max(self): num_messages_half = 5 num_messages_all = 10 qpub = QueueSimple(self.test_dir) for i in range(num_messages_all): qpub.add('{"message" : "%i"}' % i) qsub = DirectoryQueueClient(self.ch) qsub.connect() messages = qsub.get_messages(num_messages_half) assert len(messages) == num_messages_half assert qpub.count() == num_messages_half messages_read = qsub.get_messages(num_messages_half) messages_test = [{'message': str(i)} for i in range(num_messages_half, num_messages_all)] assert messages_read == messages_test assert qpub.count() == 0
def submit_state_change(job, transfer, transfer_state): """ Writes a state change message to the dirq """ msg_enabled = pylons.config.get('fts3.MonitoringMessaging', False) if not msg_enabled or msg_enabled.lower() == 'false': return publish_dn = pylons.config.get('fts3.MonitoringPublishDN', False) msg_dir = pylons.config.get('fts3.MessagingDirectory', '/var/lib/fts3') mon_dir = os.path.join(msg_dir, 'monitoring') _user_dn = job['user_dn'] if publish_dn else '' msg = dict( endpnt=pylons.config['fts3.Alias'], user_dn=_user_dn, src_url=transfer['source_surl'], dst_url=transfer['dest_surl'], vo_name=job['vo_name'], source_se=transfer['source_se'], dest_se=transfer['dest_se'], job_id=job['job_id'], file_id=transfer['file_id'], job_state=job['job_state'], file_state=transfer_state, retry_counter=0, retry_max=0, timestamp=time.time() * 1000, job_metadata=job['job_metadata'], file_metadata=transfer['file_metadata'], ) tmpfile = tempfile.NamedTemporaryFile(dir=msg_dir, delete=False) tmpfile.write("SS ") json.dump(msg, tmpfile) tmpfile.close() q = QueueSimple(path=mon_dir) q.add_path(tmpfile.name) log.debug("Sent SUBMITTED state for %s %d" % (job['job_id'], transfer['file_id']))
def test07_get(self): 'QueueSimple.get()' data = 'foo'.encode() qs = QueueSimple(self.qdir) elem = qs.add(data) qs.lock(elem) self.assertEqual(qs.get(elem), data)
def new_dirq(path, _schema): """Create a new Directory::Queue object, optionally with schema. """ kwargs = {} if opts.type == "simple": if opts.granularity is not None: kwargs['granularity'] = opts.granularity return QueueSimple(path, **kwargs) else: if _schema: schema = {'body': 'string', 'header': 'table?'} kwargs['schema'] = schema if opts.maxelts: kwargs['maxelts'] = opts.maxelts return queue.Queue(path, **kwargs)
def run_queue_once(): from dirq.QueueSimple import QueueSimple dirq = QueueSimple(QDIR) dirq.purge(30,60) did_work = False for name in dirq: if not dirq.lock(name): continue data = dirq.get(name) log.debug("item: %s", data) item = json.loads(data) signal.alarm(15) queue_ts = item['ts'] start = time.time() try : block(item['ip'], item['comment'], item['duration']) except Exception as e: #Was this a whitelisted host or similarly invalid request #Versus a server error? if hasattr(e, 'response') and e.response is not None and e.response.status_code == 400 and 'non_field_errors' in e.response.json(): log.info("Ignored API Error %s", e.response.json()) else: text = e.response.text if hasattr(e, 'response') else '' log.exception("API Error. HTTP Response: %r", text) raise end = time.time() signal.alarm(0) dirq.remove(name) log.info("block ip=%s queue_latency=%0.2f api_latency=%0.2f", item['ip'], start-queue_ts, end-start) did_work = True return did_work
class Sender(object): """A simple class for sending Accounting Records to a REST endpoint.""" def __init__(self, dest, qpath, cert, key, api_version): """Initialize a Sender.""" self._cert = cert self._key = key self._outq = QueueSimple(qpath) self._dest = dest self._api_version = api_version def send_all(self): """Send all the messages in the outgoing queue via REST.""" log.info('Found %s messages.', self._outq.count()) for msgid in self._outq: if not self._outq.lock(msgid): log.warn('Message was locked. %s will not be sent.', msgid) continue text = self._outq.get(msgid) path = '/api/%s/cloud/record' % self._api_version data = text self._rest_send('POST', path, data, {}, 202) log.info("Sent %s", msgid) time.sleep(0.1) self._outq.remove(msgid) log.info('Tidying message directory.') try: # Remove empty dirs and unlock msgs older than 5 min (default) self._outq.purge() except OSError, e: log.warn('OSError raised while purging message queue: %s', e)
class DbUnloader(object): APEL_HEADERS = {JobRecord: JOB_MSG_HEADER, SummaryRecord: SUMMARY_MSG_HEADER, SyncRecord: SYNC_MSG_HEADER, CloudRecord: CLOUD_MSG_HEADER, CloudSummaryRecord: CLOUD_SUMMARY_MSG_HEADER} RECORD_TYPES = {'VJobRecords': JobRecord, 'VSummaries': SummaryRecord, 'VSuperSummaries': SummaryRecord, 'VSyncRecords': SyncRecord, 'VCloudRecords': CloudRecord, 'VCloudSummaries': CloudSummaryRecord} # all record types for which withholding DNs is a valid option MAY_WITHHOLD_DNS = [JobRecord, SyncRecord, CloudRecord] def __init__(self, db, qpath, inc_vos=None, exc_vos=None, local=False, withhold_dns=False): self._db = db outpath = os.path.join(qpath, "outgoing") self._msgq = QueueSimple(outpath) self._inc_vos = inc_vos self._exc_vos = exc_vos self._local = local self._withhold_dns = withhold_dns def _get_base_query(self, record_type): ''' Set up a query object containing the logic which is common for all users of this DbUnloader. ''' query = Query() if record_type in (JobRecord, SummaryRecord, SyncRecord): if self._inc_vos is not None: query.VO_in = self._inc_vos log.info('Sending only these VOs: ') log.info(self._inc_vos) elif self._exc_vos is not None: log.info('Excluding these VOs: ') log.info(self._exc_vos) query.VO_notin = self._exc_vos if not self._local: query.InfrastructureType = 'grid' return query def unload_all(self, table_name, car=False): ''' Unload all records from the specified table. ''' log.info('Unloading all records from %s.' % table_name) record_type = self.RECORD_TYPES[table_name] query = self._get_base_query(record_type) msgs, records = self._write_messages(record_type, table_name, query, car) return msgs, records def unload_sync(self): ''' Unload all records from the SyncRecords table or view. ''' log.info('Writing sync messages.') query = self._get_base_query(SyncRecord) msgs = 0 records = 0 for batch in self._db.get_sync_records(query=query): records += len(batch) self._write_apel(batch) msgs += 1 return msgs, records def unload_gap(self, table_name, start, end, ur=False): ''' Unload all records from the JobRecords table whose EndTime falls within the provided dates (inclusive). ''' record_type = self.RECORD_TYPES[table_name] if record_type != JobRecord: raise ApelDbException("Can only gap publish for JobRecords.") start_tuple = [ int(x) for x in start.split('-') ] end_tuple = [ int(x) for x in end.split('-') ] # get the start of the start date start_date = datetime.date(*start_tuple) start_datetime = datetime.datetime.combine(start_date, datetime.time()) # get the end of the end date end_date = datetime.date(*end_tuple) end_datetime = datetime.datetime.combine(end_date, datetime.time()) end_datetime += datetime.timedelta(days=1) log.info('Finding records with end times between:') log.info(start_datetime) log.info(end_datetime) query = self._get_base_query(record_type) query.EndTime_gt = start_datetime query.EndTime_le = end_datetime msgs, records = self._write_messages(record_type, table_name, query, ur) return msgs, records def unload_latest(self, table_name, ur=False): ''' Unloads any records whose UpdateTime is less than the value in the LastUpdated table. Returns (number of files, number of records) ''' # special case for SuperSummaries if table_name == 'VSuperSummaries': msgs, records = self.unload_latest_super_summaries() else: record_type = self.RECORD_TYPES[table_name] query = self._get_base_query(record_type) since = self._db.get_last_updated() log.info('Getting records updated since: %s' % since) if since is not None: query.UpdateTime_gt = str(since) msgs, records = self._write_messages(record_type, table_name, query, ur) self._db.set_updated() return msgs, records def unload_latest_super_summaries(self, ur=False): ''' Special case for the SuperSummaries table. Since it is generally updated by the SummariseJobs() procedure, all records will have been updated. Instead, send all records for the current month and the preceding month. ''' table_name = 'VSuperSummaries' record_type = self.RECORD_TYPES[table_name] query = self._get_base_query(record_type) # It's actually simpler to use EarliestEndTime or LatestEndTime # to deduce the correct records. since = get_start_of_previous_month(datetime.datetime.now()) log.info('Getting summaries for months since: %s' % since.date()) if since is not None: query.EarliestEndTime_gt = str(since) msgs, records = self._write_messages(record_type, table_name, query, ur) return msgs, records def _write_messages(self, record_type, table_name, query, ur): ''' Write messsages for all the records found in the specified table, according to the logic contained in the query object. ''' if self._withhold_dns and record_type not in self.MAY_WITHHOLD_DNS: raise ApelDbException('Cannot withhold DNs for %s' % record_type.__name__) msgs = 0 records = 0 for batch in self._db.get_records(record_type, table_name, query=query): records += len(batch) if ur: self._write_xml(batch) else: self._write_apel(batch) msgs += 1 return msgs, records def _write_xml(self, records): ''' Write one message in the appropriate XML format to the outgoing message queue. This is currently enabled only for CAR. ''' buf = StringIO.StringIO() if type(records[0]) == JobRecord: XML_HEADER = '<?xml version="1.0" ?>' UR_OPEN = '<urf:UsageRecords xmlns:urf="http://eu-emi.eu/namespaces/2012/11/computerecord" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://eu-emi.eu/namespaces/2012/11/computerecord car_v1.2.xsd">' UR_CLOSE = '</urf:UsageRecords>' # elif type(records[0]) == SummaryRecord: # XML_HEADER = '<?xml version="1.0" ?>' # UR_OPEN = '<aur:SummaryRecords xmlns:aur="http://eu-emi.eu/namespaces/2012/11/aggregatedcomputerecord" xmlns:urf="http://eu-emi.eu/namespaces/2012/11/computerecord" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://eu-emi.eu/namespaces/2012/11/aggregatedcomputerecord ">' # UR_CLOSE = '</aur:SummaryRecords>' else: raise ApelDbException('Can only send URs for JobRecords.') buf.write(XML_HEADER + '\n') buf.write(UR_OPEN + '\n') buf.write('\n'.join( [ record.get_ur(self._withhold_dns) for record in records ] )) buf.write('\n' + UR_CLOSE + '\n') self._msgq.add(buf.getvalue()) buf.close() del buf def _write_apel(self, records): ''' Write one message in the APEL format to the outgoing message queue. ''' record_type = type(records[0]) buf = StringIO.StringIO() buf.write(self.APEL_HEADERS[record_type] + ' \n') buf.write('%%\n'.join( [ record.get_msg(self._withhold_dns) for record in records ] )) buf.write('%%\n') self._msgq.add(buf.getvalue()) buf.close() del buf
def test04_add(self): 'QueueSimple.add()' data = 'foo bar' qs = QueueSimple(self.qdir) elem = qs.add(data) assert open(self.qdir + '/' + elem).read() == data
def queue(ip, comment, duration): from dirq.QueueSimple import QueueSimple dirq = QueueSimple(QDIR) rec = dict(ip=ip, comment=comment, duration=duration, ts=time.time()) dirq.add(json.dumps(rec)) return True
def test11_purge_multDirMultElement(self): 'QueueSimple.purge() multiple directories & elements' qs = QueueSimple(self.qdir, granularity=1) qs.add('foo') assert qs.count() == 1 time.sleep(2) qs.add('bar') assert qs.count() == 2 assert len(os.listdir(self.qdir)) == 2 qs.purge() assert qs.count() == 2 elem = qs.first() qs.lock(elem) qs.remove(elem) assert qs.count() == 1 qs.purge() assert len(os.listdir(self.qdir)) == 1 time.sleep(2) qs.add('baz') assert len(os.listdir(self.qdir)) == 2 for elem in qs: qs.lock(elem) elem1 = qs.first() lock_path1 = self.qdir + '/' + elem1 + LOCKED_SUFFIX assert os.path.exists(lock_path1) is True os.utime(lock_path1, (time.time() - 25, time.time() - 25)) qs.purge(maxlock=10) assert os.path.exists(lock_path1) is False elem2 = qs.next() lock_path2 = self.qdir + '/' + elem2 + LOCKED_SUFFIX assert os.path.exists(lock_path2) is True
class Ssm2(stomp.ConnectionListener): ''' Minimal SSM implementation. ''' # Schema for the dirq message queue. QSCHEMA = {'body': 'string', 'signer': 'string', 'empaid': 'string?'} REJECT_SCHEMA = { 'body': 'string', 'signer': 'string?', 'empaid': 'string?', 'error': 'string' } CONNECTION_TIMEOUT = 10 def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, capath=None, check_crls=False, use_ssl=False, username=None, password=None, enc_cert=None, verify_enc_cert=True, pidfile=None): ''' Creates an SSM2 object. If a listen value is supplied, this SSM2 will be a receiver. ''' self._conn = None self._last_msg = None self._brokers = hosts_and_ports self._cert = cert self._key = key self._enc_cert = enc_cert self._capath = capath self._check_crls = check_crls self._user = username self._pwd = password self._use_ssl = use_ssl # use pwd auth if we're supplied both user and pwd self._use_pwd = username is not None and password is not None self.connected = False self._listen = listen self._dest = dest self._valid_dns = [] self._pidfile = pidfile # create the filesystem queues for accepted and rejected messages if dest is not None and listen is None: self._outq = QueueSimple(qpath) elif listen is not None: inqpath = os.path.join(qpath, 'incoming') rejectqpath = os.path.join(qpath, 'reject') self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA) self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA) else: raise Ssm2Exception('SSM must be either producer or consumer.') # check that the cert and key match if not crypto.check_cert_key(self._cert, self._key): raise Ssm2Exception('Cert and key don\'t match.') # Check that the certificate has not expired. if not crypto.verify_cert_date(self._cert): raise Ssm2Exception('Certificate %s has expired.' % self._cert) # check the server certificate provided if enc_cert is not None: log.info('Messages will be encrypted using %s', enc_cert) if not os.path.isfile(self._enc_cert): raise Ssm2Exception( 'Specified certificate file does not exist: %s.' % self._enc_cert) # Check that the encyption certificate has not expired. if not crypto.verify_cert_date(enc_cert): raise Ssm2Exception( 'Encryption certificate %s has expired. Please obtain the ' 'new one from the final server receiving your messages.' % enc_cert) if verify_enc_cert: if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls): raise Ssm2Exception( 'Failed to verify server certificate %s against CA path %s.' % (self._enc_cert, self._capath)) # If the overall SSM log level is info, we want to only # see log entries from stomp.py at the warning level and above. if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO: logging.getLogger("stomp.py").setLevel(logging.WARNING) # If the overall SSM log level is debug, we want to only # see log entries from stomp.py at the info level and above. elif logging.getLogger( "ssm.ssm2").getEffectiveLevel() == logging.DEBUG: logging.getLogger("stomp.py").setLevel(logging.INFO) def set_dns(self, dn_list): ''' Set the list of DNs which are allowed to sign incoming messages. ''' self._valid_dns = dn_list ########################################################################## # Methods called by stomppy ########################################################################## def on_send(self, frame, unused_body=None): ''' Called by stomppy when a message is sent. unused_body is only present to have a backward compatible method signature when using stomp.py v3.1.X ''' try: # Try the stomp.py v4 way first empaid = frame.headers['empa-id'] except KeyError: # Then you are most likely using stomp.py v4. # on_send is now triggered on non message frames # (such as 'CONNECT' frames) and as such without an empa-id. empaid = 'no empa-id' except AttributeError: # Then you are likely using stomp.py v3 empaid = frame['empa-id'] log.debug('Sent message: %s', empaid) def on_message(self, headers, body): ''' Called by stomppy when a message is received. Handle the message according to its content and headers. ''' try: empaid = headers['empa-id'] if empaid == 'ping': # ignore ping message log.info('Received ping message.') return except KeyError: empaid = 'noid' log.info("Received message. ID = %s", empaid) extracted_msg, signer, err_msg = self._handle_msg(body) try: # If the message is empty or the error message is not empty # then reject the message. if extracted_msg is None or err_msg is not None: if signer is None: # crypto failed signer = 'Not available.' elif extracted_msg is not None: # If there is a signer then it was rejected for not being # in the DNs list, so we can use the extracted msg, which # allows the msg to be reloaded if needed. body = extracted_msg log.warn("Message rejected: %s", err_msg) name = self._rejectq.add({ 'body': body, 'signer': signer, 'empaid': empaid, 'error': err_msg }) log.info("Message saved to reject queue as %s", name) else: # message verified ok name = self._inq.add({ 'body': extracted_msg, 'signer': signer, 'empaid': empaid }) log.info("Message saved to incoming queue as %s", name) except (IOError, OSError) as e: log.error('Failed to read or write file: %s', e) def on_error(self, unused_headers, body): ''' Called by stomppy when an error frame is received. ''' log.warn('Error message received: %s', body) raise Ssm2Exception() def on_connected(self, unused_headers, unused_body): ''' Called by stomppy when a connection is established. Track the connection. ''' self.connected = True log.info('Connected.') def on_disconnected(self): ''' Called by stomppy when disconnected from the broker. ''' log.info('Disconnected from broker.') self.connected = False def on_receipt(self, headers, unused_body): ''' Called by stomppy when the broker acknowledges receipt of a message. ''' log.info('Broker received message: %s', headers['receipt-id']) self._last_msg = headers['receipt-id'] def on_receiver_loop_completed(self, _unused_headers, _unused_body): """ Called by stompy when the receiver loop ends. This is usually trigger as part of a disconnect. """ log.debug('on_receiver_loop_completed called.') ########################################################################## # Message handling methods ########################################################################## def _handle_msg(self, text): ''' Deal with the raw message contents appropriately: - decrypt if necessary - verify signature Return plain-text message, signer's DN and an error/None. ''' if text is None or text == '': warning = 'Empty text passed to _handle_msg.' log.warn(warning) return None, None, warning # if not text.startswith('MIME-Version: 1.0'): # raise Ssm2Exception('Not a valid message.') # encrypted - this could be nicer if 'application/pkcs7-mime' in text or 'application/x-pkcs7-mime' in text: try: text = crypto.decrypt(text, self._cert, self._key) except crypto.CryptoException as e: error = 'Failed to decrypt message: %s' % e log.error(error) return None, None, error # always signed try: message, signer = crypto.verify(text, self._capath, self._check_crls) except crypto.CryptoException as e: error = 'Failed to verify message: %s' % e log.error(error) return None, None, error if signer not in self._valid_dns: warning = 'Signer not in valid DNs list: %s' % signer log.warn(warning) return None, signer, warning else: log.info('Valid signer: %s', signer) return message, signer, None def _send_msg(self, message, msgid): ''' Send one message using stomppy. The message will be signed using the host cert and key. If an encryption certificate has been supplied, the message will also be encrypted. ''' log.info('Sending message: %s', msgid) headers = { 'destination': self._dest, 'receipt': msgid, 'empa-id': msgid } if message is not None: to_send = crypto.sign(message, self._cert, self._key) if self._enc_cert is not None: to_send = crypto.encrypt(to_send, self._enc_cert) else: to_send = '' try: # Try using the v4 method signiture self._conn.send(self._dest, to_send, headers=headers) except TypeError: # If it fails, use the v3 metod signiture self._conn.send(to_send, headers=headers) def send_ping(self): ''' If a STOMP connection is left open with no activity for an hour or so, it stops responding. Stomppy 3.1.3 has two ways of handling this, but stomppy 3.0.3 (EPEL 5 and 6) has neither. To get around this, we begin and then abort a STOMP transaction to keep the connection active. ''' # Use time as transaction id to ensure uniqueness within each connection transaction_id = str(time.time()) self._conn.begin({'transaction': transaction_id}) self._conn.abort({'transaction': transaction_id}) def has_msgs(self): ''' Return True if there are any messages in the outgoing queue. ''' return self._outq.count() > 0 def send_all(self): ''' Send all the messages in the outgoing queue. ''' log.info('Found %s messages.', self._outq.count()) for msgid in self._outq: if not self._outq.lock(msgid): log.warn('Message was locked. %s will not be sent.', msgid) continue text = self._outq.get(msgid) self._send_msg(text, msgid) log.info('Waiting for broker to accept message.') while self._last_msg is None: if not self.connected: raise Ssm2Exception('Lost connection.') time.sleep(0.1) self._last_msg = None self._outq.remove(msgid) log.info('Tidying message directory.') try: # Remove empty dirs and unlock msgs older than 5 min (default) self._outq.purge() except OSError as e: log.warn('OSError raised while purging message queue: %s', e) ############################################################################ # Connection handling methods ############################################################################ def _initialise_connection(self, host, port): ''' Create the self._connection object with the appropriate properties, but don't try to start the connection. ''' log.info("Established connection to %s, port %i", host, port) if self._use_ssl: if ssl is None: raise ImportError( "SSL connection requested but the ssl module " "wasn't found.") log.info('Connecting using SSL...') else: log.warning("SSL connection not requested, your messages may be " "intercepted.") # _conn will use the default SSL version specified by stomp.py self._conn = stomp.Connection([(host, port)], use_ssl=self._use_ssl, ssl_key_file=self._key, ssl_cert_file=self._cert, timeout=Ssm2.CONNECTION_TIMEOUT) self._conn.set_listener('SSM', self) def handle_connect(self): ''' Assuming that the SSM has retrieved the details of the broker or brokers it wants to connect to, connect to one. If more than one is in the list self._network_brokers, try to connect to each in turn until successful. ''' for host, port in self._brokers: self._initialise_connection(host, port) try: self.start_connection() break except ConnectFailedException as e: # ConnectFailedException doesn't provide a message. log.warn('Failed to connect to %s:%s.', host, port) except Ssm2Exception as e: log.warn('Failed to connect to %s:%s: %s', host, port, e) if not self.connected: raise Ssm2Exception( 'Attempts to start the SSM failed. The system will exit.') def handle_disconnect(self): ''' When disconnected, attempt to reconnect using the same method as used when starting up. ''' self.connected = False # Shut down properly self.close_connection() # Sometimes the SSM will reconnect to the broker before it's properly # shut down! This prevents that. time.sleep(2) # Try again according to the same logic as the initial startup try: self.handle_connect() except Ssm2Exception: self.connected = False # If reconnection fails, admit defeat. if not self.connected: err_msg = 'Reconnection attempts failed and have been abandoned.' raise Ssm2Exception(err_msg) def start_connection(self): ''' Once self._connection exists, attempt to start it and subscribe to the relevant topics. If the timeout is reached without receiving confirmation of connection, raise an exception. ''' if self._conn is None: raise Ssm2Exception('Called start_connection() before a \ connection object was initialised.') self._conn.start() self._conn.connect(wait=True) if self._dest is not None: log.info('Will send messages to: %s', self._dest) if self._listen is not None: # Use a static ID for the subscription ID because we only ever have # one subscription within a connection and ID is only considered # to differentiate subscriptions within a connection. self._conn.subscribe(destination=self._listen, id=1, ack='auto') log.info('Subscribing to: %s', self._listen) i = 0 while not self.connected: time.sleep(0.1) if i > Ssm2.CONNECTION_TIMEOUT * 10: err = 'Timed out while waiting for connection. ' err += 'Check the connection details.' raise Ssm2Exception(err) i += 1 def close_connection(self): ''' Close the connection. This is important because it runs in a separate thread, so it can outlive the main process if it is not ended. ''' try: self._conn.disconnect() except (stomp.exception.NotConnectedException, socket.error): self._conn = None except AttributeError: # AttributeError if self._connection is None already pass log.info('SSM connection ended.') def startup(self): ''' Create the pidfile then start the connection. ''' if self._pidfile is not None: try: f = open(self._pidfile, 'w') f.write(str(os.getpid())) f.write('\n') f.close() except IOError as e: log.warn('Failed to create pidfile %s: %s', self._pidfile, e) self.handle_connect() def shutdown(self): ''' Close the connection then remove the pidfile. ''' self.close_connection() if self._pidfile is not None: try: if os.path.exists(self._pidfile): os.remove(self._pidfile) else: log.warn('pidfile %s not found.', self._pidfile) except IOError as e: log.warn('Failed to remove pidfile %s: %e', self._pidfile, e) log.warn('SSM may not start again until it is removed.')
def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, capath=None, check_crls=False, use_ssl=False, username=None, password=None, enc_cert=None, verify_enc_cert=True, pidfile=None, path_type='dirq', protocol=STOMP_MESSAGING, project=None, token=''): ''' Creates an SSM2 object. If a listen value is supplied, this SSM2 will be a receiver. ''' self._conn = None self._last_msg = None self._brokers = hosts_and_ports self._cert = cert self._key = key self._enc_cert = enc_cert self._capath = capath self._check_crls = check_crls self._user = username self._pwd = password self._use_ssl = use_ssl # use pwd auth if we're supplied both user and pwd self._use_pwd = username is not None and password is not None self.connected = False self._listen = listen self._dest = dest self._valid_dns = [] self._pidfile = pidfile # Used to differentiate between STOMP and AMS methods self._protocol = protocol # Used when interacting with an Argo Messaging Service self._project = project self._token = token if self._protocol == Ssm2.AMS_MESSAGING: self._ams = ArgoMessagingService(endpoint=self._brokers[0], token=self._token, cert=self._cert, key=self._key, project=self._project) # create the filesystem queues for accepted and rejected messages if dest is not None and listen is None: # Determine what sort of outgoing structure to make if path_type == 'dirq': if QueueSimple is None: raise ImportError("dirq path_type requested but the dirq " "module wasn't found.") self._outq = QueueSimple(qpath) elif path_type == 'directory': self._outq = MessageDirectory(qpath) else: raise Ssm2Exception('Unsupported path_type variable.') elif listen is not None: inqpath = os.path.join(qpath, 'incoming') rejectqpath = os.path.join(qpath, 'reject') # Receivers must use the dirq module, so make a quick sanity check # that dirq is installed. if Queue is None: raise ImportError("Receiving SSMs must use dirq, but the dirq " "module wasn't found.") self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA) self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA) else: raise Ssm2Exception('SSM must be either producer or consumer.') # check that the cert and key match if not crypto.check_cert_key(self._cert, self._key): raise Ssm2Exception('Cert and key don\'t match.') # Check that the certificate has not expired. if not crypto.verify_cert_date(self._cert): raise Ssm2Exception('Certificate %s has expired or will expire ' 'within a day.' % self._cert) # check the server certificate provided if enc_cert is not None: log.info('Messages will be encrypted using %s', enc_cert) if not os.path.isfile(self._enc_cert): raise Ssm2Exception( 'Specified certificate file does not exist: %s.' % self._enc_cert) # Check that the encyption certificate has not expired. if not crypto.verify_cert_date(enc_cert): raise Ssm2Exception( 'Encryption certificate %s has expired or will expire ' 'within a day. Please obtain the new one from the final ' 'server receiving your messages.' % enc_cert) if verify_enc_cert: if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls): raise Ssm2Exception( 'Failed to verify server certificate %s against CA path %s.' % (self._enc_cert, self._capath)) # If the overall SSM log level is info, we want to only # see entries from stomp.py and connectionpool at WARNING and above. if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO: logging.getLogger("stomp.py").setLevel(logging.WARNING) logging.getLogger( "requests.packages.urllib3.connectionpool").setLevel( logging.WARNING) # If the overall SSM log level is debug, we want to only # see entries from stomp.py and connectionpool at INFO above. elif logging.getLogger( "ssm.ssm2").getEffectiveLevel() == logging.DEBUG: logging.getLogger("stomp.py").setLevel(logging.INFO) logging.getLogger( "requests.packages.urllib3.connectionpool").setLevel( logging.INFO)
def process_period(config, period): period_start = period['instant'] + dateutil.relativedelta.relativedelta( seconds=-period['range_sec']) print( f"Processing year {period['year']}, month {period['month']}, " f"querying from {period['instant'].isoformat()} and going back {period['range_sec']} s to {period_start.isoformat()}." ) queries = QueryLogic(queryRange=(str(period['range_sec']) + 's')) # SSL generally not used for Prometheus access within a cluster # Docs on instant query API: https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries prom = PrometheusConnect(url=config.prometheus_server, disable_ssl=True) prom_connect_params = { 'time': period['instant'].isoformat(), 'timeout': config.query_timeout } raw_results, results, result_lengths = {}, {}, [] # iterate over each query (cputime, starttime, endtime, cores) producing raw_results['cputime'] etc. for query_name, query_string in vars(queries).items(): # Each of these raw_results is a list of dicts. Each dict in the list represents an individual data point, and contains: # 'metric': a dict of one or more key-value pairs of labels, one of which is the pod name ('exported_pod'). # 'value': a list in which the 0th element is the timestamp of the value, and 1th element is the actual value we're interested in. print(f'Executing {query_name} query: {query_string}') t1 = timer() raw_results[query_name] = prom.custom_query(query=query_string, params=prom_connect_params) t2 = timer() results[query_name] = dict(rearrange(raw_results[query_name])) result_lengths.append(len(results[query_name])) t3 = timer() print( f'Query finished in {t2 - t1} s, processed in {t3 - t2} s. Got {len(results[query_name])} items from {len(raw_results[query_name])} results. Peak RAM usage: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}K.' ) del raw_results[query_name] cputime = results['cputime'] endtime = results['endtime'] starttime = results['starttime'] cores = results['cores'] # Confirm the assumption that cputime should have the fewest entries, while starttime and cores may have additional ones # corresponding to jobs that have started but not finished yet, and endtime may have additional ones if there are pods without CPU resource requests. # We only want the jobs for which all values are available: start time, end time, CPU request. # Note that jobs which started last month and finished this month will be properly included and accounted in this month. assert len(cputime) == min( result_lengths), "cputime should be the shortest list" # However, jobs that finished last month may show up in this month's data if they are still present on the cluster this month (in Completed state). # Exclude them by filtering with a lambda (since you can't pass an argument to a function object AFAIK). endtime = dict( filter(lambda x: x[1] >= datetime.datetime.timestamp(period_start), endtime.items())) # Prepare to iterate over jobs which meet all criteria. valid_jobs = cputime.keys() & endtime.keys() # avoid sending empty records if len(valid_jobs) == 0: print('No records to process.') return sum_cputime = 0 t4 = timer() for key in valid_jobs: assert endtime[key] > starttime[ key], "job end time is before start time" # double check cputime calc of this job delta = abs(cputime[key] - (endtime[key] - starttime[key]) * cores[key]) assert delta < 0.001, "cputime calculation is inaccurate" sum_cputime += cputime[key] # CPU time as calculated here means (# cores * job duration), which apparently corresponds to # the concept of wall time in APEL accounting. It is not clear what CPU time means in APEL; # could be the actual CPU usage % integrated over the job (# cores * job duration * usage) # but this does not seem to be documented clearly. Some batch systems do not actually measure # this so it is not reported consistently or accurately. Some sites have CPU efficiency # (presumably defined as CPU time / wall time) time that is up to ~ 500% of the walltime, or # always fixed at 100%. In Kubernetes, the actual CPU usage % is tracked by metrics server # (not KSM), which is not meant to be used for monitoring or accounting purposes and is not # scraped by Prometheus. So just use walltime = cputime sum_cputime = round(sum_cputime) sum_walltime = sum_cputime print(f'total cputime: {sum_cputime}, total walltime: {sum_walltime}') # Write output to the message queue on local filesystem # https://dirq.readthedocs.io/en/latest/queuesimple.html#directory-structure dirq = QueueSimple(str(config.output_path)) summary_output = summary_message( config, year=period['year'], month=period['month'], wall_time=sum_walltime, cpu_time=sum_cputime, n_jobs=len(endtime), # this appears faster than getting min/max during the dict iteration above first_end=round(min(endtime.values())), last_end=round(max(endtime.values()))) sync_output = sync_message(config, year=period['year'], month=period['month'], n_jobs=len(endtime)) t5 = timer() summary_file = dirq.add(summary_output) sync_file = dirq.add(sync_output) print(f'Analyzed {len(endtime)} records in {t5 - t4} s.') print(f'Writing summary record to {config.output_path}/{summary_file}:') print('--------------------------------\n' + summary_output + '--------------------------------') print(f'Writing sync record to {config.output_path}/{sync_file}:') print('--------------------------------\n' + sync_output + '--------------------------------')
class DbUnloader(object): APEL_HEADERS = {JobRecord: JOB_MSG_HEADER, SummaryRecord: SUMMARY_MSG_HEADER, SyncRecord: SYNC_MSG_HEADER, CloudRecord: CLOUD_MSG_HEADER} RECORD_TYPES = {'VJobRecords': JobRecord, 'VSummaries': SummaryRecord, 'VSuperSummaries': SummaryRecord, 'VSyncRecords': SyncRecord, 'VCloudRecords': CloudRecord} def __init__(self, db, qpath, inc_vos=None, exc_vos=None, local=False): self._db = db outpath = os.path.join(qpath, "outgoing") self._msgq = QueueSimple(outpath) self._inc_vos = inc_vos self._exc_vos = exc_vos self._local = local def _get_base_query(self, record_type): ''' Set up a query object containing the logic which is common for all users of this DbUnloader. ''' query = Query() if record_type in (JobRecord, SummaryRecord, SyncRecord): if self._inc_vos is not None: query.VO_in = self._inc_vos log.info('Sending only these VOs: ') log.info(self._inc_vos) elif self._exc_vos is not None: log.info('Excluding these VOs: ') log.info(self._exc_vos) query.VO_notin = self._exc_vos if not self._local: query.InfrastructureType = 'grid' return query def unload_all(self, table_name, car=False): ''' Unload all records from the specified table. ''' log.info('Unloading all records from %s.' % table_name) record_type = self.RECORD_TYPES[table_name] query = self._get_base_query(record_type) msgs, records = self._write_messages(record_type, table_name, query, car) return msgs, records def unload_sync(self): log.info('Writing sync messages.') query = self._get_base_query(SyncRecord) msgs = 0 records = 0 for batch in self._db.get_sync_records(query=query): records += len(batch) self._write_apel(batch) msgs += 1 return msgs, records def unload_gap(self, table_name, start, end, car=False): ''' Unload all records from the JobRecords table whose EndTime falls within the provided dates (inclusive). ''' record_type = self.RECORD_TYPES[table_name] if record_type != JobRecord: raise ApelDbException("Can only gap publish for JobRecords.") start_tuple = [ int(x) for x in start.split('-') ] end_tuple = [ int(x) for x in end.split('-') ] # get the start of the start date start_date = datetime.date(*start_tuple) start_datetime = datetime.datetime.combine(start_date, datetime.time()) # get the end of the end date end_date = datetime.date(*end_tuple) end_datetime = datetime.datetime.combine(end_date, datetime.time()) end_datetime += datetime.timedelta(days=1) log.info('Finding records with end times between:') log.info(start_datetime) log.info(end_datetime) query = self._get_base_query(record_type) query.EndTime_gt = start_datetime query.EndTime_le = end_datetime msgs, records = self._write_messages(record_type, table_name, query, car) return msgs, records def unload_latest(self, table_name, car=False): ''' Unloads records from database to file. Returns the number of created files. ''' record_type = self.RECORD_TYPES[table_name] query = self._get_base_query(record_type) since = self._db.get_last_updated() log.info('Getting records updated since: %s' % since) if since is not None: query.UpdateTime_gt = str(since) msgs, records = self._write_messages(record_type, table_name, query, car) self._db.set_updated() return msgs, records def _write_messages(self, record_type, table_name, query, car): ''' Write messsages for all the records found in the specified table, according to the logic contained in the query object. ''' msgs = 0 records = 0 for batch in self._db.get_records(record_type, table_name, query=query): records += len(batch) if car: self._write_xml(batch) else: self._write_apel(batch) msgs += 1 return msgs, records def _write_xml(self, records): ''' Write one message in the appropriate XML format to the outgoing message queue. This currently works only for CAR. ''' buf = StringIO.StringIO() if type(records[0]) == JobRecord: XML_HEADER = '<?xml version="1.0" ?>' UR_OPEN = '<urf:UsageRecords xmlns:urf="http://eu-emi.eu/namespaces/2012/11/computerecord" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://eu-emi.eu/namespaces/2012/11/computerecord car_v1.2.xsd">' UR_CLOSE = '</urf:UsageRecords>' # elif type(records[0]) == SummaryRecord: # XML_HEADER = '<?xml version="1.0" ?>' # UR_OPEN = '<aur:SummaryRecords xmlns:aur="http://eu-emi.eu/namespaces/2012/11/aggregatedcomputerecord" xmlns:urf="http://eu-emi.eu/namespaces/2012/11/computerecord" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://eu-emi.eu/namespaces/2012/11/aggregatedcomputerecord ">' # UR_CLOSE = '</aur:SummaryRecords>' else: raise ApelDbException('Can only send URs for JobRecords.') buf.write(XML_HEADER + '\n') buf.write(UR_OPEN + '\n') buf.write('\n'.join( [ record.get_ur() for record in records ] )) buf.write('\n' + UR_CLOSE + '\n') self._msgq.add(buf.getvalue()) buf.close() del buf def _write_apel(self, records): ''' Write one message in the APEL format to the outgoing message queue. ''' record_type = type(records[0]) buf = StringIO.StringIO() buf.write(self.APEL_HEADERS[record_type] + ' \n') buf.write('%%\n'.join( [ record.get_msg() for record in records ] )) buf.write('%%\n') self._msgq.add(buf.getvalue()) buf.close() del buf
from dirq.QueueSimple import QueueSimple from random import randint import time path = '/tmp/test' print("start consuming...") dirq = QueueSimple(path) done = 0 while True: for name in dirq: # print("element: %s %s" % (path, name)) if not dirq.lock(name): #print("couldn't lock: %s" % name) # name = dirq.next() continue element = dirq.get(name) #print(element.keys()) print("Body: \"%s\"" % element) if randint(1, 2) % 2: done += 1 dirq.remove(name) #print('Removed') else: dirq.unlock(name) #name = dirq.next() print("consumed %i elements" % done) total_left = dirq.count() print("elements left in the queue: %d" % total_left) time.sleep(0.5)
def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, capath=None, check_crls=False, use_ssl=False, username=None, password=None, enc_cert=None, verify_enc_cert=True, pidfile=None): ''' Creates an SSM2 object. If a listen value is supplied, this SSM2 will be a receiver. ''' self._conn = None self._last_msg = None self._brokers = hosts_and_ports self._cert = cert self._key = key self._enc_cert = enc_cert self._capath = capath self._check_crls = check_crls self._user = username self._pwd = password self._use_ssl = use_ssl # use pwd auth if we're supplied both user and pwd self._use_pwd = username is not None and password is not None self.connected = False self._listen = listen self._dest = dest self._valid_dns = [] self._pidfile = pidfile # create the filesystem queues for accepted and rejected messages if dest is not None and listen is None: self._outq = QueueSimple(qpath) elif listen is not None: inqpath = os.path.join(qpath, 'incoming') rejectqpath = os.path.join(qpath, 'reject') self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA) self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA) else: raise Ssm2Exception('SSM must be either producer or consumer.') # check that the cert and key match if not crypto.check_cert_key(self._cert, self._key): raise Ssm2Exception('Cert and key don\'t match.') # Check that the certificate has not expired. if not crypto.verify_cert_date(self._cert): raise Ssm2Exception('Certificate %s has expired.' % self._cert) # check the server certificate provided if enc_cert is not None: log.info('Messages will be encrypted using %s', enc_cert) if not os.path.isfile(self._enc_cert): raise Ssm2Exception( 'Specified certificate file does not exist: %s.' % self._enc_cert) # Check that the encyption certificate has not expired. if not crypto.verify_cert_date(enc_cert): raise Ssm2Exception( 'Encryption certificate %s has expired. Please obtain the ' 'new one from the final server receiving your messages.' % enc_cert) if verify_enc_cert: if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls): raise Ssm2Exception( 'Failed to verify server certificate %s against CA path %s.' % (self._enc_cert, self._capath)) # If the overall SSM log level is info, we want to only # see log entries from stomp.py at the warning level and above. if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO: logging.getLogger("stomp.py").setLevel(logging.WARNING) # If the overall SSM log level is debug, we want to only # see log entries from stomp.py at the info level and above. elif logging.getLogger( "ssm.ssm2").getEffectiveLevel() == logging.DEBUG: logging.getLogger("stomp.py").setLevel(logging.INFO)
from time import sleep, time import datetime #from cswaUpdateCSpace import processqueueelement queue_dir = sys.argv[1] # sample producer # dirq = QueueSimple(queue_dir) # for count in range(1,101): # name = dirq.add("element %i\n" % count) # print("# added element %i as %s" % (count, name)) # sample consumer dirq = QueueSimple(queue_dir) passes = 0 items = 0 elapsedtime = time() logmessageinterval = 60 #seconds WHEN2POST = 'queue' while True: for name in dirq: if not dirq.lock(name): continue print("# reading element %s" % name) data = dirq.get(name) print data # one could use dirq.unlock(name) to only browse the queue... dirq.unlock(name)
import codecs import time import datetime import csv from dirq.QueueSimple import QueueSimple from toolbox.cswaHelpers import * from common import cspace from common.utils import deURN from cspace_django_site.main import cspace_django_site MAINCONFIG = cspace_django_site.getConfig() DIRQ = QueueSimple('/tmp/cswa') MAXLOCATIONS = 1000 import xml.etree.ElementTree as etree def getWhen2Post(config): try: when2post = config.get('info', 'when2post') except: # default is update immediately when2post = 'update' return when2post
def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None, capath=None, check_crls=False, use_ssl=False, username=None, password=None, enc_cert=None, verify_enc_cert=True, pidfile=None): ''' Creates an SSM2 object. If a listen value is supplied, this SSM2 will be a receiver. ''' self._conn = None self._last_msg = None self._brokers = hosts_and_ports self._cert = cert self._key = key self._enc_cert = enc_cert self._capath = capath self._check_crls = check_crls self._user = username self._pwd = password self._use_ssl = use_ssl # use pwd auth if we're supplied both user and pwd self._use_pwd = username is not None and password is not None self.connected = False self._listen = listen self._dest = dest self._valid_dns = [] self._pidfile = pidfile # create the filesystem queues for accepted and rejected messages if dest is not None and listen is None: self._outq = QueueSimple(qpath) elif listen is not None: inqpath = os.path.join(qpath, 'incoming') rejectqpath = os.path.join(qpath, 'reject') self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA) self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA) else: raise Ssm2Exception('SSM must be either producer or consumer.') # check that the cert and key match if not crypto.check_cert_key(self._cert, self._key): raise Ssm2Exception('Cert and key don\'t match.') # check the server certificate provided if enc_cert is not None: log.info('Messages will be encrypted using %s' % enc_cert) if not os.path.isfile(self._enc_cert): raise Ssm2Exception( 'Specified certificate file does not exist: %s.' % self._enc_cert) if verify_enc_cert: if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls): raise Ssm2Exception( 'Failed to verify server certificate %s against CA path %s.' % (self._enc_cert, self._capath))
def test10_purge_oneDirOneElement(self): 'QueueSimple.purge() one directory & element' qs = QueueSimple(self.qdir) qs.add('foo') self.assertEqual(qs.count(), 1) elem = qs.first() qs.lock(elem) elem_path_lock = self.qdir + '/' + elem + LOCKED_SUFFIX self.assert_(os.path.exists(elem_path_lock) is True) time.sleep(2) qs.purge(maxlock=1) self.assert_(os.path.exists(elem_path_lock) is False) self.assertEqual(qs.count(), 1) self.assertEqual(len(os.listdir(self.qdir)), 1)
def __init__(self, configHolder): super(DirectoryQueue, self).__init__(configHolder) self.queue = QueueSimple(self.msg_queue)
class DbUnloader(object): APEL_HEADERS = { JobRecord: JOB_MSG_HEADER, SummaryRecord: SUMMARY_MSG_HEADER, NormalisedSummaryRecord: NORMALISED_SUMMARY_MSG_HEADER, SyncRecord: SYNC_MSG_HEADER, CloudRecord: CLOUD_MSG_HEADER, CloudSummaryRecord: CLOUD_SUMMARY_MSG_HEADER } RECORD_TYPES = { 'VJobRecords': JobRecord, 'VSummaries': SummaryRecord, 'VSuperSummaries': SummaryRecord, 'VNormalisedSummaries': NormalisedSummaryRecord, 'VNormalisedSuperSummaries': NormalisedSummaryRecord, 'VSyncRecords': SyncRecord, 'VCloudRecords': CloudRecord, 'VCloudSummaries': CloudSummaryRecord, 'VStarRecords': StorageRecord } # all record types for which withholding DNs is a valid option MAY_WITHHOLD_DNS = [JobRecord, SyncRecord, CloudRecord] def __init__(self, db, qpath, inc_vos=None, exc_vos=None, local=False, withhold_dns=False): self._db = db outpath = os.path.join(qpath, "outgoing") self._msgq = QueueSimple(outpath) self._inc_vos = inc_vos self._exc_vos = exc_vos self._local = local self._withhold_dns = withhold_dns def _get_base_query(self, record_type): ''' Set up a query object containing the logic which is common for all users of this DbUnloader. ''' query = Query() if record_type in (JobRecord, SummaryRecord, NormalisedSummaryRecord, SyncRecord): if self._inc_vos is not None: query.VO_in = self._inc_vos log.info('Sending only these VOs: ') log.info(self._inc_vos) elif self._exc_vos is not None: log.info('Excluding these VOs: ') log.info(self._exc_vos) query.VO_notin = self._exc_vos if not self._local: query.InfrastructureType = 'grid' return query def unload_all(self, table_name, car=False): ''' Unload all records from the specified table. ''' log.info('Unloading all records from %s.', table_name) record_type = self.RECORD_TYPES[table_name] query = self._get_base_query(record_type) msgs, records = self._write_messages(record_type, table_name, query, car) return msgs, records def unload_sync(self): ''' Unload all records from the SyncRecords table or view. ''' log.info('Writing sync messages.') query = self._get_base_query(SyncRecord) msgs = 0 records = 0 for batch in self._db.get_sync_records(query=query): records += len(batch) self._write_apel(batch) msgs += 1 return msgs, records def unload_gap(self, table_name, start, end, ur=False): ''' Unload all records from the JobRecords table whose EndTime falls within the provided dates (inclusive). ''' record_type = self.RECORD_TYPES[table_name] if record_type != JobRecord: raise ApelDbException("Can only gap publish for JobRecords.") start_tuple = [int(x) for x in start.split('-')] end_tuple = [int(x) for x in end.split('-')] # get the start of the start date start_date = datetime.date(*start_tuple) start_datetime = datetime.datetime.combine(start_date, datetime.time()) # get the end of the end date end_date = datetime.date(*end_tuple) end_datetime = datetime.datetime.combine(end_date, datetime.time()) end_datetime += datetime.timedelta(days=1) log.info('Finding records with end times between:') log.info(start_datetime) log.info(end_datetime) query = self._get_base_query(record_type) query.EndTime_gt = start_datetime query.EndTime_le = end_datetime msgs, records = self._write_messages(record_type, table_name, query, ur) return msgs, records def unload_latest(self, table_name, ur=False): ''' Unloads any records whose UpdateTime is less than the value in the LastUpdated table. Returns (number of files, number of records) ''' # Special case for [Normalised]SuperSummaries if table_name in ('VSuperSummaries', 'VNormalisedSuperSummaries'): msgs, records = self.unload_latest_super_summaries(table_name) else: record_type = self.RECORD_TYPES[table_name] query = self._get_base_query(record_type) since = self._db.get_last_updated() log.info('Getting records updated since: %s', since) if since is not None: query.UpdateTime_gt = str(since) msgs, records = self._write_messages(record_type, table_name, query, ur) self._db.set_updated() return msgs, records def unload_latest_super_summaries( self, table_name, ur=False, ): """ Unload (normalised) super summaries for current and preceding month Special case for the [Normalised]SuperSummaries table. Since it is generally updated by the SummariseJobs() procedure, all records will have been updated. Instead, send all records for the current month and the preceding month. """ record_type = self.RECORD_TYPES[table_name] query = self._get_base_query(record_type) # It's actually simpler to use EarliestEndTime or LatestEndTime # to deduce the correct records. since = get_start_of_previous_month(datetime.datetime.now()) log.info('Getting summaries for months since: %s', since.date()) if since is not None: query.EarliestEndTime_ge = str(since) msgs, records = self._write_messages(record_type, table_name, query, ur) return msgs, records def _write_messages(self, record_type, table_name, query, ur): ''' Write messsages for all the records found in the specified table, according to the logic contained in the query object. ''' if self._withhold_dns and record_type not in self.MAY_WITHHOLD_DNS: raise ApelDbException('Cannot withhold DNs for %s' % record_type.__name__) if record_type == StorageRecord and not ur: raise ApelDbException( 'Cannot unload StorageRecords in APEL format') msgs = 0 records = 0 for batch in self._db.get_records(record_type, table_name, query=query): records += len(batch) if ur: self._write_xml(batch) else: self._write_apel(batch) msgs += 1 return msgs, records def _write_xml(self, records): ''' Write one message in the appropriate XML format to the outgoing message queue. This is currently enabled only for CAR. ''' buf = StringIO.StringIO() if type(records[0]) == JobRecord: XML_HEADER = '<?xml version="1.0" ?>' UR_OPEN = ( '<urf:UsageRecords xmlns:urf="http://eu-emi.eu/namespace' 's/2012/11/computerecord" xmlns:xsi="http://www.w3.org/2' '001/XMLSchema-instance" xsi:schemaLocation="http://eu-e' 'mi.eu/namespaces/2012/11/computerecord car_v1.2.xsd">') UR_CLOSE = '</urf:UsageRecords>' # elif type(records[0]) == SummaryRecord: # XML_HEADER = '<?xml version="1.0" ?>' # UR_OPEN = ('<aur:SummaryRecords xmlns:aur="http://eu-emi.eu/names' # 'paces/2012/11/aggregatedcomputerecord" xmlns:urf="htt' # 'p://eu-emi.eu/namespaces/2012/11/computerecord" xmlns' # ':xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:' # 'schemaLocation="http://eu-emi.eu/namespaces/2012/11/a' # 'ggregatedcomputerecord ">') # UR_CLOSE = '</aur:SummaryRecords>' elif type(records[0]) == StorageRecord: XML_HEADER = '<?xml version="1.0" ?>' UR_OPEN = ( '<sr:StorageUsageRecords xmlns:sr="http://eu-emi.eu/namespaces/2011/02/storagerecord">' ) UR_CLOSE = '</sr:StorageUsageRecords>' else: raise ApelDbException( 'Can only send URs for JobRecords and StorageRecords.') buf.write(XML_HEADER + '\n') buf.write(UR_OPEN + '\n') buf.write('\n'.join( [record.get_ur(self._withhold_dns) for record in records])) buf.write('\n' + UR_CLOSE + '\n') self._msgq.add(buf.getvalue()) buf.close() del buf def _write_apel(self, records): ''' Write one message in the APEL format to the outgoing message queue. ''' record_type = type(records[0]) buf = StringIO.StringIO() buf.write(self.APEL_HEADERS[record_type] + ' \n') buf.write('%%\n'.join( [record.get_msg(self._withhold_dns) for record in records])) buf.write('%%\n') self._msgq.add(buf.getvalue()) buf.close() del buf
# initialize logging logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') os.makedirs('/var/log/carpetbag', exist_ok=True) fh = logging.FileHandler(os.path.join('/var/log/carpetbag', 'carpetbag.log')) fh.setFormatter(logging.Formatter('%(asctime)s - %(levelname)-8s - %(message)s')) fh.setLevel(logging.DEBUG) logging.getLogger().addHandler(fh) # initialize work queue carpetbag_root = '/var/lib/carpetbag' q_root = os.path.join(carpetbag_root, 'dirq') UPLOADS = os.path.join(carpetbag_root, 'uploads') QUEUE = 'package_queue' dirq = QueueSimple(os.path.join(q_root, QUEUE)) # initialize persistent jobid jobid_file = os.path.join(carpetbag_root, 'jobid') jobid = 0 try: with open(jobid_file) as f: jobid = int(f.read()) except IOError: pass with open(jobid_file, 'w') as f: f.write(str(jobid)) # initialize database def adapt_datetime(ts): return time.mktime(ts.timetuple())
"""Tool to convert unloaded messages to loadable ones.""" from dirq.QueueSimple import QueueSimple from dirq.queue import Queue inpath = '/root/iris/messages' outpath = '/root/iris/messages/incoming' OUTQSCHEMA = {"body": "string", "signer": "string", "empaid": "string?", "error": "string?"} inq = QueueSimple(inpath) outq = Queue(outpath, schema=OUTQSCHEMA) for name in inq: if not inq.lock(name): continue data = inq.get(name) outq.add({'body': data, 'signer': 'iris', 'empaid': 'local'}) inq.remove(name)