def main(package_id, dryrun): logger.info(f'package_id={package_id}') lock = Lock('/tmp/poll_manager.lock') if lock.locked: logger.error('Lock file {} exists, exiting...'.format(lock.lock_file)) return 1 else: lock.acquire() logger.warning('Lock file {} acquired'.format(lock.lock_file)) try: scope, identifier, revision = package_id.split('.') connection = connect() event = get_package_info(connection, scope, identifier, revision) if event: qm = QueueManager() msg = f"Enqueue: {event.package} - {event.datetime} - " + \ f"{event.owner} - {event.doi} - {event.method}" logger.warning(msg) if not dryrun: qm.enqueue(event=event) else: msg = f"DRYRUN: qm.enqueue(event=event)" logger.info(msg) except AdapterRequestFailureException as e: logger.error(e) lock.release() logger.warning('Lock file {} released'.format(lock.lock_file)) return 0
def parse(url=None, fromDate=None, toDate=None, scope=properties.SCOPE): """ Parse the PASTA list of changes XML based on the query parameters provided :param url: changes URL as a String :param fromDate: fromDate as a date formatted String '%Y-%m-%dT%H:%M:%S.%f' :param toDate: toDate as a data formatted String '%Y-%m-%dT%H:%M:%S.%f' :param in_scope: in_scope filter value (only one) as a String for changes query :return: 0 if successful, 1 otherwise """ if fromDate is not None: url = url + 'fromDate=' + fromDate + '&' if toDate is not None: url = url + 'toDate=' + toDate + '&' if scope is not None: url = url + 'scope=' + scope r = adapter_utilities.requests_get_url_wrapper(url=url) if r is not None: qm = QueueManager() tree = ET.ElementTree(ET.fromstring(r.text.strip())) for dataPackage in tree.iter('dataPackage'): package = dataPackage.find('./packageId') date = dataPackage.find('./date') method = dataPackage.find('./serviceMethod') owner = dataPackage.find('./principal') doi = dataPackage.find('./doi') event = Event() event.package = package.text event.datetime = date.text event.method = method.text event.owner = owner.text event.doi = doi.text # Skip fromDate record(s) that already exist in queue if fromDate.rstrip('0') == date.text: msg = 'Skipping: {} - {} - {}'.format(package.text, date.text, method.text) logger.warn(msg) else: # Provide additional filter for multiple scope values package_scope = event.package.split('.')[0] if package_scope in properties.PASTA_WHITELIST: msg = 'Enqueue: {} - {} - {}'.format( package.text, date.text, method.text) logger.warn(msg) qm.enqueue(event=event) else: logger.info('Package {} out of scope'.format(package.text))
def enqueue_request(): QueueManager.enqueue(request) return "SUCCESS"
class TestAdapterQueue(unittest.TestCase): def setUp(self): self.qm = QueueManager(queue='test_adapter_queue.sqlite') self.build_packages() def tearDown(self): self.qm.delete_queue() def test_enqueue(self): e = Event() e.package = 'edi.3.2' e.datetime = '2017-01-03T14:30:56.673000' e.method = 'createDataPackage' e.owner = 'uid=SBC,o=LTER,dc=ecoinformatics,dc=org' e.doi = 'doi:10.5072/FK2/381addd8bfda02f8ba85329df8f903dc' self.qm.enqueue(event=e) self.assertEqual(self.qm.get_count(), 1) def test_get_head(self): self.enqueue_all() e = self.qm.get_head() self.assertEqual(e.package, self.events[0].package) def test_dequeue(self): self.enqueue_all() e = self.qm.get_head() self.qm.dequeue(package=e.package, method=e.method) e = self.qm.get_head() self.assertEqual(e.package, self.events[1].package) def test_get_last_datetime(self): self.enqueue_all() datetime = self.qm.get_last_datetime() self.assertEqual(self.events[9].datetime, datetime) def test_get_predecessor(self): self.enqueue_all() e = self.events[5] p = self.qm.get_predecessor(package=e.package) self.assertEqual(self.events[4].package, p.package) def build_packages(self): e0 = Event() e0.package = 'edi.3.2' e0.datetime = '2017-01-03T14:30:56.673000' e0.method = 'createDataPackage' e0.owner = 'uid=SBC,o=LTER,dc=ecoinformatics,dc=org' e0.doi = 'doi:10.5072/FK2/381addd8bfda02f8ba85329df8f903dc' e1 = Event() e1.package = 'edi.3002.1' e1.datetime = '2017-06-02T17:46:57.154000' e1.method = 'createDataPackage' e1.owner = 'uid=LNO,o=LTER,dc=ecoinformatics,dc=org' e1.doi = 'doi:10.5072/FK2/55fcb5e7de4634cc332d4f874d0caf73' e2 = Event() e2.package = 'edi.98.1' e2.datetime = '2017-06-14T17:20:47.138000' e2.method = 'createDataPackage' e2.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org' e2.doi = 'doi:10.5072/FK2/0ffb0cde729f2e1bf97e9a7f7acc9d57' e3 = Event() e3.package = 'edi.98.2' e3.datetime = '2017-06-14T17:45:30.938000' e3.method = 'updateDataPackage' e3.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org' e3.doi = 'doi:10.5072/FK2/c21403aa2cf1fc0535b7a3a21f3b3852' e4 = Event() e4.package = 'edi.98.3' e4.datetime = '2017-06-14T18:31:31.549000' e4.method = 'updateDataPackage' e4.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org' e4.doi = 'doi:10.5072/FK2/586c753cc9adbc6102d0a3b458cbfb1c' e5 = Event() e5.package = 'edi.98.4' e5.datetime = '2017-06-14T19:01:20.551000' e5.method = 'updateDataPackage' e5.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org' e5.doi = 'doi:10.5072/FK2/f6b49227664aaac91675a785e29bc12f' e6 = Event() e6.package = 'edi.100.1' e6.datetime = '2017-06-14T19:04:00.470000' e6.method = 'createDataPackage' e6.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org' e6.doi = 'doi:10.5072/FK2/d9b8652cd4f1a63935af87f19387351c' e7 = Event() e7.package = 'edi.100.2' e7.datetime = '2017-06-14T19:09:20.009000' e7.method = 'updateDataPackage' e7.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org' e7.doi = 'doi:10.5072/FK2/2aa459937b15c7133a48828a54b9a249' e8 = Event() e8.package = 'edi.100.1' e8.datetime = '2017-06-15T13:13:29.717000' e8.method = 'deleteDataPackage' e8.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org' e8.doi = 'doi:10.5072/FK2/d9b8652cd4f1a63935af87f19387351c' e9 = Event() e9.package = 'edi.100.2' e9.datetime = '2017-06-15T13:13:29.717000' e9.method = 'deleteDataPackage' e9.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org' e9.doi = 'doi:10.5072/FK2/2aa459937b15c7133a48828a54b9a249' self.events = (e0, e1, e2, e3, e4, e5, e6, e7, e8, e9) def enqueue_all(self): for event in self.events: self.qm.enqueue(event=event)
def parse(url=None, fromDate=None, toDate=None, scope=None): """ Parse the PASTA list of changes XML based on the query parameters provided :param url: changes URL as a String :param fromDate: fromDate as a datetime :param toDate: toDate as a datetime :param scope: scope filter value (only one) as a String for changes query :return: 0 if successful, 1 otherwise """ msg = f'parse params: url-{url}, fromDate-{fromDate}, toDate-{toDate},' + \ f' scope-{scope}' logger.info(msg) # convert to string representations fromDate = datetime.strftime(fromDate, '%Y-%m-%dT%H:%M:%S.%f') if toDate is not None: toDate = datetime.strftime(toDate, '%Y-%m-%dT%H:%M:%S.%f') # add date(s) to url if fromDate is not None: url = url + 'fromDate=' + fromDate if toDate is not None: url = url + '&toDate=' + toDate if scope is not None: url = url + '&scope=' + scope logger.info('requests_get_url_wrapper: ' + url) r = adapter_utilities.requests_get_url_wrapper(url=url, rethrow=True) if r is not None: qm = QueueManager() tree = ET.ElementTree(ET.fromstring(r.text.strip())) for dataPackage in tree.iter('dataPackage'): package = dataPackage.find('./packageId') date = dataPackage.find('./date') method = dataPackage.find('./serviceMethod') owner = dataPackage.find('./principal') doi = dataPackage.find('./doi') event = Event() event.package = package.text event.datetime = date.text event.method = method.text event.owner = owner.text event.doi = doi.text # Skip fromDate record(s) that already exist in queue if fromDate.rstrip('0') == date.text: msg = 'Skipping: {} - {} - {}'.format(package.text, date.text, method.text) logger.warning(msg) else: # Provide additional filter for multiple scope values package_scope = event.package.split('.')[0] if package_scope in properties.PASTA_WHITELIST: msg = 'Enqueue: {} - {} - {}'.format( package.text, date.text, method.text) logger.warning(msg) qm.enqueue(event=event) else: logger.info('Package {} out of scope'.format(package.text))
def enqueue_request(): QueueManager.enqueue(request) QueueManager.print_val() return "SUCCESS"