def main(package_id, dryrun):
logger.info(f'package_id={package_id}')
lock = Lock('/tmp/poll_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warning('Lock file {} acquired'.format(lock.lock_file))
try:
scope, identifier, revision = package_id.split('.')
connection = connect()
event = get_package_info(connection, scope, identifier, revision)
if event:
qm = QueueManager()
msg = f"Enqueue: {event.package} - {event.datetime} - " + \
f"{event.owner} - {event.doi} - {event.method}"
logger.warning(msg)
if not dryrun:
qm.enqueue(event=event)
else:
msg = f"DRYRUN: qm.enqueue(event=event)"
logger.info(msg)
except AdapterRequestFailureException as e:
logger.error(e)
lock.release()
logger.warning('Lock file {} released'.format(lock.lock_file))
return 0
def parse(url=None, fromDate=None, toDate=None, scope=properties.SCOPE):
"""
Parse the PASTA list of changes XML based on the query parameters provided
:param url: changes URL as a String
:param fromDate: fromDate as a date formatted String '%Y-%m-%dT%H:%M:%S.%f'
:param toDate: toDate as a data formatted String '%Y-%m-%dT%H:%M:%S.%f'
:param in_scope: in_scope filter value (only one) as a String for changes
query
:return: 0 if successful, 1 otherwise
"""
if fromDate is not None:
url = url + 'fromDate=' + fromDate + '&'
if toDate is not None:
url = url + 'toDate=' + toDate + '&'
if scope is not None:
url = url + 'scope=' + scope
r = adapter_utilities.requests_get_url_wrapper(url=url)
if r is not None:
qm = QueueManager()
tree = ET.ElementTree(ET.fromstring(r.text.strip()))
for dataPackage in tree.iter('dataPackage'):
package = dataPackage.find('./packageId')
date = dataPackage.find('./date')
method = dataPackage.find('./serviceMethod')
owner = dataPackage.find('./principal')
doi = dataPackage.find('./doi')
event = Event()
event.package = package.text
event.datetime = date.text
event.method = method.text
event.owner = owner.text
event.doi = doi.text
# Skip fromDate record(s) that already exist in queue
if fromDate.rstrip('0') == date.text:
msg = 'Skipping: {} - {} - {}'.format(package.text, date.text,
method.text)
logger.warn(msg)
else:
# Provide additional filter for multiple scope values
package_scope = event.package.split('.')[0]
if package_scope in properties.PASTA_WHITELIST:
msg = 'Enqueue: {} - {} - {}'.format(
package.text, date.text, method.text)
logger.warn(msg)
qm.enqueue(event=event)
else:
logger.info('Package {} out of scope'.format(package.text))
def enqueue_request():
QueueManager.enqueue(request)
return "SUCCESS"
class TestAdapterQueue(unittest.TestCase):
def setUp(self):
self.qm = QueueManager(queue='test_adapter_queue.sqlite')
self.build_packages()
def tearDown(self):
self.qm.delete_queue()
def test_enqueue(self):
e = Event()
e.package = 'edi.3.2'
e.datetime = '2017-01-03T14:30:56.673000'
e.method = 'createDataPackage'
e.owner = 'uid=SBC,o=LTER,dc=ecoinformatics,dc=org'
e.doi = 'doi:10.5072/FK2/381addd8bfda02f8ba85329df8f903dc'
self.qm.enqueue(event=e)
self.assertEqual(self.qm.get_count(), 1)
def test_get_head(self):
self.enqueue_all()
e = self.qm.get_head()
self.assertEqual(e.package, self.events[0].package)
def test_dequeue(self):
self.enqueue_all()
e = self.qm.get_head()
self.qm.dequeue(package=e.package, method=e.method)
e = self.qm.get_head()
self.assertEqual(e.package, self.events[1].package)
def test_get_last_datetime(self):
self.enqueue_all()
datetime = self.qm.get_last_datetime()
self.assertEqual(self.events[9].datetime, datetime)
def test_get_predecessor(self):
self.enqueue_all()
e = self.events[5]
p = self.qm.get_predecessor(package=e.package)
self.assertEqual(self.events[4].package, p.package)
def build_packages(self):
e0 = Event()
e0.package = 'edi.3.2'
e0.datetime = '2017-01-03T14:30:56.673000'
e0.method = 'createDataPackage'
e0.owner = 'uid=SBC,o=LTER,dc=ecoinformatics,dc=org'
e0.doi = 'doi:10.5072/FK2/381addd8bfda02f8ba85329df8f903dc'
e1 = Event()
e1.package = 'edi.3002.1'
e1.datetime = '2017-06-02T17:46:57.154000'
e1.method = 'createDataPackage'
e1.owner = 'uid=LNO,o=LTER,dc=ecoinformatics,dc=org'
e1.doi = 'doi:10.5072/FK2/55fcb5e7de4634cc332d4f874d0caf73'
e2 = Event()
e2.package = 'edi.98.1'
e2.datetime = '2017-06-14T17:20:47.138000'
e2.method = 'createDataPackage'
e2.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e2.doi = 'doi:10.5072/FK2/0ffb0cde729f2e1bf97e9a7f7acc9d57'
e3 = Event()
e3.package = 'edi.98.2'
e3.datetime = '2017-06-14T17:45:30.938000'
e3.method = 'updateDataPackage'
e3.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e3.doi = 'doi:10.5072/FK2/c21403aa2cf1fc0535b7a3a21f3b3852'
e4 = Event()
e4.package = 'edi.98.3'
e4.datetime = '2017-06-14T18:31:31.549000'
e4.method = 'updateDataPackage'
e4.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e4.doi = 'doi:10.5072/FK2/586c753cc9adbc6102d0a3b458cbfb1c'
e5 = Event()
e5.package = 'edi.98.4'
e5.datetime = '2017-06-14T19:01:20.551000'
e5.method = 'updateDataPackage'
e5.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e5.doi = 'doi:10.5072/FK2/f6b49227664aaac91675a785e29bc12f'
e6 = Event()
e6.package = 'edi.100.1'
e6.datetime = '2017-06-14T19:04:00.470000'
e6.method = 'createDataPackage'
e6.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e6.doi = 'doi:10.5072/FK2/d9b8652cd4f1a63935af87f19387351c'
e7 = Event()
e7.package = 'edi.100.2'
e7.datetime = '2017-06-14T19:09:20.009000'
e7.method = 'updateDataPackage'
e7.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e7.doi = 'doi:10.5072/FK2/2aa459937b15c7133a48828a54b9a249'
e8 = Event()
e8.package = 'edi.100.1'
e8.datetime = '2017-06-15T13:13:29.717000'
e8.method = 'deleteDataPackage'
e8.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e8.doi = 'doi:10.5072/FK2/d9b8652cd4f1a63935af87f19387351c'
e9 = Event()
e9.package = 'edi.100.2'
e9.datetime = '2017-06-15T13:13:29.717000'
e9.method = 'deleteDataPackage'
e9.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e9.doi = 'doi:10.5072/FK2/2aa459937b15c7133a48828a54b9a249'
self.events = (e0, e1, e2, e3, e4, e5, e6, e7, e8, e9)
def enqueue_all(self):
for event in self.events:
self.qm.enqueue(event=event)
def parse(url=None, fromDate=None, toDate=None, scope=None):
"""
Parse the PASTA list of changes XML based on the query parameters provided
:param url: changes URL as a String
:param fromDate: fromDate as a datetime
:param toDate: toDate as a datetime
:param scope: scope filter value (only one) as a String for changes
query
:return: 0 if successful, 1 otherwise
"""
msg = f'parse params: url-{url}, fromDate-{fromDate}, toDate-{toDate},' + \
f' scope-{scope}'
logger.info(msg)
# convert to string representations
fromDate = datetime.strftime(fromDate, '%Y-%m-%dT%H:%M:%S.%f')
if toDate is not None:
toDate = datetime.strftime(toDate, '%Y-%m-%dT%H:%M:%S.%f')
# add date(s) to url
if fromDate is not None:
url = url + 'fromDate=' + fromDate
if toDate is not None:
url = url + '&toDate=' + toDate
if scope is not None:
url = url + '&scope=' + scope
logger.info('requests_get_url_wrapper: ' + url)
r = adapter_utilities.requests_get_url_wrapper(url=url, rethrow=True)
if r is not None:
qm = QueueManager()
tree = ET.ElementTree(ET.fromstring(r.text.strip()))
for dataPackage in tree.iter('dataPackage'):
package = dataPackage.find('./packageId')
date = dataPackage.find('./date')
method = dataPackage.find('./serviceMethod')
owner = dataPackage.find('./principal')
doi = dataPackage.find('./doi')
event = Event()
event.package = package.text
event.datetime = date.text
event.method = method.text
event.owner = owner.text
event.doi = doi.text
# Skip fromDate record(s) that already exist in queue
if fromDate.rstrip('0') == date.text:
msg = 'Skipping: {} - {} - {}'.format(package.text, date.text,
method.text)
logger.warning(msg)
else:
# Provide additional filter for multiple scope values
package_scope = event.package.split('.')[0]
if package_scope in properties.PASTA_WHITELIST:
msg = 'Enqueue: {} - {} - {}'.format(
package.text, date.text, method.text)
logger.warning(msg)
qm.enqueue(event=event)
else:
logger.info('Package {} out of scope'.format(package.text))
def enqueue_request():
QueueManager.enqueue(request)
QueueManager.print_val()
return "SUCCESS"
