def main():
lock = Lock('/tmp/package_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warning('Lock file {} acquired'.format(lock.lock_file))
qm = QueueManager()
head = qm.get_head()
while head is not None:
logger.warning('Active package: {p}'.format(p=head.package))
skip = False
if properties.CHECK_PRE_EXISTENCE_IN_GMN and head.method in [properties.CREATE, properties.UPDATE]:
skip = gmn_exists(properties.PASTA_BASE_URL + 'metadata/eml/' + head.package.replace('.', '/'))
if skip:
logger.warning('Package already exists: {}. Skipping {}.'.format(head.package, head.method))
else:
p = Package(head)
if p.public:
logger.warning('Processing: {p}'.format(p=p.package))
resource = p.resources[properties.METADATA]
if p.method == properties.CREATE:
process_create_package(package=p)
elif p.method == properties.UPDATE:
process_update_package(package=p, queue_manager=qm)
elif p.method == properties.DELETE:
process_archive_package(package=p)
else:
msg = 'Unrecognized package event "{event}" for' \
'package: {package}'.format(event=p.method,
package=p.package)
raise(AdapterIncompleteStateException(msg))
else:
logger.warning('Package not public: {p}'.format(p=p.package))
qm.dequeue(package=head.package, method=head.method)
if properties.SLEEP_BETWEEN_PACKAGES:
time.sleep(int(properties.SLEEP_BETWEEN_PACKAGES))
head = qm.get_head()
logger.warning('Queue empty')
lock.release()
logger.warning('Lock file {} released'.format(lock.lock_file))
return 0
def main():
lock = Lock('/tmp/package_manager.lock')
if lock.locked:
logger.error('Lock file {} exists, exiting...'.format(lock.lock_file))
return 1
else:
lock.acquire()
logger.warn('Lock file {} acquired'.format(lock.lock_file))
qm = QueueManager()
head = qm.get_head()
while head is not None:
logger.warn('Active package: {p}'.format(p=head.package))
p = Package(head)
if p.public:
logger.warn('Processing: {p}'.format(p=p.package))
if p.method == properties.CREATE:
process_create_package(package=p)
elif p.method == properties.UPDATE:
process_update_package(package=p, queue_manager=qm)
elif p.method == properties.DELETE:
process_archive_package(package=p)
else:
msg = 'Unrecognized package event "{event}" for' \
'package: {package}'.format(event=p.method,
package=p.package)
raise (AdapterIncompleteStateException(msg))
else:
logger.warn('Package not public: {p}'.format(p=p.package))
qm.dequeue(package=p.package, method=p.method)
head = qm.get_head()
logger.warn('Queue empty')
lock.release()
logger.warn('Lock file {} released'.format(lock.lock_file))
return 0
class TestAdapterQueue(unittest.TestCase):
def setUp(self):
self.qm = QueueManager(queue='test_adapter_queue.sqlite')
self.build_packages()
def tearDown(self):
self.qm.delete_queue()
def test_enqueue(self):
e = Event()
e.package = 'edi.3.2'
e.datetime = '2017-01-03T14:30:56.673000'
e.method = 'createDataPackage'
e.owner = 'uid=SBC,o=LTER,dc=ecoinformatics,dc=org'
e.doi = 'doi:10.5072/FK2/381addd8bfda02f8ba85329df8f903dc'
self.qm.enqueue(event=e)
self.assertEqual(self.qm.get_count(), 1)
def test_get_head(self):
self.enqueue_all()
e = self.qm.get_head()
self.assertEqual(e.package, self.events[0].package)
def test_dequeue(self):
self.enqueue_all()
e = self.qm.get_head()
self.qm.dequeue(package=e.package, method=e.method)
e = self.qm.get_head()
self.assertEqual(e.package, self.events[1].package)
def test_get_last_datetime(self):
self.enqueue_all()
datetime = self.qm.get_last_datetime()
self.assertEqual(self.events[9].datetime, datetime)
def test_get_predecessor(self):
self.enqueue_all()
e = self.events[5]
p = self.qm.get_predecessor(package=e.package)
self.assertEqual(self.events[4].package, p.package)
def build_packages(self):
e0 = Event()
e0.package = 'edi.3.2'
e0.datetime = '2017-01-03T14:30:56.673000'
e0.method = 'createDataPackage'
e0.owner = 'uid=SBC,o=LTER,dc=ecoinformatics,dc=org'
e0.doi = 'doi:10.5072/FK2/381addd8bfda02f8ba85329df8f903dc'
e1 = Event()
e1.package = 'edi.3002.1'
e1.datetime = '2017-06-02T17:46:57.154000'
e1.method = 'createDataPackage'
e1.owner = 'uid=LNO,o=LTER,dc=ecoinformatics,dc=org'
e1.doi = 'doi:10.5072/FK2/55fcb5e7de4634cc332d4f874d0caf73'
e2 = Event()
e2.package = 'edi.98.1'
e2.datetime = '2017-06-14T17:20:47.138000'
e2.method = 'createDataPackage'
e2.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e2.doi = 'doi:10.5072/FK2/0ffb0cde729f2e1bf97e9a7f7acc9d57'
e3 = Event()
e3.package = 'edi.98.2'
e3.datetime = '2017-06-14T17:45:30.938000'
e3.method = 'updateDataPackage'
e3.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e3.doi = 'doi:10.5072/FK2/c21403aa2cf1fc0535b7a3a21f3b3852'
e4 = Event()
e4.package = 'edi.98.3'
e4.datetime = '2017-06-14T18:31:31.549000'
e4.method = 'updateDataPackage'
e4.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e4.doi = 'doi:10.5072/FK2/586c753cc9adbc6102d0a3b458cbfb1c'
e5 = Event()
e5.package = 'edi.98.4'
e5.datetime = '2017-06-14T19:01:20.551000'
e5.method = 'updateDataPackage'
e5.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e5.doi = 'doi:10.5072/FK2/f6b49227664aaac91675a785e29bc12f'
e6 = Event()
e6.package = 'edi.100.1'
e6.datetime = '2017-06-14T19:04:00.470000'
e6.method = 'createDataPackage'
e6.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e6.doi = 'doi:10.5072/FK2/d9b8652cd4f1a63935af87f19387351c'
e7 = Event()
e7.package = 'edi.100.2'
e7.datetime = '2017-06-14T19:09:20.009000'
e7.method = 'updateDataPackage'
e7.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e7.doi = 'doi:10.5072/FK2/2aa459937b15c7133a48828a54b9a249'
e8 = Event()
e8.package = 'edi.100.1'
e8.datetime = '2017-06-15T13:13:29.717000'
e8.method = 'deleteDataPackage'
e8.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e8.doi = 'doi:10.5072/FK2/d9b8652cd4f1a63935af87f19387351c'
e9 = Event()
e9.package = 'edi.100.2'
e9.datetime = '2017-06-15T13:13:29.717000'
e9.method = 'deleteDataPackage'
e9.owner = 'uid=EDI,o=LTER,dc=ecoinformatics,dc=org'
e9.doi = 'doi:10.5072/FK2/2aa459937b15c7133a48828a54b9a249'
self.events = (e0, e1, e2, e3, e4, e5, e6, e7, e8, e9)
def enqueue_all(self):
for event in self.events:
self.qm.enqueue(event=event)
