def test_mom_soap_to_job_queue(self):
"""assuming test_onXmlRPCJobReceived_no_soap passes, test the correct behaviour when called via soap xml-rpc"""
from lofar.lta.ingest.server.ingestmomadapter import MoMXMLRPCHandler
with MoMXMLRPCHandler(exchange=self.tmp_exchange.address,
mom_xmlrpc_host='localhost',
mom_xmlrpc_port=randint(2345, 4567)) as handler:
# create a tmp job receiver queue
with TemporaryQueue(
exchange=self.tmp_exchange.address) as tmp_job_queue:
with tmp_job_queue.create_frombus() as job_receiver:
# create a job...
job_xml = createJobXml('project', 0, 1, 'dp_id', 2,
'/tmp/path/to/dataproduct')
# submit the job like MoM would via xml-rpc
soap_client = SoapClient(location=handler.server_url(),
namespace="urn:pipeline.export")
soap_client.new_job(fileName='my_job_file.xml',
fileContent=job_xml)
# there should now be a job on the tmp_job_queue
# receive and check it.
job_msg = job_receiver.receive()
self.assertEqual(job_xml, job_msg.content)
def test_h5_plus_raw_file(self):
#beam formed h5 files are always accompanied by a raw file
#these should be tarred togheter
try:
project_name = 'test-project'
obs_id = 987654321
dpname = 'L%s_SAP000_SB000_bf.h5' % obs_id
rawname = dpname.replace('.h5', '.raw')
self.test_dir_path = os.path.join(os.getcwd(), 'testdir_%s' % uuid.uuid1())
def stub_GetStorageTicket(project, filename, filesize, archive_id, job_id, obs_id, check_mom_id=True, id_source='MoM'):
return { 'primary_uri_rnd': 'srm://some.site.name:8443/some/path/data/lofar/ops/projects/%s/%s/%s.tar' % (project, obs_id, dpname),
'result': 'ok',
'error': '',
'ticket': '3E0A47ED860D6339E053B316A9C3BEE2'}
ltamock.GetStorageTicket.side_effect = stub_GetStorageTicket
def stub_uploadDataAndGetSIP(archive_id, storage_ticket, filename, uri, filesize, md5_checksum, adler32_checksum, validate=True):
#return unpecified sip with proper details
from lofar.lta.ingest.server.unspecifiedSIP import makeSIP
return makeSIP(project_name, obs_id, archive_id, storage_ticket, filename, filesize, md5_checksum, adler32_checksum, 'TEST')
mommock.uploadDataAndGetSIP.side_effect = stub_uploadDataAndGetSIP
os.makedirs(self.test_dir_path)
test_file_path = os.path.join(self.test_dir_path, dpname)
with open(test_file_path, 'w') as file:
file.write(4096*'a')
raw_test_file_path = os.path.join(self.test_dir_path, dpname.replace('.h5', '.raw'))
with open(raw_test_file_path, 'w') as file:
file.write(4096*'b')
job_xml = createJobXml(testname, 123456789, obs_id, dpname, 918273645, 'localhost:%s' % test_file_path)
logger.info('job xml: %s', job_xml)
job = parseJobXml(job_xml)
pl = IngestPipeline(job, self.momclient, self.ltaclient,
exchange=self.tmp_exchange.address)
pl.run()
except Exception as e:
self.assertTrue(False, 'Unexpected exception in pipeline: %s' % e)
finally:
# the 'stub-transfered' file ended up in out local stub lta
# with the path: ltastubs._local_globus_file_path
#check extension
self.assertEqual('.tar', os.path.splitext(ltastubs._local_globus_file_path)[-1])
#check tar contents
tar = subprocess.Popen(['tar', '--list', '-f', ltastubs._local_globus_file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
tar_file_list, err = tuple(x.decode('ascii') for x in tar.communicate())
self.assertEqual(tar.returncode, 0)
logger.info('file list in tar:\n%s', tar_file_list)
self.assertTrue(os.path.basename(test_file_path) in tar_file_list)
self.assertTrue(os.path.basename(raw_test_file_path) in tar_file_list)
logger.info('all expected source files are in tar!')
os.remove(test_file_path)
os.remove(raw_test_file_path)
os.removedirs(self.test_dir_path)
def test_directory_with_odd_dataproduct_name(self):
#sometimes somebody has data in a odd directory
#and gives the dataproduct a different name than it's directory
try:
project_name = 'test-project'
obs_id = 987654321
dpname = 'my_funky_dp_name'
self.test_dir_path = os.path.join(os.getcwd(), 'testdir_%s' % uuid.uuid1(), 'my_data_dir')
def stub_uploadDataAndGetSIP(archive_id, storage_ticket, filename, uri, filesize, md5_checksum, adler32_checksum, validate=True):
#return unpecified sip with proper details
from lofar.lta.ingest.server.unspecifiedSIP import makeSIP
return makeSIP(project_name, obs_id, archive_id, storage_ticket, filename, filesize, md5_checksum, adler32_checksum, 'TEST')
mommock.uploadDataAndGetSIP.side_effect = stub_uploadDataAndGetSIP
os.makedirs(self.test_dir_path)
test_file_paths = []
for i in range(10):
test_file_path = os.path.join(self.test_dir_path, 'testfile_%s.txt' % i)
test_file_paths.append(test_file_path)
with open(test_file_path, 'w') as file:
file.write(1000*'a')
job_xml = createJobXml(testname, 123456789, obs_id, dpname, 918273645, 'localhost:%s' % self.test_dir_path)
logger.info('job xml: %s', job_xml)
job = parseJobXml(job_xml)
pl = IngestPipeline(job, self.momclient, self.ltaclient,
exchange=self.tmp_exchange.address)
pl.run()
except Exception as e:
self.assertTrue(False, 'Unexpected exception in pipeline: %s' % e)
finally:
# the 'stub-transfered' file ended up in out local stub lta
# with the path: ltastubs._local_globus_file_path
#check extension
self.assertTrue('.tar' == os.path.splitext(ltastubs._local_globus_file_path)[-1])
#check tar contents
tar = subprocess.Popen(['tar', '--list', '-f', ltastubs._local_globus_file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
tar_file_list, err = tuple(x.decode('ascii') for x in tar.communicate())
self.assertEqual(tar.returncode, 0)
logger.info('file list in tar:\n%s', tar_file_list)
for test_file_path in test_file_paths:
self.assertTrue(os.path.basename(test_file_path) in tar_file_list)
logger.info('all expected source files are in tar!')
for f in os.listdir(self.test_dir_path):
os.remove(os.path.join(self.test_dir_path, f))
os.removedirs(self.test_dir_path)
def test_single_file(self):
try:
project_name = 'test-project'
obs_id = 987654321
dpname = 'L%s_SAP000_SB000_im.h5' % obs_id
self.test_dir_path = os.path.join(os.getcwd(), 'testdir_%s' % uuid.uuid1())
def stub_GetStorageTicket(project, filename, filesize, archive_id, job_id, obs_id, check_mom_id=True, id_source='MoM'):
return { 'primary_uri_rnd': 'srm://some.site.name:8443/some/path/data/lofar/ops/projects/%s/%s/%s' % (project, obs_id, dpname),
'result': 'ok',
'error': '',
'ticket': '3E0A47ED860D6339E053B316A9C3BEE2'}
ltamock.GetStorageTicket.side_effect = stub_GetStorageTicket
def stub_uploadDataAndGetSIP(archive_id, storage_ticket, filename, uri, filesize, md5_checksum, adler32_checksum, validate=True):
#return unpecified sip with proper details
from lofar.lta.ingest.server.unspecifiedSIP import makeSIP
return makeSIP(project_name, obs_id, archive_id, storage_ticket, filename, filesize, md5_checksum, adler32_checksum, 'TEST')
mommock.uploadDataAndGetSIP.side_effect = stub_uploadDataAndGetSIP
os.makedirs(self.test_dir_path)
test_file_path = os.path.join(self.test_dir_path, dpname)
with open(test_file_path, 'w') as file:
file.write(4096*'a')
job_xml = createJobXml(testname, 123456789, obs_id, dpname, 918273645, 'localhost:%s' % test_file_path)
logger.info('job xml: %s', job_xml)
job = parseJobXml(job_xml)
pl = IngestPipeline(job, self.momclient, self.ltaclient,
exchange=self.tmp_exchange.address)
pl.run()
except Exception as e:
self.assertTrue(False, 'Unexpected exception in pipeline: %s' % e)
finally:
# the 'stub-transfered' file ended up in out local stub lta
# with the path: ltastubs._local_globus_file_path
#check extension
self.assertEqual(os.path.splitext(test_file_path)[-1],
os.path.splitext(ltastubs._local_globus_file_path)[-1])
#compare with original
with open(test_file_path) as input, open(ltastubs._local_globus_file_path) as output:
self.assertEqual(input.read(), output.read())
for f in os.listdir(self.test_dir_path):
os.remove(os.path.join(self.test_dir_path, f))
os.removedirs(self.test_dir_path)
def test_onXmlRPCJobReceived_no_soap(self):
"""test the handler routine onXmlRPCJobReceived to check if a job_xml is converted to a message and send on the correct bus"""
from lofar.lta.ingest.server.ingestmomadapter import MoMXMLRPCHandler
with MoMXMLRPCHandler(exchange=self.tmp_exchange.address,
mom_xmlrpc_host='localhost',
mom_xmlrpc_port=randint(2345, 4567)) as handler:
# create a tmp job receiver queue
with TemporaryQueue(
exchange=self.tmp_exchange.address) as tmp_job_queue:
with tmp_job_queue.create_frombus() as job_receiver:
# create a job...
job_xml = createJobXml('project', 0, 1, 'dp_id', 2,
'/tmp/path/to/dataproduct')
# and let it be handled by the handler (as if it was received via xml-rpc)
handler.onXmlRPCJobReceived('my_job_file.xml', job_xml)
# there should now be a job on the tmp_job_queue
# receive and check it.
job_msg = job_receiver.receive()
self.assertEqual(job_xml, job_msg.content)