def test_mom_soap_to_job_queue(self): """assuming test_onXmlRPCJobReceived_no_soap passes, test the correct behaviour when called via soap xml-rpc""" from lofar.lta.ingest.server.ingestmomadapter import MoMXMLRPCHandler with MoMXMLRPCHandler(exchange=self.tmp_exchange.address, mom_xmlrpc_host='localhost', mom_xmlrpc_port=randint(2345, 4567)) as handler: # create a tmp job receiver queue with TemporaryQueue( exchange=self.tmp_exchange.address) as tmp_job_queue: with tmp_job_queue.create_frombus() as job_receiver: # create a job... job_xml = createJobXml('project', 0, 1, 'dp_id', 2, '/tmp/path/to/dataproduct') # submit the job like MoM would via xml-rpc soap_client = SoapClient(location=handler.server_url(), namespace="urn:pipeline.export") soap_client.new_job(fileName='my_job_file.xml', fileContent=job_xml) # there should now be a job on the tmp_job_queue # receive and check it. job_msg = job_receiver.receive() self.assertEqual(job_xml, job_msg.content)
def test_h5_plus_raw_file(self): #beam formed h5 files are always accompanied by a raw file #these should be tarred togheter try: project_name = 'test-project' obs_id = 987654321 dpname = 'L%s_SAP000_SB000_bf.h5' % obs_id rawname = dpname.replace('.h5', '.raw') self.test_dir_path = os.path.join(os.getcwd(), 'testdir_%s' % uuid.uuid1()) def stub_GetStorageTicket(project, filename, filesize, archive_id, job_id, obs_id, check_mom_id=True, id_source='MoM'): return { 'primary_uri_rnd': 'srm://some.site.name:8443/some/path/data/lofar/ops/projects/%s/%s/%s.tar' % (project, obs_id, dpname), 'result': 'ok', 'error': '', 'ticket': '3E0A47ED860D6339E053B316A9C3BEE2'} ltamock.GetStorageTicket.side_effect = stub_GetStorageTicket def stub_uploadDataAndGetSIP(archive_id, storage_ticket, filename, uri, filesize, md5_checksum, adler32_checksum, validate=True): #return unpecified sip with proper details from lofar.lta.ingest.server.unspecifiedSIP import makeSIP return makeSIP(project_name, obs_id, archive_id, storage_ticket, filename, filesize, md5_checksum, adler32_checksum, 'TEST') mommock.uploadDataAndGetSIP.side_effect = stub_uploadDataAndGetSIP os.makedirs(self.test_dir_path) test_file_path = os.path.join(self.test_dir_path, dpname) with open(test_file_path, 'w') as file: file.write(4096*'a') raw_test_file_path = os.path.join(self.test_dir_path, dpname.replace('.h5', '.raw')) with open(raw_test_file_path, 'w') as file: file.write(4096*'b') job_xml = createJobXml(testname, 123456789, obs_id, dpname, 918273645, 'localhost:%s' % test_file_path) logger.info('job xml: %s', job_xml) job = parseJobXml(job_xml) pl = IngestPipeline(job, self.momclient, self.ltaclient, exchange=self.tmp_exchange.address) pl.run() except Exception as e: self.assertTrue(False, 'Unexpected exception in pipeline: %s' % e) finally: # the 'stub-transfered' file ended up in out local stub lta # with the path: ltastubs._local_globus_file_path #check extension self.assertEqual('.tar', os.path.splitext(ltastubs._local_globus_file_path)[-1]) #check tar contents tar = subprocess.Popen(['tar', '--list', '-f', ltastubs._local_globus_file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) tar_file_list, err = tuple(x.decode('ascii') for x in tar.communicate()) self.assertEqual(tar.returncode, 0) logger.info('file list in tar:\n%s', tar_file_list) self.assertTrue(os.path.basename(test_file_path) in tar_file_list) self.assertTrue(os.path.basename(raw_test_file_path) in tar_file_list) logger.info('all expected source files are in tar!') os.remove(test_file_path) os.remove(raw_test_file_path) os.removedirs(self.test_dir_path)
def test_directory_with_odd_dataproduct_name(self): #sometimes somebody has data in a odd directory #and gives the dataproduct a different name than it's directory try: project_name = 'test-project' obs_id = 987654321 dpname = 'my_funky_dp_name' self.test_dir_path = os.path.join(os.getcwd(), 'testdir_%s' % uuid.uuid1(), 'my_data_dir') def stub_uploadDataAndGetSIP(archive_id, storage_ticket, filename, uri, filesize, md5_checksum, adler32_checksum, validate=True): #return unpecified sip with proper details from lofar.lta.ingest.server.unspecifiedSIP import makeSIP return makeSIP(project_name, obs_id, archive_id, storage_ticket, filename, filesize, md5_checksum, adler32_checksum, 'TEST') mommock.uploadDataAndGetSIP.side_effect = stub_uploadDataAndGetSIP os.makedirs(self.test_dir_path) test_file_paths = [] for i in range(10): test_file_path = os.path.join(self.test_dir_path, 'testfile_%s.txt' % i) test_file_paths.append(test_file_path) with open(test_file_path, 'w') as file: file.write(1000*'a') job_xml = createJobXml(testname, 123456789, obs_id, dpname, 918273645, 'localhost:%s' % self.test_dir_path) logger.info('job xml: %s', job_xml) job = parseJobXml(job_xml) pl = IngestPipeline(job, self.momclient, self.ltaclient, exchange=self.tmp_exchange.address) pl.run() except Exception as e: self.assertTrue(False, 'Unexpected exception in pipeline: %s' % e) finally: # the 'stub-transfered' file ended up in out local stub lta # with the path: ltastubs._local_globus_file_path #check extension self.assertTrue('.tar' == os.path.splitext(ltastubs._local_globus_file_path)[-1]) #check tar contents tar = subprocess.Popen(['tar', '--list', '-f', ltastubs._local_globus_file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) tar_file_list, err = tuple(x.decode('ascii') for x in tar.communicate()) self.assertEqual(tar.returncode, 0) logger.info('file list in tar:\n%s', tar_file_list) for test_file_path in test_file_paths: self.assertTrue(os.path.basename(test_file_path) in tar_file_list) logger.info('all expected source files are in tar!') for f in os.listdir(self.test_dir_path): os.remove(os.path.join(self.test_dir_path, f)) os.removedirs(self.test_dir_path)
def test_single_file(self): try: project_name = 'test-project' obs_id = 987654321 dpname = 'L%s_SAP000_SB000_im.h5' % obs_id self.test_dir_path = os.path.join(os.getcwd(), 'testdir_%s' % uuid.uuid1()) def stub_GetStorageTicket(project, filename, filesize, archive_id, job_id, obs_id, check_mom_id=True, id_source='MoM'): return { 'primary_uri_rnd': 'srm://some.site.name:8443/some/path/data/lofar/ops/projects/%s/%s/%s' % (project, obs_id, dpname), 'result': 'ok', 'error': '', 'ticket': '3E0A47ED860D6339E053B316A9C3BEE2'} ltamock.GetStorageTicket.side_effect = stub_GetStorageTicket def stub_uploadDataAndGetSIP(archive_id, storage_ticket, filename, uri, filesize, md5_checksum, adler32_checksum, validate=True): #return unpecified sip with proper details from lofar.lta.ingest.server.unspecifiedSIP import makeSIP return makeSIP(project_name, obs_id, archive_id, storage_ticket, filename, filesize, md5_checksum, adler32_checksum, 'TEST') mommock.uploadDataAndGetSIP.side_effect = stub_uploadDataAndGetSIP os.makedirs(self.test_dir_path) test_file_path = os.path.join(self.test_dir_path, dpname) with open(test_file_path, 'w') as file: file.write(4096*'a') job_xml = createJobXml(testname, 123456789, obs_id, dpname, 918273645, 'localhost:%s' % test_file_path) logger.info('job xml: %s', job_xml) job = parseJobXml(job_xml) pl = IngestPipeline(job, self.momclient, self.ltaclient, exchange=self.tmp_exchange.address) pl.run() except Exception as e: self.assertTrue(False, 'Unexpected exception in pipeline: %s' % e) finally: # the 'stub-transfered' file ended up in out local stub lta # with the path: ltastubs._local_globus_file_path #check extension self.assertEqual(os.path.splitext(test_file_path)[-1], os.path.splitext(ltastubs._local_globus_file_path)[-1]) #compare with original with open(test_file_path) as input, open(ltastubs._local_globus_file_path) as output: self.assertEqual(input.read(), output.read()) for f in os.listdir(self.test_dir_path): os.remove(os.path.join(self.test_dir_path, f)) os.removedirs(self.test_dir_path)
def test_onXmlRPCJobReceived_no_soap(self): """test the handler routine onXmlRPCJobReceived to check if a job_xml is converted to a message and send on the correct bus""" from lofar.lta.ingest.server.ingestmomadapter import MoMXMLRPCHandler with MoMXMLRPCHandler(exchange=self.tmp_exchange.address, mom_xmlrpc_host='localhost', mom_xmlrpc_port=randint(2345, 4567)) as handler: # create a tmp job receiver queue with TemporaryQueue( exchange=self.tmp_exchange.address) as tmp_job_queue: with tmp_job_queue.create_frombus() as job_receiver: # create a job... job_xml = createJobXml('project', 0, 1, 'dp_id', 2, '/tmp/path/to/dataproduct') # and let it be handled by the handler (as if it was received via xml-rpc) handler.onXmlRPCJobReceived('my_job_file.xml', job_xml) # there should now be a job on the tmp_job_queue # receive and check it. job_msg = job_receiver.receive() self.assertEqual(job_xml, job_msg.content)