def get_smi_and_parse(site_catalog_file, settings, logger): '''Function to get the smi files from sftp server The smi files are picked up according to the details in the site-catalog.xml ''' dikt = gsm_lib.get_site_details_as_dict(site_catalog_file, 'data_source') site_uri = dikt['site_URI'] host, port = gsm_lib.parse_host_and_port(site_uri) site_uname = dikt['site_uname'] site_password = dikt['site_password'] site_remotepath = dikt['site_remotepath'] site_key_path = dikt['site_key_path'] site_contact_email = dikt['site_contact_email'] file_name = os.path.basename(site_remotepath) site_localpath = os.path.join(configuration_directory, file_name) # Pick up the smi file from the server and place it in the proj_root logger.info('Retrieving file: %s from %s:%s', site_remotepath, host, port) logger.info('Errors will be reported to: ' + site_contact_email) email_props_sftp = EmailProps( settings.smtp_host, settings.smtp_port, settings.sender_email, [site_contact_email], [], 'Research Subject Mapper Notification') sftp_instance = SFTPClient( host, port, site_uname, site_password, site_key_path) sftp_instance.get_file_from_uri(site_remotepath, site_localpath, email_props_sftp) return site_localpath
def test_get_site_details_as_dict(self): xml = """<?xml version='1.0' encoding='UTF-8'?> <sites_list> <site type="data_source"> <site_URI> source_sftp </site_URI> <site_uname> source_tester </site_uname> <site_password> source_pass </site_password> <site_remotepath> smi.xml </site_remotepath> <site_contact_email>[email protected] </site_contact_email> <site_key_path> </site_key_path> </site> <site type="data_destination"> <site_URI> dest_sftp </site_URI> <site_uname> dest_tester </site_uname> <site_password> dest_pass </site_password> <site_remotepath> smi.xml </site_remotepath> <site_contact_email>[email protected] </site_contact_email> <site_key_path> path </site_key_path> </site> </sites_list> """ try: pair = tempfile.mkstemp() file_path = pair[1] f = open(file_path, 'r+') f.write(xml) f.close() except: print('Unable to create temp_file: ' + file_path) good_dikt_source = { 'site_URI': 'source_sftp', 'site_uname': 'source_tester', 'site_password': '******', 'site_remotepath': 'smi.xml', 'site_contact_email': '*****@*****.**', 'site_key_path': '', } good_dikt_dest = { 'site_URI': 'dest_sftp', 'site_uname': 'dest_tester', 'site_password': '******', 'site_remotepath': 'smi.xml', 'site_contact_email': '*****@*****.**', 'site_key_path': 'path', } # Test the actual function dikt_source = gsm_lib.get_site_details_as_dict(file_path, 'data_source') dikt_dest = gsm_lib.get_site_details_as_dict(file_path, 'data_destination') #pprint.pprint(dikt) self.assertEqual(good_dikt_source, dikt_source) self.assertEqual(good_dikt_dest, dikt_dest) try: os.unlink(file_path) except: print('Unable to remove temp_file: ' + file_path)
def test_get_site_details_as_dict(self): xml = """<?xml version='1.0' encoding='UTF-8'?> <sites_list> <site type="data_source"> <site_URI> source_sftp </site_URI> <site_uname> source_tester </site_uname> <site_password> source_pass </site_password> <site_remotepath> smi.xml </site_remotepath> <site_contact_email>[email protected] </site_contact_email> <site_key_path> </site_key_path> </site> <site type="data_destination"> <site_URI> dest_sftp </site_URI> <site_uname> dest_tester </site_uname> <site_password> dest_pass </site_password> <site_remotepath> smi.xml </site_remotepath> <site_contact_email>[email protected] </site_contact_email> <site_key_path> path </site_key_path> </site> </sites_list> """ try : pair = tempfile.mkstemp() file_path = pair[1] f = open(file_path, 'r+') f.write(xml) f.close() except : print('Unable to create temp_file: ' + file_path) good_dikt_source = { 'site_URI' : 'source_sftp', 'site_uname' : 'source_tester', 'site_password' : 'source_pass', 'site_remotepath' : 'smi.xml', 'site_contact_email' : '*****@*****.**', 'site_key_path' : '', } good_dikt_dest = { 'site_URI' : 'dest_sftp', 'site_uname' : 'dest_tester', 'site_password' : 'dest_pass', 'site_remotepath' : 'smi.xml', 'site_contact_email' : '*****@*****.**', 'site_key_path' : 'path', } # Test the actual function dikt_source = gsm_lib.get_site_details_as_dict(file_path, 'data_source') dikt_dest = gsm_lib.get_site_details_as_dict(file_path, 'data_destination') #pprint.pprint(dikt) self.assertEqual(good_dikt_source, dikt_source) self.assertEqual(good_dikt_dest, dikt_dest) try : os.unlink(file_path) except : print('Unable to remove temp_file: ' + file_path)
def parse_site_details_and_send(site_catalog_file, local_file_path, action, settings, logger): if not os.path.exists(local_file_path): logging.error("subject map file " + local_file_path + " file not found") raise IOError("subject map file " + local_file_path + " file not found") dikt = gsm_lib.get_site_details_as_dict(site_catalog_file, 'data_destination') site_uri = dikt['site_URI'] host, port = gsm_lib.parse_host_and_port(site_uri) site_uname = dikt['site_uname'] site_password = dikt['site_password'] site_remotepath = dikt['site_remotepath'] site_key_path = dikt['site_key_path'] site_contact_email = dikt['site_contact_email'] sender_email = settings.sender_email # is it a file transfer or attachment email? if action == 'sftp': # Pick up the subject_map.csv and put it in the specified sftp server's remote path info = '\nSending file: [ %s] to host %s:%s %s' % (local_file_path, host, port, site_remotepath) logger.info(info) info = 'Errors will be reported to: ' + site_contact_email logger.info(info) # put the subject map csv file remote_directory = os.path.dirname(site_remotepath) remote_file_name = os.path.basename(site_remotepath) sftp_instance = SFTPClient(host, port, site_uname, site_password, site_key_path) email_props_sftp = EmailProps( settings.smtp_host, settings.smtp_port, settings.sender_email, [site_contact_email], [], 'Research Subject Mapper Notification') sftp_instance.send_file_to_uri( remote_directory, remote_file_name, local_file_path, email_props_sftp) elif action == 'email': # Send the subject_map_exceptions.csv to the contact email address as an attachment info = '\nSending file: [ %s ] as email attachement to: %s' % (local_file_path, site_contact_email) logger.info(info) mail_body = 'Hi, \n this mail contains attached exceptions.csv file.' props = EmailProps( settings.smtp_host, settings.smtp_port, settings.sender_email, [site_contact_email], [], 'Research Subject Mapper Notification', mail_body, [local_file_path]) EmailSender().send(props) else: info = 'Invalid option. Either sftp/email should be used.' logger.warn(info) return