def get_smi_and_parse(site_catalog_file): """Function to get the smi files from sftp server The smi files are picked up according to the details in the site-catalog.xml """ if not os.path.exists(site_catalog_file): raise GSMLogger().LogException( "Error: site_catalog xml file not found at \ file not found at " + site_catalog_file ) else: catalog = open(site_catalog_file, "r") site_data = etree.parse(site_catalog_file) site_num = len(site_data.findall(".//site")) gsmlogger.logger.info(str(site_num) + " total subject site entries read into tree.") """The reference site code is the current site on which generate_subject_map.py is running. As we need to get the only smi from the sftp to this site. """ for site in site_data.iter("site"): site_name = site.findtext("site_name") if site_name == "source": host, port = gsm_lib.parse_host_and_port(site.findtext("site_URI")) site_uname = site.findtext("site_uname") site_password = site.findtext("site_password") site_key_path = site.findtext("site_key_path") site_contact_email = site.findtext("site_contact_email") """Pick up the smi file from the server and place it in the proj_root """ site_remotepath = site.findtext("site_remotepath") file_name = site_remotepath.split("/")[-1] site_localpath = configuration_directory + file_name info = "Retrieving file: " + site_remotepath + " from " + host print info gsmlogger.logger.info(info) info = "Errors will be reported to: " + site_contact_email print info gsmlogger.logger.info(info) sftp_instance = SFTPClient(host, port, site_uname, site_password, site_key_path) sftp_instance.get_file_from_uri(site_remotepath, site_localpath, site_contact_email) catalog.close() gsmlogger.logger.info("site catalog XML file closed.") return site_localpath
def get_smi_and_parse(site_catalog_file, settings, logger): '''Function to get the smi files from sftp server The smi files are picked up according to the details in the site-catalog.xml ''' dikt = gsm_lib.get_site_details_as_dict(site_catalog_file, 'data_source') site_uri = dikt['site_URI'] host, port = gsm_lib.parse_host_and_port(site_uri) site_uname = dikt['site_uname'] site_password = dikt['site_password'] site_remotepath = dikt['site_remotepath'] site_key_path = dikt['site_key_path'] site_contact_email = dikt['site_contact_email'] file_name = os.path.basename(site_remotepath) site_localpath = os.path.join(configuration_directory, file_name) # Pick up the smi file from the server and place it in the proj_root logger.info('Retrieving file: %s from %s:%s', site_remotepath, host, port) logger.info('Errors will be reported to: ' + site_contact_email) email_props_sftp = EmailProps( settings.smtp_host, settings.smtp_port, settings.sender_email, [site_contact_email], [], 'Research Subject Mapper Notification') sftp_instance = SFTPClient( host, port, site_uname, site_password, site_key_path) sftp_instance.get_file_from_uri(site_remotepath, site_localpath, email_props_sftp) return site_localpath
def parse_site_details_and_send(site_catalog_file, subject_map_input, logger, settings, keep_files): """ Parses the site details from site catalog Note: uses files generated by `write_element_tree_to_file` in the main function. """ try: site_data = etree.parse(site_catalog_file) except IOError: logger.exception("Could not open Site Catalog file: '%s'. " "Check 'site_catalog' in your settings file.", site_catalog_file) raise logger.info("Processing site details and uploading subject map input files") site_num = len(site_data.findall(".//site")) logger.debug("%s total subject site entries read into tree.", site_num) for site in site_data.iter('site'): site_code = site.findtext('site_code') logger.debug("Processing site '%s'.", site_code) if site_code not in subject_map_input.keys(): logger.warning('Site code "%s" defined in Site Catalog, but no ' 'records found for that site.', site_code) continue host, port = gsm_lib.parse_host_and_port(site.findtext('site_URI')) logger.debug("Server: %s:%s", host, port) site_uname = site.findtext('site_uname') logger.debug("Username: %s", site_uname) site_key_path = site.findtext('site_key_path') site_password = site.findtext('site_password') site_contact_email = site.findtext('site_contact_email') logger.debug("Site Contact Email: %s", site_contact_email) # Pick up the correct smi file with the code and place in the # destination as smi.xml at the specified remote path site_remotepath = site.findtext('site_remotepath') site_localpath = subject_map_input[site_code] logger.info('Sending %s to %s: %s', site_localpath, host, site_remotepath) logger.debug('Any errors will be emailed to '+site_contact_email) email_props_sftp = EmailProps( settings.smtp_host, settings.smtp_port, settings.sender_email, [site_contact_email], [], 'Research Subject Mapper Notification') sftp_instance = SFTPClient( host, port, site_uname, site_password, private_key = site_key_path) sftp_instance.send_file_to_uri( site_remotepath, 'smi.xml', site_localpath, email_props_sftp) if keep_files: logger.debug('Keeping the temporary file: ' + site_localpath) else: # remove the smi.xml from the folder logger.debug('Removing the temporary file: ' + site_localpath) try: os.remove(site_localpath) except OSError as error: logger.warning(error)
def parse_site_details_and_send(site_catalog_file, local_file_path, action, settings, logger): if not os.path.exists(local_file_path): logging.error("subject map file " + local_file_path + " file not found") raise IOError("subject map file " + local_file_path + " file not found") dikt = gsm_lib.get_site_details_as_dict(site_catalog_file, 'data_destination') site_uri = dikt['site_URI'] host, port = gsm_lib.parse_host_and_port(site_uri) site_uname = dikt['site_uname'] site_password = dikt['site_password'] site_remotepath = dikt['site_remotepath'] site_key_path = dikt['site_key_path'] site_contact_email = dikt['site_contact_email'] sender_email = settings.sender_email # is it a file transfer or attachment email? if action == 'sftp': # Pick up the subject_map.csv and put it in the specified sftp server's remote path info = '\nSending file: [ %s] to host %s:%s %s' % (local_file_path, host, port, site_remotepath) logger.info(info) info = 'Errors will be reported to: ' + site_contact_email logger.info(info) # put the subject map csv file remote_directory = os.path.dirname(site_remotepath) remote_file_name = os.path.basename(site_remotepath) sftp_instance = SFTPClient(host, port, site_uname, site_password, site_key_path) email_props_sftp = EmailProps( settings.smtp_host, settings.smtp_port, settings.sender_email, [site_contact_email], [], 'Research Subject Mapper Notification') sftp_instance.send_file_to_uri( remote_directory, remote_file_name, local_file_path, email_props_sftp) elif action == 'email': # Send the subject_map_exceptions.csv to the contact email address as an attachment info = '\nSending file: [ %s ] as email attachement to: %s' % (local_file_path, site_contact_email) logger.info(info) mail_body = 'Hi, \n this mail contains attached exceptions.csv file.' props = EmailProps( settings.smtp_host, settings.smtp_port, settings.sender_email, [site_contact_email], [], 'Research Subject Mapper Notification', mail_body, [local_file_path]) EmailSender().send(props) else: info = 'Invalid option. Either sftp/email should be used.' logger.warn(info) return
def parse_site_details_and_send(site_catalog_file, subject_map_input, logger, settings, keep_files): """ Parses the site details from site catalog Note: uses files generated by `write_element_tree_to_file` in the main function. """ try: site_data = etree.parse(site_catalog_file) except IOError: logger.exception( "Could not open Site Catalog file: '%s'. " "Check 'site_catalog' in your settings file.", site_catalog_file) raise logger.info( "Processing site details and uploading subject map input files") site_num = len(site_data.findall(".//site")) logger.debug("%s total subject site entries read into tree.", site_num) for site in site_data.iter('site'): site_code = site.findtext('site_code') logger.debug("Processing site '%s'.", site_code) if site_code not in subject_map_input.keys(): logger.warning( 'Site code "%s" defined in Site Catalog, but no ' 'records found for that site.', site_code) continue host, port = gsm_lib.parse_host_and_port(site.findtext('site_URI')) logger.debug("Server: %s:%s", host, port) site_uname = site.findtext('site_uname') logger.debug("Username: %s", site_uname) site_key_path = site.findtext('site_key_path') site_password = site.findtext('site_password') site_contact_email = site.findtext('site_contact_email') logger.debug("Site Contact Email: %s", site_contact_email) # Pick up the correct smi file with the code and place in the # destination as smi.xml at the specified remote path site_remotepath = site.findtext('site_remotepath') site_localpath = subject_map_input[site_code] logger.info('Sending %s to %s: %s', site_localpath, host, site_remotepath) logger.debug('Any errors will be emailed to ' + site_contact_email) email_props_sftp = EmailProps(settings.smtp_host, settings.smtp_port, settings.sender_email, [site_contact_email], [], 'Research Subject Mapper Notification') sftp_instance = SFTPClient(host, port, site_uname, site_password, private_key=site_key_path) sftp_instance.send_file_to_uri(site_remotepath, 'smi.xml', site_localpath, email_props_sftp) if keep_files: logger.debug('Keeping the temporary file: ' + site_localpath) else: # remove the smi.xml from the folder logger.debug('Removing the temporary file: ' + site_localpath) try: os.remove(site_localpath) except OSError as error: logger.warning(error)
def parse_site_details_and_send(site_catalog_file, smi_filenames, smi_ids, gsmlogger): for smi_file_name in smi_filenames: if not os.path.exists(smi_file_name): raise GSMLogger().LogException("Error: smi file "+smi_file_name+" not found") if not os.path.exists(site_catalog_file): raise GSMLogger().LogException("Error: site_catalog xml file not found at " + site_catalog_file) r2 else: catalog = open(site_catalog_file, 'r') site_data = etree.parse(site_catalog_file) site_num = len(site_data.findall(".//site")) gsmlogger.logger.info(str(site_num) + " total subject site entries read into tree.") for site in site_data.iter('site'): site_code = site.findtext('site_code') if site_code in smi_ids: host, port = gsm_lib.parse_host_and_port(site.findtext('site_URI')) site_uname = site.findtext('site_uname') site_key_path = site.findtext('site_key_path') site_password = site.findtext('site_password') site_contact_email = site.findtext('site_contact_email') '''Pick up the correct smi file with the code and place in the destination as smi.xml at the specified remote path ''' file_name = 'smi'+site_code+'.xml' site_remotepath = site.findtext('site_remotepath') site_localpath = tmp_folder + file_name info = '\nSending '+site_localpath+' to '+host+':'+site_remotepath print info gsmlogger.logger.info(info) info = 'Any error will be reported to '+site_contact_email print info gsmlogger.logger.info(info) sftp_instance = SFTPClient(host, port, site_uname, site_password, private_key=site_key_path) sftp_instance.send_file_to_uri(site_remotepath, 'smi.xml', site_localpath, site_contact_email) if do_keep_gen_files: print ' * Keeping the temporary file: ' + site_localpath else : # remove the smi.xml from the folder try: print ' * Removing the temporary file: ' + site_localpath os.remove(site_localpath) except OSError: pass else: print 'Site code '+site_code+' does not exist' gsmlogger.logger.info('Site code does not exist') catalog.close() gsmlogger.logger.info("site catalog XML file closed.") pass
def parse_site_details_and_send(site_catalog_file, local_file_path, action): if not os.path.exists(local_file_path): raise GSMLogger().LogException("Error: subject map file " + local_file_path + " file not found") if not os.path.exists(site_catalog_file): raise GSMLogger().LogException( "Error: site_catalog xml file not found at \ file not found at " + site_catalog_file ) else: catalog = open(site_catalog_file, "r") site_data = etree.parse(site_catalog_file) site_num = len(site_data.findall(".//site")) gsmlogger.logger.info(str(site_num) + " total subject site entries read into tree.") for site in site_data.iter("site"): site_name = site.findtext("site_name") if site_name == "destination": host, port = gsm_lib.parse_host_and_port(site.findtext("site_URI")) subjectmap_uname = site.findtext("site_uname") subjectmap_password = site.findtext("site_password") subjectmap_key_path = site.findtext("site_key_path") subjectmap_contact_email = site.findtext("site_contact_email") # is it a file transfer or attachment email? if action == "sftp": """Pick up the subject_map.csv and put it in the specified sftp server's remote path """ # remote path to send the file to subjectmap_remotepath = site.findtext("site_remotepath") info = "\nSending file [" + local_file_path + "] to " + host + ":" + subjectmap_remotepath print info gsmlogger.logger.info(info) info = "Errors will be reported to: " + subjectmap_contact_email print info gsmlogger.logger.info(info) # put the subject map csv file subjectmap_remote_directory = subjectmap_remotepath.rsplit("/", 1)[0] + "/" remote_file_name = subjectmap_remotepath.split("/")[-1] sftp_instance = SFTPClient(host, port, subjectmap_uname, subjectmap_password, subjectmap_key_path) sftp_instance.send_file_to_uri( subjectmap_remote_directory, remote_file_name, local_file_path, subjectmap_contact_email ) elif action == "email": """Send the subject_map_exceptions.csv to the contact email address as an attachment """ print "\nSending file: [" + local_file_path + "] as email attachement to " + subjectmap_contact_email gsmlogger.logger.info( "Sending %s as email attachement to %s", local_file_path, subjectmap_contact_email ) # TODO change the mail body as required mail_body = "Hi, \n this mail contains attached exceptions.csv file." email_transactions().send_mail( setup["sender_email"], subjectmap_contact_email, mail_body, [local_file_path] ) else: print "Invalid option. either sftp/email should be used" gsmlogger.logger.info("Invalid option. either sftp/email should be used") catalog.close() pass