def get_smi_and_parse(site_catalog_file, settings, logger):
'''Function to get the smi files from sftp server
The smi files are picked up according to the details in the site-catalog.xml
'''
dikt = gsm_lib.get_site_details_as_dict(site_catalog_file, 'data_source')
site_uri = dikt['site_URI']
host, port = gsm_lib.parse_host_and_port(site_uri)
site_uname = dikt['site_uname']
site_password = dikt['site_password']
site_remotepath = dikt['site_remotepath']
site_key_path = dikt['site_key_path']
site_contact_email = dikt['site_contact_email']
file_name = os.path.basename(site_remotepath)
site_localpath = os.path.join(configuration_directory, file_name)
# Pick up the smi file from the server and place it in the proj_root
logger.info('Retrieving file: %s from %s:%s', site_remotepath, host, port)
logger.info('Errors will be reported to: ' + site_contact_email)
email_props_sftp = EmailProps(
settings.smtp_host,
settings.smtp_port,
settings.sender_email,
[site_contact_email],
[],
'Research Subject Mapper Notification')
sftp_instance = SFTPClient(
host,
port,
site_uname,
site_password,
site_key_path)
sftp_instance.get_file_from_uri(site_remotepath, site_localpath, email_props_sftp)
return site_localpath
def parse_site_details_and_send(site_catalog_file, local_file_path, action, settings, logger):
if not os.path.exists(local_file_path):
logging.error("subject map file " + local_file_path + " file not found")
raise IOError("subject map file " + local_file_path + " file not found")
dikt = gsm_lib.get_site_details_as_dict(site_catalog_file, 'data_destination')
site_uri = dikt['site_URI']
host, port = gsm_lib.parse_host_and_port(site_uri)
site_uname = dikt['site_uname']
site_password = dikt['site_password']
site_remotepath = dikt['site_remotepath']
site_key_path = dikt['site_key_path']
site_contact_email = dikt['site_contact_email']
sender_email = settings.sender_email
# is it a file transfer or attachment email?
if action == 'sftp':
# Pick up the subject_map.csv and put it in the specified sftp server's remote path
info = '\nSending file: [ %s] to host %s:%s %s' % (local_file_path, host, port, site_remotepath)
logger.info(info)
info = 'Errors will be reported to: ' + site_contact_email
logger.info(info)
# put the subject map csv file
remote_directory = os.path.dirname(site_remotepath)
remote_file_name = os.path.basename(site_remotepath)
sftp_instance = SFTPClient(host,
port,
site_uname,
site_password,
site_key_path)
email_props_sftp = EmailProps(
settings.smtp_host,
settings.smtp_port,
settings.sender_email,
[site_contact_email],
[],
'Research Subject Mapper Notification')
sftp_instance.send_file_to_uri(
remote_directory,
remote_file_name,
local_file_path,
email_props_sftp)
elif action == 'email':
# Send the subject_map_exceptions.csv to the contact email address as an attachment
info = '\nSending file: [ %s ] as email attachement to: %s' % (local_file_path, site_contact_email)
logger.info(info)
mail_body = 'Hi, \n this mail contains attached exceptions.csv file.'
props = EmailProps(
settings.smtp_host,
settings.smtp_port,
settings.sender_email,
[site_contact_email],
[],
'Research Subject Mapper Notification',
mail_body,
[local_file_path])
EmailSender().send(props)
else:
info = 'Invalid option. Either sftp/email should be used.'
logger.warn(info)
return
def parse_site_details_and_send(site_catalog_file, subject_map_input, logger,
settings, keep_files):
"""
Parses the site details from site catalog
Note: uses files generated by `write_element_tree_to_file` in the main
function.
"""
try:
site_data = etree.parse(site_catalog_file)
except IOError:
logger.exception(
"Could not open Site Catalog file: '%s'. "
"Check 'site_catalog' in your settings file.", site_catalog_file)
raise
logger.info(
"Processing site details and uploading subject map input files")
site_num = len(site_data.findall(".//site"))
logger.debug("%s total subject site entries read into tree.", site_num)
for site in site_data.iter('site'):
site_code = site.findtext('site_code')
logger.debug("Processing site '%s'.", site_code)
if site_code not in subject_map_input.keys():
logger.warning(
'Site code "%s" defined in Site Catalog, but no '
'records found for that site.', site_code)
continue
host, port = gsm_lib.parse_host_and_port(site.findtext('site_URI'))
logger.debug("Server: %s:%s", host, port)
site_uname = site.findtext('site_uname')
logger.debug("Username: %s", site_uname)
site_key_path = site.findtext('site_key_path')
site_password = site.findtext('site_password')
site_contact_email = site.findtext('site_contact_email')
logger.debug("Site Contact Email: %s", site_contact_email)
# Pick up the correct smi file with the code and place in the
# destination as smi.xml at the specified remote path
site_remotepath = site.findtext('site_remotepath')
site_localpath = subject_map_input[site_code]
logger.info('Sending %s to %s: %s', site_localpath, host,
site_remotepath)
logger.debug('Any errors will be emailed to ' + site_contact_email)
email_props_sftp = EmailProps(settings.smtp_host, settings.smtp_port,
settings.sender_email,
[site_contact_email], [],
'Research Subject Mapper Notification')
sftp_instance = SFTPClient(host,
port,
site_uname,
site_password,
private_key=site_key_path)
sftp_instance.send_file_to_uri(site_remotepath, 'smi.xml',
site_localpath, email_props_sftp)
if keep_files:
logger.debug('Keeping the temporary file: ' + site_localpath)
else:
# remove the smi.xml from the folder
logger.debug('Removing the temporary file: ' + site_localpath)
try:
os.remove(site_localpath)
except OSError as error:
logger.warning(error)
