def get_smi_and_parse(site_catalog_file):
    """Function to get the smi files from sftp server
    The smi files are picked up according to the details in the site-catalog.xml
    """
    if not os.path.exists(site_catalog_file):
        raise GSMLogger().LogException(
            "Error: site_catalog xml file not found at \
            file not found at "
            + site_catalog_file
        )
    else:
        catalog = open(site_catalog_file, "r")
    site_data = etree.parse(site_catalog_file)
    site_num = len(site_data.findall(".//site"))
    gsmlogger.logger.info(str(site_num) + " total subject site entries read into tree.")
    """The reference site code is the current site on which
    generate_subject_map.py is running.
    As we need to get the only smi from the sftp to this site.

    """
    for site in site_data.iter("site"):
        site_name = site.findtext("site_name")
        if site_name == "source":
            host, port = gsm_lib.parse_host_and_port(site.findtext("site_URI"))
            site_uname = site.findtext("site_uname")
            site_password = site.findtext("site_password")
            site_key_path = site.findtext("site_key_path")
            site_contact_email = site.findtext("site_contact_email")
            """Pick up the smi file from the server and place it in the proj_root

            """
            site_remotepath = site.findtext("site_remotepath")
            file_name = site_remotepath.split("/")[-1]
            site_localpath = configuration_directory + file_name

            info = "Retrieving file: " + site_remotepath + " from " + host
            print info
            gsmlogger.logger.info(info)

            info = "Errors will be reported to: " + site_contact_email
            print info
            gsmlogger.logger.info(info)

            sftp_instance = SFTPClient(host, port, site_uname, site_password, site_key_path)
            sftp_instance.get_file_from_uri(site_remotepath, site_localpath, site_contact_email)
    catalog.close()
    gsmlogger.logger.info("site catalog XML file closed.")
    return site_localpath
Example #2
0
def get_smi_and_parse(site_catalog_file, settings, logger):
    '''Function to get the smi files from sftp server
    The smi files are picked up according to the details in the site-catalog.xml
    '''
    dikt = gsm_lib.get_site_details_as_dict(site_catalog_file, 'data_source')
    site_uri = dikt['site_URI']
    host, port = gsm_lib.parse_host_and_port(site_uri)

    site_uname = dikt['site_uname']
    site_password = dikt['site_password']
    site_remotepath = dikt['site_remotepath']
    site_key_path = dikt['site_key_path']
    site_contact_email = dikt['site_contact_email']

    file_name = os.path.basename(site_remotepath)
    site_localpath = os.path.join(configuration_directory, file_name)

    # Pick up the smi file from the server and place it in the proj_root
    logger.info('Retrieving file: %s from %s:%s', site_remotepath, host, port)
    logger.info('Errors will be reported to: ' + site_contact_email)

    email_props_sftp = EmailProps(
        settings.smtp_host,
        settings.smtp_port,
        settings.sender_email,
        [site_contact_email],
        [],
        'Research Subject Mapper Notification')

    sftp_instance = SFTPClient(
        host,
        port,
        site_uname,
        site_password,
        site_key_path)

    sftp_instance.get_file_from_uri(site_remotepath, site_localpath, email_props_sftp)
    return site_localpath
def parse_site_details_and_send(site_catalog_file, subject_map_input, logger, settings, keep_files):
    """
    Parses the site details from site catalog

    Note: uses files generated by `write_element_tree_to_file` in the main
    function.
    """
    try:
        site_data = etree.parse(site_catalog_file)
    except IOError:
        logger.exception("Could not open Site Catalog file: '%s'. "
                         "Check 'site_catalog' in your settings file.",
                         site_catalog_file)
        raise

    logger.info("Processing site details and uploading subject map input files")
    site_num = len(site_data.findall(".//site"))
    logger.debug("%s total subject site entries read into tree.", site_num)

    for site in site_data.iter('site'):
        site_code = site.findtext('site_code')
        logger.debug("Processing site '%s'.", site_code)
        if site_code not in subject_map_input.keys():
            logger.warning('Site code "%s" defined in Site Catalog, but no '
                           'records found for that site.', site_code)
            continue

        host, port = gsm_lib.parse_host_and_port(site.findtext('site_URI'))
        logger.debug("Server: %s:%s", host, port)
        site_uname = site.findtext('site_uname')
        logger.debug("Username: %s", site_uname)
        site_key_path = site.findtext('site_key_path')
        site_password = site.findtext('site_password')
        site_contact_email = site.findtext('site_contact_email')
        logger.debug("Site Contact Email: %s", site_contact_email)

        # Pick up the correct smi file with the code and place in the
        # destination as smi.xml at the specified remote path
        site_remotepath = site.findtext('site_remotepath')
        site_localpath = subject_map_input[site_code]

        logger.info('Sending %s to %s: %s', site_localpath, host, site_remotepath)
        logger.debug('Any errors will be emailed to '+site_contact_email)

        email_props_sftp = EmailProps(
            settings.smtp_host,
            settings.smtp_port,
            settings.sender_email,
            [site_contact_email],
            [],
            'Research Subject Mapper Notification')

        sftp_instance = SFTPClient(
                host,
                port,
                site_uname,
                site_password,
                private_key = site_key_path)

        sftp_instance.send_file_to_uri(
                site_remotepath,
                'smi.xml',
                site_localpath,
                email_props_sftp)

        if keep_files:
            logger.debug('Keeping the temporary file: ' + site_localpath)
        else:
            # remove the smi.xml from the folder
            logger.debug('Removing the temporary file: ' + site_localpath)
            try:
                os.remove(site_localpath)
            except OSError as error:
                logger.warning(error)
Example #4
0
def parse_site_details_and_send(site_catalog_file, local_file_path, action, settings, logger):
    if not os.path.exists(local_file_path):
        logging.error("subject map file " + local_file_path + " file not found")
        raise IOError("subject map file " + local_file_path + " file not found")

    dikt = gsm_lib.get_site_details_as_dict(site_catalog_file, 'data_destination')
    site_uri = dikt['site_URI']
    host, port = gsm_lib.parse_host_and_port(site_uri)

    site_uname = dikt['site_uname']
    site_password = dikt['site_password']
    site_remotepath = dikt['site_remotepath']
    site_key_path = dikt['site_key_path']
    site_contact_email = dikt['site_contact_email']
    sender_email = settings.sender_email


    # is it a file transfer or attachment email?
    if action == 'sftp':
        # Pick up the subject_map.csv and put it in the specified sftp server's remote path
        info = '\nSending file: [ %s] to host %s:%s %s' % (local_file_path, host, port, site_remotepath)
        logger.info(info)

        info = 'Errors will be reported to: ' + site_contact_email
        logger.info(info)

        # put the subject map csv file
        remote_directory = os.path.dirname(site_remotepath)
        remote_file_name = os.path.basename(site_remotepath)

        sftp_instance = SFTPClient(host,
                                   port,
                                   site_uname,
                                   site_password,
                                   site_key_path)

        email_props_sftp = EmailProps(
            settings.smtp_host,
            settings.smtp_port,
            settings.sender_email,
            [site_contact_email],
            [],
            'Research Subject Mapper Notification')

        sftp_instance.send_file_to_uri(
                remote_directory,
                remote_file_name,
                local_file_path,
                email_props_sftp)

    elif action == 'email':
        # Send the subject_map_exceptions.csv to the contact email address as an attachment
        info = '\nSending file: [ %s ] as email attachement to: %s' % (local_file_path, site_contact_email)
        logger.info(info)

        mail_body = 'Hi, \n this mail contains attached exceptions.csv file.'
        props = EmailProps(
            settings.smtp_host,
            settings.smtp_port,
            settings.sender_email,
            [site_contact_email],
            [],
            'Research Subject Mapper Notification',
            mail_body,
            [local_file_path])

        EmailSender().send(props)
    else:
        info = 'Invalid option. Either sftp/email should be used.'
        logger.warn(info)

    return
Example #5
0
def parse_site_details_and_send(site_catalog_file, subject_map_input, logger,
                                settings, keep_files):
    """
    Parses the site details from site catalog

    Note: uses files generated by `write_element_tree_to_file` in the main
    function.
    """
    try:
        site_data = etree.parse(site_catalog_file)
    except IOError:
        logger.exception(
            "Could not open Site Catalog file: '%s'. "
            "Check 'site_catalog' in your settings file.", site_catalog_file)
        raise

    logger.info(
        "Processing site details and uploading subject map input files")
    site_num = len(site_data.findall(".//site"))
    logger.debug("%s total subject site entries read into tree.", site_num)

    for site in site_data.iter('site'):
        site_code = site.findtext('site_code')
        logger.debug("Processing site '%s'.", site_code)
        if site_code not in subject_map_input.keys():
            logger.warning(
                'Site code "%s" defined in Site Catalog, but no '
                'records found for that site.', site_code)
            continue

        host, port = gsm_lib.parse_host_and_port(site.findtext('site_URI'))
        logger.debug("Server: %s:%s", host, port)
        site_uname = site.findtext('site_uname')
        logger.debug("Username: %s", site_uname)
        site_key_path = site.findtext('site_key_path')
        site_password = site.findtext('site_password')
        site_contact_email = site.findtext('site_contact_email')
        logger.debug("Site Contact Email: %s", site_contact_email)

        # Pick up the correct smi file with the code and place in the
        # destination as smi.xml at the specified remote path
        site_remotepath = site.findtext('site_remotepath')
        site_localpath = subject_map_input[site_code]

        logger.info('Sending %s to %s: %s', site_localpath, host,
                    site_remotepath)
        logger.debug('Any errors will be emailed to ' + site_contact_email)

        email_props_sftp = EmailProps(settings.smtp_host, settings.smtp_port,
                                      settings.sender_email,
                                      [site_contact_email], [],
                                      'Research Subject Mapper Notification')

        sftp_instance = SFTPClient(host,
                                   port,
                                   site_uname,
                                   site_password,
                                   private_key=site_key_path)

        sftp_instance.send_file_to_uri(site_remotepath, 'smi.xml',
                                       site_localpath, email_props_sftp)

        if keep_files:
            logger.debug('Keeping the temporary file: ' + site_localpath)
        else:
            # remove the smi.xml from the folder
            logger.debug('Removing the temporary file: ' + site_localpath)
            try:
                os.remove(site_localpath)
            except OSError as error:
                logger.warning(error)
def parse_site_details_and_send(site_catalog_file, smi_filenames, smi_ids, gsmlogger):
    for smi_file_name in smi_filenames:
        if not os.path.exists(smi_file_name):
            raise GSMLogger().LogException("Error: smi file "+smi_file_name+" not found")

    if not os.path.exists(site_catalog_file):
        raise GSMLogger().LogException("Error: site_catalog xml file not found at "
                + site_catalog_file)
        r2
    else:
        catalog = open(site_catalog_file, 'r')

    site_data = etree.parse(site_catalog_file)
    site_num = len(site_data.findall(".//site"))
    gsmlogger.logger.info(str(site_num) + " total subject site entries read into tree.")

    for site in site_data.iter('site'):
        site_code = site.findtext('site_code')
        if site_code in smi_ids:
            host, port = gsm_lib.parse_host_and_port(site.findtext('site_URI'))
            site_uname = site.findtext('site_uname')
            site_key_path = site.findtext('site_key_path')
            site_password = site.findtext('site_password')
            site_contact_email = site.findtext('site_contact_email')

            '''Pick up the correct smi file with the code and place in the destination
            as smi.xml at the specified remote path
            '''
            file_name = 'smi'+site_code+'.xml'
            site_remotepath = site.findtext('site_remotepath')

            site_localpath = tmp_folder + file_name

            info = '\nSending '+site_localpath+' to '+host+':'+site_remotepath
            print info
            gsmlogger.logger.info(info)

            info = 'Any error will be reported to '+site_contact_email
            print info
            gsmlogger.logger.info(info)

            sftp_instance = SFTPClient(host, port, site_uname, site_password,
                                       private_key=site_key_path)
            sftp_instance.send_file_to_uri(site_remotepath, 'smi.xml',
                                           site_localpath, site_contact_email)

            if do_keep_gen_files:
                print ' * Keeping the temporary file: ' + site_localpath
            else :
                # remove the smi.xml from the folder
                try:
                    print ' * Removing the temporary file: ' + site_localpath
                    os.remove(site_localpath)
                except OSError:
                    pass
        else:
            print 'Site code '+site_code+' does not exist'
            gsmlogger.logger.info('Site code does not exist')
    catalog.close()
    gsmlogger.logger.info("site catalog XML file closed.")
    pass
def parse_site_details_and_send(site_catalog_file, local_file_path, action):
    if not os.path.exists(local_file_path):
        raise GSMLogger().LogException("Error: subject map file " + local_file_path + " file not found")
    if not os.path.exists(site_catalog_file):
        raise GSMLogger().LogException(
            "Error: site_catalog xml file not found at \
            file not found at "
            + site_catalog_file
        )
    else:
        catalog = open(site_catalog_file, "r")
    site_data = etree.parse(site_catalog_file)
    site_num = len(site_data.findall(".//site"))
    gsmlogger.logger.info(str(site_num) + " total subject site entries read into tree.")

    for site in site_data.iter("site"):
        site_name = site.findtext("site_name")
        if site_name == "destination":
            host, port = gsm_lib.parse_host_and_port(site.findtext("site_URI"))
            subjectmap_uname = site.findtext("site_uname")
            subjectmap_password = site.findtext("site_password")
            subjectmap_key_path = site.findtext("site_key_path")
            subjectmap_contact_email = site.findtext("site_contact_email")
            # is it a file transfer or attachment email?
            if action == "sftp":
                """Pick up the subject_map.csv and put it in the specified
                sftp server's remote path

                """
                # remote path to send the file to
                subjectmap_remotepath = site.findtext("site_remotepath")
                info = "\nSending file [" + local_file_path + "] to " + host + ":" + subjectmap_remotepath
                print info
                gsmlogger.logger.info(info)

                info = "Errors will be reported to: " + subjectmap_contact_email
                print info
                gsmlogger.logger.info(info)

                # put the subject map csv file
                subjectmap_remote_directory = subjectmap_remotepath.rsplit("/", 1)[0] + "/"
                remote_file_name = subjectmap_remotepath.split("/")[-1]

                sftp_instance = SFTPClient(host, port, subjectmap_uname, subjectmap_password, subjectmap_key_path)
                sftp_instance.send_file_to_uri(
                    subjectmap_remote_directory, remote_file_name, local_file_path, subjectmap_contact_email
                )

            elif action == "email":
                """Send the subject_map_exceptions.csv to the contact
                email address as an attachment

                """
                print "\nSending file: [" + local_file_path + "] as email attachement to " + subjectmap_contact_email
                gsmlogger.logger.info(
                    "Sending %s as email attachement to %s", local_file_path, subjectmap_contact_email
                )
                # TODO change the mail body as required
                mail_body = "Hi, \n this mail contains attached exceptions.csv file."
                email_transactions().send_mail(
                    setup["sender_email"], subjectmap_contact_email, mail_body, [local_file_path]
                )
            else:
                print "Invalid option. either sftp/email should be used"
                gsmlogger.logger.info("Invalid option. either sftp/email should be used")
    catalog.close()
    pass