コード例 #1
0
def main():
    # obtaining command line arguments for path to config directory
    args = parse_args()
    configuration_directory = os.path.abspath(args['configuration_directory_path'])

    if 'debug' in args:
        if args['debug']:
            debugging = args['debug']
        else:
            debugging = False
    else:
        debugging = False

    # Configure logging
    logger = configure_logging(args['verbose'], args['logfile'])

    conf_file = os.path.join(configuration_directory, 'settings.ini')
    settings = gsm_lib.get_settings(conf_file)
    gsm_lib.read_config(configuration_directory, conf_file, settings)

    if debugging:
        if settings.redcap_log_file:
            redcap_log_file = os.path.join(configuration_directory, settings.redcap_log_file)
        else:
            redcap_log_file = 'redcap.log.xml'

    # Check if xml_formatting_transform.xsl file is present/properly set in
    # setting.ini
    message2 = "Please set it with appropriate value and restart execution. " \
               "For assistance refer config-example-gsm-input/settings.ini." \
               "\nProgram will now terminate..."
    if not settings.hasoption('xml_formatting_transform_xsl'):
        message = "Required parameter xml_formatting_transform_xsl is missing " \
                  "in settings.ini. " + message2
        logger.error(message)
        raise gsm_lib.ConfigurationError(message)
    elif settings.xml_formatting_transform_xsl == "":
        message = "Required parameter xml_formatting_transform_xsl does not " \
                  "have a value in settings.ini. " + message2
        logger.error(message)
        raise gsm_lib.ConfigurationError(message)
    elif not os.path.exists(os.path.join(configuration_directory, settings.xml_formatting_transform_xsl)):
        message = "Required file xml_formatting_transform.xsl does not exist " \
                  "in {0}. Please make sure this file is included in the " \
                  "configuration directory and restart execution. For " \
                  "assistance refer config-example-gsm-input/xml_formatting_transform.xsl." \
                  "\nProgram will now terminate...".format(configuration_directory)
        logger.error(message)
        raise gsm_lib.ConfigurationError(message)

    # Initialize Redcap Interface
    rt = redcap_transactions()
    rt.configuration_directory = configuration_directory

    properties = rt.init_redcap_interface(settings, logger)
    #get data from the redcap for the fields listed in the source_data_schema.xml
    response = rt.get_data_from_redcap(properties, logger)

    if debugging:
        try:
            print('Writing REDCap response to file: ')
            with open(redcap_log_file, 'w') as log_file:
                log_file.write(response)
        except:
            print('Unable to write the REDCap response to ' + redcap_log_file)

    try:
        logger.debug(response)
        xml_tree = etree.fromstring(response)

        #XSL Transformation 1: This transformation removes junk data, rename elements and extracts site_id and adds new tag site_id
        logger.debug("Beginning XSL Transformation 1: This transformation removes junk data, rename elements and extracts site_id and adds new tag site_id")
        transform_xsl = os.path.join(configuration_directory, settings.xml_formatting_transform_xsl)
        xslt = etree.parse(transform_xsl)
        transform = etree.XSLT(xslt)
        xml_transformed = transform(xml_tree)
        xml_str = etree.tostring(xml_transformed, method='xml', pretty_print=True)
        logger.debug("XSL Transformation 1 completed.")

        #XSL Transformation 2: This transformation groups the data based on site_id
        logger.debug("Beginning XSL Transformation 2: This transformation groups the data based on site_id")
        transform2_xsl = proj_root + 'rsm/utils/groupby_siteid_transform.xsl'
        xslt = etree.parse(transform2_xsl)
        transform = etree.XSLT(xslt)
        xml_transformed2 = transform(xml_transformed)
        logger.debug("XSL Transformation 2 completed.")

        #XSL Transformation 3: This transformation removes all the nodes which are not set
        logger.debug("Beginning XSL Transformation 3: This transformation removes all the nodes which are not set")
        transform3_xsl = proj_root + 'rsm/utils/remove_junktags_transform.xsl'
        xslt = etree.parse(transform3_xsl)
        transform = etree.XSLT(xslt)
        xml_transformed3 = transform(xml_transformed2)
        logger.debug("XSL Transformation 3 completed.")

        #Prettifying the output generated by XSL Transformation
        xml_str2 = etree.tostring(xml_transformed3, method='xml', pretty_print=True)
        tree = etree.fromstring(xml_str2, etree.XMLParser(remove_blank_text=True))

        # Loop through the start_date elements and update their values
        for k in tree.iter('start_date'):
            d = datetime.datetime.strptime(k.text, "%Y-%m-%d").date()-timedelta(days=365)
            k.text = str(d)

        #writing data to smi+site_code.xml. This xml will be saved to sftp of the site as smi.xml
        do_keep_gen_files = args['keep']
        tmp_folder = gsm_lib.get_temp_path(do_keep_gen_files)

        subject_map_input = {}
        for k in tree:
            site_code = k.attrib['id']
            file_name = tmp_folder + 'smi' + site_code + '.xml'
            gsm_lib.write_element_tree_to_file(ET.ElementTree(k), file_name)
            subject_map_input[site_code] = file_name

        site_catalog_file = os.path.join(configuration_directory, settings.site_catalog)
        parse_site_details_and_send(site_catalog_file, subject_map_input, logger, settings, do_keep_gen_files)
    except Exception as xe:
        if debugging:
            print('Unable to transform data returned by REDCap:')
            print('REDCap response logged to ' + redcap_log_file)
            print('Ecountered the following error:')
            print(xe)
            exit()
        else:
            print('Unable to transform data returned by REDCap:')
            exit()
コード例 #2
0
def main():
    global configuration_directory

    # obtaining command line arguments for path to config directory
    args = parse_args()
    configuration_directory = os.path.abspath(args['configuration_directory_path'])
    do_keep_gen_files = args['keep']

    # Configure logging
    logger = configure_logging(args['verbose'], args['logfile'])

    conf_file = os.path.join(configuration_directory, 'settings.ini')
    settings = gsm_lib.get_settings(conf_file)
    gsm_lib.read_config(configuration_directory, conf_file, settings)
    site_catalog_file = os.path.join(configuration_directory, settings.site_catalog)

    # Initialize Redcap Interface
    rt = redcap_transactions()
    rt.configuration_directory = configuration_directory

    properties = rt.init_redcap_interface(settings, logger)
    # gets data from the person index for the fields listed in the source_data_schema.xml
    response = rt.get_data_from_redcap(properties, logger)
    xml_tree = etree.fromstring(response)

    #XSL Transformation : transforms the person_index data
    transform_xsl = proj_root + "bin/utils/person_index_transform.xsl"
    xslt = etree.parse(transform_xsl)
    transform = etree.XSLT(xslt)
    person_index_data = transform(xml_tree)

    # # # retrieve smi.xml from the sftp server
    smi_path = get_smi_and_parse(site_catalog_file, settings, logger)
    try:
        smi_data = etree.parse(smi_path)
    except IOError:
        logger.exception("Could not open file: " + smi_path)
        raise

    #sorting both the xml files.
    gsm_lib.sort_element_tree(smi_data)
    gsm_lib.sort_element_tree(person_index_data)
    #generating the person index dictionary
    person_index_dict = {}
    for item in person_index_data.iter('item'):
        person_index_dict[item.findtext('research_subject_id')] = [
            item.findtext('yob'),
            item.findtext('mrn'),
            item.findtext('facility_code')]
    #iterate through the smi data and generate a
    # new merged xml's for subject_map and subject_map_exceptions
    subjectmap_root = etree.Element("subject_map_records")
    subjectmap_exceptions_root = etree.Element("subject_map_exception_records")
    exceptions = False
    for item in smi_data.iter('item'):
        if item.findtext('research_subject_id') in person_index_dict.keys():
            logger.debug("Processing research_subject_id %s", item.findtext('research_subject_id'))
            if (person_index_dict[item.findtext('research_subject_id')][0] == item.findtext('yob')):
                logger.debug("yob matched for research_subject_id %s", item.findtext('research_subject_id'))
                mrn = etree.SubElement(item, "mrn")
                mrn.text = person_index_dict[item.findtext('research_subject_id')][1]
                facility_code = etree.SubElement(item, "facility_code")
                facility_code.text = person_index_dict[item.findtext('research_subject_id')][2]
                item.remove(item.find('yob'))
                subjectmap_root.append(item)

            else:
                logger.debug("yob not matched for research_subject_id %s", item.findtext('research_subject_id'))
                exception_item = etree.Element("item")
                research_subject_id = etree.SubElement(exception_item, "research_subject_id")
                research_subject_id.text = item.findtext('research_subject_id')
                if (research_subject_id.text is not None):
                    exceptions = True
                pi_yob = etree.SubElement(exception_item, "Person_Index_YOB")
                pi_yob.text = person_index_dict[item.findtext('research_subject_id')][0]
                hcvt_yob = etree.SubElement(exception_item, "HCVTarget_YOB")
                hcvt_yob.text = item.findtext('yob')
                subjectmap_exceptions_root.append(exception_item)

    #Below code transforms the xml files to csv files
    transform_xsl = proj_root + "bin/utils/xml2csv.xsl"
    xslt = etree.parse(transform_xsl)
    transform = etree.XSLT(xslt)

    tmp_folder = gsm_lib.get_temp_path(do_keep_gen_files)
    subject_map_file = tmp_folder + "subject_map.csv"
    logger.info('Using path subject map file path: ' + subject_map_file)

    try:
        subject_map_csv = open(subject_map_file, "w")
        subject_map_csv.write("%s" % '"research_subject_id","start_date","end_date","mrn","facility_code"\n')

        for item in subjectmap_root.iter("item"):
            line = '"{0}","{1}","{2}","{3}","{4}"\n'.format(
                gsm_lib.handle_blanks(item.findtext("research_subject_id")),
                gsm_lib.handle_blanks(item.findtext("start_date")),
                gsm_lib.handle_blanks(item.findtext("end_date")),
                gsm_lib.handle_blanks(item.findtext("mrn")),
                gsm_lib.handle_blanks(item.findtext("facility_code")))
            subject_map_csv.write("%s" % line)

        subject_map_csv.close()
    except IOError:
        logger.exception("Could not open file %s for write", subject_map_file)
        raise

    # remove the smi.xml from the folder because the XSLT process
    # writes data to smi.xml
    try:
        os.remove(smi_path)
    except OSError:
        logger.exception("Could not remove file %s ", smi_path)
        raise

    # send the subject_map.csv to EMR team (sftp server)
    parse_site_details_and_send(site_catalog_file, subject_map_file, 'sftp', settings, logger)
    if do_keep_gen_files:
        logger.info('Keeping the temporary file: ' + subject_map_file)
    else:
        logger.info('Removing the temporary file: ' + subject_map_file)
        os.remove(subject_map_file)

    # send subject_map_exceptions.csv as email attachment
    if exceptions:
        subject_map_exceptions_file = tmp_folder + 'subject_map_exceptions.csv'
        try:
            subject_map_exceptions_csv = open(subject_map_exceptions_file, "w")
        except IOError:
            logger.exception("Could not open file %s for write", subject_map_exceptions_file)
            raise
        subject_map_exceptions_csv.write("%s" % '"research_subject_id","person_index_yob","redcap_yob"\n')
        for item in subjectmap_exceptions_root.iter("item"):
            line = '"{0}","{1}","{2}"\n'.format(
                gsm_lib.handle_blanks(item.find("research_subject_id").text),
                gsm_lib.handle_blanks(item.find("Person_Index_YOB").text),
                gsm_lib.handle_blanks(item.find("HCVTarget_YOB").text))
            subject_map_exceptions_csv.write("%s" % line)
        subject_map_exceptions_csv.close()

        parse_site_details_and_send(site_catalog_file, subject_map_exceptions_file, 'email', settings, logger)
        if do_keep_gen_files:
            logger.info('Keeping the temporary file: ' + subject_map_exceptions_file)
        else:
            logger.info('Removing the temporary file: ' + subject_map_exceptions_file)
            os.remove(subject_map_exceptions_file)
コード例 #3
0
def main():
    global configuration_directory
    global do_keep_gen_files
    global tmp_folder

    # obtaining command line arguments for path to config directory
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-c', dest='configuration_directory_path',
        default=default_configuration_directory,
        required=False,
        help='Specify the path to the configuration directory')

    # read the optional argument `-k` for keeping the generated files
    parser.add_argument(
        '-k', '--keep',
        default=default_do_keep_gen_files,
        required=False,
        help = 'Specify `yes` to preserve the files generated during execution')


    args = vars(parser.parse_args())
    configuration_directory = args['configuration_directory_path'] + '/'
    do_keep_gen_files       = False if args['keep'] is None else True

    #setup_json = configuration_directory + 'setup.json'
    global setup
    setup = gsm_lib.read_config(configuration_directory, 'setup.json')
    site_catalog_file = configuration_directory + setup['site_catalog']
    system_log_file = setup['system_log_file']

    # Configure logging
    global gsmlogger
    gsmlogger = GSMLogger()
    gsmlogger.configure_logging(system_log_file)

    # Initialize Redcap Interface
    rt = redcap_transactions()
    rt.configuration_directory = configuration_directory

    properties = rt.init_redcap_interface(setup, gsmlogger.logger)
    transform_xsl = configuration_directory + setup['xml_formatting_tranform_xsl']
    #get data from the redcap for the fields listed in the source_data_schema.xml
    response = rt.get_data_from_redcap(properties, gsmlogger.logger)

    #XSL Transformation 1: This transformation removes junk data, rename elements and extracts site_id and adds new tag site_id
    xml_tree = etree.fromstring(response)
    xslt = etree.parse(transform_xsl)
    transform = etree.XSLT(xslt)
    xml_transformed = transform(xml_tree)
    xml_str = etree.tostring(xml_transformed, method='xml', pretty_print=True)

    #XSL Transformation 2: This transformation groups the data based on site_id
    transform2_xsl = proj_root + 'bin/utils/groupby_siteid_transform.xsl'
    xslt = etree.parse(transform2_xsl)
    transform = etree.XSLT(xslt)
    xml_transformed2 = transform(xml_transformed)

    #XSL Transformation 3: This transformation removes all the nodes which are not set
    transform3_xsl = proj_root + 'bin/utils/remove_junktags_transform.xsl'
    xslt = etree.parse(transform3_xsl)
    transform = etree.XSLT(xslt)
    xml_transformed3 = transform(xml_transformed2)

    #Prettifying the output generated by XSL Transformation
    xml_str2 = etree.tostring(xml_transformed3, method='xml', pretty_print=True)
    tree = etree.fromstring(xml_str2, etree.XMLParser(remove_blank_text=True))

    # Loop through the start_date elements and update theur values
    for k in tree.iter('start_date'):
        d = datetime.datetime.strptime(k.text, "%Y-%m-%d").date()-timedelta(days=180)
        k.text = str(d)

    #writing data to smi+site_id.xml. This xml will be saved to sftp of the site as smi.xml
    smi_filenames = []
    smi_ids = []
    tmp_folder = gsm_lib.get_temp_path(do_keep_gen_files)

    for k in tree:
        file_name = tmp_folder + 'smi' + k.attrib['id']+'.xml'
        gsm_lib.write_element_tree_to_file(ET.ElementTree(k), file_name)
        smi_filenames.append(file_name)
        smi_ids.append(k.attrib['id'])

    print 'Using smi_filenames: '
    pprint.pprint(smi_filenames)
    parse_site_details_and_send(site_catalog_file, smi_filenames, smi_ids, gsmlogger)
コード例 #4
0
def main():
    # obtaining command line arguments for path to config directory
    args = parse_args()
    configuration_directory = os.path.abspath(
        args['configuration_directory_path'])

    # Configure logging
    logger = configure_logging(args['verbose'], args['logfile'])

    conf_file = os.path.join(configuration_directory, 'settings.ini')
    settings = gsm_lib.get_settings(conf_file)
    gsm_lib.read_config(configuration_directory, conf_file, settings)

    # Check if xml_formatting_transform.xsl file is present/properly set in
    # setting.ini
    message2 = "Please set it with appropriate value and restart execution. " \
               "For assistance refer config-example-gsm-input/settings.ini." \
               "\nProgram will now terminate..."
    if not settings.hasoption('xml_formatting_transform_xsl'):
        message = "Required parameter xml_formatting_transform_xsl is missing " \
                  "in settings.ini. " + message2
        logger.error(message)
        raise gsm_lib.ConfigurationError(message)
    elif settings.xml_formatting_transform_xsl == "":
        message = "Required parameter xml_formatting_transform_xsl does not " \
                  "have a value in settings.ini. " + message2
        logger.error(message)
        raise gsm_lib.ConfigurationError(message)
    elif not os.path.exists(
            os.path.join(configuration_directory,
                         settings.xml_formatting_transform_xsl)):
        message = "Required file xml_formatting_transform.xsl does not exist " \
                  "in {0}. Please make sure this file is included in the " \
                  "configuration directory and restart execution. For " \
                  "assistance refer config-example-gsm-input/xml_formatting_transform.xsl." \
                  "\nProgram will now terminate...".format(configuration_directory)
        logger.error(message)
        raise gsm_lib.ConfigurationError(message)

    # Initialize Redcap Interface
    rt = redcap_transactions()
    rt.configuration_directory = configuration_directory

    properties = rt.init_redcap_interface(settings, logger)
    #get data from the redcap for the fields listed in the source_data_schema.xml
    response = rt.get_data_from_redcap(properties, logger)
    logger.debug(response)
    xml_tree = etree.fromstring(response)

    #XSL Transformation 1: This transformation removes junk data, rename elements and extracts site_id and adds new tag site_id
    transform_xsl = os.path.join(configuration_directory,
                                 settings.xml_formatting_transform_xsl)
    xslt = etree.parse(transform_xsl)
    transform = etree.XSLT(xslt)
    xml_transformed = transform(xml_tree)
    xml_str = etree.tostring(xml_transformed, method='xml', pretty_print=True)

    #XSL Transformation 2: This transformation groups the data based on site_id
    transform2_xsl = proj_root + 'bin/utils/groupby_siteid_transform.xsl'
    xslt = etree.parse(transform2_xsl)
    transform = etree.XSLT(xslt)
    xml_transformed2 = transform(xml_transformed)

    #XSL Transformation 3: This transformation removes all the nodes which are not set
    transform3_xsl = proj_root + 'bin/utils/remove_junktags_transform.xsl'
    xslt = etree.parse(transform3_xsl)
    transform = etree.XSLT(xslt)
    xml_transformed3 = transform(xml_transformed2)

    #Prettifying the output generated by XSL Transformation
    xml_str2 = etree.tostring(xml_transformed3,
                              method='xml',
                              pretty_print=True)
    tree = etree.fromstring(xml_str2, etree.XMLParser(remove_blank_text=True))

    # Loop through the start_date elements and update theur values
    for k in tree.iter('start_date'):
        d = datetime.datetime.strptime(k.text,
                                       "%Y-%m-%d").date() - timedelta(days=365)
        k.text = str(d)

    #writing data to smi+site_code.xml. This xml will be saved to sftp of the site as smi.xml
    do_keep_gen_files = args['keep']
    tmp_folder = gsm_lib.get_temp_path(do_keep_gen_files)

    subject_map_input = {}
    for k in tree:
        site_code = k.attrib['id']
        file_name = tmp_folder + 'smi' + site_code + '.xml'
        gsm_lib.write_element_tree_to_file(ET.ElementTree(k), file_name)
        subject_map_input[site_code] = file_name

    site_catalog_file = os.path.join(configuration_directory,
                                     settings.site_catalog)
    parse_site_details_and_send(site_catalog_file, subject_map_input, logger,
                                settings, do_keep_gen_files)
コード例 #5
0
def main():
    global configuration_directory
    global do_keep_gen_files

    # obtaining command line arguments for path to config directory
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-c",
        dest="configuration_directory_path",
        default=default_configuration_directory,
        required=False,
        help="Specify the path to the configuration directory",
    )

    # read the optional argument `-k` for keeping the generated files
    parser.add_argument(
        "-k",
        "--keep",
        default=default_do_keep_gen_files,
        required=False,
        help="Specify `yes` to preserve the files generated during execution",
    )

    args = vars(parser.parse_args())
    configuration_directory = args["configuration_directory_path"] + "/"
    do_keep_gen_files = False if args["keep"] is None else True

    # read setup options
    global setup
    setup = gsm_lib.read_config(configuration_directory, "setup.json")
    site_catalog_file = configuration_directory + setup["site_catalog"]
    system_log_file = setup["system_log_file"]

    # Configure logging
    global gsmlogger
    gsmlogger = GSMLogger()
    gsmlogger.configure_logging(system_log_file)

    # Initialize Redcap Interface
    rt = redcap_transactions()
    rt.configuration_directory = configuration_directory

    properties = rt.init_redcap_interface(setup, gsmlogger.logger)
    # gets data from the person index for the fields listed in the source_data_schema.xml
    response = rt.get_data_from_redcap(properties, gsmlogger.logger)
    xml_tree = etree.fromstring(response)

    # XSL Transformation : transforms the person_index data
    transform_xsl = proj_root + "bin/utils/person_index_transform.xsl"
    xslt = etree.parse(transform_xsl)
    transform = etree.XSLT(xslt)
    person_index_data = transform(xml_tree)

    # # # retrieve smi.xml from the sftp server
    smi_path = get_smi_and_parse(site_catalog_file)
    if not os.path.exists(smi_path):
        raise GSMLogger().LogException("Error: file " + smi_path + " not found")
    else:
        smi = open(smi_path, "r")
    # Below code merges the 2 xmls
    smi_data = etree.parse(smi_path)
    # sorting both the xml files.
    gsm_lib.sort_element_tree(smi_data)
    gsm_lib.sort_element_tree(person_index_data)
    # generating the person index dictionary
    person_index_dict = {}
    for item in person_index_data.iter("item"):
        person_index_dict[item.findtext("research_subject_id")] = [
            item.findtext("yob"),
            item.findtext("mrn"),
            item.findtext("facility_code"),
        ]
    # iterate through the smi data and generate a
    # new merged xml's for subject_map and subject_map_exceptions
    subjectmap_root = etree.Element("subject_map_records")
    subjectmap_exceptions_root = etree.Element("subject_map_exception_records")
    exceptions = False
    for item in smi_data.iter("item"):
        if item.findtext("research_subject_id") in person_index_dict.keys():
            gsmlogger.logger.debug("Processing research_subject_id %s", item.findtext("research_subject_id"))
            if person_index_dict[item.findtext("research_subject_id")][0] == item.findtext("yob"):
                gsmlogger.logger.debug("yob matched for research_subject_id %s", item.findtext("research_subject_id"))
                mrn = etree.SubElement(item, "mrn")
                mrn.text = person_index_dict[item.findtext("research_subject_id")][1]
                facility_code = etree.SubElement(item, "facility_code")
                facility_code.text = person_index_dict[item.findtext("research_subject_id")][2]
                item.remove(item.find("yob"))
                subjectmap_root.append(item)

            else:
                gsmlogger.logger.debug(
                    "yob not matched for research_subject_id %s", item.findtext("research_subject_id")
                )
                exception_item = etree.Element("item")
                research_subject_id = etree.SubElement(exception_item, "research_subject_id")
                research_subject_id.text = item.findtext("research_subject_id")
                if research_subject_id.text is not None:
                    exceptions = True
                pi_yob = etree.SubElement(exception_item, "Person_Index_YOB")
                pi_yob.text = person_index_dict[item.findtext("research_subject_id")][0]
                hcvt_yob = etree.SubElement(exception_item, "HCVTarget_YOB")
                hcvt_yob.text = item.findtext("yob")
                subjectmap_exceptions_root.append(exception_item)

    # Below code transforms the xml files to csv files
    transform_xsl = proj_root + "bin/utils/xml2csv.xsl"
    xslt = etree.parse(transform_xsl)
    transform = etree.XSLT(xslt)

    tmp_folder = gsm_lib.get_temp_path(do_keep_gen_files)
    subject_map_file = tmp_folder + "subject_map.csv"
    gsmlogger.logger.info("Using path subject map file path: " + subject_map_file)

    try:
        subject_map_csv = open(subject_map_file, "w")
        subject_map_csv.write("%s" % '"research_subject_id","start_date","end_date","mrn","facility_code"\n')

        for item in subjectmap_root.iter("item"):
            line = '"{0}","{1}","{2}","{3}","{4}"\n'.format(
                gsm_lib.handle_blanks(item.findtext("research_subject_id")),
                gsm_lib.handle_blanks(item.findtext("start_date")),
                gsm_lib.handle_blanks(item.findtext("end_date")),
                gsm_lib.handle_blanks(item.findtext("mrn")),
                gsm_lib.handle_blanks(item.findtext("facility_code")),
            )
            subject_map_csv.write("%s" % line)

        subject_map_csv.close()
    except IOError:
        raise GSMLogger().LogException("Could not open file %s for write", subject_map_file)

    # remove the smi.xml from the folder because the XSLT process
    # writes data to smi.xml
    try:
        os.remove(smi_path)
    except OSError:
        raise GSMLogger().LogException("Could not remove file %s ", smi_path)

    # send the subject_map.csv to EMR team (sftp server)
    parse_site_details_and_send(site_catalog_file, subject_map_file, "sftp")
    if do_keep_gen_files:
        print " * Keeping the temporary file: " + subject_map_file
    else:
        print " * Removing the temporary file: " + subject_map_file
        os.remove(subject_map_file)

    # send subject_map_exceptions.csv as email attachment
    if exceptions:
        subject_map_exceptions_file = tmp_folder + "subject_map_exceptions.csv"
        try:
            subject_map_exceptions_csv = open(subject_map_exceptions_file, "w")
        except IOError:
            raise GSMLogger().LogException("Could not open file %s for write", subject_map_exceptions_file)

        subject_map_exceptions_csv.write("%s" % '"research_subject_id","person_index_yob","redcap_yob"\n')
        for item in subjectmap_exceptions_root.iter("item"):
            line = '"{0}","{1}","{2}"\n'.format(
                gsm_lib.handle_blanks(item.find("research_subject_id").text),
                gsm_lib.handle_blanks(item.find("Person_Index_YOB").text),
                gsm_lib.handle_blanks(item.find("HCVTarget_YOB").text),
            )
            subject_map_exceptions_csv.write("%s" % line)
        subject_map_exceptions_csv.close()

        parse_site_details_and_send(site_catalog_file, subject_map_exceptions_file, "email")
        if do_keep_gen_files:
            print " * Keeping the temporary file: " + subject_map_exceptions_file
        else:
            print " * Removing the temporary file: " + subject_map_exceptions_file
            os.remove(subject_map_exceptions_file)