def main(): # obtaining command line arguments for path to config directory args = parse_args() configuration_directory = os.path.abspath( args['configuration_directory_path']) # Configure logging logger = configure_logging(args['verbose'], args['logfile']) conf_file = os.path.join(configuration_directory, 'settings.ini') settings = gsm_lib.get_settings(conf_file) gsm_lib.read_config(configuration_directory, conf_file, settings) # Check if xml_formatting_transform.xsl file is present/properly set in # setting.ini message2 = "Please set it with appropriate value and restart execution. " \ "For assistance refer config-example-gsm-input/settings.ini." \ "\nProgram will now terminate..." if not settings.hasoption('xml_formatting_transform_xsl'): message = "Required parameter xml_formatting_transform_xsl is missing " \ "in settings.ini. " + message2 logger.error(message) raise gsm_lib.ConfigurationError(message) elif settings.xml_formatting_transform_xsl == "": message = "Required parameter xml_formatting_transform_xsl does not " \ "have a value in settings.ini. " + message2 logger.error(message) raise gsm_lib.ConfigurationError(message) elif not os.path.exists( os.path.join(configuration_directory, settings.xml_formatting_transform_xsl)): message = "Required file xml_formatting_transform.xsl does not exist " \ "in {0}. Please make sure this file is included in the " \ "configuration directory and restart execution. For " \ "assistance refer config-example-gsm-input/xml_formatting_transform.xsl." \ "\nProgram will now terminate...".format(configuration_directory) logger.error(message) raise gsm_lib.ConfigurationError(message) # Initialize Redcap Interface rt = redcap_transactions() rt.configuration_directory = configuration_directory properties = rt.init_redcap_interface(settings, logger) #get data from the redcap for the fields listed in the source_data_schema.xml response = rt.get_data_from_redcap(properties, logger) logger.debug(response) xml_tree = etree.fromstring(response) #XSL Transformation 1: This transformation removes junk data, rename elements and extracts site_id and adds new tag site_id transform_xsl = os.path.join(configuration_directory, settings.xml_formatting_transform_xsl) xslt = etree.parse(transform_xsl) transform = etree.XSLT(xslt) xml_transformed = transform(xml_tree) xml_str = etree.tostring(xml_transformed, method='xml', pretty_print=True) #XSL Transformation 2: This transformation groups the data based on site_id transform2_xsl = proj_root + 'bin/utils/groupby_siteid_transform.xsl' xslt = etree.parse(transform2_xsl) transform = etree.XSLT(xslt) xml_transformed2 = transform(xml_transformed) #XSL Transformation 3: This transformation removes all the nodes which are not set transform3_xsl = proj_root + 'bin/utils/remove_junktags_transform.xsl' xslt = etree.parse(transform3_xsl) transform = etree.XSLT(xslt) xml_transformed3 = transform(xml_transformed2) #Prettifying the output generated by XSL Transformation xml_str2 = etree.tostring(xml_transformed3, method='xml', pretty_print=True) tree = etree.fromstring(xml_str2, etree.XMLParser(remove_blank_text=True)) # Loop through the start_date elements and update theur values for k in tree.iter('start_date'): d = datetime.datetime.strptime(k.text, "%Y-%m-%d").date() - timedelta(days=365) k.text = str(d) #writing data to smi+site_code.xml. This xml will be saved to sftp of the site as smi.xml do_keep_gen_files = args['keep'] tmp_folder = gsm_lib.get_temp_path(do_keep_gen_files) subject_map_input = {} for k in tree: site_code = k.attrib['id'] file_name = tmp_folder + 'smi' + site_code + '.xml' gsm_lib.write_element_tree_to_file(ET.ElementTree(k), file_name) subject_map_input[site_code] = file_name site_catalog_file = os.path.join(configuration_directory, settings.site_catalog) parse_site_details_and_send(site_catalog_file, subject_map_input, logger, settings, do_keep_gen_files)
def main(): # obtaining command line arguments for path to config directory args = parse_args() configuration_directory = os.path.abspath(args['configuration_directory_path']) if 'debug' in args: if args['debug']: debugging = args['debug'] else: debugging = False else: debugging = False # Configure logging logger = configure_logging(args['verbose'], args['logfile']) conf_file = os.path.join(configuration_directory, 'settings.ini') settings = gsm_lib.get_settings(conf_file) gsm_lib.read_config(configuration_directory, conf_file, settings) if debugging: if settings.redcap_log_file: redcap_log_file = os.path.join(configuration_directory, settings.redcap_log_file) else: redcap_log_file = 'redcap.log.xml' # Check if xml_formatting_transform.xsl file is present/properly set in # setting.ini message2 = "Please set it with appropriate value and restart execution. " \ "For assistance refer config-example-gsm-input/settings.ini." \ "\nProgram will now terminate..." if not settings.hasoption('xml_formatting_transform_xsl'): message = "Required parameter xml_formatting_transform_xsl is missing " \ "in settings.ini. " + message2 logger.error(message) raise gsm_lib.ConfigurationError(message) elif settings.xml_formatting_transform_xsl == "": message = "Required parameter xml_formatting_transform_xsl does not " \ "have a value in settings.ini. " + message2 logger.error(message) raise gsm_lib.ConfigurationError(message) elif not os.path.exists(os.path.join(configuration_directory, settings.xml_formatting_transform_xsl)): message = "Required file xml_formatting_transform.xsl does not exist " \ "in {0}. Please make sure this file is included in the " \ "configuration directory and restart execution. For " \ "assistance refer config-example-gsm-input/xml_formatting_transform.xsl." \ "\nProgram will now terminate...".format(configuration_directory) logger.error(message) raise gsm_lib.ConfigurationError(message) # Initialize Redcap Interface rt = redcap_transactions() rt.configuration_directory = configuration_directory properties = rt.init_redcap_interface(settings, logger) #get data from the redcap for the fields listed in the source_data_schema.xml response = rt.get_data_from_redcap(properties, logger) if debugging: try: print('Writing REDCap response to file: ') with open(redcap_log_file, 'w') as log_file: log_file.write(response) except: print('Unable to write the REDCap response to ' + redcap_log_file) try: logger.debug(response) xml_tree = etree.fromstring(response) #XSL Transformation 1: This transformation removes junk data, rename elements and extracts site_id and adds new tag site_id logger.debug("Beginning XSL Transformation 1: This transformation removes junk data, rename elements and extracts site_id and adds new tag site_id") transform_xsl = os.path.join(configuration_directory, settings.xml_formatting_transform_xsl) xslt = etree.parse(transform_xsl) transform = etree.XSLT(xslt) xml_transformed = transform(xml_tree) xml_str = etree.tostring(xml_transformed, method='xml', pretty_print=True) logger.debug("XSL Transformation 1 completed.") #XSL Transformation 2: This transformation groups the data based on site_id logger.debug("Beginning XSL Transformation 2: This transformation groups the data based on site_id") transform2_xsl = proj_root + 'rsm/utils/groupby_siteid_transform.xsl' xslt = etree.parse(transform2_xsl) transform = etree.XSLT(xslt) xml_transformed2 = transform(xml_transformed) logger.debug("XSL Transformation 2 completed.") #XSL Transformation 3: This transformation removes all the nodes which are not set logger.debug("Beginning XSL Transformation 3: This transformation removes all the nodes which are not set") transform3_xsl = proj_root + 'rsm/utils/remove_junktags_transform.xsl' xslt = etree.parse(transform3_xsl) transform = etree.XSLT(xslt) xml_transformed3 = transform(xml_transformed2) logger.debug("XSL Transformation 3 completed.") #Prettifying the output generated by XSL Transformation xml_str2 = etree.tostring(xml_transformed3, method='xml', pretty_print=True) tree = etree.fromstring(xml_str2, etree.XMLParser(remove_blank_text=True)) # Loop through the start_date elements and update their values for k in tree.iter('start_date'): d = datetime.datetime.strptime(k.text, "%Y-%m-%d").date()-timedelta(days=365) k.text = str(d) #writing data to smi+site_code.xml. This xml will be saved to sftp of the site as smi.xml do_keep_gen_files = args['keep'] tmp_folder = gsm_lib.get_temp_path(do_keep_gen_files) subject_map_input = {} for k in tree: site_code = k.attrib['id'] file_name = tmp_folder + 'smi' + site_code + '.xml' gsm_lib.write_element_tree_to_file(ET.ElementTree(k), file_name) subject_map_input[site_code] = file_name site_catalog_file = os.path.join(configuration_directory, settings.site_catalog) parse_site_details_and_send(site_catalog_file, subject_map_input, logger, settings, do_keep_gen_files) except Exception as xe: if debugging: print('Unable to transform data returned by REDCap:') print('REDCap response logged to ' + redcap_log_file) print('Ecountered the following error:') print(xe) exit() else: print('Unable to transform data returned by REDCap:') exit()
def main(): global configuration_directory global do_keep_gen_files global tmp_folder # obtaining command line arguments for path to config directory parser = argparse.ArgumentParser() parser.add_argument( '-c', dest='configuration_directory_path', default=default_configuration_directory, required=False, help='Specify the path to the configuration directory') # read the optional argument `-k` for keeping the generated files parser.add_argument( '-k', '--keep', default=default_do_keep_gen_files, required=False, help = 'Specify `yes` to preserve the files generated during execution') args = vars(parser.parse_args()) configuration_directory = args['configuration_directory_path'] + '/' do_keep_gen_files = False if args['keep'] is None else True #setup_json = configuration_directory + 'setup.json' global setup setup = gsm_lib.read_config(configuration_directory, 'setup.json') site_catalog_file = configuration_directory + setup['site_catalog'] system_log_file = setup['system_log_file'] # Configure logging global gsmlogger gsmlogger = GSMLogger() gsmlogger.configure_logging(system_log_file) # Initialize Redcap Interface rt = redcap_transactions() rt.configuration_directory = configuration_directory properties = rt.init_redcap_interface(setup, gsmlogger.logger) transform_xsl = configuration_directory + setup['xml_formatting_tranform_xsl'] #get data from the redcap for the fields listed in the source_data_schema.xml response = rt.get_data_from_redcap(properties, gsmlogger.logger) #XSL Transformation 1: This transformation removes junk data, rename elements and extracts site_id and adds new tag site_id xml_tree = etree.fromstring(response) xslt = etree.parse(transform_xsl) transform = etree.XSLT(xslt) xml_transformed = transform(xml_tree) xml_str = etree.tostring(xml_transformed, method='xml', pretty_print=True) #XSL Transformation 2: This transformation groups the data based on site_id transform2_xsl = proj_root + 'bin/utils/groupby_siteid_transform.xsl' xslt = etree.parse(transform2_xsl) transform = etree.XSLT(xslt) xml_transformed2 = transform(xml_transformed) #XSL Transformation 3: This transformation removes all the nodes which are not set transform3_xsl = proj_root + 'bin/utils/remove_junktags_transform.xsl' xslt = etree.parse(transform3_xsl) transform = etree.XSLT(xslt) xml_transformed3 = transform(xml_transformed2) #Prettifying the output generated by XSL Transformation xml_str2 = etree.tostring(xml_transformed3, method='xml', pretty_print=True) tree = etree.fromstring(xml_str2, etree.XMLParser(remove_blank_text=True)) # Loop through the start_date elements and update theur values for k in tree.iter('start_date'): d = datetime.datetime.strptime(k.text, "%Y-%m-%d").date()-timedelta(days=180) k.text = str(d) #writing data to smi+site_id.xml. This xml will be saved to sftp of the site as smi.xml smi_filenames = [] smi_ids = [] tmp_folder = gsm_lib.get_temp_path(do_keep_gen_files) for k in tree: file_name = tmp_folder + 'smi' + k.attrib['id']+'.xml' gsm_lib.write_element_tree_to_file(ET.ElementTree(k), file_name) smi_filenames.append(file_name) smi_ids.append(k.attrib['id']) print 'Using smi_filenames: ' pprint.pprint(smi_filenames) parse_site_details_and_send(site_catalog_file, smi_filenames, smi_ids, gsmlogger)