def main(): ''' Description: Provides the setup and executaion of the processor for the application. ''' # The config file is located in the same place as this script Config.read_config(os.path.dirname(__file__)) # Setup the default logger format and level. log to STDOUT logging.basicConfig(format=('%(asctime)s.%(msecs)03d %(process)d' ' %(levelname)-8s' ' %(filename)s:%(lineno)d:' '%(funcName)s -- %(message)s'), datefmt='%Y-%m-%d %H:%M:%S', level=logging.INFO, stream=sys.stdout) # Get the logger logger = logging.getLogger(__name__) # Turn down the requests and urllib3 logging otherwise they fill the log # file with mostly useless information requests_logger = logging.getLogger("requests") requests_logger.setLevel(logging.WARNING) urllib3_logger = logging.getLogger("urllib3") urllib3_logger.setLevel(logging.WARNING) # Process the command line (s_date, e_date) = parse_commandline() try: # Create the processor object processor = NARR_AuxProcessor(s_date, e_date) # Call the main processing routine processor.archive_aux_data() except Exception: logger.exception('Processing Failed') sys.exit(1) # EXIT FAILURE sys.exit(0) # EXIT SUCCESS
def main(): ''' Description: Ensures all data between start_date and end_date are up to date. Precondition: start_date and end_date are of type datetime.datetime start_date and end_date can also be of type datetime.date ''' # The config file is located in the same place as this script Config.read_config(os.path.dirname(__file__)) cmd_args = parse_arguments() setup_logging(cmd_args.debug, cmd_args.verbose) logger = logging.getLogger(__name__) try: # Determine the data that exists within the date range data = NarrData.get_next_narr_data_gen(cmd_args.start_date, cmd_args.end_date) # Determine which files are stale or missing internally. data_to_be_updated = filter(lambda x: x.need_to_update(), data) if len(data_to_be_updated) == 0: logger.info('No data found for updating archive') else: logger.info('Will download {0} files'. format(len(data_to_be_updated))) if cmd_args.report: report(list(data_to_be_updated)) else: update(data_to_be_updated) except Exception: logger.exception('Processing Failed') sys.exit(1) # EXIT FAILURE sys.exit(0) # EXIT SUCCESS
def main(): ''' Description: Ensures all data between start_date and end_date are up to date. Precondition: start_date and end_date are of type datetime.datetime start_date and end_date can also be of type datetime.date ''' # The config file is located in the same place as this script Config.read_config(os.path.dirname(__file__)) cmd_args = parse_arguments() setup_logging(cmd_args.debug, cmd_args.verbose) logger = logging.getLogger(__name__) try: # Determine the data that exists within the date range data = NarrData.get_next_narr_data_gen(cmd_args.start_date, cmd_args.end_date) # Determine which files are stale or missing internally. data_to_be_updated = filter(lambda x: x.need_to_update(), data) if len(data_to_be_updated) == 0: logger.info('No data found for updating archive') else: logger.info('Will download {0} files'.format( len(data_to_be_updated))) if cmd_args.report: report(list(data_to_be_updated)) else: update(data_to_be_updated) except Exception: logger.exception('Processing Failed') sys.exit(1) # EXIT FAILURE sys.exit(0) # EXIT SUCCESS