def ingest_multiple_dates_main(args_namespace): """ Main entry point for this script. Will parse args passed in and run ET/L as appropriate. Does not throw any exception. """ results = [] start_time = time.time() try: load_io_yaml_from_args(args_namespace.io_yaml) d = args_namespace.start_date end_date = args_namespace.end_date delta = timedelta(days=1) try: check_status() while d <= end_date: start_time = time.time() generate_random_error(0) time.sleep(100) end_time = time.time() results.append( { "date": d.strftime("%Y-%m-%d"), "status": "success", "start_time": start_time, "end_time": end_time, "error_info": {}, } ) d += delta except ProcessError as pe: results += pe.args[0]["results"] pass except (KeyboardInterrupt, KeyboardInterruptError, BaseException) as e: if type(e) in [KeyboardInterrupt, KeyboardInterruptError]: status = "cancelled" else: status = "error" results.append( { "date": args_namespace.start_date.strftime("%Y-%m-%d"), "status": status, "start_time": start_time, "end_time": time.time(), "error_info": get_error_info(status), } ) return results
def ingest_multiple_dates_main(args_namespace): """ Main entry point for this script. Will parse args passed in and run ET/L as appropriate. Does not throw any exception. """ results = [] start_time = time.time() try: load_io_yaml_from_args(args_namespace.io_yaml) d = args_namespace.start_date end_date = args_namespace.end_date delta = timedelta(days=1) try: check_status() while d <= end_date: start_time = time.time() generate_random_error(0) time.sleep(100) end_time = time.time() results.append({ 'date': d.strftime("%Y-%m-%d"), 'status': 'success', 'start_time': start_time, 'end_time': end_time, 'error_info': {} }) d += delta except ProcessError as pe: results += pe.args[0]['results'] pass except (KeyboardInterrupt, KeyboardInterruptError, BaseException) as e: if type(e) in [KeyboardInterrupt, KeyboardInterruptError]: status = 'cancelled' else: status = 'error' results.append({ 'date': args_namespace.start_date.strftime("%Y-%m-%d"), 'status': status, 'start_time': start_time, 'end_time': time.time(), 'error_info': get_error_info(status) }) return results
def ingest_multiple_dates_main(args_namespace): """ Main entry point for this script. Will parse args passed in and run ET/L as appropriate. Does not throw any exception. """ results = [] start_time = time.time() try: load_io_yaml_from_args(args_namespace.io_yaml) sc.YamlConfiguration(args_namespace.config_override, optional=True) if args_namespace.serial_stepper: etl_stepper = ETLStepper(args_namespace) else: etl_stepper = ParallelEtStepper(args_namespace) if not args_namespace.load_only: try: results += etl_stepper.execute_et_steps() except ProcessError as pe: results += pe.args[0]['results'] pass if not args_namespace.et_only: try: results += etl_stepper.execute_load_steps() except ProcessError as pe: results += pe.args[0]['results'] pass except (KeyboardInterrupt, KeyboardInterruptError, BaseException) as e: step_result = _init_step_result( ETLStep(args_namespace, args_namespace.start_date.strftime("%Y-%m-%d"), None) ) if type(e) in [KeyboardInterrupt, KeyboardInterruptError]: step_result['status'] = 'cancelled' else: step_result['error_info'] = _capture_error_info() step_result['start_time'] = start_time step_result['end_time'] = time.time() results.append(step_result) return results
def ingest_multiple_dates_main(args_namespace): """ Main entry point for this script. Will parse args passed in and run ET/L as appropriate. Does not throw any exception. """ results = [] start_time = time.time() try: load_io_yaml_from_args(args_namespace.io_yaml) sc.YamlConfiguration(args_namespace.config_override, optional=True) if args_namespace.serial_stepper: etl_stepper = ETLStepper(args_namespace) else: etl_stepper = ParallelEtStepper(args_namespace) if not args_namespace.load_only: try: results += etl_stepper.execute_et_steps() except ProcessError as pe: results += pe.args[0]['results'] pass if not args_namespace.et_only: try: results += etl_stepper.execute_load_steps() except ProcessError as pe: results += pe.args[0]['results'] pass except (KeyboardInterrupt, KeyboardInterruptError, BaseException) as e: step_result = _init_step_result( ETLStep(args_namespace, args_namespace.start_date.strftime("%Y-%m-%d"), None)) if type(e) in [KeyboardInterrupt, KeyboardInterruptError]: step_result['status'] = 'cancelled' else: step_result['error_info'] = _capture_error_info() step_result['start_time'] = start_time step_result['end_time'] = time.time() results.append(step_result) return results
def test_load_io_yaml_from_dict(): dict_str = "{'a': 'value', 'b': 'value'}" load_io_yaml_from_args(dict_str)
def test_load_io_yaml_from_int_throws_error(): with pytest.raises(ValueError): load_io_yaml_from_args("4")
def test_load_io_yaml_from_arg_with_string_file_not_found(): with pytest.raises(IOError): load_io_yaml_from_args("string_val")