def ingest_multiple_dates_main(args_namespace):
""" Main entry point for this script. Will parse args passed in and run
ET/L as appropriate.
Does not throw any exception.
"""
results = []
start_time = time.time()
try:
load_io_yaml_from_args(args_namespace.io_yaml)
d = args_namespace.start_date
end_date = args_namespace.end_date
delta = timedelta(days=1)
try:
check_status()
while d <= end_date:
start_time = time.time()
generate_random_error(0)
time.sleep(100)
end_time = time.time()
results.append(
{
"date": d.strftime("%Y-%m-%d"),
"status": "success",
"start_time": start_time,
"end_time": end_time,
"error_info": {},
}
)
d += delta
except ProcessError as pe:
results += pe.args[0]["results"]
pass
except (KeyboardInterrupt, KeyboardInterruptError, BaseException) as e:
if type(e) in [KeyboardInterrupt, KeyboardInterruptError]:
status = "cancelled"
else:
status = "error"
results.append(
{
"date": args_namespace.start_date.strftime("%Y-%m-%d"),
"status": status,
"start_time": start_time,
"end_time": time.time(),
"error_info": get_error_info(status),
}
)
return results
def ingest_multiple_dates_main(args_namespace):
""" Main entry point for this script. Will parse args passed in and run
ET/L as appropriate.
Does not throw any exception.
"""
results = []
start_time = time.time()
try:
load_io_yaml_from_args(args_namespace.io_yaml)
d = args_namespace.start_date
end_date = args_namespace.end_date
delta = timedelta(days=1)
try:
check_status()
while d <= end_date:
start_time = time.time()
generate_random_error(0)
time.sleep(100)
end_time = time.time()
results.append({
'date': d.strftime("%Y-%m-%d"),
'status': 'success',
'start_time': start_time,
'end_time': end_time,
'error_info': {}
})
d += delta
except ProcessError as pe:
results += pe.args[0]['results']
pass
except (KeyboardInterrupt, KeyboardInterruptError, BaseException) as e:
if type(e) in [KeyboardInterrupt, KeyboardInterruptError]:
status = 'cancelled'
else:
status = 'error'
results.append({
'date': args_namespace.start_date.strftime("%Y-%m-%d"),
'status': status,
'start_time': start_time,
'end_time': time.time(),
'error_info': get_error_info(status)
})
return results
def ingest_multiple_dates_main(args_namespace):
""" Main entry point for this script. Will parse args passed in and run
ET/L as appropriate.
Does not throw any exception.
"""
results = []
start_time = time.time()
try:
load_io_yaml_from_args(args_namespace.io_yaml)
sc.YamlConfiguration(args_namespace.config_override, optional=True)
if args_namespace.serial_stepper:
etl_stepper = ETLStepper(args_namespace)
else:
etl_stepper = ParallelEtStepper(args_namespace)
if not args_namespace.load_only:
try:
results += etl_stepper.execute_et_steps()
except ProcessError as pe:
results += pe.args[0]['results']
pass
if not args_namespace.et_only:
try:
results += etl_stepper.execute_load_steps()
except ProcessError as pe:
results += pe.args[0]['results']
pass
except (KeyboardInterrupt, KeyboardInterruptError, BaseException) as e:
step_result = _init_step_result(
ETLStep(args_namespace,
args_namespace.start_date.strftime("%Y-%m-%d"),
None)
)
if type(e) in [KeyboardInterrupt, KeyboardInterruptError]:
step_result['status'] = 'cancelled'
else:
step_result['error_info'] = _capture_error_info()
step_result['start_time'] = start_time
step_result['end_time'] = time.time()
results.append(step_result)
return results
def ingest_multiple_dates_main(args_namespace):
""" Main entry point for this script. Will parse args passed in and run
ET/L as appropriate.
Does not throw any exception.
"""
results = []
start_time = time.time()
try:
load_io_yaml_from_args(args_namespace.io_yaml)
sc.YamlConfiguration(args_namespace.config_override, optional=True)
if args_namespace.serial_stepper:
etl_stepper = ETLStepper(args_namespace)
else:
etl_stepper = ParallelEtStepper(args_namespace)
if not args_namespace.load_only:
try:
results += etl_stepper.execute_et_steps()
except ProcessError as pe:
results += pe.args[0]['results']
pass
if not args_namespace.et_only:
try:
results += etl_stepper.execute_load_steps()
except ProcessError as pe:
results += pe.args[0]['results']
pass
except (KeyboardInterrupt, KeyboardInterruptError, BaseException) as e:
step_result = _init_step_result(
ETLStep(args_namespace,
args_namespace.start_date.strftime("%Y-%m-%d"), None))
if type(e) in [KeyboardInterrupt, KeyboardInterruptError]:
step_result['status'] = 'cancelled'
else:
step_result['error_info'] = _capture_error_info()
step_result['start_time'] = start_time
step_result['end_time'] = time.time()
results.append(step_result)
return results
def test_load_io_yaml_from_dict():
dict_str = "{'a': 'value', 'b': 'value'}"
load_io_yaml_from_args(dict_str)
def test_load_io_yaml_from_int_throws_error():
with pytest.raises(ValueError):
load_io_yaml_from_args("4")
def test_load_io_yaml_from_arg_with_string_file_not_found():
with pytest.raises(IOError):
load_io_yaml_from_args("string_val")
def test_load_io_yaml_from_dict():
dict_str = "{'a': 'value', 'b': 'value'}"
load_io_yaml_from_args(dict_str)
def test_load_io_yaml_from_int_throws_error():
with pytest.raises(ValueError):
load_io_yaml_from_args("4")
def test_load_io_yaml_from_arg_with_string_file_not_found():
with pytest.raises(IOError):
load_io_yaml_from_args("string_val")