def test_daterange_generator_error_invalid_frequency(self):
"""
Tests the error case when the supplied frequency is invalid
"""
# frequency = None
start_date = datetime.date(1996, 1, 1)
end_date = datetime.date(1996, 1, 10)
date_gen = ParamInjector.generate_daterange("", start_date, end_date,
None)
self.assertRaises(ValueError, list, date_gen)
# frequency = "YEAR"
start_date = datetime.date(1996, 1, 1)
end_date = datetime.date(1996, 1, 10)
date_gen = ParamInjector.generate_daterange("", start_date, end_date,
"YEAR")
self.assertRaises(ValueError, list, date_gen)
# frequency = ""
start_date = datetime.date(1996, 1, 1)
end_date = datetime.date(1996, 1, 10)
date_gen = ParamInjector.generate_daterange("", start_date, end_date,
"")
self.assertRaises(ValueError, list, date_gen)
def test_daterange_generator_error_invalid_range(self):
"""
Tests the error case when no start/end date is supplied, or when either
is invalid
"""
# no end date
start_date = datetime.date(1996, 1, 1)
date_gen = ParamInjector.generate_daterange("", start_date, None)
self.assertRaises(ValueError, list, date_gen)
# no start date
end_date = datetime.date(2005, 1, 1)
date_gen = ParamInjector.generate_daterange("", None, end_date)
self.assertRaises(ValueError, list, date_gen)
# invalid start date
start_date = "1996-01-01"
end_date = datetime.date(2005, 1, 1)
date_gen = ParamInjector.generate_daterange("", start_date, end_date)
self.assertRaises(ValueError, list, date_gen)
# invalid end date
start_date = datetime.date(1996, 1, 1)
end_date = "2004-12-31"
date_gen = ParamInjector.generate_daterange("", start_date, end_date)
self.assertRaises(ValueError, list, date_gen)
def test_daterange_generator_reverse(self):
"""
Tests the date range generator in reverse
"""
# generate days from 10/01/1996 to 01/01/1996, in yyyy-mm-dd format
start_date = datetime.date(1996, 1, 10)
end_date = datetime.date(1996, 1, 1)
expected_output = list(
map(lambda x: "1996-01-" + str(x + 1).zfill(2),
reversed(range(10))))
date_gen = ParamInjector.generate_daterange("%Y-%m-%d", start_date,
end_date, "D")
self.assertEqual(expected_output, list(date_gen))
# generate months from 01/05/1996 to 01/01/1996, in yyyy/mm format
start_date = datetime.date(1996, 5, 1)
end_date = datetime.date(1996, 1, 1)
expected_output = list(
map(lambda x: "1996/" + str(x + 1).zfill(2), reversed(range(5))))
date_gen = ParamInjector.generate_daterange("%Y/%m", start_date,
end_date, "M")
self.assertEqual(expected_output, list(date_gen))
# generate years from 01/01/2005 to 01/01/1996, in yy format
start_date = datetime.date(2005, 1, 1)
end_date = datetime.date(1996, 1, 1)
expected_output = list(
map(lambda x: str(x)[-2:], reversed(range(1996, 2006))))
date_gen = ParamInjector.generate_daterange("%y", start_date, end_date)
self.assertEqual(expected_output, list(date_gen))
def test_daterange_generator_error_empty_format(self):
"""
Tests the error case when the given date format is empty
"""
start_date = datetime.date(1996, 1, 1)
end_date = datetime.date(2005, 1, 1)
date_gen = ParamInjector.generate_daterange("", start_date, end_date)
self.assertRaises(ValueError, list, date_gen)
def test_daterange_generator_limits(self):
"""
Tests the date range generator's behaviour in some corner cases where
the start/end dates could cause off-by-one errors
"""
# generate months from 01/01/1996 to 30/04/1996, in yyyy/mm format
# should include Jan to Apr, without May
start_date = datetime.date(1996, 1, 1)
end_date = datetime.date(1996, 4, 30)
expected_output = list(
map(lambda x: "1996/" + str(x + 1).zfill(2), range(4)))
date_gen = ParamInjector.generate_daterange("%Y/%m", start_date,
end_date, "M")
self.assertEqual(expected_output, list(date_gen))
# generate years from 01/01/1996 to 31/12/2004, in yy format
# should include 1996 to 2004, without 2005
start_date = datetime.date(1996, 1, 1)
end_date = datetime.date(2004, 12, 31)
expected_output = list(map(lambda x: str(x)[-2:], range(1996, 2005)))
date_gen = ParamInjector.generate_daterange("%y", start_date, end_date)
self.assertEqual(expected_output, list(date_gen))
def create_parameter_generators(probe, parameter_handlers, filter_limits=True):
"""
Loads the parameter information and creates a list of the respective
generators from the ParamInjector module, while filtering the ranges as
necessary
"""
url_injectors = []
initial_values = []
for i in [1, 2]:
# We run this code twice: the first pass will get the initial
# values for each parameter, which is used in the second pass to
# filter the ends of the limits as required
# I couldn't find a nicer way to do this
if not filter_limits and i == 2:
# Don't filter limits unless required
break
for param_index, param in enumerate(parameter_handlers):
param_type = param['parameter_type']
param_gen = None
if i == 2 and not param['filter_range']:
# We are running the "filtering" pass but this parameter
# should not be filtered
continue
entries_list = []
cons_misses = None
if i == 2:
# Configure the list of extra parameters for the range
# inference
entries_list = initial_values.copy()
entries_list[param_index] = None
cons_misses = int(param['cons_misses'])
if param_type == "process_code":
PROCESS_FORMAT = '{:07d}-{:02d}.{:04d}.{}.{:02d}.{:04d}'
first_year = int(param['first_year_proc_param'])
last_year = int(param['last_year_proc_param'])
segment_ids = param['segment_ids_proc_param'].split(",")
court_ids = param['court_ids_proc_param'].split(",")
origin_ids = param['origin_ids_proc_param'].split(",")
# turn string lists into integers
segment_ids = list(map(int, segment_ids))
court_ids = list(map(int, court_ids))
origin_ids = list(map(int, origin_ids))
max_seq = 9999999
if i == 2:
# Filter the process_code range
max_seq = RangeInference.filter_process_code(
first_year,
last_year,
segment_ids,
court_ids,
origin_ids,
probe,
entries_list,
cons_misses=cons_misses)
subparam_list = [
# sequential identifier
(0, max_seq),
# year
(first_year, last_year),
# segment identifiers
segment_ids,
# court identifiers
court_ids,
# origin identifiers
origin_ids
]
param_gen = ParamInjector.generate_format(
code_format=PROCESS_FORMAT,
param_limits=subparam_list,
verif=ParamInjector.process_code_verification,
verif_index=1)
elif param_type == "number_seq":
begin = int(param['first_num_param'])
end = int(param['last_num_param'])
if i == 2:
# Filter the number range
end = RangeInference.filter_numeric_range(
begin,
end,
probe,
entries_list,
cons_misses=cons_misses)
param_gen = ParamInjector.generate_num_sequence(
first=begin,
last=end,
step=int(param['step_num_param']),
leading=param['leading_num_param'],
)
elif param_type == 'date_seq':
begin = datetime.date.fromisoformat(
param['start_date_date_param'])
end = datetime.date.fromisoformat(param['end_date_date_param'])
frequency = param['frequency_date_param']
date_format = param['date_format_date_param']
if i == 2:
# Filter the date range
end = RangeInference.filter_daterange(
begin,
end,
probe,
frequency,
date_format,
entries_list,
cons_misses=cons_misses)
param_gen = ParamInjector.generate_daterange(
date_format=date_format,
start_date=begin,
end_date=end,
frequency=frequency,
)
elif param_type == 'alpha_seq':
# We don't do anything diferent here if it's the second
# pass, since alphabetic sequences can't be filtered
length = int(param['length_alpha_param'])
num_words = int(param['num_words_alpha_param'])
no_upper = param['no_upper_alpha_param']
param_gen = ParamInjector.generate_alpha(length=length,
num_words=num_words,
no_upper=no_upper)
elif param_type == 'value_list':
# No filtering applied to this parameter
list_values = param['value_list_param']
param_gen = ParamInjector.generate_list(elements=list_values)
elif param_type == 'const_value':
# No filtering applied to this parameter
const_value = param['value_const_param']
param_gen = ParamInjector.generate_constant(value=const_value)
else:
raise ValueError(f"Invalid parameter type: {param_type}")
if i == 2 and param_gen is not None:
# We have filtered the range for this parameter, and should
# update the generator in the list
url_injectors[param_index] = param_gen
else:
# Create a copy of the generator, to extract the first
# value. After that, add to the list of parameter
# generators
param_gen, param_gen_first = itertools.tee(param_gen)
initial_values.append(next(param_gen_first))
url_injectors.append(param_gen)
return url_injectors