def test_alphabetic_generator_all(self):
"""
Tests generation of alphabetic search patterns upper and lowercase
"""
# one letter, one word
lowercase_letters = [chr(97 + x) for x in range(0, 26)]
uppercase_letters = [chr(65 + x) for x in range(0, 26)]
expected_output = list(map(lambda x: x + "*", lowercase_letters))
expected_output += list(map(lambda x: x + "*", uppercase_letters))
pattern_gen = ParamInjector.generate_alpha(1, 1, False)
self.assertEqual(expected_output, list(pattern_gen))
# two letters, one word
# check only the first 5 and last 5 entries
expected_output_first = ["aa*", "ab*", "ac*", "ad*", "ae*"]
expected_output_last = ["ZV*", "ZW*", "ZX*", "ZY*", "ZZ*"]
pattern_gen = ParamInjector.generate_alpha(2, 1, False)
# first 5 entries
output = [next(pattern_gen) for i in range(5)]
self.assertEqual(expected_output_first, output)
# last 5 entries
last_five = deque(pattern_gen, maxlen=5)
output = [last_five.popleft() for i in range(5)]
self.assertEqual(expected_output_last, output)
# two letters, two words
# check only the first 5 and last 5 entries
expected_output_first = [
"aa* aa*", "aa* ab*", "aa* ac*", "aa* ad*", "aa* ae*"
]
expected_output_last = [
"ZZ* ZV*", "ZZ* ZW*", "ZZ* ZX*", "ZZ* ZY*", "ZZ* ZZ*"
]
pattern_gen = ParamInjector.generate_alpha(2, 2, False)
# first 5 entries
output = [next(pattern_gen) for i in range(5)]
self.assertEqual(expected_output_first, output)
# last 5 entries
last_five = deque(pattern_gen, maxlen=5)
output = [last_five.popleft() for i in range(5)]
self.assertEqual(expected_output_last, output)
def test_alphabetic_generator_error_corner_cases(self):
"""
Tests some corner cases for the alphabetic generator
"""
# two words with zero length
pattern_gen = ParamInjector.generate_alpha(2, 0)
self.assertRaises(ValueError, list, pattern_gen)
# zero words with length two
pattern_gen = ParamInjector.generate_alpha(0, 2)
self.assertRaises(ValueError, list, pattern_gen)
# negative value for length
pattern_gen = ParamInjector.generate_alpha(0, -2)
self.assertRaises(ValueError, list, pattern_gen)
# negative value for word count
pattern_gen = ParamInjector.generate_alpha(-2, 0)
self.assertRaises(ValueError, list, pattern_gen)
def test_alphabetic_generator_lower(self):
"""
Tests generation of alphabetic search patterns (all lowercase)
"""
# one letter, one word
lowercase_letters = [chr(97 + x) for x in range(0, 26)]
expected_output = list(map(lambda x: x + "*", lowercase_letters))
pattern_gen = ParamInjector.generate_alpha(1, 1)
self.assertEqual(expected_output, list(pattern_gen))
# two letters, one word
# check only the first 5 and last 5 entries
expected_output_first = ["aa*", "ab*", "ac*", "ad*", "ae*"]
expected_output_last = ["zv*", "zw*", "zx*", "zy*", "zz*"]
pattern_gen = ParamInjector.generate_alpha(2, 1)
# first 5 entries
output = [next(pattern_gen) for i in range(5)]
self.assertEqual(expected_output_first, output)
# last 5 entries
last_five = deque(pattern_gen, maxlen=5)
output = [last_five.popleft() for i in range(5)]
self.assertEqual(expected_output_last, output)
# two letters, two words
# check only the first 5 and last 5 entries
expected_output_first = [
"aa* aa*", "aa* ab*", "aa* ac*", "aa* ad*", "aa* ae*"
]
expected_output_last = [
"zz* zv*", "zz* zw*", "zz* zx*", "zz* zy*", "zz* zz*"
]
pattern_gen = ParamInjector.generate_alpha(2, 2)
# first 5 entries
output = [next(pattern_gen) for i in range(5)]
self.assertEqual(expected_output_first, output)
# last 5 entries
last_five = deque(pattern_gen, maxlen=5)
output = [last_five.popleft() for i in range(5)]
self.assertEqual(expected_output_last, output)
def create_parameter_generators(probe, parameter_handlers, filter_limits=True):
"""
Loads the parameter information and creates a list of the respective
generators from the ParamInjector module, while filtering the ranges as
necessary
"""
url_injectors = []
initial_values = []
for i in [1, 2]:
# We run this code twice: the first pass will get the initial
# values for each parameter, which is used in the second pass to
# filter the ends of the limits as required
# I couldn't find a nicer way to do this
if not filter_limits and i == 2:
# Don't filter limits unless required
break
for param_index, param in enumerate(parameter_handlers):
param_type = param['parameter_type']
param_gen = None
if i == 2 and not param['filter_range']:
# We are running the "filtering" pass but this parameter
# should not be filtered
continue
entries_list = []
cons_misses = None
if i == 2:
# Configure the list of extra parameters for the range
# inference
entries_list = initial_values.copy()
entries_list[param_index] = None
cons_misses = int(param['cons_misses'])
if param_type == "process_code":
PROCESS_FORMAT = '{:07d}-{:02d}.{:04d}.{}.{:02d}.{:04d}'
first_year = int(param['first_year_proc_param'])
last_year = int(param['last_year_proc_param'])
segment_ids = param['segment_ids_proc_param'].split(",")
court_ids = param['court_ids_proc_param'].split(",")
origin_ids = param['origin_ids_proc_param'].split(",")
# turn string lists into integers
segment_ids = list(map(int, segment_ids))
court_ids = list(map(int, court_ids))
origin_ids = list(map(int, origin_ids))
max_seq = 9999999
if i == 2:
# Filter the process_code range
max_seq = RangeInference.filter_process_code(
first_year,
last_year,
segment_ids,
court_ids,
origin_ids,
probe,
entries_list,
cons_misses=cons_misses)
subparam_list = [
# sequential identifier
(0, max_seq),
# year
(first_year, last_year),
# segment identifiers
segment_ids,
# court identifiers
court_ids,
# origin identifiers
origin_ids
]
param_gen = ParamInjector.generate_format(
code_format=PROCESS_FORMAT,
param_limits=subparam_list,
verif=ParamInjector.process_code_verification,
verif_index=1)
elif param_type == "number_seq":
begin = int(param['first_num_param'])
end = int(param['last_num_param'])
if i == 2:
# Filter the number range
end = RangeInference.filter_numeric_range(
begin,
end,
probe,
entries_list,
cons_misses=cons_misses)
param_gen = ParamInjector.generate_num_sequence(
first=begin,
last=end,
step=int(param['step_num_param']),
leading=param['leading_num_param'],
)
elif param_type == 'date_seq':
begin = datetime.date.fromisoformat(
param['start_date_date_param'])
end = datetime.date.fromisoformat(param['end_date_date_param'])
frequency = param['frequency_date_param']
date_format = param['date_format_date_param']
if i == 2:
# Filter the date range
end = RangeInference.filter_daterange(
begin,
end,
probe,
frequency,
date_format,
entries_list,
cons_misses=cons_misses)
param_gen = ParamInjector.generate_daterange(
date_format=date_format,
start_date=begin,
end_date=end,
frequency=frequency,
)
elif param_type == 'alpha_seq':
# We don't do anything diferent here if it's the second
# pass, since alphabetic sequences can't be filtered
length = int(param['length_alpha_param'])
num_words = int(param['num_words_alpha_param'])
no_upper = param['no_upper_alpha_param']
param_gen = ParamInjector.generate_alpha(length=length,
num_words=num_words,
no_upper=no_upper)
elif param_type == 'value_list':
# No filtering applied to this parameter
list_values = param['value_list_param']
param_gen = ParamInjector.generate_list(elements=list_values)
elif param_type == 'const_value':
# No filtering applied to this parameter
const_value = param['value_const_param']
param_gen = ParamInjector.generate_constant(value=const_value)
else:
raise ValueError(f"Invalid parameter type: {param_type}")
if i == 2 and param_gen is not None:
# We have filtered the range for this parameter, and should
# update the generator in the list
url_injectors[param_index] = param_gen
else:
# Create a copy of the generator, to extract the first
# value. After that, add to the list of parameter
# generators
param_gen, param_gen_first = itertools.tee(param_gen)
initial_values.append(next(param_gen_first))
url_injectors.append(param_gen)
return url_injectors
