def test_url_classes(self):
name = 'test'
url_type = HC.URL_TYPE_POST
preferred_scheme = 'https'
netloc = 'testbooru.cx'
alphabetise_get_parameters = True
match_subdomains = False
keep_matched_subdomains = False
can_produce_multiple_files = False
should_be_associated_with_files = True
path_components = []
path_components.append((ClientParsing.StringMatch(
match_type=ClientParsing.STRING_MATCH_FIXED,
match_value='post',
example_string='post'), None))
path_components.append((ClientParsing.StringMatch(
match_type=ClientParsing.STRING_MATCH_FIXED,
match_value='page.php',
example_string='page.php'), None))
parameters = {}
parameters['s'] = (ClientParsing.StringMatch(
match_type=ClientParsing.STRING_MATCH_FIXED,
match_value='view',
example_string='view'), None)
parameters['id'] = (ClientParsing.StringMatch(
match_type=ClientParsing.STRING_MATCH_FLEXIBLE,
match_value=ClientParsing.NUMERIC,
example_string='123456'), None)
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_IF_PROVIDED
referral_url_converter = None
gallery_index_type = None
gallery_index_identifier = None
gallery_index_delta = 1
example_url = 'https://testbooru.cx/post/page.php?id=123456&s=view'
#
referral_url = 'https://testbooru.cx/gallery/tags=samus_aran'
good_url = 'https://testbooru.cx/post/page.php?id=123456&s=view'
unnormalised_good_url_1 = 'https://testbooru.cx/post/page.php?id=123456&s=view&additional_gumpf=stuff'
unnormalised_good_url_2 = 'https://testbooru.cx/post/page.php?s=view&id=123456'
bad_url = 'https://wew.lad/123456'
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files)
self.assertEqual(url_class.Matches(example_url), True)
self.assertEqual(url_class.Matches(bad_url), False)
self.assertEqual(url_class.Normalise(unnormalised_good_url_1),
good_url)
self.assertEqual(url_class.Normalise(unnormalised_good_url_2),
good_url)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
referral_url)
self.assertEqual(url_class.GetReferralURL(good_url, None), None)
#
alphabetise_get_parameters = False
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files)
self.assertEqual(url_class.Normalise(unnormalised_good_url_2),
unnormalised_good_url_2)
alphabetise_get_parameters = True
#
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_NEVER
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
None)
self.assertEqual(url_class.GetReferralURL(good_url, None), None)
#
converted_referral_url = good_url.replace('testbooru.cx',
'replace.com')
transformations = []
transformations.append((ClientParsing.STRING_TRANSFORMATION_REGEX_SUB,
('testbooru.cx', 'replace.com')))
referral_url_converter = ClientParsing.StringConverter(
transformations, good_url)
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
referral_url)
self.assertEqual(url_class.GetReferralURL(good_url, None),
converted_referral_url)
#
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_CONVERTER
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
converted_referral_url)
self.assertEqual(url_class.GetReferralURL(good_url, None),
converted_referral_url)
def test_url_classes(self):
name = 'test'
url_type = HC.URL_TYPE_POST
preferred_scheme = 'https'
netloc = 'testbooru.cx'
alphabetise_get_parameters = True
match_subdomains = False
keep_matched_subdomains = False
can_produce_multiple_files = False
should_be_associated_with_files = True
keep_fragment = False
path_components = []
path_components.append((ClientStrings.StringMatch(
match_type=ClientStrings.STRING_MATCH_FIXED,
match_value='post',
example_string='post'), None))
path_components.append((ClientStrings.StringMatch(
match_type=ClientStrings.STRING_MATCH_FIXED,
match_value='page.php',
example_string='page.php'), None))
parameters = {}
parameters['s'] = (ClientStrings.StringMatch(
match_type=ClientStrings.STRING_MATCH_FIXED,
match_value='view',
example_string='view'), None)
parameters['id'] = (ClientStrings.StringMatch(
match_type=ClientStrings.STRING_MATCH_FLEXIBLE,
match_value=ClientStrings.NUMERIC,
example_string='123456'), None)
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_IF_PROVIDED
referral_url_converter = None
gallery_index_type = None
gallery_index_identifier = None
gallery_index_delta = 1
example_url = 'https://testbooru.cx/post/page.php?id=123456&s=view'
#
referral_url = 'https://testbooru.cx/gallery/tags=samus_aran'
good_url = 'https://testbooru.cx/post/page.php?id=123456&s=view'
unnormalised_good_url_1 = 'https://testbooru.cx/post/page.php?id=123456&s=view&additional_gumpf=stuff'
unnormalised_good_url_2 = 'https://testbooru.cx/post/page.php?s=view&id=123456'
bad_url = 'https://wew.lad/123456'
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files,
keep_fragment)
self.assertEqual(url_class.Matches(example_url), True)
self.assertEqual(url_class.Matches(bad_url), False)
self.assertEqual(url_class.Normalise(unnormalised_good_url_1),
good_url)
self.assertEqual(url_class.Normalise(unnormalised_good_url_2),
good_url)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
referral_url)
self.assertEqual(url_class.GetReferralURL(good_url, None), None)
#
alphabetise_get_parameters = False
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files,
keep_fragment)
self.assertEqual(url_class.Normalise(unnormalised_good_url_2),
unnormalised_good_url_2)
alphabetise_get_parameters = True
#
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_NEVER
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files,
keep_fragment)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
None)
self.assertEqual(url_class.GetReferralURL(good_url, None), None)
#
converted_referral_url = good_url.replace('testbooru.cx',
'replace.com')
conversions = []
conversions.append((ClientStrings.STRING_CONVERSION_REGEX_SUB,
('testbooru.cx', 'replace.com')))
referral_url_converter = ClientStrings.StringConverter(
conversions, good_url)
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files,
keep_fragment)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
referral_url)
self.assertEqual(url_class.GetReferralURL(good_url, None),
converted_referral_url)
#
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_CONVERTER
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files,
keep_fragment)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
converted_referral_url)
self.assertEqual(url_class.GetReferralURL(good_url, None),
converted_referral_url)
# fragment test
name = 'mega test'
url_type = HC.URL_TYPE_POST
preferred_scheme = 'https'
netloc = 'mega.nz'
alphabetise_get_parameters = True
match_subdomains = False
keep_matched_subdomains = False
can_produce_multiple_files = True
should_be_associated_with_files = True
path_components = []
path_components.append((ClientStrings.StringMatch(
match_type=ClientStrings.STRING_MATCH_FIXED,
match_value='file',
example_string='file'), None))
path_components.append((ClientStrings.StringMatch(
match_type=ClientStrings.STRING_MATCH_ANY), None))
parameters = {}
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_IF_PROVIDED
referral_url_converter = None
gallery_index_type = None
gallery_index_identifier = None
gallery_index_delta = 1
example_url = 'https://mega.nz/file/KxJHVKhT#0JPvygZDQcjBHrTWWECaDyNfXAFDyNZyE3Uonif5j-w'
keep_fragment = False
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files,
keep_fragment)
self.assertEqual(url_class.Normalise(example_url),
'https://mega.nz/file/KxJHVKhT')
keep_fragment = True
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files,
keep_fragment)
self.assertEqual(url_class.Normalise(example_url), example_url)
# single-value params test
single_value_good_url = 'https://testbooru.cx/post/page.php?id=123456&token&s=view'
single_value_bad_url = 'https://testbooru.cx/post/page.php?id=123456&bad_token&s=view'
single_value_missing_url = 'https://testbooru.cx/post/page.php?id=123456&s=view'
single_value_good_url_multiple = 'https://testbooru.cx/post/page.php?id=123456&token1&token2&s=view&token0'
single_value_good_url_alphabetical_normalised = 'https://testbooru.cx/post/page.php?id=123456&s=view&token'
single_value_good_url_multiple_alphabetical_normalised = 'https://testbooru.cx/post/page.php?id=123456&s=view&token0&token1&token2'
name = 'single value lad'
url_type = HC.URL_TYPE_POST
preferred_scheme = 'https'
netloc = 'testbooru.cx'
alphabetise_get_parameters = True
match_subdomains = False
keep_matched_subdomains = False
can_produce_multiple_files = False
should_be_associated_with_files = True
keep_fragment = False
path_components = []
path_components.append((ClientStrings.StringMatch(
match_type=ClientStrings.STRING_MATCH_FIXED,
match_value='post',
example_string='post'), None))
path_components.append((ClientStrings.StringMatch(
match_type=ClientStrings.STRING_MATCH_FIXED,
match_value='page.php',
example_string='page.php'), None))
parameters = {}
parameters['s'] = (ClientStrings.StringMatch(
match_type=ClientStrings.STRING_MATCH_FIXED,
match_value='view',
example_string='view'), None)
parameters['id'] = (ClientStrings.StringMatch(
match_type=ClientStrings.STRING_MATCH_FLEXIBLE,
match_value=ClientStrings.NUMERIC,
example_string='123456'), None)
has_single_value_parameters = True
single_value_parameters_string_match = ClientStrings.StringMatch(
match_type=ClientStrings.STRING_MATCH_REGEX,
match_value='^token.*',
example_string='token1')
example_url = single_value_good_url
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
has_single_value_parameters=has_single_value_parameters,
single_value_parameters_string_match=
single_value_parameters_string_match,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files,
keep_fragment)
self.assertEqual(url_class.Normalise(single_value_good_url),
single_value_good_url_alphabetical_normalised)
self.assertEqual(
url_class.Normalise(single_value_good_url_multiple),
single_value_good_url_multiple_alphabetical_normalised)
self.assertEqual(url_class.Matches(single_value_good_url), True)
self.assertEqual(
url_class.Matches(single_value_good_url_alphabetical_normalised),
True)
self.assertEqual(url_class.Matches(single_value_good_url_multiple),
True)
self.assertEqual(
url_class.Matches(
single_value_good_url_multiple_alphabetical_normalised), True)
self.assertEqual(url_class.Matches(single_value_bad_url), False)
self.assertEqual(url_class.Matches(single_value_missing_url), False)
url_class.SetAlphabetiseGetParameters(False)
self.assertEqual(url_class.Normalise(single_value_good_url),
single_value_good_url)
self.assertEqual(url_class.Normalise(single_value_good_url_multiple),
single_value_good_url_multiple)
self.assertEqual(url_class.Matches(single_value_good_url), True)
self.assertEqual(
url_class.Matches(single_value_good_url_alphabetical_normalised),
True)
self.assertEqual(url_class.Matches(single_value_good_url_multiple),
True)
self.assertEqual(
url_class.Matches(
single_value_good_url_multiple_alphabetical_normalised), True)
self.assertEqual(url_class.Matches(single_value_bad_url), False)
self.assertEqual(url_class.Matches(single_value_missing_url), False)