def test_hex_fail(self):
processor = ClientParsing.StringProcessor()
conversions = [(ClientParsing.STRING_CONVERSION_DECODE, 'hex')]
string_converter = ClientParsing.StringConverter(
conversions=conversions)
#
processing_steps = []
processing_steps.append(string_converter)
processing_steps.append(
ClientParsing.StringMatch(
match_type=ClientParsing.STRING_MATCH_FLEXIBLE,
match_value=ClientParsing.NUMERIC))
processor.SetProcessingSteps(processing_steps)
self.assertEqual(processor.ProcessStrings(['0123456789abcdef']), [])
#
processing_steps = []
processing_steps.append(string_converter)
processing_steps.append(ClientParsing.StringSplitter(separator=','))
processor.SetProcessingSteps(processing_steps)
self.assertEqual(processor.ProcessStrings(['0123456789abcdef']), [])
def test_basics(self):
processor = ClientParsing.StringProcessor()
self.assertEqual(processor.ProcessStrings([]), [])
self.assertEqual(processor.ProcessStrings(['test']), ['test'])
self.assertEqual(
processor.ProcessStrings(['test', 'test', '', 'test2']),
['test', 'test', '', 'test2'])
processing_steps = []
processing_steps.append(
ClientParsing.StringSplitter(separator=',', max_splits=2))
processing_steps.append(
ClientParsing.StringMatch(
match_type=ClientParsing.STRING_MATCH_FLEXIBLE,
match_value=ClientParsing.NUMERIC))
conversions = [(ClientParsing.STRING_CONVERSION_APPEND_TEXT, 'abc')]
processing_steps.append(
ClientParsing.StringConverter(conversions=conversions))
processor.SetProcessingSteps(processing_steps)
expected_result = ['1abc', '123abc']
self.assertEqual(processor.ProcessStrings(['1,a,2,3', 'test', '123']),
expected_result)
def test_url_classes(self):
name = 'test'
url_type = HC.URL_TYPE_POST
preferred_scheme = 'https'
netloc = 'testbooru.cx'
alphabetise_get_parameters = True
match_subdomains = False
keep_matched_subdomains = False
can_produce_multiple_files = False
should_be_associated_with_files = True
path_components = []
path_components.append((ClientParsing.StringMatch(
match_type=ClientParsing.STRING_MATCH_FIXED,
match_value='post',
example_string='post'), None))
path_components.append((ClientParsing.StringMatch(
match_type=ClientParsing.STRING_MATCH_FIXED,
match_value='page.php',
example_string='page.php'), None))
parameters = {}
parameters['s'] = (ClientParsing.StringMatch(
match_type=ClientParsing.STRING_MATCH_FIXED,
match_value='view',
example_string='view'), None)
parameters['id'] = (ClientParsing.StringMatch(
match_type=ClientParsing.STRING_MATCH_FLEXIBLE,
match_value=ClientParsing.NUMERIC,
example_string='123456'), None)
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_IF_PROVIDED
referral_url_converter = None
gallery_index_type = None
gallery_index_identifier = None
gallery_index_delta = 1
example_url = 'https://testbooru.cx/post/page.php?id=123456&s=view'
#
referral_url = 'https://testbooru.cx/gallery/tags=samus_aran'
good_url = 'https://testbooru.cx/post/page.php?id=123456&s=view'
unnormalised_good_url_1 = 'https://testbooru.cx/post/page.php?id=123456&s=view&additional_gumpf=stuff'
unnormalised_good_url_2 = 'https://testbooru.cx/post/page.php?s=view&id=123456'
bad_url = 'https://wew.lad/123456'
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files)
self.assertEqual(url_class.Matches(example_url), True)
self.assertEqual(url_class.Matches(bad_url), False)
self.assertEqual(url_class.Normalise(unnormalised_good_url_1),
good_url)
self.assertEqual(url_class.Normalise(unnormalised_good_url_2),
good_url)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
referral_url)
self.assertEqual(url_class.GetReferralURL(good_url, None), None)
#
alphabetise_get_parameters = False
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files)
self.assertEqual(url_class.Normalise(unnormalised_good_url_2),
unnormalised_good_url_2)
alphabetise_get_parameters = True
#
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_NEVER
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
None)
self.assertEqual(url_class.GetReferralURL(good_url, None), None)
#
converted_referral_url = good_url.replace('testbooru.cx',
'replace.com')
transformations = []
transformations.append((ClientParsing.STRING_TRANSFORMATION_REGEX_SUB,
('testbooru.cx', 'replace.com')))
referral_url_converter = ClientParsing.StringConverter(
transformations, good_url)
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
referral_url)
self.assertEqual(url_class.GetReferralURL(good_url, None),
converted_referral_url)
#
send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_CONVERTER
url_class = ClientNetworkingDomain.URLClass(
name,
url_type=url_type,
preferred_scheme=preferred_scheme,
netloc=netloc,
path_components=path_components,
parameters=parameters,
send_referral_url=send_referral_url,
referral_url_converter=referral_url_converter,
gallery_index_type=gallery_index_type,
gallery_index_identifier=gallery_index_identifier,
gallery_index_delta=gallery_index_delta,
example_url=example_url)
url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains,
alphabetise_get_parameters,
can_produce_multiple_files,
should_be_associated_with_files)
self.assertEqual(url_class.GetReferralURL(good_url, referral_url),
converted_referral_url)
self.assertEqual(url_class.GetReferralURL(good_url, None),
converted_referral_url)
def test_basics( self ):
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_BEGINNING, 1 ) ] )
self.assertEqual( string_converter.Convert( '0123456789' ), '123456789' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_END, 1 ) ] )
self.assertEqual( string_converter.Convert( '0123456789' ), '012345678' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_BEGINNING, 7 ) ] )
self.assertEqual( string_converter.Convert( '0123456789' ), '0123456' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_END, 7 ) ] )
self.assertEqual( string_converter.Convert( '0123456789' ), '3456789' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_PREPEND_TEXT, 'abc' ) ] )
self.assertEqual( string_converter.Convert( '0123456789' ), 'abc0123456789' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_APPEND_TEXT, 'xyz' ) ] )
self.assertEqual( string_converter.Convert( '0123456789' ), '0123456789xyz' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_ENCODE, 'url percent encoding' ) ] )
self.assertEqual( string_converter.Convert( '01234 56789' ), '01234%2056789' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_DECODE, 'url percent encoding' ) ] )
self.assertEqual( string_converter.Convert( '01234%2056789' ), '01234 56789' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_ENCODE, 'unicode escape characters' ) ] )
self.assertEqual( string_converter.Convert( '01234\u039456789' ), '01234\\u039456789' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_DECODE, 'unicode escape characters' ) ] )
self.assertEqual( string_converter.Convert( '01234\\u039456789' ), '01234\u039456789' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_ENCODE, 'html entities' ) ] )
self.assertEqual( string_converter.Convert( '01234&56789' ), '01234&56789' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_DECODE, 'html entities' ) ] )
self.assertEqual( string_converter.Convert( '01234&56789' ), '01234&56789' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_ENCODE, 'hex' ) ] )
self.assertEqual( string_converter.Convert( b'\xe5\xafW\xa6\x87\xf0\x89\x89O^\xce\xdeP\x04\x94X' ), 'e5af57a687f089894f5ecede50049458' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_ENCODE, 'base64' ) ] )
self.assertEqual( string_converter.Convert( b'\xe5\xafW\xa6\x87\xf0\x89\x89O^\xce\xdeP\x04\x94X' ), '5a9XpofwiYlPXs7eUASUWA==' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_REVERSE, None ) ] )
self.assertEqual( string_converter.Convert( '0123456789' ), '9876543210' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_REGEX_SUB, ( '\\d', 'd' ) ) ] )
self.assertEqual( string_converter.Convert( 'abc123' ), 'abcddd' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_DATE_DECODE, ( '%Y-%m-%d %H:%M:%S', HC.TIMEZONE_GMT, 0 ) ) ] )
self.assertEqual( string_converter.Convert( '1970-01-02 00:00:00' ), '86400' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_DATE_ENCODE, ( '%Y-%m-%d %H:%M:%S', 0 ) ) ] )
self.assertEqual( string_converter.Convert( '86400' ), '1970-01-02 00:00:00' )
#
string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_INTEGER_ADDITION, 5 ) ] )
self.assertEqual( string_converter.Convert( '4' ), '9' )
def test_compound( self ):
conversions = []
conversions.append( ( ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_BEGINNING, 1 ) )
string_converter = ClientParsing.StringConverter( conversions = conversions )
self.assertEqual( string_converter.Convert( '0123456789' ), '123456789' )
#
conversions.append( ( ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_END, 1 ) )
string_converter = ClientParsing.StringConverter( conversions = conversions )
self.assertEqual( string_converter.Convert( '0123456789' ), '12345678' )
#
conversions.append( ( ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_BEGINNING, 7 ) )
string_converter = ClientParsing.StringConverter( conversions = conversions )
self.assertEqual( string_converter.Convert( '0123456789' ), '1234567' )
#
conversions.append( ( ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_END, 6 ) )
string_converter = ClientParsing.StringConverter( conversions = conversions )
self.assertEqual( string_converter.Convert( '0123456789' ), '234567' )
#
conversions.append( ( ClientParsing.STRING_CONVERSION_PREPEND_TEXT, 'abc' ) )
string_converter = ClientParsing.StringConverter( conversions = conversions )
self.assertEqual( string_converter.Convert( '0123456789' ), 'abc234567' )
#
conversions.append( ( ClientParsing.STRING_CONVERSION_APPEND_TEXT, 'x z' ) )
string_converter = ClientParsing.StringConverter( conversions = conversions )
self.assertEqual( string_converter.Convert( '0123456789' ), 'abc234567x z' )
#
conversions.append( ( ClientParsing.STRING_CONVERSION_ENCODE, 'url percent encoding' ) )
string_converter = ClientParsing.StringConverter( conversions = conversions )
self.assertEqual( string_converter.Convert( '0123456789' ), 'abc234567x%20z' )
#
conversions.append( ( ClientParsing.STRING_CONVERSION_DECODE, 'url percent encoding' ) )
string_converter = ClientParsing.StringConverter( conversions = conversions )
self.assertEqual( string_converter.Convert( '0123456789' ), 'abc234567x z' )
#
conversions.append( ( ClientParsing.STRING_CONVERSION_REVERSE, None ) )
string_converter = ClientParsing.StringConverter( conversions = conversions )
self.assertEqual( string_converter.Convert( '0123456789' ), 'z x765432cba' )
#
conversions.append( ( ClientParsing.STRING_CONVERSION_REGEX_SUB, ( '\\d', 'd' ) ) )
string_converter = ClientParsing.StringConverter( conversions = conversions )
self.assertEqual( string_converter.Convert( '0123456789' ), 'z xddddddcba' )
def test_basics(self):
conversions = []
conversions.append(
(ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_BEGINNING, 1))
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('0123456789'), '123456789')
#
conversions.append(
(ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_END, 1))
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('0123456789'), '12345678')
#
conversions.append(
(ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_BEGINNING, 7))
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('0123456789'), '1234567')
#
conversions.append(
(ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_END, 6))
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('0123456789'), '234567')
#
conversions.append(
(ClientParsing.STRING_CONVERSION_PREPEND_TEXT, 'abc'))
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('0123456789'), 'abc234567')
#
conversions.append(
(ClientParsing.STRING_CONVERSION_APPEND_TEXT, 'x z'))
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('0123456789'),
'abc234567x z')
#
conversions.append(
(ClientParsing.STRING_CONVERSION_ENCODE, 'url percent encoding'))
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('0123456789'),
'abc234567x%20z')
#
conversions.append(
(ClientParsing.STRING_CONVERSION_DECODE, 'url percent encoding'))
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('0123456789'),
'abc234567x z')
#
conversions.append((ClientParsing.STRING_CONVERSION_REVERSE, None))
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('0123456789'),
'z x765432cba')
#
conversions.append(
(ClientParsing.STRING_CONVERSION_REGEX_SUB, ('\\d', 'd')))
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('0123456789'),
'z xddddddcba')
#
conversions = [(ClientParsing.STRING_CONVERSION_DATE_DECODE,
('%Y-%m-%d %H:%M:%S', HC.TIMEZONE_GMT, 0))]
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('1970-01-02 00:00:00'),
'86400')
#
conversions = [(ClientParsing.STRING_CONVERSION_DATE_ENCODE,
('%Y-%m-%d %H:%M:%S', 0))]
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('86400'),
'1970-01-02 00:00:00')
#
conversions = [(ClientParsing.STRING_CONVERSION_INTEGER_ADDITION, 5)]
string_converter = ClientParsing.StringConverter(
conversions=conversions)
self.assertEqual(string_converter.Convert('4'), '9')