def test_hex_fail(self): processor = ClientParsing.StringProcessor() conversions = [(ClientParsing.STRING_CONVERSION_DECODE, 'hex')] string_converter = ClientParsing.StringConverter( conversions=conversions) # processing_steps = [] processing_steps.append(string_converter) processing_steps.append( ClientParsing.StringMatch( match_type=ClientParsing.STRING_MATCH_FLEXIBLE, match_value=ClientParsing.NUMERIC)) processor.SetProcessingSteps(processing_steps) self.assertEqual(processor.ProcessStrings(['0123456789abcdef']), []) # processing_steps = [] processing_steps.append(string_converter) processing_steps.append(ClientParsing.StringSplitter(separator=',')) processor.SetProcessingSteps(processing_steps) self.assertEqual(processor.ProcessStrings(['0123456789abcdef']), [])
def test_basics(self): processor = ClientParsing.StringProcessor() self.assertEqual(processor.ProcessStrings([]), []) self.assertEqual(processor.ProcessStrings(['test']), ['test']) self.assertEqual( processor.ProcessStrings(['test', 'test', '', 'test2']), ['test', 'test', '', 'test2']) processing_steps = [] processing_steps.append( ClientParsing.StringSplitter(separator=',', max_splits=2)) processing_steps.append( ClientParsing.StringMatch( match_type=ClientParsing.STRING_MATCH_FLEXIBLE, match_value=ClientParsing.NUMERIC)) conversions = [(ClientParsing.STRING_CONVERSION_APPEND_TEXT, 'abc')] processing_steps.append( ClientParsing.StringConverter(conversions=conversions)) processor.SetProcessingSteps(processing_steps) expected_result = ['1abc', '123abc'] self.assertEqual(processor.ProcessStrings(['1,a,2,3', 'test', '123']), expected_result)
def test_url_classes(self): name = 'test' url_type = HC.URL_TYPE_POST preferred_scheme = 'https' netloc = 'testbooru.cx' alphabetise_get_parameters = True match_subdomains = False keep_matched_subdomains = False can_produce_multiple_files = False should_be_associated_with_files = True path_components = [] path_components.append((ClientParsing.StringMatch( match_type=ClientParsing.STRING_MATCH_FIXED, match_value='post', example_string='post'), None)) path_components.append((ClientParsing.StringMatch( match_type=ClientParsing.STRING_MATCH_FIXED, match_value='page.php', example_string='page.php'), None)) parameters = {} parameters['s'] = (ClientParsing.StringMatch( match_type=ClientParsing.STRING_MATCH_FIXED, match_value='view', example_string='view'), None) parameters['id'] = (ClientParsing.StringMatch( match_type=ClientParsing.STRING_MATCH_FLEXIBLE, match_value=ClientParsing.NUMERIC, example_string='123456'), None) send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_IF_PROVIDED referral_url_converter = None gallery_index_type = None gallery_index_identifier = None gallery_index_delta = 1 example_url = 'https://testbooru.cx/post/page.php?id=123456&s=view' # referral_url = 'https://testbooru.cx/gallery/tags=samus_aran' good_url = 'https://testbooru.cx/post/page.php?id=123456&s=view' unnormalised_good_url_1 = 'https://testbooru.cx/post/page.php?id=123456&s=view&additional_gumpf=stuff' unnormalised_good_url_2 = 'https://testbooru.cx/post/page.php?s=view&id=123456' bad_url = 'https://wew.lad/123456' url_class = ClientNetworkingDomain.URLClass( name, url_type=url_type, preferred_scheme=preferred_scheme, netloc=netloc, path_components=path_components, parameters=parameters, send_referral_url=send_referral_url, referral_url_converter=referral_url_converter, gallery_index_type=gallery_index_type, gallery_index_identifier=gallery_index_identifier, gallery_index_delta=gallery_index_delta, example_url=example_url) url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains, alphabetise_get_parameters, can_produce_multiple_files, should_be_associated_with_files) self.assertEqual(url_class.Matches(example_url), True) self.assertEqual(url_class.Matches(bad_url), False) self.assertEqual(url_class.Normalise(unnormalised_good_url_1), good_url) self.assertEqual(url_class.Normalise(unnormalised_good_url_2), good_url) self.assertEqual(url_class.GetReferralURL(good_url, referral_url), referral_url) self.assertEqual(url_class.GetReferralURL(good_url, None), None) # alphabetise_get_parameters = False url_class = ClientNetworkingDomain.URLClass( name, url_type=url_type, preferred_scheme=preferred_scheme, netloc=netloc, path_components=path_components, parameters=parameters, send_referral_url=send_referral_url, referral_url_converter=referral_url_converter, gallery_index_type=gallery_index_type, gallery_index_identifier=gallery_index_identifier, gallery_index_delta=gallery_index_delta, example_url=example_url) url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains, alphabetise_get_parameters, can_produce_multiple_files, should_be_associated_with_files) self.assertEqual(url_class.Normalise(unnormalised_good_url_2), unnormalised_good_url_2) alphabetise_get_parameters = True # send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_NEVER url_class = ClientNetworkingDomain.URLClass( name, url_type=url_type, preferred_scheme=preferred_scheme, netloc=netloc, path_components=path_components, parameters=parameters, send_referral_url=send_referral_url, referral_url_converter=referral_url_converter, gallery_index_type=gallery_index_type, gallery_index_identifier=gallery_index_identifier, gallery_index_delta=gallery_index_delta, example_url=example_url) url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains, alphabetise_get_parameters, can_produce_multiple_files, should_be_associated_with_files) self.assertEqual(url_class.GetReferralURL(good_url, referral_url), None) self.assertEqual(url_class.GetReferralURL(good_url, None), None) # converted_referral_url = good_url.replace('testbooru.cx', 'replace.com') transformations = [] transformations.append((ClientParsing.STRING_TRANSFORMATION_REGEX_SUB, ('testbooru.cx', 'replace.com'))) referral_url_converter = ClientParsing.StringConverter( transformations, good_url) send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_CONVERTER_IF_NONE_PROVIDED url_class = ClientNetworkingDomain.URLClass( name, url_type=url_type, preferred_scheme=preferred_scheme, netloc=netloc, path_components=path_components, parameters=parameters, send_referral_url=send_referral_url, referral_url_converter=referral_url_converter, gallery_index_type=gallery_index_type, gallery_index_identifier=gallery_index_identifier, gallery_index_delta=gallery_index_delta, example_url=example_url) url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains, alphabetise_get_parameters, can_produce_multiple_files, should_be_associated_with_files) self.assertEqual(url_class.GetReferralURL(good_url, referral_url), referral_url) self.assertEqual(url_class.GetReferralURL(good_url, None), converted_referral_url) # send_referral_url = ClientNetworkingDomain.SEND_REFERRAL_URL_ONLY_CONVERTER url_class = ClientNetworkingDomain.URLClass( name, url_type=url_type, preferred_scheme=preferred_scheme, netloc=netloc, path_components=path_components, parameters=parameters, send_referral_url=send_referral_url, referral_url_converter=referral_url_converter, gallery_index_type=gallery_index_type, gallery_index_identifier=gallery_index_identifier, gallery_index_delta=gallery_index_delta, example_url=example_url) url_class.SetURLBooleans(match_subdomains, keep_matched_subdomains, alphabetise_get_parameters, can_produce_multiple_files, should_be_associated_with_files) self.assertEqual(url_class.GetReferralURL(good_url, referral_url), converted_referral_url) self.assertEqual(url_class.GetReferralURL(good_url, None), converted_referral_url)
def test_basics( self ): string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_BEGINNING, 1 ) ] ) self.assertEqual( string_converter.Convert( '0123456789' ), '123456789' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_END, 1 ) ] ) self.assertEqual( string_converter.Convert( '0123456789' ), '012345678' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_BEGINNING, 7 ) ] ) self.assertEqual( string_converter.Convert( '0123456789' ), '0123456' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_END, 7 ) ] ) self.assertEqual( string_converter.Convert( '0123456789' ), '3456789' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_PREPEND_TEXT, 'abc' ) ] ) self.assertEqual( string_converter.Convert( '0123456789' ), 'abc0123456789' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_APPEND_TEXT, 'xyz' ) ] ) self.assertEqual( string_converter.Convert( '0123456789' ), '0123456789xyz' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_ENCODE, 'url percent encoding' ) ] ) self.assertEqual( string_converter.Convert( '01234 56789' ), '01234%2056789' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_DECODE, 'url percent encoding' ) ] ) self.assertEqual( string_converter.Convert( '01234%2056789' ), '01234 56789' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_ENCODE, 'unicode escape characters' ) ] ) self.assertEqual( string_converter.Convert( '01234\u039456789' ), '01234\\u039456789' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_DECODE, 'unicode escape characters' ) ] ) self.assertEqual( string_converter.Convert( '01234\\u039456789' ), '01234\u039456789' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_ENCODE, 'html entities' ) ] ) self.assertEqual( string_converter.Convert( '01234&56789' ), '01234&56789' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_DECODE, 'html entities' ) ] ) self.assertEqual( string_converter.Convert( '01234&56789' ), '01234&56789' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_ENCODE, 'hex' ) ] ) self.assertEqual( string_converter.Convert( b'\xe5\xafW\xa6\x87\xf0\x89\x89O^\xce\xdeP\x04\x94X' ), 'e5af57a687f089894f5ecede50049458' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_ENCODE, 'base64' ) ] ) self.assertEqual( string_converter.Convert( b'\xe5\xafW\xa6\x87\xf0\x89\x89O^\xce\xdeP\x04\x94X' ), '5a9XpofwiYlPXs7eUASUWA==' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_REVERSE, None ) ] ) self.assertEqual( string_converter.Convert( '0123456789' ), '9876543210' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_REGEX_SUB, ( '\\d', 'd' ) ) ] ) self.assertEqual( string_converter.Convert( 'abc123' ), 'abcddd' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_DATE_DECODE, ( '%Y-%m-%d %H:%M:%S', HC.TIMEZONE_GMT, 0 ) ) ] ) self.assertEqual( string_converter.Convert( '1970-01-02 00:00:00' ), '86400' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_DATE_ENCODE, ( '%Y-%m-%d %H:%M:%S', 0 ) ) ] ) self.assertEqual( string_converter.Convert( '86400' ), '1970-01-02 00:00:00' ) # string_converter = ClientParsing.StringConverter( conversions = [ ( ClientParsing.STRING_CONVERSION_INTEGER_ADDITION, 5 ) ] ) self.assertEqual( string_converter.Convert( '4' ), '9' )
def test_compound( self ): conversions = [] conversions.append( ( ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_BEGINNING, 1 ) ) string_converter = ClientParsing.StringConverter( conversions = conversions ) self.assertEqual( string_converter.Convert( '0123456789' ), '123456789' ) # conversions.append( ( ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_END, 1 ) ) string_converter = ClientParsing.StringConverter( conversions = conversions ) self.assertEqual( string_converter.Convert( '0123456789' ), '12345678' ) # conversions.append( ( ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_BEGINNING, 7 ) ) string_converter = ClientParsing.StringConverter( conversions = conversions ) self.assertEqual( string_converter.Convert( '0123456789' ), '1234567' ) # conversions.append( ( ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_END, 6 ) ) string_converter = ClientParsing.StringConverter( conversions = conversions ) self.assertEqual( string_converter.Convert( '0123456789' ), '234567' ) # conversions.append( ( ClientParsing.STRING_CONVERSION_PREPEND_TEXT, 'abc' ) ) string_converter = ClientParsing.StringConverter( conversions = conversions ) self.assertEqual( string_converter.Convert( '0123456789' ), 'abc234567' ) # conversions.append( ( ClientParsing.STRING_CONVERSION_APPEND_TEXT, 'x z' ) ) string_converter = ClientParsing.StringConverter( conversions = conversions ) self.assertEqual( string_converter.Convert( '0123456789' ), 'abc234567x z' ) # conversions.append( ( ClientParsing.STRING_CONVERSION_ENCODE, 'url percent encoding' ) ) string_converter = ClientParsing.StringConverter( conversions = conversions ) self.assertEqual( string_converter.Convert( '0123456789' ), 'abc234567x%20z' ) # conversions.append( ( ClientParsing.STRING_CONVERSION_DECODE, 'url percent encoding' ) ) string_converter = ClientParsing.StringConverter( conversions = conversions ) self.assertEqual( string_converter.Convert( '0123456789' ), 'abc234567x z' ) # conversions.append( ( ClientParsing.STRING_CONVERSION_REVERSE, None ) ) string_converter = ClientParsing.StringConverter( conversions = conversions ) self.assertEqual( string_converter.Convert( '0123456789' ), 'z x765432cba' ) # conversions.append( ( ClientParsing.STRING_CONVERSION_REGEX_SUB, ( '\\d', 'd' ) ) ) string_converter = ClientParsing.StringConverter( conversions = conversions ) self.assertEqual( string_converter.Convert( '0123456789' ), 'z xddddddcba' )
def test_basics(self): conversions = [] conversions.append( (ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_BEGINNING, 1)) string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('0123456789'), '123456789') # conversions.append( (ClientParsing.STRING_CONVERSION_REMOVE_TEXT_FROM_END, 1)) string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('0123456789'), '12345678') # conversions.append( (ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_BEGINNING, 7)) string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('0123456789'), '1234567') # conversions.append( (ClientParsing.STRING_CONVERSION_CLIP_TEXT_FROM_END, 6)) string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('0123456789'), '234567') # conversions.append( (ClientParsing.STRING_CONVERSION_PREPEND_TEXT, 'abc')) string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('0123456789'), 'abc234567') # conversions.append( (ClientParsing.STRING_CONVERSION_APPEND_TEXT, 'x z')) string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('0123456789'), 'abc234567x z') # conversions.append( (ClientParsing.STRING_CONVERSION_ENCODE, 'url percent encoding')) string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('0123456789'), 'abc234567x%20z') # conversions.append( (ClientParsing.STRING_CONVERSION_DECODE, 'url percent encoding')) string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('0123456789'), 'abc234567x z') # conversions.append((ClientParsing.STRING_CONVERSION_REVERSE, None)) string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('0123456789'), 'z x765432cba') # conversions.append( (ClientParsing.STRING_CONVERSION_REGEX_SUB, ('\\d', 'd'))) string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('0123456789'), 'z xddddddcba') # conversions = [(ClientParsing.STRING_CONVERSION_DATE_DECODE, ('%Y-%m-%d %H:%M:%S', HC.TIMEZONE_GMT, 0))] string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('1970-01-02 00:00:00'), '86400') # conversions = [(ClientParsing.STRING_CONVERSION_DATE_ENCODE, ('%Y-%m-%d %H:%M:%S', 0))] string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('86400'), '1970-01-02 00:00:00') # conversions = [(ClientParsing.STRING_CONVERSION_INTEGER_ADDITION, 5)] string_converter = ClientParsing.StringConverter( conversions=conversions) self.assertEqual(string_converter.Convert('4'), '9')