class Feed: def __init__(self) -> None: self._shortener = BitlyShortener(tokens=config.BITLY_TOKENS, max_cache_size=config.BITLY_SHORTENER_CACHE_SIZE) self._is_debug_logged = log.isEnabledFor(logging.DEBUG) @lru_cache(maxsize=config.LRU_CACHE_SIZE) def _output(self, text: bytes) -> bytes: try: xml = ElementTree.fromstring(text) except ElementTree.ParseError as exception: raise FeedError(f'Unable to parse URL content as XML: {exception}', 422) is_debug_logged = self._is_debug_logged for link_type in LINK_TYPES: link_elements = link_type.findall(xml) if not link_elements: log.debug('No %s link elements were found in XML.', link_type.NAME) continue log.debug('Found %s %s link elements in XML.', len(link_elements), link_type.NAME) long_urls = [element.link for element in link_elements] url_map = self._shortener.shorten_urls_to_dict(long_urls) for element in link_elements: long_url = element.link short_url = url_map[long_url] element.link = short_url if is_debug_logged: log.debug('Replaced %s with %s.', long_url, short_url) log.info('Output feed has %s items.', len(link_elements)) text_: bytes = ElementTree.tostring(xml) return text_ else: log.warning('No link elements were found in XML.') return text @property def cache_info(self) -> Dict[str, _CacheInfo]: info = {source.__qualname__: source.cache_info() for source in (self.feed, self._output)} info.update(self._shortener.cache_info) return info @ttl_cache(maxsize=config.TTL_CACHE_SIZE, ttl=config.TTL_CACHE_TTL) def feed(self, url: str) -> bytes: log.debug('Reading input feed having URL %s', url) request = Request(url, headers={'User-Agent': config.USER_AGENT}) try: response = urlopen(request, timeout=config.URL_TIMEOUT) text = response.read() except urllib.error.URLError as exception: raise FeedError(f'Unable to read URL: {exception}', 404) log.info('Input feed has size %s.', humanize_len(text)) if response.headers[config.CYCLE_DETECTION_HEADER_KEY] == config.CYCLE_DETECTION_HEADER_VALUE: raise FeedError(f'Cycle detected.', 400) text = self._output(text) log.info('Output feed has size %s.', humanize_len(text)) return text def log_cache_info(self): info = self.cache_info.items() info = '; '.join(f'{k}: h={v.hits},m={v.misses},ms={v.maxsize},cs={v.currsize}' for (k, v) in info) log.info('Cache info: %s', info)
def shorten_address(url): tokens_pool = ['a2039c392dd9545fc3df1002a4441d9dfc534797'] shortener = Shortener(tokens=tokens_pool, max_cache_size=8192) urls = [url] result = shortener.shorten_urls_to_dict(urls) return result
'https://arxiv.org/abs/1902.00659v1', 'https://arxiv.org/abs/1902.00655v1', 'https://arxiv.org/abs/1902.00626v1', 'https://arxiv.org/abs/1902.00624v1', 'https://arxiv.org/abs/1902.00604v1', 'https://arxiv.org/abs/1902.00577v1', 'https://arxiv.org/abs/1902.00541v1', ] BITLY_URLs = [ 'https://j.mp/websniffer', 'http://j.mp/2Bo2LVf', 'http://bit.ly/2BombJQ', 'https://cnn.it/2Ggb2ih' ] try: shortener = Shortener(tokens=tokens, max_cache_size=128) urls = random.sample(URLs, k=min(len(URLs), { 'none': 0, 'one': 1, 'some': 3, 'all': len(URLs) }['some'])) print(shortener.shorten_urls(urls)) print(shortener.shorten_urls_to_dict(urls[::-1])) except Exception: time.sleep(0.01) # Delay for longs to flush. raise
from bitlyshortener import Shortener # get access toke from bitly tokens_pool = ['f3afbf82e53c51c95dab8944bddecab753a833cb'] shortener = Shortener(tokens=tokens_pool, max_cache_size=128) # get url into list url = [] get_url = input("Enter url: ") url.append(get_url) # sort using bitly sort_url = shortener.shorten_urls_to_dict(url) print("sort url: ", sort_url)