class DoiCache(object): def __init__(self, config): cache_opts = { 'cache.type': getattr(config, 'citation_cache_type', 'file'), 'cache.data_dir': getattr(config, 'citation_cache_data_dir', None), 'cache.lock_dir': getattr(config, 'citation_cache_lock_dir', None), } self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('doi') def _raw_get_bibtex(self, doi): dx_url = "http://dx.doi.org/" + doi headers = {'Accept': 'text/bibliography; style=bibtex, application/x-bibtex'} req = requests.get(dx_url, headers=headers) return req.text def get_bibtex(self, doi): createfunc = functools.partial(self._raw_get_bibtex, doi) return self._cache.get(key=doi, createfunc=createfunc)
class DoiCache: def __init__(self, config): cache_opts = { 'cache.type': getattr(config, 'citation_cache_type', 'file'), 'cache.data_dir': getattr(config, 'citation_cache_data_dir', None), 'cache.lock_dir': getattr(config, 'citation_cache_lock_dir', None), } self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('doi') def _raw_get_bibtex(self, doi): doi_url = f"https://doi.org/{doi}" headers = {'Accept': 'application/x-bibtex'} req = requests.get(doi_url, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT) req.encoding = req.apparent_encoding return req.text def get_bibtex(self, doi): createfunc = functools.partial(self._raw_get_bibtex, doi) return self._cache.get(key=doi, createfunc=createfunc)
class RadioDns_(): """Class to handle connection to the radioDns database: listing of topics and logins, special topic rules""" CACHE_DURATION = 600 def __init__(self): self.logger = logging.getLogger('radiovisserver.radiodns') self.cache = CacheManager(**parse_cache_config_options(config.CACHE_OPTS)).get_cache('radiodns', expire=60) self.durablecache = pylibmc.Client( [config.MEMCACHED_HOST], binary=True, behaviors={ "tcp_nodelay": True, "ketama": True, } ) # CacheManager(**parse_cache_config_options(config.CACHE_OPTS)).get_cache('radiodnsdurable') def do_query(self, url, params): try: return requests.get(config.API_URL + url, data=params).json() except: # Ommit params as it's may contain passwords self.logger.error("Error trying query %s" % (url,)) return None def check_auth(self, user, password, ip): """Check an username and password""" self.logger.debug("Checking username and password for %s" % (user,)) result = self.do_query('check_auth', {'username': user, 'password': password, 'ip': ip}) if result: if result['result']: self.logger.debug("Password ok") return True else: self.logger.warning("Cannot auth: %s" % (result['error'],)) return False else: self.logger.error("No reply when check_auth ?") return False def get_channels(self, station_id): """Return the list of channels for a station. Use cachinig of 1 minute""" self.logger.debug("Getting channels of %s" % (station_id,)) def get_channels(): self.logger.debug("Query channels of %s, nothing in cache !" % (station_id,)) result = self.do_query('get_channels', {'station_id': station_id}) if result is None: self.logger.error("No reply when get_channels ?") return [] return result['list'] return self.cache.get(key="get_channels-" + station_id, createfunc=get_channels) def update_channel_topics(self): """Update the channel cache from database""" try: self.logger.debug("Updating channel topic list for durable cache.") new_topics = [] for (channel, id) in self.get_all_vis_channels(): new_topics.append(channel) self.logger.debug( "Setting radiovis_channels_topics channel topic list with %s elements." % (len(new_topics))) self.durablecache.set('radiovis_channels_topics', new_topics, time=RadioDns.CACHE_DURATION) except: e = sys.exc_info()[0] self.logger.error("Error trying to update channel topics in durable cache. %s" % (e)) return def contains_channel_topic(self, topic): """Checks if cache contains a particular channel""" try: # Normalize to ignore /image and /text topic = topic.rstrip('image').rstrip('text') channel_topics = self.durablecache.get('radiovis_channels_topics') return topic in channel_topics except: self.logger.error("Error trying to check channel topic %s in cache." % (topic)) return None def convert_fm_topic_to_gcc(self, topic): """Convert a fm topic using gcc instead of country code""" # /topic/fm/gcc/ <=> /topic/fm/cc/ . If it's a gcc, topic[13] = '/' if not topic[:10] == "/topic/fm/" or topic[13] == '/': return topic self.logger.debug("Converting %s to use gcc" % (topic,)) try: cachevalue = self.durablecache.get('radiovis_isoecc_' + topic) if cachevalue: return cachevalue except (pylibmc.ConnectionError, pylibmc.ServerDown) as e: self.logger.warning("No memcached backend is running! %s" % (e,)) def convert_topic(): splited_topic = topic.split('/') cc = splited_topic[3] self.logger.debug("Querying gcc value for %s, nothing in cache !" % (cc,)) result = self.do_query('get_gcc', {'cc': cc}) if result is None: self.logger.error("No reply when convert_fm_topic_to_gcc ?") return topic # Return the topic splited_topic[3] = result['gcc'] gcc_topic = '/'.join(splited_topic) self.logger.debug("Setting radiovis_isoecc_ to durable cache topic list with %s." % (gcc_topic)) try: self.durablecache.set('radiovis_isoecc_' + topic, gcc_topic, time=RadioDns.CACHE_DURATION) except (pylibmc.ConnectionError, pylibmc.ServerDown) as e: self.logger.warning("No memcached backend is running! %s" % (e,)) return gcc_topic return self.cache.get(key='topic-to-gcc-' + topic, createfunc=convert_topic) def check_special_matchs(self, topic, topics): """Return true if topic is in the list of topics, using specials rules (eg. fm)""" # Only a special rule for fm if not topic[:10] == "/topic/fm/": return None # Check matches using gcc version topic = self.convert_fm_topic_to_gcc(topic) for subTopic in topics: subTopicConverted = self.convert_fm_topic_to_gcc(subTopic) if subTopicConverted == topic: return subTopic return None def get_all_channels(self): """Return the list of all channels""" result = self.do_query('get_all_channels', {}) if result is None: self.logger.error("No reply when get_all_channels ?") return [] retour = [] for (topic, id) in result['list']: retour.append((self.convert_fm_topic_to_gcc(topic), id)) return retour def get_all_vis_channels(self): """Return the list of all VIS channel that have an image""" result = self.do_query('get_all_vis_channels', {}) if result is None: self.logger.error("No reply when get_all_vis_channels ?") return [] retour = [] for (topic, id) in result['list']: retour.append((self.convert_fm_topic_to_gcc(topic), id)) return retour def get_channel_default(self, id): """Return the default image, link and message for a channel""" # Get out of cache if available try: cachevalue = self.durablecache.get('get_channel_default_' + str(id)) if cachevalue: return cachevalue except (pylibmc.ConnectionError, pylibmc.ServerDown) as e: self.logger.warning("No memcached backend is running! %s" % (e,)) result = self.do_query('get_channel_default', {'id': id}) if result is None: self.logger.error("No reply when get_channel_default %s ?" % (id,)) return [] # Save to cache try: self.durablecache.set('get_channel_default_' + str(id), result['info'], time=RadioDns.CACHE_DURATION) except (pylibmc.ConnectionError, pylibmc.ServerDown) as e: self.logger.warning("No memcached backend is running! %s" % (e,)) return result['info'] def add_log(self, topic, message, headers, timestamp): """Add a log entry""" result = self.do_query('add_log', {'topic': topic, 'message': str(message), 'headers': json.dumps(headers), 'timestamp': timestamp}) if result is None: self.logger.error("No reply when add_log %s %s %s %s ?" % (topic, message, headers, timestamp,)) def cleanup_logs(self, max_age): """Clean logs""" result = self.do_query('cleanup_logs', {'max_age': max_age}) if result is None: self.logger.error("No reply when cleanup_logs ?")
class RadioDns_(): """Class to handle connection to the radioDns database: listing of topics and logins, special topic rules""" def __init__(self): self.logger = logging.getLogger('radiovisserver.radiodns') self.cache = CacheManager(**parse_cache_config_options(config.CACHE_OPTS)).get_cache('radiodns', expire=60) def do_query(self, url, params): try: return requests.get(config.API_URL + url, data=params).json() except: # Ommit params as it's may contain passwords self.logger.error("Error trying query %s" % (url, )) return None def check_auth(self, user, password, ip): """Check an username and password""" self.logger.debug("Checking username and password for %s" % (user,)) result = self.do_query('check_auth', {'username': user, 'password': password, 'ip': ip}) if result: if result['result']: self.logger.debug("Password ok") return True else: self.logger.warning("Cannot auth: %s" % (result['error'], )) return False else: self.logger.error("No reply when check_auth ?") return False def get_channels(self, station_id): """Return the list of channels for a station. Use cachinig of 1 minute""" self.logger.debug("Getting channels of %s" % (station_id,)) def get_channels(): self.logger.debug("Query channels of %s, nothing in cache !" % (station_id,)) result = self.do_query('get_channels', {'station_id': station_id}) if result is None: self.logger.error("No reply when get_channels ?") return [] return result['list'] return self.cache.get(key="get_channels-" + station_id, createfunc=get_channels) def convert_fm_topic_to_gcc(self, topic): """Convert a fm topic using gcc instead of country code""" # /topic/fm/gcc/ <=> /topic/fm/cc/ . If it's a gcc, topic[13] = '/' if not topic[:10] == "/topic/fm/" or topic[13] == '/': return topic self.logger.debug("Converting %s to use gcc" % (topic,)) def convert_topic(): splited_topic = topic.split('/') cc = splited_topic[3] self.logger.debug("Querying gcc value for %s, nothing in cache !" % (cc,)) result = self.do_query('get_gcc', {'cc': cc}) if result is None: self.logger.error("No reply when convert_fm_topic_to_gcc ?") return topic # Return the topic splited_topic[3] = result['gcc'] return '/'.join(splited_topic) return self.cache.get(key='topic-to-gcc-' + topic, createfunc=convert_topic) def check_special_matchs(self, topic, topics): """Return true if topic is in the list of topics, using specials rules (eg. fm)""" # Only a special rule for fm if not topic[:10] == "/topic/fm/": return None # Check matches using gcc version topic = self.convert_fm_topic_to_gcc(topic) for subTopic in topics: subTopicConverted = self.convert_fm_topic_to_gcc(subTopic) if subTopicConverted == topic: return subTopic return None def get_all_channels(self): """Return the list of all channels""" result = self.do_query('get_all_channels', {}) if result is None: self.logger.error("No reply when get_all_channels ?") return [] retour = [] for (topic, id) in result['list']: retour.append((self.convert_fm_topic_to_gcc(topic), id)) return retour def get_channel_default(self, id): """Return the default image, link and message for a channel""" result = self.do_query('get_channel_default', {'id': id}) if result is None: self.logger.error("No reply when get_channel_default %s ?" % (id, )) return [] return result['info'] def add_log(self, topic, message, headers, timestamp): """Add a log entry""" result = self.do_query('add_log', {'topic': topic, 'message': str(message), 'headers': json.dumps(headers), 'timestamp': timestamp }) if result is None: self.logger.error("No reply when add_log %s %s %s %s ?" % (topic, message, headers, timestamp, )) def cleanup_logs(self, max_age): """Clean logs""" result = self.do_query('cleanup_logs', {'max_age': max_age}) if result is None: self.logger.error("No reply when cleanup_logs ?")
class RadioDns_(): """Class to handle connection to the radioDns database: listing of topics and logins, special topic rules""" CACHE_DURATION = 600 def __init__(self): self.logger = logging.getLogger('radiovisserver.radiodns') self.cache = CacheManager( **parse_cache_config_options(config.CACHE_OPTS)).get_cache( 'radiodns', expire=60) self.durablecache = pylibmc.Client( [config.MEMCACHED_HOST], binary=True, behaviors={ "tcp_nodelay": True, "ketama": True, } ) # CacheManager(**parse_cache_config_options(config.CACHE_OPTS)).get_cache('radiodnsdurable') def do_query(self, url, params): try: return requests.get(config.API_URL + url, data=params).json() except: # Ommit params as it's may contain passwords self.logger.error("Error trying query %s" % (url, )) return None def check_auth(self, user, password, ip): """Check an username and password""" self.logger.debug("Checking username and password for %s" % (user, )) result = self.do_query('check_auth', { 'username': user, 'password': password, 'ip': ip }) if result: if result['result']: self.logger.debug("Password ok") return True else: self.logger.warning("Cannot auth: %s" % (result['error'], )) return False else: self.logger.error("No reply when check_auth ?") return False def get_channels(self, station_id): """Return the list of channels for a station. Use cachinig of 1 minute""" self.logger.debug("Getting channels of %s" % (station_id, )) def get_channels(): self.logger.debug("Query channels of %s, nothing in cache !" % (station_id, )) result = self.do_query('get_channels', {'station_id': station_id}) if result is None: self.logger.error("No reply when get_channels ?") return [] return result['list'] return self.cache.get(key="get_channels-" + station_id, createfunc=get_channels) def update_channel_topics(self): """Update the channel cache from database""" try: self.logger.debug("Updating channel topic list for durable cache.") new_topics = [] for (channel, id) in self.get_all_vis_channels(): new_topics.append(channel) self.logger.debug( "Setting radiovis_channels_topics channel topic list with %s elements." % (len(new_topics))) self.durablecache.set('radiovis_channels_topics', new_topics, time=RadioDns.CACHE_DURATION) except: e = sys.exc_info()[0] self.logger.error( "Error trying to update channel topics in durable cache. %s" % (e)) return def contains_channel_topic(self, topic): """Checks if cache contains a particular channel""" try: # Normalize to ignore /image and /text topic = topic.rstrip('image').rstrip('text') channel_topics = self.durablecache.get('radiovis_channels_topics') return topic in channel_topics except: self.logger.error( "Error trying to check channel topic %s in cache." % (topic)) return None def convert_fm_topic_to_gcc(self, topic): """Convert a fm topic using gcc instead of country code""" # /topic/fm/gcc/ <=> /topic/fm/cc/ . If it's a gcc, topic[13] = '/' if not topic[:10] == "/topic/fm/" or topic[13] == '/': return topic self.logger.debug("Converting %s to use gcc" % (topic, )) try: cachevalue = self.durablecache.get('radiovis_isoecc_' + topic) if cachevalue: return cachevalue except (pylibmc.ConnectionError, pylibmc.ServerDown) as e: self.logger.warning("No memcached backend is running! %s" % (e, )) def convert_topic(): splited_topic = topic.split('/') cc = splited_topic[3] self.logger.debug("Querying gcc value for %s, nothing in cache !" % (cc, )) result = self.do_query('get_gcc', {'cc': cc}) if result is None: self.logger.error("No reply when convert_fm_topic_to_gcc ?") return topic # Return the topic splited_topic[3] = result['gcc'] gcc_topic = '/'.join(splited_topic) self.logger.debug( "Setting radiovis_isoecc_ to durable cache topic list with %s." % (gcc_topic)) try: self.durablecache.set('radiovis_isoecc_' + topic, gcc_topic, time=RadioDns.CACHE_DURATION) except (pylibmc.ConnectionError, pylibmc.ServerDown) as e: self.logger.warning("No memcached backend is running! %s" % (e, )) return gcc_topic return self.cache.get(key='topic-to-gcc-' + topic, createfunc=convert_topic) def check_special_matchs(self, topic, topics): """Return true if topic is in the list of topics, using specials rules (eg. fm)""" # Only a special rule for fm if not topic[:10] == "/topic/fm/": return None # Check matches using gcc version topic = self.convert_fm_topic_to_gcc(topic) for subTopic in topics: subTopicConverted = self.convert_fm_topic_to_gcc(subTopic) if subTopicConverted == topic: return subTopic return None def get_all_channels(self): """Return the list of all channels""" result = self.do_query('get_all_channels', {}) if result is None: self.logger.error("No reply when get_all_channels ?") return [] retour = [] for (topic, id) in result['list']: retour.append((self.convert_fm_topic_to_gcc(topic), id)) return retour def get_all_vis_channels(self): """Return the list of all VIS channel that have an image""" result = self.do_query('get_all_vis_channels', {}) if result is None: self.logger.error("No reply when get_all_vis_channels ?") return [] retour = [] for (topic, id) in result['list']: retour.append((self.convert_fm_topic_to_gcc(topic), id)) return retour def get_channel_default(self, id): """Return the default image, link and message for a channel""" # Get out of cache if available try: cachevalue = self.durablecache.get('get_channel_default_' + str(id)) if cachevalue: return cachevalue except (pylibmc.ConnectionError, pylibmc.ServerDown) as e: self.logger.warning("No memcached backend is running! %s" % (e, )) result = self.do_query('get_channel_default', {'id': id}) if result is None: self.logger.error("No reply when get_channel_default %s ?" % (id, )) return [] # Save to cache try: self.durablecache.set('get_channel_default_' + str(id), result['info'], time=RadioDns.CACHE_DURATION) except (pylibmc.ConnectionError, pylibmc.ServerDown) as e: self.logger.warning("No memcached backend is running! %s" % (e, )) return result['info'] def add_log(self, topic, message, headers, timestamp): """Add a log entry""" result = self.do_query( 'add_log', { 'topic': topic, 'message': str(message), 'headers': json.dumps(headers), 'timestamp': timestamp }) if result is None: self.logger.error("No reply when add_log %s %s %s %s ?" % ( topic, message, headers, timestamp, )) def cleanup_logs(self, max_age): """Clean logs""" result = self.do_query('cleanup_logs', {'max_age': max_age}) if result is None: self.logger.error("No reply when cleanup_logs ?")