class DoiCache(object):
def __init__(self, config):
cache_opts = {
'cache.type': getattr(config, 'citation_cache_type', 'file'),
'cache.data_dir': getattr(config, 'citation_cache_data_dir', None),
'cache.lock_dir': getattr(config, 'citation_cache_lock_dir', None),
}
self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('doi')
def _raw_get_bibtex(self, doi):
dx_url = "http://dx.doi.org/" + doi
headers = {'Accept': 'text/bibliography; style=bibtex, application/x-bibtex'}
req = requests.get(dx_url, headers=headers)
return req.text
def get_bibtex(self, doi):
createfunc = functools.partial(self._raw_get_bibtex, doi)
return self._cache.get(key=doi, createfunc=createfunc)
class DoiCache:
def __init__(self, config):
cache_opts = {
'cache.type': getattr(config, 'citation_cache_type', 'file'),
'cache.data_dir': getattr(config, 'citation_cache_data_dir', None),
'cache.lock_dir': getattr(config, 'citation_cache_lock_dir', None),
}
self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache('doi')
def _raw_get_bibtex(self, doi):
doi_url = f"https://doi.org/{doi}"
headers = {'Accept': 'application/x-bibtex'}
req = requests.get(doi_url, headers=headers, timeout=DEFAULT_SOCKET_TIMEOUT)
req.encoding = req.apparent_encoding
return req.text
def get_bibtex(self, doi):
createfunc = functools.partial(self._raw_get_bibtex, doi)
return self._cache.get(key=doi, createfunc=createfunc)
class RadioDns_():
"""Class to handle connection to the radioDns database: listing of topics and logins, special topic rules"""
CACHE_DURATION = 600
def __init__(self):
self.logger = logging.getLogger('radiovisserver.radiodns')
self.cache = CacheManager(**parse_cache_config_options(config.CACHE_OPTS)).get_cache('radiodns', expire=60)
self.durablecache = pylibmc.Client(
[config.MEMCACHED_HOST],
binary=True,
behaviors={
"tcp_nodelay": True,
"ketama": True,
}
) # CacheManager(**parse_cache_config_options(config.CACHE_OPTS)).get_cache('radiodnsdurable')
def do_query(self, url, params):
try:
return requests.get(config.API_URL + url, data=params).json()
except:
# Ommit params as it's may contain passwords
self.logger.error("Error trying query %s" % (url,))
return None
def check_auth(self, user, password, ip):
"""Check an username and password"""
self.logger.debug("Checking username and password for %s" % (user,))
result = self.do_query('check_auth', {'username': user, 'password': password, 'ip': ip})
if result:
if result['result']:
self.logger.debug("Password ok")
return True
else:
self.logger.warning("Cannot auth: %s" % (result['error'],))
return False
else:
self.logger.error("No reply when check_auth ?")
return False
def get_channels(self, station_id):
"""Return the list of channels for a station. Use cachinig of 1 minute"""
self.logger.debug("Getting channels of %s" % (station_id,))
def get_channels():
self.logger.debug("Query channels of %s, nothing in cache !" % (station_id,))
result = self.do_query('get_channels', {'station_id': station_id})
if result is None:
self.logger.error("No reply when get_channels ?")
return []
return result['list']
return self.cache.get(key="get_channels-" + station_id, createfunc=get_channels)
def update_channel_topics(self):
"""Update the channel cache from database"""
try:
self.logger.debug("Updating channel topic list for durable cache.")
new_topics = []
for (channel, id) in self.get_all_vis_channels():
new_topics.append(channel)
self.logger.debug(
"Setting radiovis_channels_topics channel topic list with %s elements." % (len(new_topics)))
self.durablecache.set('radiovis_channels_topics', new_topics, time=RadioDns.CACHE_DURATION)
except:
e = sys.exc_info()[0]
self.logger.error("Error trying to update channel topics in durable cache. %s" % (e))
return
def contains_channel_topic(self, topic):
"""Checks if cache contains a particular channel"""
try:
# Normalize to ignore /image and /text
topic = topic.rstrip('image').rstrip('text')
channel_topics = self.durablecache.get('radiovis_channels_topics')
return topic in channel_topics
except:
self.logger.error("Error trying to check channel topic %s in cache." % (topic))
return None
def convert_fm_topic_to_gcc(self, topic):
"""Convert a fm topic using gcc instead of country code"""
# /topic/fm/gcc/ <=> /topic/fm/cc/ . If it's a gcc, topic[13] = '/'
if not topic[:10] == "/topic/fm/" or topic[13] == '/':
return topic
self.logger.debug("Converting %s to use gcc" % (topic,))
try:
cachevalue = self.durablecache.get('radiovis_isoecc_' + topic)
if cachevalue:
return cachevalue
except (pylibmc.ConnectionError, pylibmc.ServerDown) as e:
self.logger.warning("No memcached backend is running! %s" % (e,))
def convert_topic():
splited_topic = topic.split('/')
cc = splited_topic[3]
self.logger.debug("Querying gcc value for %s, nothing in cache !" % (cc,))
result = self.do_query('get_gcc', {'cc': cc})
if result is None:
self.logger.error("No reply when convert_fm_topic_to_gcc ?")
return topic # Return the topic
splited_topic[3] = result['gcc']
gcc_topic = '/'.join(splited_topic)
self.logger.debug("Setting radiovis_isoecc_ to durable cache topic list with %s." % (gcc_topic))
try:
self.durablecache.set('radiovis_isoecc_' + topic, gcc_topic, time=RadioDns.CACHE_DURATION)
except (pylibmc.ConnectionError, pylibmc.ServerDown) as e:
self.logger.warning("No memcached backend is running! %s" % (e,))
return gcc_topic
return self.cache.get(key='topic-to-gcc-' + topic, createfunc=convert_topic)
def check_special_matchs(self, topic, topics):
"""Return true if topic is in the list of topics, using specials rules (eg. fm)"""
# Only a special rule for fm
if not topic[:10] == "/topic/fm/":
return None
# Check matches using gcc version
topic = self.convert_fm_topic_to_gcc(topic)
for subTopic in topics:
subTopicConverted = self.convert_fm_topic_to_gcc(subTopic)
if subTopicConverted == topic:
return subTopic
return None
def get_all_channels(self):
"""Return the list of all channels"""
result = self.do_query('get_all_channels', {})
if result is None:
self.logger.error("No reply when get_all_channels ?")
return []
retour = []
for (topic, id) in result['list']:
retour.append((self.convert_fm_topic_to_gcc(topic), id))
return retour
def get_all_vis_channels(self):
"""Return the list of all VIS channel that have an image"""
result = self.do_query('get_all_vis_channels', {})
if result is None:
self.logger.error("No reply when get_all_vis_channels ?")
return []
retour = []
for (topic, id) in result['list']:
retour.append((self.convert_fm_topic_to_gcc(topic), id))
return retour
def get_channel_default(self, id):
"""Return the default image, link and message for a channel"""
# Get out of cache if available
try:
cachevalue = self.durablecache.get('get_channel_default_' + str(id))
if cachevalue:
return cachevalue
except (pylibmc.ConnectionError, pylibmc.ServerDown) as e:
self.logger.warning("No memcached backend is running! %s" % (e,))
result = self.do_query('get_channel_default', {'id': id})
if result is None:
self.logger.error("No reply when get_channel_default %s ?" % (id,))
return []
# Save to cache
try:
self.durablecache.set('get_channel_default_' + str(id), result['info'], time=RadioDns.CACHE_DURATION)
except (pylibmc.ConnectionError, pylibmc.ServerDown) as e:
self.logger.warning("No memcached backend is running! %s" % (e,))
return result['info']
def add_log(self, topic, message, headers, timestamp):
"""Add a log entry"""
result = self.do_query('add_log', {'topic': topic, 'message': str(message), 'headers': json.dumps(headers),
'timestamp': timestamp})
if result is None:
self.logger.error("No reply when add_log %s %s %s %s ?" % (topic, message, headers, timestamp,))
def cleanup_logs(self, max_age):
"""Clean logs"""
result = self.do_query('cleanup_logs', {'max_age': max_age})
if result is None:
self.logger.error("No reply when cleanup_logs ?")
class RadioDns_():
"""Class to handle connection to the radioDns database: listing of topics and logins, special topic rules"""
def __init__(self):
self.logger = logging.getLogger('radiovisserver.radiodns')
self.cache = CacheManager(**parse_cache_config_options(config.CACHE_OPTS)).get_cache('radiodns', expire=60)
def do_query(self, url, params):
try:
return requests.get(config.API_URL + url, data=params).json()
except:
# Ommit params as it's may contain passwords
self.logger.error("Error trying query %s" % (url, ))
return None
def check_auth(self, user, password, ip):
"""Check an username and password"""
self.logger.debug("Checking username and password for %s" % (user,))
result = self.do_query('check_auth', {'username': user, 'password': password, 'ip': ip})
if result:
if result['result']:
self.logger.debug("Password ok")
return True
else:
self.logger.warning("Cannot auth: %s" % (result['error'], ))
return False
else:
self.logger.error("No reply when check_auth ?")
return False
def get_channels(self, station_id):
"""Return the list of channels for a station. Use cachinig of 1 minute"""
self.logger.debug("Getting channels of %s" % (station_id,))
def get_channels():
self.logger.debug("Query channels of %s, nothing in cache !" % (station_id,))
result = self.do_query('get_channels', {'station_id': station_id})
if result is None:
self.logger.error("No reply when get_channels ?")
return []
return result['list']
return self.cache.get(key="get_channels-" + station_id, createfunc=get_channels)
def convert_fm_topic_to_gcc(self, topic):
"""Convert a fm topic using gcc instead of country code"""
# /topic/fm/gcc/ <=> /topic/fm/cc/ . If it's a gcc, topic[13] = '/'
if not topic[:10] == "/topic/fm/" or topic[13] == '/':
return topic
self.logger.debug("Converting %s to use gcc" % (topic,))
def convert_topic():
splited_topic = topic.split('/')
cc = splited_topic[3]
self.logger.debug("Querying gcc value for %s, nothing in cache !" % (cc,))
result = self.do_query('get_gcc', {'cc': cc})
if result is None:
self.logger.error("No reply when convert_fm_topic_to_gcc ?")
return topic # Return the topic
splited_topic[3] = result['gcc']
return '/'.join(splited_topic)
return self.cache.get(key='topic-to-gcc-' + topic, createfunc=convert_topic)
def check_special_matchs(self, topic, topics):
"""Return true if topic is in the list of topics, using specials rules (eg. fm)"""
# Only a special rule for fm
if not topic[:10] == "/topic/fm/":
return None
# Check matches using gcc version
topic = self.convert_fm_topic_to_gcc(topic)
for subTopic in topics:
subTopicConverted = self.convert_fm_topic_to_gcc(subTopic)
if subTopicConverted == topic:
return subTopic
return None
def get_all_channels(self):
"""Return the list of all channels"""
result = self.do_query('get_all_channels', {})
if result is None:
self.logger.error("No reply when get_all_channels ?")
return []
retour = []
for (topic, id) in result['list']:
retour.append((self.convert_fm_topic_to_gcc(topic), id))
return retour
def get_channel_default(self, id):
"""Return the default image, link and message for a channel"""
result = self.do_query('get_channel_default', {'id': id})
if result is None:
self.logger.error("No reply when get_channel_default %s ?" % (id, ))
return []
return result['info']
def add_log(self, topic, message, headers, timestamp):
"""Add a log entry"""
result = self.do_query('add_log', {'topic': topic, 'message': str(message), 'headers': json.dumps(headers), 'timestamp': timestamp })
if result is None:
self.logger.error("No reply when add_log %s %s %s %s ?" % (topic, message, headers, timestamp, ))
def cleanup_logs(self, max_age):
"""Clean logs"""
result = self.do_query('cleanup_logs', {'max_age': max_age})
if result is None:
self.logger.error("No reply when cleanup_logs ?")
class RadioDns_():
"""Class to handle connection to the radioDns database: listing of topics and logins, special topic rules"""
CACHE_DURATION = 600
def __init__(self):
self.logger = logging.getLogger('radiovisserver.radiodns')
self.cache = CacheManager(
**parse_cache_config_options(config.CACHE_OPTS)).get_cache(
'radiodns', expire=60)
self.durablecache = pylibmc.Client(
[config.MEMCACHED_HOST],
binary=True,
behaviors={
"tcp_nodelay": True,
"ketama": True,
}
) # CacheManager(**parse_cache_config_options(config.CACHE_OPTS)).get_cache('radiodnsdurable')
def do_query(self, url, params):
try:
return requests.get(config.API_URL + url, data=params).json()
except:
# Ommit params as it's may contain passwords
self.logger.error("Error trying query %s" % (url, ))
return None
def check_auth(self, user, password, ip):
"""Check an username and password"""
self.logger.debug("Checking username and password for %s" % (user, ))
result = self.do_query('check_auth', {
'username': user,
'password': password,
'ip': ip
})
if result:
if result['result']:
self.logger.debug("Password ok")
return True
else:
self.logger.warning("Cannot auth: %s" % (result['error'], ))
return False
else:
self.logger.error("No reply when check_auth ?")
return False
def get_channels(self, station_id):
"""Return the list of channels for a station. Use cachinig of 1 minute"""
self.logger.debug("Getting channels of %s" % (station_id, ))
def get_channels():
self.logger.debug("Query channels of %s, nothing in cache !" %
(station_id, ))
result = self.do_query('get_channels', {'station_id': station_id})
if result is None:
self.logger.error("No reply when get_channels ?")
return []
return result['list']
return self.cache.get(key="get_channels-" + station_id,
createfunc=get_channels)
def update_channel_topics(self):
"""Update the channel cache from database"""
try:
self.logger.debug("Updating channel topic list for durable cache.")
new_topics = []
for (channel, id) in self.get_all_vis_channels():
new_topics.append(channel)
self.logger.debug(
"Setting radiovis_channels_topics channel topic list with %s elements."
% (len(new_topics)))
self.durablecache.set('radiovis_channels_topics',
new_topics,
time=RadioDns.CACHE_DURATION)
except:
e = sys.exc_info()[0]
self.logger.error(
"Error trying to update channel topics in durable cache. %s" %
(e))
return
def contains_channel_topic(self, topic):
"""Checks if cache contains a particular channel"""
try:
# Normalize to ignore /image and /text
topic = topic.rstrip('image').rstrip('text')
channel_topics = self.durablecache.get('radiovis_channels_topics')
return topic in channel_topics
except:
self.logger.error(
"Error trying to check channel topic %s in cache." % (topic))
return None
def convert_fm_topic_to_gcc(self, topic):
"""Convert a fm topic using gcc instead of country code"""
# /topic/fm/gcc/ <=> /topic/fm/cc/ . If it's a gcc, topic[13] = '/'
if not topic[:10] == "/topic/fm/" or topic[13] == '/':
return topic
self.logger.debug("Converting %s to use gcc" % (topic, ))
try:
cachevalue = self.durablecache.get('radiovis_isoecc_' + topic)
if cachevalue:
return cachevalue
except (pylibmc.ConnectionError, pylibmc.ServerDown) as e:
self.logger.warning("No memcached backend is running! %s" % (e, ))
def convert_topic():
splited_topic = topic.split('/')
cc = splited_topic[3]
self.logger.debug("Querying gcc value for %s, nothing in cache !" %
(cc, ))
result = self.do_query('get_gcc', {'cc': cc})
if result is None:
self.logger.error("No reply when convert_fm_topic_to_gcc ?")
return topic # Return the topic
splited_topic[3] = result['gcc']
gcc_topic = '/'.join(splited_topic)
self.logger.debug(
"Setting radiovis_isoecc_ to durable cache topic list with %s."
% (gcc_topic))
try:
self.durablecache.set('radiovis_isoecc_' + topic,
gcc_topic,
time=RadioDns.CACHE_DURATION)
except (pylibmc.ConnectionError, pylibmc.ServerDown) as e:
self.logger.warning("No memcached backend is running! %s" %
(e, ))
return gcc_topic
return self.cache.get(key='topic-to-gcc-' + topic,
createfunc=convert_topic)
def check_special_matchs(self, topic, topics):
"""Return true if topic is in the list of topics, using specials rules (eg. fm)"""
# Only a special rule for fm
if not topic[:10] == "/topic/fm/":
return None
# Check matches using gcc version
topic = self.convert_fm_topic_to_gcc(topic)
for subTopic in topics:
subTopicConverted = self.convert_fm_topic_to_gcc(subTopic)
if subTopicConverted == topic:
return subTopic
return None
def get_all_channels(self):
"""Return the list of all channels"""
result = self.do_query('get_all_channels', {})
if result is None:
self.logger.error("No reply when get_all_channels ?")
return []
retour = []
for (topic, id) in result['list']:
retour.append((self.convert_fm_topic_to_gcc(topic), id))
return retour
def get_all_vis_channels(self):
"""Return the list of all VIS channel that have an image"""
result = self.do_query('get_all_vis_channels', {})
if result is None:
self.logger.error("No reply when get_all_vis_channels ?")
return []
retour = []
for (topic, id) in result['list']:
retour.append((self.convert_fm_topic_to_gcc(topic), id))
return retour
def get_channel_default(self, id):
"""Return the default image, link and message for a channel"""
# Get out of cache if available
try:
cachevalue = self.durablecache.get('get_channel_default_' +
str(id))
if cachevalue:
return cachevalue
except (pylibmc.ConnectionError, pylibmc.ServerDown) as e:
self.logger.warning("No memcached backend is running! %s" % (e, ))
result = self.do_query('get_channel_default', {'id': id})
if result is None:
self.logger.error("No reply when get_channel_default %s ?" %
(id, ))
return []
# Save to cache
try:
self.durablecache.set('get_channel_default_' + str(id),
result['info'],
time=RadioDns.CACHE_DURATION)
except (pylibmc.ConnectionError, pylibmc.ServerDown) as e:
self.logger.warning("No memcached backend is running! %s" % (e, ))
return result['info']
def add_log(self, topic, message, headers, timestamp):
"""Add a log entry"""
result = self.do_query(
'add_log', {
'topic': topic,
'message': str(message),
'headers': json.dumps(headers),
'timestamp': timestamp
})
if result is None:
self.logger.error("No reply when add_log %s %s %s %s ?" % (
topic,
message,
headers,
timestamp,
))
def cleanup_logs(self, max_age):
"""Clean logs"""
result = self.do_query('cleanup_logs', {'max_age': max_age})
if result is None:
self.logger.error("No reply when cleanup_logs ?")
