def setUp(self):
super(InfluxWriterTestCase, self).setUp()
self.ls = InfluxListenStore({ 'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME})
def setUp(self):
super(APICompatDeprecatedTestCase, self).setUp()
self.user = db_user.get_or_create('apicompatoldtestuser')
self.ls = InfluxListenStore({
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
})
def setUp(self):
super(InfluxWriterTestCase, self).setUp()
self.ls = InfluxListenStore({ 'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME}, self.app.logger)
def setUp(self):
super(APICompatTestCase, self).setUp()
self.lb_user = db_user.get_or_create(1, 'apicompattestuser')
self.lfm_user = User(
self.lb_user['id'],
self.lb_user['created'],
self.lb_user['musicbrainz_id'],
self.lb_user['auth_token'],
)
self.ls = InfluxListenStore(
{
'REDIS_HOST': current_app.config['REDIS_HOST'],
'REDIS_PORT': current_app.config['REDIS_PORT'],
'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'],
'INFLUX_HOST': current_app.config['INFLUX_HOST'],
'INFLUX_PORT': current_app.config['INFLUX_PORT'],
'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'],
}, self.app.logger)
def start(self):
self.log.info("influx-writer init")
self._verify_hosts_in_config()
if not hasattr(self.config, "INFLUX_HOST"):
self.log.error("Influx service not defined. Sleeping {0} seconds and exiting.".format(self.ERROR_RETRY_DELAY))
sleep(self.ERROR_RETRY_DELAY)
sys.exit(-1)
while True:
try:
self.ls = InfluxListenStore({ 'REDIS_HOST' : self.config.REDIS_HOST,
'REDIS_PORT' : self.config.REDIS_PORT,
'INFLUX_HOST': self.config.INFLUX_HOST,
'INFLUX_PORT': self.config.INFLUX_PORT,
'INFLUX_DB_NAME': self.config.INFLUX_DB_NAME})
self.influx = InfluxDBClient(host=self.config.INFLUX_HOST, port=self.config.INFLUX_PORT, database=self.config.INFLUX_DB_NAME)
break
except Exception as err:
self.log.error("Cannot connect to influx: %s. Retrying in 2 seconds and trying again." % str(err))
sleep(ERROR_RETRY_DELAY)
while True:
try:
self.redis = Redis(host=self.config.REDIS_HOST, port=self.config.REDIS_PORT, decode_responses=True)
self.redis.ping()
break
except Exception as err:
self.log.error("Cannot connect to redis: %s. Retrying in 2 seconds and trying again." % str(err))
sleep(ERROR_RETRY_DELAY)
while True:
self.connect_to_rabbitmq()
self.incoming_ch = self.connection.channel()
self.incoming_ch.exchange_declare(exchange=self.config.INCOMING_EXCHANGE, exchange_type='fanout')
self.incoming_ch.queue_declare(self.config.INCOMING_QUEUE, durable=True)
self.incoming_ch.queue_bind(exchange=self.config.INCOMING_EXCHANGE, queue=self.config.INCOMING_QUEUE)
self.incoming_ch.basic_consume(
lambda ch, method, properties, body: self.static_callback(ch, method, properties, body, obj=self),
queue=self.config.INCOMING_QUEUE,
)
self.unique_ch = self.connection.channel()
self.unique_ch.exchange_declare(exchange=self.config.UNIQUE_EXCHANGE, exchange_type='fanout')
self.log.info("influx-writer started")
try:
self.incoming_ch.start_consuming()
except pika.exceptions.ConnectionClosed:
self.log.info("Connection to rabbitmq closed. Re-opening.")
self.connection = None
continue
self.connection.close()
def setUp(self):
super(APICompatDeprecatedTestCase, self).setUp()
self.user = db_user.get_or_create(1, 'apicompatoldtestuser')
self.ls = InfluxListenStore({
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
}, self.app.logger)
def init_influx_connection(logger, conf):
global _influx
while True:
try:
_influx = InfluxListenStore(conf)
break
except Exception as e:
logger.error(
"Couldn't create InfluxListenStore instance: {}".format(
str(e)))
logger.error("Sleeping 2 seconds and then retrying...")
time.sleep(2)
logger.info("Successfully created InfluxListenStore instance!")
return _influx
def init_influx_connection(logger, conf):
global _influx
while True:
try:
_influx = InfluxListenStore(conf, logger)
break
except Exception as e:
logger.error(
"Couldn't create InfluxListenStore instance: {}, sleeping and trying again..."
.format(str(e)),
exc_info=True)
time.sleep(2)
logger.info("Successfully created InfluxListenStore instance!")
return _influx
def init_influx_connection():
""" Connects to influx and returns an InfluxListenStore instance """
while True:
try:
return InfluxListenStore({
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
})
except Exception as e:
logger.error(
"Couldn't create InfluxListenStore instance: {}".format(
str(e)))
logger.error("Sleeping 2 seconds and then retrying...")
time.sleep(2)
def setUp(self):
super(APICompatTestCase, self).setUp()
self.lb_user = db_user.get_or_create(1, 'apicompattestuser')
self.lfm_user = User(
self.lb_user['id'],
self.lb_user['created'],
self.lb_user['musicbrainz_id'],
self.lb_user['auth_token'],
)
self.ls = InfluxListenStore({
'REDIS_HOST': current_app.config['REDIS_HOST'],
'REDIS_PORT': current_app.config['REDIS_PORT'],
'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'],
'INFLUX_HOST': current_app.config['INFLUX_HOST'],
'INFLUX_PORT': current_app.config['INFLUX_PORT'],
'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'],
}, self.app.logger)
class APICompatDeprecatedTestCase(APICompatIntegrationTestCase):
def setUp(self):
super(APICompatDeprecatedTestCase, self).setUp()
self.user = db_user.get_or_create(1, 'apicompatoldtestuser')
self.ls = InfluxListenStore({
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
}, self.app.logger)
def handshake(self, user_name, auth_token, timestamp):
""" Makes a request to the handshake endpoint of the AudioScrobbler API and
returns the response.
"""
args = {
'hs': 'true',
'p': '1.2',
'c': 'tst',
'v': '0.1',
'u': user_name,
't': timestamp,
'a': auth_token
}
return self.client.get('/', query_string=args)
def test_handshake(self):
""" Tests handshake for a user that exists """
timestamp = int(time.time())
audioscrobbler_auth_token = _get_audioscrobbler_auth_token(self.user['auth_token'], timestamp)
r = self.handshake(self.user['musicbrainz_id'], audioscrobbler_auth_token, timestamp)
self.assert200(r)
response = r.data.decode('utf-8').split('\n')
self.assertEqual(len(response), 5)
self.assertEqual(response[0], 'OK')
self.assertEqual(len(response[1]), 32)
def test_handshake_post(self):
""" Tests POST requests to handshake endpoint """
ts = int(time.time())
args = {
'hs': 'true',
'p': '1.2',
'c': 'tst',
'v': '0.1',
'u': self.user['musicbrainz_id'],
't': ts,
'a': _get_audioscrobbler_auth_token(self.user['auth_token'], ts)
}
r = self.client.post('/', query_string=args)
self.assert200(r)
response = r.data.decode('utf-8').split('\n')
self.assertEqual(len(response), 5)
self.assertEqual(response[0], 'OK')
self.assertEqual(len(response[1]), 32)
def test_root_url_when_no_handshake(self):
""" Tests the root url when there's no handshaking taking place """
r = self.client.get('/')
self.assertStatus(r, 302)
def test_handshake_unknown_user(self):
""" Tests handshake for user that is not in the db """
r = self.handshake('', '', '')
self.assert401(r)
def test_handshake_invalid_auth(self):
""" Tests handshake when invalid authorization token is sent """
r = self.handshake(self.user['musicbrainz_id'], '', int(time.time()))
self.assert401(r)
def test_submit_listen(self):
""" Sends a valid listen after handshaking and checks if it is present in the
listenstore
"""
timestamp = int(time.time())
audioscrobbler_auth_token = _get_audioscrobbler_auth_token(self.user['auth_token'], timestamp)
r = self.handshake(self.user['musicbrainz_id'], audioscrobbler_auth_token, timestamp)
self.assert200(r)
response = r.data.decode('utf-8').split('\n')
self.assertEqual(response[0], 'OK')
sid = response[1]
data = {
's': sid,
'a[0]': 'Kishore Kumar',
't[0]': 'Saamne Ye Kaun Aya',
'o[0]': 'P',
'l[0]': 300,
'b[0]': 'Jawani Diwani',
'i[0]': int(time.time()),
}
r = self.client.post(url_for('api_compat_old.submit_listens'), data=data)
self.assert200(r)
self.assertEqual(r.data.decode('utf-8'), 'OK\n')
time.sleep(1)
to_ts = int(time.time())
listens = self.ls.fetch_listens(self.user['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 1)
def test_submit_listen_invalid_sid(self):
""" Tests endpoint for 400 Bad Request if invalid session id is sent """
sid = ''
data = {
's': sid,
'a[0]': 'Kishore Kumar',
't[0]': 'Saamne Ye Kaun Aya',
'o[0]': 'P',
'l[0]': 300,
'b[0]': 'Jawani Diwani',
'i[0]': int(time.time()),
}
r = self.client.post(url_for('api_compat_old.submit_listens'), data=data)
self.assert401(r)
self.assertEqual(r.data.decode('utf-8'), 'BADSESSION\n')
def test_submit_listen_invalid_data(self):
""" Tests endpoint for 400 Bad Request if invalid data is sent """
timestamp = int(time.time())
audioscrobbler_auth_token = _get_audioscrobbler_auth_token(self.user['auth_token'], timestamp)
r = self.handshake(self.user['musicbrainz_id'], audioscrobbler_auth_token, timestamp)
self.assert200(r)
response = r.data.decode('utf-8').split('\n')
self.assertEqual(response[0], 'OK')
sid = response[1]
# no artist in data
data = {
's': sid,
't[0]': 'Saamne Ye Kaun Aya',
'o[0]': 'P',
'l[0]': 300,
'b[0]': 'Jawani Diwani',
'i[0]': int(time.time()),
}
r = self.client.post(url_for('api_compat_old.submit_listens'), data=data)
self.assert400(r)
self.assertEqual(r.data.decode('utf-8').split()[0], 'FAILED')
# add artist and remove track name
data['a[0]'] = 'Kishore Kumar'
del data['t[0]']
r = self.client.post(url_for('api_compat_old.submit_listens'), data=data)
self.assert400(r)
self.assertEqual(r.data.decode('utf-8').split()[0], 'FAILED')
# add track name and remove timestamp
data['t[0]'] = 'Saamne Ye Kaun Aya'
del data['i[0]']
r = self.client.post(url_for('api_compat_old.submit_listens'), data=data)
self.assert400(r)
self.assertEqual(r.data.decode('utf-8').split()[0], 'FAILED')
# re-add a timestamp in ns
data['i[0]'] = int(time.time()) * 10**9
r = self.client.post(url_for('api_compat_old.submit_listens'), data=data)
self.assert400(r)
self.assertEqual(r.data.decode('utf-8').split()[0], 'FAILED')
def test_playing_now(self):
""" Tests playing now notifications """
timestamp = int(time.time())
audioscrobbler_auth_token = _get_audioscrobbler_auth_token(self.user['auth_token'], timestamp)
r = self.handshake(self.user['musicbrainz_id'], audioscrobbler_auth_token, timestamp)
self.assert200(r)
response = r.data.decode('utf-8').split('\n')
self.assertEqual(response[0], 'OK')
sid = response[1]
data = {
's': sid,
'a': 'Kishore Kumar',
't': 'Saamne Ye Kaun Aya',
'b': 'Jawani Diwani',
}
r = self.client.post(url_for('api_compat_old.submit_now_playing'), data=data)
self.assert200(r)
self.assertEqual(r.data.decode('utf-8'), 'OK\n')
def test_get_session(self):
""" Tests _get_session method in api_compat_deprecated """
s = Session.create_by_user_id(self.user['id'])
session = _get_session(s.sid)
self.assertEqual(s.sid, session.sid)
def test_get_session_which_doesnt_exist(self):
""" Make sure BadRequest is raised when we try to get a session that doesn't exists """
with self.assertRaises(BadRequest):
session = _get_session('')
def test_404(self):
r = self.client.get('/thisurldoesnotexist')
self.assert404(r)
def test_to_native_api_now_playing(self):
""" Tests _to_native_api when used with data sent to the now_playing endpoint """
data = {
's': '',
'a': 'Kishore Kumar',
't': 'Saamne Ye Kaun Aya',
'b': 'Jawani Diwani',
}
native_data = _to_native_api(data, '')
self.assertDictEqual(native_data, {
'track_metadata': {
'track_name': 'Saamne Ye Kaun Aya',
'artist_name': 'Kishore Kumar',
'release_name': 'Jawani Diwani'
}
}
)
del data['a']
native_data = _to_native_api(data, '')
self.assertIsNone(native_data)
data['a'] = 'Kishore Kumar'
del data['t']
native_data = _to_native_api(data, '')
self.assertIsNone(native_data)
data['t'] = 'Saamne Ye Kaun Aya'
del data['b']
native_data = _to_native_api(data, '')
self.assertIsNone(native_data)
def test_to_native_api(self):
""" Tests _to_native_api with data that is sent to the submission endpoint """
t = int(time.time())
data = {
's': '',
'a[0]': 'Kishore Kumar',
't[0]': 'Saamne Ye Kaun Aya',
'o[0]': 'P',
'l[0]': 300,
'b[0]': 'Jawani Diwani',
'i[0]': t,
'a[1]': 'Kishore Kumar',
't[1]': 'Wada Karo',
'o[1]': 'P',
'l[1]': 300,
'b[1]': 'Jawani Diwani',
'i[1]': t + 10,
}
self.assertDictEqual(_to_native_api(data, '[0]'), {
'listened_at': t,
'track_metadata': {
'artist_name': 'Kishore Kumar',
'track_name': 'Saamne Ye Kaun Aya',
'release_name': 'Jawani Diwani',
'additional_info': {
'source': 'P',
'track_length': 300
}
}
})
self.assertDictEqual(_to_native_api(data, '[1]'), {
'listened_at': t + 10,
'track_metadata': {
'artist_name': 'Kishore Kumar',
'track_name': 'Wada Karo',
'release_name': 'Jawani Diwani',
'additional_info': {
'source': 'P',
'track_length': 300
}
}
})
del data['a[0]']
self.assertIsNone(_to_native_api(data, '[0]'))
data['a[0]'] = 'Kishore Kumar'
del data['t[0]']
native_data = _to_native_api(data, '[0]')
self.assertIsNone(native_data)
data['t[0]'] = 'Saamne Ye Kaun Aya'
del data['b[0]']
native_data = _to_native_api(data, '[0]')
self.assertIsNone(native_data)
#!/usr/bin/env python3
# This script flushes out the listencounts that are in the 7day retention policy and sums them
# into the permanent listen count measurement.
from listenbrainz import config
from listenbrainz.listenstore import InfluxListenStore
ls = InfluxListenStore({ 'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME})
ls.update_listen_counts()
class InfluxWriterTestCase(IntegrationTestCase):
def setUp(self):
super(InfluxWriterTestCase, self).setUp()
self.ls = InfluxListenStore({
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME
})
def send_listen(self, user, filename):
with open(self.path_to_data_file(filename)) as f:
payload = json.load(f)
return self.client.post(
url_for('api_v1.submit_listen'),
data=json.dumps(payload),
headers={'Authorization': 'Token {}'.format(user['auth_token'])},
content_type='application/json')
def test_dedup(self):
user = db_user.get_or_create('testinfluxwriteruser')
# send the same listen twice
r = self.send_listen(user, 'valid_single.json')
self.assert200(r)
time.sleep(2)
r = self.send_listen(user, 'valid_single.json')
self.assert200(r)
time.sleep(2)
to_ts = int(time.time())
listens = self.ls.fetch_listens(user['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 1)
def test_dedup_user_special_characters(self):
user = db_user.get_or_create('i have a\\weird\\user, name"\n')
# send the same listen twice
r = self.send_listen(user, 'valid_single.json')
self.assert200(r)
time.sleep(2)
r = self.send_listen(user, 'valid_single.json')
self.assert200(r)
time.sleep(2)
to_ts = int(time.time())
listens = self.ls.fetch_listens(user['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 1)
def test_dedup_same_batch(self):
user = db_user.get_or_create('phifedawg')
r = self.send_listen(user, 'same_batch_duplicates.json')
self.assert200(r)
time.sleep(2)
to_ts = int(time.time())
listens = self.ls.fetch_listens(user['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 1)
def test_dedup_different_users(self):
"""
Test to make sure influx writer doesn't confuse listens with same timestamps
but different users to be duplicates
"""
user1 = db_user.get_or_create('testuser1')
user2 = db_user.get_or_create('testuser2')
r = self.send_listen(user1, 'valid_single.json')
self.assert200(r)
r = self.send_listen(user2, 'valid_single.json')
self.assert200(r)
time.sleep(2) # sleep to allow influx-writer to do its thing
to_ts = int(time.time())
listens = self.ls.fetch_listens(user1['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 1)
listens = self.ls.fetch_listens(user2['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 1)
def test_dedup_same_timestamp_different_tracks(self):
""" Test to check that if there are two tracks w/ the same timestamp,
they don't get considered as duplicates
"""
user = db_user.get_or_create('difftracksametsuser')
# send four different tracks with the same timestamp
r = self.send_listen(user, 'valid_single.json')
self.assert200(r)
r = self.send_listen(user,
'same_timestamp_diff_track_valid_single.json')
self.assert200(r)
r = self.send_listen(user,
'same_timestamp_diff_track_valid_single_2.json')
self.assert200(r)
r = self.send_listen(user,
'same_timestamp_diff_track_valid_single_3.json')
self.assert200(r)
time.sleep(2)
to_ts = int(time.time())
listens = self.ls.fetch_listens(user['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 4)
class APICompatDeprecatedTestCase(APICompatIntegrationTestCase):
def setUp(self):
super(APICompatDeprecatedTestCase, self).setUp()
self.user = db_user.get_or_create(1, 'apicompatoldtestuser')
self.ls = InfluxListenStore(
{
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME,
}, self.app.logger)
def handshake(self, user_name, auth_token, timestamp):
""" Makes a request to the handshake endpoint of the AudioScrobbler API and
returns the response.
"""
args = {
'hs': 'true',
'p': '1.2',
'c': 'tst',
'v': '0.1',
'u': user_name,
't': timestamp,
'a': auth_token
}
return self.client.get('/', query_string=args)
def test_handshake(self):
""" Tests handshake for a user that exists """
timestamp = int(time.time())
audioscrobbler_auth_token = _get_audioscrobbler_auth_token(
self.user['auth_token'], timestamp)
r = self.handshake(self.user['musicbrainz_id'],
audioscrobbler_auth_token, timestamp)
self.assert200(r)
response = r.data.decode('utf-8').split('\n')
self.assertEqual(len(response), 5)
self.assertEqual(response[0], 'OK')
self.assertEqual(len(response[1]), 32)
def test_handshake_post(self):
""" Tests POST requests to handshake endpoint """
ts = int(time.time())
args = {
'hs': 'true',
'p': '1.2',
'c': 'tst',
'v': '0.1',
'u': self.user['musicbrainz_id'],
't': ts,
'a': _get_audioscrobbler_auth_token(self.user['auth_token'], ts)
}
r = self.client.post('/', query_string=args)
self.assert200(r)
response = r.data.decode('utf-8').split('\n')
self.assertEqual(len(response), 5)
self.assertEqual(response[0], 'OK')
self.assertEqual(len(response[1]), 32)
def test_root_url_when_no_handshake(self):
""" Tests the root url when there's no handshaking taking place """
r = self.client.get('/')
self.assertStatus(r, 302)
def test_handshake_unknown_user(self):
""" Tests handshake for user that is not in the db """
r = self.handshake('', '', '')
self.assert401(r)
def test_handshake_invalid_auth(self):
""" Tests handshake when invalid authorization token is sent """
r = self.handshake(self.user['musicbrainz_id'], '', int(time.time()))
self.assert401(r)
def test_submit_listen(self):
""" Sends a valid listen after handshaking and checks if it is present in the
listenstore
"""
timestamp = int(time.time())
audioscrobbler_auth_token = _get_audioscrobbler_auth_token(
self.user['auth_token'], timestamp)
r = self.handshake(self.user['musicbrainz_id'],
audioscrobbler_auth_token, timestamp)
self.assert200(r)
response = r.data.decode('utf-8').split('\n')
self.assertEqual(response[0], 'OK')
sid = response[1]
data = {
's': sid,
'a[0]': 'Kishore Kumar',
't[0]': 'Saamne Ye Kaun Aya',
'o[0]': 'P',
'l[0]': 300,
'b[0]': 'Jawani Diwani',
'i[0]': int(time.time()),
}
r = self.client.post(url_for('api_compat_old.submit_listens'),
data=data)
self.assert200(r)
self.assertEqual(r.data.decode('utf-8'), 'OK\n')
time.sleep(1)
to_ts = int(time.time())
listens = self.ls.fetch_listens(self.user['musicbrainz_id'],
to_ts=to_ts)
self.assertEqual(len(listens), 1)
def test_submit_listen_invalid_sid(self):
""" Tests endpoint for 400 Bad Request if invalid session id is sent """
sid = ''
data = {
's': sid,
'a[0]': 'Kishore Kumar',
't[0]': 'Saamne Ye Kaun Aya',
'o[0]': 'P',
'l[0]': 300,
'b[0]': 'Jawani Diwani',
'i[0]': int(time.time()),
}
r = self.client.post(url_for('api_compat_old.submit_listens'),
data=data)
self.assert401(r)
self.assertEqual(r.data.decode('utf-8'), 'BADSESSION\n')
def test_submit_listen_invalid_data(self):
""" Tests endpoint for 400 Bad Request if invalid data is sent """
timestamp = int(time.time())
audioscrobbler_auth_token = _get_audioscrobbler_auth_token(
self.user['auth_token'], timestamp)
r = self.handshake(self.user['musicbrainz_id'],
audioscrobbler_auth_token, timestamp)
self.assert200(r)
response = r.data.decode('utf-8').split('\n')
self.assertEqual(response[0], 'OK')
sid = response[1]
# no artist in data
data = {
's': sid,
't[0]': 'Saamne Ye Kaun Aya',
'o[0]': 'P',
'l[0]': 300,
'b[0]': 'Jawani Diwani',
'i[0]': int(time.time()),
}
r = self.client.post(url_for('api_compat_old.submit_listens'),
data=data)
self.assert400(r)
self.assertEqual(r.data.decode('utf-8').split()[0], 'FAILED')
# add artist and remove track name
data['a[0]'] = 'Kishore Kumar'
del data['t[0]']
r = self.client.post(url_for('api_compat_old.submit_listens'),
data=data)
self.assert400(r)
self.assertEqual(r.data.decode('utf-8').split()[0], 'FAILED')
# add track name and remove timestamp
data['t[0]'] = 'Saamne Ye Kaun Aya'
del data['i[0]']
r = self.client.post(url_for('api_compat_old.submit_listens'),
data=data)
self.assert400(r)
self.assertEqual(r.data.decode('utf-8').split()[0], 'FAILED')
# re-add a timestamp in ns
data['i[0]'] = int(time.time()) * 10**9
r = self.client.post(url_for('api_compat_old.submit_listens'),
data=data)
self.assert400(r)
self.assertEqual(r.data.decode('utf-8').split()[0], 'FAILED')
def test_playing_now(self):
""" Tests playing now notifications """
timestamp = int(time.time())
audioscrobbler_auth_token = _get_audioscrobbler_auth_token(
self.user['auth_token'], timestamp)
r = self.handshake(self.user['musicbrainz_id'],
audioscrobbler_auth_token, timestamp)
self.assert200(r)
response = r.data.decode('utf-8').split('\n')
self.assertEqual(response[0], 'OK')
sid = response[1]
data = {
's': sid,
'a': 'Kishore Kumar',
't': 'Saamne Ye Kaun Aya',
'b': 'Jawani Diwani',
}
r = self.client.post(url_for('api_compat_old.submit_now_playing'),
data=data)
self.assert200(r)
self.assertEqual(r.data.decode('utf-8'), 'OK\n')
def test_get_session(self):
""" Tests _get_session method in api_compat_deprecated """
s = Session.create_by_user_id(self.user['id'])
session = _get_session(s.sid)
self.assertEqual(s.sid, session.sid)
def test_get_session_which_doesnt_exist(self):
""" Make sure BadRequest is raised when we try to get a session that doesn't exists """
with self.assertRaises(BadRequest):
session = _get_session('')
def test_404(self):
r = self.client.get('/thisurldoesnotexist')
self.assert404(r)
def test_to_native_api_now_playing(self):
""" Tests _to_native_api when used with data sent to the now_playing endpoint """
data = {
's': '',
'a': 'Kishore Kumar',
't': 'Saamne Ye Kaun Aya',
'b': 'Jawani Diwani',
}
native_data = _to_native_api(data, '')
self.assertDictEqual(
native_data, {
'track_metadata': {
'track_name': 'Saamne Ye Kaun Aya',
'artist_name': 'Kishore Kumar',
'release_name': 'Jawani Diwani'
}
})
del data['a']
native_data = _to_native_api(data, '')
self.assertIsNone(native_data)
data['a'] = 'Kishore Kumar'
del data['t']
native_data = _to_native_api(data, '')
self.assertIsNone(native_data)
data['t'] = 'Saamne Ye Kaun Aya'
del data['b']
native_data = _to_native_api(data, '')
self.assertIsNone(native_data)
def test_to_native_api(self):
""" Tests _to_native_api with data that is sent to the submission endpoint """
t = int(time.time())
data = {
's': '',
'a[0]': 'Kishore Kumar',
't[0]': 'Saamne Ye Kaun Aya',
'o[0]': 'P',
'l[0]': 300,
'b[0]': 'Jawani Diwani',
'i[0]': t,
'a[1]': 'Kishore Kumar',
't[1]': 'Wada Karo',
'o[1]': 'P',
'l[1]': 300,
'b[1]': 'Jawani Diwani',
'i[1]': t + 10,
}
self.assertDictEqual(
_to_native_api(data, '[0]'), {
'listened_at': t,
'track_metadata': {
'artist_name': 'Kishore Kumar',
'track_name': 'Saamne Ye Kaun Aya',
'release_name': 'Jawani Diwani',
'additional_info': {
'source': 'P',
'track_length': 300
}
}
})
self.assertDictEqual(
_to_native_api(data, '[1]'), {
'listened_at': t + 10,
'track_metadata': {
'artist_name': 'Kishore Kumar',
'track_name': 'Wada Karo',
'release_name': 'Jawani Diwani',
'additional_info': {
'source': 'P',
'track_length': 300
}
}
})
del data['a[0]']
self.assertIsNone(_to_native_api(data, '[0]'))
data['a[0]'] = 'Kishore Kumar'
del data['t[0]']
native_data = _to_native_api(data, '[0]')
self.assertIsNone(native_data)
data['t[0]'] = 'Saamne Ye Kaun Aya'
del data['b[0]']
native_data = _to_native_api(data, '[0]')
self.assertIsNone(native_data)
class APICompatTestCase(APICompatIntegrationTestCase):
def setUp(self):
super(APICompatTestCase, self).setUp()
self.lb_user = db_user.get_or_create(1, 'apicompattestuser')
self.lfm_user = User(
self.lb_user['id'],
self.lb_user['created'],
self.lb_user['musicbrainz_id'],
self.lb_user['auth_token'],
)
self.ls = InfluxListenStore({
'REDIS_HOST': current_app.config['REDIS_HOST'],
'REDIS_PORT': current_app.config['REDIS_PORT'],
'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'],
'INFLUX_HOST': current_app.config['INFLUX_HOST'],
'INFLUX_PORT': current_app.config['INFLUX_PORT'],
'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'],
}, self.app.logger)
def test_record_listen_now_playing(self):
""" Tests if listen of type 'nowplaying' is recorded correctly
if valid information is provided.
"""
token = Token.generate(self.lfm_user.api_key)
token.approve(self.lfm_user.name)
session = Session.create(token)
data = {
'method': 'track.updateNowPlaying',
'api_key': self.lfm_user.api_key,
'sk': session.sid,
'artist[0]': 'Kishore Kumar',
'track[0]': 'Saamne Ye Kaun Aya',
'album[0]': 'Jawani Diwani',
'duration[0]': 300,
'timestamp[0]': int(time.time()),
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'ok')
self.assertIsNotNone(response['lfm']['nowplaying'])
def test_get_token(self):
""" Tests if the token generated by get_token method is valid. """
data = {
'method': 'auth.gettoken',
'api_key': self.lfm_user.api_key,
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'ok')
token = Token.load(response['lfm']['token'], api_key=self.lfm_user.api_key)
self.assertIsNotNone(token)
def test_get_session(self):
""" Tests if the session key is valid and session is established correctly. """
token = Token.generate(self.lfm_user.api_key)
token.approve(self.lfm_user.name)
data = {
'method': 'auth.getsession',
'api_key': self.lfm_user.api_key,
'token': token.token,
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'ok')
self.assertEqual(response['lfm']['session']['name'], self.lfm_user.name)
session_key = Session.load(response['lfm']['session']['key'])
self.assertIsNotNone(session_key)
def test_get_session_invalid_token(self):
""" Tests if correct error codes are returned in case token supplied
is invalid during establishment of session.
"""
data = {
'method': 'auth.getsession',
'api_key': self.lfm_user.api_key,
'token': '',
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'failed')
self.assertEqual(response['lfm']['error']['@code'], '4')
def test_record_listen(self):
""" Tests if listen is recorded correctly if valid information is provided. """
token = Token.generate(self.lfm_user.api_key)
token.approve(self.lfm_user.name)
session = Session.create(token)
timestamp = int(time.time())
data = {
'method': 'track.scrobble',
'api_key': self.lfm_user.api_key,
'sk': session.sid,
'artist[0]': 'Kishore Kumar',
'track[0]': 'Saamne Ye Kaun Aya',
'album[0]': 'Jawani Diwani',
'duration[0]': 300,
'timestamp[0]': timestamp,
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'ok')
self.assertEqual(response['lfm']['scrobbles']['@accepted'], '1')
# Check if listen reached the influx listenstore
time.sleep(1)
listens = self.ls.fetch_listens(self.lb_user['musicbrainz_id'], from_ts=timestamp-1)
self.assertEqual(len(listens), 1)
def test_record_listen_multiple_listens(self):
""" Tests if multiple listens get recorded correctly in case valid information
is provided.
"""
token = Token.generate(self.lfm_user.api_key)
token.approve(self.lfm_user.name)
session = Session.create(token)
timestamp = int(time.time())
data = {
'method': 'track.scrobble',
'api_key': self.lfm_user.api_key,
'sk': session.sid,
'artist[0]': 'Kishore Kumar',
'track[0]': 'Saamne Ye Kaun Aya',
'album[0]': 'Jawani Diwani',
'duration[0]': 300,
'timestamp[0]': timestamp,
'artist[1]': 'Fifth Harmony',
'track[1]': 'Deliver',
'duration[1]': 200,
'timestamp[1]': timestamp+300,
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'ok')
self.assertEqual(response['lfm']['scrobbles']['@accepted'], '2')
# Check if listens reached the influx listenstore
time.sleep(1)
listens = self.ls.fetch_listens(self.lb_user['musicbrainz_id'], from_ts=timestamp-1)
self.assertEqual(len(listens), 2)
def test_create_response_for_single_listen(self):
""" Tests create_response_for_single_listen method in api_compat
to check if responses are generated correctly.
"""
from listenbrainz.webserver.views.api_compat import create_response_for_single_listen
timestamp = int(time.time())
original_listen = {
'artist': 'Kishore Kumar',
'track': 'Saamne Ye Kaun Aya',
'album': 'Jawani Diwani',
'duration': 300,
'timestamp': timestamp,
}
augmented_listen = {
'listened_at': timestamp,
'track_metadata': {
'artist_name': 'Kishore Kumar',
'track_name': 'Saamne Ye Kaun Aya',
'release_name': 'Jawani Diwani',
'additional_info': {
'track_length': 300
}
}
}
# If original listen and augmented listen are same
xml_response = create_response_for_single_listen(original_listen, augmented_listen, listen_type="listens")
response = xmltodict.parse(xml_response)
self.assertEqual(response['scrobble']['track']['#text'], 'Saamne Ye Kaun Aya')
self.assertEqual(response['scrobble']['track']['@corrected'], '0')
self.assertEqual(response['scrobble']['artist']['#text'], 'Kishore Kumar')
self.assertEqual(response['scrobble']['artist']['@corrected'], '0')
self.assertEqual(response['scrobble']['album']['#text'], 'Jawani Diwani')
self.assertEqual(response['scrobble']['timestamp'], str(timestamp))
# If listen type is 'playing_now'
xml_response = create_response_for_single_listen(original_listen, augmented_listen, listen_type="playing_now")
response = xmltodict.parse(xml_response)
self.assertEqual(response['nowplaying']['track']['#text'], 'Saamne Ye Kaun Aya')
self.assertEqual(response['nowplaying']['track']['@corrected'], '0')
self.assertEqual(response['nowplaying']['artist']['#text'], 'Kishore Kumar')
self.assertEqual(response['nowplaying']['artist']['@corrected'], '0')
self.assertEqual(response['nowplaying']['album']['#text'], 'Jawani Diwani')
self.assertEqual(response['nowplaying']['album']['@corrected'], '0')
self.assertEqual(response['nowplaying']['timestamp'], str(timestamp))
# If artist was corrected
original_listen['artist'] = 'Pink'
xml_response = create_response_for_single_listen(original_listen, augmented_listen, listen_type="listens")
response = xmltodict.parse(xml_response)
self.assertEqual(response['scrobble']['track']['#text'], 'Saamne Ye Kaun Aya')
self.assertEqual(response['scrobble']['track']['@corrected'], '0')
self.assertEqual(response['scrobble']['artist']['#text'], 'Kishore Kumar')
self.assertEqual(response['scrobble']['artist']['@corrected'], '1')
self.assertEqual(response['scrobble']['album']['#text'], 'Jawani Diwani')
self.assertEqual(response['scrobble']['album']['@corrected'], '0')
self.assertEqual(response['scrobble']['timestamp'], str(timestamp))
# If track was corrected
original_listen['artist'] = 'Kishore Kumar'
original_listen['track'] = 'Deliver'
xml_response = create_response_for_single_listen(original_listen, augmented_listen, listen_type="listens")
response = xmltodict.parse(xml_response)
self.assertEqual(response['scrobble']['track']['#text'], 'Saamne Ye Kaun Aya')
self.assertEqual(response['scrobble']['track']['@corrected'], '1')
self.assertEqual(response['scrobble']['artist']['#text'], 'Kishore Kumar')
self.assertEqual(response['scrobble']['artist']['@corrected'], '0')
self.assertEqual(response['scrobble']['album']['#text'], 'Jawani Diwani')
self.assertEqual(response['scrobble']['album']['@corrected'], '0')
self.assertEqual(response['scrobble']['timestamp'], str(timestamp))
# If album was corrected
original_listen['track'] = 'Saamne Ye Kaun Aya'
original_listen['album'] = 'Good Life'
xml_response = create_response_for_single_listen(original_listen, augmented_listen, listen_type="listens")
response = xmltodict.parse(xml_response)
self.assertEqual(response['scrobble']['track']['#text'], 'Saamne Ye Kaun Aya')
self.assertEqual(response['scrobble']['track']['@corrected'], '0')
self.assertEqual(response['scrobble']['artist']['#text'], 'Kishore Kumar')
self.assertEqual(response['scrobble']['artist']['@corrected'], '0')
self.assertEqual(response['scrobble']['album']['#text'], 'Jawani Diwani')
self.assertEqual(response['scrobble']['album']['@corrected'], '1')
self.assertEqual(response['scrobble']['timestamp'], str(timestamp))
class APICompatTestCase(APICompatIntegrationTestCase):
def setUp(self):
super(APICompatTestCase, self).setUp()
self.lb_user = db_user.get_or_create(1, 'apicompattestuser')
self.lfm_user = User(
self.lb_user['id'],
self.lb_user['created'],
self.lb_user['musicbrainz_id'],
self.lb_user['auth_token'],
)
self.ls = InfluxListenStore(
{
'REDIS_HOST': current_app.config['REDIS_HOST'],
'REDIS_PORT': current_app.config['REDIS_PORT'],
'REDIS_NAMESPACE': current_app.config['REDIS_NAMESPACE'],
'INFLUX_HOST': current_app.config['INFLUX_HOST'],
'INFLUX_PORT': current_app.config['INFLUX_PORT'],
'INFLUX_DB_NAME': current_app.config['INFLUX_DB_NAME'],
}, self.app.logger)
def test_record_listen_now_playing(self):
""" Tests if listen of type 'nowplaying' is recorded correctly
if valid information is provided.
"""
token = Token.generate(self.lfm_user.api_key)
token.approve(self.lfm_user.name)
session = Session.create(token)
data = {
'method': 'track.updateNowPlaying',
'api_key': self.lfm_user.api_key,
'sk': session.sid,
'artist[0]': 'Kishore Kumar',
'track[0]': 'Saamne Ye Kaun Aya',
'album[0]': 'Jawani Diwani',
'duration[0]': 300,
'timestamp[0]': int(time.time()),
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'ok')
self.assertIsNotNone(response['lfm']['nowplaying'])
def test_get_token(self):
""" Tests if the token generated by get_token method is valid. """
data = {
'method': 'auth.gettoken',
'api_key': self.lfm_user.api_key,
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'ok')
token = Token.load(response['lfm']['token'],
api_key=self.lfm_user.api_key)
self.assertIsNotNone(token)
def test_get_session(self):
""" Tests if the session key is valid and session is established correctly. """
token = Token.generate(self.lfm_user.api_key)
token.approve(self.lfm_user.name)
data = {
'method': 'auth.getsession',
'api_key': self.lfm_user.api_key,
'token': token.token,
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'ok')
self.assertEqual(response['lfm']['session']['name'],
self.lfm_user.name)
session_key = Session.load(response['lfm']['session']['key'])
self.assertIsNotNone(session_key)
def test_get_session_invalid_token(self):
""" Tests if correct error codes are returned in case token supplied
is invalid during establishment of session.
"""
data = {
'method': 'auth.getsession',
'api_key': self.lfm_user.api_key,
'token': '',
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'failed')
self.assertEqual(response['lfm']['error']['@code'], '4')
def test_record_listen(self):
""" Tests if listen is recorded correctly if valid information is provided. """
token = Token.generate(self.lfm_user.api_key)
token.approve(self.lfm_user.name)
session = Session.create(token)
timestamp = int(time.time())
data = {
'method': 'track.scrobble',
'api_key': self.lfm_user.api_key,
'sk': session.sid,
'artist[0]': 'Kishore Kumar',
'track[0]': 'Saamne Ye Kaun Aya',
'album[0]': 'Jawani Diwani',
'duration[0]': 300,
'timestamp[0]': timestamp,
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'ok')
self.assertEqual(response['lfm']['scrobbles']['@accepted'], '1')
# Check if listen reached the influx listenstore
time.sleep(1)
listens = self.ls.fetch_listens(self.lb_user['musicbrainz_id'],
from_ts=timestamp - 1)
self.assertEqual(len(listens), 1)
def test_record_listen_multiple_listens(self):
""" Tests if multiple listens get recorded correctly in case valid information
is provided.
"""
token = Token.generate(self.lfm_user.api_key)
token.approve(self.lfm_user.name)
session = Session.create(token)
timestamp = int(time.time())
data = {
'method': 'track.scrobble',
'api_key': self.lfm_user.api_key,
'sk': session.sid,
'artist[0]': 'Kishore Kumar',
'track[0]': 'Saamne Ye Kaun Aya',
'album[0]': 'Jawani Diwani',
'duration[0]': 300,
'timestamp[0]': timestamp,
'artist[1]': 'Fifth Harmony',
'track[1]': 'Deliver',
'duration[1]': 200,
'timestamp[1]': timestamp + 300,
}
r = self.client.post(url_for('api_compat.api_methods'), data=data)
self.assert200(r)
response = xmltodict.parse(r.data)
self.assertEqual(response['lfm']['@status'], 'ok')
self.assertEqual(response['lfm']['scrobbles']['@accepted'], '2')
# Check if listens reached the influx listenstore
time.sleep(1)
listens = self.ls.fetch_listens(self.lb_user['musicbrainz_id'],
from_ts=timestamp - 1)
self.assertEqual(len(listens), 2)
def test_create_response_for_single_listen(self):
""" Tests create_response_for_single_listen method in api_compat
to check if responses are generated correctly.
"""
from listenbrainz.webserver.views.api_compat import create_response_for_single_listen
timestamp = int(time.time())
original_listen = {
'artist': 'Kishore Kumar',
'track': 'Saamne Ye Kaun Aya',
'album': 'Jawani Diwani',
'duration': 300,
'timestamp': timestamp,
}
augmented_listen = {
'listened_at': timestamp,
'track_metadata': {
'artist_name': 'Kishore Kumar',
'track_name': 'Saamne Ye Kaun Aya',
'release_name': 'Jawani Diwani',
'additional_info': {
'track_length': 300
}
}
}
# If original listen and augmented listen are same
xml_response = create_response_for_single_listen(original_listen,
augmented_listen,
listen_type="listens")
response = xmltodict.parse(xml_response)
self.assertEqual(response['scrobble']['track']['#text'],
'Saamne Ye Kaun Aya')
self.assertEqual(response['scrobble']['track']['@corrected'], '0')
self.assertEqual(response['scrobble']['artist']['#text'],
'Kishore Kumar')
self.assertEqual(response['scrobble']['artist']['@corrected'], '0')
self.assertEqual(response['scrobble']['album']['#text'],
'Jawani Diwani')
self.assertEqual(response['scrobble']['timestamp'], str(timestamp))
# If listen type is 'playing_now'
xml_response = create_response_for_single_listen(
original_listen, augmented_listen, listen_type="playing_now")
response = xmltodict.parse(xml_response)
self.assertEqual(response['nowplaying']['track']['#text'],
'Saamne Ye Kaun Aya')
self.assertEqual(response['nowplaying']['track']['@corrected'], '0')
self.assertEqual(response['nowplaying']['artist']['#text'],
'Kishore Kumar')
self.assertEqual(response['nowplaying']['artist']['@corrected'], '0')
self.assertEqual(response['nowplaying']['album']['#text'],
'Jawani Diwani')
self.assertEqual(response['nowplaying']['album']['@corrected'], '0')
self.assertEqual(response['nowplaying']['timestamp'], str(timestamp))
# If artist was corrected
original_listen['artist'] = 'Pink'
xml_response = create_response_for_single_listen(original_listen,
augmented_listen,
listen_type="listens")
response = xmltodict.parse(xml_response)
self.assertEqual(response['scrobble']['track']['#text'],
'Saamne Ye Kaun Aya')
self.assertEqual(response['scrobble']['track']['@corrected'], '0')
self.assertEqual(response['scrobble']['artist']['#text'],
'Kishore Kumar')
self.assertEqual(response['scrobble']['artist']['@corrected'], '1')
self.assertEqual(response['scrobble']['album']['#text'],
'Jawani Diwani')
self.assertEqual(response['scrobble']['album']['@corrected'], '0')
self.assertEqual(response['scrobble']['timestamp'], str(timestamp))
# If track was corrected
original_listen['artist'] = 'Kishore Kumar'
original_listen['track'] = 'Deliver'
xml_response = create_response_for_single_listen(original_listen,
augmented_listen,
listen_type="listens")
response = xmltodict.parse(xml_response)
self.assertEqual(response['scrobble']['track']['#text'],
'Saamne Ye Kaun Aya')
self.assertEqual(response['scrobble']['track']['@corrected'], '1')
self.assertEqual(response['scrobble']['artist']['#text'],
'Kishore Kumar')
self.assertEqual(response['scrobble']['artist']['@corrected'], '0')
self.assertEqual(response['scrobble']['album']['#text'],
'Jawani Diwani')
self.assertEqual(response['scrobble']['album']['@corrected'], '0')
self.assertEqual(response['scrobble']['timestamp'], str(timestamp))
# If album was corrected
original_listen['track'] = 'Saamne Ye Kaun Aya'
original_listen['album'] = 'Good Life'
xml_response = create_response_for_single_listen(original_listen,
augmented_listen,
listen_type="listens")
response = xmltodict.parse(xml_response)
self.assertEqual(response['scrobble']['track']['#text'],
'Saamne Ye Kaun Aya')
self.assertEqual(response['scrobble']['track']['@corrected'], '0')
self.assertEqual(response['scrobble']['artist']['#text'],
'Kishore Kumar')
self.assertEqual(response['scrobble']['artist']['@corrected'], '0')
self.assertEqual(response['scrobble']['album']['#text'],
'Jawani Diwani')
self.assertEqual(response['scrobble']['album']['@corrected'], '1')
self.assertEqual(response['scrobble']['timestamp'], str(timestamp))
class InfluxWriterTestCase(IntegrationTestCase):
def setUp(self):
super(InfluxWriterTestCase, self).setUp()
self.ls = InfluxListenStore({ 'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'REDIS_NAMESPACE': config.REDIS_NAMESPACE,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME}, self.app.logger)
def send_listen(self, user, filename):
with open(self.path_to_data_file(filename)) as f:
payload = json.load(f)
return self.client.post(
url_for('api_v1.submit_listen'),
data = json.dumps(payload),
headers = {'Authorization': 'Token {}'.format(user['auth_token'])},
content_type = 'application/json'
)
def test_dedup(self):
user = db_user.get_or_create(1, 'testinfluxwriteruser')
# send the same listen twice
r = self.send_listen(user, 'valid_single.json')
self.assert200(r)
time.sleep(2)
r = self.send_listen(user, 'valid_single.json')
self.assert200(r)
time.sleep(2)
to_ts = int(time.time())
listens = self.ls.fetch_listens(user['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 1)
def test_dedup_user_special_characters(self):
user = db_user.get_or_create(2, 'i have a\\weird\\user, name"\n')
# send the same listen twice
r = self.send_listen(user, 'valid_single.json')
self.assert200(r)
time.sleep(2)
r = self.send_listen(user, 'valid_single.json')
self.assert200(r)
time.sleep(2)
to_ts = int(time.time())
listens = self.ls.fetch_listens(user['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 1)
def test_dedup_same_batch(self):
user = db_user.get_or_create(3, 'phifedawg')
r = self.send_listen(user, 'same_batch_duplicates.json')
self.assert200(r)
time.sleep(2)
to_ts = int(time.time())
listens = self.ls.fetch_listens(user['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 1)
def test_dedup_different_users(self):
"""
Test to make sure influx writer doesn't confuse listens with same timestamps
but different users to be duplicates
"""
user1 = db_user.get_or_create(1, 'testuser1')
user2 = db_user.get_or_create(2, 'testuser2')
r = self.send_listen(user1, 'valid_single.json')
self.assert200(r)
r = self.send_listen(user2, 'valid_single.json')
self.assert200(r)
time.sleep(2) # sleep to allow influx-writer to do its thing
to_ts = int(time.time())
listens = self.ls.fetch_listens(user1['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 1)
listens = self.ls.fetch_listens(user2['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 1)
def test_dedup_same_timestamp_different_tracks(self):
""" Test to check that if there are two tracks w/ the same timestamp,
they don't get considered as duplicates
"""
user = db_user.get_or_create(1, 'difftracksametsuser')
# send four different tracks with the same timestamp
r = self.send_listen(user, 'valid_single.json')
self.assert200(r)
r = self.send_listen(user, 'same_timestamp_diff_track_valid_single.json')
self.assert200(r)
r = self.send_listen(user, 'same_timestamp_diff_track_valid_single_2.json')
self.assert200(r)
r = self.send_listen(user, 'same_timestamp_diff_track_valid_single_3.json')
self.assert200(r)
time.sleep(2)
to_ts = int(time.time())
listens = self.ls.fetch_listens(user['musicbrainz_id'], to_ts=to_ts)
self.assertEqual(len(listens), 4)
#!/usr/bin/env python3
# This script flushes out the listencounts that are in the 7day retention policy and sums them
# into the permanent listen count measurement.
from listenbrainz import default_config as config
try:
from listenbrainz import custom_config as config
except ImportError:
pass
from listenbrainz.listenstore import InfluxListenStore
ls = InfluxListenStore({
'REDIS_HOST': config.REDIS_HOST,
'REDIS_PORT': config.REDIS_PORT,
'INFLUX_HOST': config.INFLUX_HOST,
'INFLUX_PORT': config.INFLUX_PORT,
'INFLUX_DB_NAME': config.INFLUX_DB_NAME
})
ls.update_listen_counts()
def start(self):
app = create_app()
with app.app_context():
current_app.logger.info("influx-writer init")
self._verify_hosts_in_config()
if "INFLUX_HOST" not in current_app.config:
current_app.logger.critical(
"Influx service not defined. Sleeping {0} seconds and exiting."
.format(self.ERROR_RETRY_DELAY))
sleep(self.ERROR_RETRY_DELAY)
sys.exit(-1)
while True:
try:
self.ls = InfluxListenStore(
{
'REDIS_HOST':
current_app.config['REDIS_HOST'],
'REDIS_PORT':
current_app.config['REDIS_PORT'],
'REDIS_NAMESPACE':
current_app.config['REDIS_NAMESPACE'],
'INFLUX_HOST':
current_app.config['INFLUX_HOST'],
'INFLUX_PORT':
current_app.config['INFLUX_PORT'],
'INFLUX_DB_NAME':
current_app.config['INFLUX_DB_NAME'],
},
logger=current_app.logger)
self.influx = InfluxDBClient(
host=current_app.config['INFLUX_HOST'],
port=current_app.config['INFLUX_PORT'],
database=current_app.config['INFLUX_DB_NAME'],
)
break
except Exception as err:
current_app.logger.error(
"Cannot connect to influx: %s. Retrying in 2 seconds and trying again."
% str(err),
exc_info=True)
sleep(self.ERROR_RETRY_DELAY)
while True:
try:
self.redis = Redis(host=current_app.config['REDIS_HOST'],
port=current_app.config['REDIS_PORT'],
decode_responses=True)
self.redis.ping()
self.redis_listenstore = RedisListenStore(
current_app.logger, current_app.config)
break
except Exception as err:
current_app.logger.error(
"Cannot connect to redis: %s. Retrying in 2 seconds and trying again."
% str(err),
exc_info=True)
sleep(self.ERROR_RETRY_DELAY)
while True:
self.connect_to_rabbitmq()
self.incoming_ch = self.connection.channel()
self.incoming_ch.exchange_declare(
exchange=current_app.config['INCOMING_EXCHANGE'],
exchange_type='fanout')
self.incoming_ch.queue_declare(
current_app.config['INCOMING_QUEUE'], durable=True)
self.incoming_ch.queue_bind(
exchange=current_app.config['INCOMING_EXCHANGE'],
queue=current_app.config['INCOMING_QUEUE'])
self.incoming_ch.basic_consume(
lambda ch, method, properties, body: self.static_callback(
ch, method, properties, body, obj=self),
queue=current_app.config['INCOMING_QUEUE'],
)
self.unique_ch = self.connection.channel()
self.unique_ch.exchange_declare(
exchange=current_app.config['UNIQUE_EXCHANGE'],
exchange_type='fanout')
current_app.logger.info("influx-writer started")
try:
self.incoming_ch.start_consuming()
except pika.exceptions.ConnectionClosed:
current_app.logger.warn(
"Connection to rabbitmq closed. Re-opening.",
exc_info=True)
self.connection = None
continue
self.connection.close()
