def test_to_influx(self):
listen = Listen(timestamp=int(time.time()),
user_name='testuser',
artist_msid=uuid.uuid4(),
recording_msid=uuid.uuid4(),
data={
'artist_name': 'Radiohead',
'track_name': 'True Love Waits',
'additional_info': {
'release_type': ["ALBUM", "REMIX"],
}
})
data = listen.to_influx(quote(listen.user_name))
# Make sure every value that we don't explicitly support is a string
for key in data['fields']:
if key not in Listen.SUPPORTED_KEYS:
print(key)
self.assertIsInstance(data['fields'][key], str)
# Check values
self.assertEqual(data['measurement'], quote(listen.user_name))
self.assertEqual(data['time'], listen.ts_since_epoch)
self.assertEqual(data['tags']['user_name'], listen.user_name)
self.assertEqual(data['fields']['artist_msid'], listen.artist_msid)
self.assertEqual(data['fields']['recording_msid'],
listen.recording_msid)
self.assertEqual(data['fields']['track_name'],
listen.data['track_name'])
self.assertEqual(data['fields']['artist_name'],
listen.data['artist_name'])
def create_test_data_for_timescalelistenstore(user_name: str, user_id: int, test_data_file_name: str = None):
"""Create listens for timescalelistenstore tests.
From a json file in testdata it creates Listen objects with a specified user_name for tests.
Args:
user_name: MusicBrainz username of a user.
user_id: listenbrainz row id of the user
test_data_file_name: If specified use the given file to create Listen objects.
DEFAULT = 'timescale_listenstore_test_listens.json'
Returns:
A list of Listen objects.
"""
if not test_data_file_name:
test_data_file_name = 'timescale_listenstore_test_listens.json'
test_data_file = os.path.join(TEST_DATA_PATH, test_data_file_name)
with open(test_data_file, 'r') as f:
listens = json.load(f)
test_data = []
for listen in listens['payload']:
listen['user_name'] = user_name
listen['user_id'] = user_id
test_data.append(Listen().from_json(listen))
return test_data
def test_to_timescale(self):
listen = Listen(
timestamp=int(time.time()),
user_name='testuser',
artist_msid=str(uuid.uuid4()),
dedup_tag=3,
user_id=1,
data={
'artist_name': 'Radiohead',
'track_name': 'True Love Waits',
'additional_info': {
'release_type': ["ALBUM", "REMIX"],
'recording_msid': str(uuid.uuid4()),
}
}
)
listened_at, track_name, user_name, data = listen.to_timescale()
# Check data is of type string
self.assertIsInstance(data, str)
# Convert returned data to json
json_data = ujson.loads(data)
# Check that the required fields are dumped into data
self.assertIn('track_metadata', json_data)
self.assertIn('additional_info', json_data['track_metadata'])
# Check that the required fields are dumped into data
self.assertEqual(listened_at, listen.ts_since_epoch)
self.assertEqual(track_name, listen.data['track_name'])
self.assertEqual(user_name, listen.user_name)
self.assertEqual(json_data['user_id'], listen.user_id)
self.assertEqual(json_data['track_metadata']['artist_name'], listen.data['artist_name'])
def test_update_and_get_recent_listens(self):
recent = self._redis.get_recent_listens()
self.assertEqual(recent, [])
listens = []
t = int(time.time())
for i in range(RedisListenStore.RECENT_LISTENS_MAX * 3):
listen = Listen(user_id=self.testuser['id'],
user_name=self.testuser['musicbrainz_id'],
timestamp=t - i,
data={
'artist_name': str(uuid.uuid4()),
'track_name': str(uuid.uuid4()),
'additional_info': {},
})
listens.append(listen)
self._redis.update_recent_listens(listens)
recent = self._redis.get_recent_listens()
self.assertEqual(len(recent), RedisListenStore.RECENT_LISTENS_MAX)
self.assertIsInstance(recent[0], Listen)
for i, r in enumerate(recent):
self.assertEqual(r.timestamp, listens[i].timestamp)
recent = self._redis.get_recent_listens(5)
self.assertEqual(len(recent), 5)
for i, r in enumerate(recent):
self.assertEqual(r.timestamp, listens[i].timestamp)
def generate_data(test_user_id, user_name, from_ts, num_records, inserted_ts=None):
test_data = []
artist_msid = str(uuid.uuid4())
for i in range(num_records):
if not inserted_ts:
inserted_timestamp = datetime.utcnow()
else:
inserted_timestamp = datetime.utcfromtimestamp(inserted_ts)
timestamp = datetime.utcfromtimestamp(from_ts)
item = Listen(
user_name=user_name,
user_id=test_user_id,
timestamp=timestamp,
artist_msid=artist_msid,
recording_msid=str(uuid.uuid4()),
inserted_timestamp=inserted_timestamp,
data={
'artist_name': 'Frank Ocean',
'track_name': 'Crack Rock',
'additional_info': {},
},
)
test_data.append(item)
from_ts += 1 # Add one second
if inserted_ts:
inserted_ts += 1 # Add one second
return test_data
def generate_data(test_user_id, user_name, from_ts, num_records):
test_data = []
artist_msid = str(uuid.uuid4())
if from_ts == None: #check for playing now listens
timestamp = None
else:
from_ts += 1 # Add one second
timestamp = datetime.utcfromtimestamp(from_ts)
for i in range(num_records):
item = Listen(
user_name=user_name,
user_id=test_user_id,
timestamp=timestamp,
artist_msid=artist_msid,
recording_msid=str(uuid.uuid4()),
data={
'artist_name': 'Frank Ocean',
'track_name': 'Crack Rock',
'additional_info': {},
},
)
test_data.append(item)
return test_data
def generate_data(from_date, num_records, user_name):
test_data = []
current_date = to_epoch(from_date)
artist_msid = str(uuid.uuid4())
user = db_user.get_by_mb_id(user_name)
if not user:
db_user.create(user_name)
user = db_user.get_by_mb_id(user_name)
for i in range(num_records):
current_date += 1 # Add one second
item = Listen(
user_id=user['id'],
user_name=user_name,
timestamp=datetime.utcfromtimestamp(current_date),
artist_msid=artist_msid,
recording_msid=str(uuid.uuid4()),
release_msid=str(uuid.uuid4()),
data={
'artist_name': 'Test Artist Pls ignore',
'track_name': 'Hello Goodbye',
'additional_info': {},
},
)
test_data.append(item)
return test_data
def _create_test_data(self, user_name):
test_data = []
for jdata in TEST_LISTEN_JSON:
x = ujson.loads(jdata)
x['user_name'] = user_name
test_data.append(Listen().from_json(x))
self.logstore.insert(test_data)
return len(test_data)
def convert_row(self, row):
return Listen(user_id=row[1],
user_name=row[2],
timestamp=row[3],
artist_msid=row[4],
release_msid=row[5],
recording_msid=row[6],
data=row[7])
def generate_data(test_user_id, from_ts, num_records):
test_data = []
artist_msid = str(uuid.uuid4())
for i in range(num_records):
from_ts += 1 # Add one second
item = Listen(user_id=test_user_id,
timestamp=datetime.utcfromtimestamp(from_ts),
artist_msid=artist_msid,
recording_msid=str(uuid.uuid4()))
test_data.append(item)
return test_data
def create_test_data_for_influxlistenstore(user_name):
"""Create listens for influxlistenstore tests.
From a json file 'influx_listenstore_test_listens.json' in testdata
it creates Listen objects with a specified user_name for tests.
Args:
user_name (str): MusicBrainz username of a user.
Returns:
A list of Listen objects.
"""
test_data_file = os.path.join(TEST_DATA_PATH,
'influx_listenstore_test_listens.json')
with open(test_data_file, 'r') as f:
listens = json.load(f)
test_data = []
for listen in listens['payload']:
listen['user_name'] = user_name
test_data.append(Listen().from_json(listen))
return test_data
def test_export_streaming(self, mock_fetch_listens):
self.temporary_login(self.user['login_id'])
# Three example listens, with only basic data for the purpose of this test.
# In each listen, one of {release_artist, release_msid, recording_msid}
# is missing.
listens = [
Listen(
timestamp=1539509881,
artist_msid='61746abb-76a5-465d-aee7-c4c42d61b7c4',
recording_msid='6c617681-281e-4dae-af59-8e00f93c4376',
data={
'artist_name': 'Massive Attack',
'track_name': 'The Spoils',
'additional_info': {},
},
),
Listen(
timestamp=1539441702,
release_msid='0c1d2dc3-3704-4e75-92f9-940801a1eebd',
recording_msid='7ad53fd7-5b40-4e13-b680-52716fb86d5f',
data={
'artist_name': 'Snow Patrol',
'track_name': 'Lifening',
'additional_info': {},
},
),
Listen(
timestamp=1539441531,
release_msid='7816411a-2cc6-4e43-b7a1-60ad093c2c31',
artist_msid='7e2c6fe4-3e3f-496e-961d-dce04a44f01b',
data={
'artist_name': 'Muse',
'track_name': 'Drones',
'additional_info': {},
},
),
]
# We expect three calls to fetch_listens, and we return two, one, and
# zero listens in the batch. This tests that we fetch all batches.
mock_fetch_listens.side_effect = [listens[0:2], listens[2:3], []]
r = self.client.post(url_for('profile.export_data'))
self.assert200(r)
# r.json returns None, so we decode the response manually.
results = ujson.loads(r.data.decode('utf-8'))
self.assertDictEqual(
results[0], {
'inserted_at': 0,
'listened_at': 1539509881,
'recording_msid': '6c617681-281e-4dae-af59-8e00f93c4376',
'user_name': None,
'track_metadata': {
'artist_name': 'Massive Attack',
'track_name': 'The Spoils',
'additional_info': {
'artist_msid': '61746abb-76a5-465d-aee7-c4c42d61b7c4',
'release_msid': None,
},
},
})
self.assertDictEqual(
results[1], {
'inserted_at': 0,
'listened_at': 1539441702,
'recording_msid': '7ad53fd7-5b40-4e13-b680-52716fb86d5f',
'user_name': None,
'track_metadata': {
'artist_name': 'Snow Patrol',
'track_name': 'Lifening',
'additional_info': {
'artist_msid': None,
'release_msid': '0c1d2dc3-3704-4e75-92f9-940801a1eebd',
},
},
})
self.assertDictEqual(
results[2], {
'inserted_at': 0,
'listened_at': 1539441531,
'recording_msid': None,
'user_name': None,
'track_metadata': {
'artist_name': 'Muse',
'track_name': 'Drones',
'additional_info': {
'artist_msid': '7e2c6fe4-3e3f-496e-961d-dce04a44f01b',
'release_msid': '7816411a-2cc6-4e43-b7a1-60ad093c2c31',
},
},
})