import sys
import listenbrainz.utils as utils
import time
utils.safely_import_config()
from flask import current_app
class ListenWriter:
def __init__(self):
self.redis = None
self.connection = None
self.total_inserts = 0
self.inserts = 0
self.time = 0
self.REPORT_FREQUENCY = 5000
self.DUMP_JSON_WITH_ERRORS = False
self.ERROR_RETRY_DELAY = 3 # number of seconds to wait until retrying an operation
@staticmethod
def static_callback(ch, method, properties, body, obj):
return obj.callback(ch, method, properties, body)
def connect_to_rabbitmq(self):
connection_config = {
'username': current_app.config['RABBITMQ_USERNAME'],
'password': current_app.config['RABBITMQ_PASSWORD'],
'host': current_app.config['RABBITMQ_HOST'],
'port': current_app.config['RABBITMQ_PORT'],
#!/usr/bin/env python3 import json import listenbrainz.utils as utils import logging import os import pika import sys import ujson import time from listenbrainz.utils import safely_import_config safely_import_config() from googleapiclient import discovery from googleapiclient.errors import HttpError from listenbrainz.listen_writer import ListenWriter from listenbrainz.bigquery import create_bigquery_object from listenbrainz.bigquery import NoCredentialsVariableException, NoCredentialsFileException from oauth2client.client import GoogleCredentials from redis import Redis from time import time, sleep SUBMIT_CHUNK_SIZE = 1000 # the number of listens to send to BQ in one batch # NOTE: this MUST be greater than or equal to the maximum number of listens sent to us in one # RabbitMQ batch, otherwise BigQueryWriter will submit a partial batch and send an ack for # the batch. assert (SUBMIT_CHUNK_SIZE >= 50)
#!/usr/bin/python3
import time
import listenbrainz.webserver
import json
from listenbrainz.utils import safely_import_config
safely_import_config()
from dateutil import parser
from flask import current_app
from listenbrainz.domain import spotify
from listenbrainz.webserver.views.api_tools import insert_payload, validate_listen, LISTEN_TYPE_IMPORT
from listenbrainz.db import user as db_user
from listenbrainz.db.exceptions import DatabaseException
from spotipy import SpotifyException
from werkzeug.exceptions import BadRequest, InternalServerError, ServiceUnavailable
def _convert_spotify_play_to_listen(play):
""" Converts data retrieved from the Spotify API into a listen.
Args:
play (dict): a dict that represents a listen retrieved from Spotify
Returns:
listen (dict): dict that can be submitted to ListenBrainz
"""
track = play['track']
album = track['album']
# Spotify provides microseconds, but we only give seconds to listenbrainz
listened_at = int(parser.parse(play['played_at']).timestamp())
