import sys import listenbrainz.utils as utils import time utils.safely_import_config() from flask import current_app class ListenWriter: def __init__(self): self.redis = None self.connection = None self.total_inserts = 0 self.inserts = 0 self.time = 0 self.REPORT_FREQUENCY = 5000 self.DUMP_JSON_WITH_ERRORS = False self.ERROR_RETRY_DELAY = 3 # number of seconds to wait until retrying an operation @staticmethod def static_callback(ch, method, properties, body, obj): return obj.callback(ch, method, properties, body) def connect_to_rabbitmq(self): connection_config = { 'username': current_app.config['RABBITMQ_USERNAME'], 'password': current_app.config['RABBITMQ_PASSWORD'], 'host': current_app.config['RABBITMQ_HOST'], 'port': current_app.config['RABBITMQ_PORT'],
#!/usr/bin/env python3 import json import listenbrainz.utils as utils import logging import os import pika import sys import ujson import time from listenbrainz.utils import safely_import_config safely_import_config() from googleapiclient import discovery from googleapiclient.errors import HttpError from listenbrainz.listen_writer import ListenWriter from listenbrainz.bigquery import create_bigquery_object from listenbrainz.bigquery import NoCredentialsVariableException, NoCredentialsFileException from oauth2client.client import GoogleCredentials from redis import Redis from time import time, sleep SUBMIT_CHUNK_SIZE = 1000 # the number of listens to send to BQ in one batch # NOTE: this MUST be greater than or equal to the maximum number of listens sent to us in one # RabbitMQ batch, otherwise BigQueryWriter will submit a partial batch and send an ack for # the batch. assert (SUBMIT_CHUNK_SIZE >= 50)
#!/usr/bin/python3 import time import listenbrainz.webserver import json from listenbrainz.utils import safely_import_config safely_import_config() from dateutil import parser from flask import current_app from listenbrainz.domain import spotify from listenbrainz.webserver.views.api_tools import insert_payload, validate_listen, LISTEN_TYPE_IMPORT from listenbrainz.db import user as db_user from listenbrainz.db.exceptions import DatabaseException from spotipy import SpotifyException from werkzeug.exceptions import BadRequest, InternalServerError, ServiceUnavailable def _convert_spotify_play_to_listen(play): """ Converts data retrieved from the Spotify API into a listen. Args: play (dict): a dict that represents a listen retrieved from Spotify Returns: listen (dict): dict that can be submitted to ListenBrainz """ track = play['track'] album = track['album'] # Spotify provides microseconds, but we only give seconds to listenbrainz listened_at = int(parser.parse(play['played_at']).timestamp())