def __init__(self, parent=None): super().__init__(parent) self.manager = QtNetwork.QNetworkAccessManager() self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() self.manager.finished.connect(self._process_reply) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self._init_queues() self._init_timers()
def test_query_data(self): params = { 'a&b': 'a b', 'c d': 'c&d', 'e=f': 'e=f', '': '', } data = OAuthManager._query_data(params) self.assertEqual(data, "a%26b=a+b&c+d=c%26d&e%3Df=e%3Df")
def __init__(self, parent=None): super().__init__(parent) self.manager = QtNetwork.QNetworkAccessManager() self._network_accessible_changed(self.manager.networkAccessible()) self.manager.networkAccessibleChanged.connect(self._network_accessible_changed) self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() config = get_config() self.set_transfer_timeout(config.setting['network_transfer_timeout_seconds']) self.manager.finished.connect(self._process_reply) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self._init_queues() self._init_timers()
def __init__(self, parent=None): QtCore.QObject.__init__(self, parent) self.manager = QtNetwork.QNetworkAccessManager() self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() self.manager.finished.connect(self._process_reply) self._last_request_times = {} self._active_requests = {} self._high_priority_queues = {} self._low_priority_queues = {} self._hosts = [] self._timer = QtCore.QTimer(self) self._timer.setSingleShot(True) self._timer.timeout.connect(self._run_next_task) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self.num_pending_web_requests = 0
def __init__(self, parent=None): QtCore.QObject.__init__(self, parent) self.manager = QtNetwork.QNetworkAccessManager() self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() self.manager.finished.connect(self._process_reply) self._last_request_times = defaultdict(lambda: 0) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource, } self._init_queues() self._init_timers()
class XmlWebService(QtCore.QObject): def __init__(self, parent=None): QtCore.QObject.__init__(self, parent) self.manager = QtNetwork.QNetworkAccessManager() self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() self.manager.finished.connect(self._process_reply) self._last_request_times = defaultdict(lambda: 0) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self._init_queues() self._init_timers() def _init_queues(self): self._active_requests = {} self._queues = defaultdict(lambda: defaultdict(deque)) self.num_pending_web_requests = 0 self._last_num_pending_web_requests = -1 def _init_timers(self): self._timer_run_next_task = QtCore.QTimer(self) self._timer_run_next_task.setSingleShot(True) self._timer_run_next_task.timeout.connect(self._run_next_task) self._timer_count_pending_requests = QtCore.QTimer(self) self._timer_count_pending_requests.setSingleShot(True) self._timer_count_pending_requests.timeout.connect(self._count_pending_requests) def set_cache(self, cache_size_in_mb=100): cache = QtNetwork.QNetworkDiskCache() location = QDesktopServices.storageLocation(QDesktopServices.CacheLocation) cache.setCacheDirectory(os.path.join(unicode(location), u'picard')) cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024) self.manager.setCache(cache) log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory()) log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(), cache.maximumCacheSize()) def setup_proxy(self): proxy = QtNetwork.QNetworkProxy() if config.setting["use_proxy"]: proxy.setType(QtNetwork.QNetworkProxy.HttpProxy) proxy.setHostName(config.setting["proxy_server_host"]) proxy.setPort(config.setting["proxy_server_port"]) proxy.setUser(config.setting["proxy_username"]) proxy.setPassword(config.setting["proxy_password"]) self.manager.setProxy(proxy) def _start_request_continue(self, method, host, port, path, data, handler, xml, mblogin=False, cacheloadcontrol=None, refresh=None, access_token=None, queryargs=None): url = build_qurl(host, port, path=path, mblogin=mblogin, queryargs=queryargs) request = QtNetwork.QNetworkRequest(url) if mblogin and access_token: request.setRawHeader("Authorization", "Bearer %s" % access_token) if mblogin or (method == "GET" and refresh): request.setPriority(QtNetwork.QNetworkRequest.HighPriority) request.setAttribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute, QtNetwork.QNetworkRequest.AlwaysNetwork) elif method == "PUT" or method == "DELETE": request.setPriority(QtNetwork.QNetworkRequest.HighPriority) elif cacheloadcontrol is not None: request.setAttribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute, cacheloadcontrol) request.setRawHeader("User-Agent", USER_AGENT_STRING) if xml: request.setRawHeader("Accept", "application/xml") if data is not None: if method == "POST" and host == config.setting["server_host"] and xml: request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/xml; charset=utf-8") else: request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/x-www-form-urlencoded") send = self._request_methods[method] reply = send(request, data) if data is not None else send(request) self._remember_request_time((host, port)) self._active_requests[reply] = (request, handler, xml, refresh) def _start_request(self, method, host, port, path, data, handler, xml, mblogin=False, cacheloadcontrol=None, refresh=None, queryargs=None): def start_request_continue(access_token=None): self._start_request_continue( method, host, port, path, data, handler, xml, mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, access_token=access_token, queryargs=queryargs) if mblogin and path != "/oauth2/token": self.oauth_manager.get_access_token(start_request_continue) else: start_request_continue() @staticmethod def urls_equivalent(leftUrl, rightUrl): """ Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash from QT 4.7 """ return leftUrl.port(80) == rightUrl.port(80) and \ leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort) def _handle_reply(self, reply, request, handler, xml, refresh): error = int(reply.error()) if error: log.error("Network request error for %s: %s (QT code %d, HTTP code %s)", reply.request().url().toString(QUrl.RemoveUserInfo), reply.errorString(), error, repr(reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute)) ) if handler is not None: handler(str(reply.readAll()), reply, error) else: redirect = reply.attribute(QtNetwork.QNetworkRequest.RedirectionTargetAttribute) fromCache = reply.attribute(QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute) cached = ' (CACHED)' if fromCache else '' log.debug("Received reply for %s: HTTP %d (%s) %s", reply.request().url().toString(QUrl.RemoveUserInfo), reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute), reply.attribute(QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute), cached ) if handler is not None: # Redirect if found and not infinite if redirect: url = request.url() # merge with base url (to cover the possibility of the URL being relative) redirect = url.resolved(redirect) if not XmlWebService.urls_equivalent(redirect, reply.request().url()): log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo)) redirect_host = str(redirect.host()) redirect_port = redirect.port(80) redirect_query = dict(redirect.queryItems()) redirect_path = redirect.path() original_host = str(url.host()) original_port = url.port(80) if ((original_host, original_port) in REQUEST_DELAY and (redirect_host, redirect_port) not in REQUEST_DELAY): log.debug("Setting rate limit for %s:%i to %i" % (redirect_host, redirect_port, REQUEST_DELAY[(original_host, original_port)])) REQUEST_DELAY[(redirect_host, redirect_port)] =\ REQUEST_DELAY[(original_host, original_port)] self.get(redirect_host, redirect_port, redirect_path, handler, xml, priority=True, important=True, refresh=refresh, queryargs=redirect_query, cacheloadcontrol=request.attribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute)) else: log.error("Redirect loop: %s", reply.request().url().toString(QUrl.RemoveUserInfo) ) handler(str(reply.readAll()), reply, error) elif xml: document = _read_xml(QXmlStreamReader(reply)) handler(document, reply, error) else: handler(str(reply.readAll()), reply, error) def _process_reply(self, reply): try: request, handler, xml, refresh = self._active_requests.pop(reply) except KeyError: log.error("Request not found for %s" % reply.request().url().toString(QUrl.RemoveUserInfo)) return try: self._handle_reply(reply, request, handler, xml, refresh) finally: reply.close() reply.deleteLater() def get(self, host, port, path, handler, xml=True, priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False, queryargs=None): func = partial(self._start_request, "GET", host, port, path, None, handler, xml, mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs) return self.add_task(func, host, port, priority, important=important) def post(self, host, port, path, data, handler, xml=True, priority=False, important=False, mblogin=True, queryargs=None): log.debug("POST-DATA %r", data) func = partial(self._start_request, "POST", host, port, path, data, handler, xml, mblogin, queryargs=queryargs) return self.add_task(func, host, port, priority, important=important) def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True, queryargs=None): func = partial(self._start_request, "PUT", host, port, path, data, handler, False, mblogin, queryargs=queryargs) return self.add_task(func, host, port, priority, important=important) def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True, queryargs=None): func = partial(self._start_request, "DELETE", host, port, path, None, handler, False, mblogin, queryargs=queryargs) return self.add_task(func, host, port, priority, important=important) def stop(self): for reply in self._active_requests.keys(): reply.abort() self._init_queues() def _count_pending_requests(self): count = len(self._active_requests) for prio_queue in self._queues.values(): for queue in prio_queue.values(): count += len(queue) self.num_pending_web_requests = count if count != self._last_num_pending_web_requests: self._last_num_pending_web_requests = count self.tagger.tagger_stats_changed.emit() if count: self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS) def _get_delay_to_next_request(self, hostkey): """Calculate delay to next request to hostkey (host, port) returns a tuple (wait, delay) where: wait is True if a delay is needed delay is the delay in milliseconds to next request """ interval = REQUEST_DELAY[hostkey] if not interval: log.debug("WSREQ: Starting another request to %s without delay", hostkey) return (False, 0) last_request = self._last_request_times[hostkey] if not last_request: log.debug("WSREQ: First request to %s", hostkey) self._remember_request_time(hostkey) # set it on first run return (False, interval) elapsed = (time.time() - last_request) * 1000 if elapsed >= interval: log.debug("WSREQ: Last request to %s was %d ms ago, starting another one", hostkey, elapsed) return (False, interval) delay = int(math.ceil(interval - elapsed)) log.debug("WSREQ: Last request to %s was %d ms ago, waiting %d ms before starting another one", hostkey, elapsed, delay) return (True, delay) def _remember_request_time(self, hostkey): if REQUEST_DELAY[hostkey]: self._last_request_times[hostkey] = time.time() def _run_next_task(self): delay = sys.maxsize for prio in sorted(self._queues.keys(), reverse=True): prio_queue = self._queues[prio] if not prio_queue: del(self._queues[prio]) continue for hostkey in sorted(prio_queue.keys(), key=lambda hostkey: REQUEST_DELAY[hostkey]): queue = self._queues[prio][hostkey] if not queue: del(self._queues[prio][hostkey]) continue wait, d = self._get_delay_to_next_request(hostkey) if not wait: queue.popleft()() if d < delay: delay = d if delay < sys.maxsize: self._timer_run_next_task.start(delay) def add_task(self, func, host, port, priority, important=False): hostkey = (host, port) prio = int(priority) # priority is a boolean if important: self._queues[prio][hostkey].appendleft(func) else: self._queues[prio][hostkey].append(func) if not self._timer_run_next_task.isActive(): self._timer_run_next_task.start(0) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) return (hostkey, func, prio) def remove_task(self, task): hostkey, func, prio = task try: self._queues[prio][hostkey].remove(func) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) except: pass def _get_by_id(self, entitytype, entityid, handler, inc=[], queryargs=None, priority=False, important=False, mblogin=False, refresh=False): host = config.setting["server_host"] port = config.setting["server_port"] path = "/ws/2/%s/%s" % (entitytype, entityid) if queryargs is None: queryargs = {} if inc: queryargs["inc"] = "+".join(inc) return self.get(host, port, path, handler, priority=priority, important=important, mblogin=mblogin, refresh=refresh, queryargs=queryargs) def get_release_by_id(self, releaseid, handler, inc=[], priority=False, important=False, mblogin=False, refresh=False): return self._get_by_id('release', releaseid, handler, inc, priority=priority, important=important, mblogin=mblogin, refresh=refresh) def get_track_by_id(self, trackid, handler, inc=[], priority=False, important=False, mblogin=False, refresh=False): return self._get_by_id('recording', trackid, handler, inc, priority=priority, important=important, mblogin=mblogin, refresh=refresh) def lookup_discid(self, discid, handler, priority=True, important=True, refresh=False): inc = ['artist-credits', 'labels'] return self._get_by_id('discid', discid, handler, inc, queryargs={"cdstubs": "no"}, priority=priority, important=important, refresh=refresh) def _find(self, entitytype, handler, kwargs): host = config.setting["server_host"] port = config.setting["server_port"] filters = [] query = [] for name, value in kwargs.items(): if name == 'limit': filters.append((name, str(value))) else: value = _escape_lucene_query(value).strip().lower() if value: query.append('%s:(%s)' % (name, value)) if query: filters.append(('query', ' '.join(query))) queryargs = {} for name, value in filters: value = QUrl.toPercentEncoding(unicode(value)) queryargs[str(name)] = value path = "/ws/2/%s" % (entitytype) return self.get(host, port, path, handler, queryargs=queryargs) def find_releases(self, handler, **kwargs): return self._find('release', handler, kwargs) def find_tracks(self, handler, **kwargs): return self._find('recording', handler, kwargs) def _browse(self, entitytype, handler, kwargs, inc=[], priority=False, important=False): host = config.setting["server_host"] port = config.setting["server_port"] path = "/ws/2/%s" % (entitytype) queryargs = kwargs if inc: queryargs["inc"] = "+".join(inc) return self.get(host, port, path, handler, priority=priority, important=important, queryargs=queryargs) def browse_releases(self, handler, priority=True, important=True, **kwargs): inc = ["media", "labels"] return self._browse("release", handler, kwargs, inc, priority=priority, important=important) def submit_ratings(self, ratings, handler): host = config.setting['server_host'] port = config.setting['server_port'] path = '/ws/2/rating/?client=' + CLIENT_STRING recordings = (''.join(['<recording id="%s"><user-rating>%s</user-rating></recording>' % (i[1], j*20) for i, j in ratings.items() if i[0] == 'recording'])) data = _wrap_xml_metadata('<recording-list>%s</recording-list>' % recordings) return self.post(host, port, path, data, handler, priority=True) def _encode_acoustid_args(self, args, format='xml'): filters = [] args['client'] = ACOUSTID_KEY args['clientversion'] = PICARD_VERSION_STR args['format'] = format for name, value in args.items(): value = str(QUrl.toPercentEncoding(value)) filters.append('%s=%s' % (str(name), value)) return '&'.join(filters) def query_acoustid(self, handler, **args): host, port = ACOUSTID_HOST, ACOUSTID_PORT body = self._encode_acoustid_args(args) return self.post(host, port, '/v2/lookup', body, handler, priority=False, important=False, mblogin=False) def submit_acoustid_fingerprints(self, submissions, handler): args = {'user': config.setting["acoustid_apikey"]} for i, submission in enumerate(submissions): args['fingerprint.%d' % i] = str(submission.fingerprint) args['duration.%d' % i] = str(submission.duration) args['mbid.%d' % i] = str(submission.recordingid) if submission.puid: args['puid.%d' % i] = str(submission.puid) host, port = ACOUSTID_HOST, ACOUSTID_PORT body = self._encode_acoustid_args(args, format='json') return self.post(host, port, '/v2/submit', body, handler, priority=True, important=False, mblogin=False) def download(self, host, port, path, handler, priority=False, important=False, cacheloadcontrol=None, refresh=False, queryargs=None): return self.get(host, port, path, handler, xml=False, priority=priority, important=important, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs) def get_collection(self, id, handler, limit=100, offset=0): host, port = config.setting['server_host'], config.setting['server_port'] path = "/ws/2/collection" queryargs = None if id is not None: inc = ["releases", "artist-credits", "media"] path += "/%s/releases" % (id) queryargs = {} queryargs["inc"] = "+".join(inc) queryargs["limit"] = limit queryargs["offset"] = offset return self.get(host, port, path, handler, priority=True, important=True, mblogin=True, queryargs=queryargs) def get_collection_list(self, handler): return self.get_collection(None, handler) def _collection_request(self, id, releases): while releases: ids = ";".join(releases if len(releases) <= 400 else releases[:400]) releases = releases[400:] yield "/ws/2/collection/%s/releases/%s" % (id, ids) def _get_client_queryarg(self): return {"client": CLIENT_STRING} def put_to_collection(self, id, releases, handler): host, port = config.setting['server_host'], config.setting['server_port'] for path in self._collection_request(id, releases): self.put(host, port, path, "", handler, queryargs=self._get_client_queryarg()) def delete_from_collection(self, id, releases, handler): host, port = config.setting['server_host'], config.setting['server_port'] for path in self._collection_request(id, releases): self.delete(host, port, path, handler, queryargs=self._get_client_queryarg)
class XmlWebService(QtCore.QObject): def __init__(self, parent=None): QtCore.QObject.__init__(self, parent) self.manager = QtNetwork.QNetworkAccessManager() self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() self.manager.finished.connect(self._process_reply) self._last_request_times = defaultdict(lambda: 0) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self._init_queues() self._init_timers() def _init_queues(self): self._active_requests = {} self._queues = defaultdict(lambda: defaultdict(deque)) self.num_pending_web_requests = 0 self._last_num_pending_web_requests = -1 def _init_timers(self): self._timer_run_next_task = QtCore.QTimer(self) self._timer_run_next_task.setSingleShot(True) self._timer_run_next_task.timeout.connect(self._run_next_task) self._timer_count_pending_requests = QtCore.QTimer(self) self._timer_count_pending_requests.setSingleShot(True) self._timer_count_pending_requests.timeout.connect( self._count_pending_requests) def set_cache(self, cache_size_in_mb=100): cache = QtNetwork.QNetworkDiskCache() location = QDesktopServices.storageLocation( QDesktopServices.CacheLocation) cache.setCacheDirectory(os.path.join(unicode(location), u'picard')) cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024) self.manager.setCache(cache) log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory()) log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(), cache.maximumCacheSize()) def setup_proxy(self): proxy = QtNetwork.QNetworkProxy() if config.setting["use_proxy"]: proxy.setType(QtNetwork.QNetworkProxy.HttpProxy) proxy.setHostName(config.setting["proxy_server_host"]) proxy.setPort(config.setting["proxy_server_port"]) proxy.setUser(config.setting["proxy_username"]) proxy.setPassword(config.setting["proxy_password"]) self.manager.setProxy(proxy) def _start_request_continue(self, method, host, port, path, data, handler, xml, mblogin=False, cacheloadcontrol=None, refresh=None, access_token=None, queryargs=None): url = build_qurl(host, port, path=path, mblogin=mblogin, queryargs=queryargs) request = QtNetwork.QNetworkRequest(url) if mblogin and access_token: request.setRawHeader("Authorization", "Bearer %s" % access_token) if mblogin or (method == "GET" and refresh): request.setPriority(QtNetwork.QNetworkRequest.HighPriority) request.setAttribute( QtNetwork.QNetworkRequest.CacheLoadControlAttribute, QtNetwork.QNetworkRequest.AlwaysNetwork) elif method == "PUT" or method == "DELETE": request.setPriority(QtNetwork.QNetworkRequest.HighPriority) elif cacheloadcontrol is not None: request.setAttribute( QtNetwork.QNetworkRequest.CacheLoadControlAttribute, cacheloadcontrol) request.setRawHeader("User-Agent", USER_AGENT_STRING) if xml: request.setRawHeader("Accept", "application/xml") if data is not None: if method == "POST" and host == config.setting[ "server_host"] and xml: request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/xml; charset=utf-8") else: request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/x-www-form-urlencoded") send = self._request_methods[method] reply = send(request, data) if data is not None else send(request) self._remember_request_time((host, port)) self._active_requests[reply] = (request, handler, xml, refresh) def _start_request(self, method, host, port, path, data, handler, xml, mblogin=False, cacheloadcontrol=None, refresh=None, queryargs=None): def start_request_continue(access_token=None): self._start_request_continue(method, host, port, path, data, handler, xml, mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, access_token=access_token, queryargs=queryargs) if mblogin and path != "/oauth2/token": self.oauth_manager.get_access_token(start_request_continue) else: start_request_continue() @staticmethod def urls_equivalent(leftUrl, rightUrl): """ Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash from QT 4.7 """ return leftUrl.port(80) == rightUrl.port(80) and \ leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort) @staticmethod def url_port(url): if url.scheme() == 'https': return url.port(443) return url.port(80) def _handle_reply(self, reply, request, handler, xml, refresh): error = int(reply.error()) if error: log.error( "Network request error for %s: %s (QT code %d, HTTP code %s)", reply.request().url().toString(QUrl.RemoveUserInfo), reply.errorString(), error, repr( reply.attribute( QtNetwork.QNetworkRequest.HttpStatusCodeAttribute))) if handler is not None: handler(str(reply.readAll()), reply, error) else: redirect = reply.attribute( QtNetwork.QNetworkRequest.RedirectionTargetAttribute) fromCache = reply.attribute( QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute) cached = ' (CACHED)' if fromCache else '' log.debug( "Received reply for %s: HTTP %d (%s) %s", reply.request().url().toString(QUrl.RemoveUserInfo), reply.attribute( QtNetwork.QNetworkRequest.HttpStatusCodeAttribute), reply.attribute( QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute), cached) if handler is not None: # Redirect if found and not infinite if redirect: url = request.url() # merge with base url (to cover the possibility of the URL being relative) redirect = url.resolved(redirect) if not XmlWebService.urls_equivalent( redirect, reply.request().url()): log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo)) redirect_host = str(redirect.host()) redirect_port = self.url_port(redirect) redirect_query = dict(redirect.encodedQueryItems()) redirect_path = redirect.path() original_host = str(url.host()) original_port = self.url_port(url) if ((original_host, original_port) in REQUEST_DELAY and (redirect_host, redirect_port) not in REQUEST_DELAY): log.debug( "Setting rate limit for %s:%i to %i" % (redirect_host, redirect_port, REQUEST_DELAY[ (original_host, original_port)])) REQUEST_DELAY[(redirect_host, redirect_port)] =\ REQUEST_DELAY[(original_host, original_port)] self.get(redirect_host, redirect_port, redirect_path, handler, xml, priority=True, important=True, refresh=refresh, queryargs=redirect_query, cacheloadcontrol=request.attribute( QtNetwork.QNetworkRequest. CacheLoadControlAttribute)) else: log.error( "Redirect loop: %s", reply.request().url().toString( QUrl.RemoveUserInfo)) handler(str(reply.readAll()), reply, error) elif xml: document = _read_xml(QXmlStreamReader(reply)) handler(document, reply, error) else: handler(str(reply.readAll()), reply, error) def _process_reply(self, reply): try: request, handler, xml, refresh = self._active_requests.pop(reply) except KeyError: log.error("Request not found for %s" % reply.request().url().toString(QUrl.RemoveUserInfo)) return try: self._handle_reply(reply, request, handler, xml, refresh) finally: reply.close() reply.deleteLater() def get(self, host, port, path, handler, xml=True, priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False, queryargs=None): func = partial(self._start_request, "GET", host, port, path, None, handler, xml, mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs) return self.add_task(func, host, port, priority, important=important) def post(self, host, port, path, data, handler, xml=True, priority=False, important=False, mblogin=True, queryargs=None): log.debug("POST-DATA %r", data) func = partial(self._start_request, "POST", host, port, path, data, handler, xml, mblogin, queryargs=queryargs) return self.add_task(func, host, port, priority, important=important) def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True, queryargs=None): func = partial(self._start_request, "PUT", host, port, path, data, handler, False, mblogin, queryargs=queryargs) return self.add_task(func, host, port, priority, important=important) def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True, queryargs=None): func = partial(self._start_request, "DELETE", host, port, path, None, handler, False, mblogin, queryargs=queryargs) return self.add_task(func, host, port, priority, important=important) def stop(self): for reply in self._active_requests.keys(): reply.abort() self._init_queues() def _count_pending_requests(self): count = len(self._active_requests) for prio_queue in self._queues.values(): for queue in prio_queue.values(): count += len(queue) self.num_pending_web_requests = count if count != self._last_num_pending_web_requests: self._last_num_pending_web_requests = count self.tagger.tagger_stats_changed.emit() if count: self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS) def _get_delay_to_next_request(self, hostkey): """Calculate delay to next request to hostkey (host, port) returns a tuple (wait, delay) where: wait is True if a delay is needed delay is the delay in milliseconds to next request """ interval = REQUEST_DELAY[hostkey] if not interval: log.debug("WSREQ: Starting another request to %s without delay", hostkey) return (False, 0) last_request = self._last_request_times[hostkey] if not last_request: log.debug("WSREQ: First request to %s", hostkey) self._remember_request_time(hostkey) # set it on first run return (False, interval) elapsed = (time.time() - last_request) * 1000 if elapsed >= interval: log.debug( "WSREQ: Last request to %s was %d ms ago, starting another one", hostkey, elapsed) return (False, interval) delay = int(math.ceil(interval - elapsed)) log.debug( "WSREQ: Last request to %s was %d ms ago, waiting %d ms before starting another one", hostkey, elapsed, delay) return (True, delay) def _remember_request_time(self, hostkey): if REQUEST_DELAY[hostkey]: self._last_request_times[hostkey] = time.time() def _run_next_task(self): delay = sys.maxsize for prio in sorted(self._queues.keys(), reverse=True): prio_queue = self._queues[prio] if not prio_queue: del (self._queues[prio]) continue for hostkey in sorted(prio_queue.keys(), key=lambda hostkey: REQUEST_DELAY[hostkey]): queue = self._queues[prio][hostkey] if not queue: del (self._queues[prio][hostkey]) continue wait, d = self._get_delay_to_next_request(hostkey) if not wait: queue.popleft()() if d < delay: delay = d if delay < sys.maxsize: self._timer_run_next_task.start(delay) def add_task(self, func, host, port, priority, important=False): hostkey = (host, port) prio = int(priority) # priority is a boolean if important: self._queues[prio][hostkey].appendleft(func) else: self._queues[prio][hostkey].append(func) if not self._timer_run_next_task.isActive(): self._timer_run_next_task.start(0) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) return (hostkey, func, prio) def remove_task(self, task): hostkey, func, prio = task try: self._queues[prio][hostkey].remove(func) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) except: pass def _get_by_id(self, entitytype, entityid, handler, inc=[], queryargs=None, priority=False, important=False, mblogin=False, refresh=False): host = config.setting["server_host"] port = config.setting["server_port"] path = "/ws/2/%s/%s" % (entitytype, entityid) if queryargs is None: queryargs = {} if inc: queryargs["inc"] = "+".join(inc) return self.get(host, port, path, handler, priority=priority, important=important, mblogin=mblogin, refresh=refresh, queryargs=queryargs) def get_release_by_id(self, releaseid, handler, inc=[], priority=False, important=False, mblogin=False, refresh=False): return self._get_by_id('release', releaseid, handler, inc, priority=priority, important=important, mblogin=mblogin, refresh=refresh) def get_track_by_id(self, trackid, handler, inc=[], priority=False, important=False, mblogin=False, refresh=False): return self._get_by_id('recording', trackid, handler, inc, priority=priority, important=important, mblogin=mblogin, refresh=refresh) def lookup_discid(self, discid, handler, priority=True, important=True, refresh=False): inc = ['artist-credits', 'labels'] return self._get_by_id('discid', discid, handler, inc, queryargs={"cdstubs": "no"}, priority=priority, important=important, refresh=refresh) def _find(self, entitytype, handler, kwargs): host = config.setting["server_host"] port = config.setting["server_port"] filters = [] limit = kwargs.pop("limit") if limit: filters.append(("limit", limit)) is_search = kwargs.pop("search", False) if is_search: if config.setting["use_adv_search_syntax"]: query = kwargs["query"] else: query = escape_lucene_query(kwargs["query"]).strip().lower() filters.append(("dismax", 'true')) else: query = [] for name, value in kwargs.items(): value = escape_lucene_query(value).strip().lower() if value: query.append('%s:(%s)' % (name, value)) query = ' '.join(query) if query: filters.append(("query", query)) queryargs = {} for name, value in filters: value = QUrl.toPercentEncoding(unicode(value)) queryargs[str(name)] = value path = "/ws/2/%s" % (entitytype) return self.get(host, port, path, handler, queryargs=queryargs) def find_releases(self, handler, **kwargs): return self._find('release', handler, kwargs) def find_tracks(self, handler, **kwargs): return self._find('recording', handler, kwargs) def find_artists(self, handler, **kwargs): return self._find('artist', handler, kwargs) def _browse(self, entitytype, handler, kwargs, inc=[], priority=False, important=False): host = config.setting["server_host"] port = config.setting["server_port"] path = "/ws/2/%s" % (entitytype) queryargs = kwargs if inc: queryargs["inc"] = "+".join(inc) return self.get(host, port, path, handler, priority=priority, important=important, queryargs=queryargs) def browse_releases(self, handler, priority=True, important=True, **kwargs): inc = ["media", "labels"] return self._browse("release", handler, kwargs, inc, priority=priority, important=important) def submit_ratings(self, ratings, handler): host = config.setting['server_host'] port = config.setting['server_port'] path = '/ws/2/rating/?client=' + CLIENT_STRING recordings = (''.join([ '<recording id="%s"><user-rating>%s</user-rating></recording>' % (i[1], j * 20) for i, j in ratings.items() if i[0] == 'recording' ])) data = _wrap_xml_metadata('<recording-list>%s</recording-list>' % recordings) return self.post(host, port, path, data, handler, priority=True) def _encode_acoustid_args(self, args, format='xml'): filters = [] args['client'] = ACOUSTID_KEY args['clientversion'] = PICARD_VERSION_STR args['format'] = format for name, value in args.items(): value = str(QUrl.toPercentEncoding(value)) filters.append('%s=%s' % (str(name), value)) return '&'.join(filters) def query_acoustid(self, handler, **args): host, port = ACOUSTID_HOST, ACOUSTID_PORT body = self._encode_acoustid_args(args) return self.post(host, port, '/v2/lookup', body, handler, priority=False, important=False, mblogin=False) def submit_acoustid_fingerprints(self, submissions, handler): args = {'user': config.setting["acoustid_apikey"]} for i, submission in enumerate(submissions): args['fingerprint.%d' % i] = str(submission.fingerprint) args['duration.%d' % i] = str(submission.duration) args['mbid.%d' % i] = str(submission.recordingid) if submission.puid: args['puid.%d' % i] = str(submission.puid) host, port = ACOUSTID_HOST, ACOUSTID_PORT body = self._encode_acoustid_args(args, format='json') return self.post(host, port, '/v2/submit', body, handler, priority=True, important=False, mblogin=False) def download(self, host, port, path, handler, priority=False, important=False, cacheloadcontrol=None, refresh=False, queryargs=None): return self.get(host, port, path, handler, xml=False, priority=priority, important=important, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs) def get_collection(self, id, handler, limit=100, offset=0): host, port = config.setting['server_host'], config.setting[ 'server_port'] path = "/ws/2/collection" queryargs = None if id is not None: inc = ["releases", "artist-credits", "media"] path += "/%s/releases" % (id) queryargs = {} queryargs["inc"] = "+".join(inc) queryargs["limit"] = limit queryargs["offset"] = offset return self.get(host, port, path, handler, priority=True, important=True, mblogin=True, queryargs=queryargs) def get_collection_list(self, handler): return self.get_collection(None, handler) def _collection_request(self, id, releases): while releases: ids = ";".join( releases if len(releases) <= 400 else releases[:400]) releases = releases[400:] yield "/ws/2/collection/%s/releases/%s" % (id, ids) def _get_client_queryarg(self): return {"client": CLIENT_STRING} def put_to_collection(self, id, releases, handler): host, port = config.setting['server_host'], config.setting[ 'server_port'] for path in self._collection_request(id, releases): self.put(host, port, path, "", handler, queryargs=self._get_client_queryarg()) def delete_from_collection(self, id, releases, handler): host, port = config.setting['server_host'], config.setting[ 'server_port'] for path in self._collection_request(id, releases): self.delete(host, port, path, handler, queryargs=self._get_client_queryarg())
class WebService(QtCore.QObject): PARSERS = dict() def __init__(self, parent=None): super().__init__(parent) self.manager = QtNetwork.QNetworkAccessManager() self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() self.manager.finished.connect(self._process_reply) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self._init_queues() self._init_timers() def _init_queues(self): self._active_requests = {} self._queues = defaultdict(lambda: defaultdict(deque)) self.num_pending_web_requests = 0 self._last_num_pending_web_requests = -1 def _init_timers(self): self._timer_run_next_task = QtCore.QTimer(self) self._timer_run_next_task.setSingleShot(True) self._timer_run_next_task.timeout.connect(self._run_next_task) self._timer_count_pending_requests = QtCore.QTimer(self) self._timer_count_pending_requests.setSingleShot(True) self._timer_count_pending_requests.timeout.connect(self._count_pending_requests) def set_cache(self, cache_size_in_mb=100): cache = QtNetwork.QNetworkDiskCache() location = QStandardPaths.writableLocation(QStandardPaths.CacheLocation) cache.setCacheDirectory(os.path.join(location, 'picard')) cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024) self.manager.setCache(cache) log.debug("NetworkDiskCache dir: %r size: %s / %s", cache.cacheDirectory(), cache.cacheSize(), cache.maximumCacheSize()) def setup_proxy(self): proxy = QtNetwork.QNetworkProxy() if config.setting["use_proxy"]: proxy.setType(QtNetwork.QNetworkProxy.HttpProxy) proxy.setHostName(config.setting["proxy_server_host"]) proxy.setPort(config.setting["proxy_server_port"]) proxy.setUser(config.setting["proxy_username"]) proxy.setPassword(config.setting["proxy_password"]) self.manager.setProxy(proxy) def _send_request(self, request, access_token=None): hostkey = request.get_host_key() ratecontrol.increment_requests(hostkey) request.access_token = access_token send = self._request_methods[request.method] data = request.data reply = send(request, data.encode('utf-8')) if data is not None else send(request) self._active_requests[reply] = request def _start_request(self, request): if request.mblogin and request.path != "/oauth2/token": self.oauth_manager.get_access_token(partial(self._send_request, request)) else: self._send_request(request) @staticmethod def urls_equivalent(leftUrl, rightUrl): """ Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash from QT 4.7 """ return leftUrl.port(80) == rightUrl.port(80) and \ leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort) @staticmethod def url_port(url): if url.scheme() == 'https': return url.port(443) return url.port(80) def _handle_redirect(self, reply, request, redirect): url = request.url() error = int(reply.error()) # merge with base url (to cover the possibility of the URL being relative) redirect = url.resolved(redirect) if not WebService.urls_equivalent(redirect, reply.request().url()): log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo)) redirect_host = redirect.host() redirect_port = self.url_port(redirect) redirect_query = dict(QUrlQuery(redirect).queryItems(QUrl.FullyEncoded)) redirect_path = redirect.path() original_host = url.host() original_port = self.url_port(url) original_host_key = (original_host, original_port) redirect_host_key = (redirect_host, redirect_port) ratecontrol.copy_minimal_delay(original_host_key, redirect_host_key) self.get(redirect_host, redirect_port, redirect_path, request.handler, request.parse_response_type, priority=True, important=True, refresh=request.refresh, queryargs=redirect_query, cacheloadcontrol=request.attribute(QNetworkRequest.CacheLoadControlAttribute)) else: log.error("Redirect loop: %s", reply.request().url().toString(QUrl.RemoveUserInfo) ) request.handler(reply.readAll(), reply, error) def _handle_reply(self, reply, request): hostkey = request.get_host_key() ratecontrol.decrement_requests(hostkey) self._timer_run_next_task.start(0) slow_down = False error = int(reply.error()) handler = request.handler if error: code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) code = int(code) if code else 0 errstr = reply.errorString() url = reply.request().url().toString(QUrl.RemoveUserInfo) log.error("Network request error for %s: %s (QT code %d, HTTP code %d)", url, errstr, error, code) if (not request.max_retries_reached() and (code == 503 or code == 429 # Sometimes QT returns a http status code of 200 even when there # is a service unavailable error. But it returns a QT error code # of 403 when this happens or error == 403 ) ): slow_down = True retries = request.mark_for_retry() log.debug("Retrying %s (#%d)", url, retries) self.add_request(request) elif handler is not None: handler(reply.readAll(), reply, error) slow_down = (slow_down or code >= 500) else: redirect = reply.attribute(QNetworkRequest.RedirectionTargetAttribute) fromCache = reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute) cached = ' (CACHED)' if fromCache else '' log.debug("Received reply for %s: HTTP %d (%s) %s", reply.request().url().toString(QUrl.RemoveUserInfo), reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute), cached ) if handler is not None: # Redirect if found and not infinite if redirect: self._handle_redirect(reply, request, redirect) elif request.response_parser: try: document = request.response_parser(reply) except Exception as e: url = reply.request().url().toString(QUrl.RemoveUserInfo) log.error("Unable to parse the response for %s: %s", url, e) document = reply.readAll() error = e finally: handler(document, reply, error) else: handler(reply.readAll(), reply, error) ratecontrol.adjust(hostkey, slow_down) def _process_reply(self, reply): try: request = self._active_requests.pop(reply) except KeyError: log.error("Request not found for %s", reply.request().url().toString(QUrl.RemoveUserInfo)) return try: self._handle_reply(reply, request) finally: reply.close() reply.deleteLater() def get(self, host, port, path, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE, priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False, queryargs=None): request = WSGetRequest(host, port, path, handler, parse_response_type=parse_response_type, mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs, priority=priority, important=important) return self.add_request(request) def post(self, host, port, path, data, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE, priority=False, important=False, mblogin=True, queryargs=None, request_mimetype=None): request = WSPostRequest(host, port, path, handler, parse_response_type=parse_response_type, data=data, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important, request_mimetype=request_mimetype) log.debug("POST-DATA %r", data) return self.add_request(request) def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True, queryargs=None, request_mimetype=None): request = WSPutRequest(host, port, path, handler, data=data, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important, request_mimetype=request_mimetype) return self.add_request(request) def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True, queryargs=None): request = WSDeleteRequest(host, port, path, handler, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important) return self.add_request(request) def download(self, host, port, path, handler, priority=False, important=False, cacheloadcontrol=None, refresh=False, queryargs=None): return self.get(host, port, path, handler, parse_response_type=None, priority=priority, important=important, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs) def stop(self): for reply in list(self._active_requests): reply.abort() self._init_queues() def _count_pending_requests(self): count = len(self._active_requests) for prio_queue in self._queues.values(): for queue in prio_queue.values(): count += len(queue) self.num_pending_web_requests = count if count != self._last_num_pending_web_requests: self._last_num_pending_web_requests = count self.tagger.tagger_stats_changed.emit() if count: self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS) def _run_next_task(self): delay = sys.maxsize for prio in sorted(self._queues.keys(), reverse=True): prio_queue = self._queues[prio] if not prio_queue: del(self._queues[prio]) continue for hostkey in sorted(prio_queue.keys(), key=ratecontrol.current_delay): queue = self._queues[prio][hostkey] if not queue: del(self._queues[prio][hostkey]) continue wait, d = ratecontrol.get_delay_to_next_request(hostkey) if not wait: queue.popleft()() if d < delay: delay = d if delay < sys.maxsize: self._timer_run_next_task.start(delay) def add_task(self, func, request): hostkey = request.get_host_key() prio = int(request.priority) # priority is a boolean if request.important: self._queues[prio][hostkey].appendleft(func) else: self._queues[prio][hostkey].append(func) if not self._timer_run_next_task.isActive(): self._timer_run_next_task.start(0) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) return (hostkey, func, prio) def add_request(self, request): return self.add_task(partial(self._start_request, request), request) def remove_task(self, task): hostkey, func, prio = task try: self._queues[prio][hostkey].remove(func) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) except Exception as e: log.debug(e) @classmethod def add_parser(cls, response_type, mimetype, parser): cls.PARSERS[response_type] = Parser(mimetype=mimetype, parser=parser) @classmethod def get_response_mimetype(cls, response_type): if response_type in cls.PARSERS: return cls.PARSERS[response_type].mimetype else: raise UnknownResponseParserError(response_type) @classmethod def get_response_parser(cls, response_type): if response_type in cls.PARSERS: return cls.PARSERS[response_type].parser else: raise UnknownResponseParserError(response_type)
class XmlWebService(QtCore.QObject): def __init__(self, parent=None): QtCore.QObject.__init__(self, parent) self.manager = QtNetwork.QNetworkAccessManager() self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() self.manager.finished.connect(self._process_reply) self._last_request_times = {} self._active_requests = {} self._high_priority_queues = {} self._low_priority_queues = {} self._hosts = [] self._timer = QtCore.QTimer(self) self._timer.setSingleShot(True) self._timer.timeout.connect(self._run_next_task) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self.num_pending_web_requests = 0 def set_cache(self, cache_size_in_mb=100): cache = QtNetwork.QNetworkDiskCache() location = QDesktopServices.storageLocation( QDesktopServices.CacheLocation) cache.setCacheDirectory(os.path.join(unicode(location), u'picard')) cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024) self.manager.setCache(cache) log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory()) log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(), cache.maximumCacheSize()) def setup_proxy(self): proxy = QtNetwork.QNetworkProxy() if config.setting["use_proxy"]: proxy.setType(QtNetwork.QNetworkProxy.HttpProxy) proxy.setHostName(config.setting["proxy_server_host"]) proxy.setPort(config.setting["proxy_server_port"]) proxy.setUser(config.setting["proxy_username"]) proxy.setPassword(config.setting["proxy_password"]) self.manager.setProxy(proxy) def _start_request_continue(self, method, host, port, path, data, handler, xml, mblogin=False, cacheloadcontrol=None, refresh=None, access_token=None): if (mblogin and host in MUSICBRAINZ_SERVERS and port == 80) or port == 443: urlstring = "https://%s%s" % (host, path) elif port is None or port == 80: urlstring = "http://%s%s" % (host, path) else: urlstring = "http://%s:%d%s" % (host, port, path) log.debug("%s %s", method, urlstring) url = QUrl.fromEncoded(urlstring) request = QtNetwork.QNetworkRequest(url) if mblogin and access_token: request.setRawHeader("Authorization", "Bearer %s" % access_token) if mblogin or (method == "GET" and refresh): request.setPriority(QtNetwork.QNetworkRequest.HighPriority) request.setAttribute( QtNetwork.QNetworkRequest.CacheLoadControlAttribute, QtNetwork.QNetworkRequest.AlwaysNetwork) elif method == "PUT" or method == "DELETE": request.setPriority(QtNetwork.QNetworkRequest.HighPriority) elif cacheloadcontrol is not None: request.setAttribute( QtNetwork.QNetworkRequest.CacheLoadControlAttribute, cacheloadcontrol) request.setRawHeader("User-Agent", USER_AGENT_STRING) if xml: request.setRawHeader("Accept", "application/xml") if data is not None: if method == "POST" and host == config.setting[ "server_host"] and xml: request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/xml; charset=utf-8") else: request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/x-www-form-urlencoded") send = self._request_methods[method] reply = send(request, data) if data is not None else send(request) key = (host, port) self._last_request_times[key] = time.time() self._active_requests[reply] = (request, handler, xml, refresh) def _start_request(self, method, host, port, path, data, handler, xml, mblogin=False, cacheloadcontrol=None, refresh=None): def start_request_continue(access_token=None): self._start_request_continue(method, host, port, path, data, handler, xml, mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, access_token=access_token) if mblogin and path != "/oauth2/token": self.oauth_manager.get_access_token(start_request_continue) else: start_request_continue() @staticmethod def urls_equivalent(leftUrl, rightUrl): """ Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash from QT 4.7 """ return leftUrl.port(80) == rightUrl.port(80) and \ leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort) def _process_reply(self, reply): try: request, handler, xml, refresh = self._active_requests.pop(reply) except KeyError: log.error("Request not found for %s" % reply.request().url().toString(QUrl.RemoveUserInfo)) return error = int(reply.error()) if error: log.error( "Network request error for %s: %s (QT code %d, HTTP code %s)", reply.request().url().toString(QUrl.RemoveUserInfo), reply.errorString(), error, repr( reply.attribute( QtNetwork.QNetworkRequest.HttpStatusCodeAttribute))) if handler is not None: handler(str(reply.readAll()), reply, error) else: redirect = reply.attribute( QtNetwork.QNetworkRequest.RedirectionTargetAttribute) fromCache = reply.attribute( QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute) cached = ' (CACHED)' if fromCache else '' log.debug( "Received reply for %s: HTTP %d (%s) %s", reply.request().url().toString(QUrl.RemoveUserInfo), reply.attribute( QtNetwork.QNetworkRequest.HttpStatusCodeAttribute), reply.attribute( QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute), cached) if handler is not None: # Redirect if found and not infinite if redirect and not XmlWebService.urls_equivalent( redirect, reply.request().url()): log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo)) redirect_host = str(redirect.host()) redirect_port = redirect.port(80) url = request.url() original_host = str(url.host()) original_port = url.port(80) if ((original_host, original_port) in REQUEST_DELAY and (redirect_host, redirect_port) not in REQUEST_DELAY): log.debug( "Setting rate limit for %s:%i to %i" % (redirect_host, redirect_port, REQUEST_DELAY[(original_host, original_port)])) REQUEST_DELAY[(redirect_host, redirect_port)] =\ REQUEST_DELAY[(original_host, original_port)] self.get( redirect_host, redirect_port, # retain path, query string and anchors from redirect URL redirect.toString(QUrl.RemoveAuthority | QUrl.RemoveScheme), handler, xml, priority=True, important=True, refresh=refresh, cacheloadcontrol=request.attribute( QtNetwork.QNetworkRequest.CacheLoadControlAttribute )) elif redirect: log.error( "Redirect loop: %s", reply.request().url().toString(QUrl.RemoveUserInfo)) handler(str(reply.readAll()), reply, error) elif xml: document = _read_xml(QXmlStreamReader(reply)) handler(document, reply, error) else: handler(str(reply.readAll()), reply, error) reply.close() self.num_pending_web_requests -= 1 self.tagger.tagger_stats_changed.emit() def get(self, host, port, path, handler, xml=True, priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False): func = partial(self._start_request, "GET", host, port, path, None, handler, xml, mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh) return self.add_task(func, host, port, priority, important=important) def post(self, host, port, path, data, handler, xml=True, priority=False, important=False, mblogin=True): log.debug("POST-DATA %r", data) func = partial(self._start_request, "POST", host, port, path, data, handler, xml, mblogin) return self.add_task(func, host, port, priority, important=important) def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True): func = partial(self._start_request, "PUT", host, port, path, data, handler, False, mblogin) return self.add_task(func, host, port, priority, important=important) def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True): func = partial(self._start_request, "DELETE", host, port, path, None, handler, False, mblogin) return self.add_task(func, host, port, priority, important=important) def stop(self): self._high_priority_queues = {} self._low_priority_queues = {} for reply in self._active_requests.keys(): reply.abort() def _run_next_task(self): delay = sys.maxint for key in self._hosts: queue = self._high_priority_queues.get( key) or self._low_priority_queues.get(key) if not queue: continue now = time.time() last = self._last_request_times.get(key) request_delay = REQUEST_DELAY[key] last_ms = (now - last) * 1000 if last is not None else request_delay if last_ms >= request_delay: log.debug( "Last request to %s was %d ms ago, starting another one", key, last_ms) d = request_delay queue.popleft()() else: d = int(math.ceil(request_delay - last_ms)) log.debug( "Waiting %d ms before starting another request to %s", d, key) if d < delay: delay = d if delay < sys.maxint: self._timer.start(delay) def add_task(self, func, host, port, priority, important=False): key = (host, port) if key not in self._hosts: self._hosts.append(key) if priority: queues = self._high_priority_queues else: queues = self._low_priority_queues queues.setdefault(key, deque()) if important: queues[key].appendleft(func) else: queues[key].append(func) self.num_pending_web_requests += 1 self.tagger.tagger_stats_changed.emit() if len(queues[key]) == 1: self._timer.start(0) return (key, func, priority) def remove_task(self, task): key, func, priority = task if priority: queue = self._high_priority_queues[key] else: queue = self._low_priority_queues[key] try: queue.remove(func) except: pass else: self.num_pending_web_requests -= 1 self.tagger.tagger_stats_changed.emit() def _get_by_id(self, entitytype, entityid, handler, inc=[], params=[], priority=False, important=False, mblogin=False, refresh=False): host = config.setting["server_host"] port = config.setting["server_port"] path = "/ws/2/%s/%s?inc=%s" % (entitytype, entityid, "+".join(inc)) if params: path += "&" + "&".join(params) return self.get(host, port, path, handler, priority=priority, important=important, mblogin=mblogin, refresh=refresh) def get_release_by_id(self, releaseid, handler, inc=[], priority=False, important=False, mblogin=False, refresh=False): return self._get_by_id('release', releaseid, handler, inc, priority=priority, important=important, mblogin=mblogin, refresh=refresh) def get_track_by_id(self, trackid, handler, inc=[], priority=False, important=False, mblogin=False, refresh=False): return self._get_by_id('recording', trackid, handler, inc, priority=priority, important=important, mblogin=mblogin, refresh=refresh) def lookup_discid(self, discid, handler, priority=True, important=True, refresh=False): inc = ['artist-credits', 'labels'] return self._get_by_id('discid', discid, handler, inc, params=["cdstubs=no"], priority=priority, important=important, refresh=refresh) def _find(self, entitytype, handler, kwargs): host = config.setting["server_host"] port = config.setting["server_port"] filters = [] query = [] for name, value in kwargs.items(): if name == 'limit': filters.append((name, str(value))) else: value = _escape_lucene_query(value).strip().lower() if value: query.append('%s:(%s)' % (name, value)) if query: filters.append(('query', ' '.join(query))) params = [] for name, value in filters: value = QUrl.toPercentEncoding(unicode(value)) params.append('%s=%s' % (str(name), value)) path = "/ws/2/%s/?%s" % (entitytype, "&".join(params)) return self.get(host, port, path, handler) def find_releases(self, handler, **kwargs): return self._find('release', handler, kwargs) def find_tracks(self, handler, **kwargs): return self._find('recording', handler, kwargs) def _browse(self, entitytype, handler, kwargs, inc=[], priority=False, important=False): host = config.setting["server_host"] port = config.setting["server_port"] params = "&".join(["%s=%s" % (k, v) for k, v in kwargs.items()]) path = "/ws/2/%s?%s&inc=%s" % (entitytype, params, "+".join(inc)) return self.get(host, port, path, handler, priority=priority, important=important) def browse_releases(self, handler, priority=True, important=True, **kwargs): inc = ["media", "labels"] return self._browse("release", handler, kwargs, inc, priority=priority, important=important) def submit_ratings(self, ratings, handler): host = config.setting['server_host'] port = config.setting['server_port'] path = '/ws/2/rating/?client=' + CLIENT_STRING recordings = (''.join([ '<recording id="%s"><user-rating>%s</user-rating></recording>' % (i[1], j * 20) for i, j in ratings.items() if i[0] == 'recording' ])) data = _wrap_xml_metadata('<recording-list>%s</recording-list>' % recordings) return self.post(host, port, path, data, handler, priority=True) def _encode_acoustid_args(self, args): filters = [] args['client'] = ACOUSTID_KEY args['clientversion'] = PICARD_VERSION_STR args['format'] = 'xml' for name, value in args.items(): value = str(QUrl.toPercentEncoding(value)) filters.append('%s=%s' % (str(name), value)) return '&'.join(filters) def query_acoustid(self, handler, **args): host, port = ACOUSTID_HOST, ACOUSTID_PORT body = self._encode_acoustid_args(args) return self.post(host, port, '/v2/lookup', body, handler, priority=False, important=False, mblogin=False) def submit_acoustid_fingerprints(self, submissions, handler): args = {'user': config.setting["acoustid_apikey"]} for i, submission in enumerate(submissions): args['fingerprint.%d' % i] = str(submission.fingerprint) args['duration.%d' % i] = str(submission.duration) args['mbid.%d' % i] = str(submission.recordingid) if submission.puid: args['puid.%d' % i] = str(submission.puid) host, port = ACOUSTID_HOST, ACOUSTID_PORT body = self._encode_acoustid_args(args) return self.post(host, port, '/v2/submit', body, handler, priority=True, important=False, mblogin=False) def download(self, host, port, path, handler, priority=False, important=False, cacheloadcontrol=None, refresh=False): return self.get(host, port, path, handler, xml=False, priority=priority, important=important, cacheloadcontrol=cacheloadcontrol, refresh=refresh) def get_collection(self, id, handler, limit=100, offset=0): host, port = config.setting['server_host'], config.setting[ 'server_port'] path = "/ws/2/collection" if id is not None: inc = ["releases", "artist-credits", "media"] path += "/%s/releases?inc=%s&limit=%d&offset=%d" % ( id, "+".join(inc), limit, offset) return self.get(host, port, path, handler, priority=True, important=True, mblogin=True) def get_collection_list(self, handler): return self.get_collection(None, handler) def _collection_request(self, id, releases): while releases: ids = ";".join( releases if len(releases) <= 400 else releases[:400]) releases = releases[400:] yield "/ws/2/collection/%s/releases/%s?client=%s" % (id, ids, CLIENT_STRING) def put_to_collection(self, id, releases, handler): host, port = config.setting['server_host'], config.setting[ 'server_port'] for path in self._collection_request(id, releases): self.put(host, port, path, "", handler) def delete_from_collection(self, id, releases, handler): host, port = config.setting['server_host'], config.setting[ 'server_port'] for path in self._collection_request(id, releases): self.delete(host, port, path, handler)
class WebService(QtCore.QObject): PARSERS = dict() def __init__(self, parent=None): QtCore.QObject.__init__(self, parent) self.manager = QtNetwork.QNetworkAccessManager() self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() self.manager.finished.connect(self._process_reply) self._last_request_times = defaultdict(lambda: 0) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self._init_queues() self._init_timers() def _init_queues(self): self._active_requests = {} self._queues = defaultdict(lambda: defaultdict(deque)) self.num_pending_web_requests = 0 self._last_num_pending_web_requests = -1 def _init_timers(self): self._timer_run_next_task = QtCore.QTimer(self) self._timer_run_next_task.setSingleShot(True) self._timer_run_next_task.timeout.connect(self._run_next_task) self._timer_count_pending_requests = QtCore.QTimer(self) self._timer_count_pending_requests.setSingleShot(True) self._timer_count_pending_requests.timeout.connect(self._count_pending_requests) def set_cache(self, cache_size_in_mb=100): cache = QtNetwork.QNetworkDiskCache() location = QStandardPaths.writableLocation(QStandardPaths.CacheLocation) cache.setCacheDirectory(os.path.join(location, 'picard')) cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024) self.manager.setCache(cache) log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory()) log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(), cache.maximumCacheSize()) def setup_proxy(self): proxy = QtNetwork.QNetworkProxy() if config.setting["use_proxy"]: proxy.setType(QtNetwork.QNetworkProxy.HttpProxy) proxy.setHostName(config.setting["proxy_server_host"]) proxy.setPort(config.setting["proxy_server_port"]) proxy.setUser(config.setting["proxy_username"]) proxy.setPassword(config.setting["proxy_password"]) self.manager.setProxy(proxy) @staticmethod def _adjust_throttle(hostkey, slow_down): """Adjust `REQUEST` and `CONGESTION` metrics when a HTTP request completes. Args: hostkey: `(host, port)`. slow_down: `True` if we encountered intermittent server trouble and need to slow down. """ def in_backoff_phase(hostkey): return CONGESTION_UNACK[hostkey] > CONGESTION_WINDOW_SIZE[hostkey] if slow_down: # Backoff exponentially until ~30 seconds between requests. delay = max(pow(2, REQUEST_DELAY_EXPONENT[hostkey]) * 1000, REQUEST_DELAY_MINIMUM[hostkey]) log.debug('WebService: %s: delay: %dms -> %dms.', hostkey, REQUEST_DELAY[hostkey], delay) REQUEST_DELAY[hostkey] = delay REQUEST_DELAY_EXPONENT[hostkey] = min(REQUEST_DELAY_EXPONENT[hostkey] + 1, 5) # Slow start threshold is ~1/2 of the window size up until we saw # trouble. Shrink the new window size back to 1. CONGESTION_SSTHRESH[hostkey] = int(CONGESTION_WINDOW_SIZE[hostkey] / 2.0) log.debug('WebService: %s: ssthresh: %d.', hostkey, CONGESTION_SSTHRESH[hostkey]) CONGESTION_WINDOW_SIZE[hostkey] = 1.0 log.debug('WebService: %s: cws: %.3f.', hostkey, CONGESTION_WINDOW_SIZE[hostkey]) elif not in_backoff_phase(hostkey): REQUEST_DELAY_EXPONENT[hostkey] = 0 # Coming out of backoff, so reset. # Shrink the delay between requests with each successive reply to # converge on maximum throughput. delay = max(int(REQUEST_DELAY[hostkey] / 2), REQUEST_DELAY_MINIMUM[hostkey]) if delay != REQUEST_DELAY[hostkey]: log.debug('WebService: %s: delay: %dms -> %dms.', hostkey, REQUEST_DELAY[hostkey], delay) REQUEST_DELAY[hostkey] = delay cws = CONGESTION_WINDOW_SIZE[hostkey] sst = CONGESTION_SSTHRESH[hostkey] if sst and cws >= sst: # Analogous to TCP's congestion avoidance phase. Window growth is linear. phase = 'congestion avoidance' cws = cws + (1.0 / cws) else: # Analogous to TCP's slow start phase. Window growth is exponential. phase = 'slow start' cws += 1 if CONGESTION_WINDOW_SIZE[hostkey] != cws: log.debug('WebService: %s: %s: window size %.3f -> %.3f', hostkey, phase, CONGESTION_WINDOW_SIZE[hostkey], cws) CONGESTION_WINDOW_SIZE[hostkey] = cws def _send_request(self, request, access_token=None): hostkey = request.get_host_key() # Increment the number of unack'd requests on sending a new one CONGESTION_UNACK[hostkey] += 1 log.debug("WebService: %s: outstanding reqs: %d", hostkey, CONGESTION_UNACK[hostkey]) request.access_token = access_token send = self._request_methods[request.method] data = request.data reply = send(request, data.encode('utf-8')) if data is not None else send(request) self._remember_request_time(request.get_host_key()) self._active_requests[reply] = request def _start_request(self, request): if request.mblogin and request.path != "/oauth2/token": self.oauth_manager.get_access_token(partial(self._send_request, request)) else: self._send_request(request) @staticmethod def urls_equivalent(leftUrl, rightUrl): """ Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash from QT 4.7 """ return leftUrl.port(80) == rightUrl.port(80) and \ leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort) @staticmethod def url_port(url): if url.scheme() == 'https': return url.port(443) return url.port(80) def _handle_redirect(self, reply, request, redirect): url = request.url() error = int(reply.error()) # merge with base url (to cover the possibility of the URL being relative) redirect = url.resolved(redirect) if not WebService.urls_equivalent(redirect, reply.request().url()): log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo)) redirect_host = string_(redirect.host()) redirect_port = self.url_port(redirect) redirect_query = dict(QUrlQuery(redirect).queryItems(QUrl.FullyEncoded)) redirect_path = redirect.path() original_host = string_(url.host()) original_port = self.url_port(url) original_host_key = (original_host, original_port) redirect_host_key = (redirect_host, redirect_port) if (original_host_key in REQUEST_DELAY_MINIMUM and redirect_host_key not in REQUEST_DELAY_MINIMUM): log.debug("Setting the minimum rate limit for %s to %i" % (redirect_host_key, REQUEST_DELAY_MINIMUM[original_host_key])) REQUEST_DELAY_MINIMUM[redirect_host_key] = REQUEST_DELAY_MINIMUM[original_host_key] self.get(redirect_host, redirect_port, redirect_path, request.handler, request.parse_response_type, priority=True, important=True, refresh=request.refresh, queryargs=redirect_query, cacheloadcontrol=request.attribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute)) else: log.error("Redirect loop: %s", reply.request().url().toString(QUrl.RemoveUserInfo) ) request.handler(reply.readAll(), reply, error) def _handle_reply(self, reply, request): hostkey = request.get_host_key() CONGESTION_UNACK[hostkey] -= 1 log.debug("WebService: %s: outstanding reqs: %d", hostkey, CONGESTION_UNACK[hostkey]) self._timer_run_next_task.start(0) slow_down = False error = int(reply.error()) handler = request.handler if error: code = reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute) code = int(code) if code else 0 errstr = reply.errorString() url = reply.request().url().toString(QUrl.RemoveUserInfo) log.error("Network request error for %s: %s (QT code %d, HTTP code %d)", url, errstr, error, code) if (not request.max_retries_reached() and (code == 503 or code == 429 # Sometimes QT returns a http status code of 200 even when there # is a service unavailable error. But it returns a QT error code # of 403 when this happens or error == 403 ) ): retries = request.mark_for_retry() log.debug("Retrying %s (#%d)", url, retries) self.add_task(partial(self._start_request, request), request) elif handler is not None: handler(reply.readAll(), reply, error) slow_down = True else: redirect = reply.attribute(QtNetwork.QNetworkRequest.RedirectionTargetAttribute) fromCache = reply.attribute(QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute) cached = ' (CACHED)' if fromCache else '' log.debug("Received reply for %s: HTTP %d (%s) %s", reply.request().url().toString(QUrl.RemoveUserInfo), reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute), reply.attribute(QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute), cached ) if handler is not None: # Redirect if found and not infinite if redirect: self._handle_redirect(reply, request, redirect) elif request.response_parser: try: document = request.response_parser(reply) except Exception as e: log.error("Unable to parse the response. %s", e) document = reply.readAll() finally: handler(document, reply, error) else: handler(reply.readAll(), reply, error) self._adjust_throttle(hostkey, slow_down) def _process_reply(self, reply): try: request = self._active_requests.pop(reply) except KeyError: log.error("Request not found for %s" % reply.request().url().toString(QUrl.RemoveUserInfo)) return try: self._handle_reply(reply, request) finally: reply.close() reply.deleteLater() def get(self, host, port, path, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE, priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False, queryargs=None): request = WSGetRequest(host, port, path, handler, parse_response_type=parse_response_type, mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs, priority=priority, important=important) func = partial(self._start_request, request) return self.add_task(func, request) def post(self, host, port, path, data, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE, priority=False, important=False, mblogin=True, queryargs=None): request = WSPostRequest(host, port, path, handler, parse_response_type=parse_response_type, data=data, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important) log.debug("POST-DATA %r", data) func = partial(self._start_request, request) return self.add_task(func, request) def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True, queryargs=None): request = WSPutRequest(host, port, path, handler, data=data, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important) func = partial(self._start_request, request) return self.add_task(func, request) def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True, queryargs=None): request = WSDeleteRequest(host, port, path, handler, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important) func = partial(self._start_request, request) return self.add_task(func, request) def download(self, host, port, path, handler, priority=False, important=False, cacheloadcontrol=None, refresh=False, queryargs=None): return self.get(host, port, path, handler, parse_response_type=None, priority=priority, important=important, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs) def stop(self): for reply in list(self._active_requests.keys()): reply.abort() self._init_queues() def _count_pending_requests(self): count = len(self._active_requests) for prio_queue in self._queues.values(): for queue in prio_queue.values(): count += len(queue) self.num_pending_web_requests = count if count != self._last_num_pending_web_requests: self._last_num_pending_web_requests = count self.tagger.tagger_stats_changed.emit() if count: self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS) def _get_delay_to_next_request(self, hostkey): """Calculate delay to next request to hostkey (host, port) returns a tuple (wait, delay) where: wait is True if a delay is needed delay is the delay in milliseconds to next request """ if CONGESTION_UNACK[hostkey] >= int(CONGESTION_WINDOW_SIZE[hostkey]): # We've maxed out the number of requests to `hostkey`, so wait # until responses begin to come back. (See `_timer_run_next_task` # strobe in `_handle_reply`.) return (True, sys.maxsize) interval = REQUEST_DELAY[hostkey] if not interval: log.debug("WSREQ: Starting another request to %s without delay", hostkey) return (False, 0) last_request = self._last_request_times[hostkey] if not last_request: log.debug("WSREQ: First request to %s", hostkey) self._remember_request_time(hostkey) # set it on first run return (False, interval) elapsed = (time.time() - last_request) * 1000 if elapsed >= interval: log.debug("WSREQ: Last request to %s was %d ms ago, starting another one", hostkey, elapsed) return (False, interval) delay = int(math.ceil(interval - elapsed)) log.debug("WSREQ: Last request to %s was %d ms ago, waiting %d ms before starting another one", hostkey, elapsed, delay) return (True, delay) def _remember_request_time(self, hostkey): if REQUEST_DELAY[hostkey]: self._last_request_times[hostkey] = time.time() def _run_next_task(self): delay = sys.maxsize for prio in sorted(self._queues.keys(), reverse=True): prio_queue = self._queues[prio] if not prio_queue: del(self._queues[prio]) continue for hostkey in sorted(prio_queue.keys(), key=lambda hostkey: REQUEST_DELAY[hostkey]): queue = self._queues[prio][hostkey] if not queue: del(self._queues[prio][hostkey]) continue wait, d = self._get_delay_to_next_request(hostkey) if not wait: queue.popleft()() if d < delay: delay = d if delay < sys.maxsize: self._timer_run_next_task.start(delay) def add_task(self, func, request): hostkey = request.get_host_key() prio = int(request.priority) # priority is a boolean if request.important: self._queues[prio][hostkey].appendleft(func) else: self._queues[prio][hostkey].append(func) if not self._timer_run_next_task.isActive(): self._timer_run_next_task.start(0) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) return (hostkey, func, prio) def remove_task(self, task): hostkey, func, prio = task try: self._queues[prio][hostkey].remove(func) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) except Exception as e: log.debug(e) @classmethod def add_parser(cls, response_type, mimetype, parser): cls.PARSERS[response_type] = Parser(mimetype=mimetype, parser=parser) @classmethod def get_response_mimetype(cls, response_type): if response_type in cls.PARSERS: return cls.PARSERS[response_type].mimetype else: raise UnknownResponseParserError(response_type) @classmethod def get_response_parser(cls, response_type): if response_type in cls.PARSERS: return cls.PARSERS[response_type].parser else: raise UnknownResponseParserError(response_type)
class WebService(QtCore.QObject): PARSERS = dict() def __init__(self, parent=None): QtCore.QObject.__init__(self, parent) self.manager = QtNetwork.QNetworkAccessManager() self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() self.manager.finished.connect(self._process_reply) self._last_request_times = defaultdict(lambda: 0) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self._init_queues() self._init_timers() def _init_queues(self): self._active_requests = {} self._queues = defaultdict(lambda: defaultdict(deque)) self.num_pending_web_requests = 0 self._last_num_pending_web_requests = -1 def _init_timers(self): self._timer_run_next_task = QtCore.QTimer(self) self._timer_run_next_task.setSingleShot(True) self._timer_run_next_task.timeout.connect(self._run_next_task) self._timer_count_pending_requests = QtCore.QTimer(self) self._timer_count_pending_requests.setSingleShot(True) self._timer_count_pending_requests.timeout.connect( self._count_pending_requests) def set_cache(self, cache_size_in_mb=100): cache = QtNetwork.QNetworkDiskCache() location = QStandardPaths.writableLocation( QStandardPaths.CacheLocation) cache.setCacheDirectory(os.path.join(location, 'picard')) cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024) self.manager.setCache(cache) log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory()) log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(), cache.maximumCacheSize()) def setup_proxy(self): proxy = QtNetwork.QNetworkProxy() if config.setting["use_proxy"]: proxy.setType(QtNetwork.QNetworkProxy.HttpProxy) proxy.setHostName(config.setting["proxy_server_host"]) proxy.setPort(config.setting["proxy_server_port"]) proxy.setUser(config.setting["proxy_username"]) proxy.setPassword(config.setting["proxy_password"]) self.manager.setProxy(proxy) @staticmethod def _adjust_throttle(hostkey, slow_down): """Adjust `REQUEST` and `CONGESTION` metrics when a HTTP request completes. :param hostkey: `(host, port)`. :param slow_down: `True` if we encountered intermittent server trouble and need to slow down. """ def in_backoff_phase(hostkey): return CONGESTION_UNACK[hostkey] > CONGESTION_WINDOW_SIZE[hostkey] if slow_down: # Backoff exponentially until ~30 seconds between requests. delay = max( pow(2, REQUEST_DELAY_EXPONENT[hostkey]) * 1000, REQUEST_DELAY_MINIMUM[hostkey]) log.debug('WebService: %s: delay: %dms -> %dms.', hostkey, REQUEST_DELAY[hostkey], delay) REQUEST_DELAY[hostkey] = delay REQUEST_DELAY_EXPONENT[hostkey] = min( REQUEST_DELAY_EXPONENT[hostkey] + 1, 5) # Slow start threshold is ~1/2 of the window size up until we saw # trouble. Shrink the new window size back to 1. CONGESTION_SSTHRESH[hostkey] = int( CONGESTION_WINDOW_SIZE[hostkey] / 2.0) log.debug('WebService: %s: ssthresh: %d.', hostkey, CONGESTION_SSTHRESH[hostkey]) CONGESTION_WINDOW_SIZE[hostkey] = 1.0 log.debug('WebService: %s: cws: %.3f.', hostkey, CONGESTION_WINDOW_SIZE[hostkey]) elif not in_backoff_phase(hostkey): REQUEST_DELAY_EXPONENT[ hostkey] = 0 # Coming out of backoff, so reset. # Shrink the delay between requests with each successive reply to # converge on maximum throughput. delay = max(int(REQUEST_DELAY[hostkey] / 2), REQUEST_DELAY_MINIMUM[hostkey]) if delay != REQUEST_DELAY[hostkey]: log.debug('WebService: %s: delay: %dms -> %dms.', hostkey, REQUEST_DELAY[hostkey], delay) REQUEST_DELAY[hostkey] = delay cws = CONGESTION_WINDOW_SIZE[hostkey] sst = CONGESTION_SSTHRESH[hostkey] if sst and cws >= sst: # Analogous to TCP's congestion avoidance phase. Window growth is linear. phase = 'congestion avoidance' cws = cws + (1.0 / cws) else: # Analogous to TCP's slow start phase. Window growth is exponential. phase = 'slow start' cws += 1 if CONGESTION_WINDOW_SIZE[hostkey] != cws: log.debug('WebService: %s: %s: window size %.3f -> %.3f', hostkey, phase, CONGESTION_WINDOW_SIZE[hostkey], cws) CONGESTION_WINDOW_SIZE[hostkey] = cws def _send_request(self, request, access_token=None): hostkey = request.get_host_key() # Increment the number of unack'd requests on sending a new one CONGESTION_UNACK[hostkey] += 1 log.debug("WebService: %s: outstanding reqs: %d", hostkey, CONGESTION_UNACK[hostkey]) request.access_token = access_token send = self._request_methods[request.method] data = request.data reply = send( request, data.encode('utf-8')) if data is not None else send(request) self._remember_request_time(request.get_host_key()) self._active_requests[reply] = request def _start_request(self, request): if request.mblogin and request.path != "/oauth2/token": self.oauth_manager.get_access_token( partial(self._send_request, request)) else: self._send_request(request) @staticmethod def urls_equivalent(leftUrl, rightUrl): """ Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash from QT 4.7 """ return leftUrl.port(80) == rightUrl.port(80) and \ leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort) @staticmethod def url_port(url): if url.scheme() == 'https': return url.port(443) return url.port(80) def _handle_redirect(self, reply, request, redirect): url = request.url() error = int(reply.error()) # merge with base url (to cover the possibility of the URL being relative) redirect = url.resolved(redirect) if not WebService.urls_equivalent(redirect, reply.request().url()): log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo)) redirect_host = string_(redirect.host()) redirect_port = self.url_port(redirect) redirect_query = dict( QUrlQuery(redirect).queryItems(QUrl.FullyEncoded)) redirect_path = redirect.path() original_host = string_(url.host()) original_port = self.url_port(url) original_host_key = (original_host, original_port) redirect_host_key = (redirect_host, redirect_port) if (original_host_key in REQUEST_DELAY_MINIMUM and redirect_host_key not in REQUEST_DELAY_MINIMUM): log.debug("Setting the minimum rate limit for %s to %i" % (redirect_host_key, REQUEST_DELAY_MINIMUM[original_host_key])) REQUEST_DELAY_MINIMUM[ redirect_host_key] = REQUEST_DELAY_MINIMUM[ original_host_key] self.get(redirect_host, redirect_port, redirect_path, request.handler, request.parse_response_type, priority=True, important=True, refresh=request.refresh, queryargs=redirect_query, cacheloadcontrol=request.attribute( QtNetwork.QNetworkRequest.CacheLoadControlAttribute)) else: log.error("Redirect loop: %s", reply.request().url().toString(QUrl.RemoveUserInfo)) handler(reply.readAll(), reply, error) def _handle_reply(self, reply, request): hostkey = request.get_host_key() CONGESTION_UNACK[hostkey] -= 1 log.debug("WebService: %s: outstanding reqs: %d", hostkey, CONGESTION_UNACK[hostkey]) self._timer_run_next_task.start(0) slow_down = False error = int(reply.error()) handler = request.handler if error: code = reply.attribute( QtNetwork.QNetworkRequest.HttpStatusCodeAttribute) code = int(code) if code else 0 errstr = reply.errorString() url = reply.request().url().toString(QUrl.RemoveUserInfo) log.error( "Network request error for %s: %s (QT code %d, HTTP code %d)", url, errstr, error, code) if (not request.max_retries_reached() and (code == 503 or code == 429 # Sometimes QT returns a http status code of 200 even when there # is a service unavailable error. But it returns a QT error code # of 403 when this happens or error == 403)): retries = request.mark_for_retry() log.debug("Retrying %s (#%d)", url, retries) self.add_task(partial(self._start_request, request), request) elif handler is not None: handler(reply.readAll(), reply, error) slow_down = True else: redirect = reply.attribute( QtNetwork.QNetworkRequest.RedirectionTargetAttribute) fromCache = reply.attribute( QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute) cached = ' (CACHED)' if fromCache else '' log.debug( "Received reply for %s: HTTP %d (%s) %s", reply.request().url().toString(QUrl.RemoveUserInfo), reply.attribute( QtNetwork.QNetworkRequest.HttpStatusCodeAttribute), reply.attribute( QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute), cached) if handler is not None: # Redirect if found and not infinite if redirect: self._handle_redirect(reply, request, redirect) elif request.response_parser: try: document = request.response_parser(reply) except Exception as e: log.error("Unable to parse the response. %s", e) document = reply.readAll() finally: handler(document, reply, error) else: handler(reply.readAll(), reply, error) self._adjust_throttle(hostkey, slow_down) def _process_reply(self, reply): try: request = self._active_requests.pop(reply) except KeyError: log.error("Request not found for %s" % reply.request().url().toString(QUrl.RemoveUserInfo)) return try: self._handle_reply(reply, request) finally: reply.close() reply.deleteLater() def get(self, host, port, path, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE, priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False, queryargs=None): request = WSGetRequest(host, port, path, handler, parse_response_type=parse_response_type, mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs, priority=priority, important=important) func = partial(self._start_request, request) return self.add_task(func, request) def post(self, host, port, path, data, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE, priority=False, important=False, mblogin=True, queryargs=None): request = WSPostRequest(host, port, path, handler, parse_response_type=parse_response_type, data=data, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important) log.debug("POST-DATA %r", data) func = partial(self._start_request, request) return self.add_task(func, request) def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True, queryargs=None): request = WSPutRequest(host, port, path, handler, data=data, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important) func = partial(self._start_request, request) return self.add_task(func, request) def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True, queryargs=None): request = WSDeleteRequest(host, port, path, handler, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important) func = partial(self._start_request, request) return self.add_task(func, request) def download(self, host, port, path, handler, priority=False, important=False, cacheloadcontrol=None, refresh=False, queryargs=None): return self.get(host, port, path, handler, parse_response_type=None, priority=priority, important=important, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs) def stop(self): for reply in list(self._active_requests.keys()): reply.abort() self._init_queues() def _count_pending_requests(self): count = len(self._active_requests) for prio_queue in self._queues.values(): for queue in prio_queue.values(): count += len(queue) self.num_pending_web_requests = count if count != self._last_num_pending_web_requests: self._last_num_pending_web_requests = count self.tagger.tagger_stats_changed.emit() if count: self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS) def _get_delay_to_next_request(self, hostkey): """Calculate delay to next request to hostkey (host, port) returns a tuple (wait, delay) where: wait is True if a delay is needed delay is the delay in milliseconds to next request """ if CONGESTION_UNACK[hostkey] >= int(CONGESTION_WINDOW_SIZE[hostkey]): # We've maxed out the number of requests to `hostkey`, so wait # until responses begin to come back. (See `_timer_run_next_task` # strobe in `_handle_reply`.) return (True, sys.maxsize) interval = REQUEST_DELAY[hostkey] if not interval: log.debug("WSREQ: Starting another request to %s without delay", hostkey) return (False, 0) last_request = self._last_request_times[hostkey] if not last_request: log.debug("WSREQ: First request to %s", hostkey) self._remember_request_time(hostkey) # set it on first run return (False, interval) elapsed = (time.time() - last_request) * 1000 if elapsed >= interval: log.debug( "WSREQ: Last request to %s was %d ms ago, starting another one", hostkey, elapsed) return (False, interval) delay = int(math.ceil(interval - elapsed)) log.debug( "WSREQ: Last request to %s was %d ms ago, waiting %d ms before starting another one", hostkey, elapsed, delay) return (True, delay) def _remember_request_time(self, hostkey): if REQUEST_DELAY[hostkey]: self._last_request_times[hostkey] = time.time() def _run_next_task(self): delay = sys.maxsize for prio in sorted(self._queues.keys(), reverse=True): prio_queue = self._queues[prio] if not prio_queue: del (self._queues[prio]) continue for hostkey in sorted(prio_queue.keys(), key=lambda hostkey: REQUEST_DELAY[hostkey]): queue = self._queues[prio][hostkey] if not queue: del (self._queues[prio][hostkey]) continue wait, d = self._get_delay_to_next_request(hostkey) if not wait: queue.popleft()() if d < delay: delay = d if delay < sys.maxsize: self._timer_run_next_task.start(delay) def add_task(self, func, request): hostkey = request.get_host_key() prio = int(request.priority) # priority is a boolean if request.important: self._queues[prio][hostkey].appendleft(func) else: self._queues[prio][hostkey].append(func) if not self._timer_run_next_task.isActive(): self._timer_run_next_task.start(0) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) return (hostkey, func, prio) def remove_task(self, task): hostkey, func, prio = task try: self._queues[prio][hostkey].remove(func) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) except Exception as e: log.debug(e) @classmethod def add_parser(cls, response_type, mimetype, parser): cls.PARSERS[response_type] = Parser(mimetype=mimetype, parser=parser) @classmethod def get_response_mimetype(cls, response_type): if response_type in cls.PARSERS: return cls.PARSERS[response_type].mimetype else: raise UnknownResponseParserError(response_type) @classmethod def get_response_parser(cls, response_type): if response_type in cls.PARSERS: return cls.PARSERS[response_type].parser else: raise UnknownResponseParserError(response_type)
class WebService(QtCore.QObject): PARSERS = dict() def __init__(self, parent=None): super().__init__(parent) self.manager = QtNetwork.QNetworkAccessManager() self._network_accessible_changed(self.manager.networkAccessible()) self.manager.networkAccessibleChanged.connect(self._network_accessible_changed) self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() config = get_config() self.set_transfer_timeout(config.setting['network_transfer_timeout_seconds']) self.manager.finished.connect(self._process_reply) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self._init_queues() self._init_timers() @staticmethod def http_response_code(reply): response_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) return int(response_code) if response_code else 0 @staticmethod def http_response_phrase(reply): return reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute) @staticmethod def http_response_safe_url(reply): return reply.request().url().toString(QUrl.RemoveUserInfo) def _network_accessible_changed(self, accessible): # Qt's network accessibility check sometimes fails, e.g. with VPNs on Windows. # If the accessibility is reported to be not accessible, set it to # unknown instead. Let's just try any request and handle the error. # See https://tickets.metabrainz.org/browse/PICARD-1791 if accessible == QtNetwork.QNetworkAccessManager.NotAccessible: self.manager.setNetworkAccessible(QtNetwork.QNetworkAccessManager.UnknownAccessibility) log.debug("Network accessible requested: %s, actual: %s", accessible, self.manager.networkAccessible()) def _init_queues(self): self._active_requests = {} self._queue = RequestPriorityQueue(ratecontrol) self.num_pending_web_requests = 0 def _init_timers(self): self._timer_run_next_task = QtCore.QTimer(self) self._timer_run_next_task.setSingleShot(True) self._timer_run_next_task.timeout.connect(self._run_next_task) self._timer_count_pending_requests = QtCore.QTimer(self) self._timer_count_pending_requests.setSingleShot(True) self._timer_count_pending_requests.timeout.connect(self._count_pending_requests) def set_cache(self, cache_size_in_bytes=None): if cache_size_in_bytes is None: cache_size_in_bytes = CACHE_SIZE_IN_BYTES cache = QtNetwork.QNetworkDiskCache() cache.setCacheDirectory(os.path.join(appdirs.cache_folder(), 'network')) cache.setMaximumCacheSize(cache_size_in_bytes) self.manager.setCache(cache) log.debug("NetworkDiskCache dir: %r current size: %s max size: %s", cache.cacheDirectory(), bytes2human.decimal(cache.cacheSize(), l10n=False), bytes2human.decimal(cache.maximumCacheSize(), l10n=False)) def setup_proxy(self): proxy = QtNetwork.QNetworkProxy() config = get_config() if config.setting["use_proxy"]: if config.setting["proxy_type"] == 'socks': proxy.setType(QtNetwork.QNetworkProxy.Socks5Proxy) else: proxy.setType(QtNetwork.QNetworkProxy.HttpProxy) proxy.setHostName(config.setting["proxy_server_host"]) proxy.setPort(config.setting["proxy_server_port"]) if config.setting["proxy_username"]: proxy.setUser(config.setting["proxy_username"]) if config.setting["proxy_password"]: proxy.setPassword(config.setting["proxy_password"]) self.manager.setProxy(proxy) def set_transfer_timeout(self, timeout): timeout_ms = timeout * 1000 if hasattr(self.manager, 'setTransferTimeout'): # Available since Qt 5.15 self.manager.setTransferTimeout(timeout_ms) self._transfer_timeout = 0 else: # Use fallback implementation self._transfer_timeout = timeout_ms def _send_request(self, request, access_token=None): hostkey = request.get_host_key() ratecontrol.increment_requests(hostkey) request.access_token = access_token send = self._request_methods[request.method] data = request.data reply = send(request, data.encode('utf-8')) if data is not None else send(request) self._start_transfer_timeout(reply) self._active_requests[reply] = request def _start_transfer_timeout(self, reply): if not self._transfer_timeout: return # Fallback implementation of a transfer timeout for Qt < 5.15. # Aborts a request if no data gets transferred for TRANSFER_TIMEOUT milliseconds. timer = QtCore.QTimer(self) timer.setSingleShot(True) timer.setTimerType(QtCore.Qt.PreciseTimer) timer.timeout.connect(partial(self._timeout_request, reply)) reply.finished.connect(timer.stop) reset_callback = partial(self._reset_transfer_timeout, timer) reply.uploadProgress.connect(reset_callback) reply.downloadProgress.connect(reset_callback) timer.start(self._transfer_timeout) def _reset_transfer_timeout(self, timer, bytesTransferred, bytesTotal): timer.start(self._transfer_timeout) @staticmethod def _timeout_request(reply): if reply.isRunning(): reply.abort() def _start_request(self, request): if request.mblogin and request.path != "/oauth2/token": self.oauth_manager.get_access_token(partial(self._send_request, request)) else: self._send_request(request) @staticmethod def urls_equivalent(leftUrl, rightUrl): """ Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash from QT 4.7 """ return leftUrl.port(80) == rightUrl.port(80) and \ leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort) @staticmethod def url_port(url): if url.scheme() == 'https': return url.port(443) return url.port(80) def _handle_redirect(self, reply, request, redirect): url = request.url() error = int(reply.error()) # merge with base url (to cover the possibility of the URL being relative) redirect = url.resolved(redirect) if not WebService.urls_equivalent(redirect, reply.request().url()): log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo)) redirect_host = redirect.host() redirect_port = self.url_port(redirect) redirect_query = dict(QUrlQuery(redirect).queryItems(QUrl.FullyEncoded)) redirect_path = redirect.path() original_host = url.host() original_port = self.url_port(url) original_host_key = (original_host, original_port) redirect_host_key = (redirect_host, redirect_port) ratecontrol.copy_minimal_delay(original_host_key, redirect_host_key) self.get(redirect_host, redirect_port, redirect_path, request.handler, request.parse_response_type, priority=True, important=True, refresh=request.refresh, queryargs=redirect_query, mblogin=request.mblogin, cacheloadcontrol=request.attribute(QNetworkRequest.CacheLoadControlAttribute)) else: log.error("Redirect loop: %s", self.http_response_safe_url(reply) ) request.handler(reply.readAll(), reply, error) def _handle_reply(self, reply, request): hostkey = request.get_host_key() ratecontrol.decrement_requests(hostkey) self._timer_run_next_task.start(0) slow_down = False error = int(reply.error()) handler = request.handler response_code = self.http_response_code(reply) url = self.http_response_safe_url(reply) if error: errstr = reply.errorString() log.error("Network request error for %s: %s (QT code %d, HTTP code %d)", url, errstr, error, response_code) if (not request.max_retries_reached() and (response_code == 503 or response_code == 429 # Sometimes QT returns a http status code of 200 even when there # is a service unavailable error. But it returns a QT error code # of 403 when this happens or error == 403 )): slow_down = True retries = request.mark_for_retry() log.debug("Retrying %s (#%d)", url, retries) self.add_request(request) elif handler is not None: handler(reply.readAll(), reply, error) slow_down = (slow_down or response_code >= 500) else: redirect = reply.attribute(QNetworkRequest.RedirectionTargetAttribute) from_cache = reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute) cached = ' (CACHED)' if from_cache else '' log.debug("Received reply for %s: HTTP %d (%s) %s", url, response_code, self.http_response_phrase(reply), cached ) if handler is not None: # Redirect if found and not infinite if redirect: self._handle_redirect(reply, request, redirect) elif request.response_parser: try: document = request.response_parser(reply) log.debug("Response received: %s", document) except Exception as e: log.error("Unable to parse the response for %s: %s", url, e) document = reply.readAll() error = e finally: handler(document, reply, error) else: handler(reply.readAll(), reply, error) ratecontrol.adjust(hostkey, slow_down) def _process_reply(self, reply): try: request = self._active_requests.pop(reply) except KeyError: log.error("Request not found for %s", self.http_response_safe_url(reply)) return try: self._handle_reply(reply, request) finally: reply.close() reply.deleteLater() def get(self, host, port, path, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE, priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False, queryargs=None): request = WSGetRequest(host, port, path, handler, parse_response_type=parse_response_type, mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs, priority=priority, important=important) return self.add_request(request) def post(self, host, port, path, data, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE, priority=False, important=False, mblogin=True, queryargs=None, request_mimetype=None): request = WSPostRequest(host, port, path, handler, parse_response_type=parse_response_type, data=data, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important, request_mimetype=request_mimetype) log.debug("POST-DATA %r", data) return self.add_request(request) def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True, queryargs=None, request_mimetype=None): request = WSPutRequest(host, port, path, handler, data=data, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important, request_mimetype=request_mimetype) return self.add_request(request) def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True, queryargs=None): request = WSDeleteRequest(host, port, path, handler, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important) return self.add_request(request) def download(self, host, port, path, handler, priority=False, important=False, cacheloadcontrol=None, refresh=False, queryargs=None): return self.get(host, port, path, handler, parse_response_type=None, priority=priority, important=important, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs) def stop(self): for reply in list(self._active_requests): reply.abort() self._init_queues() def _count_pending_requests(self): count = len(self._active_requests) + self._queue.count() if count != self.num_pending_web_requests: self.num_pending_web_requests = count self.tagger.tagger_stats_changed.emit() if count: self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS) def _run_next_task(self): delay = self._queue.run_ready_tasks() if delay < sys.maxsize: self._timer_run_next_task.start(delay) def add_task(self, func, request): task = RequestTask.from_request(request, func) self._queue.add_task(task, request.important) if not self._timer_run_next_task.isActive(): self._timer_run_next_task.start(0) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) return task def add_request(self, request): return self.add_task(partial(self._start_request, request), request) def remove_task(self, task): self._queue.remove_task(task) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) @classmethod def add_parser(cls, response_type, mimetype, parser): cls.PARSERS[response_type] = Parser(mimetype=mimetype, parser=parser) @classmethod def get_response_mimetype(cls, response_type): if response_type in cls.PARSERS: return cls.PARSERS[response_type].mimetype else: raise UnknownResponseParserError(response_type) @classmethod def get_response_parser(cls, response_type): if response_type in cls.PARSERS: return cls.PARSERS[response_type].parser else: raise UnknownResponseParserError(response_type)
class XmlWebService(QtCore.QObject): def __init__(self, parent=None): QtCore.QObject.__init__(self, parent) self.manager = QtNetwork.QNetworkAccessManager() self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() self.manager.finished.connect(self._process_reply) self._last_request_times = {} self._active_requests = {} self._high_priority_queues = {} self._low_priority_queues = {} self._hosts = [] self._timer = QtCore.QTimer(self) self._timer.setSingleShot(True) self._timer.timeout.connect(self._run_next_task) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self.num_pending_web_requests = 0 def set_cache(self, cache_size_in_mb=100): cache = QtNetwork.QNetworkDiskCache() location = QDesktopServices.storageLocation(QDesktopServices.CacheLocation) cache.setCacheDirectory(os.path.join(unicode(location), u'picard')) cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024) self.manager.setCache(cache) log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory()) log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(), cache.maximumCacheSize()) def setup_proxy(self): proxy = QtNetwork.QNetworkProxy() if config.setting["use_proxy"]: proxy.setType(QtNetwork.QNetworkProxy.HttpProxy) proxy.setHostName(config.setting["proxy_server_host"]) proxy.setPort(config.setting["proxy_server_port"]) proxy.setUser(config.setting["proxy_username"]) proxy.setPassword(config.setting["proxy_password"]) self.manager.setProxy(proxy) def _start_request_continue(self, method, host, port, path, data, handler, xml, mblogin=False, cacheloadcontrol=None, refresh=None, access_token=None): if mblogin and host in MUSICBRAINZ_SERVERS and port == 80: urlstring = "https://%s%s" % (host, path) else: urlstring = "http://%s:%d%s" % (host, port, path) log.debug("%s %s", method, urlstring) url = QUrl.fromEncoded(urlstring) request = QtNetwork.QNetworkRequest(url) if mblogin and access_token: request.setRawHeader("Authorization", "Bearer %s" % access_token) if mblogin or (method == "GET" and refresh): request.setPriority(QtNetwork.QNetworkRequest.HighPriority) request.setAttribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute, QtNetwork.QNetworkRequest.AlwaysNetwork) elif method == "PUT" or method == "DELETE": request.setPriority(QtNetwork.QNetworkRequest.HighPriority) elif cacheloadcontrol is not None: request.setAttribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute, cacheloadcontrol) request.setRawHeader("User-Agent", USER_AGENT_STRING) if xml: request.setRawHeader("Accept", "application/xml") if data is not None: if method == "POST" and host == config.setting["server_host"] and xml: request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/xml; charset=utf-8") else: request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/x-www-form-urlencoded") send = self._request_methods[method] reply = send(request, data) if data is not None else send(request) key = (host, port) self._last_request_times[key] = time.time() self._active_requests[reply] = (request, handler, xml, refresh) def _start_request(self, method, host, port, path, data, handler, xml, mblogin=False, cacheloadcontrol=None, refresh=None): def start_request_continue(access_token=None): self._start_request_continue( method, host, port, path, data, handler, xml, mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, access_token=access_token) if mblogin and path != "/oauth2/token": self.oauth_manager.get_access_token(start_request_continue) else: start_request_continue() @staticmethod def urls_equivalent(leftUrl, rightUrl): """ Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash from QT 4.7 """ return leftUrl.port(80) == rightUrl.port(80) and \ leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort) def _process_reply(self, reply): try: request, handler, xml, refresh = self._active_requests.pop(reply) except KeyError: log.error("Request not found for %s" % reply.request().url().toString(QUrl.RemoveUserInfo)) return error = int(reply.error()) if error: log.error("Network request error for %s: %s (QT code %d, HTTP code %s)", reply.request().url().toString(QUrl.RemoveUserInfo), reply.errorString(), error, repr(reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute)) ) if handler is not None: handler(str(reply.readAll()), reply, error) else: redirect = reply.attribute(QtNetwork.QNetworkRequest.RedirectionTargetAttribute) fromCache = reply.attribute(QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute) cached = ' (CACHED)' if fromCache else '' log.debug("Received reply for %s: HTTP %d (%s) %s", reply.request().url().toString(QUrl.RemoveUserInfo), reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute), reply.attribute(QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute), cached ) if handler is not None: # Redirect if found and not infinite if redirect and not XmlWebService.urls_equivalent(redirect, reply.request().url()): log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo)) redirect_host = str(redirect.host()) redirect_port = redirect.port(80) url = request.url() original_host = str(url.host()) original_port = url.port(80) if ((original_host, original_port) in REQUEST_DELAY and (redirect_host, redirect_port) not in REQUEST_DELAY): log.debug("Setting rate limit for %s:%i to %i" % (redirect_host, redirect_port, REQUEST_DELAY[(original_host, original_port)])) REQUEST_DELAY[(redirect_host, redirect_port)] =\ REQUEST_DELAY[(original_host, original_port)] self.get(redirect_host, redirect_port, # retain path, query string and anchors from redirect URL redirect.toString(QUrl.RemoveAuthority | QUrl.RemoveScheme), handler, xml, priority=True, important=True, refresh=refresh, cacheloadcontrol=request.attribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute)) elif redirect: log.error("Redirect loop: %s", reply.request().url().toString(QUrl.RemoveUserInfo) ) handler(str(reply.readAll()), reply, error) elif xml: document = _read_xml(QXmlStreamReader(reply)) handler(document, reply, error) else: handler(str(reply.readAll()), reply, error) reply.close() self.num_pending_web_requests -= 1 self.tagger.tagger_stats_changed.emit() def get(self, host, port, path, handler, xml=True, priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False): func = partial(self._start_request, "GET", host, port, path, None, handler, xml, mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh) return self.add_task(func, host, port, priority, important=important) def post(self, host, port, path, data, handler, xml=True, priority=False, important=False, mblogin=True): log.debug("POST-DATA %r", data) func = partial(self._start_request, "POST", host, port, path, data, handler, xml, mblogin) return self.add_task(func, host, port, priority, important=important) def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True): func = partial(self._start_request, "PUT", host, port, path, data, handler, False, mblogin) return self.add_task(func, host, port, priority, important=important) def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True): func = partial(self._start_request, "DELETE", host, port, path, None, handler, False, mblogin) return self.add_task(func, host, port, priority, important=important) def stop(self): self._high_priority_queues = {} self._low_priority_queues = {} for reply in self._active_requests.keys(): reply.abort() def _run_next_task(self): delay = sys.maxint for key in self._hosts: queue = self._high_priority_queues.get(key) or self._low_priority_queues.get(key) if not queue: continue now = time.time() last = self._last_request_times.get(key) request_delay = REQUEST_DELAY[key] last_ms = (now - last) * 1000 if last is not None else request_delay if last_ms >= request_delay: log.debug("Last request to %s was %d ms ago, starting another one", key, last_ms) d = request_delay queue.popleft()() else: d = int(math.ceil(request_delay - last_ms)) log.debug("Waiting %d ms before starting another request to %s", d, key) if d < delay: delay = d if delay < sys.maxint: self._timer.start(delay) def add_task(self, func, host, port, priority, important=False): key = (host, port) if key not in self._hosts: self._hosts.append(key) if priority: queues = self._high_priority_queues else: queues = self._low_priority_queues queues.setdefault(key, deque()) if important: queues[key].appendleft(func) else: queues[key].append(func) self.num_pending_web_requests += 1 self.tagger.tagger_stats_changed.emit() if len(queues[key]) == 1: self._timer.start(0) return (key, func, priority) def remove_task(self, task): key, func, priority = task if priority: queue = self._high_priority_queues[key] else: queue = self._low_priority_queues[key] try: queue.remove(func) except: pass else: self.num_pending_web_requests -= 1 self.tagger.tagger_stats_changed.emit() def _get_by_id(self, entitytype, entityid, handler, inc=[], params=[], priority=False, important=False, mblogin=False, refresh=False): host = config.setting["server_host"] port = config.setting["server_port"] path = "/ws/2/%s/%s?inc=%s" % (entitytype, entityid, "+".join(inc)) if params: path += "&" + "&".join(params) return self.get(host, port, path, handler, priority=priority, important=important, mblogin=mblogin, refresh=refresh) def get_release_by_id(self, releaseid, handler, inc=[], priority=False, important=False, mblogin=False, refresh=False): return self._get_by_id('release', releaseid, handler, inc, priority=priority, important=important, mblogin=mblogin, refresh=refresh) def get_track_by_id(self, trackid, handler, inc=[], priority=False, important=False, mblogin=False, refresh=False): return self._get_by_id('recording', trackid, handler, inc, priority=priority, important=important, mblogin=mblogin, refresh=refresh) def lookup_discid(self, discid, handler, priority=True, important=True, refresh=False): inc = ['artist-credits', 'labels'] return self._get_by_id('discid', discid, handler, inc, params=["cdstubs=no"], priority=priority, important=important, refresh=refresh) def _find(self, entitytype, handler, kwargs): host = config.setting["server_host"] port = config.setting["server_port"] filters = [] query = [] for name, value in kwargs.items(): if name == 'limit': filters.append((name, str(value))) else: value = _escape_lucene_query(value).strip().lower() if value: query.append('%s:(%s)' % (name, value)) if query: filters.append(('query', ' '.join(query))) params = [] for name, value in filters: value = QUrl.toPercentEncoding(unicode(value)) params.append('%s=%s' % (str(name), value)) path = "/ws/2/%s/?%s" % (entitytype, "&".join(params)) return self.get(host, port, path, handler) def find_releases(self, handler, **kwargs): return self._find('release', handler, kwargs) def find_tracks(self, handler, **kwargs): return self._find('recording', handler, kwargs) def _browse(self, entitytype, handler, kwargs, inc=[], priority=False, important=False): host = config.setting["server_host"] port = config.setting["server_port"] params = "&".join(["%s=%s" % (k, v) for k, v in kwargs.items()]) path = "/ws/2/%s?%s&inc=%s" % (entitytype, params, "+".join(inc)) return self.get(host, port, path, handler, priority=priority, important=important) def browse_releases(self, handler, priority=True, important=True, **kwargs): inc = ["media", "labels"] return self._browse("release", handler, kwargs, inc, priority=priority, important=important) def submit_ratings(self, ratings, handler): host = config.setting['server_host'] port = config.setting['server_port'] path = '/ws/2/rating/?client=' + CLIENT_STRING recordings = (''.join(['<recording id="%s"><user-rating>%s</user-rating></recording>' % (i[1], j*20) for i, j in ratings.items() if i[0] == 'recording'])) data = _wrap_xml_metadata('<recording-list>%s</recording-list>' % recordings) return self.post(host, port, path, data, handler, priority=True) def _encode_acoustid_args(self, args): filters = [] args['client'] = ACOUSTID_KEY args['clientversion'] = PICARD_VERSION_STR args['format'] = 'xml' for name, value in args.items(): value = str(QUrl.toPercentEncoding(value)) filters.append('%s=%s' % (str(name), value)) return '&'.join(filters) def query_acoustid(self, handler, **args): host, port = ACOUSTID_HOST, ACOUSTID_PORT body = self._encode_acoustid_args(args) return self.post(host, port, '/v2/lookup', body, handler, priority=False, important=False, mblogin=False) def submit_acoustid_fingerprints(self, submissions, handler): args = {'user': config.setting["acoustid_apikey"]} for i, submission in enumerate(submissions): args['fingerprint.%d' % i] = str(submission.fingerprint) args['duration.%d' % i] = str(submission.duration) args['mbid.%d' % i] = str(submission.recordingid) if submission.puid: args['puid.%d' % i] = str(submission.puid) host, port = ACOUSTID_HOST, ACOUSTID_PORT body = self._encode_acoustid_args(args) return self.post(host, port, '/v2/submit', body, handler, priority=True, important=False, mblogin=False) def download(self, host, port, path, handler, priority=False, important=False, cacheloadcontrol=None, refresh=False): return self.get(host, port, path, handler, xml=False, priority=priority, important=important, cacheloadcontrol=cacheloadcontrol, refresh=refresh) def get_collection(self, id, handler, limit=100, offset=0): host, port = config.setting['server_host'], config.setting['server_port'] path = "/ws/2/collection" if id is not None: inc = ["releases", "artist-credits", "media"] path += "/%s/releases?inc=%s&limit=%d&offset=%d" % (id, "+".join(inc), limit, offset) return self.get(host, port, path, handler, priority=True, important=True, mblogin=True) def get_collection_list(self, handler): return self.get_collection(None, handler) def _collection_request(self, id, releases): while releases: ids = ";".join(releases if len(releases) <= 400 else releases[:400]) releases = releases[400:] yield "/ws/2/collection/%s/releases/%s?client=%s" % (id, ids, CLIENT_STRING) def put_to_collection(self, id, releases, handler): host, port = config.setting['server_host'], config.setting['server_port'] for path in self._collection_request(id, releases): self.put(host, port, path, "", handler) def delete_from_collection(self, id, releases, handler): host, port = config.setting['server_host'], config.setting['server_port'] for path in self._collection_request(id, releases): self.delete(host, port, path, handler)
class WebService(QtCore.QObject): PARSERS = dict() def __init__(self, parent=None): super().__init__(parent) self.manager = QtNetwork.QNetworkAccessManager() self.oauth_manager = OAuthManager(self) self.set_cache() self.setup_proxy() self.manager.finished.connect(self._process_reply) self._request_methods = { "GET": self.manager.get, "POST": self.manager.post, "PUT": self.manager.put, "DELETE": self.manager.deleteResource } self._init_queues() self._init_timers() def _init_queues(self): self._active_requests = {} self._queues = defaultdict(lambda: defaultdict(deque)) self.num_pending_web_requests = 0 self._last_num_pending_web_requests = -1 def _init_timers(self): self._timer_run_next_task = QtCore.QTimer(self) self._timer_run_next_task.setSingleShot(True) self._timer_run_next_task.timeout.connect(self._run_next_task) self._timer_count_pending_requests = QtCore.QTimer(self) self._timer_count_pending_requests.setSingleShot(True) self._timer_count_pending_requests.timeout.connect(self._count_pending_requests) def set_cache(self, cache_size_in_mb=100): cache = QtNetwork.QNetworkDiskCache() location = QStandardPaths.writableLocation(QStandardPaths.CacheLocation) cache.setCacheDirectory(os.path.join(location, 'picard')) cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024) self.manager.setCache(cache) log.debug("NetworkDiskCache dir: %r size: %s / %s", cache.cacheDirectory(), cache.cacheSize(), cache.maximumCacheSize()) def setup_proxy(self): proxy = QtNetwork.QNetworkProxy() if config.setting["use_proxy"]: proxy.setType(QtNetwork.QNetworkProxy.HttpProxy) proxy.setHostName(config.setting["proxy_server_host"]) proxy.setPort(config.setting["proxy_server_port"]) proxy.setUser(config.setting["proxy_username"]) proxy.setPassword(config.setting["proxy_password"]) self.manager.setProxy(proxy) def _send_request(self, request, access_token=None): hostkey = request.get_host_key() ratecontrol.increment_requests(hostkey) request.access_token = access_token send = self._request_methods[request.method] data = request.data reply = send(request, data.encode('utf-8')) if data is not None else send(request) self._active_requests[reply] = request def _start_request(self, request): if request.mblogin and request.path != "/oauth2/token": self.oauth_manager.get_access_token(partial(self._send_request, request)) else: self._send_request(request) @staticmethod def urls_equivalent(leftUrl, rightUrl): """ Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash from QT 4.7 """ return leftUrl.port(80) == rightUrl.port(80) and \ leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort) @staticmethod def url_port(url): if url.scheme() == 'https': return url.port(443) return url.port(80) def _handle_redirect(self, reply, request, redirect): url = request.url() error = int(reply.error()) # merge with base url (to cover the possibility of the URL being relative) redirect = url.resolved(redirect) if not WebService.urls_equivalent(redirect, reply.request().url()): log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo)) redirect_host = redirect.host() redirect_port = self.url_port(redirect) redirect_query = dict(QUrlQuery(redirect).queryItems(QUrl.FullyEncoded)) redirect_path = redirect.path() original_host = url.host() original_port = self.url_port(url) original_host_key = (original_host, original_port) redirect_host_key = (redirect_host, redirect_port) ratecontrol.copy_minimal_delay(original_host_key, redirect_host_key) self.get(redirect_host, redirect_port, redirect_path, request.handler, request.parse_response_type, priority=True, important=True, refresh=request.refresh, queryargs=redirect_query, mblogin=request.mblogin, cacheloadcontrol=request.attribute(QNetworkRequest.CacheLoadControlAttribute)) else: log.error("Redirect loop: %s", reply.request().url().toString(QUrl.RemoveUserInfo) ) request.handler(reply.readAll(), reply, error) def _handle_reply(self, reply, request): hostkey = request.get_host_key() ratecontrol.decrement_requests(hostkey) self._timer_run_next_task.start(0) slow_down = False error = int(reply.error()) handler = request.handler if error: code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute) code = int(code) if code else 0 errstr = reply.errorString() url = reply.request().url().toString(QUrl.RemoveUserInfo) log.error("Network request error for %s: %s (QT code %d, HTTP code %d)", url, errstr, error, code) if (not request.max_retries_reached() and (code == 503 or code == 429 # Sometimes QT returns a http status code of 200 even when there # is a service unavailable error. But it returns a QT error code # of 403 when this happens or error == 403 ) ): slow_down = True retries = request.mark_for_retry() log.debug("Retrying %s (#%d)", url, retries) self.add_request(request) elif handler is not None: handler(reply.readAll(), reply, error) slow_down = (slow_down or code >= 500) else: redirect = reply.attribute(QNetworkRequest.RedirectionTargetAttribute) fromCache = reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute) cached = ' (CACHED)' if fromCache else '' log.debug("Received reply for %s: HTTP %d (%s) %s", reply.request().url().toString(QUrl.RemoveUserInfo), reply.attribute(QNetworkRequest.HttpStatusCodeAttribute), reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute), cached ) if handler is not None: # Redirect if found and not infinite if redirect: self._handle_redirect(reply, request, redirect) elif request.response_parser: try: document = request.response_parser(reply) except Exception as e: url = reply.request().url().toString(QUrl.RemoveUserInfo) log.error("Unable to parse the response for %s: %s", url, e) document = reply.readAll() error = e finally: handler(document, reply, error) else: handler(reply.readAll(), reply, error) ratecontrol.adjust(hostkey, slow_down) def _process_reply(self, reply): try: request = self._active_requests.pop(reply) except KeyError: log.error("Request not found for %s", reply.request().url().toString(QUrl.RemoveUserInfo)) return try: self._handle_reply(reply, request) finally: reply.close() reply.deleteLater() def get(self, host, port, path, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE, priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False, queryargs=None): request = WSGetRequest(host, port, path, handler, parse_response_type=parse_response_type, mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs, priority=priority, important=important) return self.add_request(request) def post(self, host, port, path, data, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE, priority=False, important=False, mblogin=True, queryargs=None, request_mimetype=None): request = WSPostRequest(host, port, path, handler, parse_response_type=parse_response_type, data=data, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important, request_mimetype=request_mimetype) log.debug("POST-DATA %r", data) return self.add_request(request) def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True, queryargs=None, request_mimetype=None): request = WSPutRequest(host, port, path, handler, data=data, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important, request_mimetype=request_mimetype) return self.add_request(request) def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True, queryargs=None): request = WSDeleteRequest(host, port, path, handler, mblogin=mblogin, queryargs=queryargs, priority=priority, important=important) return self.add_request(request) def download(self, host, port, path, handler, priority=False, important=False, cacheloadcontrol=None, refresh=False, queryargs=None): return self.get(host, port, path, handler, parse_response_type=None, priority=priority, important=important, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs) def stop(self): for reply in list(self._active_requests): reply.abort() self._init_queues() def _count_pending_requests(self): count = len(self._active_requests) for prio_queue in self._queues.values(): for queue in prio_queue.values(): count += len(queue) self.num_pending_web_requests = count if count != self._last_num_pending_web_requests: self._last_num_pending_web_requests = count self.tagger.tagger_stats_changed.emit() if count: self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS) def _run_next_task(self): delay = sys.maxsize for prio in sorted(self._queues.keys(), reverse=True): prio_queue = self._queues[prio] if not prio_queue: del(self._queues[prio]) continue for hostkey in sorted(prio_queue.keys(), key=ratecontrol.current_delay): queue = self._queues[prio][hostkey] if not queue: del(self._queues[prio][hostkey]) continue wait, d = ratecontrol.get_delay_to_next_request(hostkey) if not wait: queue.popleft()() if d < delay: delay = d if delay < sys.maxsize: self._timer_run_next_task.start(delay) def add_task(self, func, request): hostkey = request.get_host_key() prio = int(request.priority) # priority is a boolean if request.important: self._queues[prio][hostkey].appendleft(func) else: self._queues[prio][hostkey].append(func) if not self._timer_run_next_task.isActive(): self._timer_run_next_task.start(0) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) return (hostkey, func, prio) def add_request(self, request): return self.add_task(partial(self._start_request, request), request) def remove_task(self, task): hostkey, func, prio = task try: self._queues[prio][hostkey].remove(func) if not self._timer_count_pending_requests.isActive(): self._timer_count_pending_requests.start(0) except Exception as e: log.debug(e) @classmethod def add_parser(cls, response_type, mimetype, parser): cls.PARSERS[response_type] = Parser(mimetype=mimetype, parser=parser) @classmethod def get_response_mimetype(cls, response_type): if response_type in cls.PARSERS: return cls.PARSERS[response_type].mimetype else: raise UnknownResponseParserError(response_type) @classmethod def get_response_parser(cls, response_type): if response_type in cls.PARSERS: return cls.PARSERS[response_type].parser else: raise UnknownResponseParserError(response_type)