def __init__(self, parent=None):
super().__init__(parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self._init_queues()
self._init_timers()
def test_query_data(self):
params = {
'a&b': 'a b',
'c d': 'c&d',
'e=f': 'e=f',
'': '',
}
data = OAuthManager._query_data(params)
self.assertEqual(data, "a%26b=a+b&c+d=c%26d&e%3Df=e%3Df")
def __init__(self, parent=None):
super().__init__(parent)
self.manager = QtNetwork.QNetworkAccessManager()
self._network_accessible_changed(self.manager.networkAccessible())
self.manager.networkAccessibleChanged.connect(self._network_accessible_changed)
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
config = get_config()
self.set_transfer_timeout(config.setting['network_transfer_timeout_seconds'])
self.manager.finished.connect(self._process_reply)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self._init_queues()
self._init_timers()
def __init__(self, parent=None):
QtCore.QObject.__init__(self, parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._last_request_times = {}
self._active_requests = {}
self._high_priority_queues = {}
self._low_priority_queues = {}
self._hosts = []
self._timer = QtCore.QTimer(self)
self._timer.setSingleShot(True)
self._timer.timeout.connect(self._run_next_task)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self.num_pending_web_requests = 0
def __init__(self, parent=None):
super().__init__(parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self._init_queues()
self._init_timers()
def __init__(self, parent=None):
QtCore.QObject.__init__(self, parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._last_request_times = defaultdict(lambda: 0)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource,
}
self._init_queues()
self._init_timers()
def __init__(self, parent=None):
QtCore.QObject.__init__(self, parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._last_request_times = {}
self._active_requests = {}
self._high_priority_queues = {}
self._low_priority_queues = {}
self._hosts = []
self._timer = QtCore.QTimer(self)
self._timer.setSingleShot(True)
self._timer.timeout.connect(self._run_next_task)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self.num_pending_web_requests = 0
class XmlWebService(QtCore.QObject):
def __init__(self, parent=None):
QtCore.QObject.__init__(self, parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._last_request_times = defaultdict(lambda: 0)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self._init_queues()
self._init_timers()
def _init_queues(self):
self._active_requests = {}
self._queues = defaultdict(lambda: defaultdict(deque))
self.num_pending_web_requests = 0
self._last_num_pending_web_requests = -1
def _init_timers(self):
self._timer_run_next_task = QtCore.QTimer(self)
self._timer_run_next_task.setSingleShot(True)
self._timer_run_next_task.timeout.connect(self._run_next_task)
self._timer_count_pending_requests = QtCore.QTimer(self)
self._timer_count_pending_requests.setSingleShot(True)
self._timer_count_pending_requests.timeout.connect(self._count_pending_requests)
def set_cache(self, cache_size_in_mb=100):
cache = QtNetwork.QNetworkDiskCache()
location = QDesktopServices.storageLocation(QDesktopServices.CacheLocation)
cache.setCacheDirectory(os.path.join(unicode(location), u'picard'))
cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024)
self.manager.setCache(cache)
log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory())
log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(),
cache.maximumCacheSize())
def setup_proxy(self):
proxy = QtNetwork.QNetworkProxy()
if config.setting["use_proxy"]:
proxy.setType(QtNetwork.QNetworkProxy.HttpProxy)
proxy.setHostName(config.setting["proxy_server_host"])
proxy.setPort(config.setting["proxy_server_port"])
proxy.setUser(config.setting["proxy_username"])
proxy.setPassword(config.setting["proxy_password"])
self.manager.setProxy(proxy)
def _start_request_continue(self, method, host, port, path, data, handler, xml,
mblogin=False, cacheloadcontrol=None, refresh=None,
access_token=None, queryargs=None):
url = build_qurl(host, port, path=path, mblogin=mblogin,
queryargs=queryargs)
request = QtNetwork.QNetworkRequest(url)
if mblogin and access_token:
request.setRawHeader("Authorization", "Bearer %s" % access_token)
if mblogin or (method == "GET" and refresh):
request.setPriority(QtNetwork.QNetworkRequest.HighPriority)
request.setAttribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute,
QtNetwork.QNetworkRequest.AlwaysNetwork)
elif method == "PUT" or method == "DELETE":
request.setPriority(QtNetwork.QNetworkRequest.HighPriority)
elif cacheloadcontrol is not None:
request.setAttribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute,
cacheloadcontrol)
request.setRawHeader("User-Agent", USER_AGENT_STRING)
if xml:
request.setRawHeader("Accept", "application/xml")
if data is not None:
if method == "POST" and host == config.setting["server_host"] and xml:
request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/xml; charset=utf-8")
else:
request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/x-www-form-urlencoded")
send = self._request_methods[method]
reply = send(request, data) if data is not None else send(request)
self._remember_request_time((host, port))
self._active_requests[reply] = (request, handler, xml, refresh)
def _start_request(self, method, host, port, path, data, handler, xml,
mblogin=False, cacheloadcontrol=None, refresh=None,
queryargs=None):
def start_request_continue(access_token=None):
self._start_request_continue(
method, host, port, path, data, handler, xml,
mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh,
access_token=access_token, queryargs=queryargs)
if mblogin and path != "/oauth2/token":
self.oauth_manager.get_access_token(start_request_continue)
else:
start_request_continue()
@staticmethod
def urls_equivalent(leftUrl, rightUrl):
"""
Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset
that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash
from QT 4.7
"""
return leftUrl.port(80) == rightUrl.port(80) and \
leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort)
def _handle_reply(self, reply, request, handler, xml, refresh):
error = int(reply.error())
if error:
log.error("Network request error for %s: %s (QT code %d, HTTP code %s)",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.errorString(),
error,
repr(reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute))
)
if handler is not None:
handler(str(reply.readAll()), reply, error)
else:
redirect = reply.attribute(QtNetwork.QNetworkRequest.RedirectionTargetAttribute)
fromCache = reply.attribute(QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute)
cached = ' (CACHED)' if fromCache else ''
log.debug("Received reply for %s: HTTP %d (%s) %s",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute),
reply.attribute(QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute),
cached
)
if handler is not None:
# Redirect if found and not infinite
if redirect:
url = request.url()
# merge with base url (to cover the possibility of the URL being relative)
redirect = url.resolved(redirect)
if not XmlWebService.urls_equivalent(redirect, reply.request().url()):
log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo))
redirect_host = str(redirect.host())
redirect_port = redirect.port(80)
redirect_query = dict(redirect.queryItems())
redirect_path = redirect.path()
original_host = str(url.host())
original_port = url.port(80)
if ((original_host, original_port) in REQUEST_DELAY
and (redirect_host, redirect_port) not in REQUEST_DELAY):
log.debug("Setting rate limit for %s:%i to %i" %
(redirect_host, redirect_port,
REQUEST_DELAY[(original_host, original_port)]))
REQUEST_DELAY[(redirect_host, redirect_port)] =\
REQUEST_DELAY[(original_host, original_port)]
self.get(redirect_host,
redirect_port,
redirect_path,
handler, xml, priority=True, important=True, refresh=refresh, queryargs=redirect_query,
cacheloadcontrol=request.attribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute))
else:
log.error("Redirect loop: %s",
reply.request().url().toString(QUrl.RemoveUserInfo)
)
handler(str(reply.readAll()), reply, error)
elif xml:
document = _read_xml(QXmlStreamReader(reply))
handler(document, reply, error)
else:
handler(str(reply.readAll()), reply, error)
def _process_reply(self, reply):
try:
request, handler, xml, refresh = self._active_requests.pop(reply)
except KeyError:
log.error("Request not found for %s" % reply.request().url().toString(QUrl.RemoveUserInfo))
return
try:
self._handle_reply(reply, request, handler, xml, refresh)
finally:
reply.close()
reply.deleteLater()
def get(self, host, port, path, handler, xml=True, priority=False,
important=False, mblogin=False, cacheloadcontrol=None, refresh=False, queryargs=None):
func = partial(self._start_request, "GET", host, port, path, None,
handler, xml, mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh, queryargs=queryargs)
return self.add_task(func, host, port, priority, important=important)
def post(self, host, port, path, data, handler, xml=True, priority=False, important=False, mblogin=True, queryargs=None):
log.debug("POST-DATA %r", data)
func = partial(self._start_request, "POST", host, port, path, data, handler, xml, mblogin, queryargs=queryargs)
return self.add_task(func, host, port, priority, important=important)
def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True, queryargs=None):
func = partial(self._start_request, "PUT", host, port, path, data, handler, False, mblogin, queryargs=queryargs)
return self.add_task(func, host, port, priority, important=important)
def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True, queryargs=None):
func = partial(self._start_request, "DELETE", host, port, path, None, handler, False, mblogin, queryargs=queryargs)
return self.add_task(func, host, port, priority, important=important)
def stop(self):
for reply in self._active_requests.keys():
reply.abort()
self._init_queues()
def _count_pending_requests(self):
count = len(self._active_requests)
for prio_queue in self._queues.values():
for queue in prio_queue.values():
count += len(queue)
self.num_pending_web_requests = count
if count != self._last_num_pending_web_requests:
self._last_num_pending_web_requests = count
self.tagger.tagger_stats_changed.emit()
if count:
self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS)
def _get_delay_to_next_request(self, hostkey):
"""Calculate delay to next request to hostkey (host, port)
returns a tuple (wait, delay) where:
wait is True if a delay is needed
delay is the delay in milliseconds to next request
"""
interval = REQUEST_DELAY[hostkey]
if not interval:
log.debug("WSREQ: Starting another request to %s without delay", hostkey)
return (False, 0)
last_request = self._last_request_times[hostkey]
if not last_request:
log.debug("WSREQ: First request to %s", hostkey)
self._remember_request_time(hostkey) # set it on first run
return (False, interval)
elapsed = (time.time() - last_request) * 1000
if elapsed >= interval:
log.debug("WSREQ: Last request to %s was %d ms ago, starting another one", hostkey, elapsed)
return (False, interval)
delay = int(math.ceil(interval - elapsed))
log.debug("WSREQ: Last request to %s was %d ms ago, waiting %d ms before starting another one",
hostkey, elapsed, delay)
return (True, delay)
def _remember_request_time(self, hostkey):
if REQUEST_DELAY[hostkey]:
self._last_request_times[hostkey] = time.time()
def _run_next_task(self):
delay = sys.maxsize
for prio in sorted(self._queues.keys(), reverse=True):
prio_queue = self._queues[prio]
if not prio_queue:
del(self._queues[prio])
continue
for hostkey in sorted(prio_queue.keys(),
key=lambda hostkey: REQUEST_DELAY[hostkey]):
queue = self._queues[prio][hostkey]
if not queue:
del(self._queues[prio][hostkey])
continue
wait, d = self._get_delay_to_next_request(hostkey)
if not wait:
queue.popleft()()
if d < delay:
delay = d
if delay < sys.maxsize:
self._timer_run_next_task.start(delay)
def add_task(self, func, host, port, priority, important=False):
hostkey = (host, port)
prio = int(priority) # priority is a boolean
if important:
self._queues[prio][hostkey].appendleft(func)
else:
self._queues[prio][hostkey].append(func)
if not self._timer_run_next_task.isActive():
self._timer_run_next_task.start(0)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
return (hostkey, func, prio)
def remove_task(self, task):
hostkey, func, prio = task
try:
self._queues[prio][hostkey].remove(func)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
except:
pass
def _get_by_id(self, entitytype, entityid, handler, inc=[], queryargs=None,
priority=False, important=False, mblogin=False, refresh=False):
host = config.setting["server_host"]
port = config.setting["server_port"]
path = "/ws/2/%s/%s" % (entitytype, entityid)
if queryargs is None:
queryargs = {}
if inc:
queryargs["inc"] = "+".join(inc)
return self.get(host, port, path, handler,
priority=priority, important=important, mblogin=mblogin,
refresh=refresh, queryargs=queryargs)
def get_release_by_id(self, releaseid, handler, inc=[],
priority=False, important=False, mblogin=False, refresh=False):
return self._get_by_id('release', releaseid, handler, inc,
priority=priority, important=important, mblogin=mblogin, refresh=refresh)
def get_track_by_id(self, trackid, handler, inc=[],
priority=False, important=False, mblogin=False, refresh=False):
return self._get_by_id('recording', trackid, handler, inc,
priority=priority, important=important, mblogin=mblogin, refresh=refresh)
def lookup_discid(self, discid, handler, priority=True, important=True, refresh=False):
inc = ['artist-credits', 'labels']
return self._get_by_id('discid', discid, handler, inc, queryargs={"cdstubs": "no"},
priority=priority, important=important, refresh=refresh)
def _find(self, entitytype, handler, kwargs):
host = config.setting["server_host"]
port = config.setting["server_port"]
filters = []
query = []
for name, value in kwargs.items():
if name == 'limit':
filters.append((name, str(value)))
else:
value = _escape_lucene_query(value).strip().lower()
if value:
query.append('%s:(%s)' % (name, value))
if query:
filters.append(('query', ' '.join(query)))
queryargs = {}
for name, value in filters:
value = QUrl.toPercentEncoding(unicode(value))
queryargs[str(name)] = value
path = "/ws/2/%s" % (entitytype)
return self.get(host, port, path, handler, queryargs=queryargs)
def find_releases(self, handler, **kwargs):
return self._find('release', handler, kwargs)
def find_tracks(self, handler, **kwargs):
return self._find('recording', handler, kwargs)
def _browse(self, entitytype, handler, kwargs, inc=[], priority=False, important=False):
host = config.setting["server_host"]
port = config.setting["server_port"]
path = "/ws/2/%s" % (entitytype)
queryargs = kwargs
if inc:
queryargs["inc"] = "+".join(inc)
return self.get(host, port, path, handler, priority=priority,
important=important, queryargs=queryargs)
def browse_releases(self, handler, priority=True, important=True, **kwargs):
inc = ["media", "labels"]
return self._browse("release", handler, kwargs, inc, priority=priority, important=important)
def submit_ratings(self, ratings, handler):
host = config.setting['server_host']
port = config.setting['server_port']
path = '/ws/2/rating/?client=' + CLIENT_STRING
recordings = (''.join(['<recording id="%s"><user-rating>%s</user-rating></recording>' %
(i[1], j*20) for i, j in ratings.items() if i[0] == 'recording']))
data = _wrap_xml_metadata('<recording-list>%s</recording-list>' % recordings)
return self.post(host, port, path, data, handler, priority=True)
def _encode_acoustid_args(self, args, format='xml'):
filters = []
args['client'] = ACOUSTID_KEY
args['clientversion'] = PICARD_VERSION_STR
args['format'] = format
for name, value in args.items():
value = str(QUrl.toPercentEncoding(value))
filters.append('%s=%s' % (str(name), value))
return '&'.join(filters)
def query_acoustid(self, handler, **args):
host, port = ACOUSTID_HOST, ACOUSTID_PORT
body = self._encode_acoustid_args(args)
return self.post(host, port, '/v2/lookup', body, handler, priority=False, important=False, mblogin=False)
def submit_acoustid_fingerprints(self, submissions, handler):
args = {'user': config.setting["acoustid_apikey"]}
for i, submission in enumerate(submissions):
args['fingerprint.%d' % i] = str(submission.fingerprint)
args['duration.%d' % i] = str(submission.duration)
args['mbid.%d' % i] = str(submission.recordingid)
if submission.puid:
args['puid.%d' % i] = str(submission.puid)
host, port = ACOUSTID_HOST, ACOUSTID_PORT
body = self._encode_acoustid_args(args, format='json')
return self.post(host, port, '/v2/submit', body, handler, priority=True, important=False, mblogin=False)
def download(self, host, port, path, handler, priority=False,
important=False, cacheloadcontrol=None, refresh=False,
queryargs=None):
return self.get(host, port, path, handler, xml=False,
priority=priority, important=important,
cacheloadcontrol=cacheloadcontrol, refresh=refresh,
queryargs=queryargs)
def get_collection(self, id, handler, limit=100, offset=0):
host, port = config.setting['server_host'], config.setting['server_port']
path = "/ws/2/collection"
queryargs = None
if id is not None:
inc = ["releases", "artist-credits", "media"]
path += "/%s/releases" % (id)
queryargs = {}
queryargs["inc"] = "+".join(inc)
queryargs["limit"] = limit
queryargs["offset"] = offset
return self.get(host, port, path, handler, priority=True, important=True,
mblogin=True, queryargs=queryargs)
def get_collection_list(self, handler):
return self.get_collection(None, handler)
def _collection_request(self, id, releases):
while releases:
ids = ";".join(releases if len(releases) <= 400 else releases[:400])
releases = releases[400:]
yield "/ws/2/collection/%s/releases/%s" % (id, ids)
def _get_client_queryarg(self):
return {"client": CLIENT_STRING}
def put_to_collection(self, id, releases, handler):
host, port = config.setting['server_host'], config.setting['server_port']
for path in self._collection_request(id, releases):
self.put(host, port, path, "", handler,
queryargs=self._get_client_queryarg())
def delete_from_collection(self, id, releases, handler):
host, port = config.setting['server_host'], config.setting['server_port']
for path in self._collection_request(id, releases):
self.delete(host, port, path, handler,
queryargs=self._get_client_queryarg)
class XmlWebService(QtCore.QObject):
def __init__(self, parent=None):
QtCore.QObject.__init__(self, parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._last_request_times = defaultdict(lambda: 0)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self._init_queues()
self._init_timers()
def _init_queues(self):
self._active_requests = {}
self._queues = defaultdict(lambda: defaultdict(deque))
self.num_pending_web_requests = 0
self._last_num_pending_web_requests = -1
def _init_timers(self):
self._timer_run_next_task = QtCore.QTimer(self)
self._timer_run_next_task.setSingleShot(True)
self._timer_run_next_task.timeout.connect(self._run_next_task)
self._timer_count_pending_requests = QtCore.QTimer(self)
self._timer_count_pending_requests.setSingleShot(True)
self._timer_count_pending_requests.timeout.connect(
self._count_pending_requests)
def set_cache(self, cache_size_in_mb=100):
cache = QtNetwork.QNetworkDiskCache()
location = QDesktopServices.storageLocation(
QDesktopServices.CacheLocation)
cache.setCacheDirectory(os.path.join(unicode(location), u'picard'))
cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024)
self.manager.setCache(cache)
log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory())
log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(),
cache.maximumCacheSize())
def setup_proxy(self):
proxy = QtNetwork.QNetworkProxy()
if config.setting["use_proxy"]:
proxy.setType(QtNetwork.QNetworkProxy.HttpProxy)
proxy.setHostName(config.setting["proxy_server_host"])
proxy.setPort(config.setting["proxy_server_port"])
proxy.setUser(config.setting["proxy_username"])
proxy.setPassword(config.setting["proxy_password"])
self.manager.setProxy(proxy)
def _start_request_continue(self,
method,
host,
port,
path,
data,
handler,
xml,
mblogin=False,
cacheloadcontrol=None,
refresh=None,
access_token=None,
queryargs=None):
url = build_qurl(host,
port,
path=path,
mblogin=mblogin,
queryargs=queryargs)
request = QtNetwork.QNetworkRequest(url)
if mblogin and access_token:
request.setRawHeader("Authorization", "Bearer %s" % access_token)
if mblogin or (method == "GET" and refresh):
request.setPriority(QtNetwork.QNetworkRequest.HighPriority)
request.setAttribute(
QtNetwork.QNetworkRequest.CacheLoadControlAttribute,
QtNetwork.QNetworkRequest.AlwaysNetwork)
elif method == "PUT" or method == "DELETE":
request.setPriority(QtNetwork.QNetworkRequest.HighPriority)
elif cacheloadcontrol is not None:
request.setAttribute(
QtNetwork.QNetworkRequest.CacheLoadControlAttribute,
cacheloadcontrol)
request.setRawHeader("User-Agent", USER_AGENT_STRING)
if xml:
request.setRawHeader("Accept", "application/xml")
if data is not None:
if method == "POST" and host == config.setting[
"server_host"] and xml:
request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader,
"application/xml; charset=utf-8")
else:
request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader,
"application/x-www-form-urlencoded")
send = self._request_methods[method]
reply = send(request, data) if data is not None else send(request)
self._remember_request_time((host, port))
self._active_requests[reply] = (request, handler, xml, refresh)
def _start_request(self,
method,
host,
port,
path,
data,
handler,
xml,
mblogin=False,
cacheloadcontrol=None,
refresh=None,
queryargs=None):
def start_request_continue(access_token=None):
self._start_request_continue(method,
host,
port,
path,
data,
handler,
xml,
mblogin=mblogin,
cacheloadcontrol=cacheloadcontrol,
refresh=refresh,
access_token=access_token,
queryargs=queryargs)
if mblogin and path != "/oauth2/token":
self.oauth_manager.get_access_token(start_request_continue)
else:
start_request_continue()
@staticmethod
def urls_equivalent(leftUrl, rightUrl):
"""
Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset
that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash
from QT 4.7
"""
return leftUrl.port(80) == rightUrl.port(80) and \
leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort)
@staticmethod
def url_port(url):
if url.scheme() == 'https':
return url.port(443)
return url.port(80)
def _handle_reply(self, reply, request, handler, xml, refresh):
error = int(reply.error())
if error:
log.error(
"Network request error for %s: %s (QT code %d, HTTP code %s)",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.errorString(), error,
repr(
reply.attribute(
QtNetwork.QNetworkRequest.HttpStatusCodeAttribute)))
if handler is not None:
handler(str(reply.readAll()), reply, error)
else:
redirect = reply.attribute(
QtNetwork.QNetworkRequest.RedirectionTargetAttribute)
fromCache = reply.attribute(
QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute)
cached = ' (CACHED)' if fromCache else ''
log.debug(
"Received reply for %s: HTTP %d (%s) %s",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.attribute(
QtNetwork.QNetworkRequest.HttpStatusCodeAttribute),
reply.attribute(
QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute),
cached)
if handler is not None:
# Redirect if found and not infinite
if redirect:
url = request.url()
# merge with base url (to cover the possibility of the URL being relative)
redirect = url.resolved(redirect)
if not XmlWebService.urls_equivalent(
redirect,
reply.request().url()):
log.debug("Redirect to %s requested",
redirect.toString(QUrl.RemoveUserInfo))
redirect_host = str(redirect.host())
redirect_port = self.url_port(redirect)
redirect_query = dict(redirect.encodedQueryItems())
redirect_path = redirect.path()
original_host = str(url.host())
original_port = self.url_port(url)
if ((original_host, original_port) in REQUEST_DELAY
and (redirect_host, redirect_port)
not in REQUEST_DELAY):
log.debug(
"Setting rate limit for %s:%i to %i" %
(redirect_host, redirect_port, REQUEST_DELAY[
(original_host, original_port)]))
REQUEST_DELAY[(redirect_host, redirect_port)] =\
REQUEST_DELAY[(original_host, original_port)]
self.get(redirect_host,
redirect_port,
redirect_path,
handler,
xml,
priority=True,
important=True,
refresh=refresh,
queryargs=redirect_query,
cacheloadcontrol=request.attribute(
QtNetwork.QNetworkRequest.
CacheLoadControlAttribute))
else:
log.error(
"Redirect loop: %s",
reply.request().url().toString(
QUrl.RemoveUserInfo))
handler(str(reply.readAll()), reply, error)
elif xml:
document = _read_xml(QXmlStreamReader(reply))
handler(document, reply, error)
else:
handler(str(reply.readAll()), reply, error)
def _process_reply(self, reply):
try:
request, handler, xml, refresh = self._active_requests.pop(reply)
except KeyError:
log.error("Request not found for %s" %
reply.request().url().toString(QUrl.RemoveUserInfo))
return
try:
self._handle_reply(reply, request, handler, xml, refresh)
finally:
reply.close()
reply.deleteLater()
def get(self,
host,
port,
path,
handler,
xml=True,
priority=False,
important=False,
mblogin=False,
cacheloadcontrol=None,
refresh=False,
queryargs=None):
func = partial(self._start_request,
"GET",
host,
port,
path,
None,
handler,
xml,
mblogin,
cacheloadcontrol=cacheloadcontrol,
refresh=refresh,
queryargs=queryargs)
return self.add_task(func, host, port, priority, important=important)
def post(self,
host,
port,
path,
data,
handler,
xml=True,
priority=False,
important=False,
mblogin=True,
queryargs=None):
log.debug("POST-DATA %r", data)
func = partial(self._start_request,
"POST",
host,
port,
path,
data,
handler,
xml,
mblogin,
queryargs=queryargs)
return self.add_task(func, host, port, priority, important=important)
def put(self,
host,
port,
path,
data,
handler,
priority=True,
important=False,
mblogin=True,
queryargs=None):
func = partial(self._start_request,
"PUT",
host,
port,
path,
data,
handler,
False,
mblogin,
queryargs=queryargs)
return self.add_task(func, host, port, priority, important=important)
def delete(self,
host,
port,
path,
handler,
priority=True,
important=False,
mblogin=True,
queryargs=None):
func = partial(self._start_request,
"DELETE",
host,
port,
path,
None,
handler,
False,
mblogin,
queryargs=queryargs)
return self.add_task(func, host, port, priority, important=important)
def stop(self):
for reply in self._active_requests.keys():
reply.abort()
self._init_queues()
def _count_pending_requests(self):
count = len(self._active_requests)
for prio_queue in self._queues.values():
for queue in prio_queue.values():
count += len(queue)
self.num_pending_web_requests = count
if count != self._last_num_pending_web_requests:
self._last_num_pending_web_requests = count
self.tagger.tagger_stats_changed.emit()
if count:
self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS)
def _get_delay_to_next_request(self, hostkey):
"""Calculate delay to next request to hostkey (host, port)
returns a tuple (wait, delay) where:
wait is True if a delay is needed
delay is the delay in milliseconds to next request
"""
interval = REQUEST_DELAY[hostkey]
if not interval:
log.debug("WSREQ: Starting another request to %s without delay",
hostkey)
return (False, 0)
last_request = self._last_request_times[hostkey]
if not last_request:
log.debug("WSREQ: First request to %s", hostkey)
self._remember_request_time(hostkey) # set it on first run
return (False, interval)
elapsed = (time.time() - last_request) * 1000
if elapsed >= interval:
log.debug(
"WSREQ: Last request to %s was %d ms ago, starting another one",
hostkey, elapsed)
return (False, interval)
delay = int(math.ceil(interval - elapsed))
log.debug(
"WSREQ: Last request to %s was %d ms ago, waiting %d ms before starting another one",
hostkey, elapsed, delay)
return (True, delay)
def _remember_request_time(self, hostkey):
if REQUEST_DELAY[hostkey]:
self._last_request_times[hostkey] = time.time()
def _run_next_task(self):
delay = sys.maxsize
for prio in sorted(self._queues.keys(), reverse=True):
prio_queue = self._queues[prio]
if not prio_queue:
del (self._queues[prio])
continue
for hostkey in sorted(prio_queue.keys(),
key=lambda hostkey: REQUEST_DELAY[hostkey]):
queue = self._queues[prio][hostkey]
if not queue:
del (self._queues[prio][hostkey])
continue
wait, d = self._get_delay_to_next_request(hostkey)
if not wait:
queue.popleft()()
if d < delay:
delay = d
if delay < sys.maxsize:
self._timer_run_next_task.start(delay)
def add_task(self, func, host, port, priority, important=False):
hostkey = (host, port)
prio = int(priority) # priority is a boolean
if important:
self._queues[prio][hostkey].appendleft(func)
else:
self._queues[prio][hostkey].append(func)
if not self._timer_run_next_task.isActive():
self._timer_run_next_task.start(0)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
return (hostkey, func, prio)
def remove_task(self, task):
hostkey, func, prio = task
try:
self._queues[prio][hostkey].remove(func)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
except:
pass
def _get_by_id(self,
entitytype,
entityid,
handler,
inc=[],
queryargs=None,
priority=False,
important=False,
mblogin=False,
refresh=False):
host = config.setting["server_host"]
port = config.setting["server_port"]
path = "/ws/2/%s/%s" % (entitytype, entityid)
if queryargs is None:
queryargs = {}
if inc:
queryargs["inc"] = "+".join(inc)
return self.get(host,
port,
path,
handler,
priority=priority,
important=important,
mblogin=mblogin,
refresh=refresh,
queryargs=queryargs)
def get_release_by_id(self,
releaseid,
handler,
inc=[],
priority=False,
important=False,
mblogin=False,
refresh=False):
return self._get_by_id('release',
releaseid,
handler,
inc,
priority=priority,
important=important,
mblogin=mblogin,
refresh=refresh)
def get_track_by_id(self,
trackid,
handler,
inc=[],
priority=False,
important=False,
mblogin=False,
refresh=False):
return self._get_by_id('recording',
trackid,
handler,
inc,
priority=priority,
important=important,
mblogin=mblogin,
refresh=refresh)
def lookup_discid(self,
discid,
handler,
priority=True,
important=True,
refresh=False):
inc = ['artist-credits', 'labels']
return self._get_by_id('discid',
discid,
handler,
inc,
queryargs={"cdstubs": "no"},
priority=priority,
important=important,
refresh=refresh)
def _find(self, entitytype, handler, kwargs):
host = config.setting["server_host"]
port = config.setting["server_port"]
filters = []
limit = kwargs.pop("limit")
if limit:
filters.append(("limit", limit))
is_search = kwargs.pop("search", False)
if is_search:
if config.setting["use_adv_search_syntax"]:
query = kwargs["query"]
else:
query = escape_lucene_query(kwargs["query"]).strip().lower()
filters.append(("dismax", 'true'))
else:
query = []
for name, value in kwargs.items():
value = escape_lucene_query(value).strip().lower()
if value:
query.append('%s:(%s)' % (name, value))
query = ' '.join(query)
if query:
filters.append(("query", query))
queryargs = {}
for name, value in filters:
value = QUrl.toPercentEncoding(unicode(value))
queryargs[str(name)] = value
path = "/ws/2/%s" % (entitytype)
return self.get(host, port, path, handler, queryargs=queryargs)
def find_releases(self, handler, **kwargs):
return self._find('release', handler, kwargs)
def find_tracks(self, handler, **kwargs):
return self._find('recording', handler, kwargs)
def find_artists(self, handler, **kwargs):
return self._find('artist', handler, kwargs)
def _browse(self,
entitytype,
handler,
kwargs,
inc=[],
priority=False,
important=False):
host = config.setting["server_host"]
port = config.setting["server_port"]
path = "/ws/2/%s" % (entitytype)
queryargs = kwargs
if inc:
queryargs["inc"] = "+".join(inc)
return self.get(host,
port,
path,
handler,
priority=priority,
important=important,
queryargs=queryargs)
def browse_releases(self,
handler,
priority=True,
important=True,
**kwargs):
inc = ["media", "labels"]
return self._browse("release",
handler,
kwargs,
inc,
priority=priority,
important=important)
def submit_ratings(self, ratings, handler):
host = config.setting['server_host']
port = config.setting['server_port']
path = '/ws/2/rating/?client=' + CLIENT_STRING
recordings = (''.join([
'<recording id="%s"><user-rating>%s</user-rating></recording>' %
(i[1], j * 20) for i, j in ratings.items() if i[0] == 'recording'
]))
data = _wrap_xml_metadata('<recording-list>%s</recording-list>' %
recordings)
return self.post(host, port, path, data, handler, priority=True)
def _encode_acoustid_args(self, args, format='xml'):
filters = []
args['client'] = ACOUSTID_KEY
args['clientversion'] = PICARD_VERSION_STR
args['format'] = format
for name, value in args.items():
value = str(QUrl.toPercentEncoding(value))
filters.append('%s=%s' % (str(name), value))
return '&'.join(filters)
def query_acoustid(self, handler, **args):
host, port = ACOUSTID_HOST, ACOUSTID_PORT
body = self._encode_acoustid_args(args)
return self.post(host,
port,
'/v2/lookup',
body,
handler,
priority=False,
important=False,
mblogin=False)
def submit_acoustid_fingerprints(self, submissions, handler):
args = {'user': config.setting["acoustid_apikey"]}
for i, submission in enumerate(submissions):
args['fingerprint.%d' % i] = str(submission.fingerprint)
args['duration.%d' % i] = str(submission.duration)
args['mbid.%d' % i] = str(submission.recordingid)
if submission.puid:
args['puid.%d' % i] = str(submission.puid)
host, port = ACOUSTID_HOST, ACOUSTID_PORT
body = self._encode_acoustid_args(args, format='json')
return self.post(host,
port,
'/v2/submit',
body,
handler,
priority=True,
important=False,
mblogin=False)
def download(self,
host,
port,
path,
handler,
priority=False,
important=False,
cacheloadcontrol=None,
refresh=False,
queryargs=None):
return self.get(host,
port,
path,
handler,
xml=False,
priority=priority,
important=important,
cacheloadcontrol=cacheloadcontrol,
refresh=refresh,
queryargs=queryargs)
def get_collection(self, id, handler, limit=100, offset=0):
host, port = config.setting['server_host'], config.setting[
'server_port']
path = "/ws/2/collection"
queryargs = None
if id is not None:
inc = ["releases", "artist-credits", "media"]
path += "/%s/releases" % (id)
queryargs = {}
queryargs["inc"] = "+".join(inc)
queryargs["limit"] = limit
queryargs["offset"] = offset
return self.get(host,
port,
path,
handler,
priority=True,
important=True,
mblogin=True,
queryargs=queryargs)
def get_collection_list(self, handler):
return self.get_collection(None, handler)
def _collection_request(self, id, releases):
while releases:
ids = ";".join(
releases if len(releases) <= 400 else releases[:400])
releases = releases[400:]
yield "/ws/2/collection/%s/releases/%s" % (id, ids)
def _get_client_queryarg(self):
return {"client": CLIENT_STRING}
def put_to_collection(self, id, releases, handler):
host, port = config.setting['server_host'], config.setting[
'server_port']
for path in self._collection_request(id, releases):
self.put(host,
port,
path,
"",
handler,
queryargs=self._get_client_queryarg())
def delete_from_collection(self, id, releases, handler):
host, port = config.setting['server_host'], config.setting[
'server_port']
for path in self._collection_request(id, releases):
self.delete(host,
port,
path,
handler,
queryargs=self._get_client_queryarg())
class WebService(QtCore.QObject):
PARSERS = dict()
def __init__(self, parent=None):
super().__init__(parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self._init_queues()
self._init_timers()
def _init_queues(self):
self._active_requests = {}
self._queues = defaultdict(lambda: defaultdict(deque))
self.num_pending_web_requests = 0
self._last_num_pending_web_requests = -1
def _init_timers(self):
self._timer_run_next_task = QtCore.QTimer(self)
self._timer_run_next_task.setSingleShot(True)
self._timer_run_next_task.timeout.connect(self._run_next_task)
self._timer_count_pending_requests = QtCore.QTimer(self)
self._timer_count_pending_requests.setSingleShot(True)
self._timer_count_pending_requests.timeout.connect(self._count_pending_requests)
def set_cache(self, cache_size_in_mb=100):
cache = QtNetwork.QNetworkDiskCache()
location = QStandardPaths.writableLocation(QStandardPaths.CacheLocation)
cache.setCacheDirectory(os.path.join(location, 'picard'))
cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024)
self.manager.setCache(cache)
log.debug("NetworkDiskCache dir: %r size: %s / %s",
cache.cacheDirectory(), cache.cacheSize(),
cache.maximumCacheSize())
def setup_proxy(self):
proxy = QtNetwork.QNetworkProxy()
if config.setting["use_proxy"]:
proxy.setType(QtNetwork.QNetworkProxy.HttpProxy)
proxy.setHostName(config.setting["proxy_server_host"])
proxy.setPort(config.setting["proxy_server_port"])
proxy.setUser(config.setting["proxy_username"])
proxy.setPassword(config.setting["proxy_password"])
self.manager.setProxy(proxy)
def _send_request(self, request, access_token=None):
hostkey = request.get_host_key()
ratecontrol.increment_requests(hostkey)
request.access_token = access_token
send = self._request_methods[request.method]
data = request.data
reply = send(request, data.encode('utf-8')) if data is not None else send(request)
self._active_requests[reply] = request
def _start_request(self, request):
if request.mblogin and request.path != "/oauth2/token":
self.oauth_manager.get_access_token(partial(self._send_request, request))
else:
self._send_request(request)
@staticmethod
def urls_equivalent(leftUrl, rightUrl):
"""
Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset
that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash
from QT 4.7
"""
return leftUrl.port(80) == rightUrl.port(80) and \
leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort)
@staticmethod
def url_port(url):
if url.scheme() == 'https':
return url.port(443)
return url.port(80)
def _handle_redirect(self, reply, request, redirect):
url = request.url()
error = int(reply.error())
# merge with base url (to cover the possibility of the URL being relative)
redirect = url.resolved(redirect)
if not WebService.urls_equivalent(redirect, reply.request().url()):
log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo))
redirect_host = redirect.host()
redirect_port = self.url_port(redirect)
redirect_query = dict(QUrlQuery(redirect).queryItems(QUrl.FullyEncoded))
redirect_path = redirect.path()
original_host = url.host()
original_port = self.url_port(url)
original_host_key = (original_host, original_port)
redirect_host_key = (redirect_host, redirect_port)
ratecontrol.copy_minimal_delay(original_host_key, redirect_host_key)
self.get(redirect_host,
redirect_port,
redirect_path,
request.handler, request.parse_response_type, priority=True, important=True,
refresh=request.refresh, queryargs=redirect_query,
cacheloadcontrol=request.attribute(QNetworkRequest.CacheLoadControlAttribute))
else:
log.error("Redirect loop: %s",
reply.request().url().toString(QUrl.RemoveUserInfo)
)
request.handler(reply.readAll(), reply, error)
def _handle_reply(self, reply, request):
hostkey = request.get_host_key()
ratecontrol.decrement_requests(hostkey)
self._timer_run_next_task.start(0)
slow_down = False
error = int(reply.error())
handler = request.handler
if error:
code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
code = int(code) if code else 0
errstr = reply.errorString()
url = reply.request().url().toString(QUrl.RemoveUserInfo)
log.error("Network request error for %s: %s (QT code %d, HTTP code %d)",
url, errstr, error, code)
if (not request.max_retries_reached()
and (code == 503
or code == 429
# Sometimes QT returns a http status code of 200 even when there
# is a service unavailable error. But it returns a QT error code
# of 403 when this happens
or error == 403
)
):
slow_down = True
retries = request.mark_for_retry()
log.debug("Retrying %s (#%d)", url, retries)
self.add_request(request)
elif handler is not None:
handler(reply.readAll(), reply, error)
slow_down = (slow_down or code >= 500)
else:
redirect = reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
fromCache = reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute)
cached = ' (CACHED)' if fromCache else ''
log.debug("Received reply for %s: HTTP %d (%s) %s",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.attribute(QNetworkRequest.HttpStatusCodeAttribute),
reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute),
cached
)
if handler is not None:
# Redirect if found and not infinite
if redirect:
self._handle_redirect(reply, request, redirect)
elif request.response_parser:
try:
document = request.response_parser(reply)
except Exception as e:
url = reply.request().url().toString(QUrl.RemoveUserInfo)
log.error("Unable to parse the response for %s: %s", url, e)
document = reply.readAll()
error = e
finally:
handler(document, reply, error)
else:
handler(reply.readAll(), reply, error)
ratecontrol.adjust(hostkey, slow_down)
def _process_reply(self, reply):
try:
request = self._active_requests.pop(reply)
except KeyError:
log.error("Request not found for %s", reply.request().url().toString(QUrl.RemoveUserInfo))
return
try:
self._handle_reply(reply, request)
finally:
reply.close()
reply.deleteLater()
def get(self, host, port, path, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE,
priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False,
queryargs=None):
request = WSGetRequest(host, port, path, handler, parse_response_type=parse_response_type,
mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh,
queryargs=queryargs, priority=priority, important=important)
return self.add_request(request)
def post(self, host, port, path, data, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE,
priority=False, important=False, mblogin=True, queryargs=None, request_mimetype=None):
request = WSPostRequest(host, port, path, handler, parse_response_type=parse_response_type,
data=data, mblogin=mblogin, queryargs=queryargs,
priority=priority, important=important,
request_mimetype=request_mimetype)
log.debug("POST-DATA %r", data)
return self.add_request(request)
def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True,
queryargs=None, request_mimetype=None):
request = WSPutRequest(host, port, path, handler, data=data, mblogin=mblogin,
queryargs=queryargs, priority=priority,
important=important, request_mimetype=request_mimetype)
return self.add_request(request)
def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True,
queryargs=None):
request = WSDeleteRequest(host, port, path, handler, mblogin=mblogin,
queryargs=queryargs, priority=priority, important=important)
return self.add_request(request)
def download(self, host, port, path, handler, priority=False,
important=False, cacheloadcontrol=None, refresh=False,
queryargs=None):
return self.get(host, port, path, handler, parse_response_type=None,
priority=priority, important=important,
cacheloadcontrol=cacheloadcontrol, refresh=refresh,
queryargs=queryargs)
def stop(self):
for reply in list(self._active_requests):
reply.abort()
self._init_queues()
def _count_pending_requests(self):
count = len(self._active_requests)
for prio_queue in self._queues.values():
for queue in prio_queue.values():
count += len(queue)
self.num_pending_web_requests = count
if count != self._last_num_pending_web_requests:
self._last_num_pending_web_requests = count
self.tagger.tagger_stats_changed.emit()
if count:
self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS)
def _run_next_task(self):
delay = sys.maxsize
for prio in sorted(self._queues.keys(), reverse=True):
prio_queue = self._queues[prio]
if not prio_queue:
del(self._queues[prio])
continue
for hostkey in sorted(prio_queue.keys(),
key=ratecontrol.current_delay):
queue = self._queues[prio][hostkey]
if not queue:
del(self._queues[prio][hostkey])
continue
wait, d = ratecontrol.get_delay_to_next_request(hostkey)
if not wait:
queue.popleft()()
if d < delay:
delay = d
if delay < sys.maxsize:
self._timer_run_next_task.start(delay)
def add_task(self, func, request):
hostkey = request.get_host_key()
prio = int(request.priority) # priority is a boolean
if request.important:
self._queues[prio][hostkey].appendleft(func)
else:
self._queues[prio][hostkey].append(func)
if not self._timer_run_next_task.isActive():
self._timer_run_next_task.start(0)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
return (hostkey, func, prio)
def add_request(self, request):
return self.add_task(partial(self._start_request, request), request)
def remove_task(self, task):
hostkey, func, prio = task
try:
self._queues[prio][hostkey].remove(func)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
except Exception as e:
log.debug(e)
@classmethod
def add_parser(cls, response_type, mimetype, parser):
cls.PARSERS[response_type] = Parser(mimetype=mimetype, parser=parser)
@classmethod
def get_response_mimetype(cls, response_type):
if response_type in cls.PARSERS:
return cls.PARSERS[response_type].mimetype
else:
raise UnknownResponseParserError(response_type)
@classmethod
def get_response_parser(cls, response_type):
if response_type in cls.PARSERS:
return cls.PARSERS[response_type].parser
else:
raise UnknownResponseParserError(response_type)
class XmlWebService(QtCore.QObject):
def __init__(self, parent=None):
QtCore.QObject.__init__(self, parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._last_request_times = {}
self._active_requests = {}
self._high_priority_queues = {}
self._low_priority_queues = {}
self._hosts = []
self._timer = QtCore.QTimer(self)
self._timer.setSingleShot(True)
self._timer.timeout.connect(self._run_next_task)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self.num_pending_web_requests = 0
def set_cache(self, cache_size_in_mb=100):
cache = QtNetwork.QNetworkDiskCache()
location = QDesktopServices.storageLocation(
QDesktopServices.CacheLocation)
cache.setCacheDirectory(os.path.join(unicode(location), u'picard'))
cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024)
self.manager.setCache(cache)
log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory())
log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(),
cache.maximumCacheSize())
def setup_proxy(self):
proxy = QtNetwork.QNetworkProxy()
if config.setting["use_proxy"]:
proxy.setType(QtNetwork.QNetworkProxy.HttpProxy)
proxy.setHostName(config.setting["proxy_server_host"])
proxy.setPort(config.setting["proxy_server_port"])
proxy.setUser(config.setting["proxy_username"])
proxy.setPassword(config.setting["proxy_password"])
self.manager.setProxy(proxy)
def _start_request_continue(self,
method,
host,
port,
path,
data,
handler,
xml,
mblogin=False,
cacheloadcontrol=None,
refresh=None,
access_token=None):
if (mblogin and host in MUSICBRAINZ_SERVERS
and port == 80) or port == 443:
urlstring = "https://%s%s" % (host, path)
elif port is None or port == 80:
urlstring = "http://%s%s" % (host, path)
else:
urlstring = "http://%s:%d%s" % (host, port, path)
log.debug("%s %s", method, urlstring)
url = QUrl.fromEncoded(urlstring)
request = QtNetwork.QNetworkRequest(url)
if mblogin and access_token:
request.setRawHeader("Authorization", "Bearer %s" % access_token)
if mblogin or (method == "GET" and refresh):
request.setPriority(QtNetwork.QNetworkRequest.HighPriority)
request.setAttribute(
QtNetwork.QNetworkRequest.CacheLoadControlAttribute,
QtNetwork.QNetworkRequest.AlwaysNetwork)
elif method == "PUT" or method == "DELETE":
request.setPriority(QtNetwork.QNetworkRequest.HighPriority)
elif cacheloadcontrol is not None:
request.setAttribute(
QtNetwork.QNetworkRequest.CacheLoadControlAttribute,
cacheloadcontrol)
request.setRawHeader("User-Agent", USER_AGENT_STRING)
if xml:
request.setRawHeader("Accept", "application/xml")
if data is not None:
if method == "POST" and host == config.setting[
"server_host"] and xml:
request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader,
"application/xml; charset=utf-8")
else:
request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader,
"application/x-www-form-urlencoded")
send = self._request_methods[method]
reply = send(request, data) if data is not None else send(request)
key = (host, port)
self._last_request_times[key] = time.time()
self._active_requests[reply] = (request, handler, xml, refresh)
def _start_request(self,
method,
host,
port,
path,
data,
handler,
xml,
mblogin=False,
cacheloadcontrol=None,
refresh=None):
def start_request_continue(access_token=None):
self._start_request_continue(method,
host,
port,
path,
data,
handler,
xml,
mblogin=mblogin,
cacheloadcontrol=cacheloadcontrol,
refresh=refresh,
access_token=access_token)
if mblogin and path != "/oauth2/token":
self.oauth_manager.get_access_token(start_request_continue)
else:
start_request_continue()
@staticmethod
def urls_equivalent(leftUrl, rightUrl):
"""
Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset
that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash
from QT 4.7
"""
return leftUrl.port(80) == rightUrl.port(80) and \
leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort)
def _process_reply(self, reply):
try:
request, handler, xml, refresh = self._active_requests.pop(reply)
except KeyError:
log.error("Request not found for %s" %
reply.request().url().toString(QUrl.RemoveUserInfo))
return
error = int(reply.error())
if error:
log.error(
"Network request error for %s: %s (QT code %d, HTTP code %s)",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.errorString(), error,
repr(
reply.attribute(
QtNetwork.QNetworkRequest.HttpStatusCodeAttribute)))
if handler is not None:
handler(str(reply.readAll()), reply, error)
else:
redirect = reply.attribute(
QtNetwork.QNetworkRequest.RedirectionTargetAttribute)
fromCache = reply.attribute(
QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute)
cached = ' (CACHED)' if fromCache else ''
log.debug(
"Received reply for %s: HTTP %d (%s) %s",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.attribute(
QtNetwork.QNetworkRequest.HttpStatusCodeAttribute),
reply.attribute(
QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute),
cached)
if handler is not None:
# Redirect if found and not infinite
if redirect and not XmlWebService.urls_equivalent(
redirect,
reply.request().url()):
log.debug("Redirect to %s requested",
redirect.toString(QUrl.RemoveUserInfo))
redirect_host = str(redirect.host())
redirect_port = redirect.port(80)
url = request.url()
original_host = str(url.host())
original_port = url.port(80)
if ((original_host, original_port) in REQUEST_DELAY and
(redirect_host, redirect_port) not in REQUEST_DELAY):
log.debug(
"Setting rate limit for %s:%i to %i" %
(redirect_host, redirect_port,
REQUEST_DELAY[(original_host, original_port)]))
REQUEST_DELAY[(redirect_host, redirect_port)] =\
REQUEST_DELAY[(original_host, original_port)]
self.get(
redirect_host,
redirect_port,
# retain path, query string and anchors from redirect URL
redirect.toString(QUrl.RemoveAuthority
| QUrl.RemoveScheme),
handler,
xml,
priority=True,
important=True,
refresh=refresh,
cacheloadcontrol=request.attribute(
QtNetwork.QNetworkRequest.CacheLoadControlAttribute
))
elif redirect:
log.error(
"Redirect loop: %s",
reply.request().url().toString(QUrl.RemoveUserInfo))
handler(str(reply.readAll()), reply, error)
elif xml:
document = _read_xml(QXmlStreamReader(reply))
handler(document, reply, error)
else:
handler(str(reply.readAll()), reply, error)
reply.close()
self.num_pending_web_requests -= 1
self.tagger.tagger_stats_changed.emit()
def get(self,
host,
port,
path,
handler,
xml=True,
priority=False,
important=False,
mblogin=False,
cacheloadcontrol=None,
refresh=False):
func = partial(self._start_request,
"GET",
host,
port,
path,
None,
handler,
xml,
mblogin,
cacheloadcontrol=cacheloadcontrol,
refresh=refresh)
return self.add_task(func, host, port, priority, important=important)
def post(self,
host,
port,
path,
data,
handler,
xml=True,
priority=False,
important=False,
mblogin=True):
log.debug("POST-DATA %r", data)
func = partial(self._start_request, "POST", host, port, path, data,
handler, xml, mblogin)
return self.add_task(func, host, port, priority, important=important)
def put(self,
host,
port,
path,
data,
handler,
priority=True,
important=False,
mblogin=True):
func = partial(self._start_request, "PUT", host, port, path, data,
handler, False, mblogin)
return self.add_task(func, host, port, priority, important=important)
def delete(self,
host,
port,
path,
handler,
priority=True,
important=False,
mblogin=True):
func = partial(self._start_request, "DELETE", host, port, path, None,
handler, False, mblogin)
return self.add_task(func, host, port, priority, important=important)
def stop(self):
self._high_priority_queues = {}
self._low_priority_queues = {}
for reply in self._active_requests.keys():
reply.abort()
def _run_next_task(self):
delay = sys.maxint
for key in self._hosts:
queue = self._high_priority_queues.get(
key) or self._low_priority_queues.get(key)
if not queue:
continue
now = time.time()
last = self._last_request_times.get(key)
request_delay = REQUEST_DELAY[key]
last_ms = (now -
last) * 1000 if last is not None else request_delay
if last_ms >= request_delay:
log.debug(
"Last request to %s was %d ms ago, starting another one",
key, last_ms)
d = request_delay
queue.popleft()()
else:
d = int(math.ceil(request_delay - last_ms))
log.debug(
"Waiting %d ms before starting another request to %s", d,
key)
if d < delay:
delay = d
if delay < sys.maxint:
self._timer.start(delay)
def add_task(self, func, host, port, priority, important=False):
key = (host, port)
if key not in self._hosts:
self._hosts.append(key)
if priority:
queues = self._high_priority_queues
else:
queues = self._low_priority_queues
queues.setdefault(key, deque())
if important:
queues[key].appendleft(func)
else:
queues[key].append(func)
self.num_pending_web_requests += 1
self.tagger.tagger_stats_changed.emit()
if len(queues[key]) == 1:
self._timer.start(0)
return (key, func, priority)
def remove_task(self, task):
key, func, priority = task
if priority:
queue = self._high_priority_queues[key]
else:
queue = self._low_priority_queues[key]
try:
queue.remove(func)
except:
pass
else:
self.num_pending_web_requests -= 1
self.tagger.tagger_stats_changed.emit()
def _get_by_id(self,
entitytype,
entityid,
handler,
inc=[],
params=[],
priority=False,
important=False,
mblogin=False,
refresh=False):
host = config.setting["server_host"]
port = config.setting["server_port"]
path = "/ws/2/%s/%s?inc=%s" % (entitytype, entityid, "+".join(inc))
if params:
path += "&" + "&".join(params)
return self.get(host,
port,
path,
handler,
priority=priority,
important=important,
mblogin=mblogin,
refresh=refresh)
def get_release_by_id(self,
releaseid,
handler,
inc=[],
priority=False,
important=False,
mblogin=False,
refresh=False):
return self._get_by_id('release',
releaseid,
handler,
inc,
priority=priority,
important=important,
mblogin=mblogin,
refresh=refresh)
def get_track_by_id(self,
trackid,
handler,
inc=[],
priority=False,
important=False,
mblogin=False,
refresh=False):
return self._get_by_id('recording',
trackid,
handler,
inc,
priority=priority,
important=important,
mblogin=mblogin,
refresh=refresh)
def lookup_discid(self,
discid,
handler,
priority=True,
important=True,
refresh=False):
inc = ['artist-credits', 'labels']
return self._get_by_id('discid',
discid,
handler,
inc,
params=["cdstubs=no"],
priority=priority,
important=important,
refresh=refresh)
def _find(self, entitytype, handler, kwargs):
host = config.setting["server_host"]
port = config.setting["server_port"]
filters = []
query = []
for name, value in kwargs.items():
if name == 'limit':
filters.append((name, str(value)))
else:
value = _escape_lucene_query(value).strip().lower()
if value:
query.append('%s:(%s)' % (name, value))
if query:
filters.append(('query', ' '.join(query)))
params = []
for name, value in filters:
value = QUrl.toPercentEncoding(unicode(value))
params.append('%s=%s' % (str(name), value))
path = "/ws/2/%s/?%s" % (entitytype, "&".join(params))
return self.get(host, port, path, handler)
def find_releases(self, handler, **kwargs):
return self._find('release', handler, kwargs)
def find_tracks(self, handler, **kwargs):
return self._find('recording', handler, kwargs)
def _browse(self,
entitytype,
handler,
kwargs,
inc=[],
priority=False,
important=False):
host = config.setting["server_host"]
port = config.setting["server_port"]
params = "&".join(["%s=%s" % (k, v) for k, v in kwargs.items()])
path = "/ws/2/%s?%s&inc=%s" % (entitytype, params, "+".join(inc))
return self.get(host,
port,
path,
handler,
priority=priority,
important=important)
def browse_releases(self,
handler,
priority=True,
important=True,
**kwargs):
inc = ["media", "labels"]
return self._browse("release",
handler,
kwargs,
inc,
priority=priority,
important=important)
def submit_ratings(self, ratings, handler):
host = config.setting['server_host']
port = config.setting['server_port']
path = '/ws/2/rating/?client=' + CLIENT_STRING
recordings = (''.join([
'<recording id="%s"><user-rating>%s</user-rating></recording>' %
(i[1], j * 20) for i, j in ratings.items() if i[0] == 'recording'
]))
data = _wrap_xml_metadata('<recording-list>%s</recording-list>' %
recordings)
return self.post(host, port, path, data, handler, priority=True)
def _encode_acoustid_args(self, args):
filters = []
args['client'] = ACOUSTID_KEY
args['clientversion'] = PICARD_VERSION_STR
args['format'] = 'xml'
for name, value in args.items():
value = str(QUrl.toPercentEncoding(value))
filters.append('%s=%s' % (str(name), value))
return '&'.join(filters)
def query_acoustid(self, handler, **args):
host, port = ACOUSTID_HOST, ACOUSTID_PORT
body = self._encode_acoustid_args(args)
return self.post(host,
port,
'/v2/lookup',
body,
handler,
priority=False,
important=False,
mblogin=False)
def submit_acoustid_fingerprints(self, submissions, handler):
args = {'user': config.setting["acoustid_apikey"]}
for i, submission in enumerate(submissions):
args['fingerprint.%d' % i] = str(submission.fingerprint)
args['duration.%d' % i] = str(submission.duration)
args['mbid.%d' % i] = str(submission.recordingid)
if submission.puid:
args['puid.%d' % i] = str(submission.puid)
host, port = ACOUSTID_HOST, ACOUSTID_PORT
body = self._encode_acoustid_args(args)
return self.post(host,
port,
'/v2/submit',
body,
handler,
priority=True,
important=False,
mblogin=False)
def download(self,
host,
port,
path,
handler,
priority=False,
important=False,
cacheloadcontrol=None,
refresh=False):
return self.get(host,
port,
path,
handler,
xml=False,
priority=priority,
important=important,
cacheloadcontrol=cacheloadcontrol,
refresh=refresh)
def get_collection(self, id, handler, limit=100, offset=0):
host, port = config.setting['server_host'], config.setting[
'server_port']
path = "/ws/2/collection"
if id is not None:
inc = ["releases", "artist-credits", "media"]
path += "/%s/releases?inc=%s&limit=%d&offset=%d" % (
id, "+".join(inc), limit, offset)
return self.get(host,
port,
path,
handler,
priority=True,
important=True,
mblogin=True)
def get_collection_list(self, handler):
return self.get_collection(None, handler)
def _collection_request(self, id, releases):
while releases:
ids = ";".join(
releases if len(releases) <= 400 else releases[:400])
releases = releases[400:]
yield "/ws/2/collection/%s/releases/%s?client=%s" % (id, ids,
CLIENT_STRING)
def put_to_collection(self, id, releases, handler):
host, port = config.setting['server_host'], config.setting[
'server_port']
for path in self._collection_request(id, releases):
self.put(host, port, path, "", handler)
def delete_from_collection(self, id, releases, handler):
host, port = config.setting['server_host'], config.setting[
'server_port']
for path in self._collection_request(id, releases):
self.delete(host, port, path, handler)
class WebService(QtCore.QObject):
PARSERS = dict()
def __init__(self, parent=None):
QtCore.QObject.__init__(self, parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._last_request_times = defaultdict(lambda: 0)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self._init_queues()
self._init_timers()
def _init_queues(self):
self._active_requests = {}
self._queues = defaultdict(lambda: defaultdict(deque))
self.num_pending_web_requests = 0
self._last_num_pending_web_requests = -1
def _init_timers(self):
self._timer_run_next_task = QtCore.QTimer(self)
self._timer_run_next_task.setSingleShot(True)
self._timer_run_next_task.timeout.connect(self._run_next_task)
self._timer_count_pending_requests = QtCore.QTimer(self)
self._timer_count_pending_requests.setSingleShot(True)
self._timer_count_pending_requests.timeout.connect(self._count_pending_requests)
def set_cache(self, cache_size_in_mb=100):
cache = QtNetwork.QNetworkDiskCache()
location = QStandardPaths.writableLocation(QStandardPaths.CacheLocation)
cache.setCacheDirectory(os.path.join(location, 'picard'))
cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024)
self.manager.setCache(cache)
log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory())
log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(),
cache.maximumCacheSize())
def setup_proxy(self):
proxy = QtNetwork.QNetworkProxy()
if config.setting["use_proxy"]:
proxy.setType(QtNetwork.QNetworkProxy.HttpProxy)
proxy.setHostName(config.setting["proxy_server_host"])
proxy.setPort(config.setting["proxy_server_port"])
proxy.setUser(config.setting["proxy_username"])
proxy.setPassword(config.setting["proxy_password"])
self.manager.setProxy(proxy)
@staticmethod
def _adjust_throttle(hostkey, slow_down):
"""Adjust `REQUEST` and `CONGESTION` metrics when a HTTP request completes.
Args:
hostkey: `(host, port)`.
slow_down: `True` if we encountered intermittent server trouble
and need to slow down.
"""
def in_backoff_phase(hostkey):
return CONGESTION_UNACK[hostkey] > CONGESTION_WINDOW_SIZE[hostkey]
if slow_down:
# Backoff exponentially until ~30 seconds between requests.
delay = max(pow(2, REQUEST_DELAY_EXPONENT[hostkey]) * 1000,
REQUEST_DELAY_MINIMUM[hostkey])
log.debug('WebService: %s: delay: %dms -> %dms.', hostkey, REQUEST_DELAY[hostkey],
delay)
REQUEST_DELAY[hostkey] = delay
REQUEST_DELAY_EXPONENT[hostkey] = min(REQUEST_DELAY_EXPONENT[hostkey] + 1, 5)
# Slow start threshold is ~1/2 of the window size up until we saw
# trouble. Shrink the new window size back to 1.
CONGESTION_SSTHRESH[hostkey] = int(CONGESTION_WINDOW_SIZE[hostkey] / 2.0)
log.debug('WebService: %s: ssthresh: %d.', hostkey, CONGESTION_SSTHRESH[hostkey])
CONGESTION_WINDOW_SIZE[hostkey] = 1.0
log.debug('WebService: %s: cws: %.3f.', hostkey, CONGESTION_WINDOW_SIZE[hostkey])
elif not in_backoff_phase(hostkey):
REQUEST_DELAY_EXPONENT[hostkey] = 0 # Coming out of backoff, so reset.
# Shrink the delay between requests with each successive reply to
# converge on maximum throughput.
delay = max(int(REQUEST_DELAY[hostkey] / 2), REQUEST_DELAY_MINIMUM[hostkey])
if delay != REQUEST_DELAY[hostkey]:
log.debug('WebService: %s: delay: %dms -> %dms.', hostkey, REQUEST_DELAY[hostkey],
delay)
REQUEST_DELAY[hostkey] = delay
cws = CONGESTION_WINDOW_SIZE[hostkey]
sst = CONGESTION_SSTHRESH[hostkey]
if sst and cws >= sst:
# Analogous to TCP's congestion avoidance phase. Window growth is linear.
phase = 'congestion avoidance'
cws = cws + (1.0 / cws)
else:
# Analogous to TCP's slow start phase. Window growth is exponential.
phase = 'slow start'
cws += 1
if CONGESTION_WINDOW_SIZE[hostkey] != cws:
log.debug('WebService: %s: %s: window size %.3f -> %.3f', hostkey, phase,
CONGESTION_WINDOW_SIZE[hostkey], cws)
CONGESTION_WINDOW_SIZE[hostkey] = cws
def _send_request(self, request, access_token=None):
hostkey = request.get_host_key()
# Increment the number of unack'd requests on sending a new one
CONGESTION_UNACK[hostkey] += 1
log.debug("WebService: %s: outstanding reqs: %d", hostkey, CONGESTION_UNACK[hostkey])
request.access_token = access_token
send = self._request_methods[request.method]
data = request.data
reply = send(request, data.encode('utf-8')) if data is not None else send(request)
self._remember_request_time(request.get_host_key())
self._active_requests[reply] = request
def _start_request(self, request):
if request.mblogin and request.path != "/oauth2/token":
self.oauth_manager.get_access_token(partial(self._send_request, request))
else:
self._send_request(request)
@staticmethod
def urls_equivalent(leftUrl, rightUrl):
"""
Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset
that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash
from QT 4.7
"""
return leftUrl.port(80) == rightUrl.port(80) and \
leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort)
@staticmethod
def url_port(url):
if url.scheme() == 'https':
return url.port(443)
return url.port(80)
def _handle_redirect(self, reply, request, redirect):
url = request.url()
error = int(reply.error())
# merge with base url (to cover the possibility of the URL being relative)
redirect = url.resolved(redirect)
if not WebService.urls_equivalent(redirect, reply.request().url()):
log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo))
redirect_host = string_(redirect.host())
redirect_port = self.url_port(redirect)
redirect_query = dict(QUrlQuery(redirect).queryItems(QUrl.FullyEncoded))
redirect_path = redirect.path()
original_host = string_(url.host())
original_port = self.url_port(url)
original_host_key = (original_host, original_port)
redirect_host_key = (redirect_host, redirect_port)
if (original_host_key in REQUEST_DELAY_MINIMUM
and redirect_host_key not in REQUEST_DELAY_MINIMUM):
log.debug("Setting the minimum rate limit for %s to %i" %
(redirect_host_key, REQUEST_DELAY_MINIMUM[original_host_key]))
REQUEST_DELAY_MINIMUM[redirect_host_key] = REQUEST_DELAY_MINIMUM[original_host_key]
self.get(redirect_host,
redirect_port,
redirect_path,
request.handler, request.parse_response_type, priority=True, important=True,
refresh=request.refresh, queryargs=redirect_query,
cacheloadcontrol=request.attribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute))
else:
log.error("Redirect loop: %s",
reply.request().url().toString(QUrl.RemoveUserInfo)
)
request.handler(reply.readAll(), reply, error)
def _handle_reply(self, reply, request):
hostkey = request.get_host_key()
CONGESTION_UNACK[hostkey] -= 1
log.debug("WebService: %s: outstanding reqs: %d", hostkey, CONGESTION_UNACK[hostkey])
self._timer_run_next_task.start(0)
slow_down = False
error = int(reply.error())
handler = request.handler
if error:
code = reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute)
code = int(code) if code else 0
errstr = reply.errorString()
url = reply.request().url().toString(QUrl.RemoveUserInfo)
log.error("Network request error for %s: %s (QT code %d, HTTP code %d)",
url, errstr, error, code)
if (not request.max_retries_reached()
and (code == 503
or code == 429
# Sometimes QT returns a http status code of 200 even when there
# is a service unavailable error. But it returns a QT error code
# of 403 when this happens
or error == 403
)
):
retries = request.mark_for_retry()
log.debug("Retrying %s (#%d)", url, retries)
self.add_task(partial(self._start_request, request), request)
elif handler is not None:
handler(reply.readAll(), reply, error)
slow_down = True
else:
redirect = reply.attribute(QtNetwork.QNetworkRequest.RedirectionTargetAttribute)
fromCache = reply.attribute(QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute)
cached = ' (CACHED)' if fromCache else ''
log.debug("Received reply for %s: HTTP %d (%s) %s",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute),
reply.attribute(QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute),
cached
)
if handler is not None:
# Redirect if found and not infinite
if redirect:
self._handle_redirect(reply, request, redirect)
elif request.response_parser:
try:
document = request.response_parser(reply)
except Exception as e:
log.error("Unable to parse the response. %s", e)
document = reply.readAll()
finally:
handler(document, reply, error)
else:
handler(reply.readAll(), reply, error)
self._adjust_throttle(hostkey, slow_down)
def _process_reply(self, reply):
try:
request = self._active_requests.pop(reply)
except KeyError:
log.error("Request not found for %s" % reply.request().url().toString(QUrl.RemoveUserInfo))
return
try:
self._handle_reply(reply, request)
finally:
reply.close()
reply.deleteLater()
def get(self, host, port, path, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE,
priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False,
queryargs=None):
request = WSGetRequest(host, port, path, handler, parse_response_type=parse_response_type,
mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh,
queryargs=queryargs, priority=priority, important=important)
func = partial(self._start_request, request)
return self.add_task(func, request)
def post(self, host, port, path, data, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE,
priority=False, important=False, mblogin=True, queryargs=None):
request = WSPostRequest(host, port, path, handler, parse_response_type=parse_response_type,
data=data, mblogin=mblogin, queryargs=queryargs,
priority=priority, important=important)
log.debug("POST-DATA %r", data)
func = partial(self._start_request, request)
return self.add_task(func, request)
def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True,
queryargs=None):
request = WSPutRequest(host, port, path, handler, data=data, mblogin=mblogin,
queryargs=queryargs, priority=priority, important=important)
func = partial(self._start_request, request)
return self.add_task(func, request)
def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True,
queryargs=None):
request = WSDeleteRequest(host, port, path, handler, mblogin=mblogin,
queryargs=queryargs, priority=priority, important=important)
func = partial(self._start_request, request)
return self.add_task(func, request)
def download(self, host, port, path, handler, priority=False,
important=False, cacheloadcontrol=None, refresh=False,
queryargs=None):
return self.get(host, port, path, handler, parse_response_type=None,
priority=priority, important=important,
cacheloadcontrol=cacheloadcontrol, refresh=refresh,
queryargs=queryargs)
def stop(self):
for reply in list(self._active_requests.keys()):
reply.abort()
self._init_queues()
def _count_pending_requests(self):
count = len(self._active_requests)
for prio_queue in self._queues.values():
for queue in prio_queue.values():
count += len(queue)
self.num_pending_web_requests = count
if count != self._last_num_pending_web_requests:
self._last_num_pending_web_requests = count
self.tagger.tagger_stats_changed.emit()
if count:
self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS)
def _get_delay_to_next_request(self, hostkey):
"""Calculate delay to next request to hostkey (host, port)
returns a tuple (wait, delay) where:
wait is True if a delay is needed
delay is the delay in milliseconds to next request
"""
if CONGESTION_UNACK[hostkey] >= int(CONGESTION_WINDOW_SIZE[hostkey]):
# We've maxed out the number of requests to `hostkey`, so wait
# until responses begin to come back. (See `_timer_run_next_task`
# strobe in `_handle_reply`.)
return (True, sys.maxsize)
interval = REQUEST_DELAY[hostkey]
if not interval:
log.debug("WSREQ: Starting another request to %s without delay", hostkey)
return (False, 0)
last_request = self._last_request_times[hostkey]
if not last_request:
log.debug("WSREQ: First request to %s", hostkey)
self._remember_request_time(hostkey) # set it on first run
return (False, interval)
elapsed = (time.time() - last_request) * 1000
if elapsed >= interval:
log.debug("WSREQ: Last request to %s was %d ms ago, starting another one", hostkey, elapsed)
return (False, interval)
delay = int(math.ceil(interval - elapsed))
log.debug("WSREQ: Last request to %s was %d ms ago, waiting %d ms before starting another one",
hostkey, elapsed, delay)
return (True, delay)
def _remember_request_time(self, hostkey):
if REQUEST_DELAY[hostkey]:
self._last_request_times[hostkey] = time.time()
def _run_next_task(self):
delay = sys.maxsize
for prio in sorted(self._queues.keys(), reverse=True):
prio_queue = self._queues[prio]
if not prio_queue:
del(self._queues[prio])
continue
for hostkey in sorted(prio_queue.keys(),
key=lambda hostkey: REQUEST_DELAY[hostkey]):
queue = self._queues[prio][hostkey]
if not queue:
del(self._queues[prio][hostkey])
continue
wait, d = self._get_delay_to_next_request(hostkey)
if not wait:
queue.popleft()()
if d < delay:
delay = d
if delay < sys.maxsize:
self._timer_run_next_task.start(delay)
def add_task(self, func, request):
hostkey = request.get_host_key()
prio = int(request.priority) # priority is a boolean
if request.important:
self._queues[prio][hostkey].appendleft(func)
else:
self._queues[prio][hostkey].append(func)
if not self._timer_run_next_task.isActive():
self._timer_run_next_task.start(0)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
return (hostkey, func, prio)
def remove_task(self, task):
hostkey, func, prio = task
try:
self._queues[prio][hostkey].remove(func)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
except Exception as e:
log.debug(e)
@classmethod
def add_parser(cls, response_type, mimetype, parser):
cls.PARSERS[response_type] = Parser(mimetype=mimetype, parser=parser)
@classmethod
def get_response_mimetype(cls, response_type):
if response_type in cls.PARSERS:
return cls.PARSERS[response_type].mimetype
else:
raise UnknownResponseParserError(response_type)
@classmethod
def get_response_parser(cls, response_type):
if response_type in cls.PARSERS:
return cls.PARSERS[response_type].parser
else:
raise UnknownResponseParserError(response_type)
class WebService(QtCore.QObject):
PARSERS = dict()
def __init__(self, parent=None):
QtCore.QObject.__init__(self, parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._last_request_times = defaultdict(lambda: 0)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self._init_queues()
self._init_timers()
def _init_queues(self):
self._active_requests = {}
self._queues = defaultdict(lambda: defaultdict(deque))
self.num_pending_web_requests = 0
self._last_num_pending_web_requests = -1
def _init_timers(self):
self._timer_run_next_task = QtCore.QTimer(self)
self._timer_run_next_task.setSingleShot(True)
self._timer_run_next_task.timeout.connect(self._run_next_task)
self._timer_count_pending_requests = QtCore.QTimer(self)
self._timer_count_pending_requests.setSingleShot(True)
self._timer_count_pending_requests.timeout.connect(
self._count_pending_requests)
def set_cache(self, cache_size_in_mb=100):
cache = QtNetwork.QNetworkDiskCache()
location = QStandardPaths.writableLocation(
QStandardPaths.CacheLocation)
cache.setCacheDirectory(os.path.join(location, 'picard'))
cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024)
self.manager.setCache(cache)
log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory())
log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(),
cache.maximumCacheSize())
def setup_proxy(self):
proxy = QtNetwork.QNetworkProxy()
if config.setting["use_proxy"]:
proxy.setType(QtNetwork.QNetworkProxy.HttpProxy)
proxy.setHostName(config.setting["proxy_server_host"])
proxy.setPort(config.setting["proxy_server_port"])
proxy.setUser(config.setting["proxy_username"])
proxy.setPassword(config.setting["proxy_password"])
self.manager.setProxy(proxy)
@staticmethod
def _adjust_throttle(hostkey, slow_down):
"""Adjust `REQUEST` and `CONGESTION` metrics when a HTTP request completes.
:param hostkey: `(host, port)`.
:param slow_down: `True` if we encountered intermittent server trouble
and need to slow down.
"""
def in_backoff_phase(hostkey):
return CONGESTION_UNACK[hostkey] > CONGESTION_WINDOW_SIZE[hostkey]
if slow_down:
# Backoff exponentially until ~30 seconds between requests.
delay = max(
pow(2, REQUEST_DELAY_EXPONENT[hostkey]) * 1000,
REQUEST_DELAY_MINIMUM[hostkey])
log.debug('WebService: %s: delay: %dms -> %dms.', hostkey,
REQUEST_DELAY[hostkey], delay)
REQUEST_DELAY[hostkey] = delay
REQUEST_DELAY_EXPONENT[hostkey] = min(
REQUEST_DELAY_EXPONENT[hostkey] + 1, 5)
# Slow start threshold is ~1/2 of the window size up until we saw
# trouble. Shrink the new window size back to 1.
CONGESTION_SSTHRESH[hostkey] = int(
CONGESTION_WINDOW_SIZE[hostkey] / 2.0)
log.debug('WebService: %s: ssthresh: %d.', hostkey,
CONGESTION_SSTHRESH[hostkey])
CONGESTION_WINDOW_SIZE[hostkey] = 1.0
log.debug('WebService: %s: cws: %.3f.', hostkey,
CONGESTION_WINDOW_SIZE[hostkey])
elif not in_backoff_phase(hostkey):
REQUEST_DELAY_EXPONENT[
hostkey] = 0 # Coming out of backoff, so reset.
# Shrink the delay between requests with each successive reply to
# converge on maximum throughput.
delay = max(int(REQUEST_DELAY[hostkey] / 2),
REQUEST_DELAY_MINIMUM[hostkey])
if delay != REQUEST_DELAY[hostkey]:
log.debug('WebService: %s: delay: %dms -> %dms.', hostkey,
REQUEST_DELAY[hostkey], delay)
REQUEST_DELAY[hostkey] = delay
cws = CONGESTION_WINDOW_SIZE[hostkey]
sst = CONGESTION_SSTHRESH[hostkey]
if sst and cws >= sst:
# Analogous to TCP's congestion avoidance phase. Window growth is linear.
phase = 'congestion avoidance'
cws = cws + (1.0 / cws)
else:
# Analogous to TCP's slow start phase. Window growth is exponential.
phase = 'slow start'
cws += 1
if CONGESTION_WINDOW_SIZE[hostkey] != cws:
log.debug('WebService: %s: %s: window size %.3f -> %.3f',
hostkey, phase, CONGESTION_WINDOW_SIZE[hostkey], cws)
CONGESTION_WINDOW_SIZE[hostkey] = cws
def _send_request(self, request, access_token=None):
hostkey = request.get_host_key()
# Increment the number of unack'd requests on sending a new one
CONGESTION_UNACK[hostkey] += 1
log.debug("WebService: %s: outstanding reqs: %d", hostkey,
CONGESTION_UNACK[hostkey])
request.access_token = access_token
send = self._request_methods[request.method]
data = request.data
reply = send(
request,
data.encode('utf-8')) if data is not None else send(request)
self._remember_request_time(request.get_host_key())
self._active_requests[reply] = request
def _start_request(self, request):
if request.mblogin and request.path != "/oauth2/token":
self.oauth_manager.get_access_token(
partial(self._send_request, request))
else:
self._send_request(request)
@staticmethod
def urls_equivalent(leftUrl, rightUrl):
"""
Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset
that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash
from QT 4.7
"""
return leftUrl.port(80) == rightUrl.port(80) and \
leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort)
@staticmethod
def url_port(url):
if url.scheme() == 'https':
return url.port(443)
return url.port(80)
def _handle_redirect(self, reply, request, redirect):
url = request.url()
error = int(reply.error())
# merge with base url (to cover the possibility of the URL being relative)
redirect = url.resolved(redirect)
if not WebService.urls_equivalent(redirect, reply.request().url()):
log.debug("Redirect to %s requested",
redirect.toString(QUrl.RemoveUserInfo))
redirect_host = string_(redirect.host())
redirect_port = self.url_port(redirect)
redirect_query = dict(
QUrlQuery(redirect).queryItems(QUrl.FullyEncoded))
redirect_path = redirect.path()
original_host = string_(url.host())
original_port = self.url_port(url)
original_host_key = (original_host, original_port)
redirect_host_key = (redirect_host, redirect_port)
if (original_host_key in REQUEST_DELAY_MINIMUM
and redirect_host_key not in REQUEST_DELAY_MINIMUM):
log.debug("Setting the minimum rate limit for %s to %i" %
(redirect_host_key,
REQUEST_DELAY_MINIMUM[original_host_key]))
REQUEST_DELAY_MINIMUM[
redirect_host_key] = REQUEST_DELAY_MINIMUM[
original_host_key]
self.get(redirect_host,
redirect_port,
redirect_path,
request.handler,
request.parse_response_type,
priority=True,
important=True,
refresh=request.refresh,
queryargs=redirect_query,
cacheloadcontrol=request.attribute(
QtNetwork.QNetworkRequest.CacheLoadControlAttribute))
else:
log.error("Redirect loop: %s",
reply.request().url().toString(QUrl.RemoveUserInfo))
handler(reply.readAll(), reply, error)
def _handle_reply(self, reply, request):
hostkey = request.get_host_key()
CONGESTION_UNACK[hostkey] -= 1
log.debug("WebService: %s: outstanding reqs: %d", hostkey,
CONGESTION_UNACK[hostkey])
self._timer_run_next_task.start(0)
slow_down = False
error = int(reply.error())
handler = request.handler
if error:
code = reply.attribute(
QtNetwork.QNetworkRequest.HttpStatusCodeAttribute)
code = int(code) if code else 0
errstr = reply.errorString()
url = reply.request().url().toString(QUrl.RemoveUserInfo)
log.error(
"Network request error for %s: %s (QT code %d, HTTP code %d)",
url, errstr, error, code)
if (not request.max_retries_reached() and
(code == 503 or code == 429
# Sometimes QT returns a http status code of 200 even when there
# is a service unavailable error. But it returns a QT error code
# of 403 when this happens
or error == 403)):
retries = request.mark_for_retry()
log.debug("Retrying %s (#%d)", url, retries)
self.add_task(partial(self._start_request, request), request)
elif handler is not None:
handler(reply.readAll(), reply, error)
slow_down = True
else:
redirect = reply.attribute(
QtNetwork.QNetworkRequest.RedirectionTargetAttribute)
fromCache = reply.attribute(
QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute)
cached = ' (CACHED)' if fromCache else ''
log.debug(
"Received reply for %s: HTTP %d (%s) %s",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.attribute(
QtNetwork.QNetworkRequest.HttpStatusCodeAttribute),
reply.attribute(
QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute),
cached)
if handler is not None:
# Redirect if found and not infinite
if redirect:
self._handle_redirect(reply, request, redirect)
elif request.response_parser:
try:
document = request.response_parser(reply)
except Exception as e:
log.error("Unable to parse the response. %s", e)
document = reply.readAll()
finally:
handler(document, reply, error)
else:
handler(reply.readAll(), reply, error)
self._adjust_throttle(hostkey, slow_down)
def _process_reply(self, reply):
try:
request = self._active_requests.pop(reply)
except KeyError:
log.error("Request not found for %s" %
reply.request().url().toString(QUrl.RemoveUserInfo))
return
try:
self._handle_reply(reply, request)
finally:
reply.close()
reply.deleteLater()
def get(self,
host,
port,
path,
handler,
parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE,
priority=False,
important=False,
mblogin=False,
cacheloadcontrol=None,
refresh=False,
queryargs=None):
request = WSGetRequest(host,
port,
path,
handler,
parse_response_type=parse_response_type,
mblogin=mblogin,
cacheloadcontrol=cacheloadcontrol,
refresh=refresh,
queryargs=queryargs,
priority=priority,
important=important)
func = partial(self._start_request, request)
return self.add_task(func, request)
def post(self,
host,
port,
path,
data,
handler,
parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE,
priority=False,
important=False,
mblogin=True,
queryargs=None):
request = WSPostRequest(host,
port,
path,
handler,
parse_response_type=parse_response_type,
data=data,
mblogin=mblogin,
queryargs=queryargs,
priority=priority,
important=important)
log.debug("POST-DATA %r", data)
func = partial(self._start_request, request)
return self.add_task(func, request)
def put(self,
host,
port,
path,
data,
handler,
priority=True,
important=False,
mblogin=True,
queryargs=None):
request = WSPutRequest(host,
port,
path,
handler,
data=data,
mblogin=mblogin,
queryargs=queryargs,
priority=priority,
important=important)
func = partial(self._start_request, request)
return self.add_task(func, request)
def delete(self,
host,
port,
path,
handler,
priority=True,
important=False,
mblogin=True,
queryargs=None):
request = WSDeleteRequest(host,
port,
path,
handler,
mblogin=mblogin,
queryargs=queryargs,
priority=priority,
important=important)
func = partial(self._start_request, request)
return self.add_task(func, request)
def download(self,
host,
port,
path,
handler,
priority=False,
important=False,
cacheloadcontrol=None,
refresh=False,
queryargs=None):
return self.get(host,
port,
path,
handler,
parse_response_type=None,
priority=priority,
important=important,
cacheloadcontrol=cacheloadcontrol,
refresh=refresh,
queryargs=queryargs)
def stop(self):
for reply in list(self._active_requests.keys()):
reply.abort()
self._init_queues()
def _count_pending_requests(self):
count = len(self._active_requests)
for prio_queue in self._queues.values():
for queue in prio_queue.values():
count += len(queue)
self.num_pending_web_requests = count
if count != self._last_num_pending_web_requests:
self._last_num_pending_web_requests = count
self.tagger.tagger_stats_changed.emit()
if count:
self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS)
def _get_delay_to_next_request(self, hostkey):
"""Calculate delay to next request to hostkey (host, port)
returns a tuple (wait, delay) where:
wait is True if a delay is needed
delay is the delay in milliseconds to next request
"""
if CONGESTION_UNACK[hostkey] >= int(CONGESTION_WINDOW_SIZE[hostkey]):
# We've maxed out the number of requests to `hostkey`, so wait
# until responses begin to come back. (See `_timer_run_next_task`
# strobe in `_handle_reply`.)
return (True, sys.maxsize)
interval = REQUEST_DELAY[hostkey]
if not interval:
log.debug("WSREQ: Starting another request to %s without delay",
hostkey)
return (False, 0)
last_request = self._last_request_times[hostkey]
if not last_request:
log.debug("WSREQ: First request to %s", hostkey)
self._remember_request_time(hostkey) # set it on first run
return (False, interval)
elapsed = (time.time() - last_request) * 1000
if elapsed >= interval:
log.debug(
"WSREQ: Last request to %s was %d ms ago, starting another one",
hostkey, elapsed)
return (False, interval)
delay = int(math.ceil(interval - elapsed))
log.debug(
"WSREQ: Last request to %s was %d ms ago, waiting %d ms before starting another one",
hostkey, elapsed, delay)
return (True, delay)
def _remember_request_time(self, hostkey):
if REQUEST_DELAY[hostkey]:
self._last_request_times[hostkey] = time.time()
def _run_next_task(self):
delay = sys.maxsize
for prio in sorted(self._queues.keys(), reverse=True):
prio_queue = self._queues[prio]
if not prio_queue:
del (self._queues[prio])
continue
for hostkey in sorted(prio_queue.keys(),
key=lambda hostkey: REQUEST_DELAY[hostkey]):
queue = self._queues[prio][hostkey]
if not queue:
del (self._queues[prio][hostkey])
continue
wait, d = self._get_delay_to_next_request(hostkey)
if not wait:
queue.popleft()()
if d < delay:
delay = d
if delay < sys.maxsize:
self._timer_run_next_task.start(delay)
def add_task(self, func, request):
hostkey = request.get_host_key()
prio = int(request.priority) # priority is a boolean
if request.important:
self._queues[prio][hostkey].appendleft(func)
else:
self._queues[prio][hostkey].append(func)
if not self._timer_run_next_task.isActive():
self._timer_run_next_task.start(0)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
return (hostkey, func, prio)
def remove_task(self, task):
hostkey, func, prio = task
try:
self._queues[prio][hostkey].remove(func)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
except Exception as e:
log.debug(e)
@classmethod
def add_parser(cls, response_type, mimetype, parser):
cls.PARSERS[response_type] = Parser(mimetype=mimetype, parser=parser)
@classmethod
def get_response_mimetype(cls, response_type):
if response_type in cls.PARSERS:
return cls.PARSERS[response_type].mimetype
else:
raise UnknownResponseParserError(response_type)
@classmethod
def get_response_parser(cls, response_type):
if response_type in cls.PARSERS:
return cls.PARSERS[response_type].parser
else:
raise UnknownResponseParserError(response_type)
class WebService(QtCore.QObject):
PARSERS = dict()
def __init__(self, parent=None):
super().__init__(parent)
self.manager = QtNetwork.QNetworkAccessManager()
self._network_accessible_changed(self.manager.networkAccessible())
self.manager.networkAccessibleChanged.connect(self._network_accessible_changed)
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
config = get_config()
self.set_transfer_timeout(config.setting['network_transfer_timeout_seconds'])
self.manager.finished.connect(self._process_reply)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self._init_queues()
self._init_timers()
@staticmethod
def http_response_code(reply):
response_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
return int(response_code) if response_code else 0
@staticmethod
def http_response_phrase(reply):
return reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
@staticmethod
def http_response_safe_url(reply):
return reply.request().url().toString(QUrl.RemoveUserInfo)
def _network_accessible_changed(self, accessible):
# Qt's network accessibility check sometimes fails, e.g. with VPNs on Windows.
# If the accessibility is reported to be not accessible, set it to
# unknown instead. Let's just try any request and handle the error.
# See https://tickets.metabrainz.org/browse/PICARD-1791
if accessible == QtNetwork.QNetworkAccessManager.NotAccessible:
self.manager.setNetworkAccessible(QtNetwork.QNetworkAccessManager.UnknownAccessibility)
log.debug("Network accessible requested: %s, actual: %s", accessible, self.manager.networkAccessible())
def _init_queues(self):
self._active_requests = {}
self._queue = RequestPriorityQueue(ratecontrol)
self.num_pending_web_requests = 0
def _init_timers(self):
self._timer_run_next_task = QtCore.QTimer(self)
self._timer_run_next_task.setSingleShot(True)
self._timer_run_next_task.timeout.connect(self._run_next_task)
self._timer_count_pending_requests = QtCore.QTimer(self)
self._timer_count_pending_requests.setSingleShot(True)
self._timer_count_pending_requests.timeout.connect(self._count_pending_requests)
def set_cache(self, cache_size_in_bytes=None):
if cache_size_in_bytes is None:
cache_size_in_bytes = CACHE_SIZE_IN_BYTES
cache = QtNetwork.QNetworkDiskCache()
cache.setCacheDirectory(os.path.join(appdirs.cache_folder(), 'network'))
cache.setMaximumCacheSize(cache_size_in_bytes)
self.manager.setCache(cache)
log.debug("NetworkDiskCache dir: %r current size: %s max size: %s",
cache.cacheDirectory(),
bytes2human.decimal(cache.cacheSize(), l10n=False),
bytes2human.decimal(cache.maximumCacheSize(), l10n=False))
def setup_proxy(self):
proxy = QtNetwork.QNetworkProxy()
config = get_config()
if config.setting["use_proxy"]:
if config.setting["proxy_type"] == 'socks':
proxy.setType(QtNetwork.QNetworkProxy.Socks5Proxy)
else:
proxy.setType(QtNetwork.QNetworkProxy.HttpProxy)
proxy.setHostName(config.setting["proxy_server_host"])
proxy.setPort(config.setting["proxy_server_port"])
if config.setting["proxy_username"]:
proxy.setUser(config.setting["proxy_username"])
if config.setting["proxy_password"]:
proxy.setPassword(config.setting["proxy_password"])
self.manager.setProxy(proxy)
def set_transfer_timeout(self, timeout):
timeout_ms = timeout * 1000
if hasattr(self.manager, 'setTransferTimeout'): # Available since Qt 5.15
self.manager.setTransferTimeout(timeout_ms)
self._transfer_timeout = 0
else: # Use fallback implementation
self._transfer_timeout = timeout_ms
def _send_request(self, request, access_token=None):
hostkey = request.get_host_key()
ratecontrol.increment_requests(hostkey)
request.access_token = access_token
send = self._request_methods[request.method]
data = request.data
reply = send(request, data.encode('utf-8')) if data is not None else send(request)
self._start_transfer_timeout(reply)
self._active_requests[reply] = request
def _start_transfer_timeout(self, reply):
if not self._transfer_timeout:
return
# Fallback implementation of a transfer timeout for Qt < 5.15.
# Aborts a request if no data gets transferred for TRANSFER_TIMEOUT milliseconds.
timer = QtCore.QTimer(self)
timer.setSingleShot(True)
timer.setTimerType(QtCore.Qt.PreciseTimer)
timer.timeout.connect(partial(self._timeout_request, reply))
reply.finished.connect(timer.stop)
reset_callback = partial(self._reset_transfer_timeout, timer)
reply.uploadProgress.connect(reset_callback)
reply.downloadProgress.connect(reset_callback)
timer.start(self._transfer_timeout)
def _reset_transfer_timeout(self, timer, bytesTransferred, bytesTotal):
timer.start(self._transfer_timeout)
@staticmethod
def _timeout_request(reply):
if reply.isRunning():
reply.abort()
def _start_request(self, request):
if request.mblogin and request.path != "/oauth2/token":
self.oauth_manager.get_access_token(partial(self._send_request, request))
else:
self._send_request(request)
@staticmethod
def urls_equivalent(leftUrl, rightUrl):
"""
Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset
that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash
from QT 4.7
"""
return leftUrl.port(80) == rightUrl.port(80) and \
leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort)
@staticmethod
def url_port(url):
if url.scheme() == 'https':
return url.port(443)
return url.port(80)
def _handle_redirect(self, reply, request, redirect):
url = request.url()
error = int(reply.error())
# merge with base url (to cover the possibility of the URL being relative)
redirect = url.resolved(redirect)
if not WebService.urls_equivalent(redirect, reply.request().url()):
log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo))
redirect_host = redirect.host()
redirect_port = self.url_port(redirect)
redirect_query = dict(QUrlQuery(redirect).queryItems(QUrl.FullyEncoded))
redirect_path = redirect.path()
original_host = url.host()
original_port = self.url_port(url)
original_host_key = (original_host, original_port)
redirect_host_key = (redirect_host, redirect_port)
ratecontrol.copy_minimal_delay(original_host_key, redirect_host_key)
self.get(redirect_host,
redirect_port,
redirect_path,
request.handler, request.parse_response_type, priority=True, important=True,
refresh=request.refresh, queryargs=redirect_query, mblogin=request.mblogin,
cacheloadcontrol=request.attribute(QNetworkRequest.CacheLoadControlAttribute))
else:
log.error("Redirect loop: %s",
self.http_response_safe_url(reply)
)
request.handler(reply.readAll(), reply, error)
def _handle_reply(self, reply, request):
hostkey = request.get_host_key()
ratecontrol.decrement_requests(hostkey)
self._timer_run_next_task.start(0)
slow_down = False
error = int(reply.error())
handler = request.handler
response_code = self.http_response_code(reply)
url = self.http_response_safe_url(reply)
if error:
errstr = reply.errorString()
log.error("Network request error for %s: %s (QT code %d, HTTP code %d)",
url, errstr, error, response_code)
if (not request.max_retries_reached()
and (response_code == 503
or response_code == 429
# Sometimes QT returns a http status code of 200 even when there
# is a service unavailable error. But it returns a QT error code
# of 403 when this happens
or error == 403
)):
slow_down = True
retries = request.mark_for_retry()
log.debug("Retrying %s (#%d)", url, retries)
self.add_request(request)
elif handler is not None:
handler(reply.readAll(), reply, error)
slow_down = (slow_down or response_code >= 500)
else:
redirect = reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
from_cache = reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute)
cached = ' (CACHED)' if from_cache else ''
log.debug("Received reply for %s: HTTP %d (%s) %s",
url,
response_code,
self.http_response_phrase(reply),
cached
)
if handler is not None:
# Redirect if found and not infinite
if redirect:
self._handle_redirect(reply, request, redirect)
elif request.response_parser:
try:
document = request.response_parser(reply)
log.debug("Response received: %s", document)
except Exception as e:
log.error("Unable to parse the response for %s: %s", url, e)
document = reply.readAll()
error = e
finally:
handler(document, reply, error)
else:
handler(reply.readAll(), reply, error)
ratecontrol.adjust(hostkey, slow_down)
def _process_reply(self, reply):
try:
request = self._active_requests.pop(reply)
except KeyError:
log.error("Request not found for %s", self.http_response_safe_url(reply))
return
try:
self._handle_reply(reply, request)
finally:
reply.close()
reply.deleteLater()
def get(self, host, port, path, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE,
priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False,
queryargs=None):
request = WSGetRequest(host, port, path, handler, parse_response_type=parse_response_type,
mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh,
queryargs=queryargs, priority=priority, important=important)
return self.add_request(request)
def post(self, host, port, path, data, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE,
priority=False, important=False, mblogin=True, queryargs=None, request_mimetype=None):
request = WSPostRequest(host, port, path, handler, parse_response_type=parse_response_type,
data=data, mblogin=mblogin, queryargs=queryargs,
priority=priority, important=important,
request_mimetype=request_mimetype)
log.debug("POST-DATA %r", data)
return self.add_request(request)
def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True,
queryargs=None, request_mimetype=None):
request = WSPutRequest(host, port, path, handler, data=data, mblogin=mblogin,
queryargs=queryargs, priority=priority,
important=important, request_mimetype=request_mimetype)
return self.add_request(request)
def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True,
queryargs=None):
request = WSDeleteRequest(host, port, path, handler, mblogin=mblogin,
queryargs=queryargs, priority=priority, important=important)
return self.add_request(request)
def download(self, host, port, path, handler, priority=False,
important=False, cacheloadcontrol=None, refresh=False,
queryargs=None):
return self.get(host, port, path, handler, parse_response_type=None,
priority=priority, important=important,
cacheloadcontrol=cacheloadcontrol, refresh=refresh,
queryargs=queryargs)
def stop(self):
for reply in list(self._active_requests):
reply.abort()
self._init_queues()
def _count_pending_requests(self):
count = len(self._active_requests) + self._queue.count()
if count != self.num_pending_web_requests:
self.num_pending_web_requests = count
self.tagger.tagger_stats_changed.emit()
if count:
self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS)
def _run_next_task(self):
delay = self._queue.run_ready_tasks()
if delay < sys.maxsize:
self._timer_run_next_task.start(delay)
def add_task(self, func, request):
task = RequestTask.from_request(request, func)
self._queue.add_task(task, request.important)
if not self._timer_run_next_task.isActive():
self._timer_run_next_task.start(0)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
return task
def add_request(self, request):
return self.add_task(partial(self._start_request, request), request)
def remove_task(self, task):
self._queue.remove_task(task)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
@classmethod
def add_parser(cls, response_type, mimetype, parser):
cls.PARSERS[response_type] = Parser(mimetype=mimetype, parser=parser)
@classmethod
def get_response_mimetype(cls, response_type):
if response_type in cls.PARSERS:
return cls.PARSERS[response_type].mimetype
else:
raise UnknownResponseParserError(response_type)
@classmethod
def get_response_parser(cls, response_type):
if response_type in cls.PARSERS:
return cls.PARSERS[response_type].parser
else:
raise UnknownResponseParserError(response_type)
class XmlWebService(QtCore.QObject):
def __init__(self, parent=None):
QtCore.QObject.__init__(self, parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._last_request_times = {}
self._active_requests = {}
self._high_priority_queues = {}
self._low_priority_queues = {}
self._hosts = []
self._timer = QtCore.QTimer(self)
self._timer.setSingleShot(True)
self._timer.timeout.connect(self._run_next_task)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self.num_pending_web_requests = 0
def set_cache(self, cache_size_in_mb=100):
cache = QtNetwork.QNetworkDiskCache()
location = QDesktopServices.storageLocation(QDesktopServices.CacheLocation)
cache.setCacheDirectory(os.path.join(unicode(location), u'picard'))
cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024)
self.manager.setCache(cache)
log.debug("NetworkDiskCache dir: %s", cache.cacheDirectory())
log.debug("NetworkDiskCache size: %s / %s", cache.cacheSize(),
cache.maximumCacheSize())
def setup_proxy(self):
proxy = QtNetwork.QNetworkProxy()
if config.setting["use_proxy"]:
proxy.setType(QtNetwork.QNetworkProxy.HttpProxy)
proxy.setHostName(config.setting["proxy_server_host"])
proxy.setPort(config.setting["proxy_server_port"])
proxy.setUser(config.setting["proxy_username"])
proxy.setPassword(config.setting["proxy_password"])
self.manager.setProxy(proxy)
def _start_request_continue(self, method, host, port, path, data, handler, xml,
mblogin=False, cacheloadcontrol=None, refresh=None,
access_token=None):
if mblogin and host in MUSICBRAINZ_SERVERS and port == 80:
urlstring = "https://%s%s" % (host, path)
else:
urlstring = "http://%s:%d%s" % (host, port, path)
log.debug("%s %s", method, urlstring)
url = QUrl.fromEncoded(urlstring)
request = QtNetwork.QNetworkRequest(url)
if mblogin and access_token:
request.setRawHeader("Authorization", "Bearer %s" % access_token)
if mblogin or (method == "GET" and refresh):
request.setPriority(QtNetwork.QNetworkRequest.HighPriority)
request.setAttribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute,
QtNetwork.QNetworkRequest.AlwaysNetwork)
elif method == "PUT" or method == "DELETE":
request.setPriority(QtNetwork.QNetworkRequest.HighPriority)
elif cacheloadcontrol is not None:
request.setAttribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute,
cacheloadcontrol)
request.setRawHeader("User-Agent", USER_AGENT_STRING)
if xml:
request.setRawHeader("Accept", "application/xml")
if data is not None:
if method == "POST" and host == config.setting["server_host"] and xml:
request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/xml; charset=utf-8")
else:
request.setHeader(QtNetwork.QNetworkRequest.ContentTypeHeader, "application/x-www-form-urlencoded")
send = self._request_methods[method]
reply = send(request, data) if data is not None else send(request)
key = (host, port)
self._last_request_times[key] = time.time()
self._active_requests[reply] = (request, handler, xml, refresh)
def _start_request(self, method, host, port, path, data, handler, xml,
mblogin=False, cacheloadcontrol=None, refresh=None):
def start_request_continue(access_token=None):
self._start_request_continue(
method, host, port, path, data, handler, xml,
mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh,
access_token=access_token)
if mblogin and path != "/oauth2/token":
self.oauth_manager.get_access_token(start_request_continue)
else:
start_request_continue()
@staticmethod
def urls_equivalent(leftUrl, rightUrl):
"""
Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset
that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash
from QT 4.7
"""
return leftUrl.port(80) == rightUrl.port(80) and \
leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort)
def _process_reply(self, reply):
try:
request, handler, xml, refresh = self._active_requests.pop(reply)
except KeyError:
log.error("Request not found for %s" % reply.request().url().toString(QUrl.RemoveUserInfo))
return
error = int(reply.error())
if error:
log.error("Network request error for %s: %s (QT code %d, HTTP code %s)",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.errorString(),
error,
repr(reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute))
)
if handler is not None:
handler(str(reply.readAll()), reply, error)
else:
redirect = reply.attribute(QtNetwork.QNetworkRequest.RedirectionTargetAttribute)
fromCache = reply.attribute(QtNetwork.QNetworkRequest.SourceIsFromCacheAttribute)
cached = ' (CACHED)' if fromCache else ''
log.debug("Received reply for %s: HTTP %d (%s) %s",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.attribute(QtNetwork.QNetworkRequest.HttpStatusCodeAttribute),
reply.attribute(QtNetwork.QNetworkRequest.HttpReasonPhraseAttribute),
cached
)
if handler is not None:
# Redirect if found and not infinite
if redirect and not XmlWebService.urls_equivalent(redirect, reply.request().url()):
log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo))
redirect_host = str(redirect.host())
redirect_port = redirect.port(80)
url = request.url()
original_host = str(url.host())
original_port = url.port(80)
if ((original_host, original_port) in REQUEST_DELAY
and (redirect_host, redirect_port) not in REQUEST_DELAY):
log.debug("Setting rate limit for %s:%i to %i" %
(redirect_host, redirect_port,
REQUEST_DELAY[(original_host, original_port)]))
REQUEST_DELAY[(redirect_host, redirect_port)] =\
REQUEST_DELAY[(original_host, original_port)]
self.get(redirect_host,
redirect_port,
# retain path, query string and anchors from redirect URL
redirect.toString(QUrl.RemoveAuthority | QUrl.RemoveScheme),
handler, xml, priority=True, important=True, refresh=refresh,
cacheloadcontrol=request.attribute(QtNetwork.QNetworkRequest.CacheLoadControlAttribute))
elif redirect:
log.error("Redirect loop: %s",
reply.request().url().toString(QUrl.RemoveUserInfo)
)
handler(str(reply.readAll()), reply, error)
elif xml:
document = _read_xml(QXmlStreamReader(reply))
handler(document, reply, error)
else:
handler(str(reply.readAll()), reply, error)
reply.close()
self.num_pending_web_requests -= 1
self.tagger.tagger_stats_changed.emit()
def get(self, host, port, path, handler, xml=True, priority=False,
important=False, mblogin=False, cacheloadcontrol=None, refresh=False):
func = partial(self._start_request, "GET", host, port, path, None,
handler, xml, mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh)
return self.add_task(func, host, port, priority, important=important)
def post(self, host, port, path, data, handler, xml=True, priority=False, important=False, mblogin=True):
log.debug("POST-DATA %r", data)
func = partial(self._start_request, "POST", host, port, path, data, handler, xml, mblogin)
return self.add_task(func, host, port, priority, important=important)
def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True):
func = partial(self._start_request, "PUT", host, port, path, data, handler, False, mblogin)
return self.add_task(func, host, port, priority, important=important)
def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True):
func = partial(self._start_request, "DELETE", host, port, path, None, handler, False, mblogin)
return self.add_task(func, host, port, priority, important=important)
def stop(self):
self._high_priority_queues = {}
self._low_priority_queues = {}
for reply in self._active_requests.keys():
reply.abort()
def _run_next_task(self):
delay = sys.maxint
for key in self._hosts:
queue = self._high_priority_queues.get(key) or self._low_priority_queues.get(key)
if not queue:
continue
now = time.time()
last = self._last_request_times.get(key)
request_delay = REQUEST_DELAY[key]
last_ms = (now - last) * 1000 if last is not None else request_delay
if last_ms >= request_delay:
log.debug("Last request to %s was %d ms ago, starting another one", key, last_ms)
d = request_delay
queue.popleft()()
else:
d = int(math.ceil(request_delay - last_ms))
log.debug("Waiting %d ms before starting another request to %s", d, key)
if d < delay:
delay = d
if delay < sys.maxint:
self._timer.start(delay)
def add_task(self, func, host, port, priority, important=False):
key = (host, port)
if key not in self._hosts:
self._hosts.append(key)
if priority:
queues = self._high_priority_queues
else:
queues = self._low_priority_queues
queues.setdefault(key, deque())
if important:
queues[key].appendleft(func)
else:
queues[key].append(func)
self.num_pending_web_requests += 1
self.tagger.tagger_stats_changed.emit()
if len(queues[key]) == 1:
self._timer.start(0)
return (key, func, priority)
def remove_task(self, task):
key, func, priority = task
if priority:
queue = self._high_priority_queues[key]
else:
queue = self._low_priority_queues[key]
try:
queue.remove(func)
except:
pass
else:
self.num_pending_web_requests -= 1
self.tagger.tagger_stats_changed.emit()
def _get_by_id(self, entitytype, entityid, handler, inc=[], params=[],
priority=False, important=False, mblogin=False, refresh=False):
host = config.setting["server_host"]
port = config.setting["server_port"]
path = "/ws/2/%s/%s?inc=%s" % (entitytype, entityid, "+".join(inc))
if params:
path += "&" + "&".join(params)
return self.get(host, port, path, handler,
priority=priority, important=important, mblogin=mblogin, refresh=refresh)
def get_release_by_id(self, releaseid, handler, inc=[],
priority=False, important=False, mblogin=False, refresh=False):
return self._get_by_id('release', releaseid, handler, inc,
priority=priority, important=important, mblogin=mblogin, refresh=refresh)
def get_track_by_id(self, trackid, handler, inc=[],
priority=False, important=False, mblogin=False, refresh=False):
return self._get_by_id('recording', trackid, handler, inc,
priority=priority, important=important, mblogin=mblogin, refresh=refresh)
def lookup_discid(self, discid, handler, priority=True, important=True, refresh=False):
inc = ['artist-credits', 'labels']
return self._get_by_id('discid', discid, handler, inc, params=["cdstubs=no"],
priority=priority, important=important, refresh=refresh)
def _find(self, entitytype, handler, kwargs):
host = config.setting["server_host"]
port = config.setting["server_port"]
filters = []
query = []
for name, value in kwargs.items():
if name == 'limit':
filters.append((name, str(value)))
else:
value = _escape_lucene_query(value).strip().lower()
if value:
query.append('%s:(%s)' % (name, value))
if query:
filters.append(('query', ' '.join(query)))
params = []
for name, value in filters:
value = QUrl.toPercentEncoding(unicode(value))
params.append('%s=%s' % (str(name), value))
path = "/ws/2/%s/?%s" % (entitytype, "&".join(params))
return self.get(host, port, path, handler)
def find_releases(self, handler, **kwargs):
return self._find('release', handler, kwargs)
def find_tracks(self, handler, **kwargs):
return self._find('recording', handler, kwargs)
def _browse(self, entitytype, handler, kwargs, inc=[], priority=False, important=False):
host = config.setting["server_host"]
port = config.setting["server_port"]
params = "&".join(["%s=%s" % (k, v) for k, v in kwargs.items()])
path = "/ws/2/%s?%s&inc=%s" % (entitytype, params, "+".join(inc))
return self.get(host, port, path, handler, priority=priority, important=important)
def browse_releases(self, handler, priority=True, important=True, **kwargs):
inc = ["media", "labels"]
return self._browse("release", handler, kwargs, inc, priority=priority, important=important)
def submit_ratings(self, ratings, handler):
host = config.setting['server_host']
port = config.setting['server_port']
path = '/ws/2/rating/?client=' + CLIENT_STRING
recordings = (''.join(['<recording id="%s"><user-rating>%s</user-rating></recording>' %
(i[1], j*20) for i, j in ratings.items() if i[0] == 'recording']))
data = _wrap_xml_metadata('<recording-list>%s</recording-list>' % recordings)
return self.post(host, port, path, data, handler, priority=True)
def _encode_acoustid_args(self, args):
filters = []
args['client'] = ACOUSTID_KEY
args['clientversion'] = PICARD_VERSION_STR
args['format'] = 'xml'
for name, value in args.items():
value = str(QUrl.toPercentEncoding(value))
filters.append('%s=%s' % (str(name), value))
return '&'.join(filters)
def query_acoustid(self, handler, **args):
host, port = ACOUSTID_HOST, ACOUSTID_PORT
body = self._encode_acoustid_args(args)
return self.post(host, port, '/v2/lookup', body, handler, priority=False, important=False, mblogin=False)
def submit_acoustid_fingerprints(self, submissions, handler):
args = {'user': config.setting["acoustid_apikey"]}
for i, submission in enumerate(submissions):
args['fingerprint.%d' % i] = str(submission.fingerprint)
args['duration.%d' % i] = str(submission.duration)
args['mbid.%d' % i] = str(submission.recordingid)
if submission.puid:
args['puid.%d' % i] = str(submission.puid)
host, port = ACOUSTID_HOST, ACOUSTID_PORT
body = self._encode_acoustid_args(args)
return self.post(host, port, '/v2/submit', body, handler, priority=True, important=False, mblogin=False)
def download(self, host, port, path, handler, priority=False,
important=False, cacheloadcontrol=None, refresh=False):
return self.get(host, port, path, handler, xml=False, priority=priority,
important=important, cacheloadcontrol=cacheloadcontrol, refresh=refresh)
def get_collection(self, id, handler, limit=100, offset=0):
host, port = config.setting['server_host'], config.setting['server_port']
path = "/ws/2/collection"
if id is not None:
inc = ["releases", "artist-credits", "media"]
path += "/%s/releases?inc=%s&limit=%d&offset=%d" % (id, "+".join(inc), limit, offset)
return self.get(host, port, path, handler, priority=True, important=True, mblogin=True)
def get_collection_list(self, handler):
return self.get_collection(None, handler)
def _collection_request(self, id, releases):
while releases:
ids = ";".join(releases if len(releases) <= 400 else releases[:400])
releases = releases[400:]
yield "/ws/2/collection/%s/releases/%s?client=%s" % (id, ids, CLIENT_STRING)
def put_to_collection(self, id, releases, handler):
host, port = config.setting['server_host'], config.setting['server_port']
for path in self._collection_request(id, releases):
self.put(host, port, path, "", handler)
def delete_from_collection(self, id, releases, handler):
host, port = config.setting['server_host'], config.setting['server_port']
for path in self._collection_request(id, releases):
self.delete(host, port, path, handler)
class WebService(QtCore.QObject):
PARSERS = dict()
def __init__(self, parent=None):
super().__init__(parent)
self.manager = QtNetwork.QNetworkAccessManager()
self.oauth_manager = OAuthManager(self)
self.set_cache()
self.setup_proxy()
self.manager.finished.connect(self._process_reply)
self._request_methods = {
"GET": self.manager.get,
"POST": self.manager.post,
"PUT": self.manager.put,
"DELETE": self.manager.deleteResource
}
self._init_queues()
self._init_timers()
def _init_queues(self):
self._active_requests = {}
self._queues = defaultdict(lambda: defaultdict(deque))
self.num_pending_web_requests = 0
self._last_num_pending_web_requests = -1
def _init_timers(self):
self._timer_run_next_task = QtCore.QTimer(self)
self._timer_run_next_task.setSingleShot(True)
self._timer_run_next_task.timeout.connect(self._run_next_task)
self._timer_count_pending_requests = QtCore.QTimer(self)
self._timer_count_pending_requests.setSingleShot(True)
self._timer_count_pending_requests.timeout.connect(self._count_pending_requests)
def set_cache(self, cache_size_in_mb=100):
cache = QtNetwork.QNetworkDiskCache()
location = QStandardPaths.writableLocation(QStandardPaths.CacheLocation)
cache.setCacheDirectory(os.path.join(location, 'picard'))
cache.setMaximumCacheSize(cache_size_in_mb * 1024 * 1024)
self.manager.setCache(cache)
log.debug("NetworkDiskCache dir: %r size: %s / %s",
cache.cacheDirectory(), cache.cacheSize(),
cache.maximumCacheSize())
def setup_proxy(self):
proxy = QtNetwork.QNetworkProxy()
if config.setting["use_proxy"]:
proxy.setType(QtNetwork.QNetworkProxy.HttpProxy)
proxy.setHostName(config.setting["proxy_server_host"])
proxy.setPort(config.setting["proxy_server_port"])
proxy.setUser(config.setting["proxy_username"])
proxy.setPassword(config.setting["proxy_password"])
self.manager.setProxy(proxy)
def _send_request(self, request, access_token=None):
hostkey = request.get_host_key()
ratecontrol.increment_requests(hostkey)
request.access_token = access_token
send = self._request_methods[request.method]
data = request.data
reply = send(request, data.encode('utf-8')) if data is not None else send(request)
self._active_requests[reply] = request
def _start_request(self, request):
if request.mblogin and request.path != "/oauth2/token":
self.oauth_manager.get_access_token(partial(self._send_request, request))
else:
self._send_request(request)
@staticmethod
def urls_equivalent(leftUrl, rightUrl):
"""
Lazy method to determine whether two QUrls are equivalent. At the moment it assumes that if ports are unset
that they are port 80 - in absence of a URL normalization function in QUrl or ability to use qHash
from QT 4.7
"""
return leftUrl.port(80) == rightUrl.port(80) and \
leftUrl.toString(QUrl.RemovePort) == rightUrl.toString(QUrl.RemovePort)
@staticmethod
def url_port(url):
if url.scheme() == 'https':
return url.port(443)
return url.port(80)
def _handle_redirect(self, reply, request, redirect):
url = request.url()
error = int(reply.error())
# merge with base url (to cover the possibility of the URL being relative)
redirect = url.resolved(redirect)
if not WebService.urls_equivalent(redirect, reply.request().url()):
log.debug("Redirect to %s requested", redirect.toString(QUrl.RemoveUserInfo))
redirect_host = redirect.host()
redirect_port = self.url_port(redirect)
redirect_query = dict(QUrlQuery(redirect).queryItems(QUrl.FullyEncoded))
redirect_path = redirect.path()
original_host = url.host()
original_port = self.url_port(url)
original_host_key = (original_host, original_port)
redirect_host_key = (redirect_host, redirect_port)
ratecontrol.copy_minimal_delay(original_host_key, redirect_host_key)
self.get(redirect_host,
redirect_port,
redirect_path,
request.handler, request.parse_response_type, priority=True, important=True,
refresh=request.refresh, queryargs=redirect_query, mblogin=request.mblogin,
cacheloadcontrol=request.attribute(QNetworkRequest.CacheLoadControlAttribute))
else:
log.error("Redirect loop: %s",
reply.request().url().toString(QUrl.RemoveUserInfo)
)
request.handler(reply.readAll(), reply, error)
def _handle_reply(self, reply, request):
hostkey = request.get_host_key()
ratecontrol.decrement_requests(hostkey)
self._timer_run_next_task.start(0)
slow_down = False
error = int(reply.error())
handler = request.handler
if error:
code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
code = int(code) if code else 0
errstr = reply.errorString()
url = reply.request().url().toString(QUrl.RemoveUserInfo)
log.error("Network request error for %s: %s (QT code %d, HTTP code %d)",
url, errstr, error, code)
if (not request.max_retries_reached()
and (code == 503
or code == 429
# Sometimes QT returns a http status code of 200 even when there
# is a service unavailable error. But it returns a QT error code
# of 403 when this happens
or error == 403
)
):
slow_down = True
retries = request.mark_for_retry()
log.debug("Retrying %s (#%d)", url, retries)
self.add_request(request)
elif handler is not None:
handler(reply.readAll(), reply, error)
slow_down = (slow_down or code >= 500)
else:
redirect = reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
fromCache = reply.attribute(QNetworkRequest.SourceIsFromCacheAttribute)
cached = ' (CACHED)' if fromCache else ''
log.debug("Received reply for %s: HTTP %d (%s) %s",
reply.request().url().toString(QUrl.RemoveUserInfo),
reply.attribute(QNetworkRequest.HttpStatusCodeAttribute),
reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute),
cached
)
if handler is not None:
# Redirect if found and not infinite
if redirect:
self._handle_redirect(reply, request, redirect)
elif request.response_parser:
try:
document = request.response_parser(reply)
except Exception as e:
url = reply.request().url().toString(QUrl.RemoveUserInfo)
log.error("Unable to parse the response for %s: %s", url, e)
document = reply.readAll()
error = e
finally:
handler(document, reply, error)
else:
handler(reply.readAll(), reply, error)
ratecontrol.adjust(hostkey, slow_down)
def _process_reply(self, reply):
try:
request = self._active_requests.pop(reply)
except KeyError:
log.error("Request not found for %s", reply.request().url().toString(QUrl.RemoveUserInfo))
return
try:
self._handle_reply(reply, request)
finally:
reply.close()
reply.deleteLater()
def get(self, host, port, path, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE,
priority=False, important=False, mblogin=False, cacheloadcontrol=None, refresh=False,
queryargs=None):
request = WSGetRequest(host, port, path, handler, parse_response_type=parse_response_type,
mblogin=mblogin, cacheloadcontrol=cacheloadcontrol, refresh=refresh,
queryargs=queryargs, priority=priority, important=important)
return self.add_request(request)
def post(self, host, port, path, data, handler, parse_response_type=DEFAULT_RESPONSE_PARSER_TYPE,
priority=False, important=False, mblogin=True, queryargs=None, request_mimetype=None):
request = WSPostRequest(host, port, path, handler, parse_response_type=parse_response_type,
data=data, mblogin=mblogin, queryargs=queryargs,
priority=priority, important=important,
request_mimetype=request_mimetype)
log.debug("POST-DATA %r", data)
return self.add_request(request)
def put(self, host, port, path, data, handler, priority=True, important=False, mblogin=True,
queryargs=None, request_mimetype=None):
request = WSPutRequest(host, port, path, handler, data=data, mblogin=mblogin,
queryargs=queryargs, priority=priority,
important=important, request_mimetype=request_mimetype)
return self.add_request(request)
def delete(self, host, port, path, handler, priority=True, important=False, mblogin=True,
queryargs=None):
request = WSDeleteRequest(host, port, path, handler, mblogin=mblogin,
queryargs=queryargs, priority=priority, important=important)
return self.add_request(request)
def download(self, host, port, path, handler, priority=False,
important=False, cacheloadcontrol=None, refresh=False,
queryargs=None):
return self.get(host, port, path, handler, parse_response_type=None,
priority=priority, important=important,
cacheloadcontrol=cacheloadcontrol, refresh=refresh,
queryargs=queryargs)
def stop(self):
for reply in list(self._active_requests):
reply.abort()
self._init_queues()
def _count_pending_requests(self):
count = len(self._active_requests)
for prio_queue in self._queues.values():
for queue in prio_queue.values():
count += len(queue)
self.num_pending_web_requests = count
if count != self._last_num_pending_web_requests:
self._last_num_pending_web_requests = count
self.tagger.tagger_stats_changed.emit()
if count:
self._timer_count_pending_requests.start(COUNT_REQUESTS_DELAY_MS)
def _run_next_task(self):
delay = sys.maxsize
for prio in sorted(self._queues.keys(), reverse=True):
prio_queue = self._queues[prio]
if not prio_queue:
del(self._queues[prio])
continue
for hostkey in sorted(prio_queue.keys(),
key=ratecontrol.current_delay):
queue = self._queues[prio][hostkey]
if not queue:
del(self._queues[prio][hostkey])
continue
wait, d = ratecontrol.get_delay_to_next_request(hostkey)
if not wait:
queue.popleft()()
if d < delay:
delay = d
if delay < sys.maxsize:
self._timer_run_next_task.start(delay)
def add_task(self, func, request):
hostkey = request.get_host_key()
prio = int(request.priority) # priority is a boolean
if request.important:
self._queues[prio][hostkey].appendleft(func)
else:
self._queues[prio][hostkey].append(func)
if not self._timer_run_next_task.isActive():
self._timer_run_next_task.start(0)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
return (hostkey, func, prio)
def add_request(self, request):
return self.add_task(partial(self._start_request, request), request)
def remove_task(self, task):
hostkey, func, prio = task
try:
self._queues[prio][hostkey].remove(func)
if not self._timer_count_pending_requests.isActive():
self._timer_count_pending_requests.start(0)
except Exception as e:
log.debug(e)
@classmethod
def add_parser(cls, response_type, mimetype, parser):
cls.PARSERS[response_type] = Parser(mimetype=mimetype, parser=parser)
@classmethod
def get_response_mimetype(cls, response_type):
if response_type in cls.PARSERS:
return cls.PARSERS[response_type].mimetype
else:
raise UnknownResponseParserError(response_type)
@classmethod
def get_response_parser(cls, response_type):
if response_type in cls.PARSERS:
return cls.PARSERS[response_type].parser
else:
raise UnknownResponseParserError(response_type)
