class OperationsManager(QObject):
__instance__ = None
@classmethod
def get_instance(cls, parent=None):
if cls.__instance__ is None:
cls.__instance__ = OperationsManager(parent=parent)
return cls.__instance__
supported_ops = [
'TestMargins', 'FindAmazonMatches', 'GetMyFeesEstimate',
'UpdateAmazonListing', 'SearchAmazon'
]
operation_complete = pyqtSignal(int)
status_message = pyqtSignal(str)
def __init__(self, parent=None):
super(OperationsManager, self).__init__(parent=parent)
self.dbsession = Session()
self.network_manager = QNetworkAccessManager(self)
self.scheduled = {}
self.processing = False
self.running = False
self._callbacks = {}
listen(self.dbsession, 'before_commit', self._before_commit_listener)
# Set up the Amazon api's and throttling managers
self.mwsapi = mws.Throttler(mws.Products(mwskeys.accesskey,
mwskeys.secretkey,
mwskeys.sellerid),
limits=mws.PRODUCTS_LIMITS,
blocking=True)
self.paapi = mws.Throttler(mws.ProductAdvertising(
pakeys.accesskey, pakeys.secretkey, pakeys.associatetag),
limits=mws.PRODUCT_ADVTERTISING_LIMITS,
blocking=True)
self.mwsapi.api.make_request = self.make_request
self.paapi.api.make_request = self.make_request
# Set the priority limits
for operation in self.mwsapi.limits:
self.mwsapi.set_priority_quota(
operation,
priority=0,
quota=self.mwsapi.limits[operation].quota_max - 2)
self.mwsapi.set_priority_quota(operation, priority=10, quota=2)
# Schedule the next operations
self.load_next()
def register_callback(self, op, callback):
self._callbacks[op] = callback
def _before_commit_listener(self, session):
"""Checks if any Operations have been added/modified in the session. If so, calls load_next()."""
for item in chain(session.new, session.dirty, session.deleted):
if isinstance(item, Operation):
break
else:
return
self.load_next()
def start(self):
"""Starts processing operations in the database."""
self.status_message.emit('Begin processing operations...')
self.running = True
self.load_next()
def stop(self):
"""Remove all pending operations from the queue."""
self.status_message.emit('Stopping all operations.')
self.running = False
for timer_id, op in self.scheduled.items():
self.killTimer(timer_id)
self.scheduled = {}
def load_next(self):
"""Load and schedule the next operation of each type listed in self.supported_ops."""
if self.running:
min_priority = 0
else:
min_priority = 1
for op_name in self.supported_ops:
eligible_ops = self.dbsession.query(Operation).\
filter(Operation.complete == False).\
filter(Operation.error == False).\
filter(Operation.operation == op_name).\
filter(Operation.priority >= min_priority)
# Get the highest-priority event older than the current time
next_op = eligible_ops.filter(Operation.scheduled <= func.now()).\
order_by(Operation.priority.desc()).\
order_by(Operation.scheduled).\
first()
# If nothing is overdue, schedule event with the nearest scheduled time
if next_op is None:
next_op = eligible_ops.order_by(Operation.scheduled.asc()).\
order_by(Operation.priority.desc()).\
first()
# If no more ops of this type, skip to the next one
if next_op is None:
continue
# Make sure only one operation of this type is scheduled at a time
for timer_id, sched_op in {
k: v
for k, v in self.scheduled.items()
}.items():
if sched_op.operation == next_op.operation:
self.killTimer(timer_id)
self.scheduled.pop(timer_id)
self.schedule_op(next_op)
def schedule_op(self, op, wait=None):
"""Get the required wait and set a timer for the given operation."""
if wait is None:
# Get next available time from the throttler
throttled_wait = self.get_wait(op.operation, op.priority)
delta = op.scheduled - arrow.utcnow().naive
scheduled_wait = delta.total_seconds()
wait = max(throttled_wait, scheduled_wait, 0)
# Schedule the timer
timer_id = self.startTimer(wait * 1000 * 1.1)
self.scheduled[timer_id] = op
def get_wait(self, operation, priority):
"""Return the wait time, in seconds, before the given api_call can be executed. If priority is negative,
return the restore rate of the operation.
"""
if operation == 'FindAmazonMatches':
return self.mwsapi.request_wait('ListMatchingProducts', priority) if priority >= 0 else \
self.mwsapi.limits['ListMatchingProducts'].restore_rate
elif operation == 'TestMargins' or operation == 'GetMyFeesEstimate':
return self.mwsapi.request_wait('GetMyFeesEstimate', priority) if priority >= 0 else \
self.mwsapi.limits['GetMyFeesEstimate'].restore_rate
elif operation == 'UpdateAmazonListing':
if priority >= 0:
return self.paapi.request_wait('ItemLookup', priority) \
+ self.mwsapi.request_wait('GetLowestOfferListingsForASIN', priority)
else:
return self.paapi.limits['ItemLookup'].restore_rate \
+ self.mwsapi.limits['GetLowestOfferListingsForASIN'].restore_rate
elif operation == 'SearchAmazon':
return self.mwsapi.request_wait('ListMatchingProducts', priority) if priority >= 0 else \
self.mwsapi.limits['ListMatchingProducts'].restore_rate
else:
if priority >= 0:
return max(self.mwsapi.request_wait(operation, priority),
self.paapi.request_wait(operation, priority))
else:
return max(getattr(self.mwsapi.limits, operation, 0),
getattr(self.paapi.limits, operation, 0))
def timerEvent(self, event):
"""Do the operation scheduled by the given timer."""
if self.processing:
event.ignore()
return
# Kill the timer and get the associated operation
timer_id = event.timerId()
self.killTimer(timer_id)
op = self.scheduled.pop(timer_id)
# If the op was deleted just load the next one
try:
op.id
except ObjectDeletedError:
self.load_next()
return
# Handle the operation
status_message = '%s: \'%s\', priority=%s' % (
time.asctime(), op.operation, op.priority)
handler = getattr(self, op.operation, None)
if handler is None:
op.error = True
op.message = 'No handler found.'
self.dbsession.commit()
status_message += ', error: No handler found.'
self.status_message.emit(status_message)
return
self.processing = True
handler(op)
self.processing = False
self.dbsession.commit()
if op.complete or op.error and op in self._callbacks:
self._callbacks[op](op)
if op.complete:
status_message += ': %s' % (op.message or 'complete.')
self.operation_complete.emit(op.id)
elif op.error:
status_message += ', error: %s' % op.message
elif not op.complete and not op.error:
status_message += ': %s' % op.message or ''
self.status_message.emit(status_message)
def is_error_response(self, reply, op):
"""Test if the response is an error, and take appropriate action."""
status_code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
error = reply.error()
if status_code == 200:
return False
# Try to parse the error response, if there is one
msg = 'Status code %s, ' % status_code
try:
parser = ErrorResponseParser(reply.readAll().data().decode())
msg += '%s - %s' % (parser.code, parser.message)
except ParseError:
msg += 'no parsable response.'
op.message = msg
# 400 usually means an invalid parameter
if status_code in [400]:
op.error = True
# 401, 403, 404 means there was a problem with the keys, signature, or address used
elif status_code in [401, 403, 404]:
self.stop()
# 500 or 503 usually means internal service error or throttling
elif status_code in [500, 503]:
self.stop()
QTimer.singleShot(5 * 60 * 1000, self.start)
# Connection reset, timed out, network session failure, etc
elif error in [
QNetworkReply.ConnectionRefusedError,
QNetworkReply.TimeoutError,
QNetworkReply.ServiceUnavailableError,
QNetworkReply.NetworkSessionFailedError,
QNetworkReply.OperationCanceledError
]:
self.stop()
QTimer.singleShot(15 * 60 * 1000, self.start)
msg += ' Connection reset, timed out, or service unavailable. Waiting 15 minutes.'
return True
def make_request(self, *args, **kwargs):
"""Make the actual network request."""
url = QUrl(kwargs['url'])
request = QNetworkRequest(url)
for k, v in kwargs['headers'].items():
request.setRawHeader(k.encode(), v.encode())
start = arrow.utcnow().timestamp
reply = self.network_manager.sendCustomRequest(
request, kwargs['method'].encode(), kwargs['data'])
while reply.isRunning():
now = arrow.utcnow().timestamp
if now - start > 30:
print('make_request timed out.')
reply.abort()
else:
QCoreApplication.processEvents(QEventLoop.AllEvents, 100)
return reply
def TestMargins(self, op):
"""Look at the potential profit margin for a listing, based on available sources. Add the listing to a list
if the margin meets a minimum threshold.
Parameters: confidence= The minimum confidence level for sources to be considered.
threshold= The minimum profit margin to be added to the list.
list= The name of the list to add matches to.
"""
amz_listing = op.listing
params = op.params
if not amz_listing.price or not amz_listing.quantity:
op.complete = True
return
min_confidence = params.get('confidence', 0)
# Get the lowest vendor cost available
vnd_unit_cost = self.dbsession.query(func.min(VendorListing.unit_price * (1 + Vendor.tax_rate + Vendor.ship_rate))).\
join(LinkedProducts, LinkedProducts.vnd_listing_id == Listing.id).\
filter(LinkedProducts.amz_listing_id == amz_listing.id,
LinkedProducts.confidence >= min_confidence,
Vendor.id == Listing.vendor_id).\
scalar()
if vnd_unit_cost is None:
op.complete = True
return
# Test the margin based solely on cost
cost = vnd_unit_cost * amz_listing.quantity
profit = amz_listing.price - cost
if profit / cost < params['threshold']:
op.complete = True
return
# Now get fees
price_point = dbhelpers.get_or_create(self.dbsession,
AmzPriceAndFees,
amz_listing_id=amz_listing.id,
price=amz_listing.price)
if price_point.fba is None:
self.GetMyFeesEstimate(op)
fba = price_point.fba or amz_listing.price * .25
prep = price_point.prep or 0
ship = price_point.ship or 0
# Calculate the margin
cost = vnd_unit_cost * amz_listing.quantity + prep + ship
profit = amz_listing.price - cost - fba
if profit / cost < params['threshold']:
op.complete = True
return
# Add to the list
dbhelpers.add_ids_to_list(self.dbsession,
listing_ids=[amz_listing.id],
list_name=params['list'])
op.complete = True
def SearchAmazon(self, op):
"""Find Amazon listings based on given search terms. Add the results to a list.
Parameters: terms= A string containing search terms.
addtolist= A list name to add results to.
"""
params = op.params
r = self.mwsapi.ListMatchingProducts(
priority=op.priority,
MarketplaceId=self.mwsapi.api.market_id(),
Query=params['terms'])
if self.is_error_response(r, op):
return
parser = ListMatchingProductsParser(r.readAll().data().decode())
for product in parser.products:
# Update the product's info
amz_listing = dbhelpers.get_or_create(self.dbsession,
AmazonListing,
sku=product.asin)
product.update(amz_listing)
# Update the product's category
category = dbhelpers.get_or_create_category(
self.dbsession, product.product_category_id,
product.product_group)
amz_listing.category = category
# Schedule an update to fill in the rest of the product info
self.dbsession.add(
Operation.UpdateAmazonListing(listing=amz_listing,
priority=op.priority))
# Add to list
if 'addtolist' in params:
add_list = dbhelpers.get_or_create(self.dbsession,
List,
name=params['addtolist'],
is_amazon=True)
dbhelpers.get_or_create(self.dbsession,
ListMembership,
list=add_list,
listing=amz_listing)
op.complete = True
def FindAmazonMatches(self, op):
"""Query Amazon for products matching a given listing.
Parameters: linkif: create a link only if the conditions are met
conf= match confidence greater than or equal to the given value.
testmargins: Create a TestMargins operation if the conditions are met.
salesrank= Maximum sales rank
list= The name of the list given to TestMargins
threshold= The minimum margin threshold given to TestMargins
"""
vnd_listing = op.listing
params = op.params
title = str(vnd_listing.title).replace(vnd_listing.brand, '').replace(
vnd_listing.model, '').strip()
query = ' '.join([vnd_listing.brand, vnd_listing.model, title])
r = self.mwsapi.ListMatchingProducts(
priority=op.priority,
MarketplaceId=self.mwsapi.api.market_id(),
Query=query)
if self.is_error_response(r, op):
return
parser = ListMatchingProductsParser(r.readAll().data().decode())
for product in parser.products:
# Update the product info
amz_listing = dbhelpers.get_or_create(self.dbsession,
AmazonListing,
sku=product.asin)
product.update(amz_listing)
# Update the product category
category = dbhelpers.get_or_create_category(
self.dbsession, product.product_category_id,
product.product_group)
amz_listing.category = category
# Create a link between the two listings. If it doesn't meet the criteria, expunge() it below.
link = dbhelpers.link_products(self.dbsession,
amz=amz_listing,
vnd=vnd_listing)
# Link criteria - it meets the threshold, or no threshold was provided
add_cond_1 = 'linkif' in params \
and 'conf' in params['linkif'] \
and link.confidence >= float(params['linkif']['conf'])
add_cond_2 = 'linkif' not in params
if add_cond_1 or add_cond_2:
# Test margins?
if 'testmargins' in params:
if 'salesrank' not in params['testmargins'] \
or (product.salesrank and product.salesrank <= params['testmargins']['salesrank']):
update_op = Operation.UpdateAmazonListing(
listing=amz_listing,
params={'testmargins': params['testmargins']},
priority=op.priority)
self.dbsession.add(update_op)
else:
self.dbsession.expunge(link)
op.message = '%s links found.' % len(vnd_listing.amz_links)
op.complete = True
def GetMyFeesEstimate(self, op):
"""Get an FBA fees estimate for the given listing.
Parameters: price= Update all price points at the given price. If not provided, update all price points
at the current listing price. Create a new price point if none exist.
fees= Set the fees to the given value. If not provided, request fees from Amazon.
"""
amz_listing = op.listing
params = op.params
if 'price' in params:
price = float(params['price'])
else:
price = amz_listing.price
if 'fees' in params:
fba_fees = float(params['fees'])
else:
feerequest = {
'MarketplaceId': self.mwsapi.api.market_id(),
'IdType': 'ASIN',
'IdValue': amz_listing.sku,
'Identifier': 'prwlr',
'IsAmazonFulfilled': 'true',
'PriceToEstimateFees.ListingPrice.CurrencyCode': 'USD',
'PriceToEstimateFees.ListingPrice.Amount': price
}
r = self.mwsapi.GetMyFeesEstimate(
FeesEstimateRequestList=[feerequest])
if self.is_error_response(r, op):
return
parser = GetMyFeesEstimateParser(r.readAll().data().decode())
fees = next(parser.get_fees())
if fees is None or fees['status'] != 'Success':
op.error = True
op.message = fees['errormessage']
return
else:
fba_fees = fees['amount']
# Get all the price points this fee request applies to
price_point = None
for price_point in self.dbsession.query(AmzPriceAndFees).filter_by(
amz_listing_id=amz_listing.id, price=price):
price_point.fba = fba_fees
if price_point is None:
self.dbsession.add(
AmzPriceAndFees(amz_listing_id=amz_listing.id,
price=price,
fba=fba_fees))
op.complete = True
def UpdateAmazonListing(self, op):
"""Update pricing, salesrank, offers, and merchant info for listing, then add to product history.
Parameters: log= Add the new product data to the log
repeat= Repeat this operation after the given number of minutes.
testmargins: Create a TestMargins operation if the criteria are met.
salesrank= Sales Rank must be below the given value.
threshold= The minimum require profit margin to add to the list.
list= The name of the list to add matches to.
"""
amz_listing = op.listing
params = op.params
r = self.paapi.ItemLookup(
priority=op.priority,
ItemId=amz_listing.sku,
ResponseGroup='OfferFull,SalesRank,ItemAttributes')
if self.is_error_response(r, op):
return
parser = ItemLookupParser(r.readAll().data().decode())
product = parser.product
if product:
product.update(amz_listing)
else:
code = parser.xpath_get('.//Error/Code')
message = parser.xpath_get('.//Error/Message')
op.error = True
op.message = 'ItemLookup failed for ASIN %s: %s. %s' % (
amz_listing.sku, code, message)
return
parser.product.update(amz_listing)
# Update the merchant
merchant = dbhelpers.get_or_create(self.dbsession,
AmazonMerchant,
name=parser.product.merchant
or 'N/A')
amz_listing.merchant = merchant
# ItemLookup tells us the current buy box price, but not including shipping. Call GetLowestOffListings
# to get the lowest offer INCLUDING shipping. This is *probably* the buy box price
r = self.mwsapi.GetLowestOfferListingsForASIN(
priority=op.priority,
MarketplaceId=self.mwsapi.api.market_id(),
ASINList=[amz_listing.sku],
ItemCondition='New')
if self.is_error_response(r, op):
return
parser = GetLowestOfferListingsForASINParser(
r.readAll().data().decode())
result = next(parser.get_product_info())
if result['error']:
op.error = True
op.message = result['message']
return
else:
amz_listing.price = max(amz_listing.price or 0, result['price']
or 0) or None
# amz_listing.hasprime = result['prime']
# Test margins?
if 'testmargins' in params:
if 'salesrank' not in params['testmargins'] \
or (amz_listing.salesrank and amz_listing.salesrank <= params['testmargins']['salesrank']):
test_op = Operation.TestMargins(listing=amz_listing,
params=params['testmargins'],
priority=op.priority)
self.dbsession.add(test_op)
if 'log' in params and params['log'] == True:
self.dbsession.add(
AmzProductHistory(amz_listing_id=amz_listing.id,
salesrank=amz_listing.salesrank,
hasprime=amz_listing.hasprime,
price=amz_listing.price,
merchant_id=amz_listing.merchant_id,
offers=amz_listing.offers,
timestamp=func.now()))
op.message = None
if 'repeat' in params and params['repeat'] > 0:
op.scheduled = datetime.utcnow() + timedelta(
minutes=params['repeat'])
else:
op.complete = True
class VAbstractNetworkClient(QObject):
"""Позволяет приложению отправлять сетевые запросы и получать на них ответы.
Вся работа с сетью должна быть инкапсулирована в его наследниках.
"""
@staticmethod
def contentTypeFrom(reply: QNetworkReply, default=None):
"""Определяет и возвращает MIME-тип содержимого (со всеми вспомогательными данными, напр., кодировкой)
из http-заголовка `Content-type` в ответе `reply`.
Если тип содержимого определить невозможно, возвращает `default`.
"""
assert reply
contentType = reply.header(QNetworkRequest.ContentTypeHeader)
if contentType:
assert isinstance(contentType, str) # TODO: Delete me!
return contentType
return default
@staticmethod
def encodingFrom(reply: QNetworkReply, default: str = "utf-8") -> str:
"""Определяет и возвращает кодировку содержимого из http-заголовка `Content-type` в ответе `reply`.
Если кодировку определить невозможно, возвращает `default`.
"""
missing = object()
contentType = VAbstractNetworkClient.contentTypeFrom(reply, missing)
if contentType is missing:
return default
try:
charset = contentType.split(";")[1]
assert "charset" in charset
encoding = charset.split("=")[1]
return encoding.strip()
except:
return default
@staticmethod
def waitForFinished(reply: QNetworkReply, timeout: int = -1):
"""Блокирует вызывающий метод на время, пока не будет завершен сетевой ответ `reply` (то есть пока
не испустится сигнал `reply.finished`), или пока не истечет `timeout` миллисекунд.
Если `timeout` меньше 0 (по умолчанию), то по данному таймеру блокировка отменяться не будет.
"""
if reply.isFinished():
return
event_loop = QEventLoop()
reply.finished.connect(event_loop.quit)
if timeout >= 0:
timer = QTimer()
timer.setInterval(timeout)
timer.setSingleShot(True)
timer.timeout.connect(event_loop.quit)
# Если блокировка отменится до истечения таймера, то при выходе из метода таймер остановится и уничтожится.
timer.start()
event_loop.exec()
reply.finished.disconnect(event_loop.quit)
networkAccessManagerChanged = pyqtSignal(QNetworkAccessManager,
arguments=['manager'])
"""Сигнал об изменении менеджера доступа к сети.
:param QNetworkAccessManager manager: Новый менеджер доступа к сети.
"""
baseUrlChanged = pyqtSignal(QUrl, arguments=['url'])
"""Сигнал об изменении базового url-а.
:param QUrl url: Новый базовый url.
"""
replyFinished = pyqtSignal(QNetworkReply, arguments=['reply'])
"""Сигнал о завершении ответа на сетевой запрос.
:param QNetworkReply reply: Завершенный сетевой запрос.
"""
def __init__(self, parent: QObject = None):
super().__init__(parent)
self.__networkAccessManager = QNetworkAccessManager(parent=self)
self.__baseUrl = QUrl()
def getNetworkAccessManager(self) -> QNetworkAccessManager:
"""Возвращает менеджер доступа к сети."""
return self.__networkAccessManager
def setNetworkAccessManager(self, manager: QNetworkAccessManager):
"""Устанавливает менеджер доступа к сети."""
assert manager
if manager is self.__networkAccessManager:
return
if self.__networkAccessManager.parent() is self:
self.__networkAccessManager.deleteLater()
self.__networkAccessManager = manager
self.networkAccessManagerChanged.emit(manager)
networkAccessManager = pyqtProperty(type=QNetworkAccessManager,
fget=getNetworkAccessManager,
fset=setNetworkAccessManager,
notify=networkAccessManagerChanged,
doc="Менеджер доступа к сети.")
def getBaseUrl(self) -> QUrl:
"""Возвращает базовый url."""
return QUrl(self.__baseUrl)
def setBaseUrl(self, url: QUrl):
"""Устанавливает базовый url."""
if url == self.__baseUrl:
return
self.__baseUrl = QUrl(url)
self.baseUrlChanged.emit(url)
baseUrl = pyqtProperty(type=QUrl,
fget=getBaseUrl,
fset=setBaseUrl,
notify=baseUrlChanged,
doc="Базовый url.")
def _connectReplySignals(self, reply: QNetworkReply):
"""Соединяет сигналы ответа с сигналами клиента."""
reply.finished.connect(lambda: self.replyFinished.emit(reply))
# TODO: Добавить сюда подключение остальных сигналов.
def _get(self, request: QNetworkRequest) -> QNetworkReply:
"""Запускает отправку GET-запроса и возвращает ответ :class:`QNetworkReply` на него."""
reply = self.__networkAccessManager.get(request)
self._connectReplySignals(reply)
return reply
def _head(self, request: QNetworkRequest) -> QNetworkReply:
"""Запускает отправку HEAD-запроса и возвращает ответ :class:`QNetworkReply` на него."""
reply = self.__networkAccessManager.head(request)
self._connectReplySignals(reply)
return reply
def _post(self, request: QNetworkRequest, data=None) -> QNetworkReply:
"""Запускает отправку POST-запроса и возвращает ответ :class:`QNetworkReply` на него.
_post(self, request: QNetworkRequest) -> QNetworkReply.
_post(self, request: QNetworkRequest, data: bytes) -> QNetworkReply.
_post(self, request: QNetworkRequest, data: bytearray) -> QNetworkReply.
_post(self, request: QNetworkRequest, data: QByteArray) -> QNetworkReply.
_post(self, request: QNetworkRequest, data: QIODevice) -> QNetworkReply.
_post(self, request: QNetworkRequest, data: QHttpMultiPart) -> QNetworkReply.
"""
if data is not None:
reply = self.__networkAccessManager.post(request, data)
else:
reply = self.__networkAccessManager.sendCustomRequest(
request, b"POST")
self._connectReplySignals(reply)
return reply
def _put(self, request: QNetworkRequest, data=None) -> QNetworkReply:
"""Запускает отправку PUT-запроса и возвращает ответ :class:`QNetworkReply` на него.
_put(self, request: QNetworkRequest) -> QNetworkReply.
_put(self, request: QNetworkRequest, data: bytes) -> QNetworkReply.
_put(self, request: QNetworkRequest, data: bytearray) -> QNetworkReply.
_put(self, request: QNetworkRequest, data: QByteArray) -> QNetworkReply.
_put(self, request: QNetworkRequest, data: QIODevice) -> QNetworkReply.
_put(self, request: QNetworkRequest, data: QHttpMultiPart) -> QNetworkReply.
"""
if data is not None:
reply = self.__networkAccessManager.put(request, data)
else:
reply = self.__networkAccessManager.sendCustomRequest(
request, b"PUT")
self._connectReplySignals(reply)
return reply
def _delete(self, request: QNetworkRequest, data=None) -> QNetworkReply:
"""Запускает отправку DELETE-запроса и возвращает ответ :class:`QNetworkReply` на него.
_delete(self, request: QNetworkRequest) -> QNetworkReply.
_delete(self, request: QNetworkRequest, data: bytes) -> QNetworkReply.
_delete(self, request: QNetworkRequest, data: bytearray) -> QNetworkReply.
_delete(self, request: QNetworkRequest, data: QByteArray) -> QNetworkReply.
_delete(self, request: QNetworkRequest, data: QIODevice) -> QNetworkReply.
_delete(self, request: QNetworkRequest, data: QHttpMultiPart) -> QNetworkReply.
"""
if data is not None:
reply = self.__networkAccessManager.deleteResource(request)
else:
reply = self.__networkAccessManager.sendCustomRequest(
request, b"DELETE")
self._connectReplySignals(reply)
return reply
def _sendCustomRequest(self,
request: QNetworkRequest,
verb: bytes,
data=None) -> QNetworkReply:
"""Запускает отправку пользовательского запроса и возвращает ответ :class:`QNetworkReply` на него.
_sendCustomRequest(self, request: QNetworkRequest, verb: bytes) -> QNetworkReply.
_sendCustomRequest(self, request: QNetworkRequest, verb: bytes, data: bytes) -> QNetworkReply.
_sendCustomRequest(self, request: QNetworkRequest, verb: bytes, data: bytearray) -> QNetworkReply.
_sendCustomRequest(self, request: QNetworkRequest, verb: bytes, data: QByteArray) -> QNetworkReply.
_sendCustomRequest(self, request: QNetworkRequest, verb: bytes, data: QIODevice) -> QNetworkReply.
_sendCustomRequest(self, request: QNetworkRequest, verb: bytes, data: QHttpMultiPart) -> QNetworkReply.
"""
reply = self.__networkAccessManager.sendCustomRequest(
request, verb, data)
self._connectReplySignals(reply)
return reply
class GreasemonkeyBridge(QObject):
"""Object to be exposed to greasemonkey clients in javascript."""
requestFinished = pyqtSignal(QVariant)
def __init__(self, wc, profile):
super().__init__()
self.wc = wc
self.profile = profile
# It would be nice to use the qnam from the current profile but
# that ain't an option and it looks like webengine doesn't even
# use it anymore. So we have to try copy various things ourself.
self.nam = QNetworkAccessManager(self)
self.cookiejar = CookieJarWrapper(self, self.profile.cookieStore())
self.nam.setCookieJar(self.cookiejar)
def handle_xhr_reply(self, reply, index):
ret = {}
ret['_qute_gm_request_index'] = index
ret['status'] = reply.attribute(
QNetworkRequest.HttpStatusCodeAttribute)
ret['statusText'] = reply.attribute(
QNetworkRequest.HttpReasonPhraseAttribute)
ret['responseText'] = reply.readAll()
# list of QByteArray tuples
heads = reply.rawHeaderPairs()
pyheads = [(str(h[0], encoding='ascii'), str(h[1], encoding='ascii'))
for h in heads]
for k, v in pyheads:
print("{}: {}".format(k, v))
ret['responseHeaders'] = dict(pyheads)
ret['finalUrl'] = reply.url()
self.requestFinished.emit(ret)
@pyqtSlot(QVariant, result=QVariant)
def GM_xmlhttpRequest(self, details):
# we could actually mock a XMLHttpRequest to support progress
# signals but who really uses them?
# https://wiki.greasespot.net/GM_xmlhttpRequest
# qtwebchannel.js calls JSON.stringify in QWebChannel.send() so any
# method attributes of arguments (eg {'onload':function(...){...;}) are
# stripped.
# * handle method, url, headers, data
# * figure out what headers we need to automatically set (referer, ...)
# * can we use some qt thing (page.get()?) to do ^
# * should probably check how cookies are handled
# chrome/tampermonkey sends cookies (for the requested domain,
# duh) with GM_xhr requests
# https://openuserjs.org/
# https://greasyfork.org/en/scripts
# tampermoney on chrome prompts when a script tries to do a
# cross-origin request.
print("==============================================")
print("GM_xmlhttpRequest")
print(details)
if not 'url' in details:
return
request_index = details['_qute_gm_request_index']
if not request_index:
log.greasemonkey.error(("GM_xmlhttpRequest received request "
"without nonce, skipping."))
return
if objreg.get('host-blocker').is_blocked(QUrl(details['url'])):
return
# TODO: url might be relative, need to fix on the JS side.
request = QNetworkRequest(QUrl(details['url']))
request.setOriginatingObject(self)
# The C++ docs say the default is to not follow any redirects.
request.setAttribute(QNetworkRequest.RedirectionTargetAttribute,
QNetworkRequest.NoLessSafeRedirectPolicy)
# TODO: Ensure these headers are encoded to spec if containing eg
# unicodes
if 'headers' in details:
for k, v in details['headers'].items():
# With this script: https://raw.githubusercontent.com/evazion/translate-pixiv-tags/master/translate-pixiv-tags.user.js
# One of the headers it 'X-Twitter-Polling': True, which was
# causing the below to error out because v is a bool. Not sure
# where that is coming from or what value twitter expects.
# That script is patching jquery so try with unpatched jquery
# and see what it does.
request.setRawHeader(k.encode('ascii'), str(v).encode('ascii'))
# TODO: Should we allow xhr to set user-agent?
if not request.header(QNetworkRequest.UserAgentHeader):
request.setHeader(QNetworkRequest.UserAgentHeader,
self.profile.httpUserAgent())
payload = details.get('data', None)
if payload:
# Should check encoding from content-type header?
payload = payload.encode('utf-8')
reply = self.nam.sendCustomRequest(
request,
details.get('method', 'GET').encode('ascii'), payload)
if reply.isFinished():
self.handle_xhr_reply(reply, request_index)
else:
reply.finished.connect(
functools.partial(self.handle_xhr_reply, reply, request_index))
