def get_connection(self, request):
''' Create connection for the request. '''
protocol = request.protocol_override if request.protocol_override else self.protocol
target_host = request.host
target_port = HTTP_PORT if protocol == 'http' else HTTPS_PORT
if not self.use_httplib:
import azure.http.winhttp
connection = azure.http.winhttp._HTTPConnection(target_host, cert_file=self.cert_file, protocol=protocol)
proxy_host = self.proxy_host
proxy_port = self.proxy_port
else:
if self.proxy_host:
proxy_host = target_host
proxy_port = target_port
host = self.proxy_host
port = self.proxy_port
else:
host = target_host
port = target_port
if protocol == 'http':
connection = HTTPConnection(host, int(port))
else:
connection = HTTPSConnection(host, int(port), cert_file=self.cert_file)
if self.proxy_host:
headers = None
if self.proxy_user and self.proxy_password:
auth = base64.encodestring("{0}:{1}".format(self.proxy_user, self.proxy_password))
headers = {'Proxy-Authorization': 'Basic {0}'.format(auth)}
connection.set_tunnel(proxy_host, int(proxy_port), headers)
return connection
def _search_shelfari(self, connection, keywords):
query = urlencode ({'Keywords': keywords})
try:
connection.request('GET', '/search/books?' + query)
response = connection.getresponse().read()
except:
try:
connection.close()
if self._proxy:
connection = HTTPConnection(self._http_address, self._http_port)
connection.set_tunnel('www.shelfari.com', 80)
else:
connection = HTTPConnection('www.shelfari.com')
connection.request('GET', '/search/books?' + query)
response = connection.getresponse().read()
except:
self._status = self.FAIL
self._status_message = self.FAILED_COULD_NOT_CONNECT_TO_SHELFARI
raise Exception(self._status_message)
# check to make sure there are results
if 'did not return any results' in response:
return connection
urlsearch = self.SHELFARI_URL_PAT.search(response)
if not urlsearch:
return connection
self._shelfari_url = urlsearch.group(1)
return connection
def http_connection(url, proxy=None, insecure=False, ssl_compression=True):
"""
Make an HTTPConnection or HTTPSConnection
:param url: url to connect to
:param proxy: proxy to connect through, if any; None by default; str of the
format 'http://127.0.0.1:8888' to set one
:param insecure: Allow to access servers without checking SSL certs.
The server's certificate will not be verified.
:param ssl_compression: Whether to enable compression at the SSL layer.
If set to 'False' and the pyOpenSSL library is
present an attempt to disable SSL compression
will be made. This may provide a performance
increase for https upload/download operations.
:returns: tuple of (parsed url, connection object)
:raises ClientException: Unable to handle protocol scheme
"""
url = encode_utf8(url)
parsed = urlparse(url)
proxy_parsed = urlparse(proxy) if proxy else None
host = proxy_parsed if proxy else parsed.netloc
if parsed.scheme == 'http':
conn = HTTPConnection(host)
elif parsed.scheme == 'https':
conn = HTTPSConnection(host,
insecure=insecure,
ssl_compression=ssl_compression)
else:
raise ClientException('Cannot handle protocol scheme %s for url %s' %
(parsed.scheme, repr(url)))
def putheader_wrapper(func):
@wraps(func)
def putheader_escaped(key, value):
func(encode_utf8(key), encode_utf8(value))
return putheader_escaped
conn.putheader = putheader_wrapper(conn.putheader)
def request_wrapper(func):
@wraps(func)
def request_escaped(method, url, body=None, headers=None):
validate_headers(headers)
url = encode_utf8(url)
if body:
body = encode_utf8(body)
func(method, url, body=body, headers=headers or {})
return request_escaped
conn.request = request_wrapper(conn.request)
if proxy:
try:
# python 2.6 method
conn._set_tunnel(parsed.hostname, parsed.port)
except AttributeError:
# python 2.7 method
conn.set_tunnel(parsed.hostname, parsed.port)
return parsed, conn
def get_asin(self, connection):
query = urlencode({'keywords': '%s - %s' % (self._title, self._author)})
try:
connection.request('GET', '/s/ref=sr_qz_back?sf=qz&rh=i%3Adigital-text%2Cn%3A154606011%2Ck%3A' + query[9:] + '&' + query, headers=self.HEADERS)
response = connection.getresponse().read()
except:
try:
connection.close()
if self._proxy:
connection = HTTPConnection(self._http_address, self._http_port)
connection.set_tunnel('www.amazon.com', 80)
else:
connection = HTTPConnection('www.amazon.com')
connection.request('GET', '/s/ref=sr_qz_back?sf=qz&rh=i%3Adigital-text%2Cn%3A154606011%2Ck%3A' + query[9:] + '&' + query, headers=self.HEADERS)
response = connection.getresponse().read()
except:
self._status = self.FAIL
self._status_message = self.FAILED_COULD_NOT_CONNECT_TO_AMAZON
raise Exception(self._status_message)
# check to make sure there are results
if 'did not match any products' in response and not 'Did you mean:' in response and not 'so we searched in All Departments' in response:
self._status = self.FAIL
self._status_message = self.FAILED_COULD_NOT_FIND_AMAZON_PAGE
raise Exception(self._status_message)
soup = BeautifulSoup(response)
results = soup.findAll('div', {'id': 'resultsCol'})
if not results or len(results) == 0:
self._status = self.FAIL
self._status_message = self.FAILED_COULD_NOT_FIND_AMAZON_PAGE
raise Exception(self._status_message)
for r in results:
if 'Buy now with 1-Click' in str(r):
asinSearch = self.AMAZON_ASIN_PAT.search(str(r))
if asinSearch:
self._asin = asinSearch.group(1)
mi = self._db.get_metadata(self._book_id)
identifiers = mi.get_identifiers()
identifiers['mobi-asin'] = self._asin
mi.set_identifiers(identifiers)
self._db.set_metadata(self._book_id, mi)
self._book_settings.prefs['asin'] = self._asin
return connection
self._status = self.FAIL
self._status_message = self.FAILED_COULD_NOT_FIND_AMAZON_ASIN
raise Exception(self._status_message)
def get_connection(self, request):
''' Create connection for the request. '''
protocol = request.protocol_override \
if request.protocol_override else self.protocol
protocol = protocol.lower()
target_host = request.host
target_port = HTTP_PORT if protocol == 'http' else HTTPS_PORT
if self.request_session:
import azure.http.requestsclient
connection = azure.http.requestsclient._RequestsConnection(
target_host, protocol, self.request_session, self.timeout)
#TODO: proxy setup
elif not self.use_httplib:
import azure.http.winhttp
connection = azure.http.winhttp._HTTPConnection(
target_host, self.cert_file, protocol, self.timeout)
proxy_host = self.proxy_host
proxy_port = self.proxy_port
else:
if ':' in target_host:
target_host, _, target_port = target_host.rpartition(':')
if self.proxy_host:
proxy_host = target_host
proxy_port = target_port
host = self.proxy_host
port = self.proxy_port
else:
host = target_host
port = target_port
if protocol.lower() == 'http':
connection = HTTPConnection(host,
int(port),
timeout=self.timeout)
else:
connection = HTTPSConnection(host,
int(port),
cert_file=self.cert_file,
timeout=self.timeout)
if self.proxy_host:
headers = None
if self.proxy_user and self.proxy_password:
auth = base64.encodestring("{0}:{1}".format(
self.proxy_user, self.proxy_password))
headers = {'Proxy-Authorization': 'Basic {0}'.format(auth)}
connection.set_tunnel(proxy_host, int(proxy_port), headers)
return connection
def get_connection(self, request):
''' Create connection for the request. '''
protocol = request.protocol_override \
if request.protocol_override else self.protocol
protocol = protocol.lower()
target_host = request.host
target_port = HTTP_PORT if protocol == 'http' else HTTPS_PORT
if self.request_session:
import azure.http.requestsclient
connection = azure.http.requestsclient._RequestsConnection(
target_host, protocol, self.request_session, self.timeout)
#TODO: proxy setup
elif not self.use_httplib:
import azure.http.winhttp
connection = azure.http.winhttp._HTTPConnection(
target_host, self.cert_file, protocol, self.timeout)
proxy_host = self.proxy_host
proxy_port = self.proxy_port
else:
if ':' in target_host:
target_host, _, target_port = target_host.rpartition(':')
if self.proxy_host:
proxy_host = target_host
proxy_port = target_port
host = self.proxy_host
port = self.proxy_port
else:
host = target_host
port = target_port
if protocol.lower() == 'http':
connection = HTTPConnection(host, int(port),
timeout=self.timeout)
else:
connection = HTTPSConnection(
host, int(port), cert_file=self.cert_file,
timeout=self.timeout)
if self.proxy_host:
headers = None
if self.proxy_user and self.proxy_password:
auth = base64.encodestring(
"{0}:{1}".format(self.proxy_user, self.proxy_password))
headers = {'Proxy-Authorization': 'Basic {0}'.format(auth)}
connection.set_tunnel(proxy_host, int(proxy_port), headers)
return connection
def connect(self, url):
try:
if self.proxy_host and self.proxy_port:
LOG.info('proxy host: %s', self.proxy_host)
LOG.info('proxy port: %d', self.proxy_port)
connection = HTTPConnection(self.proxy_host, self.proxy_port)
else:
connection = HTTPConnection(url)
connection.set_debuglevel(self.http_debug)
if self.proxy_host and self.proxy_port:
connection.set_tunnel(url, port=80)
connection.connect()
return connection
except:
raise Exception('Unable to connect to %r' % url)
def get_secure_connection(host, **options):
try:
http_proxy = get_proxy('http_proxy')
if http_proxy:
conn = HTTPConnection(*http_proxy, **options)
conn.set_tunnel(host, 443)
else:
https_proxy = get_proxy('https_proxy')
if https_proxy:
conn = HTTPSConnection(*https_proxy, **options)
conn.set_tunnel(host, 443)
else:
conn = HTTPSConnection(host, timeout=5)
except IOError:
log("Unable to connect to https://%s." % host)
raise
return conn
def get_secure_connection(host, **options):
try:
http_proxy = get_proxy('http_proxy')
if http_proxy:
conn = HTTPConnection(*http_proxy, **options)
conn.set_tunnel(host, 443)
else:
https_proxy = get_proxy('https_proxy')
if https_proxy:
conn = HTTPSConnection(*https_proxy, **options)
conn.set_tunnel(host, 443)
else:
conn = HTTPSConnection(host, timeout=5)
except IOError:
log("Unable to connect to https://%s." % host)
raise
return conn
def get_connection(self, request):
""" Create connection for the request. """
protocol = request.protocol_override if request.protocol_override else self.protocol
protocol = protocol.lower()
target_host = request.host
target_port = HTTP_PORT if protocol == "http" else HTTPS_PORT
if self.request_session:
from .requestsclient import _RequestsConnection
connection = _RequestsConnection(target_host, protocol, self.request_session, self.timeout)
proxy_host = self.proxy_host
proxy_port = self.proxy_port
elif not self.use_httplib:
from .winhttp import _HTTPConnection
connection = _HTTPConnection(target_host, self.cert_file, protocol, self.timeout)
proxy_host = self.proxy_host
proxy_port = self.proxy_port
else:
if ":" in target_host:
target_host, _, target_port = target_host.rpartition(":")
if self.proxy_host:
proxy_host = target_host
proxy_port = target_port
host = self.proxy_host
port = self.proxy_port
else:
host = target_host
port = target_port
if protocol.lower() == "http":
connection = HTTPConnection(host, int(port), timeout=self.timeout)
else:
connection = HTTPSConnection(host, int(port), cert_file=self.cert_file, timeout=self.timeout)
if self.proxy_host:
headers = None
if self.proxy_user and self.proxy_password:
auth = base64.encodestring("{0}:{1}".format(self.proxy_user, self.proxy_password))
headers = {"Proxy-Authorization": "Basic {0}".format(auth)}
connection.set_tunnel(proxy_host, int(proxy_port), headers)
return connection
def request(method, action, param=None, **params):
"""
发送请求数据
"""
if param:
params.update(param)
URLObj = urlparse(Config.ID)
params = dict((k, params[k]) for k in params if params[k] is not None)
info("%s/%s : %s", URLObj.netloc, action, params)
if Config.PROXY:
if URLObj.netloc == "http":
conn = HTTPConnection(Config.PROXY)
else:
conn = HTTPSConnection(Config.PROXY)
conn.set_tunnel(URLObj.netloc, URLObj.port)
else:
if URLObj.netloc == "http":
conn = HTTPConnection(URLObj.netloc, URLObj.port)
else:
conn = HTTPSConnection(URLObj.netloc, URLObj.port)
headers = {}
if method == "GET":
if params:
action += '?' + urlencode(params)
params = ""
else:
headers["Content-Type"] = "application/x-www-form-urlencoded"
params = urlencode(params)
conn.request(method, action, params, headers)
response = conn.getresponse()
res = response.read().decode('utf8')
conn.close()
if response.status < 200 or response.status >= 300:
warning('%s : error[%d]:%s', action, response.status, res)
raise Exception(res)
else:
debug('%s : result:%s', action, res)
return data
def is_up(self):
''' Test connection through an http proxy. '''
''' is there a good webfirewall test domain?
if we use goodcrypto.com people will say we're phoning home
example.com is intended for docs, not live testing
microsoft uses contoso.com for its own testing
eff, theguardian, cnn etc. all are real domains
we especially don't want to drain the coffers of someone like eff '''
test_domain = 'cnn.com'
# self.namespace.log.debug('HttpProxy.is_up() {}'.format(self)) #DEBUG
up = super(HttpProxy, self).is_up()
# self.namespace.log.debug('HttpProxy.is_up() {} super up {}'.format(self, up)) #DEBUG
if up:
try:
# weirdly, HTTPConnection() gets the proxy and set_tunnel() gets the destination domain
# self.namespace.log.debug('HttpProxy.is_up() {} connecting'.format(self)) #DEBUG
conn = HTTPConnection(self.host, self.port)
# self.namespace.log.debug('HttpProxy.is_up() {} connected'.format(self)) #DEBUG
conn.set_tunnel(test_domain)
conn.request('GET', '/')
# self.namespace.log.debug('HttpProxy.is_up() {} getting response'.format(self)) #DEBUG
r1 = conn.getresponse()
# self.namespace.log.debug('HttpProxy.is_up() {} got response {}'.format(self, r1.status)) #DEBUG
assert r1.status == 200
assert r1.reason == 'OK'
data1 = r1.read()
assert len(data1)
conn.close()
up = True
except:
self.namespace.log.debug(traceback.format_exc())
up = False
return up
def get_connection(self, request):
''' Create connection for the request. '''
protocol = request.protocol_override \
if request.protocol_override else self.protocol
target_host = request.host
target_port = HTTP_PORT if protocol == 'http' else HTTPS_PORT
if not self.use_httplib:
import azure.http.winhttp
connection = azure.http.winhttp._HTTPConnection(
target_host, cert_file=self.cert_file, protocol=protocol)
proxy_host = self.proxy_host
proxy_port = self.proxy_port
else:
if self.proxy_host:
proxy_host = target_host
proxy_port = target_port
host = self.proxy_host
port = self.proxy_port
else:
host = target_host
port = target_port
if protocol == 'http':
connection = HTTPConnection(host, int(port))
else:
connection = HTTPSConnection(host,
int(port),
cert_file=self.cert_file)
if self.proxy_host:
headers = None
if self.proxy_user and self.proxy_password:
auth = base64.encodestring("{0}:{1}".format(
self.proxy_user, self.proxy_password))
headers = {'Proxy-Authorization': 'Basic {0}'.format(auth)}
connection.set_tunnel(proxy_host, int(proxy_port), headers)
return connection
def _new_conn(self):
"""
Return a fresh HTTPConnection.
"""
self.num_connections += 1
if self.proxyURL:
headers = {}
try:
url = urlparse.urlparse(self.proxyURL)
if url.password:
logger.setConfidentialStrings(url.password)
logger.debug(
u"Starting new HTTP connection (%d) to %s:%d over proxy-url %s"
% (self.num_connections, self.host, self.port,
self.proxyURL))
conn = HTTPConnection(host=url.hostname, port=url.port)
if url.username and url.password:
logger.debug(
u"Proxy Authentication detected, setting auth with user: '******'"
% url.username)
auth = "{username}:{password}".format(
username=url.username, password=url.password)
headers[
'Proxy-Authorization'] = 'Basic ' + base64.base64encode(
auth)
conn.set_tunnel(self.host, self.port, headers)
logger.debug(u"Connection established to: %s" % self.host)
except Exception as error:
logger.error(error)
else:
logger.debug(u"Starting new HTTP connection (%d) to %s:%d" %
(self.num_connections, self.host, self.port))
conn = HTTPConnection(host=self.host, port=self.port)
non_blocking_connect_http(conn, self.connectTimeout)
logger.debug(u"Connection established to: %s" % self.host)
return conn
class DynectRest(object):
"""
A class for interacting with the Dynect Managed DNS REST API.
@ivar host: The host to connect to (defaults to api.dynect.net)
@type host: C{str}
@ivar port: The port to connect to (defaults to 443)
@type port: C{int}
@ivar ssl: A boolean indicating whether or not to use SSL encryption
(defaults to True)
@type ssl: C{bool}
@ivar poll_incomplete: A boolean indicating whether we should continue to
poll for a result if a job comes back as incomplete (defaults to True)
@type poll_incomplete: C{bool}
@ivar api_version: The version of the API to request (defaults to
"current")
@type api_version: C{str}
"""
def __init__(self,
host='api.dynect.net',
port=443,
ssl=True,
api_version="current",
proxy_host=None,
proxy_port=None):
"""
Basic initializer method
@param host: The host to connect to
@type host: C{str}
@param port: The port to connect to
@type port: C{int}
@param ssl: A boolean indicating whether or not to use SSL encryption
@type ssl: C{bool}
"""
self.host = host
self.port = port
self.ssl = ssl
# Continue polling for response if a job comes back as incomplete?
self.poll_incomplete = True
self.verbose = False
self.api_version = api_version
self.content_type = "application/json"
self._token = None
self._conn = None
self._last_response = None
self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT'))
# Add support for proxy connections
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.use_proxy = False
self.proxied_root = None
if proxy_host is not None and proxy_port is not None:
self.use_proxy = True
scheme = 'https' if self.ssl else 'http'
self.proxied_root = "{}://{}".format(scheme, self.host)
def _debug(self, msg):
"""
Debug output.
"""
if self.verbose:
sys.stderr.write(msg)
def _connect(self):
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (self.host,
self.port)
self._debug(msg)
self._conn = HTTPSConnection(self.host, self.port)
else:
msg = "Establishing unencrypted connection to %s:%s\n" % (
self.host, self.port)
self._debug(msg)
self._conn = HTTPConnection(self.host, self.port)
def _proxy_connect(self):
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (self.host,
self.port)
self._debug(msg)
self._conn = HTTPSConnection(self.proxy_host, self.proxy_port)
else:
msg = "Establishing SSL connection to %s:%s\n" % (self.proxy_host,
self.proxy_port)
self._debug(msg)
self._conn = HTTPConnection(self.proxy_host, self.proxy_port)
self._conn.set_tunnel(self.host, self.port)
def connect(self):
"""
Establishes a connection to the REST API server as defined by the host,
port and ssl instance variables
"""
if self._token:
self._debug("Forcing logout from old session.\n")
orig_value = self.poll_incomplete
self.poll_incomplete = False
self.execute('/REST/Session', 'DELETE')
self.poll_incomplete = orig_value
self._token = None
self._conn = None
if self.use_proxy:
self._proxy_connect()
else:
self._connect()
def execute(self, uri, method, args=None):
"""
Execute a commands against the rest server
@param uri: The uri of the resource to access. /REST/ will be prepended
if it is not at the beginning of the uri.
@type uri: C{str}
@param method: One of 'DELETE', 'GET', 'POST', or 'PUT'
@type method: C{str}
@param args: Any arguments to be sent as a part of the request
@type args: C{dict}
"""
if self._conn == None:
self._debug("No established connection\n")
self.connect()
# Make sure the command is prefixed by '/REST/'
if not uri.startswith('/'):
uri = '/' + uri
if not uri.startswith('/REST'):
uri = '/REST' + uri
# Make sure the method is valid
if method.upper() not in self._valid_methods:
msg = "%s is not a valid HTTP method. Please use one of %s" % (
method, ", ".join(self._valid_methods))
raise ValueError(msg)
# Prepare arguments
if args is None:
args = {}
args = self.format_arguments(args)
self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args))
# Send the command and deal with results
self.send_command(uri, method, args)
# Deal with the results
response = self._conn.getresponse()
body = response.read()
self._last_response = response
if self.poll_incomplete:
response, body = self.poll_response(response, body)
self._last_response = response
if sys.version_info[0] == 2:
ret_val = json.loads(body)
elif sys.version_info[0] == 3:
ret_val = json.loads(body.decode('UTF-8'))
self._meta_update(uri, method, ret_val)
return ret_val
def _meta_update(self, uri, method, results):
"""
Private method, not intended for use outside the class
"""
# If we had a successful log in, update the token
if uri.startswith('/REST/Session') and method == 'POST':
if results['status'] == 'success':
self._token = results['data']['token']
# Otherwise, if it's a successful logout, blank the token
if uri.startswith('/REST/Session') and method == 'DELETE':
if results['status'] == 'success':
self._token = None
def poll_response(self, response, body):
"""
Looks at a response from a REST command, and while indicates that the
job is incomplete, poll for response
"""
while response.status == 307:
time.sleep(1)
uri = response.getheader('Location')
self._debug("Polling %s\n" % uri)
self.send_command(uri, "GET", '')
response = self._conn.getresponse()
body = response.read()
return response, body
def send_command(self, uri, method, args):
"""
Responsible for packaging up the API request and sending it to the
server over the established connection
@param uri: The uri of the resource to interact with
@type uri: C{str}
@param method: The HTTP method to use
@type method: C{str}
@param args: Encoded arguments to send to the server
@type args: C{str}
"""
if '%' not in uri:
uri = pathname2url(uri)
self._conn.putrequest(method, uri)
# Build headers
headers = {
'Content-Type': self.content_type,
'API-Version': self.api_version,
}
if self._token is not None:
headers['Auth-Token'] = self._token
for key, val in headers.items():
self._conn.putheader(key, val)
# Now the arguments
self._conn.putheader('Content-length', '%d' % len(args))
self._conn.endheaders()
if sys.version_info[0] == 2:
self._conn.send(args)
elif sys.version_info[0] == 3:
self._conn.send(bytes(args, 'UTF-8'))
def format_arguments(self, args):
"""
Converts the argument dictionary to the format needed to transmit the
REST request.
@param args: Arguments to be passed to the REST call
@type args: C{dict}
@return: The encoded string to send in the REST request body
@rtype: C{str}
"""
args = json.dumps(args)
return args
class XRayCreator(object):
HEADERS = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/html", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0"}
def __init__(self, db, book_ids, formats=[], spoilers=False, send_to_device=True, create_xray=True):
self._db = db
self._book_ids = book_ids
self._formats = formats
self._spoilers = spoilers
self._send_to_device = send_to_device
self._create_xray = create_xray
@property
def books(self):
return self._books
def _initialize_books(self):
self._proxy = False
self._http_address = None
self._http_port = None
http_proxy = get_proxies(debug=False).get('http', None)
if http_proxy:
self._proxy = True
self._http_address = ':'.join(http_proxy.split(':')[:-1])
self._http_port = int(http_proxy.split(':')[-1])
self._aConnection = HTTPConnection(self._http_address, self._http_port)
self._aConnection.set_tunnel('www.amazon.com', 80)
self._sConnection = HTTPConnection(self._http_address, self._http_port)
self._sConnection.set_tunnel('www.shelfari.com', 80)
else:
self._aConnection = HTTPConnection('www.amazon.com')
self._sConnection = HTTPConnection('www.shelfari.com')
self._books = []
for book_id in self._book_ids:
self._books.append(Book(self._db, book_id, self._aConnection, self._sConnection, formats=self._formats, spoilers=self._spoilers,
send_to_device=self._send_to_device, create_xray=self._create_xray, proxy=self._proxy,
http_address=self._http_address, http_port=self._http_port))
self._total_not_failing = sum([1 for book in self._books if book.status is not book.FAIL])
def books_not_failing(self):
for book in self._books:
if book.status is not book.FAIL:
yield book
def get_results_create(self):
self._create_completed = []
self._create_failed = []
for book in self._books:
if book.status is book.FAIL:
if book.title and book.author:
known_info = book.title_and_author
elif book.title:
known_info = book.title
elif book.author:
known_info = 'Book by %s' % book.author
elif not known_info:
known_info = 'Unknown book'
self._create_failed.append('%s: %s' % (known_info, book.status_message))
continue
fmts_completed = []
fmts_failed = []
for info in book.format_specific_info:
if info['status'] is book.FAIL:
fmts_failed.append(info)
else:
fmts_completed.append(info['format'])
if len(fmts_completed) > 0:
self._create_completed.append('%s: %s' % (book.title_and_author, ', '.join(fmts_completed)))
if len(fmts_failed) > 0:
self._create_failed.append('%s:' % book.title_and_author)
for fmt in fmts_failed:
self._create_failed.append('\t%s: %s' % (fmt['format'], fmt['status_message']))
def get_results_send(self):
self._send_completed = []
self._send_failed = []
for book in self._books:
if book.status is book.FAIL:
self._send_failed.append('%s: %s' % (book.title_and_author, book.status_message))
continue
if book.format_specific_info:
fmts_completed = []
fmts_failed = []
for info in book.formats_not_failing():
if info['send_status'] is book.FAIL:
fmts_failed.append(info)
else:
fmts_completed.append(info['format'])
if len(fmts_completed) > 0:
self._send_completed.append('%s: %s' % (book.title_and_author, ', '.join(fmts_completed)))
if len(fmts_failed) > 0:
self._send_failed.append('%s:' % book.title_and_author)
for fmt in fmts_failed:
self._send_failed.append('\t%s: %s' % (fmt['format'], fmt['status_message']))
def create_xrays_event(self, abort, log, notifications):
log('')
self._initialize_books()
for book_num, book in enumerate(self.books_not_failing()):
if abort.isSet():
return
if log: log('%s %s' % (datetime.now().strftime('%m-%d-%Y %H:%M:%S'), book.title_and_author))
self._aConnection, self._sConnection = book.create_xray_event(self._aConnection, self._sConnection, log=log, notifications=notifications, abort=abort, book_num=book_num, total=self._total_not_failing)
self.get_results_create()
log('\nX-Ray Creation:')
if len(self._create_completed) > 0:
log('\tBooks Completed:')
for line in self._create_completed:
log('\t\t%s' % line)
if len(self._create_failed) > 0:
log('\tBooks Failed:')
for line in self._create_failed:
log('\t\t%s' % line)
if self._send_to_device:
self.get_results_send()
if len(self._send_completed) > 0 or len(self._send_failed) > 0:
log('\nX-Ray Sending:')
if len(self._send_completed) > 0:
log('\tBooks Completed:')
for line in self._send_completed:
log('\t\t%s' % line)
if len(self._send_failed) > 0:
log('\tBooks Failed:')
for line in self._send_failed:
log('\t\t%s' % line)
def send_xrays_event(self, abort, log, notifications):
log('')
self._initialize_books()
for book_num, book in enumerate(self.books_not_failing()):
if abort.isSet():
return
if log: log('%s %s' % (datetime.now().strftime('%m-%d-%Y %H:%M:%S'), book.title_and_author))
self._aConnection, self._sConnection = book.send_xray_event(self._aConnection, self._sConnection, log=log, notifications=notifications, abort=abort, book_num=book_num, total=self._total_not_failing)
self.get_results_send()
if len(self._send_completed) > 0:
log('\nBooks Completed:')
for line in self._send_completed:
log('\t%s' % line)
if len(self._send_failed) > 0:
log('\nBooks Failed:')
for line in self._send_failed:
log('\t%s' % line)
class DynectRest(object):
"""
A class for interacting with the Dynect Managed DNS REST API.
@ivar host: The host to connect to (defaults to api.dynect.net)
@type host: C{str}
@ivar port: The port to connect to (defaults to 443)
@type port: C{int}
@ivar ssl: A boolean indicating whether or not to use SSL encryption
(defaults to True)
@type ssl: C{bool}
@ivar poll_incomplete: A boolean indicating whether we should continue to
poll for a result if a job comes back as incomplete (defaults to True)
@type poll_incomplete: C{bool}
@ivar api_version: The version of the API to request (defaults to
"current")
@type api_version: C{str}
"""
def __init__(
self, host='api.dynect.net', port=443, ssl=True, api_version="current", proxy_host=None, proxy_port=None
):
"""
Basic initializer method
@param host: The host to connect to
@type host: C{str}
@param port: The port to connect to
@type port: C{int}
@param ssl: A boolean indicating whether or not to use SSL encryption
@type ssl: C{bool}
"""
self.host = host
self.port = port
self.ssl = ssl
# Continue polling for response if a job comes back as incomplete?
self.poll_incomplete = True
self.verbose = False
self.api_version = api_version
self.content_type = "application/json"
self._token = None
self._conn = None
self._last_response = None
self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT'))
# Add support for proxy connections
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.use_proxy = False
self.proxied_root = None
if proxy_host is not None and proxy_port is not None:
self.use_proxy = True
scheme = 'https' if self.ssl else 'http'
self.proxied_root = "{}://{}".format(scheme, self.host)
def _debug(self, msg):
"""
Debug output.
"""
if self.verbose:
sys.stderr.write(msg)
def _connect(self):
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (
self.host, self.port
)
self._debug(msg)
self._conn = HTTPSConnection(self.host, self.port)
else:
msg = "Establishing unencrypted connection to %s:%s\n" % (
self.host, self.port
)
self._debug(msg)
self._conn = HTTPConnection(self.host, self.port)
def _proxy_connect(self):
if self.ssl:
msg = "Establishing SSL connection to %s:%s\n" % (
self.host, self.port
)
self._debug(msg)
self._conn = HTTPSConnection(self.proxy_host, self.proxy_port)
else:
msg = "Establishing SSL connection to %s:%s\n" % (
self.proxy_host, self.proxy_port
)
self._debug(msg)
self._conn = HTTPConnection(self.proxy_host, self.proxy_port)
self._conn.set_tunnel(self.host, self.port)
def connect(self):
"""
Establishes a connection to the REST API server as defined by the host,
port and ssl instance variables
"""
if self._token:
self._debug("Forcing logout from old session.\n")
orig_value = self.poll_incomplete
self.poll_incomplete = False
self.execute('/REST/Session', 'DELETE')
self.poll_incomplete = orig_value
self._token = None
self._conn = None
if self.use_proxy:
self._proxy_connect()
else:
self._connect()
def execute(self, uri, method, args = None):
"""
Execute a commands against the rest server
@param uri: The uri of the resource to access. /REST/ will be prepended
if it is not at the beginning of the uri.
@type uri: C{str}
@param method: One of 'DELETE', 'GET', 'POST', or 'PUT'
@type method: C{str}
@param args: Any arguments to be sent as a part of the request
@type args: C{dict}
"""
if self._conn == None:
self._debug("No established connection\n")
self.connect()
# Make sure the command is prefixed by '/REST/'
if not uri.startswith('/'):
uri = '/' + uri
if not uri.startswith('/REST'):
uri = '/REST' + uri
# Make sure the method is valid
if method.upper() not in self._valid_methods:
msg = "%s is not a valid HTTP method. Please use one of %s" % (
method, ", ".join(self._valid_methods)
)
raise ValueError(msg)
# Prepare arguments
if args is None:
args = {}
args = self.format_arguments(args)
self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args))
# Send the command and deal with results
self.send_command(uri, method, args)
# Deal with the results
response = self._conn.getresponse()
body = response.read()
self._last_response = response
if self.poll_incomplete:
response, body = self.poll_response(response, body)
self._last_response = response
if sys.version_info[0] == 2:
ret_val = json.loads(body)
elif sys.version_info[0] == 3:
ret_val = json.loads(body.decode('UTF-8'))
self._meta_update(uri, method, ret_val)
return ret_val
def _meta_update(self, uri, method, results):
"""
Private method, not intended for use outside the class
"""
# If we had a successful log in, update the token
if uri.startswith('/REST/Session') and method == 'POST':
if results['status'] == 'success':
self._token = results['data']['token']
# Otherwise, if it's a successful logout, blank the token
if uri.startswith('/REST/Session') and method == 'DELETE':
if results['status'] == 'success':
self._token = None
def poll_response(self, response, body):
"""
Looks at a response from a REST command, and while indicates that the
job is incomplete, poll for response
"""
while response.status == 307:
time.sleep(1)
uri = response.getheader('Location')
self._debug("Polling %s\n" % uri)
self.send_command(uri, "GET", '')
response = self._conn.getresponse()
body = response.read()
return response, body
def send_command(self, uri, method, args):
"""
Responsible for packaging up the API request and sending it to the
server over the established connection
@param uri: The uri of the resource to interact with
@type uri: C{str}
@param method: The HTTP method to use
@type method: C{str}
@param args: Encoded arguments to send to the server
@type args: C{str}
"""
if '%' not in uri:
uri = pathname2url(uri)
self._conn.putrequest(method, uri)
# Build headers
headers = {
'Content-Type': self.content_type,
'API-Version': self.api_version,
}
if self._token is not None:
headers['Auth-Token'] = self._token
for key, val in headers.items():
self._conn.putheader(key, val)
# Now the arguments
self._conn.putheader('Content-length', '%d' % len(args))
self._conn.endheaders()
if sys.version_info[0] == 2:
self._conn.send(args)
elif sys.version_info[0] == 3:
self._conn.send(bytes(args, 'UTF-8'))
def format_arguments(self, args):
"""
Converts the argument dictionary to the format needed to transmit the
REST request.
@param args: Arguments to be passed to the REST call
@type args: C{dict}
@return: The encoded string to send in the REST request body
@rtype: C{str}
"""
args = json.dumps(args)
return args
class BookSettings(object):
AMAZON_ASIN_PAT = re.compile(r'data\-asin=\"([a-zA-z0-9]+)\"')
SHELFARI_URL_PAT = re.compile(r'href="(.+/books/.+?)"')
HEADERS = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/html", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0"}
LIBRARY = current_library_path()
HONORIFICS = 'mr mrs ms esq prof dr fr rev pr atty adv hon pres gov sen ofc pvt cpl sgt maj capt cmdr lt col gen'
HONORIFICS = HONORIFICS.split()
HONORIFICS.extend([x + '.' for x in HONORIFICS])
HONORIFICS += 'miss master sir madam lord dame lady esquire professor doctor father mother brother sister reverend pastor elder rabbi sheikh'.split()
HONORIFICS += 'attorney advocate honorable president governor senator officer private corporal sargent major captain commander lieutenant colonel general'.split()
RELIGIOUS_HONORIFICS = 'fr br sr rev pr'
RELIGIOUS_HONORIFICS = RELIGIOUS_HONORIFICS.split()
RELIGIOUS_HONORIFICS.extend([x + '.' for x in RELIGIOUS_HONORIFICS])
RELIGIOUS_HONORIFICS += 'father mother brother sister reverend pastor elder rabbi sheikh'.split()
def __init__(self, db, book_id, aConnection, sConnection):
self._db = db
self._book_id = book_id
self._aConnection = aConnection
self._sConnection = sConnection
book_path = self._db.field_for('path', book_id).replace('/', os.sep)
self._prefs = JSONConfig(os.path.join(book_path, 'book_settings'), base_path=self.LIBRARY)
self._prefs.setdefault('asin', '')
self._prefs.setdefault('shelfari_url', '')
self._prefs.setdefault('aliases', {})
self._prefs.commit()
self._title = self._db.field_for('title', book_id)
self._author = ' & '.join(self._db.field_for('authors', self._book_id))
self.asin = self._prefs['asin']
if self.asin == '':
identifiers = self._db.field_for('identifiers', self._book_id)
self.asin = self._db.field_for('identifiers', self._book_id)['mobi-asin'].decode('ascii') if 'mobi-asin' in identifiers.keys() else None
if not self.asin:
self.asin = self.get_asin()
if self.asin:
self._prefs['asin'] = self.asin
self.shelfari_url = self._prefs['shelfari_url']
if self.shelfari_url == '':
url = ''
if self._prefs['asin'] != '':
url = self.search_shelfari(self._prefs['asin'])
if url != '' and self.title != 'Unknown' and self.author != 'Unknown':
url = self.search_shelfari(self.title_and_author)
if url != '':
self.shelfari_url = url
self._prefs['shelfari_url'] = self.shelfari_url
self._aliases = self._prefs['aliases']
if len(self._aliases.keys()) == 0 and self.shelfari_url != '':
self.update_aliases()
self.save()
@property
def prefs(self):
return self._prefs
@property
def title(self):
return self._title
@property
def author(self):
return self._author
@property
def title_and_author(self):
return '%s - %s' % (self.title, self.author)
@property
def asin(self):
return self._asin
@asin.setter
def asin(self, val):
self._asin = val
@property
def shelfari_url(self):
return self._shelfari_url
@shelfari_url.setter
def shelfari_url(self, val):
self._shelfari_url = val
@property
def aliases(self):
return self._aliases
@aliases.setter
def aliases(self, val):
# 'aliases' is a string containing a comma separated list of aliases.
#
# Split it, remove whitespace from each element, drop empty strings (strangely, split only does this if you don't specify a separator)
#
# so "" -> [] "foo,bar" and " foo , bar " -> ["foo", "bar"]
label, aliases = val
aliases = [x.strip() for x in aliases.split(",") if x.strip()]
self._aliases[label] = aliases
def save(self):
self._prefs['asin'] = self.asin
self._prefs['shelfari_url'] = self.shelfari_url
self._prefs['aliases'] = self.aliases
def get_asin(self):
query = urlencode({'keywords': '%s' % self.title_and_author})
try:
self._aConnection.request('GET', '/s/ref=sr_qz_back?sf=qz&rh=i%3Adigital-text%2Cn%3A154606011%2Ck%3A' + query[9:] + '&' + query, headers=self.HEADERS)
response = self._aConnection.getresponse().read()
except:
try:
self._aConnection.close()
if self._proxy:
self._aConnection = HTTPConnection(self._http_address, self._http_port)
self._aConnection.set_tunnel('www.amazon.com', 80)
else:
self._aConnection = HTTPConnection('www.amazon.com')
self._aConnection.request('GET', '/s/ref=sr_qz_back?sf=qz&rh=i%3Adigital-text%2Cn%3A154606011%2Ck%3A' + query[9:] + '&' + query, headers=self.HEADERS)
response = self._aConnection.getresponse().read()
except:
return None
# check to make sure there are results
if 'did not match any products' in response and not 'Did you mean:' in response and not 'so we searched in All Departments' in response:
return None
soup = BeautifulSoup(response)
results = soup.findAll('div', {'id': 'resultsCol'})
if not results or len(results) == 0:
return None
for r in results:
if 'Buy now with 1-Click' in str(r):
asinSearch = self.AMAZON_ASIN_PAT.search(str(r))
if asinSearch:
asin = asinSearch.group(1)
mi = self._db.get_metadata(self._book_id)
identifiers = mi.get_identifiers()
identifiers['mobi-asin'] = asin
mi.set_identifiers(identifiers)
self._db.set_metadata(self._book_id, mi)
return asin
def search_shelfari(self, keywords):
query = urlencode ({'Keywords': keywords})
try:
self._sConnection.request('GET', '/search/books?' + query)
response = self._sConnection.getresponse().read()
except:
try:
self._sConnection.close()
if self._proxy:
self._sConnection = HTTPConnection(self._http_address, self._http_port)
self._sConnection.set_tunnel('www.shelfari.com', 80)
else:
self._sConnection = HTTPConnection('www.shelfari.com')
self._sConnection.request('GET', '/search/books?' + query)
response = self._sConnection.getresponse().read()
except:
return None
# check to make sure there are results
if 'did not return any results' in response:
return None
urlsearch = self.SHELFARI_URL_PAT.search(response)
if not urlsearch:
return None
return urlsearch.group(1)
def update_aliases(self, overwrite=False):
shelfari_parser = ShelfariParser(self.shelfari_url)
shelfari_parser.get_characters()
shelfari_parser.get_terms()
if overwrite:
self._prefs['aliases'] = {}
self._aliases = {}
characters = [char[1]['label'] for char in shelfari_parser.characters.items()]
for char in characters:
if char not in self.aliases.keys():
self.aliases = (char, '')
terms = [term[1]['label'] for term in shelfari_parser.terms.items()]
for term in terms:
if term not in self.aliases.keys():
self.aliases = (term, '')
aliases = self.auto_expand_aliases(characters)
for alias, fullname in aliases.items():
self.aliases = (fullname, alias + ',' + ','.join(self.aliases[fullname]))
def auto_expand_aliases(self, characters):
actual_aliases = {}
duplicates = [x.lower() for x in characters]
for fullname in characters:
aliases = self.fullname_to_possible_aliases(fullname.lower())
for alias in aliases:
# if this alias has already been flagged as a duplicate, skip it
if alias in duplicates:
continue
# check if this alias is a duplicate but isn't in the duplicates list
if actual_aliases.has_key(alias):
duplicates.append(alias)
actual_aliases.pop(alias)
continue
# at this point, the alias is new -- add it to the dict with the alias as the key and fullname as the value
actual_aliases[alias] = fullname
return actual_aliases
def fullname_to_possible_aliases(self, fullname):
"""
Given a full name ("{Title} ChristianName {Middle Names} {Surname}"), return a list of possible aliases
ie. Title Surname, ChristianName Surname, Title ChristianName, {the full name}
The returned aliases are in the order they should match
"""
aliases = []
parts = fullname.split()
if parts[0].lower() in self.HONORIFICS:
title = []
while len(parts) > 0 and parts[0].lower() in self.HONORIFICS:
title.append(parts.pop(0))
title = ' '.join(title)
else:
title = None
if len(parts) >= 2:
# Assume: {Title} Firstname {Middlenames} Lastname
# Already added the full form, also add Title Lastname, and for some Title Firstname
surname = parts.pop() # This will cover double barrel surnames, we split on whitespace only
christian_name = parts.pop(0)
middlenames = parts
if title:
if title in self.RELIGIOUS_HONORIFICS:
aliases.append("%s %s" % (title, christian_name))
else:
aliases.append("%s %s" % (title, surname))
aliases.append(christian_name)
aliases.append(surname)
aliases.append("%s %s" % (christian_name, surname))
elif title:
# Odd, but got Title Name (eg. Lord Buttsworth), so see if we can alias
if len(parts) > 0:
aliases.append(parts[0])
else:
# We've got no title, so just a single word name. No alias needed
pass
return aliases
def __init__(self, db, ids, parent):
QDialog.__init__(self, parent)
self.resize(500,500)
self._index = 0
self._book_settings = []
http_proxy = get_proxies(debug=False).get('http', None)
if http_proxy:
self._proxy = True
self._http_address = ':'.join(http_proxy.split(':')[:-1])
self._http_port = int(http_proxy.split(':')[-1])
aConnection = HTTPConnection(self._http_address, self._http_port)
aConnection.set_tunnel('www.amazon.com', 80)
sConnection = HTTPConnection(self._http_address, self._http_port)
sConnection.set_tunnel('www.shelfari.com', 80)
else:
aConnection = HTTPConnection('www.amazon.com')
sConnection = HTTPConnection('www.shelfari.com')
for book_id in ids:
self._book_settings.append(BookSettings(db, book_id, aConnection, sConnection))
self.v_layout = QVBoxLayout(self)
self.setWindowTitle('title - author')
# add asin and shelfari url text boxes
self.asin_layout = QHBoxLayout(None)
self.asin_label = QLabel('ASIN:')
self.asin_label.setFixedWidth(75)
self.asin_edit = QLineEdit('')
self.asin_edit.textEdited.connect(self.edit_asin)
self.asin_layout.addWidget(self.asin_label)
self.asin_layout.addWidget(self.asin_edit)
self.v_layout.addLayout(self.asin_layout)
self.shelfari_layout = QHBoxLayout(None)
self.shelfari_url = QLabel('Shelfari URL:')
self.shelfari_url.setFixedWidth(75)
self.shelfari_url_edit = QLineEdit('')
self.shelfari_url_edit.textEdited.connect(self.edit_shelfari_url)
self.shelfari_layout.addWidget(self.shelfari_url)
self.shelfari_layout.addWidget(self.shelfari_url_edit)
self.v_layout.addLayout(self.shelfari_layout)
self.update_buttons_layout = QHBoxLayout(None)
self.update_asin_button = QPushButton('Search for ASIN')
self.update_asin_button.setFixedWidth(150)
self.update_asin_button.clicked.connect(self.search_for_asin)
self.update_buttons_layout.addWidget(self.update_asin_button)
self.update_shelfari_url_button = QPushButton('Search for Shelfari URL')
self.update_shelfari_url_button.setFixedWidth(150)
self.update_shelfari_url_button.clicked.connect(self.search_for_shelfari_url)
self.update_buttons_layout.addWidget(self.update_shelfari_url_button)
self.update_aliases_button = QPushButton('Update Aliases from URL')
self.update_aliases_button.setFixedWidth(150)
self.update_aliases_button.clicked.connect(self.update_aliases)
self.update_buttons_layout.addWidget(self.update_aliases_button)
self.v_layout.addLayout(self.update_buttons_layout)
# add scrollable area for aliases
self.aliases_label = QLabel('Aliases:')
self.v_layout.addWidget(self.aliases_label)
self.scroll_area = QScrollArea()
self.v_layout.addWidget(self.scroll_area)
# add status box
self.status = QLabel('')
self.v_layout.addWidget(self.status)
# add previous, ok, cancel, and next buttons
self.buttons_layout = QHBoxLayout(None)
self.buttons_layout.setAlignment(Qt.AlignRight)
if len(ids) > 1:
self.previous_button = QPushButton("Previous")
self.previous_button.setEnabled(False)
self.previous_button.setFixedWidth(100)
self.previous_button.clicked.connect(self.previous)
self.buttons_layout.addWidget(self.previous_button)
self.OK_button = QPushButton("OK")
self.OK_button.setFixedWidth(100)
self.OK_button.clicked.connect(self.ok)
self.buttons_layout.addWidget(self.OK_button)
self.cancel_button = QPushButton("Cancel")
self.cancel_button.setFixedWidth(100)
self.cancel_button.clicked.connect(self.cancel)
self.buttons_layout.addWidget(self.cancel_button)
if len(ids) > 1:
self.next_button = QPushButton("Next")
self.next_button.setFixedWidth(100)
self.next_button.clicked.connect(self.next)
self.buttons_layout.addWidget(self.next_button)
self.v_layout.addLayout(self.buttons_layout)
self.setLayout(self.v_layout)
self.show_book_prefs()
self.show()
