def get_connection(self, request): ''' Create connection for the request. ''' protocol = request.protocol_override if request.protocol_override else self.protocol target_host = request.host target_port = HTTP_PORT if protocol == 'http' else HTTPS_PORT if not self.use_httplib: import azure.http.winhttp connection = azure.http.winhttp._HTTPConnection(target_host, cert_file=self.cert_file, protocol=protocol) proxy_host = self.proxy_host proxy_port = self.proxy_port else: if self.proxy_host: proxy_host = target_host proxy_port = target_port host = self.proxy_host port = self.proxy_port else: host = target_host port = target_port if protocol == 'http': connection = HTTPConnection(host, int(port)) else: connection = HTTPSConnection(host, int(port), cert_file=self.cert_file) if self.proxy_host: headers = None if self.proxy_user and self.proxy_password: auth = base64.encodestring("{0}:{1}".format(self.proxy_user, self.proxy_password)) headers = {'Proxy-Authorization': 'Basic {0}'.format(auth)} connection.set_tunnel(proxy_host, int(proxy_port), headers) return connection
def _search_shelfari(self, connection, keywords): query = urlencode ({'Keywords': keywords}) try: connection.request('GET', '/search/books?' + query) response = connection.getresponse().read() except: try: connection.close() if self._proxy: connection = HTTPConnection(self._http_address, self._http_port) connection.set_tunnel('www.shelfari.com', 80) else: connection = HTTPConnection('www.shelfari.com') connection.request('GET', '/search/books?' + query) response = connection.getresponse().read() except: self._status = self.FAIL self._status_message = self.FAILED_COULD_NOT_CONNECT_TO_SHELFARI raise Exception(self._status_message) # check to make sure there are results if 'did not return any results' in response: return connection urlsearch = self.SHELFARI_URL_PAT.search(response) if not urlsearch: return connection self._shelfari_url = urlsearch.group(1) return connection
def http_connection(url, proxy=None, insecure=False, ssl_compression=True): """ Make an HTTPConnection or HTTPSConnection :param url: url to connect to :param proxy: proxy to connect through, if any; None by default; str of the format 'http://127.0.0.1:8888' to set one :param insecure: Allow to access servers without checking SSL certs. The server's certificate will not be verified. :param ssl_compression: Whether to enable compression at the SSL layer. If set to 'False' and the pyOpenSSL library is present an attempt to disable SSL compression will be made. This may provide a performance increase for https upload/download operations. :returns: tuple of (parsed url, connection object) :raises ClientException: Unable to handle protocol scheme """ url = encode_utf8(url) parsed = urlparse(url) proxy_parsed = urlparse(proxy) if proxy else None host = proxy_parsed if proxy else parsed.netloc if parsed.scheme == 'http': conn = HTTPConnection(host) elif parsed.scheme == 'https': conn = HTTPSConnection(host, insecure=insecure, ssl_compression=ssl_compression) else: raise ClientException('Cannot handle protocol scheme %s for url %s' % (parsed.scheme, repr(url))) def putheader_wrapper(func): @wraps(func) def putheader_escaped(key, value): func(encode_utf8(key), encode_utf8(value)) return putheader_escaped conn.putheader = putheader_wrapper(conn.putheader) def request_wrapper(func): @wraps(func) def request_escaped(method, url, body=None, headers=None): validate_headers(headers) url = encode_utf8(url) if body: body = encode_utf8(body) func(method, url, body=body, headers=headers or {}) return request_escaped conn.request = request_wrapper(conn.request) if proxy: try: # python 2.6 method conn._set_tunnel(parsed.hostname, parsed.port) except AttributeError: # python 2.7 method conn.set_tunnel(parsed.hostname, parsed.port) return parsed, conn
def get_asin(self, connection): query = urlencode({'keywords': '%s - %s' % (self._title, self._author)}) try: connection.request('GET', '/s/ref=sr_qz_back?sf=qz&rh=i%3Adigital-text%2Cn%3A154606011%2Ck%3A' + query[9:] + '&' + query, headers=self.HEADERS) response = connection.getresponse().read() except: try: connection.close() if self._proxy: connection = HTTPConnection(self._http_address, self._http_port) connection.set_tunnel('www.amazon.com', 80) else: connection = HTTPConnection('www.amazon.com') connection.request('GET', '/s/ref=sr_qz_back?sf=qz&rh=i%3Adigital-text%2Cn%3A154606011%2Ck%3A' + query[9:] + '&' + query, headers=self.HEADERS) response = connection.getresponse().read() except: self._status = self.FAIL self._status_message = self.FAILED_COULD_NOT_CONNECT_TO_AMAZON raise Exception(self._status_message) # check to make sure there are results if 'did not match any products' in response and not 'Did you mean:' in response and not 'so we searched in All Departments' in response: self._status = self.FAIL self._status_message = self.FAILED_COULD_NOT_FIND_AMAZON_PAGE raise Exception(self._status_message) soup = BeautifulSoup(response) results = soup.findAll('div', {'id': 'resultsCol'}) if not results or len(results) == 0: self._status = self.FAIL self._status_message = self.FAILED_COULD_NOT_FIND_AMAZON_PAGE raise Exception(self._status_message) for r in results: if 'Buy now with 1-Click' in str(r): asinSearch = self.AMAZON_ASIN_PAT.search(str(r)) if asinSearch: self._asin = asinSearch.group(1) mi = self._db.get_metadata(self._book_id) identifiers = mi.get_identifiers() identifiers['mobi-asin'] = self._asin mi.set_identifiers(identifiers) self._db.set_metadata(self._book_id, mi) self._book_settings.prefs['asin'] = self._asin return connection self._status = self.FAIL self._status_message = self.FAILED_COULD_NOT_FIND_AMAZON_ASIN raise Exception(self._status_message)
def get_connection(self, request): ''' Create connection for the request. ''' protocol = request.protocol_override \ if request.protocol_override else self.protocol protocol = protocol.lower() target_host = request.host target_port = HTTP_PORT if protocol == 'http' else HTTPS_PORT if self.request_session: import azure.http.requestsclient connection = azure.http.requestsclient._RequestsConnection( target_host, protocol, self.request_session, self.timeout) #TODO: proxy setup elif not self.use_httplib: import azure.http.winhttp connection = azure.http.winhttp._HTTPConnection( target_host, self.cert_file, protocol, self.timeout) proxy_host = self.proxy_host proxy_port = self.proxy_port else: if ':' in target_host: target_host, _, target_port = target_host.rpartition(':') if self.proxy_host: proxy_host = target_host proxy_port = target_port host = self.proxy_host port = self.proxy_port else: host = target_host port = target_port if protocol.lower() == 'http': connection = HTTPConnection(host, int(port), timeout=self.timeout) else: connection = HTTPSConnection(host, int(port), cert_file=self.cert_file, timeout=self.timeout) if self.proxy_host: headers = None if self.proxy_user and self.proxy_password: auth = base64.encodestring("{0}:{1}".format( self.proxy_user, self.proxy_password)) headers = {'Proxy-Authorization': 'Basic {0}'.format(auth)} connection.set_tunnel(proxy_host, int(proxy_port), headers) return connection
def get_connection(self, request): ''' Create connection for the request. ''' protocol = request.protocol_override \ if request.protocol_override else self.protocol protocol = protocol.lower() target_host = request.host target_port = HTTP_PORT if protocol == 'http' else HTTPS_PORT if self.request_session: import azure.http.requestsclient connection = azure.http.requestsclient._RequestsConnection( target_host, protocol, self.request_session, self.timeout) #TODO: proxy setup elif not self.use_httplib: import azure.http.winhttp connection = azure.http.winhttp._HTTPConnection( target_host, self.cert_file, protocol, self.timeout) proxy_host = self.proxy_host proxy_port = self.proxy_port else: if ':' in target_host: target_host, _, target_port = target_host.rpartition(':') if self.proxy_host: proxy_host = target_host proxy_port = target_port host = self.proxy_host port = self.proxy_port else: host = target_host port = target_port if protocol.lower() == 'http': connection = HTTPConnection(host, int(port), timeout=self.timeout) else: connection = HTTPSConnection( host, int(port), cert_file=self.cert_file, timeout=self.timeout) if self.proxy_host: headers = None if self.proxy_user and self.proxy_password: auth = base64.encodestring( "{0}:{1}".format(self.proxy_user, self.proxy_password)) headers = {'Proxy-Authorization': 'Basic {0}'.format(auth)} connection.set_tunnel(proxy_host, int(proxy_port), headers) return connection
def connect(self, url): try: if self.proxy_host and self.proxy_port: LOG.info('proxy host: %s', self.proxy_host) LOG.info('proxy port: %d', self.proxy_port) connection = HTTPConnection(self.proxy_host, self.proxy_port) else: connection = HTTPConnection(url) connection.set_debuglevel(self.http_debug) if self.proxy_host and self.proxy_port: connection.set_tunnel(url, port=80) connection.connect() return connection except: raise Exception('Unable to connect to %r' % url)
def get_secure_connection(host, **options): try: http_proxy = get_proxy('http_proxy') if http_proxy: conn = HTTPConnection(*http_proxy, **options) conn.set_tunnel(host, 443) else: https_proxy = get_proxy('https_proxy') if https_proxy: conn = HTTPSConnection(*https_proxy, **options) conn.set_tunnel(host, 443) else: conn = HTTPSConnection(host, timeout=5) except IOError: log("Unable to connect to https://%s." % host) raise return conn
def get_connection(self, request): """ Create connection for the request. """ protocol = request.protocol_override if request.protocol_override else self.protocol protocol = protocol.lower() target_host = request.host target_port = HTTP_PORT if protocol == "http" else HTTPS_PORT if self.request_session: from .requestsclient import _RequestsConnection connection = _RequestsConnection(target_host, protocol, self.request_session, self.timeout) proxy_host = self.proxy_host proxy_port = self.proxy_port elif not self.use_httplib: from .winhttp import _HTTPConnection connection = _HTTPConnection(target_host, self.cert_file, protocol, self.timeout) proxy_host = self.proxy_host proxy_port = self.proxy_port else: if ":" in target_host: target_host, _, target_port = target_host.rpartition(":") if self.proxy_host: proxy_host = target_host proxy_port = target_port host = self.proxy_host port = self.proxy_port else: host = target_host port = target_port if protocol.lower() == "http": connection = HTTPConnection(host, int(port), timeout=self.timeout) else: connection = HTTPSConnection(host, int(port), cert_file=self.cert_file, timeout=self.timeout) if self.proxy_host: headers = None if self.proxy_user and self.proxy_password: auth = base64.encodestring("{0}:{1}".format(self.proxy_user, self.proxy_password)) headers = {"Proxy-Authorization": "Basic {0}".format(auth)} connection.set_tunnel(proxy_host, int(proxy_port), headers) return connection
def request(method, action, param=None, **params): """ 发送请求数据 """ if param: params.update(param) URLObj = urlparse(Config.ID) params = dict((k, params[k]) for k in params if params[k] is not None) info("%s/%s : %s", URLObj.netloc, action, params) if Config.PROXY: if URLObj.netloc == "http": conn = HTTPConnection(Config.PROXY) else: conn = HTTPSConnection(Config.PROXY) conn.set_tunnel(URLObj.netloc, URLObj.port) else: if URLObj.netloc == "http": conn = HTTPConnection(URLObj.netloc, URLObj.port) else: conn = HTTPSConnection(URLObj.netloc, URLObj.port) headers = {} if method == "GET": if params: action += '?' + urlencode(params) params = "" else: headers["Content-Type"] = "application/x-www-form-urlencoded" params = urlencode(params) conn.request(method, action, params, headers) response = conn.getresponse() res = response.read().decode('utf8') conn.close() if response.status < 200 or response.status >= 300: warning('%s : error[%d]:%s', action, response.status, res) raise Exception(res) else: debug('%s : result:%s', action, res) return data
def is_up(self): ''' Test connection through an http proxy. ''' ''' is there a good webfirewall test domain? if we use goodcrypto.com people will say we're phoning home example.com is intended for docs, not live testing microsoft uses contoso.com for its own testing eff, theguardian, cnn etc. all are real domains we especially don't want to drain the coffers of someone like eff ''' test_domain = 'cnn.com' # self.namespace.log.debug('HttpProxy.is_up() {}'.format(self)) #DEBUG up = super(HttpProxy, self).is_up() # self.namespace.log.debug('HttpProxy.is_up() {} super up {}'.format(self, up)) #DEBUG if up: try: # weirdly, HTTPConnection() gets the proxy and set_tunnel() gets the destination domain # self.namespace.log.debug('HttpProxy.is_up() {} connecting'.format(self)) #DEBUG conn = HTTPConnection(self.host, self.port) # self.namespace.log.debug('HttpProxy.is_up() {} connected'.format(self)) #DEBUG conn.set_tunnel(test_domain) conn.request('GET', '/') # self.namespace.log.debug('HttpProxy.is_up() {} getting response'.format(self)) #DEBUG r1 = conn.getresponse() # self.namespace.log.debug('HttpProxy.is_up() {} got response {}'.format(self, r1.status)) #DEBUG assert r1.status == 200 assert r1.reason == 'OK' data1 = r1.read() assert len(data1) conn.close() up = True except: self.namespace.log.debug(traceback.format_exc()) up = False return up
def get_connection(self, request): ''' Create connection for the request. ''' protocol = request.protocol_override \ if request.protocol_override else self.protocol target_host = request.host target_port = HTTP_PORT if protocol == 'http' else HTTPS_PORT if not self.use_httplib: import azure.http.winhttp connection = azure.http.winhttp._HTTPConnection( target_host, cert_file=self.cert_file, protocol=protocol) proxy_host = self.proxy_host proxy_port = self.proxy_port else: if self.proxy_host: proxy_host = target_host proxy_port = target_port host = self.proxy_host port = self.proxy_port else: host = target_host port = target_port if protocol == 'http': connection = HTTPConnection(host, int(port)) else: connection = HTTPSConnection(host, int(port), cert_file=self.cert_file) if self.proxy_host: headers = None if self.proxy_user and self.proxy_password: auth = base64.encodestring("{0}:{1}".format( self.proxy_user, self.proxy_password)) headers = {'Proxy-Authorization': 'Basic {0}'.format(auth)} connection.set_tunnel(proxy_host, int(proxy_port), headers) return connection
def _new_conn(self): """ Return a fresh HTTPConnection. """ self.num_connections += 1 if self.proxyURL: headers = {} try: url = urlparse.urlparse(self.proxyURL) if url.password: logger.setConfidentialStrings(url.password) logger.debug( u"Starting new HTTP connection (%d) to %s:%d over proxy-url %s" % (self.num_connections, self.host, self.port, self.proxyURL)) conn = HTTPConnection(host=url.hostname, port=url.port) if url.username and url.password: logger.debug( u"Proxy Authentication detected, setting auth with user: '******'" % url.username) auth = "{username}:{password}".format( username=url.username, password=url.password) headers[ 'Proxy-Authorization'] = 'Basic ' + base64.base64encode( auth) conn.set_tunnel(self.host, self.port, headers) logger.debug(u"Connection established to: %s" % self.host) except Exception as error: logger.error(error) else: logger.debug(u"Starting new HTTP connection (%d) to %s:%d" % (self.num_connections, self.host, self.port)) conn = HTTPConnection(host=self.host, port=self.port) non_blocking_connect_http(conn, self.connectTimeout) logger.debug(u"Connection established to: %s" % self.host) return conn
class DynectRest(object): """ A class for interacting with the Dynect Managed DNS REST API. @ivar host: The host to connect to (defaults to api.dynect.net) @type host: C{str} @ivar port: The port to connect to (defaults to 443) @type port: C{int} @ivar ssl: A boolean indicating whether or not to use SSL encryption (defaults to True) @type ssl: C{bool} @ivar poll_incomplete: A boolean indicating whether we should continue to poll for a result if a job comes back as incomplete (defaults to True) @type poll_incomplete: C{bool} @ivar api_version: The version of the API to request (defaults to "current") @type api_version: C{str} """ def __init__(self, host='api.dynect.net', port=443, ssl=True, api_version="current", proxy_host=None, proxy_port=None): """ Basic initializer method @param host: The host to connect to @type host: C{str} @param port: The port to connect to @type port: C{int} @param ssl: A boolean indicating whether or not to use SSL encryption @type ssl: C{bool} """ self.host = host self.port = port self.ssl = ssl # Continue polling for response if a job comes back as incomplete? self.poll_incomplete = True self.verbose = False self.api_version = api_version self.content_type = "application/json" self._token = None self._conn = None self._last_response = None self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT')) # Add support for proxy connections self.proxy_host = proxy_host self.proxy_port = proxy_port self.use_proxy = False self.proxied_root = None if proxy_host is not None and proxy_port is not None: self.use_proxy = True scheme = 'https' if self.ssl else 'http' self.proxied_root = "{}://{}".format(scheme, self.host) def _debug(self, msg): """ Debug output. """ if self.verbose: sys.stderr.write(msg) def _connect(self): if self.ssl: msg = "Establishing SSL connection to %s:%s\n" % (self.host, self.port) self._debug(msg) self._conn = HTTPSConnection(self.host, self.port) else: msg = "Establishing unencrypted connection to %s:%s\n" % ( self.host, self.port) self._debug(msg) self._conn = HTTPConnection(self.host, self.port) def _proxy_connect(self): if self.ssl: msg = "Establishing SSL connection to %s:%s\n" % (self.host, self.port) self._debug(msg) self._conn = HTTPSConnection(self.proxy_host, self.proxy_port) else: msg = "Establishing SSL connection to %s:%s\n" % (self.proxy_host, self.proxy_port) self._debug(msg) self._conn = HTTPConnection(self.proxy_host, self.proxy_port) self._conn.set_tunnel(self.host, self.port) def connect(self): """ Establishes a connection to the REST API server as defined by the host, port and ssl instance variables """ if self._token: self._debug("Forcing logout from old session.\n") orig_value = self.poll_incomplete self.poll_incomplete = False self.execute('/REST/Session', 'DELETE') self.poll_incomplete = orig_value self._token = None self._conn = None if self.use_proxy: self._proxy_connect() else: self._connect() def execute(self, uri, method, args=None): """ Execute a commands against the rest server @param uri: The uri of the resource to access. /REST/ will be prepended if it is not at the beginning of the uri. @type uri: C{str} @param method: One of 'DELETE', 'GET', 'POST', or 'PUT' @type method: C{str} @param args: Any arguments to be sent as a part of the request @type args: C{dict} """ if self._conn == None: self._debug("No established connection\n") self.connect() # Make sure the command is prefixed by '/REST/' if not uri.startswith('/'): uri = '/' + uri if not uri.startswith('/REST'): uri = '/REST' + uri # Make sure the method is valid if method.upper() not in self._valid_methods: msg = "%s is not a valid HTTP method. Please use one of %s" % ( method, ", ".join(self._valid_methods)) raise ValueError(msg) # Prepare arguments if args is None: args = {} args = self.format_arguments(args) self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args)) # Send the command and deal with results self.send_command(uri, method, args) # Deal with the results response = self._conn.getresponse() body = response.read() self._last_response = response if self.poll_incomplete: response, body = self.poll_response(response, body) self._last_response = response if sys.version_info[0] == 2: ret_val = json.loads(body) elif sys.version_info[0] == 3: ret_val = json.loads(body.decode('UTF-8')) self._meta_update(uri, method, ret_val) return ret_val def _meta_update(self, uri, method, results): """ Private method, not intended for use outside the class """ # If we had a successful log in, update the token if uri.startswith('/REST/Session') and method == 'POST': if results['status'] == 'success': self._token = results['data']['token'] # Otherwise, if it's a successful logout, blank the token if uri.startswith('/REST/Session') and method == 'DELETE': if results['status'] == 'success': self._token = None def poll_response(self, response, body): """ Looks at a response from a REST command, and while indicates that the job is incomplete, poll for response """ while response.status == 307: time.sleep(1) uri = response.getheader('Location') self._debug("Polling %s\n" % uri) self.send_command(uri, "GET", '') response = self._conn.getresponse() body = response.read() return response, body def send_command(self, uri, method, args): """ Responsible for packaging up the API request and sending it to the server over the established connection @param uri: The uri of the resource to interact with @type uri: C{str} @param method: The HTTP method to use @type method: C{str} @param args: Encoded arguments to send to the server @type args: C{str} """ if '%' not in uri: uri = pathname2url(uri) self._conn.putrequest(method, uri) # Build headers headers = { 'Content-Type': self.content_type, 'API-Version': self.api_version, } if self._token is not None: headers['Auth-Token'] = self._token for key, val in headers.items(): self._conn.putheader(key, val) # Now the arguments self._conn.putheader('Content-length', '%d' % len(args)) self._conn.endheaders() if sys.version_info[0] == 2: self._conn.send(args) elif sys.version_info[0] == 3: self._conn.send(bytes(args, 'UTF-8')) def format_arguments(self, args): """ Converts the argument dictionary to the format needed to transmit the REST request. @param args: Arguments to be passed to the REST call @type args: C{dict} @return: The encoded string to send in the REST request body @rtype: C{str} """ args = json.dumps(args) return args
class XRayCreator(object): HEADERS = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/html", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0"} def __init__(self, db, book_ids, formats=[], spoilers=False, send_to_device=True, create_xray=True): self._db = db self._book_ids = book_ids self._formats = formats self._spoilers = spoilers self._send_to_device = send_to_device self._create_xray = create_xray @property def books(self): return self._books def _initialize_books(self): self._proxy = False self._http_address = None self._http_port = None http_proxy = get_proxies(debug=False).get('http', None) if http_proxy: self._proxy = True self._http_address = ':'.join(http_proxy.split(':')[:-1]) self._http_port = int(http_proxy.split(':')[-1]) self._aConnection = HTTPConnection(self._http_address, self._http_port) self._aConnection.set_tunnel('www.amazon.com', 80) self._sConnection = HTTPConnection(self._http_address, self._http_port) self._sConnection.set_tunnel('www.shelfari.com', 80) else: self._aConnection = HTTPConnection('www.amazon.com') self._sConnection = HTTPConnection('www.shelfari.com') self._books = [] for book_id in self._book_ids: self._books.append(Book(self._db, book_id, self._aConnection, self._sConnection, formats=self._formats, spoilers=self._spoilers, send_to_device=self._send_to_device, create_xray=self._create_xray, proxy=self._proxy, http_address=self._http_address, http_port=self._http_port)) self._total_not_failing = sum([1 for book in self._books if book.status is not book.FAIL]) def books_not_failing(self): for book in self._books: if book.status is not book.FAIL: yield book def get_results_create(self): self._create_completed = [] self._create_failed = [] for book in self._books: if book.status is book.FAIL: if book.title and book.author: known_info = book.title_and_author elif book.title: known_info = book.title elif book.author: known_info = 'Book by %s' % book.author elif not known_info: known_info = 'Unknown book' self._create_failed.append('%s: %s' % (known_info, book.status_message)) continue fmts_completed = [] fmts_failed = [] for info in book.format_specific_info: if info['status'] is book.FAIL: fmts_failed.append(info) else: fmts_completed.append(info['format']) if len(fmts_completed) > 0: self._create_completed.append('%s: %s' % (book.title_and_author, ', '.join(fmts_completed))) if len(fmts_failed) > 0: self._create_failed.append('%s:' % book.title_and_author) for fmt in fmts_failed: self._create_failed.append('\t%s: %s' % (fmt['format'], fmt['status_message'])) def get_results_send(self): self._send_completed = [] self._send_failed = [] for book in self._books: if book.status is book.FAIL: self._send_failed.append('%s: %s' % (book.title_and_author, book.status_message)) continue if book.format_specific_info: fmts_completed = [] fmts_failed = [] for info in book.formats_not_failing(): if info['send_status'] is book.FAIL: fmts_failed.append(info) else: fmts_completed.append(info['format']) if len(fmts_completed) > 0: self._send_completed.append('%s: %s' % (book.title_and_author, ', '.join(fmts_completed))) if len(fmts_failed) > 0: self._send_failed.append('%s:' % book.title_and_author) for fmt in fmts_failed: self._send_failed.append('\t%s: %s' % (fmt['format'], fmt['status_message'])) def create_xrays_event(self, abort, log, notifications): log('') self._initialize_books() for book_num, book in enumerate(self.books_not_failing()): if abort.isSet(): return if log: log('%s %s' % (datetime.now().strftime('%m-%d-%Y %H:%M:%S'), book.title_and_author)) self._aConnection, self._sConnection = book.create_xray_event(self._aConnection, self._sConnection, log=log, notifications=notifications, abort=abort, book_num=book_num, total=self._total_not_failing) self.get_results_create() log('\nX-Ray Creation:') if len(self._create_completed) > 0: log('\tBooks Completed:') for line in self._create_completed: log('\t\t%s' % line) if len(self._create_failed) > 0: log('\tBooks Failed:') for line in self._create_failed: log('\t\t%s' % line) if self._send_to_device: self.get_results_send() if len(self._send_completed) > 0 or len(self._send_failed) > 0: log('\nX-Ray Sending:') if len(self._send_completed) > 0: log('\tBooks Completed:') for line in self._send_completed: log('\t\t%s' % line) if len(self._send_failed) > 0: log('\tBooks Failed:') for line in self._send_failed: log('\t\t%s' % line) def send_xrays_event(self, abort, log, notifications): log('') self._initialize_books() for book_num, book in enumerate(self.books_not_failing()): if abort.isSet(): return if log: log('%s %s' % (datetime.now().strftime('%m-%d-%Y %H:%M:%S'), book.title_and_author)) self._aConnection, self._sConnection = book.send_xray_event(self._aConnection, self._sConnection, log=log, notifications=notifications, abort=abort, book_num=book_num, total=self._total_not_failing) self.get_results_send() if len(self._send_completed) > 0: log('\nBooks Completed:') for line in self._send_completed: log('\t%s' % line) if len(self._send_failed) > 0: log('\nBooks Failed:') for line in self._send_failed: log('\t%s' % line)
class DynectRest(object): """ A class for interacting with the Dynect Managed DNS REST API. @ivar host: The host to connect to (defaults to api.dynect.net) @type host: C{str} @ivar port: The port to connect to (defaults to 443) @type port: C{int} @ivar ssl: A boolean indicating whether or not to use SSL encryption (defaults to True) @type ssl: C{bool} @ivar poll_incomplete: A boolean indicating whether we should continue to poll for a result if a job comes back as incomplete (defaults to True) @type poll_incomplete: C{bool} @ivar api_version: The version of the API to request (defaults to "current") @type api_version: C{str} """ def __init__( self, host='api.dynect.net', port=443, ssl=True, api_version="current", proxy_host=None, proxy_port=None ): """ Basic initializer method @param host: The host to connect to @type host: C{str} @param port: The port to connect to @type port: C{int} @param ssl: A boolean indicating whether or not to use SSL encryption @type ssl: C{bool} """ self.host = host self.port = port self.ssl = ssl # Continue polling for response if a job comes back as incomplete? self.poll_incomplete = True self.verbose = False self.api_version = api_version self.content_type = "application/json" self._token = None self._conn = None self._last_response = None self._valid_methods = set(('DELETE', 'GET', 'POST', 'PUT')) # Add support for proxy connections self.proxy_host = proxy_host self.proxy_port = proxy_port self.use_proxy = False self.proxied_root = None if proxy_host is not None and proxy_port is not None: self.use_proxy = True scheme = 'https' if self.ssl else 'http' self.proxied_root = "{}://{}".format(scheme, self.host) def _debug(self, msg): """ Debug output. """ if self.verbose: sys.stderr.write(msg) def _connect(self): if self.ssl: msg = "Establishing SSL connection to %s:%s\n" % ( self.host, self.port ) self._debug(msg) self._conn = HTTPSConnection(self.host, self.port) else: msg = "Establishing unencrypted connection to %s:%s\n" % ( self.host, self.port ) self._debug(msg) self._conn = HTTPConnection(self.host, self.port) def _proxy_connect(self): if self.ssl: msg = "Establishing SSL connection to %s:%s\n" % ( self.host, self.port ) self._debug(msg) self._conn = HTTPSConnection(self.proxy_host, self.proxy_port) else: msg = "Establishing SSL connection to %s:%s\n" % ( self.proxy_host, self.proxy_port ) self._debug(msg) self._conn = HTTPConnection(self.proxy_host, self.proxy_port) self._conn.set_tunnel(self.host, self.port) def connect(self): """ Establishes a connection to the REST API server as defined by the host, port and ssl instance variables """ if self._token: self._debug("Forcing logout from old session.\n") orig_value = self.poll_incomplete self.poll_incomplete = False self.execute('/REST/Session', 'DELETE') self.poll_incomplete = orig_value self._token = None self._conn = None if self.use_proxy: self._proxy_connect() else: self._connect() def execute(self, uri, method, args = None): """ Execute a commands against the rest server @param uri: The uri of the resource to access. /REST/ will be prepended if it is not at the beginning of the uri. @type uri: C{str} @param method: One of 'DELETE', 'GET', 'POST', or 'PUT' @type method: C{str} @param args: Any arguments to be sent as a part of the request @type args: C{dict} """ if self._conn == None: self._debug("No established connection\n") self.connect() # Make sure the command is prefixed by '/REST/' if not uri.startswith('/'): uri = '/' + uri if not uri.startswith('/REST'): uri = '/REST' + uri # Make sure the method is valid if method.upper() not in self._valid_methods: msg = "%s is not a valid HTTP method. Please use one of %s" % ( method, ", ".join(self._valid_methods) ) raise ValueError(msg) # Prepare arguments if args is None: args = {} args = self.format_arguments(args) self._debug("uri: %s, method: %s, args: %s\n" % (uri, method, args)) # Send the command and deal with results self.send_command(uri, method, args) # Deal with the results response = self._conn.getresponse() body = response.read() self._last_response = response if self.poll_incomplete: response, body = self.poll_response(response, body) self._last_response = response if sys.version_info[0] == 2: ret_val = json.loads(body) elif sys.version_info[0] == 3: ret_val = json.loads(body.decode('UTF-8')) self._meta_update(uri, method, ret_val) return ret_val def _meta_update(self, uri, method, results): """ Private method, not intended for use outside the class """ # If we had a successful log in, update the token if uri.startswith('/REST/Session') and method == 'POST': if results['status'] == 'success': self._token = results['data']['token'] # Otherwise, if it's a successful logout, blank the token if uri.startswith('/REST/Session') and method == 'DELETE': if results['status'] == 'success': self._token = None def poll_response(self, response, body): """ Looks at a response from a REST command, and while indicates that the job is incomplete, poll for response """ while response.status == 307: time.sleep(1) uri = response.getheader('Location') self._debug("Polling %s\n" % uri) self.send_command(uri, "GET", '') response = self._conn.getresponse() body = response.read() return response, body def send_command(self, uri, method, args): """ Responsible for packaging up the API request and sending it to the server over the established connection @param uri: The uri of the resource to interact with @type uri: C{str} @param method: The HTTP method to use @type method: C{str} @param args: Encoded arguments to send to the server @type args: C{str} """ if '%' not in uri: uri = pathname2url(uri) self._conn.putrequest(method, uri) # Build headers headers = { 'Content-Type': self.content_type, 'API-Version': self.api_version, } if self._token is not None: headers['Auth-Token'] = self._token for key, val in headers.items(): self._conn.putheader(key, val) # Now the arguments self._conn.putheader('Content-length', '%d' % len(args)) self._conn.endheaders() if sys.version_info[0] == 2: self._conn.send(args) elif sys.version_info[0] == 3: self._conn.send(bytes(args, 'UTF-8')) def format_arguments(self, args): """ Converts the argument dictionary to the format needed to transmit the REST request. @param args: Arguments to be passed to the REST call @type args: C{dict} @return: The encoded string to send in the REST request body @rtype: C{str} """ args = json.dumps(args) return args
class BookSettings(object): AMAZON_ASIN_PAT = re.compile(r'data\-asin=\"([a-zA-z0-9]+)\"') SHELFARI_URL_PAT = re.compile(r'href="(.+/books/.+?)"') HEADERS = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/html", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0"} LIBRARY = current_library_path() HONORIFICS = 'mr mrs ms esq prof dr fr rev pr atty adv hon pres gov sen ofc pvt cpl sgt maj capt cmdr lt col gen' HONORIFICS = HONORIFICS.split() HONORIFICS.extend([x + '.' for x in HONORIFICS]) HONORIFICS += 'miss master sir madam lord dame lady esquire professor doctor father mother brother sister reverend pastor elder rabbi sheikh'.split() HONORIFICS += 'attorney advocate honorable president governor senator officer private corporal sargent major captain commander lieutenant colonel general'.split() RELIGIOUS_HONORIFICS = 'fr br sr rev pr' RELIGIOUS_HONORIFICS = RELIGIOUS_HONORIFICS.split() RELIGIOUS_HONORIFICS.extend([x + '.' for x in RELIGIOUS_HONORIFICS]) RELIGIOUS_HONORIFICS += 'father mother brother sister reverend pastor elder rabbi sheikh'.split() def __init__(self, db, book_id, aConnection, sConnection): self._db = db self._book_id = book_id self._aConnection = aConnection self._sConnection = sConnection book_path = self._db.field_for('path', book_id).replace('/', os.sep) self._prefs = JSONConfig(os.path.join(book_path, 'book_settings'), base_path=self.LIBRARY) self._prefs.setdefault('asin', '') self._prefs.setdefault('shelfari_url', '') self._prefs.setdefault('aliases', {}) self._prefs.commit() self._title = self._db.field_for('title', book_id) self._author = ' & '.join(self._db.field_for('authors', self._book_id)) self.asin = self._prefs['asin'] if self.asin == '': identifiers = self._db.field_for('identifiers', self._book_id) self.asin = self._db.field_for('identifiers', self._book_id)['mobi-asin'].decode('ascii') if 'mobi-asin' in identifiers.keys() else None if not self.asin: self.asin = self.get_asin() if self.asin: self._prefs['asin'] = self.asin self.shelfari_url = self._prefs['shelfari_url'] if self.shelfari_url == '': url = '' if self._prefs['asin'] != '': url = self.search_shelfari(self._prefs['asin']) if url != '' and self.title != 'Unknown' and self.author != 'Unknown': url = self.search_shelfari(self.title_and_author) if url != '': self.shelfari_url = url self._prefs['shelfari_url'] = self.shelfari_url self._aliases = self._prefs['aliases'] if len(self._aliases.keys()) == 0 and self.shelfari_url != '': self.update_aliases() self.save() @property def prefs(self): return self._prefs @property def title(self): return self._title @property def author(self): return self._author @property def title_and_author(self): return '%s - %s' % (self.title, self.author) @property def asin(self): return self._asin @asin.setter def asin(self, val): self._asin = val @property def shelfari_url(self): return self._shelfari_url @shelfari_url.setter def shelfari_url(self, val): self._shelfari_url = val @property def aliases(self): return self._aliases @aliases.setter def aliases(self, val): # 'aliases' is a string containing a comma separated list of aliases. # # Split it, remove whitespace from each element, drop empty strings (strangely, split only does this if you don't specify a separator) # # so "" -> [] "foo,bar" and " foo , bar " -> ["foo", "bar"] label, aliases = val aliases = [x.strip() for x in aliases.split(",") if x.strip()] self._aliases[label] = aliases def save(self): self._prefs['asin'] = self.asin self._prefs['shelfari_url'] = self.shelfari_url self._prefs['aliases'] = self.aliases def get_asin(self): query = urlencode({'keywords': '%s' % self.title_and_author}) try: self._aConnection.request('GET', '/s/ref=sr_qz_back?sf=qz&rh=i%3Adigital-text%2Cn%3A154606011%2Ck%3A' + query[9:] + '&' + query, headers=self.HEADERS) response = self._aConnection.getresponse().read() except: try: self._aConnection.close() if self._proxy: self._aConnection = HTTPConnection(self._http_address, self._http_port) self._aConnection.set_tunnel('www.amazon.com', 80) else: self._aConnection = HTTPConnection('www.amazon.com') self._aConnection.request('GET', '/s/ref=sr_qz_back?sf=qz&rh=i%3Adigital-text%2Cn%3A154606011%2Ck%3A' + query[9:] + '&' + query, headers=self.HEADERS) response = self._aConnection.getresponse().read() except: return None # check to make sure there are results if 'did not match any products' in response and not 'Did you mean:' in response and not 'so we searched in All Departments' in response: return None soup = BeautifulSoup(response) results = soup.findAll('div', {'id': 'resultsCol'}) if not results or len(results) == 0: return None for r in results: if 'Buy now with 1-Click' in str(r): asinSearch = self.AMAZON_ASIN_PAT.search(str(r)) if asinSearch: asin = asinSearch.group(1) mi = self._db.get_metadata(self._book_id) identifiers = mi.get_identifiers() identifiers['mobi-asin'] = asin mi.set_identifiers(identifiers) self._db.set_metadata(self._book_id, mi) return asin def search_shelfari(self, keywords): query = urlencode ({'Keywords': keywords}) try: self._sConnection.request('GET', '/search/books?' + query) response = self._sConnection.getresponse().read() except: try: self._sConnection.close() if self._proxy: self._sConnection = HTTPConnection(self._http_address, self._http_port) self._sConnection.set_tunnel('www.shelfari.com', 80) else: self._sConnection = HTTPConnection('www.shelfari.com') self._sConnection.request('GET', '/search/books?' + query) response = self._sConnection.getresponse().read() except: return None # check to make sure there are results if 'did not return any results' in response: return None urlsearch = self.SHELFARI_URL_PAT.search(response) if not urlsearch: return None return urlsearch.group(1) def update_aliases(self, overwrite=False): shelfari_parser = ShelfariParser(self.shelfari_url) shelfari_parser.get_characters() shelfari_parser.get_terms() if overwrite: self._prefs['aliases'] = {} self._aliases = {} characters = [char[1]['label'] for char in shelfari_parser.characters.items()] for char in characters: if char not in self.aliases.keys(): self.aliases = (char, '') terms = [term[1]['label'] for term in shelfari_parser.terms.items()] for term in terms: if term not in self.aliases.keys(): self.aliases = (term, '') aliases = self.auto_expand_aliases(characters) for alias, fullname in aliases.items(): self.aliases = (fullname, alias + ',' + ','.join(self.aliases[fullname])) def auto_expand_aliases(self, characters): actual_aliases = {} duplicates = [x.lower() for x in characters] for fullname in characters: aliases = self.fullname_to_possible_aliases(fullname.lower()) for alias in aliases: # if this alias has already been flagged as a duplicate, skip it if alias in duplicates: continue # check if this alias is a duplicate but isn't in the duplicates list if actual_aliases.has_key(alias): duplicates.append(alias) actual_aliases.pop(alias) continue # at this point, the alias is new -- add it to the dict with the alias as the key and fullname as the value actual_aliases[alias] = fullname return actual_aliases def fullname_to_possible_aliases(self, fullname): """ Given a full name ("{Title} ChristianName {Middle Names} {Surname}"), return a list of possible aliases ie. Title Surname, ChristianName Surname, Title ChristianName, {the full name} The returned aliases are in the order they should match """ aliases = [] parts = fullname.split() if parts[0].lower() in self.HONORIFICS: title = [] while len(parts) > 0 and parts[0].lower() in self.HONORIFICS: title.append(parts.pop(0)) title = ' '.join(title) else: title = None if len(parts) >= 2: # Assume: {Title} Firstname {Middlenames} Lastname # Already added the full form, also add Title Lastname, and for some Title Firstname surname = parts.pop() # This will cover double barrel surnames, we split on whitespace only christian_name = parts.pop(0) middlenames = parts if title: if title in self.RELIGIOUS_HONORIFICS: aliases.append("%s %s" % (title, christian_name)) else: aliases.append("%s %s" % (title, surname)) aliases.append(christian_name) aliases.append(surname) aliases.append("%s %s" % (christian_name, surname)) elif title: # Odd, but got Title Name (eg. Lord Buttsworth), so see if we can alias if len(parts) > 0: aliases.append(parts[0]) else: # We've got no title, so just a single word name. No alias needed pass return aliases
def __init__(self, db, ids, parent): QDialog.__init__(self, parent) self.resize(500,500) self._index = 0 self._book_settings = [] http_proxy = get_proxies(debug=False).get('http', None) if http_proxy: self._proxy = True self._http_address = ':'.join(http_proxy.split(':')[:-1]) self._http_port = int(http_proxy.split(':')[-1]) aConnection = HTTPConnection(self._http_address, self._http_port) aConnection.set_tunnel('www.amazon.com', 80) sConnection = HTTPConnection(self._http_address, self._http_port) sConnection.set_tunnel('www.shelfari.com', 80) else: aConnection = HTTPConnection('www.amazon.com') sConnection = HTTPConnection('www.shelfari.com') for book_id in ids: self._book_settings.append(BookSettings(db, book_id, aConnection, sConnection)) self.v_layout = QVBoxLayout(self) self.setWindowTitle('title - author') # add asin and shelfari url text boxes self.asin_layout = QHBoxLayout(None) self.asin_label = QLabel('ASIN:') self.asin_label.setFixedWidth(75) self.asin_edit = QLineEdit('') self.asin_edit.textEdited.connect(self.edit_asin) self.asin_layout.addWidget(self.asin_label) self.asin_layout.addWidget(self.asin_edit) self.v_layout.addLayout(self.asin_layout) self.shelfari_layout = QHBoxLayout(None) self.shelfari_url = QLabel('Shelfari URL:') self.shelfari_url.setFixedWidth(75) self.shelfari_url_edit = QLineEdit('') self.shelfari_url_edit.textEdited.connect(self.edit_shelfari_url) self.shelfari_layout.addWidget(self.shelfari_url) self.shelfari_layout.addWidget(self.shelfari_url_edit) self.v_layout.addLayout(self.shelfari_layout) self.update_buttons_layout = QHBoxLayout(None) self.update_asin_button = QPushButton('Search for ASIN') self.update_asin_button.setFixedWidth(150) self.update_asin_button.clicked.connect(self.search_for_asin) self.update_buttons_layout.addWidget(self.update_asin_button) self.update_shelfari_url_button = QPushButton('Search for Shelfari URL') self.update_shelfari_url_button.setFixedWidth(150) self.update_shelfari_url_button.clicked.connect(self.search_for_shelfari_url) self.update_buttons_layout.addWidget(self.update_shelfari_url_button) self.update_aliases_button = QPushButton('Update Aliases from URL') self.update_aliases_button.setFixedWidth(150) self.update_aliases_button.clicked.connect(self.update_aliases) self.update_buttons_layout.addWidget(self.update_aliases_button) self.v_layout.addLayout(self.update_buttons_layout) # add scrollable area for aliases self.aliases_label = QLabel('Aliases:') self.v_layout.addWidget(self.aliases_label) self.scroll_area = QScrollArea() self.v_layout.addWidget(self.scroll_area) # add status box self.status = QLabel('') self.v_layout.addWidget(self.status) # add previous, ok, cancel, and next buttons self.buttons_layout = QHBoxLayout(None) self.buttons_layout.setAlignment(Qt.AlignRight) if len(ids) > 1: self.previous_button = QPushButton("Previous") self.previous_button.setEnabled(False) self.previous_button.setFixedWidth(100) self.previous_button.clicked.connect(self.previous) self.buttons_layout.addWidget(self.previous_button) self.OK_button = QPushButton("OK") self.OK_button.setFixedWidth(100) self.OK_button.clicked.connect(self.ok) self.buttons_layout.addWidget(self.OK_button) self.cancel_button = QPushButton("Cancel") self.cancel_button.setFixedWidth(100) self.cancel_button.clicked.connect(self.cancel) self.buttons_layout.addWidget(self.cancel_button) if len(ids) > 1: self.next_button = QPushButton("Next") self.next_button.setFixedWidth(100) self.next_button.clicked.connect(self.next) self.buttons_layout.addWidget(self.next_button) self.v_layout.addLayout(self.buttons_layout) self.setLayout(self.v_layout) self.show_book_prefs() self.show()