def run_app(app, *, host='0.0.0.0', port=None, shutdown_timeout=60.0, ssl_context=None, print=print, backlog=128, access_log_format=None, access_log=access_logger): """Run an app locally""" if port is None: if not ssl_context: port = 8080 else: port = 8443 loop = app.loop make_handler_kwargs = dict() if access_log_format is not None: make_handler_kwargs['access_log_format'] = access_log_format handler = app.make_handler(access_log=access_log, **make_handler_kwargs) loop.run_until_complete(app.startup()) srv = loop.run_until_complete( loop.create_server(handler, host, port, ssl=ssl_context, backlog=backlog)) scheme = 'https' if ssl_context else 'http' base_url = URL('{}://localhost'.format(scheme)).with_port(port) if isinstance(host, str): urls = [base_url.with_host(host)] else: urls = [base_url.with_host(h) for h in host] print("======== Running on {} ========\n" "(Press CTRL+C to quit)".format(', '.join(str(u) for u in urls))) try: loop.run_forever() except KeyboardInterrupt: # pragma: no cover pass finally: srv.close() loop.run_until_complete(srv.wait_closed()) loop.run_until_complete(app.shutdown()) loop.run_until_complete(handler.shutdown(shutdown_timeout)) loop.run_until_complete(app.cleanup()) loop.close()
def __init__(self, host="0.0.0.0", port=8080, ssl_context=None, netmem_dict:dict=None): super().__init__() self.host = host self.port = port self.ssl_context = ssl_context self.netmem = netmem_dict # Reference to the hosting networkmemory self._app = None # type: web.Application self._handler = None # type: web.Server self._srv = None # type: asyncio.base_events.Server self._active_ws_updates_sockets = [] # type: [web.WebSocketResponse] self._active_ws_whole_sockets = [] # type: [web.WebSocketResponse] scheme = 'https' if self.ssl_context else 'http' url = URL('{}://localhost'.format(scheme)) self.url_base = url.with_host(socket.gethostname()).with_port(self.port) self.url_ws_updates = self.url_base.join(URL(WsServerConnector.WS_UPDATES)) self.url_ws_whole = self.url_base.join(URL(WsServerConnector.WS_WHOLE)) self.url_html_view = self.url_base.join(URL(WsServerConnector.HTML_VIEW)) self.log.info("{} : Hosting html view at {}".format(self, self.url_html_view)) self.log.info("{} : Hosting websocket memory updates at {}".format(self, self.url_ws_updates)) self.log.info("{} : Hosting websocket memory complete reflection at {}".format(self, self.url_ws_whole))
async def start(self): logger.debug('start') if not self.has_app(): logger.debug('does not have app') return if not self.auto_start: logger.debug('should not start automatically') return app = self.get_app() handler = app.make_handler() loop = asyncio.get_event_loop() server = loop.create_server( handler, self.host, self.port, ssl=self.ssl_context, backlog=self.backlog, ) srv, startup_res = await asyncio.gather( server, app.startup(), loop=loop, ) scheme = 'https' if self.ssl_context else 'http' url = URL('{}://localhost'.format(scheme)) url = url.with_host(self.host).with_port(self.port) logger.debug('======== Running on {} ========\n' '(Press CTRL+C to quit)'.format(url)) self.server = srv self.handler = handler
def test_with_host_non_ascii(): url = URL("http://example.com:123") url2 = url.with_host("историк.рф") assert url2.raw_host == "xn--h1aagokeh.xn--p1ai" assert url2.host == "историк.рф" assert url2.raw_authority == "xn--h1aagokeh.xn--p1ai:123" assert url2.authority == "историк.рф:123"
def test_with_host_percent_encoded(): url = URL("http://%25cf%2580%cf%80:%25cf%2580%cf%[email protected]:123") url2 = url.with_host("%cf%80.org") assert url2.raw_host == "%cf%80.org" assert url2.host == "%cf%80.org" assert url2.raw_authority == "%25cf%2580%CF%80:%25cf%2580%CF%80@%cf%80.org:123" assert url2.authority == "%cf%80π:%cf%80π@%cf%80.org:123"
async def proxy_connection(proxy: TCPProxy, amqp_url: URL, loop): url = amqp_url.with_host("localhost", ).with_port(proxy.proxy_port, ) connection = Connection(url, loop=loop) await connection.connect() try: yield connection finally: await connection.close()
def test_port_override_with_ipv6(openapi3_schema_url, simple_openapi, mocker): resp = Response() resp.status_code = 200 resp._content = json.dumps(simple_openapi).encode("utf-8") mocker.patch("requests.get", return_value=resp) url = URL(openapi3_schema_url) parts = list(map(int, url.host.split("."))) ipv6_host = "2002:{:02x}{:02x}:{:02x}{:02x}::".format(*parts) ipv6 = url.with_host("[%s]" % ipv6_host) schema = schemathesis.from_uri(str(ipv6), validate_schema=False, port=8081) operation = next(schema.get_all_operations()).ok() assert operation.base_url == "http://[2002:7f00:1::]:8081/schema.yaml"
def run(self, host='localhost', port=None, shutdown_timeout=60.0, ssl_context=None, print=print, backlog=128, access_log_format=None, access_log=None): """Run an app locally""" if port is None: if not ssl_context: port = 8080 else: port = 8443 loop = self._app.loop make_handler_kwargs = dict() if access_log_format is not None: make_handler_kwargs['access_log_format'] = access_log_format handler = self._app.make_handler(access_log=access_log, **make_handler_kwargs) loop.run_until_complete(self._app.startup()) srv = loop.run_until_complete( loop.create_server(handler, host, port, ssl=ssl_context, backlog=backlog)) scheme = 'https' if ssl_context else 'http' url = URL('{}://localhost'.format(scheme)) url = url.with_host(host).with_port(port) print("======== Running on {} ========\n" "(Press CTRL+C to quit)".format(url)) try: loop.run_forever() except KeyboardInterrupt: # pragma: no cover pass finally: print("run here") srv.close() loop.run_until_complete(srv.wait_closed()) loop.run_until_complete(self._app.shutdown()) loop.run_until_complete(handler.shutdown(shutdown_timeout)) loop.run_until_complete(self._app.cleanup()) loop.close()
def entity_picture(self) -> str: """Return the gamer pic.""" if not self.data: return None # Xbox sometimes returns a domain that uses a wrong certificate which creates issues # with loading the image. # The correct domain is images-eds-ssl which can just be replaced # to point to the correct image, with the correct domain and certificate. # We need to also remove the 'mode=Padding' query because with it, it results in an error 400. url = URL(self.data.display_pic) if url.host == "images-eds.xboxlive.com": url = url.with_host("images-eds-ssl.xboxlive.com") query = dict(url.query) query.pop("mode", None) return str(url.with_query(query))
def __init__(self, url): self.logger = logging.getLogger(__name__) endpoint = 'http://oss-cn-beijing-internal.aliyuncs.com' auth = oss2.Auth('LTAIwHXVgRkPRxcw', 'PCm6kdpeyjAG2gJz4B4C2js3TIifrc') self.bucket = oss2.Bucket(auth, endpoint, 'weegotr-statics') u = URL(url) path = u.path if not u.path.startswith('/') else u.path[1:] exists = self.bucket.object_exists(path) if not exists: url = u.with_host('exp.cdn-hotels.com') content = BytesIO(self.session.get(str(url)).content) put_result = self.bucket.put_object(path, content) if put_result.status >= 300: print({'status': put_result.status, 'text': put_result.resp}) else: print({'status': put_result.status, 'file': u.path})
async def handle(request, session, redis): # Lookup the backend on redis. key = '{}:{}'.format(request.url.host, request.url.port) value = await redis.get(key) if not value: raise web.HTTPNotFound backend_host, backend_port = value.decode().split(':') backend_port = int(backend_port) url = request.url.with_host(backend_host).with_port(backend_port) # Send an equivalent request to the backend. headers = request.headers.copy() headers['Host'] = '{}:{}'.format(backend_host, backend_port) data = request.content if request.has_body else None async with session.request(request.method, url, allow_redirects=False, data=data, headers=headers) as resp: # Create a new response. out_resp = web.Response(status=resp.status) # Copy received headers into the new response. for k, v in resp.headers.items(): out_resp.headers[k] = v # Fix 'Location' header on redirect. if int(resp.status / 100) == 3 and 'Location' in resp.headers: location = URL(resp.headers['Location']) if location.host: location = location.with_host(request.url.host) location = location.with_port(request.url.port) out_resp.headers['Location'] = str(location) # Send headers. await out_resp.prepare(request) # Stream response back to the client. while True: chunk = await resp.content.readany() if not chunk: break out_resp.write(chunk) out_resp.write_eof() return out_resp
def __init__(self, doc, db): self.db = db self.hcom_id = doc.get("hcom_id", "") self["capture_id"] = str(doc["_id"]) self["room_type"] = doc["name"].split("(")[0] self["occupancy_info"] = doc["occupancy"] self["room_size"] = self.get_roomsize(doc["description"][:2]) if "bed_types_str" in doc: self["bed_type"] = doc["bed_types_str"] elif doc["description"]: self["bed_type"] = self.get_bedtype(doc["description"][0], doc["bed_types"]) else: self["bed_type"] = "" self["bed_size"] = self.format_bedsize(doc["bed_metrics"]) self["room_desc"] = "\n".join(doc["description"]) self["facilities"] = [{ "facility": value } for value in doc["amenities"]] images = [] for img in doc["images"]: try: u = URL(img.get("image_url") or img["url"]) if u.host == "exp.cdn-hotels.com": u = u.with_host("img3.weegotr.com") executor.submit(Gallery, str(u)) images.append({"image_url": str(u)}) except Exception: pass self["gallery"] = [{"image_url": img["image_url"]} for img in images] executor.map(lambda x: Gallery["image_url"], doc["images"], chunksize=10) self["add_bed_type"] = [{ "type": extra, "size": None } for extra in doc["bed_extra_types"]] try: self["room_type_en"] = self.get_en_name(doc["room_type_code"]) except KeyError: logger.error("room_type_code dosent exits, %s", self["capture_id"]) self.update(self.service(doc["description"]))
def run_app(app, *, host='0.0.0.0', port=None, shutdown_timeout=60.0, ssl_context=None, print=print, backlog=128): """Run an app locally""" if port is None: if not ssl_context: port = 8080 else: port = 8443 loop = app.loop handler = app.make_handler() server = loop.create_server(handler, host, port, ssl=ssl_context, backlog=backlog) srv, startup_res = loop.run_until_complete( asyncio.gather(server, app.startup(), loop=loop)) scheme = 'https' if ssl_context else 'http' url = URL('{}://localhost'.format(scheme)) url = url.with_host(host).with_port(port) print("======== Running on {} ========\n" "(Press CTRL+C to quit)".format(url)) try: loop.run_forever() except KeyboardInterrupt: # pragma: no cover pass finally: srv.close() loop.run_until_complete(srv.wait_closed()) loop.run_until_complete(app.shutdown()) loop.run_until_complete(handler.finish_connections(shutdown_timeout)) loop.run_until_complete(app.cleanup()) loop.close()
async def new_request(*args, **kwargs): assert len(args) == 3 url = args[2] if isinstance(url, str): url = URL(url) if url.host == "director-v2": from pytest_simcore.helpers.utils_docker import get_service_published_port log.debug("MOCKING _request [before] url=%s", url) new_port = int(get_service_published_port("director-v2", 8000)) url = url.with_host("172.17.0.1").with_port(new_port) log.debug("MOCKING _request [after] url=%s kwargs=%s", url, str(kwargs)) args = args[0], args[1], url return await old_request(*args, **kwargs)
def run_app(app, *, host='0.0.0.0', port=None, shutdown_timeout=60.0, ssl_context=None, print=print, backlog=128, access_log_format=None, access_log=access_logger): """Run an app locally""" if port is None: if not ssl_context: port = 8080 else: port = 8443 loop = app.loop make_handler_kwargs = dict() if access_log_format is not None: make_handler_kwargs['access_log_format'] = access_log_format handler = app.make_handler(access_log=access_log, **make_handler_kwargs) server = loop.create_server(handler, host, port, ssl=ssl_context, backlog=backlog) srv, startup_res = loop.run_until_complete(asyncio.gather(server, app.startup(), loop=loop)) scheme = 'https' if ssl_context else 'http' url = URL('{}://localhost'.format(scheme)) url = url.with_host(host).with_port(port) print("======== Running on {} ========\n" "(Press CTRL+C to quit)".format(url)) try: loop.run_forever() except KeyboardInterrupt: # pragma: no cover pass finally: srv.close() loop.run_until_complete(srv.wait_closed()) loop.run_until_complete(app.shutdown()) loop.run_until_complete(handler.finish_connections(shutdown_timeout)) loop.run_until_complete(app.cleanup()) loop.close()
def run_app(app, *, shutdown_timeout=60.0, ssl_context=None, print=print, backlog=128, access_log_format=None, access_log=access_logger, servers=1): """Run an app locally""" app['env'] = os.environ.get('AIOWEB_ENV', 'development') conf = {'host': '0.0.0.0', 'port': 8000} conf_reader = ConfigReader('config/server.yml') if conf_reader.config: conf.update(conf_reader.config.get(app['env'])) host = conf['host'] port = conf['port'] servers = conf.get('servers', 1) unix_socket = conf.get('unix') if port is None: if not ssl_context: port = 8000 else: port = 8443 loop = app.loop make_handler_kwargs = dict() if access_log_format is not None: make_handler_kwargs['access_log_format'] = access_log_format handler = app.make_handler(access_log=access_log, **make_handler_kwargs) loop.run_until_complete(app.startup()) if unix_socket: sockets = [] if servers == 1: sockets = [unix_socket] else: path, ext = os.path.splitext(unix_socket) for i in range(1, servers + 1): sockets.append("{}.{:02d}{}".format(path, i, ext)) for socket in sockets: try: os.unlink(socket) except: pass srv = loop.run_until_complete( loop.create_unix_server(handler, path=socket, ssl=ssl_context, backlog=backlog)) os.chmod(socket, 0o664) print("======== Running on unix://{} ========".format(socket)) print("\n(Press CTRL+C to quit)") # try: # os.unlink(unix_socket) # except: # pass # srv = loop.run_until_complete(loop.create_unix_server(handler, path=unix_socket, # ssl=ssl_context, # backlog=backlog)) # os.chmod(unix_socket, 0o664) # print("======== Running on unix://{} ========\n" # "(Press CTRL+C to quit)".format(unix_socket)) else: srv = loop.run_until_complete( loop.create_server(handler, host, port, ssl=ssl_context, backlog=backlog)) scheme = 'https' if ssl_context else 'http' url = URL('{}://localhost'.format(scheme)) url = url.with_host(host).with_port(port) print("======== Running on {} ========\n" "(Press CTRL+C to quit)".format(url)) try: loop.run_forever() except KeyboardInterrupt: # pragma: no cover pass finally: srv.close() loop.run_until_complete(srv.wait_closed()) loop.run_until_complete(app.shutdown()) loop.run_until_complete(handler.shutdown(shutdown_timeout)) loop.run_until_complete(app.cleanup()) loop.close()
def test_with_host_non_ascii(): url = URL("http://example.com:123") url2 = url.with_host("историк.рф") assert url2.raw_host == "xn--h1aagokeh.xn--p1ai" assert url2.host == "историк.рф"
def test_with_host_empty(): url = URL("http://example.com:123") with pytest.raises(ValueError): url.with_host("")
def test_with_host(): url = URL('http://example.com:123') assert str(url.with_host('example.org')) == 'http://example.org:123'
def test_with_host_non_ascii(): url = URL('http://example.com:123') url2 = url.with_host('историк.рф') assert url2.raw_host == 'xn--h1aagokeh.xn--p1ai' assert url2.host == 'историк.рф'
def test_with_host_invalid_type(): url = URL("http://example.com:123") with pytest.raises(TypeError): url.with_host(None)
def test_from_str_with_host_ipv4(): url = URL('http://host:80') url = url.with_host('192.168.1.1') assert url.raw_host == '192.168.1.1'
def run_app(app, *, host=None, port=None, path=None, sock=None, shutdown_timeout=60.0, ssl_context=None, print=print, backlog=128, access_log_format=None, access_log=access_logger, loop=None): """Run an app locally""" user_supplied_loop = loop is not None if loop is None: loop = asyncio.get_event_loop() make_handler_kwargs = dict() if access_log_format is not None: make_handler_kwargs['access_log_format'] = access_log_format handler = app.make_handler(loop=loop, access_log=access_log, **make_handler_kwargs) loop.run_until_complete(app.startup()) scheme = 'https' if ssl_context else 'http' base_url = URL('{}://localhost'.format(scheme)).with_port(port) if path is None: paths = () elif isinstance(path, (str, bytes, bytearray, memoryview))\ or not isinstance(path, Iterable): paths = (path, ) else: paths = path if sock is None: socks = () elif not isinstance(sock, Iterable): socks = (sock, ) else: socks = sock if host is None: if (paths or socks) and not port: hosts = () else: hosts = ("0.0.0.0", ) elif isinstance(host, (str, bytes, bytearray, memoryview))\ or not isinstance(host, Iterable): hosts = (host, ) else: hosts = host if hosts and port is None: port = 8443 if ssl_context else 8080 server_creations = [] uris = [str(base_url.with_host(host)) for host in hosts] if hosts: # Multiple hosts bound to same server is available in most loop # implementations, but only send multiple if we have multiple. host_binding = hosts[0] if len(hosts) == 1 else hosts server_creations.append( loop.create_server(handler, host_binding, port, ssl=ssl_context, backlog=backlog)) for path in paths: # Most loop implementations don't support multiple paths bound in same # server, so create a server for each. server_creations.append( loop.create_unix_server(handler, path, ssl=ssl_context, backlog=backlog)) uris.append('{}://unix:{}:'.format(scheme, path)) # Clean up prior socket path if stale and not abstract. # CPython 3.5.3+'s event loop already does this. See # https://github.com/python/asyncio/issues/425 if path[0] not in (0, '\x00'): # pragma: no branch try: if stat.S_ISSOCK(os.stat(path).st_mode): os.remove(path) except FileNotFoundError: pass for sock in socks: server_creations.append( loop.create_server(handler, sock=sock, ssl=ssl_context, backlog=backlog)) if hasattr(socket, 'AF_UNIX') and sock.family == socket.AF_UNIX: uris.append('{}://unix:{}:'.format(scheme, sock.getsockname())) else: host, port = sock.getsockname() uris.append(str(base_url.with_host(host).with_port(port))) servers = loop.run_until_complete( asyncio.gather(*server_creations, loop=loop)) print("======== Running on {} ========\n" "(Press CTRL+C to quit)".format(', '.join(uris))) try: loop.run_forever() except KeyboardInterrupt: # pragma: no cover pass finally: server_closures = [] for srv in servers: srv.close() server_closures.append(srv.wait_closed()) loop.run_until_complete(asyncio.gather(*server_closures, loop=loop)) loop.run_until_complete(app.shutdown()) loop.run_until_complete(handler.shutdown(shutdown_timeout)) loop.run_until_complete(app.cleanup()) if not user_supplied_loop: loop.close()
def start_app(app, *, shutdown_timeout=60.0, ssl_context=None, port=None, host="0.0.0.0", socket=None, servers=1, log="log/server.log", log_level="debug", print=print, backlog=128, access_log_format=None, access_log=access_logger): loop = app.loop app['env'] = os.environ.get('AIOWEB_ENV', 'development') if not port: port = 8000 if not ssl_context else 8443 make_handler_kwargs = dict() if access_log_format is not None: make_handler_kwargs['access_log_format'] = access_log_format handler = app.make_handler(access_log=access_log, **make_handler_kwargs) loop.run_until_complete(app.startup()) if socket: sockets = [] if servers == 1: sockets = [socket] else: path, ext = os.path.splitext(socket) for i in range(1, servers + 1): sockets.append("{}.{:02d}{}".format(path, i, ext)) for socket in sockets: try: os.unlink(socket) except: pass srv = loop.run_until_complete( loop.create_unix_server(handler, path=socket, ssl=ssl_context, backlog=backlog)) os.chmod(socket, 0o664) print("======== Running on unix://{} ========".format(socket)) print("\n(Press CTRL+C to quit)") else: srv = loop.run_until_complete( loop.create_server(handler, host, port, ssl=ssl_context, backlog=backlog)) scheme = 'https' if ssl_context else 'http' url = URL('{}://localhost'.format(scheme)) url = url.with_host(host).with_port(port) print("======== Running on {} ========\n" "(Press CTRL+C to quit)".format(url)) try: loop.run_forever() except KeyboardInterrupt: # pragma: no cover pass finally: srv.close() loop.run_until_complete(srv.wait_closed()) loop.run_until_complete(handler.shutdown(shutdown_timeout)) loop.run_until_complete(app.shutdown()) loop.run_until_complete(app.cleanup()) loop.close()
def test_from_str_with_host_ipv4(): url = URL("http://host:80") url = url.with_host("192.168.1.1") assert url.raw_host == "192.168.1.1"
def test_from_str_with_host_ipv6(): url = URL("http://host:80") url = url.with_host("::1") assert url.raw_host == "::1"
def _hotel_parse(self, response): url = URL(response.url) if response.xpath('//h2[@class="narrow-results-header"]/text()'): self.logger.debug('Drop response(%s)', response.url) return if url.path == '/search.do' and response.status == 302: location = URL(response.headers.get('Location').decode()) yield Request( str(url.with_path(location.path).with_query(location.query)), self._hotel_parse, meta=response.meta ) return hotels_id = url.path.strip('/').split('/')[0][2:] hotels_id = url.query.get('hotel-id') or hotels_id loader = ItemLoader(item=Hotels(), response=response) en_url = response.xpath(hotels_xp.EN_US_URL).extract_first() PREFIX = self.site_prefix.get(url.host.split('.')[-1]) if PREFIX == 'CN': en_url = url.with_host('www.hotels.com') \ .with_query({'pos': 'HCOM_US', 'locale': 'en_US'}) en_url = str(en_url) # yield Request( # en_url, # self._hotel_parse, # meta={ # # 'proxy': '172.17.1.198:1118' # } # ) if response.meta.get('crawl_reviews', True): review_url = url.with_scheme(url.scheme) \ .with_host(url.host) \ .join( URL( '/'.join([ '', 'hotel', hotels_id, 'reviews', '' ]) ) ) yield Request( str(review_url), self._review_parse, meta={ 'hotels_id': hotels_id, # 'proxy': '172.17.1.198:1118' } ) loader.add_value('url', response.url) loader.add_value('us_url', en_url) if response.meta.get('crawl_rooms', True): now = datetime.now() checkin = now + timedelta(days=1) checkout = now + timedelta(days=3) details_query = { 'q-check-in': checkin.strftime('%Y-%m-%d'), 'q-check-out': checkout.strftime('%Y-%m-%d'), # 'hotel-id': hotels_id, 'q-room-0-adults': 2, 'q-room-0-children': 0, 'tab': 'description', } u = url.with_scheme('https') \ .with_query(details_query) yield Request( str(u), self._rooms_parse, meta={ 'hotels_id': hotels_id, 'locale': PREFIX.lower(), 'dont_redirect': True, } ) yield Request( str(u.with_host('www.hotels.cn')), self._rooms_parse, meta={ 'hotels_id': hotels_id, 'locale': 'en', 'dont_redirect': True, # 'proxy': '172.17.1.198:1118' } ) supplier_obj_id = response.meta.get('statics.hotels.id') supplier_name = response.meta.get('statics.hotels.supplier') if supplier_obj_id: loader.add_value('statics_hotels_id', supplier_obj_id) loader.add_value('statics_hotels_supplier', supplier_name) loader.add_value('hotels_id', hotels_id) loader.add_xpath('title', hotels_xp.TITLE) loader.add_xpath('name', hotels_xp.NAME) position = take_first(response, hotels_xp.POSITION) # loader.add_xpath('latitude', hotels_xp.LATITUDE) # loader.add_xpath('longitude', hotels_xp.LONGITUDE) latitude, longitude = position.split(',') loader.add_value('latitude', latitude) loader.add_xpath('longitude', longitude) loader.add_xpath('star', hotels_xp.STAR, re='(\d+)') loader.add_value('address', self._address(response)) loader.add_xpath('address_text', hotels_xp.ADDRESS) loader.add_xpath('price', hotels_xp.PRICE) loader.add_xpath('city', hotels_xp.ADDRESS_LOCALITY) loader.add_xpath('country', hotels_xp.ADDRESS_COUNTRY) loader.add_xpath('telephone', hotels_xp.TELEPHONE) # loader.add_xpath('rating', '') # loader.add_xpath('tripadvisor', '') loader.add_xpath('landmarks', hotels_xp.LANDMARKS) loader.add_xpath('traffic_tips', hotels_xp.TRAFFIC_TIPS) loader.add_value('notice', self._notice(response, PREFIX)) IN_STORE_SERVICES = getattr( hotels_xp, PREFIX + '_IN_STORE_SERVICE_FACT' ) loader.add_value( 'in_store_service_facilities', service_to_dict(response.xpath(IN_STORE_SERVICES), hotels_xp) ) ROOM_SERVICES = getattr(hotels_xp, PREFIX + '_ROOM_SERVICE_FACT') loader.add_value( 'room_service_facilities', service_to_dict(response.xpath(ROOM_SERVICES), hotels_xp) ) loader.add_xpath('around', hotels_xp.AROUND) pictures = self._pictures(response.xpath(hotels_xp.PICTURES)) loader.add_value('pictures', pictures) loader.add_xpath('short_introduction', hotels_xp.SHORT_INTRODUCTION) loader.add_xpath('amenities', hotels_xp.AMENITIES) loader.add_xpath('for_families', hotels_xp.FOR_FAMILIES) loader.add_value('locale', PREFIX.lower()) loader.add_value('summary', self._summary(response)) yield loader.load_item()
def __init__( self, db=None, hotel_id="", jset_id="", hotels_cn_id="", bonotel="", hotelspro="", hotelbeds="", bookings_id="", jactravel="", city=None, capture_url=None, comments_url=None, capture_id=None, ): if (not hotel_id and not jset_id and not hotels_cn_id and not bookings_id): raise HotelNotFound("<>") self.city = city self._capture_url_ = capture_url self._capture_id = capture_id self.db = db or MongoClient(settings.MONGO).scripture self.captured_urls_host = "http://scripture.weegotr.com/captured_from" self._doc = { "capture_id": capture_id, "capture_url": capture_url, "hotels_id": hotels_cn_id, "quote_ids": [], } if bonotel: self._doc["quote_ids"].append({ "hotel_id": bonotel, "quoter": "59b20729a389295de92f4952" }) if hotel_id: self._doc["quote_ids"].append({ "hotel_id": hotel_id, "quoter": "59afe1e0a389295de92f47f8" }) if hotelspro: self._doc["quote_ids"].append({ "hotel_id": hotelspro, "quoter": "59b0b048a389295de92f4815" }) if hotelbeds: self._doc["quote_ids"].append({ "hotel_id": hotelbeds, "quoter": "59c1f0f8e1812033000e1de8" }) if jactravel: self._doc["quote_ids"].append({ "hotel_id": jactravel, "quoter": "59b0b035a389295de92f4814" }) self.__loaded_from_jset = False self.__loaded_from_rxml = False self.__loaded_from_hotels_cn = False self.__comments_url = comments_url self._jset_id = jset_id self._hotel_id = hotel_id self._hotels_cn_id = hotels_cn_id if hotel_id: self.from_rxml(hotel_id) if jset_id: self.from_jset(jset_id) if hotels_cn_id: self.from_hotels_cn(hotels_cn_id) if bookings_id: self.from_bookings(bookings_id) host = URL("http://feifanweige.com/").with_path("api/internal") if os.environ.get("SCRIPTURE") == "SANDBOX": self.host = host.with_host("172.16.4.110:8289") else: self.host = host.with_host( "api.cn-beijing.aliyuncs.feifanweige.com")
def run_app(app, *, host=None, port=None, path=None, shutdown_timeout=60.0, ssl_context=None, print=print, backlog=128, access_log_format=None, access_log=access_logger): """Run an app locally""" loop = app.loop make_handler_kwargs = dict() if access_log_format is not None: make_handler_kwargs['access_log_format'] = access_log_format handler = app.make_handler(access_log=access_log, **make_handler_kwargs) loop.run_until_complete(app.startup()) scheme = 'https' if ssl_context else 'http' base_url = URL('{}://localhost'.format(scheme)).with_port(port) if path is None: paths = () elif isinstance(path, (str, bytes, bytearray, memoryview))\ or not isinstance(path, Iterable): paths = (path,) else: paths = path if host is None: if paths and not port: hosts = () else: hosts = ("0.0.0.0",) elif isinstance(host, (str, bytes, bytearray, memoryview))\ or not isinstance(host, Iterable): hosts = (host,) else: hosts = host if hosts and port is None: port = 8443 if ssl_context else 8080 server_creations = [] uris = [str(base_url.with_host(host)) for host in hosts] if hosts: # Multiple hosts bound to same server is available in most loop # implementations, but only send multiple if we have multiple. host_binding = hosts[0] if len(hosts) == 1 else hosts server_creations.append( loop.create_server( handler, host_binding, port, ssl=ssl_context, backlog=backlog ) ) for path in paths: # Most loop implementations don't support multiple paths bound in same # server, so create a server for each. server_creations.append( loop.create_unix_server( handler, path, ssl=ssl_context, backlog=backlog ) ) uris.append('{}://unix:{}:'.format(scheme, path)) # Clean up prior socket path if stale and not abstract. # CPython 3.5.3+'s event loop already does this. See # https://github.com/python/asyncio/issues/425 if path[0] not in (0, '\x00'): # pragma: no branch try: if stat.S_ISSOCK(os.stat(path).st_mode): os.remove(path) except FileNotFoundError: pass servers = loop.run_until_complete( asyncio.gather(*server_creations, loop=loop) ) print("======== Running on {} ========\n" "(Press CTRL+C to quit)".format(', '.join(uris))) try: loop.run_forever() except KeyboardInterrupt: # pragma: no cover pass finally: server_closures = [] for srv in servers: srv.close() server_closures.append(srv.wait_closed()) loop.run_until_complete(asyncio.gather(*server_closures, loop=loop)) loop.run_until_complete(app.shutdown()) loop.run_until_complete(handler.shutdown(shutdown_timeout)) loop.run_until_complete(app.cleanup()) loop.close()
def test_from_str_with_host_ipv6(): url = URL('http://host:80') url = url.with_host('::1') assert url.raw_host == '::1'
def test_with_host_empty(): url = URL('http://example.com:123') with pytest.raises(ValueError): url.with_host('')
def to_ipv6(url): url = URL(url) parts = list(map(int, url.host.split("."))) ipv6_host = "2002:{:02x}{:02x}:{:02x}{:02x}::".format(*parts) return str(url.with_host("[%s]" % ipv6_host))
def test_with_host_invalid_type(): url = URL('http://example.com:123') with pytest.raises(TypeError): url.with_host(None)
def _make_server_creators(handler, *, loop, ssl_context, host, port, path, sock, backlog): scheme = 'https' if ssl_context else 'http' base_url = URL('{}://localhost'.format(scheme)).with_port(port) if path is None: paths = () elif isinstance(path, (str, bytes, bytearray, memoryview))\ or not isinstance(path, Iterable): paths = (path,) else: paths = path if sock is None: socks = () elif not isinstance(sock, Iterable): socks = (sock,) else: socks = sock if host is None: if (paths or socks) and not port: hosts = () else: hosts = ("0.0.0.0",) elif isinstance(host, (str, bytes, bytearray, memoryview))\ or not isinstance(host, Iterable): hosts = (host,) else: hosts = host if hosts and port is None: port = 8443 if ssl_context else 8080 server_creations = [] uris = [str(base_url.with_host(host)) for host in hosts] if hosts: # Multiple hosts bound to same server is available in most loop # implementations, but only send multiple if we have multiple. host_binding = hosts[0] if len(hosts) == 1 else hosts server_creations.append( loop.create_server( handler, host_binding, port, ssl=ssl_context, backlog=backlog ) ) for path in paths: # Most loop implementations don't support multiple paths bound in same # server, so create a server for each. server_creations.append( loop.create_unix_server( handler, path, ssl=ssl_context, backlog=backlog ) ) uris.append('{}://unix:{}:'.format(scheme, path)) # Clean up prior socket path if stale and not abstract. # CPython 3.5.3+'s event loop already does this. See # https://github.com/python/asyncio/issues/425 if path[0] not in (0, '\x00'): # pragma: no branch try: if stat.S_ISSOCK(os.stat(path).st_mode): os.remove(path) except FileNotFoundError: pass for sock in socks: server_creations.append( loop.create_server( handler, sock=sock, ssl=ssl_context, backlog=backlog ) ) if hasattr(socket, 'AF_UNIX') and sock.family == socket.AF_UNIX: uris.append('{}://unix:{}:'.format(scheme, sock.getsockname())) else: host, port = sock.getsockname() uris.append(str(base_url.with_host(host).with_port(port))) return server_creations, uris
def test_with_host(): url = URL("http://example.com:123") assert str(url.with_host("example.org")) == "http://example.org:123"
def _make_server_creators(handler, *, loop, ssl_context, host, port, path, sock, backlog): scheme = 'https' if ssl_context else 'http' base_url = URL('{}://localhost'.format(scheme)).with_port(port) if path is None: paths = () elif isinstance(path, (str, bytes, bytearray, memoryview))\ or not isinstance(path, Iterable): paths = (path, ) else: paths = path if sock is None: socks = () elif not isinstance(sock, Iterable): socks = (sock, ) else: socks = sock if host is None: if (paths or socks) and not port: hosts = () else: hosts = ("0.0.0.0", ) elif isinstance(host, (str, bytes, bytearray, memoryview))\ or not isinstance(host, Iterable): hosts = (host, ) else: hosts = host if hosts and port is None: port = 8443 if ssl_context else 8080 server_creations = [] uris = [str(base_url.with_host(host)) for host in hosts] if hosts: # Multiple hosts bound to same server is available in most loop # implementations, but only send multiple if we have multiple. host_binding = hosts[0] if len(hosts) == 1 else hosts server_creations.append( loop.create_server(handler, host_binding, port, ssl=ssl_context, backlog=backlog)) for path in paths: # Most loop implementations don't support multiple paths bound in same # server, so create a server for each. server_creations.append( loop.create_unix_server(handler, path, ssl=ssl_context, backlog=backlog)) uris.append('{}://unix:{}:'.format(scheme, path)) # Clean up prior socket path if stale and not abstract. # CPython 3.5.3+'s event loop already does this. See # https://github.com/python/asyncio/issues/425 if path[0] not in (0, '\x00'): # pragma: no branch try: if stat.S_ISSOCK(os.stat(path).st_mode): os.remove(path) except FileNotFoundError: pass for sock in socks: server_creations.append( loop.create_server(handler, sock=sock, ssl=ssl_context, backlog=backlog)) if hasattr(socket, 'AF_UNIX') and sock.family == socket.AF_UNIX: uris.append('{}://unix:{}:'.format(scheme, sock.getsockname())) else: host, port = sock.getsockname() uris.append(str(base_url.with_host(host).with_port(port))) return server_creations, uris