def run_app(app,
*,
host='0.0.0.0',
port=None,
shutdown_timeout=60.0,
ssl_context=None,
print=print,
backlog=128,
access_log_format=None,
access_log=access_logger):
"""Run an app locally"""
if port is None:
if not ssl_context:
port = 8080
else:
port = 8443
loop = app.loop
make_handler_kwargs = dict()
if access_log_format is not None:
make_handler_kwargs['access_log_format'] = access_log_format
handler = app.make_handler(access_log=access_log, **make_handler_kwargs)
loop.run_until_complete(app.startup())
srv = loop.run_until_complete(
loop.create_server(handler,
host,
port,
ssl=ssl_context,
backlog=backlog))
scheme = 'https' if ssl_context else 'http'
base_url = URL('{}://localhost'.format(scheme)).with_port(port)
if isinstance(host, str):
urls = [base_url.with_host(host)]
else:
urls = [base_url.with_host(h) for h in host]
print("======== Running on {} ========\n"
"(Press CTRL+C to quit)".format(', '.join(str(u) for u in urls)))
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
pass
finally:
srv.close()
loop.run_until_complete(srv.wait_closed())
loop.run_until_complete(app.shutdown())
loop.run_until_complete(handler.shutdown(shutdown_timeout))
loop.run_until_complete(app.cleanup())
loop.close()
def __init__(self, host="0.0.0.0", port=8080, ssl_context=None, netmem_dict:dict=None):
super().__init__()
self.host = host
self.port = port
self.ssl_context = ssl_context
self.netmem = netmem_dict # Reference to the hosting networkmemory
self._app = None # type: web.Application
self._handler = None # type: web.Server
self._srv = None # type: asyncio.base_events.Server
self._active_ws_updates_sockets = [] # type: [web.WebSocketResponse]
self._active_ws_whole_sockets = [] # type: [web.WebSocketResponse]
scheme = 'https' if self.ssl_context else 'http'
url = URL('{}://localhost'.format(scheme))
self.url_base = url.with_host(socket.gethostname()).with_port(self.port)
self.url_ws_updates = self.url_base.join(URL(WsServerConnector.WS_UPDATES))
self.url_ws_whole = self.url_base.join(URL(WsServerConnector.WS_WHOLE))
self.url_html_view = self.url_base.join(URL(WsServerConnector.HTML_VIEW))
self.log.info("{} : Hosting html view at {}".format(self, self.url_html_view))
self.log.info("{} : Hosting websocket memory updates at {}".format(self, self.url_ws_updates))
self.log.info("{} : Hosting websocket memory complete reflection at {}".format(self, self.url_ws_whole))
async def start(self):
logger.debug('start')
if not self.has_app():
logger.debug('does not have app')
return
if not self.auto_start:
logger.debug('should not start automatically')
return
app = self.get_app()
handler = app.make_handler()
loop = asyncio.get_event_loop()
server = loop.create_server(
handler,
self.host,
self.port,
ssl=self.ssl_context,
backlog=self.backlog,
)
srv, startup_res = await asyncio.gather(
server,
app.startup(),
loop=loop,
)
scheme = 'https' if self.ssl_context else 'http'
url = URL('{}://localhost'.format(scheme))
url = url.with_host(self.host).with_port(self.port)
logger.debug('======== Running on {} ========\n'
'(Press CTRL+C to quit)'.format(url))
self.server = srv
self.handler = handler
def test_with_host_non_ascii():
url = URL("http://example.com:123")
url2 = url.with_host("историк.рф")
assert url2.raw_host == "xn--h1aagokeh.xn--p1ai"
assert url2.host == "историк.рф"
assert url2.raw_authority == "xn--h1aagokeh.xn--p1ai:123"
assert url2.authority == "историк.рф:123"
def test_with_host_percent_encoded():
url = URL("http://%25cf%2580%cf%80:%25cf%2580%cf%[email protected]:123")
url2 = url.with_host("%cf%80.org")
assert url2.raw_host == "%cf%80.org"
assert url2.host == "%cf%80.org"
assert url2.raw_authority == "%25cf%2580%CF%80:%25cf%2580%CF%80@%cf%80.org:123"
assert url2.authority == "%cf%80π:%cf%80π@%cf%80.org:123"
async def proxy_connection(proxy: TCPProxy, amqp_url: URL, loop):
url = amqp_url.with_host("localhost", ).with_port(proxy.proxy_port, )
connection = Connection(url, loop=loop)
await connection.connect()
try:
yield connection
finally:
await connection.close()
def test_port_override_with_ipv6(openapi3_schema_url, simple_openapi, mocker):
resp = Response()
resp.status_code = 200
resp._content = json.dumps(simple_openapi).encode("utf-8")
mocker.patch("requests.get", return_value=resp)
url = URL(openapi3_schema_url)
parts = list(map(int, url.host.split(".")))
ipv6_host = "2002:{:02x}{:02x}:{:02x}{:02x}::".format(*parts)
ipv6 = url.with_host("[%s]" % ipv6_host)
schema = schemathesis.from_uri(str(ipv6), validate_schema=False, port=8081)
operation = next(schema.get_all_operations()).ok()
assert operation.base_url == "http://[2002:7f00:1::]:8081/schema.yaml"
def run(self,
host='localhost',
port=None,
shutdown_timeout=60.0,
ssl_context=None,
print=print,
backlog=128,
access_log_format=None,
access_log=None):
"""Run an app locally"""
if port is None:
if not ssl_context:
port = 8080
else:
port = 8443
loop = self._app.loop
make_handler_kwargs = dict()
if access_log_format is not None:
make_handler_kwargs['access_log_format'] = access_log_format
handler = self._app.make_handler(access_log=access_log,
**make_handler_kwargs)
loop.run_until_complete(self._app.startup())
srv = loop.run_until_complete(
loop.create_server(handler,
host,
port,
ssl=ssl_context,
backlog=backlog))
scheme = 'https' if ssl_context else 'http'
url = URL('{}://localhost'.format(scheme))
url = url.with_host(host).with_port(port)
print("======== Running on {} ========\n"
"(Press CTRL+C to quit)".format(url))
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
pass
finally:
print("run here")
srv.close()
loop.run_until_complete(srv.wait_closed())
loop.run_until_complete(self._app.shutdown())
loop.run_until_complete(handler.shutdown(shutdown_timeout))
loop.run_until_complete(self._app.cleanup())
loop.close()
def entity_picture(self) -> str:
"""Return the gamer pic."""
if not self.data:
return None
# Xbox sometimes returns a domain that uses a wrong certificate which creates issues
# with loading the image.
# The correct domain is images-eds-ssl which can just be replaced
# to point to the correct image, with the correct domain and certificate.
# We need to also remove the 'mode=Padding' query because with it, it results in an error 400.
url = URL(self.data.display_pic)
if url.host == "images-eds.xboxlive.com":
url = url.with_host("images-eds-ssl.xboxlive.com")
query = dict(url.query)
query.pop("mode", None)
return str(url.with_query(query))
def __init__(self, url):
self.logger = logging.getLogger(__name__)
endpoint = 'http://oss-cn-beijing-internal.aliyuncs.com'
auth = oss2.Auth('LTAIwHXVgRkPRxcw', 'PCm6kdpeyjAG2gJz4B4C2js3TIifrc')
self.bucket = oss2.Bucket(auth, endpoint, 'weegotr-statics')
u = URL(url)
path = u.path if not u.path.startswith('/') else u.path[1:]
exists = self.bucket.object_exists(path)
if not exists:
url = u.with_host('exp.cdn-hotels.com')
content = BytesIO(self.session.get(str(url)).content)
put_result = self.bucket.put_object(path, content)
if put_result.status >= 300:
print({'status': put_result.status, 'text': put_result.resp})
else:
print({'status': put_result.status, 'file': u.path})
async def handle(request, session, redis):
# Lookup the backend on redis.
key = '{}:{}'.format(request.url.host, request.url.port)
value = await redis.get(key)
if not value:
raise web.HTTPNotFound
backend_host, backend_port = value.decode().split(':')
backend_port = int(backend_port)
url = request.url.with_host(backend_host).with_port(backend_port)
# Send an equivalent request to the backend.
headers = request.headers.copy()
headers['Host'] = '{}:{}'.format(backend_host, backend_port)
data = request.content if request.has_body else None
async with session.request(request.method, url,
allow_redirects=False,
data=data,
headers=headers) as resp:
# Create a new response.
out_resp = web.Response(status=resp.status)
# Copy received headers into the new response.
for k, v in resp.headers.items():
out_resp.headers[k] = v
# Fix 'Location' header on redirect.
if int(resp.status / 100) == 3 and 'Location' in resp.headers:
location = URL(resp.headers['Location'])
if location.host:
location = location.with_host(request.url.host)
location = location.with_port(request.url.port)
out_resp.headers['Location'] = str(location)
# Send headers.
await out_resp.prepare(request)
# Stream response back to the client.
while True:
chunk = await resp.content.readany()
if not chunk:
break
out_resp.write(chunk)
out_resp.write_eof()
return out_resp
def __init__(self, doc, db):
self.db = db
self.hcom_id = doc.get("hcom_id", "")
self["capture_id"] = str(doc["_id"])
self["room_type"] = doc["name"].split("(")[0]
self["occupancy_info"] = doc["occupancy"]
self["room_size"] = self.get_roomsize(doc["description"][:2])
if "bed_types_str" in doc:
self["bed_type"] = doc["bed_types_str"]
elif doc["description"]:
self["bed_type"] = self.get_bedtype(doc["description"][0],
doc["bed_types"])
else:
self["bed_type"] = ""
self["bed_size"] = self.format_bedsize(doc["bed_metrics"])
self["room_desc"] = "\n".join(doc["description"])
self["facilities"] = [{
"facility": value
} for value in doc["amenities"]]
images = []
for img in doc["images"]:
try:
u = URL(img.get("image_url") or img["url"])
if u.host == "exp.cdn-hotels.com":
u = u.with_host("img3.weegotr.com")
executor.submit(Gallery, str(u))
images.append({"image_url": str(u)})
except Exception:
pass
self["gallery"] = [{"image_url": img["image_url"]} for img in images]
executor.map(lambda x: Gallery["image_url"],
doc["images"],
chunksize=10)
self["add_bed_type"] = [{
"type": extra,
"size": None
} for extra in doc["bed_extra_types"]]
try:
self["room_type_en"] = self.get_en_name(doc["room_type_code"])
except KeyError:
logger.error("room_type_code dosent exits, %s", self["capture_id"])
self.update(self.service(doc["description"]))
def run_app(app,
*,
host='0.0.0.0',
port=None,
shutdown_timeout=60.0,
ssl_context=None,
print=print,
backlog=128):
"""Run an app locally"""
if port is None:
if not ssl_context:
port = 8080
else:
port = 8443
loop = app.loop
handler = app.make_handler()
server = loop.create_server(handler,
host,
port,
ssl=ssl_context,
backlog=backlog)
srv, startup_res = loop.run_until_complete(
asyncio.gather(server, app.startup(), loop=loop))
scheme = 'https' if ssl_context else 'http'
url = URL('{}://localhost'.format(scheme))
url = url.with_host(host).with_port(port)
print("======== Running on {} ========\n"
"(Press CTRL+C to quit)".format(url))
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
pass
finally:
srv.close()
loop.run_until_complete(srv.wait_closed())
loop.run_until_complete(app.shutdown())
loop.run_until_complete(handler.finish_connections(shutdown_timeout))
loop.run_until_complete(app.cleanup())
loop.close()
async def new_request(*args, **kwargs):
assert len(args) == 3
url = args[2]
if isinstance(url, str):
url = URL(url)
if url.host == "director-v2":
from pytest_simcore.helpers.utils_docker import get_service_published_port
log.debug("MOCKING _request [before] url=%s", url)
new_port = int(get_service_published_port("director-v2", 8000))
url = url.with_host("172.17.0.1").with_port(new_port)
log.debug("MOCKING _request [after] url=%s kwargs=%s", url,
str(kwargs))
args = args[0], args[1], url
return await old_request(*args, **kwargs)
def run_app(app, *, host='0.0.0.0', port=None,
shutdown_timeout=60.0, ssl_context=None,
print=print, backlog=128, access_log_format=None,
access_log=access_logger):
"""Run an app locally"""
if port is None:
if not ssl_context:
port = 8080
else:
port = 8443
loop = app.loop
make_handler_kwargs = dict()
if access_log_format is not None:
make_handler_kwargs['access_log_format'] = access_log_format
handler = app.make_handler(access_log=access_log,
**make_handler_kwargs)
server = loop.create_server(handler, host, port, ssl=ssl_context,
backlog=backlog)
srv, startup_res = loop.run_until_complete(asyncio.gather(server,
app.startup(),
loop=loop))
scheme = 'https' if ssl_context else 'http'
url = URL('{}://localhost'.format(scheme))
url = url.with_host(host).with_port(port)
print("======== Running on {} ========\n"
"(Press CTRL+C to quit)".format(url))
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
pass
finally:
srv.close()
loop.run_until_complete(srv.wait_closed())
loop.run_until_complete(app.shutdown())
loop.run_until_complete(handler.finish_connections(shutdown_timeout))
loop.run_until_complete(app.cleanup())
loop.close()
def run_app(app,
*,
shutdown_timeout=60.0,
ssl_context=None,
print=print,
backlog=128,
access_log_format=None,
access_log=access_logger,
servers=1):
"""Run an app locally"""
app['env'] = os.environ.get('AIOWEB_ENV', 'development')
conf = {'host': '0.0.0.0', 'port': 8000}
conf_reader = ConfigReader('config/server.yml')
if conf_reader.config:
conf.update(conf_reader.config.get(app['env']))
host = conf['host']
port = conf['port']
servers = conf.get('servers', 1)
unix_socket = conf.get('unix')
if port is None:
if not ssl_context:
port = 8000
else:
port = 8443
loop = app.loop
make_handler_kwargs = dict()
if access_log_format is not None:
make_handler_kwargs['access_log_format'] = access_log_format
handler = app.make_handler(access_log=access_log, **make_handler_kwargs)
loop.run_until_complete(app.startup())
if unix_socket:
sockets = []
if servers == 1:
sockets = [unix_socket]
else:
path, ext = os.path.splitext(unix_socket)
for i in range(1, servers + 1):
sockets.append("{}.{:02d}{}".format(path, i, ext))
for socket in sockets:
try:
os.unlink(socket)
except:
pass
srv = loop.run_until_complete(
loop.create_unix_server(handler,
path=socket,
ssl=ssl_context,
backlog=backlog))
os.chmod(socket, 0o664)
print("======== Running on unix://{} ========".format(socket))
print("\n(Press CTRL+C to quit)")
# try:
# os.unlink(unix_socket)
# except:
# pass
# srv = loop.run_until_complete(loop.create_unix_server(handler, path=unix_socket,
# ssl=ssl_context,
# backlog=backlog))
# os.chmod(unix_socket, 0o664)
# print("======== Running on unix://{} ========\n"
# "(Press CTRL+C to quit)".format(unix_socket))
else:
srv = loop.run_until_complete(
loop.create_server(handler,
host,
port,
ssl=ssl_context,
backlog=backlog))
scheme = 'https' if ssl_context else 'http'
url = URL('{}://localhost'.format(scheme))
url = url.with_host(host).with_port(port)
print("======== Running on {} ========\n"
"(Press CTRL+C to quit)".format(url))
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
pass
finally:
srv.close()
loop.run_until_complete(srv.wait_closed())
loop.run_until_complete(app.shutdown())
loop.run_until_complete(handler.shutdown(shutdown_timeout))
loop.run_until_complete(app.cleanup())
loop.close()
def test_with_host_non_ascii():
url = URL("http://example.com:123")
url2 = url.with_host("историк.рф")
assert url2.raw_host == "xn--h1aagokeh.xn--p1ai"
assert url2.host == "историк.рф"
def test_with_host_empty():
url = URL("http://example.com:123")
with pytest.raises(ValueError):
url.with_host("")
def test_with_host():
url = URL('http://example.com:123')
assert str(url.with_host('example.org')) == 'http://example.org:123'
def test_with_host_non_ascii():
url = URL('http://example.com:123')
url2 = url.with_host('историк.рф')
assert url2.raw_host == 'xn--h1aagokeh.xn--p1ai'
assert url2.host == 'историк.рф'
def test_with_host_invalid_type():
url = URL("http://example.com:123")
with pytest.raises(TypeError):
url.with_host(None)
def test_from_str_with_host_ipv4():
url = URL('http://host:80')
url = url.with_host('192.168.1.1')
assert url.raw_host == '192.168.1.1'
def test_with_host():
url = URL('http://example.com:123')
assert str(url.with_host('example.org')) == 'http://example.org:123'
def run_app(app,
*,
host=None,
port=None,
path=None,
sock=None,
shutdown_timeout=60.0,
ssl_context=None,
print=print,
backlog=128,
access_log_format=None,
access_log=access_logger,
loop=None):
"""Run an app locally"""
user_supplied_loop = loop is not None
if loop is None:
loop = asyncio.get_event_loop()
make_handler_kwargs = dict()
if access_log_format is not None:
make_handler_kwargs['access_log_format'] = access_log_format
handler = app.make_handler(loop=loop,
access_log=access_log,
**make_handler_kwargs)
loop.run_until_complete(app.startup())
scheme = 'https' if ssl_context else 'http'
base_url = URL('{}://localhost'.format(scheme)).with_port(port)
if path is None:
paths = ()
elif isinstance(path, (str, bytes, bytearray, memoryview))\
or not isinstance(path, Iterable):
paths = (path, )
else:
paths = path
if sock is None:
socks = ()
elif not isinstance(sock, Iterable):
socks = (sock, )
else:
socks = sock
if host is None:
if (paths or socks) and not port:
hosts = ()
else:
hosts = ("0.0.0.0", )
elif isinstance(host, (str, bytes, bytearray, memoryview))\
or not isinstance(host, Iterable):
hosts = (host, )
else:
hosts = host
if hosts and port is None:
port = 8443 if ssl_context else 8080
server_creations = []
uris = [str(base_url.with_host(host)) for host in hosts]
if hosts:
# Multiple hosts bound to same server is available in most loop
# implementations, but only send multiple if we have multiple.
host_binding = hosts[0] if len(hosts) == 1 else hosts
server_creations.append(
loop.create_server(handler,
host_binding,
port,
ssl=ssl_context,
backlog=backlog))
for path in paths:
# Most loop implementations don't support multiple paths bound in same
# server, so create a server for each.
server_creations.append(
loop.create_unix_server(handler,
path,
ssl=ssl_context,
backlog=backlog))
uris.append('{}://unix:{}:'.format(scheme, path))
# Clean up prior socket path if stale and not abstract.
# CPython 3.5.3+'s event loop already does this. See
# https://github.com/python/asyncio/issues/425
if path[0] not in (0, '\x00'): # pragma: no branch
try:
if stat.S_ISSOCK(os.stat(path).st_mode):
os.remove(path)
except FileNotFoundError:
pass
for sock in socks:
server_creations.append(
loop.create_server(handler,
sock=sock,
ssl=ssl_context,
backlog=backlog))
if hasattr(socket, 'AF_UNIX') and sock.family == socket.AF_UNIX:
uris.append('{}://unix:{}:'.format(scheme, sock.getsockname()))
else:
host, port = sock.getsockname()
uris.append(str(base_url.with_host(host).with_port(port)))
servers = loop.run_until_complete(
asyncio.gather(*server_creations, loop=loop))
print("======== Running on {} ========\n"
"(Press CTRL+C to quit)".format(', '.join(uris)))
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
pass
finally:
server_closures = []
for srv in servers:
srv.close()
server_closures.append(srv.wait_closed())
loop.run_until_complete(asyncio.gather(*server_closures, loop=loop))
loop.run_until_complete(app.shutdown())
loop.run_until_complete(handler.shutdown(shutdown_timeout))
loop.run_until_complete(app.cleanup())
if not user_supplied_loop:
loop.close()
def start_app(app,
*,
shutdown_timeout=60.0,
ssl_context=None,
port=None,
host="0.0.0.0",
socket=None,
servers=1,
log="log/server.log",
log_level="debug",
print=print,
backlog=128,
access_log_format=None,
access_log=access_logger):
loop = app.loop
app['env'] = os.environ.get('AIOWEB_ENV', 'development')
if not port:
port = 8000 if not ssl_context else 8443
make_handler_kwargs = dict()
if access_log_format is not None:
make_handler_kwargs['access_log_format'] = access_log_format
handler = app.make_handler(access_log=access_log, **make_handler_kwargs)
loop.run_until_complete(app.startup())
if socket:
sockets = []
if servers == 1:
sockets = [socket]
else:
path, ext = os.path.splitext(socket)
for i in range(1, servers + 1):
sockets.append("{}.{:02d}{}".format(path, i, ext))
for socket in sockets:
try:
os.unlink(socket)
except:
pass
srv = loop.run_until_complete(
loop.create_unix_server(handler,
path=socket,
ssl=ssl_context,
backlog=backlog))
os.chmod(socket, 0o664)
print("======== Running on unix://{} ========".format(socket))
print("\n(Press CTRL+C to quit)")
else:
srv = loop.run_until_complete(
loop.create_server(handler,
host,
port,
ssl=ssl_context,
backlog=backlog))
scheme = 'https' if ssl_context else 'http'
url = URL('{}://localhost'.format(scheme))
url = url.with_host(host).with_port(port)
print("======== Running on {} ========\n"
"(Press CTRL+C to quit)".format(url))
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
pass
finally:
srv.close()
loop.run_until_complete(srv.wait_closed())
loop.run_until_complete(handler.shutdown(shutdown_timeout))
loop.run_until_complete(app.shutdown())
loop.run_until_complete(app.cleanup())
loop.close()
def test_from_str_with_host_ipv4():
url = URL("http://host:80")
url = url.with_host("192.168.1.1")
assert url.raw_host == "192.168.1.1"
def test_from_str_with_host_ipv6():
url = URL("http://host:80")
url = url.with_host("::1")
assert url.raw_host == "::1"
def test_from_str_with_host_ipv4():
url = URL("http://host:80")
url = url.with_host("192.168.1.1")
assert url.raw_host == "192.168.1.1"
def _hotel_parse(self, response):
url = URL(response.url)
if response.xpath('//h2[@class="narrow-results-header"]/text()'):
self.logger.debug('Drop response(%s)', response.url)
return
if url.path == '/search.do' and response.status == 302:
location = URL(response.headers.get('Location').decode())
yield Request(
str(url.with_path(location.path).with_query(location.query)),
self._hotel_parse,
meta=response.meta
)
return
hotels_id = url.path.strip('/').split('/')[0][2:]
hotels_id = url.query.get('hotel-id') or hotels_id
loader = ItemLoader(item=Hotels(), response=response)
en_url = response.xpath(hotels_xp.EN_US_URL).extract_first()
PREFIX = self.site_prefix.get(url.host.split('.')[-1])
if PREFIX == 'CN':
en_url = url.with_host('www.hotels.com') \
.with_query({'pos': 'HCOM_US', 'locale': 'en_US'})
en_url = str(en_url)
# yield Request(
# en_url,
# self._hotel_parse,
# meta={
# # 'proxy': '172.17.1.198:1118'
# }
# )
if response.meta.get('crawl_reviews', True):
review_url = url.with_scheme(url.scheme) \
.with_host(url.host) \
.join(
URL(
'/'.join([
'',
'hotel',
hotels_id,
'reviews',
''
])
)
)
yield Request(
str(review_url),
self._review_parse,
meta={
'hotels_id': hotels_id,
# 'proxy': '172.17.1.198:1118'
}
)
loader.add_value('url', response.url)
loader.add_value('us_url', en_url)
if response.meta.get('crawl_rooms', True):
now = datetime.now()
checkin = now + timedelta(days=1)
checkout = now + timedelta(days=3)
details_query = {
'q-check-in': checkin.strftime('%Y-%m-%d'),
'q-check-out': checkout.strftime('%Y-%m-%d'),
# 'hotel-id': hotels_id,
'q-room-0-adults': 2,
'q-room-0-children': 0,
'tab': 'description',
}
u = url.with_scheme('https') \
.with_query(details_query)
yield Request(
str(u),
self._rooms_parse,
meta={
'hotels_id': hotels_id,
'locale': PREFIX.lower(),
'dont_redirect': True,
}
)
yield Request(
str(u.with_host('www.hotels.cn')),
self._rooms_parse,
meta={
'hotels_id': hotels_id,
'locale': 'en',
'dont_redirect': True,
# 'proxy': '172.17.1.198:1118'
}
)
supplier_obj_id = response.meta.get('statics.hotels.id')
supplier_name = response.meta.get('statics.hotels.supplier')
if supplier_obj_id:
loader.add_value('statics_hotels_id', supplier_obj_id)
loader.add_value('statics_hotels_supplier', supplier_name)
loader.add_value('hotels_id', hotels_id)
loader.add_xpath('title', hotels_xp.TITLE)
loader.add_xpath('name', hotels_xp.NAME)
position = take_first(response, hotels_xp.POSITION)
# loader.add_xpath('latitude', hotels_xp.LATITUDE)
# loader.add_xpath('longitude', hotels_xp.LONGITUDE)
latitude, longitude = position.split(',')
loader.add_value('latitude', latitude)
loader.add_xpath('longitude', longitude)
loader.add_xpath('star', hotels_xp.STAR, re='(\d+)')
loader.add_value('address', self._address(response))
loader.add_xpath('address_text', hotels_xp.ADDRESS)
loader.add_xpath('price', hotels_xp.PRICE)
loader.add_xpath('city', hotels_xp.ADDRESS_LOCALITY)
loader.add_xpath('country', hotels_xp.ADDRESS_COUNTRY)
loader.add_xpath('telephone', hotels_xp.TELEPHONE)
# loader.add_xpath('rating', '')
# loader.add_xpath('tripadvisor', '')
loader.add_xpath('landmarks', hotels_xp.LANDMARKS)
loader.add_xpath('traffic_tips', hotels_xp.TRAFFIC_TIPS)
loader.add_value('notice', self._notice(response, PREFIX))
IN_STORE_SERVICES = getattr(
hotels_xp,
PREFIX + '_IN_STORE_SERVICE_FACT'
)
loader.add_value(
'in_store_service_facilities',
service_to_dict(response.xpath(IN_STORE_SERVICES), hotels_xp)
)
ROOM_SERVICES = getattr(hotels_xp, PREFIX + '_ROOM_SERVICE_FACT')
loader.add_value(
'room_service_facilities',
service_to_dict(response.xpath(ROOM_SERVICES), hotels_xp)
)
loader.add_xpath('around', hotels_xp.AROUND)
pictures = self._pictures(response.xpath(hotels_xp.PICTURES))
loader.add_value('pictures', pictures)
loader.add_xpath('short_introduction', hotels_xp.SHORT_INTRODUCTION)
loader.add_xpath('amenities', hotels_xp.AMENITIES)
loader.add_xpath('for_families', hotels_xp.FOR_FAMILIES)
loader.add_value('locale', PREFIX.lower())
loader.add_value('summary', self._summary(response))
yield loader.load_item()
def __init__(
self,
db=None,
hotel_id="",
jset_id="",
hotels_cn_id="",
bonotel="",
hotelspro="",
hotelbeds="",
bookings_id="",
jactravel="",
city=None,
capture_url=None,
comments_url=None,
capture_id=None,
):
if (not hotel_id and not jset_id and not hotels_cn_id
and not bookings_id):
raise HotelNotFound("<>")
self.city = city
self._capture_url_ = capture_url
self._capture_id = capture_id
self.db = db or MongoClient(settings.MONGO).scripture
self.captured_urls_host = "http://scripture.weegotr.com/captured_from"
self._doc = {
"capture_id": capture_id,
"capture_url": capture_url,
"hotels_id": hotels_cn_id,
"quote_ids": [],
}
if bonotel:
self._doc["quote_ids"].append({
"hotel_id": bonotel,
"quoter": "59b20729a389295de92f4952"
})
if hotel_id:
self._doc["quote_ids"].append({
"hotel_id": hotel_id,
"quoter": "59afe1e0a389295de92f47f8"
})
if hotelspro:
self._doc["quote_ids"].append({
"hotel_id": hotelspro,
"quoter": "59b0b048a389295de92f4815"
})
if hotelbeds:
self._doc["quote_ids"].append({
"hotel_id": hotelbeds,
"quoter": "59c1f0f8e1812033000e1de8"
})
if jactravel:
self._doc["quote_ids"].append({
"hotel_id": jactravel,
"quoter": "59b0b035a389295de92f4814"
})
self.__loaded_from_jset = False
self.__loaded_from_rxml = False
self.__loaded_from_hotels_cn = False
self.__comments_url = comments_url
self._jset_id = jset_id
self._hotel_id = hotel_id
self._hotels_cn_id = hotels_cn_id
if hotel_id:
self.from_rxml(hotel_id)
if jset_id:
self.from_jset(jset_id)
if hotels_cn_id:
self.from_hotels_cn(hotels_cn_id)
if bookings_id:
self.from_bookings(bookings_id)
host = URL("http://feifanweige.com/").with_path("api/internal")
if os.environ.get("SCRIPTURE") == "SANDBOX":
self.host = host.with_host("172.16.4.110:8289")
else:
self.host = host.with_host(
"api.cn-beijing.aliyuncs.feifanweige.com")
def run_app(app, *, host=None, port=None, path=None,
shutdown_timeout=60.0, ssl_context=None,
print=print, backlog=128, access_log_format=None,
access_log=access_logger):
"""Run an app locally"""
loop = app.loop
make_handler_kwargs = dict()
if access_log_format is not None:
make_handler_kwargs['access_log_format'] = access_log_format
handler = app.make_handler(access_log=access_log,
**make_handler_kwargs)
loop.run_until_complete(app.startup())
scheme = 'https' if ssl_context else 'http'
base_url = URL('{}://localhost'.format(scheme)).with_port(port)
if path is None:
paths = ()
elif isinstance(path, (str, bytes, bytearray, memoryview))\
or not isinstance(path, Iterable):
paths = (path,)
else:
paths = path
if host is None:
if paths and not port:
hosts = ()
else:
hosts = ("0.0.0.0",)
elif isinstance(host, (str, bytes, bytearray, memoryview))\
or not isinstance(host, Iterable):
hosts = (host,)
else:
hosts = host
if hosts and port is None:
port = 8443 if ssl_context else 8080
server_creations = []
uris = [str(base_url.with_host(host)) for host in hosts]
if hosts:
# Multiple hosts bound to same server is available in most loop
# implementations, but only send multiple if we have multiple.
host_binding = hosts[0] if len(hosts) == 1 else hosts
server_creations.append(
loop.create_server(
handler, host_binding, port, ssl=ssl_context, backlog=backlog
)
)
for path in paths:
# Most loop implementations don't support multiple paths bound in same
# server, so create a server for each.
server_creations.append(
loop.create_unix_server(
handler, path, ssl=ssl_context, backlog=backlog
)
)
uris.append('{}://unix:{}:'.format(scheme, path))
# Clean up prior socket path if stale and not abstract.
# CPython 3.5.3+'s event loop already does this. See
# https://github.com/python/asyncio/issues/425
if path[0] not in (0, '\x00'): # pragma: no branch
try:
if stat.S_ISSOCK(os.stat(path).st_mode):
os.remove(path)
except FileNotFoundError:
pass
servers = loop.run_until_complete(
asyncio.gather(*server_creations, loop=loop)
)
print("======== Running on {} ========\n"
"(Press CTRL+C to quit)".format(', '.join(uris)))
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
pass
finally:
server_closures = []
for srv in servers:
srv.close()
server_closures.append(srv.wait_closed())
loop.run_until_complete(asyncio.gather(*server_closures, loop=loop))
loop.run_until_complete(app.shutdown())
loop.run_until_complete(handler.shutdown(shutdown_timeout))
loop.run_until_complete(app.cleanup())
loop.close()
def test_from_str_with_host_ipv6():
url = URL('http://host:80')
url = url.with_host('::1')
assert url.raw_host == '::1'
def test_from_str_with_host_ipv4():
url = URL('http://host:80')
url = url.with_host('192.168.1.1')
assert url.raw_host == '192.168.1.1'
def test_from_str_with_host_ipv6():
url = URL('http://host:80')
url = url.with_host('::1')
assert url.raw_host == '::1'
def test_with_host_non_ascii():
url = URL('http://example.com:123')
url2 = url.with_host('историк.рф')
assert url2.raw_host == 'xn--h1aagokeh.xn--p1ai'
assert url2.host == 'историк.рф'
def test_with_host_empty():
url = URL('http://example.com:123')
with pytest.raises(ValueError):
url.with_host('')
def to_ipv6(url):
url = URL(url)
parts = list(map(int, url.host.split(".")))
ipv6_host = "2002:{:02x}{:02x}:{:02x}{:02x}::".format(*parts)
return str(url.with_host("[%s]" % ipv6_host))
def test_with_host_invalid_type():
url = URL('http://example.com:123')
with pytest.raises(TypeError):
url.with_host(None)
def _make_server_creators(handler, *, loop, ssl_context,
host, port, path, sock, backlog):
scheme = 'https' if ssl_context else 'http'
base_url = URL('{}://localhost'.format(scheme)).with_port(port)
if path is None:
paths = ()
elif isinstance(path, (str, bytes, bytearray, memoryview))\
or not isinstance(path, Iterable):
paths = (path,)
else:
paths = path
if sock is None:
socks = ()
elif not isinstance(sock, Iterable):
socks = (sock,)
else:
socks = sock
if host is None:
if (paths or socks) and not port:
hosts = ()
else:
hosts = ("0.0.0.0",)
elif isinstance(host, (str, bytes, bytearray, memoryview))\
or not isinstance(host, Iterable):
hosts = (host,)
else:
hosts = host
if hosts and port is None:
port = 8443 if ssl_context else 8080
server_creations = []
uris = [str(base_url.with_host(host)) for host in hosts]
if hosts:
# Multiple hosts bound to same server is available in most loop
# implementations, but only send multiple if we have multiple.
host_binding = hosts[0] if len(hosts) == 1 else hosts
server_creations.append(
loop.create_server(
handler, host_binding, port, ssl=ssl_context, backlog=backlog
)
)
for path in paths:
# Most loop implementations don't support multiple paths bound in same
# server, so create a server for each.
server_creations.append(
loop.create_unix_server(
handler, path, ssl=ssl_context, backlog=backlog
)
)
uris.append('{}://unix:{}:'.format(scheme, path))
# Clean up prior socket path if stale and not abstract.
# CPython 3.5.3+'s event loop already does this. See
# https://github.com/python/asyncio/issues/425
if path[0] not in (0, '\x00'): # pragma: no branch
try:
if stat.S_ISSOCK(os.stat(path).st_mode):
os.remove(path)
except FileNotFoundError:
pass
for sock in socks:
server_creations.append(
loop.create_server(
handler, sock=sock, ssl=ssl_context, backlog=backlog
)
)
if hasattr(socket, 'AF_UNIX') and sock.family == socket.AF_UNIX:
uris.append('{}://unix:{}:'.format(scheme, sock.getsockname()))
else:
host, port = sock.getsockname()
uris.append(str(base_url.with_host(host).with_port(port)))
return server_creations, uris
def test_from_str_with_host_ipv6():
url = URL("http://host:80")
url = url.with_host("::1")
assert url.raw_host == "::1"
def test_with_host():
url = URL("http://example.com:123")
assert str(url.with_host("example.org")) == "http://example.org:123"
def _make_server_creators(handler, *, loop, ssl_context, host, port, path,
sock, backlog):
scheme = 'https' if ssl_context else 'http'
base_url = URL('{}://localhost'.format(scheme)).with_port(port)
if path is None:
paths = ()
elif isinstance(path, (str, bytes, bytearray, memoryview))\
or not isinstance(path, Iterable):
paths = (path, )
else:
paths = path
if sock is None:
socks = ()
elif not isinstance(sock, Iterable):
socks = (sock, )
else:
socks = sock
if host is None:
if (paths or socks) and not port:
hosts = ()
else:
hosts = ("0.0.0.0", )
elif isinstance(host, (str, bytes, bytearray, memoryview))\
or not isinstance(host, Iterable):
hosts = (host, )
else:
hosts = host
if hosts and port is None:
port = 8443 if ssl_context else 8080
server_creations = []
uris = [str(base_url.with_host(host)) for host in hosts]
if hosts:
# Multiple hosts bound to same server is available in most loop
# implementations, but only send multiple if we have multiple.
host_binding = hosts[0] if len(hosts) == 1 else hosts
server_creations.append(
loop.create_server(handler,
host_binding,
port,
ssl=ssl_context,
backlog=backlog))
for path in paths:
# Most loop implementations don't support multiple paths bound in same
# server, so create a server for each.
server_creations.append(
loop.create_unix_server(handler,
path,
ssl=ssl_context,
backlog=backlog))
uris.append('{}://unix:{}:'.format(scheme, path))
# Clean up prior socket path if stale and not abstract.
# CPython 3.5.3+'s event loop already does this. See
# https://github.com/python/asyncio/issues/425
if path[0] not in (0, '\x00'): # pragma: no branch
try:
if stat.S_ISSOCK(os.stat(path).st_mode):
os.remove(path)
except FileNotFoundError:
pass
for sock in socks:
server_creations.append(
loop.create_server(handler,
sock=sock,
ssl=ssl_context,
backlog=backlog))
if hasattr(socket, 'AF_UNIX') and sock.family == socket.AF_UNIX:
uris.append('{}://unix:{}:'.format(scheme, sock.getsockname()))
else:
host, port = sock.getsockname()
uris.append(str(base_url.with_host(host).with_port(port)))
return server_creations, uris
def test_with_host():
url = URL("http://example.com:123")
assert str(url.with_host("example.org")) == "http://example.org:123"
