def write(self, key, value, disable_zip=False):
if key.endswith(".json") or key.endswith(".zip"):
Log.error("Expecting a pure key")
try:
if hasattr(value, "read"):
if disable_zip:
storage = self.bucket.new_key(str(key + ".json"))
string_length = len(value)
headers = {"Content-Type": mimetype.JSON}
else:
storage = self.bucket.new_key(str(key + ".json.gz"))
string_length = len(value)
value = convert.bytes2zip(value)
headers = {"Content-Type": mimetype.GZIP}
file_length = len(value)
Log.note(
"Sending contents with length {{file_length|comma}} (from string with length {{string_length|comma}})",
file_length=file_length,
string_length=string_length,
)
value.seek(0)
storage.set_contents_from_file(value, headers=headers)
if self.settings.public:
storage.set_acl("public-read")
return
if len(value) > 20 * 1000 and not disable_zip:
self.bucket.delete_key(str(key + ".json"))
self.bucket.delete_key(str(key + ".json.gz"))
if is_binary(value):
value = convert.bytes2zip(value)
key += ".json.gz"
else:
value = convert.bytes2zip(value).encode("utf8")
key += ".json.gz"
headers = {"Content-Type": mimetype.GZIP}
else:
self.bucket.delete_key(str(key + ".json.gz"))
if is_binary(value):
key += ".json"
else:
key += ".json"
headers = {"Content-Type": mimetype.JSON}
storage = self.bucket.new_key(str(key))
storage.set_contents_from_string(value, headers=headers)
if self.settings.public:
storage.set_acl("public-read")
except Exception as e:
Log.error(
"Problem writing {{bytes}} bytes to {{key}} in {{bucket}}",
key=key,
bucket=self.bucket.name,
bytes=len(value),
cause=e,
)
def write(self, key, value, disable_zip=False):
if key.endswith(".json") or key.endswith(".zip"):
Log.error("Expecting a pure key")
try:
if hasattr(value, "read"):
if disable_zip:
storage = self.bucket.new_key(key + ".json")
string_length = len(value)
else:
storage = self.bucket.new_key(key + ".json.gz")
string_length = len(value)
value = convert.bytes2zip(value)
file_length = len(value)
Log.note(
"Sending contents with length {{file_length|comma}} (from string with length {{string_length|comma}})",
file_length=file_length,
string_length=string_length)
value.seek(0)
storage.set_contents_from_file(value)
if self.settings.public:
storage.set_acl('public-read')
return
if len(value) > 20 * 1000 and not disable_zip:
self.bucket.delete_key(key + ".json")
self.bucket.delete_key(key + ".json.gz")
if isinstance(value, str):
value = convert.bytes2zip(value)
key += ".json.gz"
else:
value = convert.bytes2zip(convert.unicode2utf8(value))
key += ".json.gz"
else:
self.bucket.delete_key(key + ".json.gz")
if isinstance(value, str):
key += ".json"
else:
key += ".json"
storage = self.bucket.new_key(key)
storage.set_contents_from_string(value)
if self.settings.public:
storage.set_acl('public-read')
except Exception, e:
Log.error(
"Problem writing {{bytes}} bytes to {{key}} in {{bucket}}",
key=key,
bucket=self.bucket.name,
bytes=len(value),
cause=e)
def write(self, key, value, disable_zip=False):
if key.endswith(".json") or key.endswith(".zip"):
Log.error("Expecting a pure key")
try:
if hasattr(value, "read"):
if disable_zip:
storage = self.bucket.new_key(key + ".json")
string_length = len(value)
else:
storage = self.bucket.new_key(key + ".json.gz")
string_length = len(value)
value = convert.bytes2zip(value)
file_length = len(value)
Log.note("Sending contents with length {{file_length|comma}} (from string with length {{string_length|comma}})", file_length= file_length, string_length=string_length)
value.seek(0)
storage.set_contents_from_file(value)
if self.settings.public:
storage.set_acl('public-read')
return
if len(value) > 20 * 1000 and not disable_zip:
self.bucket.delete_key(key + ".json")
self.bucket.delete_key(key + ".json.gz")
if isinstance(value, str):
value = convert.bytes2zip(value)
key += ".json.gz"
else:
value = convert.bytes2zip(convert.unicode2utf8(value))
key += ".json.gz"
else:
self.bucket.delete_key(key + ".json.gz")
if isinstance(value, str):
key += ".json"
else:
key += ".json"
storage = self.bucket.new_key(key)
storage.set_contents_from_string(value)
if self.settings.public:
storage.set_acl('public-read')
except Exception, e:
Log.error(
"Problem writing {{bytes}} bytes to {{key}} in {{bucket}}",
key=key,
bucket=self.bucket.name,
bytes=len(value),
cause=e
)
retry = wrap(retry)
if isinstance(retry.sleep, Duration):
retry.sleep = retry.sleep.seconds
set_default(retry, {"times": 1, "sleep": 0})
if b'json' in kwargs:
kwargs[b'data'] = convert.value2json(kwargs[b'json']).encode("utf8")
del kwargs[b'json']
try:
headers = kwargs[b"headers"] = unwrap(
coalesce(wrap(kwargs)[b"headers"], {}))
set_default(headers, {b"accept-encoding": b"compress, gzip"})
if zip and len(coalesce(kwargs.get(b"data"))) > 1000:
compressed = convert.bytes2zip(kwargs[b"data"])
headers[b'content-encoding'] = b'gzip'
kwargs[b"data"] = compressed
_to_ascii_dict(headers)
else:
_to_ascii_dict(headers)
except Exception, e:
Log.error("Request setup failure on {{url}}", url=url, cause=e)
errors = []
for r in range(retry.times):
if r:
Till(seconds=retry.sleep).wait()
try:
elif isinstance(retry, Number):
retry = Dict(times=retry, sleep=SECOND)
else:
retry = wrap(retry)
set_default(retry.sleep, {"times": 1, "sleep": 0})
if b'json' in kwargs:
kwargs[b'data'] = convert.value2json(kwargs[b'json']).encode("utf8")
del kwargs[b'json']
try:
headers = kwargs[b"headers"] = unwrap(coalesce(wrap(kwargs)[b"headers"], {}))
set_default(headers, {b"accept-encoding": b"compress, gzip"})
if zip and len(coalesce(kwargs.get(b"data"))) > 1000:
compressed = convert.bytes2zip(kwargs[b"data"])
headers[b'content-encoding'] = b'gzip'
kwargs[b"data"] = compressed
_to_ascii_dict(headers)
else:
_to_ascii_dict(headers)
except Exception, e:
Log.error("Request setup failure on {{url}}", url=url, cause=e)
errors = []
for r in range(retry.times):
if r:
Thread.sleep(retry.sleep)
try:
def request(method, url, zip=None, retry=None, **kwargs):
"""
JUST LIKE requests.request() BUT WITH DEFAULT HEADERS AND FIXES
DEMANDS data IS ONE OF:
* A JSON-SERIALIZABLE STRUCTURE, OR
* LIST OF JSON-SERIALIZABLE STRUCTURES, OR
* None
Parameters
* zip - ZIP THE REQUEST BODY, IF BIG ENOUGH
* json - JSON-SERIALIZABLE STRUCTURE
* retry - {"times": x, "sleep": y} STRUCTURE
THE BYTE_STRINGS (b"") ARE NECESSARY TO PREVENT httplib.py FROM **FREAKING OUT**
IT APPEARS requests AND httplib.py SIMPLY CONCATENATE STRINGS BLINDLY, WHICH
INCLUDES url AND headers
"""
global _warning_sent
if not default_headers and not _warning_sent:
_warning_sent = True
Log.warning(
"The pyLibrary.env.http module was meant to add extra "
"default headers to all requests, specifically the 'Referer' "
"header with a URL to the project. Use the `pyLibrary.debug.constants.set()` "
"function to set `pyLibrary.env.http.default_headers`")
if isinstance(url, list):
# TRY MANY URLS
failures = []
for remaining, u in jx.countdown(url):
try:
response = request(method, u, zip=zip, retry=retry, **kwargs)
if Math.round(response.status_code,
decimal=-2) not in [400, 500]:
return response
if not remaining:
return response
except Exception as e:
e = Except.wrap(e)
failures.append(e)
Log.error(u"Tried {{num}} urls", num=len(url), cause=failures)
if 'session' in kwargs:
session = kwargs['session']
del kwargs['session']
sess = Null
else:
sess = session = sessions.Session()
session.headers.update(default_headers)
with closing(sess):
if zip is None:
zip = ZIP_REQUEST
if isinstance(url, text_type):
# httplib.py WILL **FREAK OUT** IF IT SEES ANY UNICODE
url = url.encode('ascii')
_to_ascii_dict(kwargs)
timeout = kwargs['timeout'] = coalesce(kwargs.get('timeout'),
default_timeout)
if retry == None:
retry = Data(times=1, sleep=0)
elif isinstance(retry, Number):
retry = Data(times=retry, sleep=1)
else:
retry = wrap(retry)
if isinstance(retry.sleep, Duration):
retry.sleep = retry.sleep.seconds
set_default(retry, {"times": 1, "sleep": 0})
if 'json' in kwargs:
kwargs['data'] = value2json(kwargs['json']).encode('utf8')
del kwargs['json']
try:
headers = kwargs['headers'] = unwrap(
coalesce(kwargs.get('headers'), {}))
set_default(headers, {'Accept-Encoding': 'compress, gzip'})
if zip and len(coalesce(kwargs.get('data'))) > 1000:
compressed = convert.bytes2zip(kwargs['data'])
headers['content-encoding'] = 'gzip'
kwargs['data'] = compressed
_to_ascii_dict(headers)
else:
_to_ascii_dict(headers)
except Exception as e:
Log.error(u"Request setup failure on {{url}}", url=url, cause=e)
errors = []
for r in range(retry.times):
if r:
Till(seconds=retry.sleep).wait()
try:
if DEBUG:
Log.note(u"http {{method}} to {{url}}",
method=method,
url=url)
return session.request(method=method, url=url, **kwargs)
except Exception as e:
errors.append(Except.wrap(e))
if " Read timed out." in errors[0]:
Log.error(
u"Tried {{times}} times: Timeout failure (timeout was {{timeout}}",
timeout=timeout,
times=retry.times,
cause=errors[0])
else:
Log.error(u"Tried {{times}} times: Request failure of {{url}}",
url=url,
times=retry.times,
cause=errors[0])
def request(method, url, **kwargs):
"""
JUST LIKE requests.request() BUT WITH DEFAULT HEADERS AND FIXES
DEMANDS data IS ONE OF:
* A JSON-SERIALIZABLE STRUCTURE, OR
* LIST OF JSON-SERIALIZABLE STRUCTURES, OR
* None
THE BYTE_STRINGS (b"") ARE NECESSARY TO PREVENT httplib.py FROM **FREAKING OUT**
IT APPEARS requests AND httplib.py SIMPLY CONCATENATE STRINGS BLINDLY, WHICH
INCLUDES url AND headers
"""
global _warning_sent
if not default_headers and not _warning_sent:
_warning_sent = True
Log.warning("The pyLibrary.env.http module was meant to add extra "
"default headers to all requests, specifically the 'From' "
"header with a URL to the project, or email of developer. "
"Use the constants.set() function to set pyLibrary.env.http.default_headers"
)
session = sessions.Session()
session.headers.update(default_headers)
if isinstance(url, unicode):
# httplib.py WILL **FREAK OUT** IF IT SEES ANY UNICODE
url = url.encode("ascii")
# if "data" not in kwargs:
# pass
# elif kwargs["data"] == None:
# pass
# elif isinstance(kwargs["data"], basestring):
# Log.error("Expecting `data` to be a structure")
# elif isinstance(kwargs["data"], list):
# #CR-DELIMITED JSON IS ALSO ACCEPTABLE
# kwargs["data"] = b"\n".join(convert.unicode2utf8(convert.value2json(d)) for d in kwargs["data"])
# else:
# kwargs["data"] = convert.unicode2utf8(convert.value2json(kwargs["data"]))
_to_ascii_dict(kwargs)
timeout = kwargs[b'timeout'] = coalesce(kwargs.get(b'timeout'), default_timeout)
try:
if len(coalesce(kwargs.get(b"data"))) > 1000:
compressed = convert.bytes2zip(kwargs[b"data"])
if b"headers" not in kwargs:
kwargs[b"headers"] = {}
kwargs[b"headers"][b'content-encoding'] = b'gzip'
kwargs[b"data"] = compressed
_to_ascii_dict(kwargs[b"headers"])
return session.request(method=method, url=url, **kwargs)
else:
_to_ascii_dict(kwargs.get(b"headers"))
return session.request(method=method, url=url, **kwargs)
except Exception, e:
if " Read timed out." in e:
Log.error("Timeout failure (timeout was {{timeout}}", timeout=timeout, cause=e)
else:
Log.error("Request failure", e)
def request(method, url, headers=None, zip=None, retry=None, **kwargs):
"""
JUST LIKE requests.request() BUT WITH DEFAULT HEADERS AND FIXES
DEMANDS data IS ONE OF:
* A JSON-SERIALIZABLE STRUCTURE, OR
* LIST OF JSON-SERIALIZABLE STRUCTURES, OR
* None
Parameters
* zip - ZIP THE REQUEST BODY, IF BIG ENOUGH
* json - JSON-SERIALIZABLE STRUCTURE
* retry - {"times": x, "sleep": y} STRUCTURE
THE BYTE_STRINGS (b"") ARE NECESSARY TO PREVENT httplib.py FROM **FREAKING OUT**
IT APPEARS requests AND httplib.py SIMPLY CONCATENATE STRINGS BLINDLY, WHICH
INCLUDES url AND headers
"""
global _warning_sent
global request_count
if not _warning_sent and not default_headers:
Log.warning(
text(
"The pyLibrary.env.http module was meant to add extra " +
"default headers to all requests, specifically the 'Referer' "
+
"header with a URL to the project. Use the `pyLibrary.debug.constants.set()` "
+ "function to set `pyLibrary.env.http.default_headers`"))
_warning_sent = True
if is_list(url):
# TRY MANY URLS
failures = []
for remaining, u in jx.countdown(url):
try:
response = request(method, u, retry=retry, **kwargs)
if mo_math.round(response.status_code,
decimal=-2) not in [400, 500]:
return response
if not remaining:
return response
except Exception as e:
e = Except.wrap(e)
failures.append(e)
Log.error(u"Tried {{num}} urls", num=len(url), cause=failures)
if 'session' in kwargs:
session = kwargs['session']
del kwargs['session']
sess = Null
else:
sess = session = sessions.Session()
with closing(sess):
if PY2 and is_text(url):
# httplib.py WILL **FREAK OUT** IF IT SEES ANY UNICODE
url = url.encode('ascii')
try:
set_default(kwargs, {"zip": zip, "retry": retry}, DEFAULTS)
_to_ascii_dict(kwargs)
# HEADERS
headers = kwargs['headers'] = unwrap(
set_default(headers, session.headers, default_headers))
_to_ascii_dict(headers)
del kwargs['headers']
# RETRY
retry = wrap(kwargs['retry'])
if isinstance(retry, Number):
retry = set_default({"times": retry}, DEFAULTS['retry'])
if isinstance(retry.sleep, Duration):
retry.sleep = retry.sleep.seconds
del kwargs['retry']
# JSON
if 'json' in kwargs:
kwargs['data'] = value2json(kwargs['json']).encode('utf8')
del kwargs['json']
# ZIP
set_default(headers, {'Accept-Encoding': 'compress, gzip'})
if kwargs['zip'] and len(coalesce(kwargs.get('data'))) > 1000:
compressed = convert.bytes2zip(kwargs['data'])
headers['content-encoding'] = 'gzip'
kwargs['data'] = compressed
del kwargs['zip']
except Exception as e:
Log.error(u"Request setup failure on {{url}}", url=url, cause=e)
errors = []
for r in range(retry.times):
if r:
Till(seconds=retry.sleep).wait()
try:
DEBUG and Log.note(u"http {{method|upper}} to {{url}}",
method=method,
url=text(url))
request_count += 1
return session.request(method=method,
headers=headers,
url=str(url),
**kwargs)
except Exception as e:
e = Except.wrap(e)
if retry['http'] and str(url).startswith(
"https://"
) and "EOF occurred in violation of protocol" in e:
url = URL("http://" + str(url)[8:])
Log.note(
"Changed {{url}} to http due to SSL EOF violation.",
url=str(url))
errors.append(e)
if " Read timed out." in errors[0]:
Log.error(
u"Tried {{times}} times: Timeout failure (timeout was {{timeout}}",
timeout=kwargs['timeout'],
times=retry.times,
cause=errors[0])
else:
Log.error(u"Tried {{times}} times: Request failure of {{url}}",
url=url,
times=retry.times,
cause=errors[0])
def request(method, url, headers=None, zip=None, retry=None, **kwargs):
"""
JUST LIKE requests.request() BUT WITH DEFAULT HEADERS AND FIXES
DEMANDS data IS ONE OF:
* A JSON-SERIALIZABLE STRUCTURE, OR
* LIST OF JSON-SERIALIZABLE STRUCTURES, OR
* None
Parameters
* zip - ZIP THE REQUEST BODY, IF BIG ENOUGH
* json - JSON-SERIALIZABLE STRUCTURE
* retry - {"times": x, "sleep": y} STRUCTURE
THE BYTE_STRINGS (b"") ARE NECESSARY TO PREVENT httplib.py FROM **FREAKING OUT**
IT APPEARS requests AND httplib.py SIMPLY CONCATENATE STRINGS BLINDLY, WHICH
INCLUDES url AND headers
"""
global _warning_sent
global request_count
if not _warning_sent and not default_headers:
Log.warning(text_type(
"The pyLibrary.env.http module was meant to add extra " +
"default headers to all requests, specifically the 'Referer' " +
"header with a URL to the project. Use the `pyLibrary.debug.constants.set()` " +
"function to set `pyLibrary.env.http.default_headers`"
))
_warning_sent = True
if is_list(url):
# TRY MANY URLS
failures = []
for remaining, u in jx.countdown(url):
try:
response = request(method, u, retry=retry, **kwargs)
if mo_math.round(response.status_code, decimal=-2) not in [400, 500]:
return response
if not remaining:
return response
except Exception as e:
e = Except.wrap(e)
failures.append(e)
Log.error(u"Tried {{num}} urls", num=len(url), cause=failures)
if 'session' in kwargs:
session = kwargs['session']
del kwargs['session']
sess = Null
else:
sess = session = sessions.Session()
with closing(sess):
if PY2 and is_text(url):
# httplib.py WILL **FREAK OUT** IF IT SEES ANY UNICODE
url = url.encode('ascii')
try:
set_default(kwargs, {"zip":zip, "retry": retry}, DEFAULTS)
_to_ascii_dict(kwargs)
# HEADERS
headers = kwargs['headers'] = unwrap(set_default(headers, session.headers, default_headers))
_to_ascii_dict(headers)
del kwargs['headers']
# RETRY
retry = wrap(kwargs['retry'])
if isinstance(retry, Number):
retry = set_default({"times":retry}, DEFAULTS['retry'])
if isinstance(retry.sleep, Duration):
retry.sleep = retry.sleep.seconds
del kwargs['retry']
# JSON
if 'json' in kwargs:
kwargs['data'] = value2json(kwargs['json']).encode('utf8')
del kwargs['json']
# ZIP
set_default(headers, {'Accept-Encoding': 'compress, gzip'})
if kwargs['zip'] and len(coalesce(kwargs.get('data'))) > 1000:
compressed = convert.bytes2zip(kwargs['data'])
headers['content-encoding'] = 'gzip'
kwargs['data'] = compressed
del kwargs['zip']
except Exception as e:
Log.error(u"Request setup failure on {{url}}", url=url, cause=e)
errors = []
for r in range(retry.times):
if r:
Till(seconds=retry.sleep).wait()
try:
DEBUG and Log.note(u"http {{method|upper}} to {{url}}", method=method, url=text_type(url))
request_count += 1
return session.request(method=method, headers=headers, url=str(url), **kwargs)
except Exception as e:
e = Except.wrap(e)
if retry['http'] and str(url).startswith("https://") and "EOF occurred in violation of protocol" in e:
url = URL("http://" + str(url)[8:])
Log.note("Changed {{url}} to http due to SSL EOF violation.", url=str(url))
errors.append(e)
if " Read timed out." in errors[0]:
Log.error(u"Tried {{times}} times: Timeout failure (timeout was {{timeout}}", timeout=kwargs['timeout'], times=retry.times, cause=errors[0])
else:
Log.error(u"Tried {{times}} times: Request failure of {{url}}", url=url, times=retry.times, cause=errors[0])
def request(method, url, **kwargs):
"""
JUST LIKE requests.request() BUT WITH DEFAULT HEADERS AND FIXES
DEMANDS data IS ONE OF:
* A JSON-SERIALIZABLE STRUCTURE, OR
* LIST OF JSON-SERIALIZABLE STRUCTURES, OR
* None
THE BYTE_STRINGS (b"") ARE NECESSARY TO PREVENT httplib.py FROM **FREAKING OUT**
IT APPEARS requests AND httplib.py SIMPLY CONCATENATE STRINGS BLINDLY, WHICH
INCLUDES url AND headers
"""
global _warning_sent
if not default_headers and not _warning_sent:
_warning_sent = True
Log.warning(
"The pyLibrary.env.http module was meant to add extra "
"default headers to all requests, specifically the 'From' "
"header with a URL to the project, or email of developer. "
"Use the constants.set() function to set pyLibrary.env.http.default_headers"
)
session = sessions.Session()
session.headers.update(default_headers)
if isinstance(url, unicode):
# httplib.py WILL **FREAK OUT** IF IT SEES ANY UNICODE
url = url.encode("ascii")
# if "data" not in kwargs:
# pass
# elif kwargs["data"] == None:
# pass
# elif isinstance(kwargs["data"], basestring):
# Log.error("Expecting `data` to be a structure")
# elif isinstance(kwargs["data"], list):
# #CR-DELIMITED JSON IS ALSO ACCEPTABLE
# kwargs["data"] = b"\n".join(convert.unicode2utf8(convert.value2json(d)) for d in kwargs["data"])
# else:
# kwargs["data"] = convert.unicode2utf8(convert.value2json(kwargs["data"]))
_to_ascii_dict(kwargs)
timeout = kwargs[b'timeout'] = coalesce(kwargs.get(b'timeout'),
default_timeout)
try:
if len(coalesce(kwargs.get(b"data"))) > 1000:
compressed = convert.bytes2zip(kwargs[b"data"])
if b"headers" not in kwargs:
kwargs[b"headers"] = {}
kwargs[b"headers"][b'content-encoding'] = b'gzip'
kwargs[b"data"] = compressed
_to_ascii_dict(kwargs[b"headers"])
return session.request(method=method, url=url, **kwargs)
else:
_to_ascii_dict(kwargs.get(b"headers"))
return session.request(method=method, url=url, **kwargs)
except Exception, e:
if " Read timed out." in e:
Log.error("Timeout failure (timeout was {{timeout}}",
timeout=timeout,
cause=e)
else:
Log.error("Request failure", e)
def request(method, url, zip=None, retry=None, **kwargs):
"""
JUST LIKE requests.request() BUT WITH DEFAULT HEADERS AND FIXES
DEMANDS data IS ONE OF:
* A JSON-SERIALIZABLE STRUCTURE, OR
* LIST OF JSON-SERIALIZABLE STRUCTURES, OR
* None
Parameters
* zip - ZIP THE REQUEST BODY, IF BIG ENOUGH
* json - JSON-SERIALIZABLE STRUCTURE
* retry - {"times": x, "sleep": y} STRUCTURE
THE BYTE_STRINGS (b"") ARE NECESSARY TO PREVENT httplib.py FROM **FREAKING OUT**
IT APPEARS requests AND httplib.py SIMPLY CONCATENATE STRINGS BLINDLY, WHICH
INCLUDES url AND headers
"""
global _warning_sent
if not default_headers and not _warning_sent:
_warning_sent = True
Log.warning(
"The pyLibrary.env.http module was meant to add extra "
"default headers to all requests, specifically the 'Referer' "
"header with a URL to the project. Use the `pyLibrary.debug.constants.set()` "
"function to set `pyLibrary.env.http.default_headers`"
)
if isinstance(url, list):
# TRY MANY URLS
failures = []
for remaining, u in jx.countdown(url):
try:
response = request(method, u, zip=zip, retry=retry, **kwargs)
if Math.round(response.status_code, decimal=-2) not in [400, 500]:
return response
if not remaining:
return response
except Exception as e:
e = Except.wrap(e)
failures.append(e)
Log.error("Tried {{num}} urls", num=len(url), cause=failures)
if b"session" in kwargs:
session = kwargs[b"session"]
del kwargs[b"session"]
else:
session = sessions.Session()
session.headers.update(default_headers)
if zip is None:
zip = ZIP_REQUEST
if isinstance(url, unicode):
# httplib.py WILL **FREAK OUT** IF IT SEES ANY UNICODE
url = url.encode("ascii")
_to_ascii_dict(kwargs)
timeout = kwargs[b'timeout'] = coalesce(kwargs.get(b'timeout'), default_timeout)
if retry == None:
retry = Data(times=1, sleep=0)
elif isinstance(retry, Number):
retry = Data(times=retry, sleep=1)
else:
retry = wrap(retry)
if isinstance(retry.sleep, Duration):
retry.sleep = retry.sleep.seconds
set_default(retry, {"times": 1, "sleep": 0})
if b'json' in kwargs:
kwargs[b'data'] = convert.value2json(kwargs[b'json']).encode("utf8")
del kwargs[b'json']
try:
headers = kwargs[b"headers"] = unwrap(coalesce(wrap(kwargs)[b"headers"], {}))
set_default(headers, {b"accept-encoding": b"compress, gzip"})
if zip and len(coalesce(kwargs.get(b"data"))) > 1000:
compressed = convert.bytes2zip(kwargs[b"data"])
headers[b'content-encoding'] = b'gzip'
kwargs[b"data"] = compressed
_to_ascii_dict(headers)
else:
_to_ascii_dict(headers)
except Exception as e:
Log.error("Request setup failure on {{url}}", url=url, cause=e)
errors = []
for r in range(retry.times):
if r:
Till(seconds=retry.sleep).wait()
try:
if DEBUG:
Log.note("http {{method}} to {{url}}", method=method, url=url)
return session.request(method=method, url=url, **kwargs)
except Exception as e:
errors.append(Except.wrap(e))
if " Read timed out." in errors[0]:
Log.error("Tried {{times}} times: Timeout failure (timeout was {{timeout}}", timeout=timeout, times=retry.times, cause=errors[0])
else:
Log.error("Tried {{times}} times: Request failure of {{url}}", url=url, times=retry.times, cause=errors[0])
