def download_file(url, dst=None, callback=None):
if sys.stdout.isatty():
callback = callback or _reporthook
else:
callback = callback or _nullhook
dst = dst or tempfile.mkstemp()[1]
# Assuming the KA Lite version is included in user agent because of an
# intention to create stats on learningequality.org
from kalite.version import user_agent
response = requests.get(
url,
allow_redirects=True,
stream=True,
headers={"user-agent": user_agent()}
)
# If a destination is set, then we'll write a file and send back updates
if dst:
chunk_size = 1024
with open(dst, 'wb') as fd:
for chunk_number, chunk in enumerate(response.iter_content(chunk_size)):
fd.write(chunk)
bytes_fetched = chunk_number * chunk_size
if 'content-length' not in response.headers:
fraction = 0.0
elif int(response.headers['content-length']) == 0:
fraction = 0.0
else:
total_size = float(response.headers['content-length'])
fraction = min(float(bytes_fetched) / total_size, 1.0)
callback(fraction)
return response
def central_server_down_or_error(error_msg):
""" If the central server is down, return a context that says so.
Otherwise, pass along the actual error returned by the central server.
error_msg: a string
"""
if error_msg:
from kalite.version import user_agent
if requests.get(settings.CENTRAL_SERVER_URL, headers={"user-agent": user_agent()}).status_code != 200:
return {"error_msg": _("Central Server is not reachable; please try again after some time.")}
else:
return {"error_msg": error_msg}
def post(self, path, payload={}, *args, **kwargs):
from kalite.version import user_agent
if self.verbose:
print "CLIENT: post %s" % path
return requests.post(
self.path_to_url(path),
data=json.dumps(payload),
headers={"user-agent": user_agent()}
)
def am_i_online(url,
expected_val=None,
search_string=None,
timeout=5,
allow_redirects=True):
"""Test whether we are online or not.
returns True or False.
Eats all exceptions!
"""
assert not (search_string and expected_val is not None
), "Search string and expected value cannot both be set"
from kalite.version import user_agent
try:
if not search_string and expected_val is None:
response = requests.head(url, headers={"user-agent": user_agent()})
else:
response = requests.get(url,
timeout=timeout,
allow_redirects=allow_redirects,
headers={"user-agent": user_agent()})
# Validate that response came from the requested url
if response.status_code != 200:
return False
elif not allow_redirects and response.url != url:
return False
# Check the output, if expected values are specified
if expected_val is not None:
return expected_val == response.text
elif search_string:
return search_string in response.text
return True
except Exception as e:
return False
def get(self, path, payload={}, *args, **kwargs):
from kalite.version import user_agent
# add a random parameter to ensure the request is not cached
payload["_"] = uuid.uuid4().hex
query = urllib.urlencode(payload)
if self.verbose:
logging.debug("CLIENT: get %s" % path)
kwargs['headers'] = kwargs.get('headers', {})
kwargs['headers']["user-agent"] = user_agent()
return requests.get(
self.path_to_url(path) + "?" + query,
*args,
**kwargs
)
def central_server_down_or_error(error_msg):
""" If the central server is down, return a context that says so.
Otherwise, pass along the actual error returned by the central server.
error_msg: a string
"""
if error_msg:
from kalite.version import user_agent
if requests.get(settings.CENTRAL_SERVER_URL,
headers={
"user-agent": user_agent()
}).status_code != 200:
return {
"error_msg":
_("Central Server is not reachable; please try again after some time."
)
}
else:
return {"error_msg": error_msg}
def am_i_online():
"""Test whether we are online or not.
returns True or False.
Eats all exceptions! <- great :( /benjaoming
"""
from kalite.version import user_agent
timeout = 15 # seconds
url = urljoin(settings.CENTRAL_SERVER_URL, reverse("get_server_info"))
try:
# Based on experience, 5 seconds is too little
response = requests.get(url,
timeout=timeout,
allow_redirects=False,
headers={"user-agent": user_agent()})
# Validate that response came from the requested url
if response.status_code != 200:
logger.warning(
"Unexpected response detecting online status: {}".format(
response))
return False
return True
except ReadTimeout:
logger.info(
("Assuming offline status, timeout={} seconds, timed out while "
"fetching {}").format(timeout, url))
return False
except ConnectionError:
logger.info(
"Assuming offline status, connection error while fetching {}".
format(url))
return False
except Exception as e:
logger.warning(
"Unhandled exception when detecting if online: {}".format(e))
return False
def download_file(url, dst=None, callback=None, fp=None, max_retries=5):
assert not (dst and fp)
callback = callback or _nullhook
from requests.adapters import HTTPAdapter
s = requests.Session()
# Define the way that we do retries.
# retries = 5
# backoff = 0.2
# sum(b * (2 ^ (r - 1)) for r in range(1,6))
# =2.4 seconds total retry backoff
# socket timeout is 20 (see above)
# = 102.4 seconds on an unconnected line
retries = Retry(
total=max_retries,
connect=max_retries,
read=max_retries,
backoff_factor=0.2,
)
s.mount('http://', HTTPAdapter(max_retries=retries))
# Assuming the KA Lite version is included in user agent because of an
# intention to create stats on learningequality.org
from kalite.version import user_agent
# Notice that we deliberate aren't using the ``timeout`` kwarg here, we
# will allow the stream to hang forever when a connection is disrupted
# but a download has already started. This is to not have to write "resume"
# logic on top of our retry logic.
response = s.get(
url,
allow_redirects=True,
stream=True,
# timeout=DOWNLOAD_SOCKET_TIMEOUT,
headers={"user-agent": user_agent()})
response.raise_for_status()
# Don't do this things until passed the raise_for_status() point
# If not called with a file pointer or destination, create a new temporary
# file
# If a destination is set, then we'll write a file and send back updates
if dst:
fp = open(dst, 'wb')
if not (dst or fp):
fp, dst = tempfile.mkstemp()[1]
chunk_size = 1024 * 50 # 50 KB
for chunk_number, chunk in enumerate(response.iter_content(chunk_size)):
fp.write(chunk)
bytes_fetched = chunk_number * chunk_size
if 'content-length' not in response.headers:
fraction = 0.0
elif int(response.headers['content-length']) == 0:
fraction = 0.0
else:
total_size = float(response.headers['content-length'])
fraction = min(float(bytes_fetched) / total_size, 1.0)
callback(fraction)
# Many operations expect a file pointer at 0 after having written the file
# successfully. For instance if it's passed on to a ZipFile object.
fp.seek(0)
# Verify file existence
dst = fp.name or None
if dst:
if os.path.isfile(dst):
size_on_disk = os.path.getsize(dst)
else:
size_on_disk = 0
if 'content-length' in response.headers:
size_header = int(response.headers['content-length'])
size_header = 0
if size_on_disk <= 0 or (size_header and size_on_disk != size_header):
logger.error(
("Error downloading {url}, incorrect file size, disk full? "
"Expected {header}, got {disk}").format(
url=url,
header=size_header,
disk=size_header,
))
raise RuntimeError("Download failed to write correct file.")
return response
def download_file(url, dst=None, callback=None, fp=None, max_retries=5):
assert not (dst and fp)
callback = callback or _nullhook
from requests.adapters import HTTPAdapter
s = requests.Session()
# Define the way that we do retries.
# retries = 5
# backoff = 0.2
# sum(b * (2 ^ (r - 1)) for r in range(1,6))
# =2.4 seconds total retry backoff
# socket timeout is 20 (see above)
# = 102.4 seconds on an unconnected line
retries = Retry(
total=max_retries,
connect=max_retries,
read=max_retries,
backoff_factor=0.2,
)
s.mount('http://', HTTPAdapter(max_retries=retries))
# Assuming the KA Lite version is included in user agent because of an
# intention to create stats on learningequality.org
from kalite.version import user_agent
# Notice that we deliberate aren't using the ``timeout`` kwarg here, we
# will allow the stream to hang forever when a connection is disrupted
# but a download has already started. This is to not have to write "resume"
# logic on top of our retry logic.
response = s.get(
url,
allow_redirects=True,
stream=True,
# timeout=DOWNLOAD_SOCKET_TIMEOUT,
headers={"user-agent": user_agent()}
)
response.raise_for_status()
# Don't do this things until passed the raise_for_status() point
# If not called with a file pointer or destination, create a new temporary
# file
# If a destination is set, then we'll write a file and send back updates
if dst:
fp = open(dst, 'wb')
if not (dst or fp):
fp, dst = tempfile.mkstemp()[1]
chunk_size = 1024 * 50 # 50 KB
for chunk_number, chunk in enumerate(response.iter_content(chunk_size)):
fp.write(chunk)
bytes_fetched = chunk_number * chunk_size
if 'content-length' not in response.headers:
fraction = 0.0
elif int(response.headers['content-length']) == 0:
fraction = 0.0
else:
total_size = float(response.headers['content-length'])
fraction = min(float(bytes_fetched) / total_size, 1.0)
callback(fraction)
# Many operations expect a file pointer at 0 after having written the file
# successfully. For instance if it's passed on to a ZipFile object.
fp.seek(0)
# Verify file existence
dst = fp.name or None
if dst:
if os.path.isfile(dst):
size_on_disk = os.path.getsize(dst)
else:
size_on_disk = 0
if 'content-length' in response.headers:
size_header = int(response.headers['content-length'])
size_header = 0
if size_on_disk <=0 or (size_header and size_on_disk != size_header):
logger.error((
"Error downloading {url}, incorrect file size, disk full? "
"Expected {header}, got {disk}").format(
url=url,
header=size_header,
disk=size_header,
)
)
raise RuntimeError("Download failed to write correct file.")
return response
