def spark_dist():
"""Returns the directory location of the local spark distribution.
Fetches it if it doesn't exist."""
app = spark_app()
spark_uri = app['labels']['SPARK_URI']
# <spark>.tgz
basename = posixpath.basename(spark_uri)
# <spark>
root = posixpath.splitext(basename)[0]
# data/
data_dir = _spark_dist_dir()
if not os.path.exists(data_dir):
os.makedirs(data_dir)
# data/<spark.tgz>
spark_archive = os.path.join(data_dir, basename)
# data/<spark>
spark_dir = os.path.join(data_dir, root)
# data/tmp
data_tmp_dir = os.path.join(data_dir, 'tmp')
# only download spark if data/<spark> doesn't yet exist
if not os.path.exists(spark_dir):
# download archive
print('Spark distribution {} not found locally.'.format(root))
print('It looks like this is your first time running Spark!')
print('Downloading {}...'.format(spark_uri))
resp = http.request('GET', spark_uri, stream=True)
resp.raise_for_status()
# write to data/<spark.tgz>
with open(spark_archive, 'wb') as spark_archive_file:
for block in resp:
spark_archive_file.write(block)
# extract to data/tmp/<spark>
print('Extracting spark distribution {}...'.format(spark_archive))
tf = tarfile.open(spark_archive)
tf.extractall(data_tmp_dir)
tf.close()
# move from data/tmp/<spark> to data/<spark>
spark_tmp = os.path.join(data_tmp_dir, root)
shutil.copytree(spark_tmp, spark_dir)
# clean up data/tmp/<spark> and data/<spark.tgz>
shutil.rmtree(spark_tmp)
os.remove(spark_archive)
print('Successfully fetched spark distribution {}!'.format(spark_uri))
return spark_dir
def spark_dist():
"""Returns the directory location of the local spark distribution.
Fetches it if it doesn't exist."""
app = spark_app()
spark_uri = app['labels']['SPARK_URI']
# <spark>.tgz
basename = posixpath.basename(spark_uri)
# <spark>
root = posixpath.splitext(basename)[0]
# data/
data_dir = _spark_dist_dir()
if not os.path.exists(data_dir):
os.makedirs(data_dir)
# data/<spark.tgz>
spark_archive = os.path.join(data_dir, basename)
# data/<spark>
spark_dir = os.path.join(data_dir, root)
# data/tmp
data_tmp_dir = os.path.join(data_dir, 'tmp')
# only download spark if data/<spark> doesn't yet exist
if not os.path.exists(spark_dir):
# download archive
print('Spark distribution {} not found locally.'.format(root))
print('It looks like this is your first time running Spark!')
print('Downloading {}...'.format(spark_uri))
resp = http.request('GET', spark_uri, stream=True)
resp.raise_for_status()
# write to data/<spark.tgz>
with open(spark_archive, 'wb') as spark_archive_file:
for block in resp:
spark_archive_file.write(block)
# extract to data/tmp/<spark>
print('Extracting spark distribution {}...'.format(spark_archive))
tf = tarfile.open(spark_archive)
tf.extractall(data_tmp_dir)
tf.close()
# move from data/tmp/<spark> to data/<spark>
spark_tmp = os.path.join(data_tmp_dir, root)
shutil.copytree(spark_tmp, spark_dir)
# clean up data/tmp/<spark> and data/<spark.tgz>
shutil.rmtree(spark_tmp)
os.remove(spark_archive)
print('Successfully fetched spark distribution {}!'.format(spark_uri))
return spark_dir
def _assert_request_timeout(requests_mock, timeout, toml_config,
expected_timeout):
resp = Response()
resp.status_code = 200
requests_mock.return_value = resp
method = 'GET'
url = 'https://www.example.com'
http.request(method,
url,
timeout=timeout,
headers={},
toml_config=toml_config)
requests_mock.assert_called_once_with(method=method,
url=url,
auth=None,
timeout=expected_timeout,
headers={},
verify=None)
def http_request(self, method, url, exit_on_failure=True, **kwargs):
try:
verify = True
if self.insecure_ssl:
verify = False
r = http.request(method, url, verify=verify, is_success=_default_is_success, **kwargs)
self.vlog_request(r)
if r.status_code == 404:
return FailedRequest(404, method, url, "Resource at " + url + " was not found (Status Code: 404)")
return r
except Exception as e:
if exit_on_failure:
raise e
else:
self.vlog(e)
return FailedRequest(0, method, url)
def http_request(self, method, url, exit_on_failure=True, **kwargs):
try:
verify = True
if self.insecure_ssl:
verify = False
r = http.request(method,
url,
verify=verify,
is_success=_default_is_success,
**kwargs)
self.vlog_request(r)
if r.status_code == 404:
return FailedRequest(
404, method, url,
'Resource at ' + url + ' was not found (Status Code: 404)')
return r
except Exception as e:
if exit_on_failure:
raise e
else:
self.vlog(e)
return FailedRequest(0, method, url)
def request(self, method: str, url: str, **kwargs: Any) -> http.requests.Response:
return http.request(method, url, toml_config=self.toml_config, **kwargs)