def find_file(self, filename):
"""
Try to find file or dir in search_path if it was specified. Helps finding files
in non-CLI environments or relative to config path
:param filename: file basename to find
:type filename: str
"""
if not filename:
return filename
filename = os.path.expanduser(filename)
if os.path.exists(filename):
return filename
elif filename.lower().startswith("http://") or filename.lower().startswith("https://"):
parsed_url = parse.urlparse(filename)
downloader = ExceptionalDownloader()
self.log.info("Downloading %s", filename)
tmp_f_name, http_msg = downloader.get(filename)
cd_header = http_msg.get('Content-Disposition', '')
dest = cd_header.split('filename=')[-1] if cd_header and 'filename=' in cd_header else ''
if not dest:
dest = os.path.basename(parsed_url.path)
fname, ext = os.path.splitext(dest) if dest else (parsed_url.hostname.replace(".", "_"), '.file')
dest = self.create_artifact(fname, ext)
self.log.debug("Moving %s to %s", tmp_f_name, dest)
shutil.move(tmp_f_name, dest)
return dest
elif self.file_search_paths:
for dirname in self.file_search_paths:
location = os.path.join(dirname, filename)
if os.path.exists(location):
self.log.warning("Guessed location from search paths for %s: %s", filename, location)
return location
self.log.warning("Could not find location at path: %s", filename)
return filename
def _download_logs(self):
for session in self.router.master.sessions():
assert isinstance(session, Session)
for log in session.get_logs():
self.log.info("Downloading %s from the cloud", log['filename'])
cloud_dir = os.path.join(self.engine.artifacts_dir,
'cloud-artifacts')
if not os.path.exists(cloud_dir):
os.makedirs(cloud_dir)
dest = os.path.join(cloud_dir, log['filename'])
dwn = ExceptionalDownloader(self.engine.get_http_client())
with ProgressBarContext() as pbar:
try:
dwn.get(log['dataUrl'],
dest,
reporthook=pbar.download_callback)
except BaseException:
self.log.debug("Error is: %s", traceback.format_exc())
self.log.warning("Failed to download from %s",
log['dataUrl'])
continue
if log['filename'].startswith(
'artifacts') and log['filename'].endswith('.zip'):
with zipfile.ZipFile(dest) as zipf:
for name in zipf.namelist():
ext = name.split('.')[-1].lower()
if ext in ('har', 'jpg', 'js', 'html', 'css'):
self.log.debug("Extracting %s to %s", name,
cloud_dir)
zipf.extract(name, cloud_dir)
def find_file(self, filename):
"""
Try to find file or dir in search_path if it was specified. Helps finding files
in non-CLI environments or relative to config path
Return path is full and mustn't treat with abspath/etc.
:param filename: file basename to find
:type filename: str
"""
if not filename:
return filename
if filename.lower().startswith(
"http://") or filename.lower().startswith("https://"):
parsed_url = parse.urlparse(filename)
downloader = ExceptionalDownloader(self.get_http_client())
self.log.info("Downloading %s", filename)
tmp_f_name, headers = downloader.get(filename)
cd_header = headers.get('Content-Disposition', '')
dest = cd_header.split(
'filename='
)[-1] if cd_header and 'filename=' in cd_header else ''
if dest.startswith('"') and dest.endswith('"') or dest.startswith(
"'") and dest.endswith("'"):
dest = dest[1:-1]
elif not dest:
dest = os.path.basename(parsed_url.path)
fname, ext = os.path.splitext(dest) if dest else (
parsed_url.hostname.replace(".", "_"), '.file')
dest = self.create_artifact(fname, ext)
self.log.debug("Moving %s to %s", tmp_f_name, dest)
shutil.move(tmp_f_name, dest)
return dest
else:
filename = os.path.expanduser(
filename
) # expanding of '~' is required for check of existence
# check filename 'as is' and all combinations of file_search_path/filename
for dirname in [""] + self.file_search_paths:
location = os.path.join(dirname, filename)
if os.path.exists(location):
if dirname:
self.log.warning(
"Guessed location from search paths for %s: %s",
filename, location)
return get_full_path(location)
self.log.warning("Could not find location at path: %s", filename)
return filename
def find_file(self, filename):
"""
Try to find file or dir in search_path if it was specified. Helps finding files
in non-CLI environments or relative to config path
Return path is full and mustn't treat with abspath/etc.
:param filename: file basename to find
:type filename: str
"""
if not filename:
return filename
if filename.lower().startswith("http://") or filename.lower().startswith("https://"):
parsed_url = parse.urlparse(filename)
downloader = ExceptionalDownloader(self.get_http_client())
self.log.info("Downloading %s", filename)
tmp_f_name, headers = downloader.get(filename)
cd_header = headers.get('Content-Disposition', '')
dest = cd_header.split('filename=')[-1] if cd_header and 'filename=' in cd_header else ''
if dest.startswith('"') and dest.endswith('"') or dest.startswith("'") and dest.endswith("'"):
dest = dest[1:-1]
elif not dest:
dest = os.path.basename(parsed_url.path)
fname, ext = os.path.splitext(dest) if dest else (parsed_url.hostname.replace(".", "_"), '.file')
dest = self.create_artifact(fname, ext)
self.log.debug("Moving %s to %s", tmp_f_name, dest)
shutil.move(tmp_f_name, dest)
return dest
else:
filename = os.path.expanduser(filename) # expanding of '~' is required for check of existence
# check filename 'as is' and all combinations of file_search_path/filename
for dirname in [""] + self.file_search_paths:
location = os.path.join(dirname, filename)
if os.path.exists(location):
if dirname:
self.log.warning("Guessed location from search paths for %s: %s", filename, location)
return get_full_path(location)
self.log.warning("Could not find location at path: %s", filename)
return filename