def _run_virtualenv(self, generator):
generator.output_path = self.test_folder
save_files(self.test_folder, generator.content)
# Generate the list of commands to execute
shell_commands = [
self.commands.dump_env.format(filename=self.env_before),
self.commands.find_program.format(program="conan",
variable="__conan_pre_path__"),
self.commands.find_program.format(program=self.app,
variable="__exec_pre_path__"),
self.commands.activate,
self.commands.dump_env.format(filename=self.env_activated),
self.commands.find_program.format(program="conan",
variable="__conan_env_path__"),
self.commands.find_program.format(program=self.app,
variable="__exec_env_path__"),
self.commands.deactivate,
self.commands.dump_env.format(filename=self.env_after),
self.commands.find_program.format(program="conan",
variable="__conan_post_path__"),
self.commands.find_program.format(program=self.app,
variable="__exec_post_path__"),
"",
]
# Execute
with environment_append({"PATH": [
self.ori_path,
]}):
shell = subprocess.Popen(self.commands.shell,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.test_folder)
(stdout, stderr) = shell.communicate(
to_file_bytes("\n".join(shell_commands)))
stdout, stderr = decode_text(stdout), decode_text(stderr)
# Consistency checks
self.assertFalse(
stderr, "Running shell resulted in error, output:\n%s" % stdout)
env_before = _load_env_file(
os.path.join(self.test_folder, self.env_before))
env_after = _load_env_file(
os.path.join(self.test_folder, self.env_after))
there_was_ps1 = os.getenv("PS1")
# FIXME: Not the best behavior
# The deactivate sets PS1 always, but sometimes it didn't exist previously
if platform.system() == "Darwin" or not there_was_ps1:
env_after.pop(six.u("PS1"),
None) # TODO: FIXME: Needed for the test to pass
env_after.pop("PS1",
None) # TODO: FIXME: Needed for the test to pass
self.assertDictEqual(env_before,
env_after) # Environment restored correctly
return stdout, _load_env_file(
os.path.join(self.test_folder, self.env_activated))
def _run_virtualenv(self, generator):
with environment_append({"PATH": [
self.ori_path,
]}):
# FIXME: I need this context because restore values for the 'deactivate' script are
# generated at the 'generator.content' and not when the 'activate' is called.
save_files(self.test_folder, generator.content)
# Generate the list of commands to execute
shell_commands = [
self.commands.dump_env.format(filename=self.env_before),
self.commands.find_program.format(
program="conan", variable="__conan_pre_path__"),
self.commands.find_program.format(
program=self.app, variable="__exec_pre_path__"),
self.commands.activate,
self.commands.dump_env.format(filename=self.env_activated),
self.commands.find_program.format(
program="conan", variable="__conan_env_path__"),
self.commands.find_program.format(
program=self.app, variable="__exec_env_path__"),
self.commands.deactivate,
self.commands.dump_env.format(filename=self.env_after),
self.commands.find_program.format(
program="conan", variable="__conan_post_path__"),
self.commands.find_program.format(
program=self.app, variable="__exec_post_path__"),
"",
]
# Execute
shell = subprocess.Popen(self.commands.shell,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.test_folder)
(stdout, stderr) = shell.communicate(
to_file_bytes("\n".join(shell_commands)))
stdout, stderr = decode_text(stdout), decode_text(stderr)
# Consistency checks
self.assertFalse(
stderr, "Running shell resulted in error, output:\n%s" % stdout)
env_before = _load_env_file(
os.path.join(self.test_folder, self.env_before))
env_after = _load_env_file(
os.path.join(self.test_folder, self.env_after))
if platform.system() == "Darwin":
env_after.pop(six.u("PS1"),
None) # TODO: FIXME: Needed for the test to pass
env_after.pop("PS1",
None) # TODO: FIXME: Needed for the test to pass
self.assertDictEqual(env_before,
env_after) # Environment restored correctly
return stdout, _load_env_file(
os.path.join(self.test_folder, self.env_activated))
def vcvars_dict(settings, arch=None, compiler_version=None, force=False, filter_known_paths=False):
cmd = vcvars_command(settings, arch=arch,
compiler_version=compiler_version, force=force) + " && echo __BEGINS__ && set"
ret = decode_text(subprocess.check_output(cmd, shell=True))
new_env = {}
start_reached = False
for line in ret.splitlines():
if not start_reached:
if "__BEGINS__" in line:
start_reached = True
continue
name_var, value = line.split("=", 1)
new_env[name_var] = value
if filter_known_paths:
def relevant_path(path):
path = path.replace("\\", "/").lower()
keywords = "msbuild", "visual", "microsoft", "/msvc/", "/vc/", "system32", "windows"
return any(word in path for word in keywords)
path = new_env.get("PATH", "").split(";")
path = [entry for entry in path if relevant_path(entry)]
new_env["PATH"] = ";".join(path)
return new_env
def _get_json(self, url, data=None):
t1 = time.time()
headers = self.custom_headers
if data: # POST request
headers.update({'Content-type': 'application/json',
'Accept': 'text/plain',
'Accept': 'application/json'})
response = self.requester.post(url, auth=self.auth, headers=headers,
verify=self.verify_ssl,
stream=True,
data=json.dumps(data))
else:
response = self.requester.get(url, auth=self.auth, headers=headers,
verify=self.verify_ssl,
stream=True)
duration = time.time() - t1
method = "POST" if data else "GET"
log_client_rest_api_call(url, method, duration, headers)
if response.status_code != 200: # Error message is text
response.charset = "utf-8" # To be able to access ret.text (ret.content are bytes)
raise get_exception_from_error(response.status_code)(response.text)
result = json.loads(decode_text(response.content))
if not isinstance(result, dict):
raise ConanException("Unexpected server response %s" % result)
return result
def get_recipe_manifest(self, ref):
# If revision not specified, check latest
if not ref.revision:
ref = self.get_latest_recipe_revision(ref)
url = self.router.recipe_manifest(ref)
content = self._get_remote_file_contents(url)
return FileTreeManifest.loads(decode_text(content))
def run(command: List[str], working_dir: Optional[types.PATH] = None) -> str:
log.debug(f"Run command: {' '.join(command)}")
out, _ = subprocess.Popen(command, stdout=subprocess.PIPE,
cwd=working_dir).communicate()
out = decode_text(out)
log.debug(out)
return out
def excluded_files(self):
ret = []
try:
file_paths = [
os.path.normpath(
os.path.join(os.path.relpath(folder, self.folder),
el)).replace("\\", "/")
for folder, dirpaths, fs in walk(self.folder)
for el in fs + dirpaths
]
if file_paths:
p = subprocess.Popen(['git', 'check-ignore', '--stdin'],
stdout=PIPE,
stdin=PIPE,
stderr=STDOUT,
cwd=self.folder)
paths = to_file_bytes("\n".join(file_paths))
grep_stdout = decode_text(p.communicate(input=paths)[0])
ret = grep_stdout.splitlines()
except (CalledProcessError, FileNotFoundError) as e:
if self._output:
self._output.warn("Error checking excluded git files: %s. "
"Ignoring excluded files" % e)
ret = []
return ret
def _get_json(self, url, data=None):
t1 = time.time()
headers = self.custom_headers
if data: # POST request
headers.update({
'Content-type': 'application/json',
'Accept': 'text/plain',
'Accept': 'application/json'
})
response = self.requester.post(url,
auth=self.auth,
headers=headers,
verify=self.verify_ssl,
stream=True,
data=json.dumps(data))
else:
response = self.requester.get(url,
auth=self.auth,
headers=headers,
verify=self.verify_ssl,
stream=True)
duration = time.time() - t1
method = "POST" if data else "GET"
log_client_rest_api_call(url, method, duration, headers)
if response.status_code != 200: # Error message is text
response.charset = "utf-8" # To be able to access ret.text (ret.content are bytes)
raise get_exception_from_error(response.status_code)(response.text)
result = json.loads(decode_text(response.content))
if not isinstance(result, dict):
raise ConanException("Unexpected server response %s" % result)
return result
def inner(*argc, **argv):
ret = method(*argc, **argv)
if ret.status_code != 200:
ret.charset = "utf-8" # To be able to access ret.text (ret.content are bytes)
text = ret.text if ret.status_code != 404 else "404 Not found"
raise get_exception_from_error(ret.status_code)(text)
return deserializer(ret.content) if deserializer else decode_text(ret.content)
def inner(*argc, **argv):
ret = method(*argc, **argv)
if ret.status_code != 200:
ret.charset = "utf-8" # To be able to access ret.text (ret.content are bytes)
text = ret.text if ret.status_code != 404 else "404 Not found"
raise get_exception_from_error(ret.status_code)(text)
return deserializer(ret.content) if deserializer else decode_text(ret.content)
def get_json(self, url, data=None):
headers = self.custom_headers
if data: # POST request
headers.update({'Content-type': 'application/json',
'Accept': 'text/plain',
'Accept': 'application/json'})
logger.debug("REST: post: %s" % url)
response = self.requester.post(url, auth=self.auth, headers=headers,
verify=self.verify_ssl,
stream=True,
data=json.dumps(data))
else:
logger.debug("REST: get: %s" % url)
response = self.requester.get(url, auth=self.auth, headers=headers,
verify=self.verify_ssl,
stream=True)
if response.status_code != 200: # Error message is text
response.charset = "utf-8" # To be able to access ret.text (ret.content are bytes)
raise get_exception_from_error(response.status_code)(response.text)
result = json.loads(decode_text(response.content))
if not isinstance(result, dict):
raise ConanException("Unexpected server response %s" % result)
return result
def write(self, data, front=None, back=None, newline=False, error=False):
if six.PY2:
if isinstance(data, str):
data = decode_text(data) # Keep python 2 compatibility
if self._color and (front or back):
data = "%s%s%s%s" % (front or '', back
or '', data, Style.RESET_ALL)
# https://github.com/conan-io/conan/issues/4277
# Windows output locks produce IOErrors
for _ in range(3):
try:
if error:
self._write_err(data, newline)
else:
self._write(data, newline)
break
except IOError:
import time
time.sleep(0.02)
except UnicodeError:
data = data.encode("utf8").decode("ascii", "ignore")
self._stream.flush()
def get_package_info(self, pref, headers):
url = self.router.package_info(pref)
cache = (pref.revision != DEFAULT_REVISION_V1)
content = self._get_remote_file_contents(url,
use_cache=cache,
headers=headers)
return ConanInfo.loads(decode_text(content))
def get_json(self, url, data=None):
headers = self.custom_headers
if data: # POST request
headers.update({'Content-type': 'application/json',
'Accept': 'text/plain',
'Accept': 'application/json'})
logger.debug("REST: post: %s" % url)
response = self.requester.post(url, auth=self.auth, headers=headers,
verify=self.verify_ssl,
stream=True,
data=json.dumps(data))
else:
logger.debug("REST: get: %s" % url)
response = self.requester.get(url, auth=self.auth, headers=headers,
verify=self.verify_ssl,
stream=True)
if response.status_code != 200: # Error message is text
response.charset = "utf-8" # To be able to access ret.text (ret.content are bytes)
raise get_exception_from_error(response.status_code)(response_to_str(response))
content = decode_text(response.content)
content_type = response.headers.get("Content-Type")
if content_type != 'application/json':
raise ConanException("%s\n\nResponse from remote is not json, but '%s'"
% (content, content_type))
try: # This can fail, if some proxy returns 200 and an html message
result = json.loads(content)
except Exception:
raise ConanException("Remote responded with broken json: %s" % content)
if not isinstance(result, dict):
raise ConanException("Unexpected server response %s" % result)
return result
def _get_path(self, npm_package_files, path):
files = npm_package_files.keys()
def is_dir(the_path):
if the_path == ".":
return True
for _the_file in files:
if the_path == _the_file:
return False
elif _the_file.startswith(the_path):
return True
raise NotFoundException("The specified path doesn't exist")
if is_dir(path):
ret = []
for the_file in files:
if path == "." or the_file.startswith(path):
tmp = the_file[len(path) - 1:].split("/", 1)[0]
if tmp not in ret:
ret.append(tmp)
return sorted(ret)
else:
contents = self._get_file_contents(npm_package_files, [path])
content = contents[path]
return decode_text(content)
def _get_path(self, url, path):
urls = self._get_file_to_url_dict(url)
def is_dir(the_path):
if the_path == ".":
return True
for _the_file in urls:
if the_path == _the_file:
return False
elif _the_file.startswith(the_path):
return True
raise NotFoundException("The specified path doesn't exist")
if is_dir(path):
ret = []
for the_file in urls:
if path == "." or the_file.startswith(path):
tmp = the_file[len(path) - 1:].split("/", 1)[0]
if tmp not in ret:
ret.append(tmp)
return sorted(ret)
else:
downloader = FileDownloader(self.requester, None, self.verify_ssl,
self._config)
auth, _ = self._file_server_capabilities(urls[path])
content = downloader.download(urls[path], auth=auth)
return decode_text(content)
def vcvars_dict(settings, arch=None, compiler_version=None, force=False, filter_known_paths=False):
cmd = vcvars_command(settings, arch=arch,
compiler_version=compiler_version, force=force) + " && echo __BEGINS__ && set"
ret = decode_text(subprocess.check_output(cmd, shell=True))
new_env = {}
start_reached = False
for line in ret.splitlines():
if not start_reached:
if "__BEGINS__" in line:
start_reached = True
continue
name_var, value = line.split("=", 1)
new_env[name_var] = value
if filter_known_paths:
def relevant_path(path):
path = path.replace("\\", "/").lower()
keywords = "msbuild", "visual", "microsoft", "/msvc/", "/vc/", "system32", "windows"
return any(word in path for word in keywords)
path = new_env.get("PATH", "").split(";")
path = [entry for entry in path if relevant_path(entry)]
new_env["PATH"] = ";".join(path)
return new_env
def parse_authorization_value(self, header_value):
"""Parse header_value and return kwargs to apply bottle
method parameters"""
if header_value is None:
return None
username, password = decode_text(base64.b64decode(header_value)).split(":", 1)
ret = UserPasswordPair(username, password)
return {self.keyword: ret}
def get_meson_version():
try:
out = version_runner(["meson", "--version"])
version_line = decode_text(out).split('\n', 1)[0]
version_str = version_line.rsplit(' ', 1)[-1]
return Version(version_str)
except Exception:
return Version("0.0.0")
def get_version():
try:
out, err = subprocess.Popen(["svn", "--version"], stdout=subprocess.PIPE).communicate()
version_line = decode_text(out).split('\n', 1)[0]
version_str = version_line.split(' ', 3)[2]
return Version(version_str)
except Exception as e:
raise ConanException("Error retrieving SVN version: '{}'".format(e))
def get_version(cls):
try:
out = version_runner([cls.cmd_command, "--version"])
version_line = decode_text(out).split('\n', 1)[0]
version_str = version_line.split(' ', 3)[2]
return Version(version_str)
except Exception as e:
raise ConanException("Error retrieving {} version: '{}'".format(cls.cmd_command, e))
def get_version(cls):
try:
out, _ = subprocess.Popen([cls.cmd_command, "--version"], stdout=subprocess.PIPE).communicate()
version_line = decode_text(out).split('\n', 1)[0]
version_str = version_line.split(' ', 3)[2]
return Version(version_str)
except Exception as e:
raise ConanException("Error retrieving {} version: '{}'".format(cls.cmd_command, e))
def get_version():
try:
out = version_runner(["cmake", "--version"])
version_line = decode_text(out).split('\n', 1)[0]
version_str = version_line.rsplit(' ', 1)[-1]
return Version(version_str)
except Exception as e:
raise ConanException("Error retrieving CMake version: '{}'".format(e))
def run(self, command):
command = "%s %s" % (self.cmd_command, command)
with chdir(self.folder) if self.folder else no_op():
with environment_append({"LC_ALL": "en_US.UTF-8"}) if self._force_eng else no_op():
if not self._runner:
return decode_text(subprocess.check_output(command, shell=True).strip())
else:
return self._runner(command)
def get_recipe_path(self, ref, path):
url = self.router.recipe_snapshot(ref)
files = self._get_file_list_json(url)
if self._is_dir(path, files):
return self._list_dir_contents(path, files)
else:
url = self.router.recipe_file(ref, path)
content = self._get_remote_file_contents(url)
return decode_text(content)
def get_recipe_manifest(self, ref):
"""Gets a FileTreeManifest from conans"""
# Obtain the files from npm package
npm_package_files = self._download_recipe_npm(ref)
contents = self._get_file_contents(npm_package_files, [CONAN_MANIFEST])
# Unroll generator and decode (plain text)
contents = {key: decode_text(value) for key, value in contents.items()}
return FileTreeManifest.loads(contents[CONAN_MANIFEST])
def get_package_manifest(self, pref):
"""Gets a FileTreeManifest from a package"""
pref = pref.copy_with_revs(None, None)
# Obtain the files from npm package
npm_package_files = self._download_package_npm(pref)
contents = self._get_file_contents(npm_package_files, [CONAN_MANIFEST])
# Unroll generator and decode (plain text)
contents = {key: decode_text(value) for key, value in contents.items()}
return FileTreeManifest.loads(contents[CONAN_MANIFEST])
def get_package_path(self, pref, path):
"""Gets a file content or a directory list"""
url = self.router.package_snapshot(pref)
files = self._get_file_list_json(url)
if self._is_dir(path, files):
return self._list_dir_contents(path, files)
else:
url = self.router.package_file(pref, path)
content = self._get_remote_file_contents(url)
return decode_text(content)
def __call__(self, command, output, cwd=None):
""" There are two options, with or without you (sorry, U2 pun :)
With or without output. Probably the Popen approach would be fine for both cases
but I found it more error prone, slower, problems with very large outputs (typical
when building C/C++ projects...) so I prefer to keep the os.system one for
most cases, in which the user does not want to capture the output, and the Popen
for cases they want
"""
if output is True:
if not cwd:
return os.system(command)
else:
try:
old_dir = os.getcwd()
os.chdir(cwd)
result = os.system(command)
except Exception as e:
raise ConanException("Error while executing '%s'\n\t%s" %
(command, str(e)))
finally:
os.chdir(old_dir)
return result
else:
proc = Popen(command,
shell=True,
stdout=PIPE,
stderr=STDOUT,
cwd=cwd)
if hasattr(output, "write"):
while True:
line = proc.stdout.readline()
if not line:
break
output.write(decode_text(line))
out, err = proc.communicate()
if hasattr(output, "write"):
if out:
output.write(decode_text(out))
if err:
output.write(decode_text(err))
return proc.returncode
def deserialize(data):
tmp = json.loads(decode_text(data))
ret = SearchInfo()
for conan_ref, packages in tmp.items():
conan_ref = ConanFileReference.loads(conan_ref)
ret[conan_ref] = {}
for package_id, info in packages.items():
ret[conan_ref][package_id] = ConanInfo.deserialize(info)
return ret
def deserialize(data):
tmp = json.loads(decode_text(data))
ret = SearchInfo()
for conan_ref, packages in tmp.items():
conan_ref = ConanFileReference.loads(conan_ref)
ret[conan_ref] = {}
for package_id, info in packages.items():
ret[conan_ref][package_id] = ConanInfo.deserialize(info)
return ret
def get_recipe_path(self, ref, path):
url = self.router.recipe_snapshot(ref)
files = self._get_file_list_json(url)
if self._is_dir(path, files):
return self._list_dir_contents(path, files)
else:
url = self.router.recipe_file(ref, path)
cache = (ref.revision != DEFAULT_REVISION_V1)
content = self._get_remote_file_contents(url, use_cache=cache)
return decode_text(content)
def vcvars_dict(settings, arch=None, compiler_version=None, force=False, filter_known_paths=False,
vcvars_ver=None, winsdk_version=None, only_diff=True):
known_path_lists = ("include", "lib", "libpath", "path")
cmd = vcvars_command(settings, arch=arch,
compiler_version=compiler_version, force=force,
vcvars_ver=vcvars_ver, winsdk_version=winsdk_version)
cmd += " && echo __BEGINS__ && set"
ret = decode_text(subprocess.check_output(cmd, shell=True))
new_env = {}
start_reached = False
for line in ret.splitlines():
line = line.strip()
if not start_reached:
if "__BEGINS__" in line:
start_reached = True
continue
if line == "\n" or not line:
continue
try:
name_var, value = line.split("=", 1)
new_value = value.split(os.pathsep) if name_var.lower() in known_path_lists else value
# Return only new vars & changed ones, but only with the changed elements if the var is
# a list
if only_diff:
old_value = os.environ.get(name_var)
if name_var.lower() == "path":
old_values_lower = [v.lower() for v in old_value.split(os.pathsep)]
# Clean all repeated entries, not append if the element was already there
new_env[name_var] = [v for v in new_value if v.lower() not in old_values_lower]
elif old_value and value.endswith(os.pathsep + old_value):
# The new value ends with separator and the old value, is a list,
# get only the new elements
new_env[name_var] = value[:-(len(old_value) + 1)].split(os.pathsep)
elif value != old_value:
# Only if the vcvars changed something, we return the variable,
# otherwise is not vcvars related
new_env[name_var] = new_value
else:
new_env[name_var] = new_value
except ValueError:
pass
if filter_known_paths:
def relevant_path(path):
path = path.replace("\\", "/").lower()
keywords = "msbuild", "visual", "microsoft", "/msvc/", "/vc/", "system32", "windows"
return any(word in path for word in keywords)
path = new_env.get("PATH", "").split(";")
path = [entry for entry in path if relevant_path(entry)]
new_env["PATH"] = ";".join(path)
return new_env
def get_conan_digest(self, conan_reference):
"""Gets a FileTreeManifest from conans"""
# Obtain the URLs
url = "%s/conans/%s/digest" % (self._remote_api_url, "/".join(conan_reference))
urls = self._get_json(url)
# Get the digest
contents = self.download_files(urls)
contents = {key: decode_text(value) for key, value in dict(contents).items()} # Unroll generator and decode shas (plain text)
return FileTreeManifest.loads(contents[CONAN_MANIFEST])
def __call__(self, command, output, cwd=None):
if output is True:
return os.system(command)
else:
proc = Popen(command, shell=True, stdout=PIPE, stderr=STDOUT, cwd=cwd)
if hasattr(output, "write"):
while True:
line = proc.stdout.readline()
if not line:
break
output.write(decode_text(line))
out, err = proc.communicate()
if hasattr(output, "write"):
if out:
output.write(decode_text(out))
if err:
output.write(decode_text(err))
return proc.returncode
def __call__(self, command, output, cwd=None):
""" There are two options, with or without you (sorry, U2 pun :)
With or without output. Probably the Popen approach would be fine for both cases
but I found it more error prone, slower, problems with very large outputs (typical
when building C/C++ projects...) so I prefer to keep the os.system one for
most cases, in which the user does not want to capture the output, and the Popen
for cases they want
"""
if output is True:
if not cwd:
return os.system(command)
else:
try:
old_dir = os.getcwd()
os.chdir(cwd)
result = os.system(command)
except Exception as e:
raise ConanException("Error while executing '%s'\n\t%s" % (command, str(e)))
finally:
os.chdir(old_dir)
return result
else:
proc = Popen(command, shell=True, stdout=PIPE, stderr=STDOUT, cwd=cwd)
if hasattr(output, "write"):
while True:
line = proc.stdout.readline()
if not line:
break
output.write(decode_text(line))
out, err = proc.communicate()
if hasattr(output, "write"):
if out:
output.write(decode_text(out))
if err:
output.write(decode_text(err))
return proc.returncode
def get_package_manifest(self, package_reference):
"""Gets a FileTreeManifest from a package"""
# Obtain the URLs
url = "%s/conans/%s/packages/%s/digest" % (self._remote_api_url,
"/".join(package_reference.conan),
package_reference.package_id)
urls = self._get_file_to_url_dict(url)
# Get the digest
contents = self.download_files(urls)
# Unroll generator and decode shas (plain text)
contents = {key: decode_text(value) for key, value in dict(contents).items()}
return FileTreeManifest.loads(contents[CONAN_MANIFEST])
def get_upload_urls(self, paths_sizes, user=None):
'''Get the urls for upload the specified files using s3 signed request.
returns a dict with this structure: {"filepath": "http://..."}
paths_sizes is a dict of {path: size_in_bytes} '''
assert isinstance(paths_sizes, dict)
ret = {}
for filepath, filesize in paths_sizes.items():
url_path = os.path.relpath(filepath, self._store_folder)
url_path = url_path.replace("\\", "/")
# FALTA SIZE DEL FICHERO PARA EL UPLOAD URL!
signature = self.updown_auth_manager.get_token_for(url_path, user, filesize)
url = "%s/%s?signature=%s" % (self.base_url, url_path, decode_text(signature))
ret[filepath] = url
return ret
def _get_json(self, url, data=None):
if data: # POST request
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
headers.update(self.custom_headers)
response = self.requester.post(url, auth=self.auth, headers=headers,
verify=self.VERIFY_SSL,
stream=True,
data=json.dumps(data))
else:
response = self.requester.get(url, auth=self.auth, headers=self.custom_headers,
verify=self.VERIFY_SSL,
stream=True)
if response.status_code != 200: # Error message is text
response.charset = "utf-8" # To be able to access ret.text (ret.content are bytes)
raise get_exception_from_error(response.status_code)(response.text)
return json.loads(decode_text(response.content))
def get_download_urls(self, paths, user=None):
'''Get the urls for download the specified files using s3 signed request.
returns a dict with this structure: {"filepath": "http://..."}
paths is a list of path files '''
assert isinstance(paths, list)
ret = {}
for filepath in paths:
url_path = os.path.relpath(filepath, self.base_storage_path)
url_path = url_path.replace("\\", "/")
# FALTA SIZE DEL FICHERO PARA EL UPLOAD URL!
signature = self.updown_auth_manager.get_token_for(url_path, user)
url = "%s/%s?signature=%s" % (self.base_url, url_path, decode_text(signature))
ret[filepath] = url
return ret
def get_package_info(self, package_reference):
"""Gets a ConanInfo file from a package"""
url = "%s/conans/%s/packages/%s/download_urls" % (self._remote_api_url,
"/".join(package_reference.conan),
package_reference.package_id)
urls = self._get_json(url)
if not urls:
raise NotFoundException("Package not found!")
if CONANINFO not in urls:
raise NotFoundException("Package %s doesn't have the %s file!" % (package_reference,
CONANINFO))
# Get the info (in memory)
contents = self.download_files({CONANINFO: urls[CONANINFO]})
# Unroll generator and decode shas (plain text)
contents = {key: decode_text(value) for key, value in dict(contents).items()}
return ConanInfo.loads(contents[CONANINFO])
def get_stream_lines(the_stream):
while True:
line = the_stream.readline()
if not line:
break
decoded_line = decode_text(line)
if stream_output and self._log_run_to_output:
try:
stream_output.write(decoded_line)
except UnicodeEncodeError: # be agressive on text encoding
decoded_line = decoded_line.encode("latin-1", "ignore").decode("latin-1",
"ignore")
stream_output.write(decoded_line)
if log_handler:
# Write decoded in PY2 causes some ASCII encoding problems
# tried to open the log_handler binary but same result.
log_handler.write(line if six.PY2 else decoded_line)
def write(self, data, front=None, back=None, newline=False):
if six.PY2:
if isinstance(data, str):
data = decode_text(data) # Keep python 2 compatibility
if self._color and (front or back):
color = "%s%s" % (front or '', back or '')
end = (Style.RESET_ALL + "\n") if newline else Style.RESET_ALL # @UndefinedVariable
data = "%s%s%s" % (color, data, end)
else:
if newline:
data = "%s\n" % data
try:
self._stream.write(data)
except UnicodeError:
data = data.encode("utf8").decode("ascii", "ignore")
self._stream.write(data)
self._stream.flush()
def get_path(self, conan_reference, package_id, path):
"""Gets a file content or a directory list"""
if not package_id:
url = "%s/conans/%s/download_urls" % (self._remote_api_url, "/".join(conan_reference))
else:
url = "%s/conans/%s/packages/%s/download_urls" % (self._remote_api_url,
"/".join(conan_reference),
package_id)
try:
urls = self._get_file_to_url_dict(url)
except NotFoundException:
if package_id:
raise NotFoundException("Package %s:%s not found" % (conan_reference, package_id))
else:
raise NotFoundException("Recipe %s not found" % str(conan_reference))
def is_dir(the_path):
if the_path == ".":
return True
for the_file in urls.keys():
if the_path == the_file:
return False
elif the_file.startswith(the_path):
return True
raise NotFoundException("The specified path doesn't exist")
if is_dir(path):
ret = []
for the_file in urls.keys():
if path == "." or the_file.startswith(path):
tmp = the_file[len(path)-1:].split("/", 1)[0]
if tmp not in ret:
ret.append(tmp)
return sorted(ret)
else:
downloader = Downloader(self.requester, None, self.verify_ssl)
auth, _ = self._file_server_capabilities(urls[path])
content = downloader.download(urls[path], auth=auth)
return decode_text(content)
def vswhere(all_=False, prerelease=False, products=None, requires=None, version="", latest=False,
legacy=False, property_="", nologo=True):
# 'version' option only works if Visual Studio 2017 is installed:
# https://github.com/Microsoft/vswhere/issues/91
products = list() if products is None else products
requires = list() if requires is None else requires
if legacy and (products or requires):
raise ConanException("The 'legacy' parameter cannot be specified with either the "
"'products' or 'requires' parameter")
program_files = os.environ.get("ProgramFiles(x86)", os.environ.get("ProgramFiles"))
vswhere_path = ""
if program_files:
vswhere_path = os.path.join(program_files, "Microsoft Visual Studio", "Installer",
"vswhere.exe")
if not os.path.isfile(vswhere_path):
raise ConanException("Cannot locate 'vswhere'")
else:
raise ConanException("Cannot locate 'Program Files'/'Program Files (x86)' directory")
arguments = list()
arguments.append(vswhere_path)
# Output json format
arguments.append("-format")
arguments.append("json")
if all_:
arguments.append("-all")
if prerelease:
arguments.append("-prerelease")
if products:
arguments.append("-products")
arguments.extend(products)
if requires:
arguments.append("-requires")
arguments.extend(requires)
if len(version) is not 0:
arguments.append("-version")
arguments.append(version)
if latest:
arguments.append("-latest")
if legacy:
arguments.append("-legacy")
if len(property_) is not 0:
arguments.append("-property")
arguments.append(property_)
if nologo:
arguments.append("-nologo")
try:
output = subprocess.check_output(arguments)
vswhere_out = decode_text(output).strip()
except (ValueError, subprocess.CalledProcessError, UnicodeDecodeError) as e:
raise ConanException("vswhere error: %s" % str(e))
return json.loads(vswhere_out)
