def run(self, command_line, user_io=None, ignore_error=False):
""" run a single command as in the command line.
If user or password is filled, user_io will be mocked to return this
tuple if required
"""
self.init_dynamic_vars(user_io)
command = Command(self.client_cache, self.user_io, self.runner, self.remote_manager, self.search_manager)
args = shlex.split(command_line)
current_dir = os.getcwd()
os.chdir(self.current_folder)
old_modules = list(sys.modules.keys())
try:
error = command.run(args)
finally:
os.chdir(current_dir)
# Reset sys.modules to its prev state. A .copy() DOES NOT WORK
added_modules = set(sys.modules).difference(old_modules)
for added in added_modules:
sys.modules.pop(added, None)
if not ignore_error and error:
logger.error(self.user_io.out)
raise Exception("Command failed:\n%s" % command_line)
self.all_output += str(self.user_io.out)
return error
def run(self, command_line, user_io=None, ignore_error=False):
""" run a single command as in the command line.
If user or password is filled, user_io will be mocked to return this
tuple if required
"""
self.init_dynamic_vars(user_io)
command = Command(self.client_cache, self.user_io, self.runner, self.remote_manager, self.search_manager)
args = shlex.split(command_line)
current_dir = os.getcwd()
os.chdir(self.current_folder)
old_modules = list(sys.modules.keys())
try:
error = command.run(args)
finally:
os.chdir(current_dir)
# Reset sys.modules to its prev state. A .copy() DOES NOT WORK
added_modules = set(sys.modules).difference(old_modules)
for added in added_modules:
sys.modules.pop(added, None)
if not ignore_error and error:
logger.error(self.user_io.out)
raise Exception("Command failed:\n%s" % command_line)
return error
def run(self, command_line, user_io=None, assert_error=False):
""" run a single command as in the command line.
If user or password is filled, user_io will be mocked to return this
tuple if required
"""
conan = self.get_conan_api(user_io)
self.api = conan
command = Command(conan)
args = shlex.split(command_line)
current_dir = os.getcwd()
os.chdir(self.current_folder)
old_path = sys.path[:]
old_modules = list(sys.modules.keys())
try:
error = command.run(args)
finally:
sys.path = old_path
os.chdir(current_dir)
# Reset sys.modules to its prev state. A .copy() DOES NOT WORK
added_modules = set(sys.modules).difference(old_modules)
for added in added_modules:
sys.modules.pop(added, None)
self._handle_cli_result(command_line, assert_error=assert_error, error=error)
return error
def run(self,
command_line,
user_io=None,
ignore_error=False,
should_error=False):
""" run a single command as in the command line.
If user or password is filled, user_io will be mocked to return this
tuple if required
"""
output, requester = self.init_dynamic_vars(user_io)
with tools.environment_append(self.client_cache.conan_config.env_vars):
# Settings preprocessor
interactive = not get_env("CONAN_NON_INTERACTIVE", False)
conan = Conan(self.client_cache,
self.user_io,
self.runner,
self.remote_manager,
self.hook_manager,
interactive=interactive)
outputer = CommandOutputer(self.user_io, self.client_cache)
command = Command(conan, self.client_cache, self.user_io, outputer)
args = shlex.split(command_line)
current_dir = os.getcwd()
os.chdir(self.current_folder)
old_path = sys.path[:]
sys.path.append(os.path.join(self.client_cache.conan_folder, "python"))
old_modules = list(sys.modules.keys())
old_output, old_requester = set_global_instances(output, requester)
try:
error = command.run(args)
finally:
set_global_instances(old_output, old_requester)
sys.path = old_path
os.chdir(current_dir)
# Reset sys.modules to its prev state. A .copy() DOES NOT WORK
added_modules = set(sys.modules).difference(old_modules)
for added in added_modules:
sys.modules.pop(added, None)
if not ignore_error and error and not should_error:
exc_message = "\n{command_header}\n{command}\n{output_header}\n{output}\n{output_footer}\n".format(
command_header='{:-^80}'.format(" Command failed: "),
output_header='{:-^80}'.format(" Output: "),
output_footer='-' * 80,
command=command_line,
output=self.user_io.out)
raise Exception(exc_message)
if should_error and not error:
raise Exception("This command should have failed: %s" %
command_line)
self.all_output += str(self.user_io.out)
return error
def run_cli(self, command_line, assert_error=False):
conan = self.get_conan_api()
self.api = conan
if os.getenv("CONAN_V2_CLI"):
command = Cli(conan)
else:
command = Command(conan)
args = shlex.split(command_line)
current_dir = os.getcwd()
os.chdir(self.current_folder)
old_path = sys.path[:]
old_modules = list(sys.modules.keys())
try:
error = command.run(args)
finally:
sys.path = old_path
os.chdir(current_dir)
# Reset sys.modules to its prev state. A .copy() DOES NOT WORK
added_modules = set(sys.modules).difference(old_modules)
for added in added_modules:
sys.modules.pop(added, None)
self._handle_cli_result(command_line,
assert_error=assert_error,
error=error)
return error
def conan_command(output_stream):
# This snippet reproduces code from conans.client.command.main, we cannot directly
# use it because in case of error it is exiting the python interpreter :/
old_stdout, old_stderr = sys.stdout, sys.stderr
sys.stdout, sys.stderr = output_stream, output_stream
conan_api, cache, user_io = Conan.factory()
if Version(conan_version) >= "1.16":
cmd = Command(conan_api)
else:
user_io.out._stream = output_stream
outputer = CommandOutputer(user_io, cache)
cmd = Command(conan_api, cache, user_io, outputer)
try:
yield cmd
finally:
if Version(conan_version) < "1.13":
conan_api._remote_manager._auth_manager._localdb.connection.close(
) # Close sqlite3
sys.stdout, sys.stderr = old_stdout, old_stderr
def run(self, command_line, user_io=None, ignore_error=False):
""" run a single command as in the command line.
If user or password is filled, user_io will be mocked to return this
tuple if required
"""
self.init_dynamic_vars(user_io)
with tools.environment_append(self.client_cache.conan_config.env_vars):
# Settings preprocessor
interactive = not get_env("CONAN_NON_INTERACTIVE", False)
conan = Conan(self.client_cache,
self.user_io,
self.runner,
self.remote_manager,
self.search_manager,
settings_preprocessor,
interactive=interactive)
outputer = CommandOutputer(self.user_io, self.client_cache)
command = Command(conan, self.client_cache, self.user_io, outputer)
args = shlex.split(command_line)
current_dir = os.getcwd()
os.chdir(self.current_folder)
old_path = sys.path[:]
sys.path.append(os.path.join(self.client_cache.conan_folder, "python"))
old_modules = list(sys.modules.keys())
try:
error = command.run(args)
finally:
sys.path = old_path
os.chdir(current_dir)
# Reset sys.modules to its prev state. A .copy() DOES NOT WORK
added_modules = set(sys.modules).difference(old_modules)
for added in added_modules:
sys.modules.pop(added, None)
if not ignore_error and error:
logger.error(self.user_io.out)
print(self.user_io.out)
raise Exception("Command failed:\n%s" % command_line)
self.all_output += str(self.user_io.out)
return error
def run(self, command_line, user_io=None, ignore_error=False):
""" run a single command as in the command line.
If user or password is filled, user_io will be mocked to return this
tuple if required
"""
self.init_dynamic_vars(user_io)
command = Command(self.paths, self.user_io, self.runner, self.remote_manager, self.localdb)
args = shlex.split(command_line)
current_dir = os.getcwd()
os.chdir(self.current_folder)
try:
error = command.run(args)
finally:
os.chdir(current_dir)
if not ignore_error and error:
logger.error(self.user_io.out)
raise Exception("Command failed:\n%s" % command_line)
return error
def run(self, command_line, user_io=None, ignore_error=False):
""" run a single command as in the command line.
If user or password is filled, user_io will be mocked to return this
tuple if required
"""
self.init_dynamic_vars(user_io)
command = Command(self.paths, self.user_io, self.runner, self.remote_manager)
args = shlex.split(command_line)
current_dir = os.getcwd()
os.chdir(self.current_folder)
try:
error = command.run(args)
finally:
os.chdir(current_dir)
if not ignore_error and error:
logger.error(self.user_io.out)
raise Exception("Command failed:\n%s" % command_line)
return error
def run(self, command_line, user_io=None, assert_error=False):
""" run a single command as in the command line.
If user or password is filled, user_io will be mocked to return this
tuple if required
"""
conan = self.get_conan_api(user_io)
self.api = conan
command = Command(conan)
args = shlex.split(command_line)
current_dir = os.getcwd()
os.chdir(self.current_folder)
old_path = sys.path[:]
old_modules = list(sys.modules.keys())
try:
error = command.run(args)
finally:
sys.path = old_path
os.chdir(current_dir)
# Reset sys.modules to its prev state. A .copy() DOES NOT WORK
added_modules = set(sys.modules).difference(old_modules)
for added in added_modules:
sys.modules.pop(added, None)
if (assert_error and not error) or (not assert_error and error):
if assert_error:
msg = " Command succeeded (failure expected): "
else:
msg = " Command failed (unexpectedly): "
exc_message = "\n{header}\n{cmd}\n{output_header}\n{output}\n{output_footer}\n".format(
header='{:-^80}'.format(msg),
output_header='{:-^80}'.format(" Output: "),
output_footer='-'*80,
cmd=command_line,
output=self.out
)
raise Exception(exc_message)
return error
def run(self, command_line, user_io=None, ignore_error=False):
""" run a single command as in the command line.
If user or password is filled, user_io will be mocked to return this
tuple if required
"""
self.init_dynamic_vars(user_io)
with tools.environment_append(self.client_cache.conan_config.env_vars):
# Settings preprocessor
interactive = not get_env("CONAN_NON_INTERACTIVE", False)
conan = Conan(self.client_cache, self.user_io, self.runner, self.remote_manager,
self.search_manager, settings_preprocessor, interactive=interactive)
outputer = CommandOutputer(self.user_io, self.client_cache)
command = Command(conan, self.client_cache, self.user_io, outputer)
args = shlex.split(command_line)
current_dir = os.getcwd()
os.chdir(self.current_folder)
old_path = sys.path[:]
sys.path.append(os.path.join(self.client_cache.conan_folder, "python"))
old_modules = list(sys.modules.keys())
try:
error = command.run(args)
finally:
sys.path = old_path
os.chdir(current_dir)
# Reset sys.modules to its prev state. A .copy() DOES NOT WORK
added_modules = set(sys.modules).difference(old_modules)
for added in added_modules:
sys.modules.pop(added, None)
if not ignore_error and error:
logger.error(self.user_io.out)
print(self.user_io.out)
raise Exception("Command failed:\n%s" % command_line)
self.all_output += str(self.user_io.out)
return error
def main(pypi, to_arti):
remote = "conan-hdim" # conan-hdim: http://cytosplore.lumc.nl:8081/artifactory/api/conan/conan-local
remote_url_base = "http://cytosplore.lumc.nl:8081/artifactory/conan-local"
version = "1.0.0"
package_reference = "nptsne/" + version + "@lkeb/stable"
url_reference = "lkeb/nptsne/" + version + "/stable"
packages_queries = ["os=Windows", "os=Linux", "os=Macos"]
python_versions = ["3.6", "3.7"]
conan_api, cache, user_io = Conan.factory()
command = Command(conan_api)
if os.path.exists('./linuxwheels'):
shutil.rmtree('./linuxwheels')
os.makedirs('./linuxwheels')
if os.path.exists('./wheels'):
shutil.rmtree('./wheels')
os.makedirs('./wheels')
for vers in python_versions:
ver_dir = './{}'.format(vers)
if os.path.exists(ver_dir):
shutil.rmtree(ver_dir)
# Get the conan tgz for each package
for query in packages_queries:
print(package_reference, query, '\n')
ret = conan_api.search_packages(package_reference,
remote_name=remote,
query=query)
result_list = ret['results']
package_list = result_list[0]['items'][0]['packages']
for package in package_list:
if package['options'].get('python_version',
None) in python_versions:
pyver = package['options']['python_version']
print(query, pyver, package['id'])
reference = "{}:{}".format(package_reference, package['id'])
print(reference)
url_path = "{}/{}/0/package/{}/0/conan_package.tgz".format(
remote_url_base, url_reference, package['id'])
local_filename, headers = urllib.request.urlretrieve(
url_path, filename=pyver + "_" + package['id'] + ".tgz")
print(local_filename)
# Open the complete package tgz file.
# Using the TarFile init does not work - settings problem?
# Simply using tarfile.open seems to be robust.
conantar = tarfile.open(local_filename)
print("Get members")
mems = conantar.getmembers()
print("total contents: ", len(mems))
#Extract the wheel from the dist directory.
whls = [
mem for mem in mems if mem.name.startswith("dist")
and mem.name.endswith(".whl")
]
print("wheels: ", len(whls))
conantar.extract(whls[0], path=pyver)
# Skip linux for pypi upload
# TODO make manylinux2010 wheels
if "linux" in whls[0].name:
shutil.copy(os.path.join('./', pyver, whls[0].name),
'./linuxwheels')
else:
shutil.copy(os.path.join('./', pyver, whls[0].name),
'./wheels')
conantar.close()
if to_arti:
# Don't forget to setup the netrc with login & password for the artifactory
c = pycurl.Curl()
for wheel_name in os.listdir('./linuxwheels'):
c.setopt(
c.URL,
'http://cytosplore.lumc.nl:8081/artifactory/wheels/nptsne/{}'.
format(wheel_name))
c.setopt(c.UPLOAD, 1)
c.setopt(c.NETRC, 1)
with open('./linuxwheels/{}'.format(wheel_name), 'rb') as file:
c.setopt(c.READDATA, file)
c.perform()
print('Uploaded {}'.format(wheel_name))
c.close()
if pypi == "testpypi":
subprocess.run([
"twine", "upload", "--verbose", "-r", "testpypi",
"wheels/*{}*".format(version)
])
if pypi == "releasepypi":
subprocess.run(
["twine", "upload", "--verbose", "wheels/*{}*".format(version)])
]
if __name__ == "__main__":
target_remote = sys.argv[1]
conan_api, _, _ = Conan.factory()
# for pkg in rehost_packages:
for pkg in rehost_packages:
# Which remote should we be getting this from?
remote = None
if "bincrafters" in pkg:
remote = "bincrafters"
else:
remote = "conan-center"
install_args = ["install", "-r", remote,
"--build=missing", pkg]
print("conan {}".format(" ".join(install_args)))
cmd = Command(conan_api)
error = cmd.run(install_args)
if error != 0:
raise RuntimeError("Result is not zero, but {}".format(error))
upload_args = ["upload", "-r", target_remote,
"--all", "-c", pkg]
print("conan {}".format(" ".join(upload_args)))
cmd = Command(conan_api)
error = cmd.run(upload_args)
if error != 0:
raise RuntimeError("Result is not zero, but {}".format(error))
def __init__(self, conan_user_home: types.PATH, cwd: types.PATH):
self._cwd = cwd
self.stream = StringIO()
output = ConanOutput(self.stream, self.stream, color=False)
conan_api = ConanAPIV1(cache_folder=conan_user_home, output=output)
self.cmd = Command(conan_api)
class ConanWrapper:
_cwd: types.PATH
_conan_user_home: types.PATH
def __init__(self, conan_user_home: types.PATH, cwd: types.PATH):
self._cwd = cwd
self.stream = StringIO()
output = ConanOutput(self.stream, self.stream, color=False)
conan_api = ConanAPIV1(cache_folder=conan_user_home, output=output)
self.cmd = Command(conan_api)
def run(self, command: List[str], cwd: Optional[types.PATH] = None):
log.debug(f"Run command '{' '.join(command)}'")
self.stream.truncate(0)
try:
with chdir(cwd or self._cwd):
r = self.cmd.run(command)
except Exception as e:
# Conan execution failed
log.error(f"Command unexpectedly failed: {e}")
return -1, str(e)
else:
return r, self.stream.getvalue()
def inspect(self, recipe, attributes):
# Return options attribute and default_options as dictionary
with temp_file() as json_file:
cmd = ['inspect']
for it in attributes:
cmd.extend(['-a', it])
cmd.extend(['--json', json_file, recipe.conanfile])
self.run(cmd)
data = json.loads(tools.load(json_file))
return tuple([data[it] for it in attributes])
def export(self, recipe):
log.info(f"Export recipe {recipe.ref}")
self.run(command=["export", recipe.conanfile, f"{recipe.ref}@"])
@staticmethod
def get_reference(nodes, node_id):
node = nodes[node_id]
reference = node['ref']
ref = ConanFileReference.loads(reference)
return ref
def requirements(self, recipe, profile):
options_cmd = []
if recipe.options:
for opt in recipe.options:
options_cmd.extend(['-o', opt])
with temp_file('conan.lock') as output_file:
cmd = ['lock', 'create', '--build', '--profile', profile, '--lockfile-out', output_file] \
+ options_cmd + ["--reference", f"{recipe.ref}@"]
try:
r, out = self.run(cmd)
if 'Invalid configuration' in out:
return None, None, None
elif r != 0:
log.warning(f"Error in recipe {recipe.ref}: {out}")
return None, None, None
content = json.loads(tools.load(output_file))
nodes = content['graph_lock']['nodes']
for _, node in nodes.items():
reference = node['ref']
ref = ConanFileReference.loads(reference)
if ref == recipe.ref:
reqs = [
self.get_reference(nodes, it)
for it in node.get('requires', [])
]
breqs = [
self.get_reference(nodes, it)
for it in node.get('build_requires', [])
]
pyreqs = [
self.get_reference(nodes, it)
for it in node.get('python_requires', [])
]
return reqs, breqs, pyreqs
assert False, "Never get here!"
except Exception as e:
log.error(f"Error in recipe {recipe.ref}!!! {e}")
log.error(f" - command: {' '.join(cmd)}")
log.error(f" - recipe: {recipe.ref}")
log.error(f" - profile: {os.path.basename(profile)}")
log.error(r)
return None, None, None
