def fire_command_and_show_stdout(self,
commands: List[Union[str, List[str]]],
cwd: str = None,
env: Dict[str, Any] = None,
check_exit_code: bool = True,
timeout: int = None,
log_entry: bool = False) -> int:
#get cwd
if cwd is None:
cwd = os.getcwd()
# fetch the current user environment variables and updates with the ones from the caller
actual_env = dict(os.environ)
if env is not None:
for k, v in env.items():
actual_env[k] = v
# create tempfile
with self.create_temp_directory_with(
"pmakeup-command-") as absolute_temp_dir:
try:
filepath = self.create_temp_file(directory=absolute_temp_dir,
file_prefix="cmd_",
file_suffix=".cmd",
executable_for_owner=True)
self._write_cmd_bat_file(filepath, actual_env, commands)
actual_command = f"""cmd.exe /C \"{filepath}\""""
log_method = logging.critical if log_entry else logging.debug
log_method(f"Executing {actual_command}")
with open(filepath, "r") as f:
log_method(f"in file \"{filepath}\" = \n{f.read()}")
if len(os.getcwd()) > 258:
raise ValueError(
f"{os.getcwd()} path is too long. needs to be at most 258"
)
if len(cwd) > 258:
raise ValueError(
f"{cwd} path is too long. needs to be at most 258")
result = subprocess.run(args=actual_command,
cwd=cwd,
shell=True,
capture_output=False,
timeout=timeout,
env=actual_env)
if check_exit_code and result.returncode != 0:
raise pm.PMakeupException(
f"cwd=\"{cwd}\" command=\"{actual_command}\" exit=\"{result.returncode}\""
)
return result.returncode
finally:
os.unlink(filepath)
def get_variable(self, name: str) -> Any:
"""
Ensure the user has passed a variable.
If not, raises an exception
:param name: the variable name to check
:raises PMakeupException: if the variable is not found
"""
if name not in self.get_shared_variables():
raise pm.PMakeupException(f"Variable {name} not found")
return self.get_shared_variables()[name]
def _get_platform(self) -> pm.IOSSystem:
"""
get the current operating system type
:return: structure providing you specific operating system methods
"""
if os.name == "nt":
return pm.WindowsOSSystem(self)
elif os.name == "posix":
return pm.LinuxOSSystem(self)
else:
raise pm.PMakeupException(f"Cannot identify platform!")
def process_targets(self):
"""
Function used to process in the correct order. If the user requested to show the help for this file,
the function will show it and return it
It will call the function declared in declare_target
"""
def show_target_help():
if self._model.info_description is not None:
print(self._model.info_description)
for a_i, a_target_name in enumerate(self._model.available_targets):
a_target_descriptor = self._model.available_targets[a_target_name]
print(f" - {a_i}. {a_target_name}: {a_target_descriptor.description}")
def perform_target(name: str, descriptor: pm.TargetDescriptor):
if name in already_done:
# do nothing if the node has already been processed
return
if name in doing:
raise ValueError(f"Cyclic dependencies detected!")
doing.add(name)
out_edges = list(self._model.target_network.edges(name))
if len(out_edges) == 0:
# the node has no dependencies: perform the task
descriptor.function()
else:
# G.edges([0, 2])
# OutEdgeDataView([(0, 1), (2, 3)])
for sink in map(lambda x: x[1], out_edges):
perform_target(sink, self._model.available_targets[sink])
# we have satisfied all requirements. Perform this target
descriptor.function()
# mark the node as "already done"
doing.remove(name)
already_done.add(name)
if self._model.should_show_target_help:
show_target_help()
else:
doing = set()
already_done = set()
logging.info(f"Available targets are {', '.join(self._model.available_targets.keys())}")
for i, target_name in enumerate(self._model.requested_target_names):
if target_name not in self._model.available_targets:
raise pm.PMakeupException(
f"Invalid target {target_name}. Available targets are {', '.join(self._model.available_targets.keys())}")
target_descriptor = self._model.available_targets[target_name]
self._log_command(f"Executing target \"{target_descriptor.name}\"")
perform_target(target_descriptor.name, target_descriptor)
def get_env_variable(self, name: str) -> str:
code, stdout, _ = self.fire_command_and_capture_stdout(
commands=[f"echo %{name}%"],
log_entry=True,
)
stdout = stdout.strip()
if len(stdout) == 0:
raise pm.PMakeupException(
f"Cannot find the environment variable \"{name}\" for user \"{self.get_current_username()}\""
)
return stdout
def get_env_variable(self, name: str) -> str:
exit_code, stdout, _ = self.execute_command(
commands=[f"printenv {name}"],
capture_stdout=True,
show_output_on_screen=False,
check_exit_code=False,
)
if exit_code != 0:
raise pm.PMakeupException(
f"Cannot find the environment variable \"{name}\" for user \"{self.get_current_username()}\""
)
return stdout.strip()
def cd_into_directories(self, folder: pm.path, prefix: str, folder_format: str, error_if_mismatch: bool = True):
"""
Inside the given folder, there can be several folders, each of them with the same format. We cd into the "latest" one.
How can we determine which is the "latest" one? Via folder_format. it is a string that is either:
- "number": an integer number
- "semver2": a semantic versionign string;
We fetch the "latest" by looking at the one with the greater value. If the folder contains a folder which it is not compliant
with folder_format, it is either ignored or rase error
:param folder: folder where several folders are located
:param prefix: a string that prefix folder_format
:param folder_format: either "number" or "semver2"
:param error_if_mismatch: if a folder is not compliant with folder_format, if true we will generate an exception
:return:
"""
try:
p = self.abs_path(folder)
self._log_command(f"Cd'ing into the \"latest\" directory in folder \"{p}\" according to criterion \"{folder_format}\"")
self._disable_log_command = True
self.cd(folder)
folders = dict()
for subfolder in self.files.ls_only_directories(p):
if not subfolder.startswith(prefix):
if error_if_mismatch:
raise pm.PMakeupException(f"subfolder \"{subfolder}\" in \"{p}\" does not start with \"{prefix}\"")
else:
continue
subfolder_id = subfolder[len(prefix):]
try:
if folder_format == "semver2":
folders[Version(subfolder_id)] = subfolder
elif folder_format == "number":
folders[int(subfolder_id)] = subfolder
else:
raise pm.InvalidScenarioPMakeupException(f"invalid folder_format \"{folder_format}\"")
except Exception as e:
if error_if_mismatch:
raise e
else:
continue
# fetch the "latest" by fetching the greater value in "folders"
latest_folder = list(sorted(folders.keys()))[0]
self.cd(folders[latest_folder])
finally:
self._disable_log_command = False
def build():
log.echo("Building...", foreground="blue")
if core.on_linux():
log.echo("building for linux", foreground="blue")
operating_system.execute_stdout_on_screen([
f"source {paths.path('venv', 'bin', 'activate')}",
f"python setup.py bdist_wheel", f"deactivate"
])
elif core.on_windows():
log.echo(f"building for windows in {paths.cwd()}", foreground="blue")
operating_system.execute_stdout_on_screen([
f"python setup.py bdist_wheel",
])
else:
raise pm.PMakeupException()
def require_pmakeup_version(self, lowerbound: str) -> None:
"""
Check if the current version of pmakeup is greater or equal than the given one.
If the current version of pmakeup is not compliant with this constraint, an error is generated
:param lowerbound: the minimum version this script is compliant with
"""
system_version = Version(pm.version.VERSION)
script_version = Version(lowerbound)
self._log_command(
f"Checking if script minimum pmakeup version {script_version} >= {system_version}"
)
if script_version > system_version:
raise pm.PMakeupException(
f"The script requires at least version {script_version} to be installed. Current version is {system_version}"
)
def semantic_version_2_only_core(self, filename: str) -> Version:
"""
A function that can be used within ::get_latest_version_in_folder
:param filename: the absolute path of a file that contains a version
:return: the version
"""
regex = r"\d+\.\d+\.\d+"
b = os.path.basename(filename)
m = re.search(regex, b)
logging.debug(f"checking if \"{filename}\" satisfies \"{regex}\"")
if m is None:
raise pm.PMakeupException(
f"Cannot find the regex {regex} within file \"{b}\"!")
logging.debug(f"yes: \"{m.group(0)}\"")
return Version(m.group(0))
def find_executable_in_program_directories(self, program_name: str, fail_if_program_is_not_found: bool = False) -> \
Optional[pm.path]:
"""
Find a program ouside the path as well. Paths is still considered
:param program_name: name of the program to look for
:param fail_if_program_is_not_found: if true, we will raise an exception if the program is not found
:return: first absolute path of the program found. None if we did not find the program
"""
self._log_command(
f"""Find the executable \"{program_name}\" in the place where the operating system usually puts installed programs...""")
result = self.platform.find_executable_in_program_directories(
program_name=program_name,
)
if result is None and fail_if_program_is_not_found:
raise pm.PMakeupException(f"We could not find the program \"{program_name}\" on the system!")
return result
def quasi_semantic_version_2_only_core(self, filename: str) -> Version:
"""
A function that can be used within ::get_latest_version_in_folder.
It accepts values like "1.0.0", but also "1.0" and "1"
:param filename: the absolute path of a file that contains a version
:return: the version
"""
regex = r"\d+(?:\.\d+(?:\.\d+)?)?"
b = os.path.basename(filename)
m = re.search(regex, b)
if m is None:
raise pm.PMakeupException(
f"Cannot find the regex {regex} within file \"{b}\"!")
result = m.group(0)
if len(result.split(".")) == 2:
result += ".0"
if len(result.split(".")) == 1:
result += ".0.0"
return Version(result)
def upload_to_pypi():
TWINE_PYPI_PASSWORD = files.read_file_content("TWINE_PYPI_PASSWORD")
log.echo("Uploading to pypi ...", foreground="blue")
latest_version, file_list = get_latest_version_in_folder(
"dist", version_fetcher=semantic_version_2_only_core)
upload_files = ' '.join(map(lambda x: f"\"{x}\"", file_list))
log.echo(f"File to upload is {upload_files}...", foreground="blue")
if core.on_linux():
log.echo("Uploading for linux", foreground="blue")
operating_system.execute_stdout_on_screen([
f"twine upload --verbose --non-interactive --username \"{TWINE_PYPI_USER}\" --password \"{TWINE_PYPI_PASSWORD}\" {upload_files}",
])
elif core.on_windows():
log.echo("Uploading for windows", foreground="blue")
operating_system.execute_stdout_on_screen([
f"twine upload --verbose --non-interactive --username \"{TWINE_PYPI_USER}\" --password \"{TWINE_PYPI_PASSWORD}\" {upload_files}",
])
else:
raise pm.PMakeupException()
def get_column_of_table_by_name(self, table: List[List[str]],
column_name: str) -> List[str]:
"""
Select a single column from the table, generated by ::convert_table
We assumed the first row of the table is a header, contaiing the column names
:param table: the table generated by ::convert_table
:param column_name: name of the column to return.
:return: the column requested
"""
header = table[0]
column_index = None
for index, name in enumerate(header):
if name == column_name:
column_index = index
break
if column_index is None:
raise pm.PMakeupException(
f"Cannot find column named '{column_name}' in header: {', '.join(header)}"
)
return self.get_column_of_table(table, column_index)
def upload_to_test_pypi():
testpypi = paths.get_absolute_file_till_root("TWINE_TEST_PYPI_PASSWORD")
TWINE_TEST_PYPI_PASSWORD = files.read_file_content(testpypi)
log.echo("Uploading to test pypi...", foreground="blue")
latest_version, file_list = get_latest_version_in_folder(
"dist", version_fetcher=semantic_version_2_only_core)
upload_files = ' '.join(map(lambda x: f"\"{x}\"", file_list))
if core.on_linux():
log.echo("Uploading for linux", foreground="blue")
operating_system.execute_stdout_on_screen([
#"source venv/bin/activate",
f"twine upload --verbose --repository testpypi --username \"{TWINE_TEST_PYPI_USER}\" --password \"{TWINE_TEST_PYPI_PASSWORD}\" {upload_files}",
#"deactivate"
])
elif core.on_windows():
log.echo("Uploading for windows", foreground="blue")
operating_system.execute_stdout_on_screen([
#"venv/Scripts/activate.bat",
f"twine upload --verbose --repository testpypi --username \"{TWINE_TEST_PYPI_USER}\" --password \"{TWINE_TEST_PYPI_PASSWORD}\" {upload_files}",
#"venv/Scripts/deactivate.bat"
])
else:
raise pm.PMakeupException()
def execute_command(self,
commands: List[Union[str, List[str]]],
show_output_on_screen: bool,
capture_stdout: bool,
cwd: str = None,
env: Dict[str, Any] = None,
check_exit_code: bool = True,
timeout: int = None,
execute_as_admin: bool = False,
admin_password: str = None,
log_entry: bool = False) -> Tuple[int, str, str]:
# fetch the current user environment variables and updates with the ones from the caller
actual_env = dict(os.environ)
if env is None:
env = {}
for k, v in env.items():
actual_env[k] = v
# create tempfile
with self.create_temp_directory_with(
"pmakeup-command-") as absolute_temp_dir:
filepath = self.create_temp_file(directory=absolute_temp_dir,
file_prefix="cmd_",
file_suffix=".bash",
executable_for_owner=True)
with open(filepath, "w") as f:
# put the commands in the temp file
f.write("#!/bin/bash\n")
for cmd in commands:
if isinstance(cmd, str):
cmd_str = cmd
elif isinstance(cmd, list):
cmd_str = ' '.join(cmd)
else:
raise TypeError(
f"Invalid type of command {type(cmd)}!")
f.write(cmd_str)
f.write("\n")
stdout_filepath = os.path.join(absolute_temp_dir, "stdout.txt")
stderr_filepath = os.path.join(absolute_temp_dir, "stderr.txt")
if len(actual_env) > 0:
env_part = ','.join(actual_env.keys())
env_part = f"--preserve-env={env_part}"
else:
env_part = ""
# Now execute file
if execute_as_admin:
if admin_password:
password_file = self.create_temp_file(
directory=absolute_temp_dir, file_prefix="stdin-")
with open(password_file, "w") as f:
f.write(f"{admin_password}\n")
if show_output_on_screen and capture_stdout:
actual_command = f"""cat '{password_file}' | sudo {env_part} --stdin --login bash '{filepath}'"""
actual_capture_output = True
actual_read_stdout = False
elif show_output_on_screen and not capture_stdout:
actual_command = f"""cat '{password_file}' | sudo {env_part} --stdin --login bash '{filepath}'"""
actual_capture_output = False
actual_read_stdout = False
elif not show_output_on_screen and capture_stdout:
actual_command = f"""cat '{password_file}' | sudo {env_part} --stdin --login bash '{filepath}' > {stdout_filepath} 2>{stderr_filepath}"""
actual_capture_output = False
actual_read_stdout = True
else:
actual_command = f"""cat '{password_file}' | sudo {env_part} --stdin --login bash '{filepath}' 2>&1 > /dev/null"""
actual_capture_output = False
actual_read_stdout = False
else:
if show_output_on_screen and capture_stdout:
actual_command = f"""sudo {env_part} --login bash '{filepath}'"""
actual_capture_output = True
actual_read_stdout = False
elif show_output_on_screen and not capture_stdout:
actual_command = f"""sudo {env_part} --login bash '{filepath}'"""
actual_capture_output = False
actual_read_stdout = False
elif not show_output_on_screen and capture_stdout:
actual_command = f"""sudo {env_part} --login bash '{filepath}' > {stdout_filepath} 2>{stderr_filepath}"""
actual_capture_output = False
actual_read_stdout = True
else:
actual_command = f"""sudo {env_part} --login bash '{filepath}' 2>&1 > /dev/null"""
actual_capture_output = False
actual_read_stdout = False
else:
if show_output_on_screen and capture_stdout:
actual_command = f"""sudo {env_part} --user {self.get_current_username()} --login bash '{filepath}'"""
actual_capture_output = True
actual_read_stdout = False
elif show_output_on_screen and not capture_stdout:
actual_command = f"""sudo {env_part} --user {self.get_current_username()} --login bash '{filepath}'"""
actual_capture_output = False
actual_read_stdout = False
elif not show_output_on_screen and capture_stdout:
actual_command = f"""sudo {env_part} --user {self.get_current_username()} --login bash '{filepath}' > {stdout_filepath} 2>{stderr_filepath}"""
actual_capture_output = False
actual_read_stdout = True
else:
actual_command = f"""sudo {env_part} --user {self.get_current_username()} --login bash '{filepath}' 2>&1 > /dev/null"""
actual_capture_output = False
actual_read_stdout = False
if log_entry:
log_method = logging.info
else:
log_method = logging.debug
log_method(f"Executing {actual_command}")
with open(filepath, "r") as f:
log_method(f"in file \"{filepath}\" = \n{f.read()}")
result = subprocess.run(args=actual_command,
cwd=cwd,
shell=True,
capture_output=actual_capture_output,
timeout=timeout,
env=env)
if check_exit_code and result.returncode != 0:
raise pm.PMakeupException(
f"cwd=\"{cwd}\" command=\"{actual_command}\" exit=\"{result.returncode}\""
)
if actual_capture_output:
stdout = self._convert_stdout(result.stdout)
stderr = self._convert_stdout(result.stderr)
elif actual_read_stdout:
with open(stdout_filepath) as f:
stdout = self._convert_stdout(f.read())
with open(stderr_filepath) as f:
stderr = self._convert_stdout(f.read())
else:
stdout = ""
stderr = ""
return result.returncode, stdout, stderr
def fire_admin_command_and_wait(self,
commands: List[Union[str, List[str]]],
cwd: str = None,
env: Dict[str, Any] = None,
check_exit_code: bool = True,
timeout: int = None,
log_entry: bool = False,
credential_type: str = None,
credential: any = None) -> int:
# get cwd
if cwd is None:
cwd = os.getcwd()
# fetch the current user environment variables and updates with the ones from the caller
actual_env = dict(os.environ)
if env is not None:
for k, v in env.items():
actual_env[k] = v
# create tempfile
with self.create_temp_directory_with(
"pmakeup-command-") as absolute_temp_dir:
try:
filepath = self.create_temp_file(directory=absolute_temp_dir,
file_prefix="cmd_",
file_suffix=".cmd",
executable_for_owner=True)
self._write_cmd_bat_file(filepath, actual_env, commands)
# Now execute file
if credential_type == 'password':
admin_password: str = credential
actual_command = f"""powershell.exe -Command \"Start-Process -FilePath 'cmd.exe' -ArgumentList '/C','{filepath} > nul 2>&1' -WorkingDirectory '{cwd}' -Wait -Verb RunAs\""""
else:
raise ValueError(
f"invalid credential type {credential_type}")
log_method = logging.critical if log_entry else logging.debug
log_method(f"Executing {actual_command}")
with open(filepath, "r") as f:
log_method(f"in file \"{filepath}\" = \n{f.read()}")
if len(os.getcwd()) > 258:
raise ValueError(
f"{os.getcwd()} path is too long. needs to be at most 258"
)
if len(cwd) > 258:
raise ValueError(
f"{cwd} path is too long. needs to be at most 258")
result = subprocess.run(args=actual_command,
cwd=cwd,
shell=True,
capture_output=False,
timeout=timeout,
env=actual_env)
if check_exit_code and result.returncode != 0:
raise pm.PMakeupException(
f"cwd=\"{cwd}\" command=\"{actual_command}\" exit=\"{result.returncode}\""
)
return result.returncode
finally:
os.unlink(filepath)
