def build_web_files(prod, project_path):
print('Building web...')
npm_mode = 'prod' if prod else 'dev'
work_dir = os.path.join(project_path, 'web-src')
process_utils.invoke('npm install', work_dir)
process_utils.invoke('npm run build:' + npm_mode, work_dir)
print('Done')
def rebuild_root(parent_project_path, mvn_opts, silent=True):
command = "mvn clean install " + mvn_opts
if silent:
process_utils.invoke(command, parent_project_path, exit_on_failure=True)
else:
process_utils.invoke_attached(command, parent_project_path)
def create_version_file():
if 'TRAVIS_BRANCH' in os.environ:
current_branch = os.environ['TRAVIS_BRANCH']
else:
current_branch = process_utils.invoke(
'git rev-parse --abbrev-ref HEAD').strip()
npm_version = get_npm_version()
if current_branch == 'stable':
last_tag = process_utils.invoke(
'git describe --exclude dev --abbrev=0 --tags').strip()
last_tag_version = parse_semver_str(last_tag)
if (last_tag_version[0] == npm_version[0]) and (last_tag_version[1]
== npm_version[1]):
new_version = [
last_tag_version[0], last_tag_version[1],
last_tag_version[2] + 1
]
else:
new_version = npm_version
new_version = '.'.join([str(v) for v in new_version])
else:
git_hash = process_utils.invoke('git rev-parse --short HEAD').strip()
new_version = str(npm_version[0])
new_version += '.' + str(npm_version[1] + 1)
new_version += '.0-'
new_version += current_branch + '@' + git_hash
file_utils.write_file(VERSION_FILE, new_version)
def rebuild_root(parent_project_path, mvn_opts, silent=True):
command = "mvn clean install " + mvn_opts
if silent:
process_utils.invoke(command,
parent_project_path,
exit_on_failure=True)
else:
process_utils.invoke_attached(command, parent_project_path)
def verify(self, username, password):
try:
process_utils.invoke(['htpasswd', '-bv', self.path, username, password], check_stderr=False)
return True
except ExecutionException as e:
if e.exit_code in [3, 5, 6]:
return False
raise e
def rebuild(parent_project_path, projects, mvn_opts, silent=True):
if not projects:
print("No projects to build, skipping")
return None
project_roots = analyze_project_roots(projects)
levels = []
if len(set(project_roots.values())) > 1:
dependency_levels = split_by_dependencies(projects, project_roots)
print("Complex structure detected. Rebuilding in several steps")
for i in range(0, len(dependency_levels)):
level = dependency_levels[i]
grouped_by_root = {}
for project in level:
root = project_roots[project]
if not (root in grouped_by_root):
grouped_by_root[root] = []
grouped_by_root[root].append(project)
print("Step " + str(i) + ":")
for root_path, child_projects in grouped_by_root.items():
rel_root_path = os.path.relpath(root_path, parent_project_path)
print('\t' + rel_root_path + ': ' +
str(collections.to_strings(child_projects)))
levels.append((root_path, child_projects))
else:
root_value = list(project_roots.values())[0]
levels.append((root_value, project_roots.keys()))
for root_path, child_projects in levels:
project_names = [(":" + project.artifact_id)
for project in child_projects]
project_names_string = ",".join(project_names)
root_pom_path = os.path.join(root_path, 'pom.xml')
command = 'mvn clean install -f {} {} -pl {}'.format(
root_pom_path, mvn_opts, project_names_string)
if silent:
process_utils.invoke(command,
parent_project_path,
exit_on_failure=True)
else:
process_utils.invoke_attached(command, parent_project_path)
def _resolve_default(default, username, audit_name, working_dir):
if not default:
return default
script = False
if isinstance(default, dict) and 'script' in default:
string_value = default['script']
script = True
elif isinstance(default, str):
string_value = default
else:
return default
resolved_string_value = resolve_env_vars(string_value, full_match=True)
if resolved_string_value == string_value:
resolved_string_value = replace_auth_vars(string_value, username,
audit_name)
if script:
has_variables = string_value != resolved_string_value
shell = read_bool_from_config('shell',
default,
default=not has_variables)
output = process_utils.invoke(resolved_string_value,
working_dir,
shell=shell)
return output.strip()
return resolved_string_value
def is_installed():
try:
output = process_utils.invoke(
['htpasswd', '-nbp', 'some_user', 'some_password'],
check_stderr=False)
return output.strip().endswith('some_user:some_password')
except FileNotFoundError:
return False
def get_local_changed_files(self, abs_path, ignore_unversioned=True):
status_info = process_utils.invoke(['svn', 'status', '--xml', abs_path])
entries_xpath = '*/entry'
found_results = xml_utils.find_in_string(status_info, [entries_xpath])
entries = found_results[entries_xpath]
return self.svn_xml_status_to_files(entries, abs_path, ignore_unversioned)
def get_revision_changed_files(self, abs_path, from_revision, to_revision):
status_info = process_utils.invoke(
['svn', 'diff', '--summarize', '--xml', '-r' + from_revision + ':' + to_revision, abs_path])
entries_xpath = '*/path'
found_results = xml_utils.find_in_string(status_info, [entries_xpath])
entries = found_results[entries_xpath]
return self.svn_xml_diff_to_files(entries, abs_path)
def rebuild(parent_project_path, projects, mvn_opts, silent=True):
if not projects:
print("No projects to build, skipping")
return None
project_roots = analyze_project_roots(projects)
levels = []
if len(set(project_roots.values())) > 1:
dependency_levels = split_by_dependencies(projects, project_roots)
print("Complex structure detected. Rebuilding in several steps")
for i in range(0, len(dependency_levels)):
level = dependency_levels[i]
grouped_by_root = {}
for project in level:
root = project_roots[project]
if not (root in grouped_by_root):
grouped_by_root[root] = []
grouped_by_root[root].append(project)
print("Step " + str(i) + ":")
for root_path, child_projects in grouped_by_root.items():
rel_root_path = os.path.relpath(root_path, parent_project_path)
print('\t' + rel_root_path + ': ' + str(collections.to_strings(child_projects)))
levels.append((root_path, child_projects))
else:
root_value = list(project_roots.values())[0]
levels.append((root_value, project_roots.keys()))
for root_path, child_projects in levels:
project_names = [(":" + project.artifact_id) for project in child_projects]
project_names_string = ",".join(project_names)
root_pom_path = os.path.join(root_path, 'pom.xml')
command = 'mvn clean install -f {} {} -pl {}'.format(root_pom_path, mvn_opts, project_names_string)
if silent:
process_utils.invoke(command, parent_project_path, exit_on_failure=True)
else:
process_utils.invoke_attached(command, parent_project_path)
def get_local_changed_files(self, abs_path, ignore_unversioned=True):
status_info = process_utils.invoke(
['svn', 'status', '--xml', abs_path])
entries_xpath = '*/entry'
found_results = xml_utils.find_in_string(status_info, [entries_xpath])
entries = found_results[entries_xpath]
return self.svn_xml_status_to_files(entries, abs_path,
ignore_unversioned)
def get_revision(self, project_path):
svn_info = process_utils.invoke(['svn', 'info', project_path])
info_lines = svn_info.split('\n')
revision_prefix = 'Revision: '
for line in info_lines:
if line.startswith(revision_prefix):
return line[len(revision_prefix):]
raise Exception("Couldn't get svn revision in " + project_path)
def get_revision(self, project_path):
svn_info = process_utils.invoke(['svn', 'info', project_path])
info_lines = svn_info.split('\n')
revision_prefix = 'Revision: '
for line in info_lines:
if line.startswith(revision_prefix):
return line[len(revision_prefix):]
raise Exception("Couldn't get svn revision in " + project_path)
def get_revision_changed_files(self, abs_path, from_revision, to_revision):
status_info = process_utils.invoke([
'svn', 'diff', '--summarize', '--xml',
'-r' + from_revision + ':' + to_revision, abs_path
])
entries_xpath = '*/path'
found_results = xml_utils.find_in_string(status_info, [entries_xpath])
entries = found_results[entries_xpath]
return self.svn_xml_diff_to_files(entries, abs_path)
def get_local_changed_files(self, abs_path, ignore_unversioned=True):
result = set()
status_info = process_utils.invoke(['git', 'status', '-s'], abs_path, ['?'])
if ignore_unversioned:
changed_files = self.parse_changed_files(status_info, abs_path, ['?'])
else:
changed_files = self.parse_changed_files(status_info, abs_path)
result.update(changed_files)
return list(result)
def get_values(self, parameter_values):
for param_name in self._required_parameters:
value = parameter_values.get(param_name)
if is_empty(value):
return []
script = fill_parameter_values(self._parameter_configs,
self._script_template, parameter_values)
try:
script_output = process_utils.invoke(script)
except:
LOGGER.exception('Failed to execute script')
return []
script_output = script_output.rstrip('\n')
return script_output.split('\n')
def get_values(self, parameter_values):
for param_name in self._required_parameters:
value = parameter_values.get(param_name)
if is_empty(value):
return []
parameters = self._parameters_supplier()
script = fill_parameter_values(parameters, self._script_template, parameter_values)
try:
script_output = process_utils.invoke(script)
except Exception as e:
LOGGER.warning('Failed to execute script. ' + str(e))
return []
script_output = script_output.rstrip('\n')
return script_output.split('\n')
def get_values(self, parameter_values):
for param_name in self._required_parameters:
value = parameter_values.get(param_name)
if is_empty(value):
return []
parameters = self._parameters_supplier()
script = fill_parameter_values(parameters, self._script_template, parameter_values)
try:
script_output = process_utils.invoke(script)
except Exception as e:
LOGGER.warn('Failed to execute script. ' + str(e))
return []
script_output = script_output.rstrip('\n')
return script_output.split('\n')
def _resolve_default(default, username, audit_name, working_dir):
if not default:
return default
script = False
if isinstance(default, dict) and 'script' in default:
string_value = default['script']
script = True
elif isinstance(default, str):
string_value = default
else:
return default
resolved_string_value = resolve_env_vars(string_value, full_match=True)
if resolved_string_value == string_value:
resolved_string_value = replace_auth_vars(string_value, username, audit_name)
if not script:
return resolved_string_value
output = process_utils.invoke(resolved_string_value, working_dir)
return output.strip()
def _resolve_default(default, username, audit_name, working_dir):
if not default:
return default
script = False
if isinstance(default, dict) and 'script' in default:
string_value = default['script']
script = True
elif isinstance(default, str):
string_value = default
else:
return default
resolved_string_value = resolve_env_vars(string_value, full_match=True)
if resolved_string_value == string_value:
resolved_string_value = replace_auth_vars(string_value, username, audit_name)
if not script:
return resolved_string_value
output = process_utils.invoke(resolved_string_value, working_dir)
return output.strip()
def __init__(self, script) -> None:
script_output = process_utils.invoke(script)
script_output = script_output.rstrip('\n')
self._values = script_output.split('\n')
def from_json(file_path, json_string, pty_enabled_default=False):
json_object = json.loads(json_string)
config = Config()
config.name = read_name(file_path, json_string)
config.script_body = json_object.get("script_path")
config.description = json_object.get("description")
config.working_directory = json_object.get("working_directory")
config.redirect = json_object.get("redirect")
requires_terminal = json_object.get("requires_terminal")
if requires_terminal is not None:
if requires_terminal == True:
config.requires_terminal = True
elif requires_terminal == False:
config.requires_terminal = False
else:
raise Exception(
"'requires_terminal' parameter should be True or False")
else:
config.requires_terminal = pty_enabled_default
parameters_json = json_object.get("parameters")
if parameters_json is not None:
for parameter_json in parameters_json:
parameter = Parameter()
parameter.set_name(parameter_json.get("name"))
parameter.set_param(parameter_json.get("param"))
parameter.set_no_value(parameter_json.get("no_value"))
parameter.set_description(parameter_json.get("description"))
parameter.set_required(parameter_json.get("required"))
parameter.set_default(parameter_json.get("default"))
parameter.set_min(parameter_json.get("min"))
parameter.set_max(parameter_json.get("max"))
values = parameter_json.get("values")
if values:
if isinstance(values, list):
parameter.set_values(values)
elif "script" in values:
script_output = process_utils.invoke(values["script"])
script_output = script_output.rstrip("\n")
derived_values = script_output.split("\n")
parameter.set_values(derived_values)
else:
raise Exception("Unsupported values")
type = parameter_json.get("type")
if type:
parameter.set_type(type)
constant = parameter_json.get("constant")
if constant == True:
if not parameter.get_default():
raise Exception(
"Constant should have default value specified")
parameter.set_constant(constant)
config.add_parameter(parameter)
return config
def send(self, parameters, environment_variables=None):
full_command = self.command + parameters
process_utils.invoke(full_command, environment_variables=environment_variables)
def send(self, parameters, environment_variables=None):
full_command = self.command + parameters
process_utils.invoke(full_command,
environment_variables=environment_variables)
def get_revision_changed_files(self, abs_path, from_revision, to_revision):
diff_info = process_utils.invoke(
['git', 'diff', '--name-status', from_revision, to_revision], abs_path)
return self.parse_changed_files(diff_info, abs_path)
def get_revision(self, project_path):
return process_utils.invoke(['git', 'rev-parse', 'HEAD'], project_path).strip()
def from_json(file_path, json_object, pty_enabled_default=False):
config = Config()
config.name = read_name(file_path, json_object)
config.script_command = json_object.get("script_path")
config.description = json_object.get("description")
config.working_directory = json_object.get("working_directory")
config.requires_terminal = read_boolean("requires_terminal", json_object,
pty_enabled_default)
config.bash_formatting = read_boolean(
"bash_formatting", json_object,
os_utils.is_linux() or os_utils.is_mac())
config.allowed_users = json_object.get('allowed_users')
if config.allowed_users is None:
config.allowed_users = ANY_USER
output_files = json_object.get("output_files")
if output_files:
config.output_files = output_files
parameters_json = json_object.get("parameters")
if parameters_json is not None:
for parameter_json in parameters_json:
parameter = Parameter()
parameter.set_name(parameter_json.get('name'))
parameter.set_param(parameter_json.get('param'))
parameter.set_no_value(parameter_json.get('no_value'))
parameter.set_description(parameter_json.get('description'))
parameter.set_required(parameter_json.get('required'))
parameter.set_default(parameter_json.get('default'))
parameter.set_min(parameter_json.get('min'))
parameter.set_max(parameter_json.get('max'))
parameter.secure = read_boolean('secure', parameter_json)
parameter.separator = parameter_json.get('separator', ',')
parameter.multiple_arguments = read_boolean('multiple_arguments',
parameter_json,
default=False)
values = parameter_json.get("values")
if values:
if isinstance(values, list):
parameter.set_values(values)
elif "script" in values:
script_output = process_utils.invoke(values["script"])
script_output = script_output.rstrip("\n")
derived_values = script_output.split("\n")
parameter.set_values(derived_values)
else:
raise Exception("Unsupported values")
type = parameter_json.get('type')
if type:
parameter.type = type
constant = parameter_json.get("constant")
if constant is True:
if not parameter.get_default():
raise Exception(
"Constant should have default value specified")
parameter.set_constant(constant)
config.add_parameter(parameter)
return config
def from_json(file_path, json_string, pty_enabled_default=False):
json_object = json.loads(json_string)
config = Config()
config.name = read_name(file_path, json_string)
config.script_command = json_object.get("script_path")
config.description = json_object.get("description")
config.working_directory = json_object.get("working_directory")
config.requires_terminal = read_boolean("requires_terminal", json_object,
pty_enabled_default)
config.bash_formatting = read_boolean(
"bash_formatting", json_object,
os_utils.is_linux() or os_utils.is_mac())
config.kill_on_disconnect = read_boolean('kill_on_disconnect', json_object,
True)
output_files = json_object.get("output_files")
if output_files:
config.output_files = output_files
parameters_json = json_object.get("parameters")
if parameters_json is not None:
for parameter_json in parameters_json:
parameter = Parameter()
parameter.set_name(parameter_json.get("name"))
parameter.set_param(parameter_json.get("param"))
parameter.set_no_value(parameter_json.get("no_value"))
parameter.set_description(parameter_json.get("description"))
parameter.set_required(parameter_json.get("required"))
parameter.set_default(parameter_json.get("default"))
parameter.set_min(parameter_json.get("min"))
parameter.set_max(parameter_json.get("max"))
parameter.secure = read_boolean('secure', parameter_json)
values = parameter_json.get("values")
if values:
if isinstance(values, list):
parameter.set_values(values)
elif "script" in values:
script_output = process_utils.invoke(values["script"])
script_output = script_output.rstrip("\n")
derived_values = script_output.split("\n")
parameter.set_values(derived_values)
else:
raise Exception("Unsupported values")
type = parameter_json.get('type')
if type:
parameter.type = type
constant = parameter_json.get("constant")
if constant is True:
if not parameter.get_default():
raise Exception(
"Constant should have default value specified")
parameter.set_constant(constant)
config.add_parameter(parameter)
return config
def __init__(self, script) -> None:
script_output = process_utils.invoke(script)
script_output = script_output.rstrip('\n')
self._values = script_output.split('\n')
def __init__(self, script) -> None:
script_output = process_utils.invoke(script)
script_output = script_output.rstrip('\n')
self._values = [
line for line in script_output.split('\n') if not is_empty(line)
]