def assert_downloadable_files(self, prepared_files, original_files):
prepared_files_dict = {os.path.basename(f): f for f in prepared_files}
original_files_dict = {os.path.basename(f): f for f in original_files}
self.assertEqual(original_files_dict.keys(), prepared_files_dict.keys())
for filename in original_files_dict.keys():
prepared_file = prepared_files_dict[filename]
original_file = original_files_dict[filename]
prepared_content = file_utils.read_file(prepared_file)
original_content = file_utils.read_file(original_file)
self.assertEqual(original_content, prepared_content, 'Different content for file ' + filename)
def __introduce_access_config(context):
conf_folder = os.path.join(context.conf_folder, 'runners')
if not os.path.exists(conf_folder):
return
conf_files = [os.path.join(conf_folder, file)
for file in os.listdir(conf_folder)
if file.lower().endswith('.json')]
for conf_file in conf_files:
content = file_utils.read_file(conf_file)
json_object = json.loads(content, object_pairs_hook=OrderedDict)
if ('output_files' not in json_object) or ('parameters' not in json_object):
continue
output_files = json_object['output_files']
parameter_names = [p['name'] for p in json_object['parameters'] if not is_blank(p.get('name'))]
changed = False
for i in range(len(output_files)):
output_file = output_files[i]
for param_name in parameter_names:
output_file = re.sub('\$\$\$' + param_name, '${' + param_name + '}', output_file)
if output_file != output_files[i]:
output_files[i] = output_file
changed = True
if changed:
_write_json(conf_file, json_object, content)
def _visit_script_configs(self, visitor):
configs_dir = self._script_configs_folder
files = os.listdir(configs_dir)
configs = [file for file in files if file.lower().endswith(".json")]
result = []
for config_path in configs:
path = os.path.join(configs_dir, config_path)
try:
content = file_utils.read_file(path)
visit_result = visitor(path, content)
if visit_result is not None:
result.append(visit_result)
except StopIteration as e:
if e.value is not None:
result.append(e.value)
except:
LOGGER.exception("Couldn't read the file: " + config_path)
return result
def _assert_files(self, expected_files, actual_files, expected_path_names=None):
if expected_path_names is None:
expected_path_names = {}
expected_form_files = {}
for key, file in expected_files.items():
if key in expected_path_names:
path_name = expected_path_names[key]
else:
path_name = file[0]
form_file = HttpFormFile(file[0], os.path.join(_UPLOADS_FOLDER, path_name))
put_multivalue(expected_form_files, key, form_file)
self.assertEqual(expected_form_files, actual_files)
for key, file in expected_files.items():
actual_file = actual_files[key]
if isinstance(actual_file, list):
actual_file = find_any(actual_file, lambda f: f.filename == file[0])
if actual_file is None:
self.fail('Failed to find actual file for ' + str(file))
actual_path = actual_file.path
expected_content = file[1]
byte_content = isinstance(expected_content, bytes)
actual_content = file_utils.read_file(actual_path, byte_content=byte_content)
self.assertEqual(expected_content, actual_content)
def bld_to_list(file_path):
"""Takes a BLD file path and returns an OrderedList of comments, new lines
and dict(rule) in the file."""
# Abort quickly if its a bad JSON file.
bld_as_json = su.read_build_file(file_path)
# Format each rule one by one. Preserve the comments only outside the body
# of rule dictionary.
lines = [l.strip() for l in fu.read_file(file_path).split('\n')]
formatted = []
while lines:
line = lines.pop(0)
if not line or line.startswith(COMMENT_CHAR):
formatted.append(line)
elif line.startswith(QUOTE_CHAR):
name = _rule_name(line)
#TODO: Improve it to retain comments inside a BLD rule as well.
formatted.append({RULE_NAME_KEY: name, RULE_BODY_KEY: bld_as_json[name]})
# Skip remaining lines of this rule now.
_strip_rule_body(lines)
else:
raise ValueError('Illegal text %s found in file %s.' % (line, file_path))
# Do a sanity check, formatting shouldn't change build file's semantics.
temp_file = fu.get_temp_file()
fu.write_file(temp_file, list_to_bld_string(formatted))
formatted_bld_as_json = su.read_build_file(temp_file)
assert formatted_bld_as_json == bld_as_json
return formatted
def __introduce_access_config(context):
file_path = context.conf_file
if not os.path.exists(file_path):
return
content = file_utils.read_file(file_path)
json_object = json.loads(content, object_pairs_hook=OrderedDict)
def move_to_access(field, parent_object):
if 'access' not in json_object:
json_object['access'] = {}
json_object['access'][field] = parent_object[field]
del parent_object[field]
changed = False
if 'auth' in json_object:
auth_object = json_object['auth']
if 'allowed_users' in auth_object:
move_to_access('allowed_users', auth_object)
changed = True
fields = ['admin_users', 'trusted_ips']
for field in fields:
if field in json_object:
changed = True
move_to_access(field, json_object)
if changed:
_write_json(file_path, json_object, content)
def _write_post_execution_info(log_file_path, exit_code):
file_content = file_utils.read_file(log_file_path, keep_newlines=True)
file_parts = file_content.split(OUTPUT_STARTED_MARKER + os.linesep, 1)
parameters_text = file_parts[0]
parameters_text += 'exit_code:' + str(exit_code) + os.linesep
new_content = parameters_text + OUTPUT_STARTED_MARKER + os.linesep + file_parts[1]
file_utils.write_file(log_file_path, new_content.encode(ENCODING), byte_content=True)
def test_simple_trace(self):
sys_log_file = open(self.vi.sys_log, "w")
subprocess.Popen("strace ls",
stderr=sys_log_file, stdout=sys_log_file,
shell=True)
sys_log_file.close()
sleep(5) # wait for strace output to be available
sys_logs = fu.read_file(self.vi.sys_log)
for sys_call in SYS_CALLS:
self.assertIn(sys_call, sys_logs)
def get_secret(temp_folder):
secret_file = os.path.join(temp_folder, 'secret.dat')
if os.path.exists(secret_file):
secret = file_utils.read_file(secret_file, byte_content=True)
if secret:
return secret
secret = os.urandom(256)
file_utils.write_file(secret_file, secret, byte_content=True)
return secret
def get_tornado_secret():
secret_file = os.path.join("temp", "secret.dat")
if os.path.exists(secret_file):
secret = file_utils.read_file(secret_file, byte_content=True)
if secret:
return secret
secret = os.urandom(256)
file_utils.write_file(secret_file, secret, byte_content=True)
return secret
def _validate_config(test_case, expected_filename, expected_body):
configs_path = os.path.join(test_utils.temp_folder, 'runners')
path = os.path.join(configs_path, expected_filename)
all_paths = str(os.listdir(configs_path))
test_case.assertTrue(
os.path.exists(path),
'Failed to find path ' + path + '. Existing paths: ' + all_paths)
actual_body = json.loads(file_utils.read_file(path))
test_case.assertEqual(expected_body, actual_body)
def _read_old_migrations(temp_folder):
file_path = os.path.join(temp_folder, 'migrations.txt')
if not os.path.exists(file_path):
return []
content = file_utils.read_file(file_path)
if not content:
return []
return [id.strip() for id in content.split('\n') if id.strip()]
def test_simple_trace(self):
sys_log_file = open(self.vi.sys_log, "w")
subprocess.Popen("strace ls",
stderr=sys_log_file,
stdout=sys_log_file,
shell=True)
sys_log_file.close()
sleep(5) # wait for strace output to be available
sys_logs = fu.read_file(self.vi.sys_log)
for sys_call in SYS_CALLS:
self.assertIn(sys_call, sys_logs)
def _read_old_migrations(temp_folder):
file_path = os.path.join(temp_folder, 'migrations.txt')
if not os.path.exists(file_path):
return []
content = file_utils.read_file(file_path)
if not content:
return []
return [id.strip() for id in content.split('\n') if id.strip()]
def from_json(conf_path):
if os.path.exists(conf_path):
file_content = file_utils.read_file(conf_path)
else:
file_content = "{}"
config = ServerConfig()
json_object = json.loads(file_content)
address = "0.0.0.0"
port = 5000
ssl = json_object.get("ssl")
if ssl is not None:
key_path = model_helper.read_obligatory(ssl, 'key_path', ' for ssl')
cert_path = model_helper.read_obligatory(ssl, 'cert_path', ' for ssl')
config.ssl = True
config.ssl_key_path = key_path
config.ssl_cert_path = cert_path
port = 5443
if json_object.get("address"):
address = json_object.get("address")
config.address = address
if json_object.get("port"):
port = json_object.get("port")
config.port = port
if json_object.get('title'):
config.title = json_object.get('title')
auth_config = json_object.get('auth')
admin_users = _parse_admin_users(json_object)
if auth_config:
config.authenticator = create_authenticator(auth_config)
allowed_users = auth_config.get('allowed_users')
auth_type = config.authenticator.auth_type
if auth_type == 'google_oauth' and allowed_users is None:
raise Exception('auth.allowed_users field is mandatory for ' +
auth_type)
config.authorizer = _create_authorizer(allowed_users, admin_users)
else:
config.authorizer = _create_authorizer('*', admin_users)
config.alerts_config = parse_alerts_config(json_object)
config.logging_config = parse_logging_config(json_object)
return config
def find_log(self, execution_id):
self._renew_files_cache()
file = self._ids_to_file_map.get(execution_id)
if file is None:
LOGGER.warning('find_log: file for %s id not found', execution_id)
return None
file_content = file_utils.read_file(os.path.join(self._output_folder, file),
keep_newlines=True)
log = file_content.split(OUTPUT_STARTED_MARKER, 1)[1]
return _lstrip_any_linesep(log)
def _write_post_execution_info(log_file_path, exit_code):
file_content = file_utils.read_file(log_file_path, keep_newlines=True)
file_parts = file_content.split(OUTPUT_STARTED_MARKER + os.linesep, 1)
parameters_text = file_parts[0]
parameters_text += 'exit_code:' + str(exit_code) + os.linesep
new_content = parameters_text + OUTPUT_STARTED_MARKER + os.linesep + file_parts[
1]
file_utils.write_file(log_file_path,
new_content.encode(ENCODING),
byte_content=True)
def verify_config_files(self, expected_jobs: Sequence[SchedulingJob]):
expected_files = [get_job_filename(job) for job in expected_jobs]
schedules_dir = os.path.join(test_utils.temp_folder, 'schedules')
test_utils.assert_dir_files(expected_files, schedules_dir, self)
for job in expected_jobs:
job_path = os.path.join(schedules_dir, get_job_filename(job))
content = file_utils.read_file(job_path)
restored_job = json.loads(content)
self.assertEqual(restored_job, job.as_serializable_dict())
def get(self):
try:
s_id = self.get_query_argument('strategy_id')
strategy = self.session.query(Strategy).filter(
Strategy.id == s_id).one()
except KeyError:
return self.set_status(404)
strategy = Strategy.to_dict(strategy)
strategy['code'] = f.read_file(strategy['code_location'])
print(strategy['code'])
resp = {'status': True, 'data': strategy}
self.finish(resp)
def decrypt_file(cipher: str, file_path : str, key: str):
clean_file = file_utils.unmerk(file_path)
fileBytes = file_utils.read_file(clean_file)
# returns chosen cipher class
algo = mux(cipher).Cipher(key)
# Bytes to write to modified file
if algo:
# decrypt zipped folder
print("DECRYPTING FILE BYTES",'\n')
newBytes = bytes(algo.decrypt(bytes(fileBytes)))
# write encrypted bytes and rename zip
file_utils.write_file(clean_file,newBytes)
print(f"{clean_file} has been unMERKed")
def from_json(conf_path):
if os.path.exists(conf_path):
file_content = file_utils.read_file(conf_path)
else:
file_content = "{}"
config = ServerConfig()
json_object = json.loads(file_content)
port = 5000
ssl = json_object.get("ssl")
if ssl is not None:
key_path = ssl.get("key_path")
cert_path = ssl.get("cert_path")
if not key_path:
raise Exception("key_path is required for ssl")
if not cert_path:
raise Exception("cert_path is required for ssl")
config.ssl = True
config.ssl_key_path = key_path
config.ssl_cert_path = cert_path
port = 5443
if json_object.get("port"):
port = json_object.get("port")
config.port = port
if json_object.get("auth"):
auth_object = json_object.get("auth")
auth_type = auth_object.get("type")
if not auth_type:
raise Exception("Auth type should be specified")
auth_type = auth_type.strip().lower()
if auth_type == "ldap":
from auth.auth_ldap import LdapAuthorizer
config.authorizer = LdapAuthorizer(auth_object)
else:
raise Exception(auth_type + " auth is not supported")
config.alerts_config = parse_alerts_config(json_object)
return config
def encrypt_file(cipher: str, file_path : str, key: str):
fileBytes = file_utils.read_file(file_path)
# returns chosen cipher class
algo = mux(cipher).Cipher(key)
# Bytes to write to modified file
if algo:
# encrypt zipped folder
print("ENCRYPTING FILE BYTES",'\n')
newBytes = bytes(algo.encrypt(bytes(fileBytes)))
# write encrypted bytes and rename zip
file_utils.write_file(file_path,newBytes)
MERKed_file = file_utils.merk(file_path)
print(f"{file_path} has been MERKed:",MERKed_file)
def _write_post_execution_info(execution_id, log_file_path,
post_execution_info_provider):
exit_code = post_execution_info_provider.get_exit_code(execution_id)
if exit_code is None:
return
file_content = file_utils.read_file(log_file_path)
file_parts = file_content.split(OUTPUT_STARTED_MARKER + '\n', 1)
parameters_text = file_parts[0]
parameters_text += 'exit_code:' + str(exit_code) + '\n'
new_content = parameters_text + OUTPUT_STARTED_MARKER + '\n' + file_parts[
1]
file_utils.write_file(log_file_path, new_content)
def encrypt_folder(cipher: str, folder_path : str, key: str):
# Create new zip and delete old folder
oldFolder, newZip = file_utils.zip_folder(folder_path)
file_utils.delete_folder(oldFolder)
fileBytes = file_utils.read_file(newZip)
# returns chosen cipher class
algo = mux(cipher).Cipher(key)
# Bytes to write to modified file
if algo:
# encrypt zipped folder
newBytes = bytes(algo.encrypt(bytes(fileBytes)))
# write encrypted bytes and rename zip
file_utils.write_file(newZip,newBytes)
MERKed_file = file_utils.merk(newZip)
print(f"{folder_path} has been MERKed:",MERKed_file)
def _path_to_json(self, path):
if path is None:
return None
path = file_utils.normalize_path(path, self._config_folder)
if os.path.exists(path):
try:
file_content = file_utils.read_file(path)
return json.loads(file_content)
except:
LOGGER.exception('Failed to load included file ' + path)
return None
else:
LOGGER.warning('Failed to load included file, path does not exist: ' + path)
return None
def _path_to_json(self, path):
if path is None:
return None
path = file_utils.normalize_path(path, self._config_folder)
if os.path.exists(path):
try:
file_content = file_utils.read_file(path)
return json.loads(file_content)
except:
LOGGER.exception('Failed to load included file ' + path)
return None
else:
LOGGER.warning('Failed to load included file, path does not exist: ' + path)
return None
def visit_configs(visitor):
configs_dir = CONFIGS_FOLDER
files = os.listdir(configs_dir)
configs = [file for file in files if file.lower().endswith(".json")]
result = []
for config_path in configs:
path = os.path.join(configs_dir, config_path)
content = file_utils.read_file(path)
visit_result = visitor(path, content)
if visit_result is not None:
result.append(visit_result)
return result
def _write_post_execution_info(execution_id, log_file_path,
post_execution_info_provider):
exit_code = post_execution_info_provider.get_exit_code(execution_id)
if exit_code is None:
return
file_content = file_utils.read_file(log_file_path, keep_newlines=True)
file_parts = file_content.split(OUTPUT_STARTED_MARKER + os.linesep, 1)
parameters_text = file_parts[0]
parameters_text += 'exit_code:' + str(exit_code) + os.linesep
new_content = parameters_text + OUTPUT_STARTED_MARKER + os.linesep + file_parts[
1]
file_utils.write_file(log_file_path,
new_content.encode(ENCODING),
byte_content=True)
def test_update_script_config(self):
self.start_server(12345, '127.0.0.1')
xsrf_token = self.get_xsrf_token(self._admin_session)
script_path = test_utils.create_file('my_script.py')
test_utils.write_script_config(
{
'name': 's1',
'script_path': script_path
}, 's1', self.runners_folder)
response = self._admin_session.put(
'http://127.0.0.1:12345/admin/scripts',
data={
'filename':
's1.json',
'config':
json.dumps({
'name': 'new name',
'script': {
'mode': 'new_code',
'path': script_path,
'code': 'abcdef'
}
})
},
headers={'X-XSRFToken': xsrf_token},
)
self.assertEqual(200, response.status_code)
conf_response = self.request(
'get', 'http://127.0.0.1:12345/admin/scripts/new%20name',
self._admin_session)
self.assertEqual(
{
'config': {
'name': 'new name',
'script_path': script_path
},
'filename': 's1.json'
}, conf_response)
script_content = file_utils.read_file(script_path)
self.assertEqual('abcdef', script_content)
def get_import_paths(project_path):
imports = set()
class HtmlImportSearcher(HTMLParser):
def handle_starttag(self, tag, attrs):
if tag == 'script':
for attr in attrs:
if attr[0] == 'src':
imports.add(attr[1])
if tag == 'link':
for attr in attrs:
if attr[0] == 'href':
imports.add(attr[1])
web_folder = os.path.join(project_path, 'web')
for file in os.listdir(web_folder):
if not file.endswith('.html'):
continue
file_path = os.path.join(web_folder, file)
parser = HtmlImportSearcher()
parser.feed(file_utils.read_file(file_path))
css_folder = os.path.join(web_folder, 'css')
for file in os.listdir(css_folder):
if not file.endswith('.css'):
continue
file_path = os.path.join(css_folder, file)
fonts_paths = extract_font_urls_from_css(file_path)
for path in fonts_paths:
imports.add(os.path.join('css', path))
lib_paths = []
for import_path in imports:
if '/libs/' in import_path:
lib_path = import_path.replace('/', os.path.sep)
lib_path = os.path.join(web_folder, lib_path)
lib_paths.append(lib_path)
return lib_paths
def __introduce_access_config(context):
conf_folder = os.path.join(context.conf_folder, 'runners')
if not os.path.exists(conf_folder):
return
conf_files = [
os.path.join(conf_folder, file) for file in os.listdir(conf_folder)
if file.lower().endswith('.json')
]
for conf_file in conf_files:
content = file_utils.read_file(conf_file)
json_object = json.loads(content, object_pairs_hook=OrderedDict)
if ('output_files' not in json_object) or ('parameters'
not in json_object):
continue
output_files = json_object['output_files']
parameter_names = [
p['name'] for p in json_object['parameters']
if not is_blank(p.get('name'))
]
changed = False
for i in range(len(output_files)):
output_file = output_files[i]
if not isinstance(output_file, str):
continue
for param_name in parameter_names:
output_file = re.sub('\\$\\$\\$' + param_name,
'${' + param_name + '}', output_file)
if output_file != output_files[i]:
output_files[i] = output_file
changed = True
if changed:
_write_json(conf_file, json_object, content)
def restore_jobs(schedules_folder):
files = [file for file in os.listdir(schedules_folder) if file.endswith('.json')]
job_dict = {}
ids = [] # list of ALL ids, including broken configs
for file in files:
try:
content = file_utils.read_file(os.path.join(schedules_folder, file))
job_json = json.loads(content)
ids.append(job_json['id'])
job = scheduling_job.from_dict(job_json)
job_dict[job.id] = job
except:
LOGGER.exception('Failed to parse schedule file: ' + file)
return job_dict, ids
def __migrate_user_id(context):
output_folder = os.path.join(context.log_folder, 'processes')
if not os.path.exists(output_folder):
return
log_files = [
os.path.join(output_folder, file) for file in os.listdir(output_folder)
if file.lower().endswith('.log')
]
for log_file in log_files:
(correct, parameters_text
) = ExecutionLoggingService._read_parameters_text(log_file)
if not correct:
continue
parameters = ExecutionLoggingService._parse_history_parameters(
parameters_text)
if not parameters or ('user' not in parameters):
continue
if ('user_id' in parameters) and ('user_name' in parameters):
continue
file_content = file_utils.read_file(log_file, keep_newlines=True)
file_parts = file_content.split(
execution.logging.OUTPUT_STARTED_MARKER + os.linesep, 1)
parameters_text = file_parts[0]
user = parameters['user']
if 'user_id' not in parameters:
parameters_text += 'user_id:' + user + os.linesep
if 'user_name' not in parameters:
parameters_text += 'user_name:' + user + os.linesep
new_content = parameters_text + execution.logging.OUTPUT_STARTED_MARKER + os.linesep + file_parts[
1]
file_utils.write_file(log_file,
new_content.encode(execution.logging.ENCODING),
byte_content=True)
def _restore_state(self):
if not os.path.exists(self.dump_file):
LOGGER.info('OAuth dump file is missing. Nothing to restore')
return
dump_data = file_utils.read_file(self.dump_file)
dump_json = json.loads(dump_data)
for user_state in dump_json:
username = user_state.get('username')
if not username:
LOGGER.warning('Missing username in ' + str(user_state))
continue
state = _UserState(username)
self._users[username] = state
state.groups = user_state.get('groups', [])
state.last_auth_update = user_state.get('last_auth_update')
state.last_visit = user_state.get('last_visit')
def test_create_script_config(self):
self.start_server(12345, '127.0.0.1')
xsrf_token = self.get_xsrf_token(self._admin_session)
response = self._admin_session.post(
'http://127.0.0.1:12345/admin/scripts',
data={
'filename':
'whatever',
'config':
json.dumps({
'name': 'test conf',
'script': {
'mode': 'upload_script',
'path': 'whatever'
}
})
},
files={'uploadedScript': ('my.py', b'script content')},
headers={'X-XSRFToken': xsrf_token},
)
self.assertEqual(200, response.status_code)
expected_script_path = os.path.join(test_utils.temp_folder, 'conf',
'scripts', 'my.py')
conf_response = self.request(
'get', 'http://127.0.0.1:12345/admin/scripts/test%20conf',
self._admin_session)
self.assertEqual(
{
'config': {
'name': 'test conf',
'script_path': expected_script_path
},
'filename': 'test_conf.json'
}, conf_response)
script_content = file_utils.read_file(expected_script_path)
self.assertEqual('script content', script_content)
def _load_script_code_by_config(self, plain_config):
script_path = plain_config.get(SCRIPT_PATH_FIELD)
if is_blank(script_path):
raise InvalidFileException('', 'Script path is not specified')
command = process_utils.split_command(
script_path, plain_config.get(WORKING_DIR_FIELD))
binary_files = []
for argument in command:
if file_utils.exists(argument):
if file_utils.is_binary(argument):
binary_files.append(argument)
continue
return {
'code': file_utils.read_file(argument),
'file_path': argument
}
if binary_files:
if len(binary_files) == 1:
return {
'code': None,
'file_path': binary_files[0],
'code_edit_error': 'Cannot edit binary file'
}
raise InvalidFileException(
'command', 'Cannot choose which binary file to edit: ' +
str(binary_files))
if len(command) == 1:
return {
'code': None,
'file_path': command[0],
'code_edit_error': 'Script path does not exist'
}
raise InvalidFileException(
'command',
'Failed to find script path in command "' + script_path + '"')
def test_attach_trace(self):
sys_log_file = open(self.vi.sys_log, "w")
vdisplay = ffm.start_xvfb()
driver, _, proc = ffm.get_browser(self.vi.ff_log)
subprocess.Popen("strace -p %d" % proc.pid,
stderr=sys_log_file, stdout=sys_log_file,
shell=True)
driver.get("https://google.com")
sleep(1)
driver.close()
proc.terminate()
driver.quit()
ffm.stop_xvfb(vdisplay)
sys_log_file.close()
sleep(5) # wait for strace output to be available
sys_logs = fu.read_file(self.vi.sys_log)
for sys_call in SYS_CALLS:
self.assertIn(sys_call, sys_logs,
"""Cannot trace process, make sure you set up
ptrace permissions in /proc/sys/kernel/yama/ptrace_scope as
described in etc/setup.sh.""")
def __introduce_access_config():
file_path = os.path.join('conf', 'conf.json')
if not os.path.exists(file_path):
return
content = file_utils.read_file(file_path)
json_object = json.loads(content)
def move_to_access(field, parent_object):
if 'access' not in json_object:
json_object['access'] = {}
json_object['access'][field] = parent_object[field]
del parent_object[field]
changed = False
if 'auth' in json_object:
auth_object = json_object['auth']
if 'allowed_users' in auth_object:
move_to_access('allowed_users', auth_object)
changed = True
fields = ['admin_users', 'trusted_ips']
for field in fields:
if field in json_object:
changed = True
move_to_access(field, json_object)
if changed:
space_matches = re.findall('^\s+', content, flags=re.MULTILINE)
if space_matches:
indent_string = space_matches[0].replace('\t', ' ')
indent = min(len(indent_string), 8)
else:
indent = 4
with open(file_path, 'w') as fp:
json.dump(json_object, fp, indent=indent)
def visit_script_configs(visitor):
configs_dir = SCRIPT_CONFIGS_FOLDER
files = os.listdir(configs_dir)
configs = [file for file in files if file.lower().endswith(".json")]
result = []
for config_path in configs:
path = os.path.join(configs_dir, config_path)
try:
content = file_utils.read_file(path)
visit_result = visitor(path, content)
if visit_result is not None:
result.append(visit_result)
except:
LOGGER.exception("Couldn't read the file: " + config_path)
return result
def _load_runner_files(conf_folder):
runners_folder = os.path.join(conf_folder, 'runners')
if not os.path.exists(runners_folder):
return
conf_files = [os.path.join(runners_folder, file)
for file in os.listdir(runners_folder)
if file.lower().endswith('.json')]
result = []
for conf_file in conf_files:
content = file_utils.read_file(conf_file)
try:
json_object = custom_json.loads(content, object_pairs_hook=OrderedDict)
result.append((conf_file, json_object, content))
except Exception:
LOGGER.exception('Failed to load file for migration: ' + conf_file)
continue
return result
def test_attach_trace(self):
sys_log_file = open(self.vi.sys_log, "w")
vdisplay = ffm.start_xvfb()
driver, _, proc = ffm.get_browser(self.vi.ff_log)
subprocess.Popen("strace -p %d" % proc.pid,
stderr=sys_log_file,
stdout=sys_log_file,
shell=True)
driver.get("https://google.com")
sleep(1)
driver.close()
proc.terminate()
driver.quit()
ffm.stop_xvfb(vdisplay)
sys_log_file.close()
sleep(5) # wait for strace output to be available
sys_logs = fu.read_file(self.vi.sys_log)
for sys_call in SYS_CALLS:
self.assertIn(
sys_call, sys_logs,
"""Cannot trace process, make sure you set up
ptrace permissions in /proc/sys/kernel/yama/ptrace_scope as
described in etc/setup.sh.""")
def __migrate_user_id(context):
output_folder = os.path.join(context.log_folder, 'processes')
if not os.path.exists(output_folder):
return
log_files = [os.path.join(output_folder, file)
for file in os.listdir(output_folder)
if file.lower().endswith('.log')]
for log_file in log_files:
(correct, parameters_text) = ExecutionLoggingService._read_parameters_text(log_file)
if not correct:
continue
parameters = ExecutionLoggingService._parse_history_parameters(parameters_text)
if not parameters or ('user' not in parameters):
continue
if ('user_id' in parameters) and ('user_name' in parameters):
continue
file_content = file_utils.read_file(log_file, keep_newlines=True)
file_parts = file_content.split(execution.logging.OUTPUT_STARTED_MARKER + os.linesep, 1)
parameters_text = file_parts[0]
user = parameters['user']
if 'user_id' not in parameters:
parameters_text += 'user_id:' + user + os.linesep
if 'user_name' not in parameters:
parameters_text += 'user_name:' + user + os.linesep
new_content = parameters_text + execution.logging.OUTPUT_STARTED_MARKER + os.linesep + file_parts[1]
file_utils.write_file(log_file, new_content.encode(execution.logging.ENCODING), byte_content=True)
break
if parent_path == ROOT_PROJECT_PATH:
break
parent_path = os.path.dirname(parent_path)
new_in_progress = set(pom_paths)
home_folder = os.path.expanduser('~')
unique_name = get_unique_name(ROOT_PROJECT_PATH)
in_progress_file = os.path.join(home_folder, '.incremaven', unique_name)
prev_in_progress = []
if os.path.exists(in_progress_file):
prev_in_progress = file_utils.read_file(in_progress_file).split("\n")
prev_in_progress = filter(lambda line: line != "", prev_in_progress)
for pom_path in prev_in_progress:
if os.path.exists(pom_path):
pom_paths.add(pom_path)
file_utils.write_file(in_progress_file, "\n".join(pom_paths))
projects = common.to_mvn_projects(pom_paths, ROOT_PROJECT_PATH, ROOT_ONLY)
to_rebuild = []
to_install = []
for project in projects:
build_date = mvn_utils.target_build_date(project)
def __migrate_old_files(context):
output_folder = os.path.join(context.log_folder, 'processes')
if not os.path.exists(output_folder):
return
log_files = [os.path.join(output_folder, file)
for file in os.listdir(output_folder)
if file.lower().endswith('.log')]
def is_new_format(log_file):
with open(log_file, 'r') as f:
first_line = f.readline().strip()
if not first_line.startswith('id:'):
return False
for line in f:
if line.strip() == execution.logging.OUTPUT_STARTED_MARKER:
return True
return False
old_files = [log_file for log_file in log_files if not is_new_format(log_file)]
if not old_files:
return
existing_ids = set()
for file in log_files:
correct, parameters_text = ExecutionLoggingService._read_parameters_text(file)
if not correct:
continue
parameters = ExecutionLoggingService._parse_history_parameters(parameters_text)
if not parameters or 'id' not in parameters:
continue
existing_ids.add(parameters['id'])
id_generator = (str(id) for id in itertools.count())
id_generator = filter(lambda id: id not in existing_ids, id_generator)
for old_file in old_files:
log_basename = os.path.basename(old_file)
filename = os.path.splitext(log_basename)[0]
match = re.fullmatch('(.+)_([^_]+)_((\d\d)(\d\d)(\d\d)_(\d\d)(\d\d)(\d\d))', filename)
if match:
script_name = match.group(1)
username = match.group(2)
start_time = datetime.strptime(match.group(3), '%y%m%d_%H%M%S')
id = next(id_generator)
else:
script_name = 'unknown'
username = '******'
start_time = sec_to_datetime(os.path.getctime(old_file))
id = next(id_generator)
new_begin = ''
new_begin += 'id:' + id + '\n'
new_begin += 'user_name:' + username + '\n'
new_begin += 'user_id:' + username + '\n'
new_begin += 'script:' + script_name + '\n'
new_begin += 'start_time:' + str(to_millis(start_time)) + '\n'
new_begin += 'command:unknown' + '\n'
new_begin += execution.logging.OUTPUT_STARTED_MARKER + '\n'
file_content = file_utils.read_file(old_file)
file_content = new_begin + file_content
file_utils.write_file(old_file, file_content)
def read_logs_only(self, log_file):
content = file_utils.read_file(log_file, keep_newlines=True)
self.assertTrue(OUTPUT_STARTED_MARKER in content)
log_start = content.index(OUTPUT_STARTED_MARKER) + len(OUTPUT_STARTED_MARKER) + 1
return content[log_start:]
def _replace_line_separators(files, original, new):
for file in files:
content = file_utils.read_file(file, byte_content=True)
replaced_content = content.decode('utf-8').replace(original, new).encode('utf-8')
file_utils.write_file(file, replaced_content, byte_content=True)
def read_log(self):
if self.file_path and os.path.exists(self.file_path):
return file_utils.read_file(self.file_path, keep_newlines=True)
return None
def create_db_from_schema(db_file, schema_file=cm.DB_SCHEMA):
schema = fu.read_file(schema_file)
with sq.connect(db_file, timeout=cm.DB_CONN_TIMOEUT) as conn:
cursor = conn.cursor()
cursor.executescript(schema)
if not mvn_utils.is_built(unchanged_project):
print('project ' + str(unchanged_project) + ' was cleaned, sending to rebuild')
changed_projects.append(unchanged_project)
continue
mvn_utils.fast_install(unchanged_project, MAVEN_REPO_PATH)
mvn_utils.rebuild(ROOT_PROJECT_PATH, changed_projects, MVN_OPTS, silent=False)
current_revision = vcs_gateway.get_revision(ROOT_PROJECT_PATH)
info_file_path = os.path.join(ROOT_PROJECT_PATH, "_ci_rebuild.info")
if os.path.exists(info_file_path):
last_revision = file_utils.read_file(info_file_path).strip()
if last_revision != current_revision:
try:
incremental_rebuild(last_revision, current_revision)
except mvn_utils.IncorrectConfigException as e:
print('ERROR! {}'.format(e))
sys.exit(-1)
else:
print("Svn revision is the same. Skipping rebuild")
else:
print("No previous revision found, rebuilding the whole root project...")
mvn_utils.rebuild_root(ROOT_PROJECT_PATH, MVN_OPTS, silent=False)
file_utils.write_file(info_file_path, current_revision)
def from_json(conf_path, temp_folder):
if os.path.exists(conf_path):
file_content = file_utils.read_file(conf_path)
else:
file_content = "{}"
config = ServerConfig()
json_object = json.loads(file_content)
address = "0.0.0.0"
port = 5000
ssl = json_object.get("ssl")
if ssl is not None:
key_path = model_helper.read_obligatory(ssl, 'key_path', ' for ssl')
cert_path = model_helper.read_obligatory(ssl, 'cert_path', ' for ssl')
config.ssl = True
config.ssl_key_path = key_path
config.ssl_cert_path = cert_path
port = 5443
if json_object.get("address"):
address = json_object.get("address")
config.address = address
if json_object.get("port"):
port = json_object.get("port")
config.port = port
if json_object.get('title'):
config.title = json_object.get('title')
access_config = json_object.get('access')
if access_config:
allowed_users = access_config.get('allowed_users')
user_groups = model_helper.read_dict(access_config, 'groups')
else:
allowed_users = None
user_groups = {}
auth_config = json_object.get('auth')
if auth_config:
config.authenticator = create_authenticator(auth_config, temp_folder)
auth_type = config.authenticator.auth_type
if auth_type == 'google_oauth' and allowed_users is None:
raise Exception('auth.allowed_users field is mandatory for ' + auth_type)
def_trusted_ips = []
def_admins = []
else:
def_trusted_ips = ['127.0.0.1', '::1']
def_admins = def_trusted_ips
if access_config:
config.trusted_ips = strip(read_list(access_config, 'trusted_ips', default=def_trusted_ips))
admin_users = _parse_admin_users(access_config, default_admins=def_admins)
else:
config.trusted_ips = def_trusted_ips
admin_users = def_admins
config.allowed_users = _prepare_allowed_users(allowed_users, admin_users, user_groups)
config.alerts_config = json_object.get('alerts')
config.callbacks_config = json_object.get('callbacks')
config.logging_config = parse_logging_config(json_object)
config.user_groups = user_groups
config.admin_users = admin_users
config.max_request_size_mb = read_int_from_config('max_request_size', json_object, default=10)
return config
def test_write_to_file(self):
filename = self.new_temp_file('write_test.txt')
random_str = ut.rand_str(100)
fu.write_to_file(filename, random_str)
self.assertEqual(random_str, fu.read_file(filename))
def test_read_file(self):
file_content = fu.read_file(os.path.realpath(__file__))
if 'whatever written here' not in file_content:
self.fail('Cannot read itself')
def _load_groups(self, groups_file):
if not os.path.exists(groups_file):
return {}
content = file_utils.read_file(groups_file)
return json.loads(content)
