def check_text_files(obtained_fn, expected_fn, fix_callback=lambda x: x, encoding=None): """ Compare two files contents. If the files differ, show the diff and write a nice HTML diff file into the data directory. :param Path obtained_fn: path to obtained file during current testing. :param Path expected_fn: path to the expected file, obtained from previous testing. :param str encoding: encoding used to open the files. :param callable fix_callback: A callback to "fix" the contents of the obtained (first) file. This callback receives a list of strings (lines) and must also return a list of lines, changed as needed. The resulting lines will be used to compare with the contents of expected_fn. """ __tracebackhide__ = True obtained_fn = Path(obtained_fn) expected_fn = Path(expected_fn) obtained_lines = fix_callback(obtained_fn.read_text(encoding=encoding).splitlines()) expected_lines = expected_fn.read_text(encoding=encoding).splitlines() if obtained_lines != expected_lines: diff_lines = list( difflib.unified_diff(expected_lines, obtained_lines, lineterm="") ) if len(diff_lines) <= 500: html_fn = obtained_fn.with_suffix(".diff.html") try: differ = difflib.HtmlDiff() html_diff = differ.make_file( fromlines=expected_lines, fromdesc=expected_fn, tolines=obtained_lines, todesc=obtained_fn, ) except Exception as e: html_fn = "(failed to generate html diff: %s)" % e else: html_fn.write_text(html_diff, encoding="UTF-8") diff = ["FILES DIFFER:", str(expected_fn), str(obtained_fn)] diff += ["HTML DIFF: %s" % html_fn] diff += diff_lines raise AssertionError("\n".join(diff)) else: # difflib has exponential scaling and for thousands of lines it starts to take minutes to render # the HTML diff. msg = [ "Files are different, but diff is too big ({} lines)".format( len(diff_lines) ), "- obtained: {}".format(obtained_fn), "- expected: {}".format(expected_fn), ] raise AssertionError("\n".join(msg))
def check_text_files(obtained_fn, expected_fn, fix_callback=lambda x: x, encoding=None): """ Compare two files contents. If the files differ, show the diff and write a nice HTML diff file into the data directory. :param Path obtained_fn: path to obtained file during current testing. :param Path expected_fn: path to the expected file, obtained from previous testing. :param str encoding: encoding used to open the files. :param callable fix_callback: A callback to "fix" the contents of the obtained (first) file. This callback receives a list of strings (lines) and must also return a list of lines, changed as needed. The resulting lines will be used to compare with the contents of expected_fn. """ __tracebackhide__ = True obtained_fn = Path(obtained_fn) expected_fn = Path(expected_fn) obtained_lines = fix_callback(obtained_fn.read_text(encoding=encoding).splitlines()) expected_lines = expected_fn.read_text(encoding=encoding).splitlines() if obtained_lines != expected_lines: diff_lines = list(difflib.unified_diff(expected_lines, obtained_lines)) if len(diff_lines) <= 500: html_fn = obtained_fn.with_suffix(".diff.html") try: differ = difflib.HtmlDiff() html_diff = differ.make_file( fromlines=expected_lines, fromdesc=expected_fn, tolines=obtained_lines, todesc=obtained_fn, ) except Exception as e: html_fn = "(failed to generate html diff: %s)" % e else: html_fn.write_text(html_diff, encoding="UTF-8") diff = ["FILES DIFFER:", str(expected_fn), str(obtained_fn)] diff += ["HTML DIFF: %s" % html_fn] diff += diff_lines raise AssertionError("\n".join(diff)) else: # difflib has exponential scaling and for thousands of lines it starts to take minutes to render # the HTML diff. msg = [ "Files are different, but diff is too big (%s lines)" % (len(diff_lines),), "- obtained: %s" % (obtained_fn,), "- expected: %s" % (expected_fn,), ] raise AssertionError("\n".join(msg))
def test_create_stack_with_changeset(self, patched_create_change_set, patched_update_term): """Test create_stack, force changeset, termination protection.""" stack_name = 'fake_stack' template_path = Path('./tests/cfngin/fixtures/cfn_template.yaml') template = Template(body=template_path.read_text()) parameters = [] tags = [] changeset_id = 'CHANGESETID' patched_create_change_set.return_value = ([], changeset_id) self.stubber.add_response('execute_change_set', {}, {'ChangeSetName': changeset_id}) with self.stubber: self.provider.create_stack(stack_name, template, parameters, tags, force_change_set=True, termination_protection=True) self.stubber.assert_no_pending_responses() patched_create_change_set.assert_called_once_with( self.provider.cloudformation, stack_name, template, parameters, tags, 'CREATE', service_role=self.provider.service_role) patched_update_term.assert_called_once_with(stack_name, True)
def handle(cls, name, args): # type: (str, Dict[str, Any]) -> None """Perform the actual test. Relies on .cfnlintrc file to be located beside the Runway config file. """ cfnlintrc = Path('./.cfnlintrc') if not cfnlintrc.is_file(): LOGGER.error('File must exist to use this test: %s', cfnlintrc) sys.exit(1) # prevent duplicate log messages by not passing to the root logger logging.getLogger('cfnlint').propagate = False try: with argv(*['cfn-lint'] + args.get('cli_args', [])): runpy.run_module('cfnlint', run_name='__main__') except SystemExit as err: # this call will always result in SystemExit if err.code != 0: # ignore zero exit codes but re-raise for non-zero if not (yaml.safe_load(cfnlintrc.read_text()) or {}).get('templates'): LOGGER.warning('cfnlintrc is missing a "templates" ' 'section which is required by cfn-lint') raise
def convert_reorientparams_save_to_mat_script(): from pathlib2 import Path import shutil shutil.copy('/computation/convert_to_mat_file_template.m', '/computation/convert_to_mat_file.m') path = Path('/computation/convert_to_mat_file.m') text = path.read_text() text = text.replace('x_mm', str(template_dict['reorient_params_x_mm'])) text = text.replace('y_mm', str(template_dict['reorient_params_y_mm'])) text = text.replace('z_mm', str(template_dict['reorient_params_z_mm'])) text = text.replace('pitch', str(template_dict['reorient_params_pitch'])) text = text.replace('roll', str(template_dict['reorient_params_roll'])) text = text.replace('yaw', str(template_dict['reorient_params_yaw'])) text = text.replace('x_scaling', str(template_dict['reorient_params_x_scaling'])) text = text.replace('y_scaling', str(template_dict['reorient_params_y_scaling'])) text = text.replace('z_scaling', str(template_dict['reorient_params_z_scaling'])) text = text.replace('x_affine', str(template_dict['reorient_params_x_affine'])) text = text.replace('y_affine', str(template_dict['reorient_params_y_affine'])) text = text.replace('z_affine', str(template_dict['reorient_params_z_affine'])) path.write_text(text) # Run reorient.m script using spm12 standalone and Matlab MCR with stdchannel_redirected(sys.stderr, os.devnull): spm.SPMCommand.set_mlab_paths( matlab_cmd= '/opt/spm12/run_spm12.sh /opt/mcr/v95 script /computation/convert_to_mat_file.m', use_mcr=True)
def _build_wp(self): try: path = '{}{}/'.format(self.APACHE['path'], self._name) print(t.bold_yellow('> getting wordpress')) download(self.WP_PATH, self._name) print(t.bold_yellow('> uncompressing')) ZipFile(self._name, 'r').extractall() remove(self._name) print(t.bold_yellow('> setting up site')) move('wordpress', path) chown(path, 'www-data', 'www-data') rename('{}wp-config-sample.php'.format(path), '{}wp-config.php'.format(path)) path = Path('{}wp-config.php'.format(path)) text = path.read_text() text = text.replace('database_name_here', self._name) text = text.replace('username_here', self._name) text = text.replace('password_here', self._psw) text = text.replace('put your unique phrase here', self._random_str(50)) path.write_text(text) except Exception as e: self._rollback() print( t.bold_red( 'Error trying to build wordpress context, rolling back database changes...' ), e) exit(3)
def test_copy_or_create(conf_dir): src_file = Path(conf_dir, "temp.x.in") dst_file = Path(conf_dir, "temp.x.out") # file doesn't exist, check file created msi_update.copy_or_create(src_file, dst_file, u"!!!") assert dst_file.exists() content = dst_file.read_text() assert content == "!!!" # files exists check file copied src_file.write_text(u"+++") msi_update.copy_or_create(src_file, dst_file, u"!!!") assert dst_file.exists() content = dst_file.read_text() assert content == "+++"
def generate_ssh_key_instance(self, path, storage): """Generate ssh key from file.""" private_key_path = Path(path) instance = SshKey(private_key=private_key_path.read_text(), label=private_key_path.name) self.validate_ssh_key(instance, storage) return instance
class SNMPtrapd(object): def __init__(self, port): self.snmptrapd_proc = None self.port = port self.snmptrapd_log = Path(tc_parameters.WORKING_DIR, "snmptrapd_log") self.snmptrapd_stdout_path = Path(tc_parameters.WORKING_DIR, "snmptrapd_stdout") self.snmptrapd_stderr_path = Path(tc_parameters.WORKING_DIR, "snmptrapd_stderr") def wait_for_snmptrapd_log_creation(self): return self.snmptrapd_log.exists() def wait_for_snmptrapd_startup(self): return "NET-SNMP version" in self.snmptrapd_log.read_text() def start(self): if self.snmptrapd_proc is not None: return self.snmptrapd_proc = ProcessExecutor().start( [ "snmptrapd", "-f", "--disableAuthorization=yes", "-C", "-On", "--doNotLogTraps=no", self.port, "-LF", "6-6", os.path.relpath(str(self.snmptrapd_log)), ], self.snmptrapd_stdout_path, self.snmptrapd_stderr_path, ) wait_until_true(self.wait_for_snmptrapd_log_creation) wait_until_true(self.wait_for_snmptrapd_startup) return self.snmptrapd_proc.is_running() def stop(self): if self.snmptrapd_proc is None: return self.snmptrapd_proc.terminate() try: self.snmptrapd_proc.wait(4) except TimeoutExpired: self.snmptrapd_proc.kill() self.snmptrapd_proc = None def get_port(self): return self.port def get_logs(self): file_reader = DestinationReader(FileIO) return file_reader.read_all_logs(self.snmptrapd_log)
def generate_ssh_key_instance(self, path, storage): """Generate ssh key from file.""" private_key_path = Path(path) instance = SshKey( private_key=private_key_path.read_text(), label=private_key_path.name ) self.validate_ssh_key(instance, storage) return instance
def update_model_desc(self, new_model_desc_file=None): """ Change the task's model_desc """ execution = self._get_task_property('execution') p = Path(new_model_desc_file) if not p.is_file(): raise IOError('mode_desc file %s cannot be found' % new_model_desc_file) new_model_desc = p.read_text() model_desc_key = list(execution.model_desc.keys())[0] if execution.model_desc else 'design' execution.model_desc[model_desc_key] = new_model_desc res = self._edit(execution=execution) return res.response
def remove_federation_block(remote): server_conf_path = "/opt/spire/conf/server/server.conf" copy_file_from_remote(remote, server_conf_path) server_conf = Path(server_conf_path) server_conf_content = server_conf.read_text() start_index = server_conf_content.find("\n\n federation") federation_limit = "}\n }\n }\n\n" end_index = server_conf_content.find(federation_limit, start_index) + len(federation_limit) federation_block = server_conf_content[start_index:end_index] server_conf_content = server_conf_content.replace(federation_block, "") server_conf.write_text(server_conf_content) send_file_to_remote(remote, server_conf_path)
def convert_and_run_reorient_script(input_file): from pathlib2 import Path import shutil from nipype.interfaces import spm shutil.copy('/computation/reorient_template.m', '/computation/reorient.m') path = Path('/computation/reorient.m') text = path.read_text() text = text.replace('input_file', input_file) path.write_text(text) # Run convert_to_mat_file.m script using spm12 standalone and Matlab MCR with stdchannel_redirected(sys.stderr, os.devnull): spm.SPMCommand.set_mlab_paths(matlab_cmd='/opt/spm12/run_spm12.sh /opt/mcr/v95 script /computation/reorient_job.m', use_mcr=True)
def _get_modules(context, filter_path=None, filter_module=None, filter_changes=True): if filter_module is None: working_modules = [] filter_changes = filter_changes if FAB_SKIP_DELTA not in os.environ else False working_dirs = _run_local( context, "pwd", hide='out').stdout.encode("utf8").strip().split('/') if working_dirs[-1] == "asystem": for filtered_module in filter( lambda module_tmp: isdir(module_tmp) and (not filter_changes or _run_local(context, "git status --porcelain {}". format(module_tmp), DIR_ROOT, hide='out').stdout), glob.glob('*/*')): working_modules.append(filtered_module) else: root_dir_index = working_dirs.index("asystem") if (root_dir_index + 2) < len(working_dirs): working_modules.append(working_dirs[root_dir_index + 1] + "/" + working_dirs[root_dir_index + 2]) else: for nested_modules in filter( lambda module_tmp: isdir(module_tmp), glob.glob('*')): working_modules.append("{}/{}".format( working_dirs[root_dir_index + 1], nested_modules)) working_modules[:] = [ module for module in working_modules if filter_path is None or glob.glob("{}/{}/{}*".format(DIR_ROOT, module, filter_path)) ] grouped_modules = {} for module in working_modules: group_path = Path(join(DIR_ROOT, module, ".group")) group = group_path.read_text().strip() if group_path.exists( ) else "ZZZZZ" if group not in grouped_modules: grouped_modules[group] = [module] else: grouped_modules[group].append(module) sorted_modules = [] for group in sorted(grouped_modules): sorted_modules.extend(grouped_modules[group]) return sorted_modules else: return [filter_module] if filter_path is None or os.path.exists( join(DIR_ROOT, filter_module, filter_path)) else []
class SerializedEntity: # pragma: no cover -- TODO requires functional tests. def __init__(self, filename, current_version, entity_name='data', unlink=False, readable=False): self.filename = Path(filename) self.version = current_version self.entity_name = entity_name self.unlink = unlink self.readable = readable self.entity = None def load(self): if not self.filename.exists(): return None data = self.filename.read_text() savedata = jsonpickle.decode(data, keys=True) if savedata['version'] > self.version: raise RuntimeError( "Stored data version {0} is newer than currently supported {1}" .format(savedata['version'], self.version)) self.entity = savedata[self.entity_name] if self.unlink: os.unlink(str(self.filename)) return self.entity def reset(self, new_value=None): self.entity = new_value def save(self, entity=None): if entity is None: entity = self.entity if entity is None: return savedata = {'version': self.version, self.entity_name: entity} if self.readable: data = jsonpickle.encode(savedata, indent=2, keys=True) else: data = jsonpickle.encode(savedata, keys=True) self.filename.write_bytes(data.encode('utf-8', 'replace')) def __enter__(self): self.load() return self def __exit__(self, *args, **kwargs): self.save()
def DebuggerSwitch(prod_mode: bool): file_data = [] for filename in os.listdir("./cogs"): if filename.endswith(".py"): file_directory = os.path.join("./cogs", filename) path = Path(file_directory) text = path.read_text() true_check = re.findall("prod_mode = True", text) false_check = re.findall("prod_mode = False", text) if true_check: text = text.replace("prod_mode = True", "prod_mode = " + str(prod_mode)) elif false_check: text = text.replace("prod_mode = False", "prod_mode = " + str(prod_mode)) path.write_text(text)
def decrypt(self, input_file): slogverify_stdout = Path(tc_parameters.WORKING_DIR, "slogverify_stdout") slogverify_stderr = Path(tc_parameters.WORKING_DIR, "slogverify_stderr") encrypted = Path(tc_parameters.WORKING_DIR, input_file) decrypted = Path(tc_parameters.WORKING_DIR, "decrypted.txt") CommandExecutor().run( [ self.slogverify, self.decryption_key, encrypted, self.cmac, decrypted ], slogverify_stdout, slogverify_stderr, ) return decrypted.read_text().rstrip("\n").split("\n")
def add_federation_block(trust_domain, bundle_endpoint, remote): federation_path = "/mnt/c-spiffe/integration_test/resources/federation.conf" federation_config_content = Path(federation_path).read_text() if federation_config_content.find(trust_domain) == -1: federation_config_content = update_federation_block(trust_domain, bundle_endpoint) server_conf_path = "/opt/spire/conf/server/server.conf" copy_file_from_remote(remote, server_conf_path) server_conf = Path(server_conf_path) server_conf_content = server_conf.read_text() start_index = server_conf_content.find("server {") end_index = server_conf_content.find("}", start_index)-1 current_value = server_conf_content[start_index:end_index] new_value = current_value + "\n\n" + federation_config_content + "\n" server_conf_content = server_conf_content.replace(current_value, new_value) server_conf.write_text(server_conf_content) send_file_to_remote(remote, server_conf_path)
def read_config(cls, filename=None): # pragma: no cover -- TODO """ Reads configuration from JSON file. Default location is XDG_DATA_PATH/taskwarrior/config.json If file does not exist, returns default configuration values. For possible configuration fields see documentation of Config class. """ filename = Path(filename or xdg.save_data_path('taskwarrior') / 'config.json') data = {} if filename.exists(): data = json.loads(filename.read_text()) taskfile = data.get('taskfile') if taskfile: taskfile = os.path.expanduser(taskfile) return cls( taskfile=taskfile, separator=data.get('separator'), stop_alias=data.get('stop_alias'), resume_alias=data.get('resume_alias'), )
def main(with_noreg): """Test automatic scanning / registering""" entry_point_map = manager.get_entry_map(groups='reentry_test', ep_names=['test-plugin', 'test-noreg', 'builtin']) data_file = Path(get_datafile()) assert entry_point_map, 'The \'reentry_test\' entry point group was not found\nMap:\n{}\n\nData File: {}\n\nContents:\n{}'.format( manager.get_entry_map(), str(data_file), data_file.read_text()) try: test_entry_point = entry_point_map['reentry_test']['test-plugin'] builtin_entry_point = entry_point_map['reentry_test']['builtin'] if with_noreg: # note: `reentry scan` for this work noreg_entry_point = entry_point_map['reentry_test']['test-noreg'] except Exception as err: print('datafile: {}'.format(str(data_file))) print('\nCurrent relevant entry point map:\n\n') print(manager.format_map(entry_point_map)) print('\n') scan_map = manager.scan(groups=['reentry_test'], commit=False) print('\nFull entry point map after scan:\n\n') print(manager.format_map(scan_map)) raise err plugin_class = test_entry_point.load() builtin_class = builtin_entry_point.load() assert plugin_class.test_string == 'TEST', 'The test string was incorrect' assert builtin_class.test_string == 'TEST', 'The test string was incorrect' if with_noreg: noreg_class = noreg_entry_point.load() assert noreg_class.test_string == 'TEST', 'The test string was incorrect' plugin_list = [ep.load() for ep in manager.iter_entry_points('reentry_test')] assert plugin_class in plugin_list, 'iter_entry_points found differing test entry point from get_entry_map.' assert builtin_class in plugin_list, 'iter_entry_points found differing test entry point from get_entry_map.' if with_noreg: assert noreg_class in plugin_list, 'iter_entry_points found differing test entry point from get_entry_map.'
def content_accessed(self): if not self._cache_accessed: self.access() file = Path(self.path_accessed) self._cache_accessed = file.read_text() return self._cache_accessed
def content_summarized(self): from place.post import PostBase file = Path(self.path_summarized) json_str = file.read_text() return PostBase(json_str)
def loads(file: p, *args, **kwargs): if safe: # If safe mode is enabled return safe_load(file.read_text(), Loader=Loader) else: # Or it isn't return load(file.read_text(), Loader=Loader)
import sendText from pathlib2 import Path import time import os f = open('config.txt', 'w') f.write("+17202357772\n") f.write("destgps:(50,50)\n") f.close() initialise = True while (True): my_file = Path("/home/linaro/Documents/TextMessage") if my_file.is_file(): content = my_file.read_text() os.remove(str(my_file)) print("sending: ", content) sendText.text(str(content)) time.sleep(10)
class Reader: """References Reader Pleasant interface for .csv reading. """ def __init__(self, file_path: str): """Constructor :param file_path: path to the csv file """ self._file_path = file_path self._file = Path(file_path) if not self._file.exists(): raise FileNotFoundError self._columns = self._rows = -1 self._load_stats() def _load_stats(self) -> None: """Load stats of the .csv file for future usage """ self._rows = 0 for line in self._file.read_text(Csv.encoding).split('\n'): if self._columns == -1: self._columns = len(line.split(Csv.separator)) self._rows += 1 def read_content( self, skip_header: Optional[bool] = False, encoding: Optional[str] = Csv.encoding ) -> Iterator[List[str]]: """Read .csv content :param skip_header: :param encoding: :return: an iterator with each line as a list of its columns """ is_header_skipped = False for line in self._file.read_text(encoding).split('\n'): # skipping header if needed if skip_header and not is_header_skipped: is_header_skipped = True continue # removing empty lines if not line: continue # from a row, removing trailing \n in all fields values = [field for field, _ in re.findall( Parsing.parse_regex, line )][:-1] # if the data is surrounded by '"', remove them yield list(map( lambda field: field[1:-1] if field.startswith(Csv.delimiter) and field.endswith(Csv.delimiter) else field, values )) @property def columns(self) -> int: """Getter for `_columns` :return: the number of columns in the file """ return self._columns @property def rows(self) -> int: """Getter for `_rows` :return: the number of rows in the file """ return self._rows
else: K_Force = 4 / (beta * (dE_wind + 0.4 * dE_wind)**2) #print(i,K_Force) K_s.append(K_Force) ## creation of a file that will be used in the creation of a Potential Mean Force (PMF) profile # Handle errors while calling os.remove() with open("pmf_data.txt", "a+") as test_ks: test_ks.write("{:.4f} {:.4f} \n".format(K_Force * 0.000446253514585234, i)) shutil.copy(template, "{}/CV{:.3f}US".format(new_direc_cv, i)) path = Path("{}/CV{:.3f}US".format(new_direc_cv, i)) text = path.read_text() text = text.replace("topoledit", "./{}".format(topol_file)) text = text.replace("coordfiledit", "CV{:.3f}.pdb".format(i)) text = text.replace("nametoedit", "CV{:.3f}".format(i)) text = text.replace("cvvaltoedit", "{}".format(i)) text = text.replace("constantktoedit", "{:.4f}".format(K_Force * 0.000446253514585234)) path.write_text(text) #write job to execution with open("CV{:.3f}/job".format(i), "w") as job: job.write('''#!/bin/bash #SBATCH --time=160:00:00 #SBATCH --nodes=1 #SBATCH --ntasks-per-node=8 #SBATCH --cpus-per-task=2 #SBATCH --mem=32GB
class VersionUpdater(object): """ Version number synchronisation interface. Updates the version information in * setup.json * aiida_vasp/__init__.py to the current version number. The current version number is either parsed from the output of ``git describe --tags --match v*.*.*``, or if the command fails for any reason, from setup.json. The current version number is decided on init, syncronization can be executed by calling ``.sync()``. """ version_pat = re.compile(r'\d+.\d+.\d+(-(alpha|beta|rc)(.\d+){0,3}){0,1}') init_version_pat = re.compile(r'(__version__ = )([\'"])(.*?)([\'"])', re.DOTALL | re.MULTILINE) replace_tmpl = r'\1\g<2>{}\4' def __init__(self): """Initialize with documents that should be kept up to date and actual version.""" self.top_level_init = Path(subpath('reentry', '__init__.py')) self.setup_json = Path(subpath('setup.json')) self.version = self.get_version() def write_to_init(self): init_content = self.top_level_init.read_text() self.top_level_init.write_text( re.sub(self.init_version_pat, self.new_version_str, init_content, re.DOTALL | re.MULTILINE)) def write_to_setup(self): """Write the updated version number to setup.json.""" with open(str(self.setup_json), 'r') as setup_fo: # preserve order setup = json.load(setup_fo, object_pairs_hook=collections.OrderedDict) setup['version'] = str(self.version) with open(str(self.setup_json), 'w') as setup_fo: json.dump(setup, setup_fo, indent=4, separators=(',', ': ')) @property def new_version_str(self): return self.replace_tmpl.format(str(self.version)) @property def setup_version(self): """Grab the parsed version from setup.json.""" with open(str(self.setup_json), 'r') as setup_fo: setup = json.load(setup_fo) try: version_string = setup['version'] except KeyError: raise AttributeError('No version found in setup.json') return version.parse(version_string) @property def init_version(self): """Grab the parsed version from the init file.""" match = re.search(self.init_version_pat, self.top_level_init.read_text()) if not match: raise AttributeError( 'No __version__ found in top-level __init__.py') return version.parse(match.groups()[2]) @property def tag_version(self): """Get the current version number from ``git describe``, fall back to setup.json.""" try: describe_byte_string = subprocess.check_output( ['git', 'describe', '--tags', '--match', 'v*.*.*']) match = re.search(self.version_pat, describe_byte_string.decode(encoding='UTF-8')) version_string = match.string[match.pos:match.end()] return version.parse(version_string) except subprocess.CalledProcessError: return self.setup_version def get_version(self): return max(self.setup_version, self.init_version, self.tag_version) def sync(self): if self.version > self.init_version: self.write_to_init() if self.version > self.setup_version: self.write_to_setup()
try: from pathlib2 import Path except ImportError: from pathlib import Path from clckwrkbdgr import xdg from clckwrkbdgr import commands import clckwrkbdgr.jobsequence.context trace = context = clckwrkbdgr.jobsequence.context.init( verbose_var='DOTFILES_SETUP_VERBOSE', skip_platforms='Windows', ) if not commands.has_sudo_rights(): trace.info('Have not sudo rights, skipping.') context.done() etc_config_file = Path('/etc/X11/app-defaults/XXkb') if not etc_config_file.exists(): context.die('{0} is not found, cannot add XXkb settings.'.format(etc_config_file)) # TODO can we create this file if it is absent? Xresources = xdg.XDG_CONFIG_HOME/'Xresources' local_XXkb = set([line for line in Xresources.read_text().splitlines() if line.startswith('XXkb.')]) missing = local_XXkb - set(etc_config_file.read_text().splitlines()) if not missing: context.done() trace.error('These XXkb config lines are not present in {0}:'.format(etc_config_file)) for line in missing: print(line) sys.exit(1) # TODO way to fix automatically with sudo.
class LocalConfig(object): def __init__(self): # self.lk_config_dir_string = str(Path.home()) + '/' + '.config/lk' self.lk_config_dir_string = ConfigUtil().lk_config_dir self.lk_config_dir_path = Path(self.lk_config_dir_string) # self.lk_config_file_path_string = self.lk_config_dir_string + '/' + 'lk_config.yaml' self.lk_config_file_path_string = ConfigUtil().user_lk_config_file_path self.lk_config_file_path = Path(self.lk_config_file_path_string) @property def not_found(self): if not self.lk_config_file_path.exists(): return True else: return False @property def config_dict(self): config_string = self.lk_config_file_path.read_text() config_dict = yaml.load(config_string) return config_dict def add_remote_repo(self, remote_repo): self.create_config(remote_repo=remote_repo) def default_commands_repo(self): try: return self.config_dict['default_commands_repo'] except IOError: self.create_config() return self.config_dict['default_commands_repo'] # raise LocalConfigNotFound def create_config(self, remote_repo=None): if not self.lk_config_dir_path.exists(): Path.mkdir(self.lk_config_dir_path, parents=True, exist_ok=True) if self.lk_config_file_path.exists(): print('# Config file exists at: {config_file_path}'.format( config_file_path=self.lk_config_file_path_string)) if app_confirm('Do you want to override?'): pass else: click.Abort() if remote_repo is None: remote_repo_value = app_config.DEFAULT_COMMANDS_REPO else: remote_repo_value = remote_repo config_odict = odict([('default_commands_repo', remote_repo_value)]) setup_yaml() config_yaml_string = yaml.dump(config_odict, default_flow_style=False) self.lk_config_file_path.write_text(config_yaml_string.decode('utf-8'))
def cleandata(file_name): from pathlib2 import Path path = Path(file_name) text = path.read_text() text = text.replace(',', ' ') path.write_text(text)
def _release(context): modules = _get_modules(context, "docker-compose.yml") for module in modules: if FAB_SKIP_TESTS not in os.environ: _pull(context, filter_module=module) _build(context, filter_module=module) _unittest(context, filter_module=module) _systest(context, filter_module=module) _get_versions_next_release() if FAB_SKIP_GIT not in os.environ: print("Tagging repository ...") _run_local( context, "git add -A && git commit -m 'Update asystem-{}' && git tag -a {} -m 'Release asystem-{}'" .format(_get_versions()[0], _get_versions()[0], _get_versions()[0]), env={"HOME": os.environ["HOME"]}) for module in modules: for host in _get_hosts(context, module): _clean(context, filter_module=module) _pull(context, filter_module=module, filter_host=host, is_release=True) _build(context, filter_module=module, is_release=True) _package(context, filter_module=module) _print_header(module, "release") group_path = Path(join(DIR_ROOT, module, ".group")) if group_path.exists() and group_path.read_text().strip( ).isnumeric() and int(group_path.read_text().strip()) >= 0: _run_local(context, "mkdir -p target/release", module) _run_local(context, "cp -rvfp docker-compose.yml target/release", module, hide='err', warn=True) if isfile(join(DIR_ROOT, module, "Dockerfile")): file_image = "{}-{}.tar.gz".format(_name(module), _get_versions()[0]) print("docker -> target/release/{}".format(file_image)) _run_local( context, "docker image save -o {} {}:{}".format( file_image, _name(module), _get_versions()[0]), join(module, "target/release")) if glob.glob( join(DIR_ROOT, module, "target/package/main/resources/*")): _run_local( context, "cp -rvfp target/package/main/resources/* target/release", module) else: _run_local(context, "mkdir -p target/release/config", module) hosts = set( filter( len, _run_local( context, "find {} -type d ! -name '.*' -mindepth 1 -maxdepth 1" .format(DIR_ROOT), hide='out').stdout.replace( DIR_ROOT + "/", "").replace("_", "\n").split("\n"))) Path( join(DIR_ROOT, module, "target/release/hosts")).write_text("\n".join(hosts) + "\n") if glob.glob(join(DIR_ROOT, module, "target/package/run*")): _run_local(context, "cp -rvfp target/package/run* target/release", module) else: _run_local(context, "touch target/release/run.sh", module) _print_header("{}/{}".format(host, _name(module)), "release") ssh_pass = _ssh_pass(context, host) install = "{}/{}/{}".format(DIR_INSTALL, module, _get_versions()[0]) print("Copying release to {} ... ".format(host)) _run_local( context, "{}ssh -q root@{} 'rm -rf {} && mkdir -p {}'".format( ssh_pass, host, install, install)) _run_local( context, "{}scp -qpr $(find target/release -maxdepth 1 -type f) root@{}:{}" .format(ssh_pass, host, install), module) _run_local( context, "{}scp -qpr target/release/config root@{}:{}".format( ssh_pass, host, install), module) print("Installing release to {} ... ".format(host)) _run_local( context, "{}ssh -q root@{} 'chmod +x {}/run.sh && {}/run.sh'". format(ssh_pass, host, install, install)) _run_local( context, "{}ssh -q root@{} 'docker system prune --volumes -f'". format(ssh_pass, host), hide='err', warn=True) _run_local( context, "{}ssh -q root@{} 'find $(dirname {}) -maxdepth 1 -mindepth 1 2>/dev/null | sort | " "head -n $(($(find $(dirname {}) -maxdepth 1 -mindepth 1 2>/dev/null | wc -l) - 2)) | xargs rm -rf'" .format(ssh_pass, host, install, install), hide='err', warn=True) _print_footer("{}/{}".format(host, _name(module)), "release") else: print("Module ignored") _print_footer(module, "release") _get_versions_next_snapshot() if FAB_SKIP_GIT not in os.environ: print("Pushing repository ...") _run_local( context, "git add -A && git commit -m 'Update asystem-{}' && git push --all && git push origin --tags" .format(_get_versions()[0], _get_versions()[0], _get_versions()[0]), env={"HOME": os.environ["HOME"]})
class SNMPtrapd(object): TRAP_LOG_PREFIX = 'LIGHT_TEST_SNMP_TRAP_RECEIVED:' def __init__(self, port): self.snmptrapd_proc = None self.port = port self.snmptrapd_log = Path(tc_parameters.WORKING_DIR, "snmptrapd_log") self.snmptrapd_stdout_path = Path(tc_parameters.WORKING_DIR, "snmptrapd_stdout") self.snmptrapd_stderr_path = Path(tc_parameters.WORKING_DIR, "snmptrapd_stderr") def wait_for_snmptrapd_log_creation(self): return self.snmptrapd_log.exists() def wait_for_snmptrapd_startup(self): return "NET-SNMP version" in self.snmptrapd_log.read_text() def start(self): if self.snmptrapd_proc is not None: return self.snmptrapd_proc = ProcessExecutor().start( [ "snmptrapd", "-f", "--disableAuthorization=yes", "-C", "-m ALL", "-A", "-Ddump", "-On", "--doNotLogTraps=no", "--authCommunity=log public", self.port, "-d", "-Lf", os.path.relpath(str(self.snmptrapd_log)), "-F", "{}%v\n".format(self.TRAP_LOG_PREFIX), ], self.snmptrapd_stdout_path, self.snmptrapd_stderr_path, ) wait_until_true(self.wait_for_snmptrapd_log_creation) wait_until_true(self.wait_for_snmptrapd_startup) return self.snmptrapd_proc.is_running() def stop(self): if self.snmptrapd_proc is None: return self.snmptrapd_proc.terminate() try: self.snmptrapd_proc.wait(4) except TimeoutExpired: self.snmptrapd_proc.kill() self.snmptrapd_proc = None def get_port(self): return self.port def get_traps(self): file_reader = DestinationReader(FileIO) logs = file_reader.read_all_logs(self.snmptrapd_log) trap_list = [] for log_line in logs: res = re.match('({})(.*)'.format(self.TRAP_LOG_PREFIX), log_line) if (res): trap_list += res.group(2).rstrip().split("\t") return sorted(trap_list) def get_raw_traps(self): file_reader = DestinationReader(FileIO) return file_reader.read_all_logs(self.snmptrapd_log)