def postgres(schema, tables, data_directory, psql_path, **params): psql = local.get('psql', psql_path) data_directory = Path(data_directory) logger.info('Initializing PostgreSQL...') engine = init_database( 'postgresql', params, schema, isolation_level='AUTOCOMMIT' ) query = "COPY {} FROM STDIN WITH (FORMAT CSV, HEADER TRUE, DELIMITER ',')" database = params['database'] for table in tables: src = data_directory / '{}.csv'.format(table) load = psql[ '--host', params['host'], '--port', params['port'], '--username', params['user'], '--dbname', database, '--command', query.format(table), ] with local.env(PGPASSWORD=params['password']): with src.open('r') as f: load(stdin=f) engine.execute('VACUUM FULL ANALYZE')
def get_gpus(): gpu_count = local["lspci"]().count( "VGA" ) #BEWARE this doesn't reflect true gpu count remember the ASPEED device in lspci gpu_array = [] if gpu_count == 1: return [], [], [] for gpu_num in range(0, gpu_count - 1): gpu_fields = {} arg_arr = [ '-i', str(gpu_num), '--query-gpu=gpu_name,gpu_bus_id,vbios_version,serial,memory.total', '--format=csv,noheader,nounits' ] gpu_info = local.get("nvidia-smi").run(arg_arr)[1].encode("ascii") gpu_arr = gpu_info.strip().split(", ") gpu_fields["Name"] = gpu_arr[0] gpu_fields['PCIID'] = gpu_arr[1] gpu_fields['Bios Version'] = gpu_arr[2] gpu_fields['Serial'] = gpu_arr[3] gpu_fields['Memory'] = gpu_arr[4] gpu_array.append(gpu_fields) return { "GPUs": gpu_array }, gpu_array, { "Name": "GPUs", "children": add_tree_specific_fields(gpu_array, 'gpu') }
def test_get(self): assert str(local['ls']) == str(local.get('ls')) assert str(local['ls']) == str(local.get('non_exist1N9', 'ls')) with pytest.raises(CommandNotFound): local.get("non_exist1N9") with pytest.raises(CommandNotFound): local.get("non_exist1N9", "non_exist1N8") with pytest.raises(CommandNotFound): local.get("non_exist1N9", "/tmp/non_exist1N8")
def __init__(self, fzf_path=None): ''' A systemError will be raised if fzf is not found. ''' try: if fzf_path: self.fzf = local.get(fzf_path) else: self.fzf = local['fzf'] except plumbum.CommandNotFound: raise SystemError("Cannot find 'fzf' ( {0} )".format(FZF_URL))
def run(self): from plumbum import local, FG with local.cwd("rust"): try: cargo = local.get('cargo') rustc = local.get('rustc') except Exception as e: print("Installing Rust...") rustup = local['curl']['--proto']['=https']['--tlsv1.2'][ '-sSf']['https://sh.rustup.rs'] sh = local['sh'] if os.isatty(sys.stdout.fileno()): (rustup | sh) & FG else: (rustup | sh['-s']['--']['-y'])() cargo = local.get( 'cargo', os.environ.get('HOME', 'root') + '/.cargo/bin/cargo') finally: if debug: cargo["build"]["--features"][ "bindings cli null-plugins"] & FG else: cargo["build"]["--release"]["--features"][ "bindings cli null-plugins"] & FG local['mkdir']("-p", py_target_dir) sys.path.append("python/tools") import add_swig_directives add_swig_directives.run(include_dir + "/dqcsim-py.h", py_target_dir + "/dqcsim.i") local["swig"]["-v"]["-python"]["-py3"]["-outdir"][py_target_dir]["-o"][ py_target_dir + "/dqcsim.c"][py_target_dir + "/dqcsim.i"] & FG _build.run(self)
def postgres(schema, tables, data_directory, psql_path, **params): psql = local.get('psql', psql_path) data_directory = Path(data_directory) logger.info('Initializing PostgreSQL...') engine = init_database('postgresql', params, schema, isolation_level='AUTOCOMMIT') query = "COPY {} FROM STDIN WITH (FORMAT CSV, HEADER TRUE, DELIMITER ',')" database = params['database'] for table in tables: src = data_directory / '{}.csv'.format(table) load = psql['--host', params['host'], '--port', params['port'], '--username', params['user'], '--dbname', database, '--command', query.format(table)] with local.env(PGPASSWORD=params['password']): with src.open('r') as f: load(stdin=f) engine.execute('VACUUM FULL ANALYZE')
def main(self, *svc_names): logs = [ self.parent.logs_dir / svc.name / 'current' for svc in self.parent.svc_map(svc_names or self.parent.svcs) ] if self.debug: logs.append(self.parent.logs_dir / '.s6-svscan' / 'current') if self.follow: with suppress(KeyboardInterrupt): try: mtail = local.get('lnav', 'multitail') except CommandNotFound: tail[['-n', self.lines, '-F'] + logs].run_fg() else: mtail[logs].run_fg() else: for log in logs: if log.is_file(): tail['-vn', self.lines, log].run_fg() print('\n')
def test_get(self): self.assertEqual(str(local['ls']),str(local.get('ls'))) self.assertEqual(str(local['ls']),str(local.get('non_exist1N9', 'ls'))) self.assertRaises(CommandNotFound, lambda: local.get("non_exist1N9")) self.assertRaises(CommandNotFound, lambda: local.get("non_exist1N9", "non_exist1N8")) self.assertRaises(CommandNotFound, lambda: local.get("non_exist1N9", "/tmp/non_exist1N8"))
DATA = { "py3": True, "make_secret": lambda: sha1(os.urandom(48)).hexdigest(), "myvar": "awesome", "what": "world", "project_name": "Copier", "version": "2.0.0", "description": "A library for rendering projects templates", } COPIER_CMD = local.get( # Allow debugging in VSCode # HACK https://github.com/microsoft/vscode-python/issues/14222 str(Path(sys.executable).parent / "copier.cmd"), str(Path(sys.executable).parent / "copier"), # Poetry installs the executable as copier.cmd in Windows "copier.cmd", "copier", ) # Executing copier this way allows to debug subprocesses using debugpy # See https://github.com/microsoft/debugpy/issues/596#issuecomment-824643237 COPIER_PATH = (sys.executable, "-m", "copier") # Helpers to use with tests designed for old copier bracket envops defaults BRACKET_ENVOPS = { "autoescape": False, "block_end_string": "%]", "block_start_string": "[%", "comment_end_string": "#]",
DYN_O_MITE_DEFAULTS = dict( secure_server_option='datacenter', pem_key_file='conf/dynomite.pem', data_store=0, datastore_connections=1, ) INTERNODE_LISTEN = 8101 CLIENT_LISTEN = 8102 REDIS_PORT = 1212 STATS_PORT = 22222 BASE_IPADDRESS = quad2int('127.0.1.1') RING_SIZE = 2**32 SETTLE_TIME = 5 redis = local.get('./test/_binaries/redis-server', 'redis-server') with local.cwd('src'): # Removed './test/_binaries/dynomite' dynomite = local.get('./dynomite') @contextmanager def launch_redis(ip): logfile = 'logs/redis_{}.log'.format(ip) f = (redis['--requirePass', 'testpass', '--masterauth', 'testpass', '--bind', ip, '--port', REDIS_PORT] > logfile) & BG(-9) try: #print("Redis future: {}".format(str(f))) #print("Redis started, pid: {}".format(f.proc.pid)) yield RedisNode(ip, REDIS_PORT) finally:
DYN_O_MITE_DEFAULTS = dict( secure_server_option='datacenter', pem_key_file='conf/dynomite.pem', data_store=0, datastore_connections=1, ) INTERNODE_LISTEN = 8101 CLIENT_LISTEN = 8102 REDIS_PORT = 1212 STATS_PORT = 22222 BASE_IPADDRESS = quad2int('127.0.1.1') RING_SIZE = 2**32 SETTLE_TIME = 5 redis = local.get('./test/_binaries/redis-server', 'redis-server') with local.cwd('src'): # Removed './test/_binaries/dynomite' #dynomite = local.get('./dynomite') @contextmanager def launch_redis(ip): logfile = 'logs/redis_{}.log'.format(ip) f = (redis['--requirePass', 'testpass', '--masterauth', 'testpass', '--bind', ip, '--port', REDIS_PORT] > logfile) & BG(-9) try: yield RedisNode(ip, REDIS_PORT) finally: f.proc.kill() f.wait() def pick_tokens(count, start_offset):
# pansearch_full_path = "{{ pansearch_full_path }}" pansearch_full_path="/var/log" catalogs_list = pansearch_full_path.split(' ') #working_dir="/wrk/pansearch" working_dir="./" mask_finded_results = False scanning_only_gz = False #"{{ PANSEARCH_SCANNING_ONLY_GZ }}" unzip_before = False send_email_with_results = True write_pan_to_log = False write_cvv_to_log = True current_hostname = local.get("hostname")() stoplist = ["pareq","pares","audispd","panHash","INT_REF","</MD>","a3=","irn"] cc_validator = """4[0-9]{12}([0-9]{3})|[25][1-7][0-9]{14}|6(011|5[0-9][0-9])[0-9]{12}|3[47][0-9]{13}|3(0[0-5]|[68][0-9])[0-9]{11}|(2131|1800|35\d{3})\d{11}""" scan_failed = { 1: "not running at this time", 2: "completed"} scan_status = 0 overall_before_status_lock = join(working_dir, "pansearch.lock") log_file = join(working_dir, "pansearch_scanning.log") timestamp_file = join(working_dir, "pansearch_timestamp.conf") files_to_current_scan = join(working_dir, "pansearch_files_to_scan.db") pansearch_files_list = join(working_dir, "pansearch.db") log_file_gz = join(working_dir, "pansearch_scanning.log.gz")
def haup(name): local.get('haup')[name] & FG
def hadown(name): local.get('hadown')[name] & FG
INTERFACE64='1', BINARY='64'), Job(TOOLCHAIN='mingw', TARGET='NEHALEM', ARCH='x86', INTERFACE64='0', BINARY='32'), Job(TOOLCHAIN='mingw', TARGET='HASWELL', ARCH='x86-64', INTERFACE64='1', BINARY='64') ] with local.cwd(local.cwd / 'OpenBLAS'): for job in jobs: with local.env(PATH=(mingw64 if job.ARCH == 'x86-64' else mingw32) + ';C:/msys64/usr/bin/'): make = local.get('make') make['TARGET={0}'.format(job.TARGET), 'USE_THREAD=0', 'NO_LAPACK=1', 'NO_LAPACKE=1', 'ONLY_CBLAS=1', 'INTERFACE64={0}'.format(job.INTERFACE64), 'BINARY={0}'.format(job.BINARY)] & FG make['PREFIX=../opt/{0}/{1}/{2}/{3}'. format(platform.system().lower(), job.TOOLCHAIN, job. TARGET, job.ARCH), 'install'] & FG make['clean'] & FG