def install(vpn_opts: ClientOpts, svc_opts: UnixServiceOpts, auto_startup: bool, auto_dnsmasq: bool, dnsmasq: bool, auto_connman_dhcp: bool, force: bool): executor = VPNClientExecutor(vpn_opts).probe(log_lvl=logger.INFO) dns_resolver = executor.device.dns_resolver if not dnsmasq and not dns_resolver.is_connman(): logger.error('Only support dnsmasq as DNS resolver in first version') sys.exit(ErrorCode.NOT_YET_SUPPORTED) if executor.is_installed(silent=True): if force: logger.warn( 'VPN service is already installed. Try to remove then reinstall...' ) executor.do_uninstall(keep_vpn=False, keep_dnsmasq=True) else: logger.error('VPN service is already installed') sys.exit(ErrorCode.VPN_ALREADY_INSTALLED) if dnsmasq and not dns_resolver.is_dnsmasq_available( ) and not dns_resolver.is_connman(): executor.device.install_dnsmasq(auto_dnsmasq) logger.info( f'Installing VPN client into [{vpn_opts.vpn_dir}] and register service[{svc_opts.service_name}]...' ) executor.do_install(svc_opts, auto_startup, auto_connman_dhcp) logger.done()
def detail(vpn_opts: ClientOpts, accounts): if accounts is None or len(accounts) == 0: logger.error('Must provide at least account') sys.exit(ErrorCode.INVALID_ARGUMENT) VPNClientExecutor(vpn_opts, adhoc_task=True).require_install().probe().exec_command( 'AccountGet', params=accounts, log_lvl=logger.INFO)
def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except Warning as w: logger.warning(str(w), exc_info=True) except Exception as e: logger.error(str(e), exc_info=True)
def __with_bit_depth(audio_segment, bit_depth): """Return a version of <audio_segment> with updated bit depth.""" if bit_depth % 8 != 0: log.error( "Bit depth should be a multiple of 8, used value here is {0}". format(bit_depth)) return audio_segment.set_sample_width(bit_depth // 8)
def generate_sequence(oui: list, seq: list, quantity: int): length = len(oui) + len(seq) if length < 6: logger.error( f'Combination between OUI and next MAC sequence is invalid [{oui + seq}]' ) sys.exit(10) mac = oui + (seq if length == 6 else seq[len(oui):6]) return (increase(mac, -1) for _ in range(quantity))
def query_data(self): """Retrieve data points from database""" try: self.check_db() qr = QueryBuilder(measurement=self._measurement) rs = InfluxService.db_client.query(query=qr.generate_query()) return list(rs.get_points()) except Exception as e: app_logger.error(str(e))
def delete(vpn_opts: ClientOpts, accounts): logger.info( f'Delete VPN account [{accounts}] and stop/disable VPN service if it\'s a current VPN connection...' ) if accounts is None or len(accounts) == 0: logger.error('Must provide at least account') sys.exit(ErrorCode.INVALID_ARGUMENT) VPNClientExecutor(vpn_opts).require_install().probe( log_lvl=logger.INFO).do_delete(accounts) logger.done()
def parseYamlFile(absoluteFilePath): with open(absoluteFilePath) as f: try: inputConfig = yaml.load(f) except yaml.YAMLError as e: logger.error(str(e)) sys.exit(1) logger.debug("Parsed config:\n{}".format(json.dumps(inputConfig))) return inputConfig
def write_data(): _dict = jsn.load() if not _dict: log.error("{0} is empty".format(tml.value('json', section='data', subkey='fname'))) data = [] for key in _dict: data.append(list(map(__convert, map(_read, (_dict[key], key)), repeat(DTYPE)))) log.debug("{0} couples data/label have been retrieved".format(len(data))) npz.write(np.asarray(data))
def _path_to_best_model(dpath): """Return path to best model amongst models in <dpath>.""" fpaths = pth.__list_files(dpath) if not fpaths: log.error("\"{0}\" does not contain any file or is not a directory".format(dpath)) return get_vloss = lambda fpath: float(pth.__no_extension(fpath).split('-')[-1]) vlosses = list(map(get_vloss, fpaths)) bmdl_idx = vlosses.index(min(vlosses)) return fpaths[bmdl_idx]
def pre_exec(self, silent=False, log_lvl=logger.DEBUG, **kwargs): logger.log(log_lvl, 'Start VPN Client if not yet running...') if not self.is_installed(silent, log_lvl): return if self.pid_handler.is_running(): self._prev_is_run = True return SystemHelper.exec_command(f'{self.opts.vpnclient} start', log_lvl=logger.down_lvl(log_lvl)) time.sleep(1) if not self.pid_handler.is_running(log_lvl=logger.down_lvl(log_lvl)): logger.error('Unable start VPN Client') sys.exit(ErrorCode.VPN_START_FAILED)
def get_command(self, ctx, cmd_name): name = cmd_name.encode( 'ascii', 'replace') if sys.version_info[0] == 2 else cmd_name try: module = self.commands.get(name) if module is None: logger.error(f'Unsupported command "{name}"') click.echo(click.get_current_context().get_help()) sys.exit(10) return __import__(to_module([module, 'cmd_' + name]), None, None, ['cli']).cli except ImportError as err: logger.error("Load command failed {}::{}".format(name, str(err))) sys.exit(10)
def __validate(file): keys, n, d = {}, 0, 0 for row in file: n += 1 keys[row] = keys[row] + [n] if row in keys else [n] for k, v in keys.items(): if len(v) > 1: logger.warn(f'Duplicated key: {k.strip()} in lines {v}') d += 1 if d == 0: logger.success('No duplication') sys.exit(0) logger.error(f'Duplicated {d} keys') sys.exit(20)
def loop_interval(condition: Callable[[], bool], error_if_timeout: str, pre_func: Callable[[], NoReturn] = lambda: None, max_retries: int = 5, interval: float = 1, throttle=0, exit_if_error=False): for c in range(max_retries + 1): pre_func() time.sleep(throttle) if condition(): return time.sleep(interval) msg = f'{error_if_timeout} after {max_retries * interval}(s)' if exit_if_error: logger.error(msg) sys.exit(ErrorCode.TIMEOUT) else: raise TimeoutError(msg)
def __copy(nic, asix1, asix2, oui, byte_fmt, sep): check = len(list(filter(lambda x: x, [asix1, asix2, oui]))) if check > 1: logger.error( 'Option [asix1, asix2, oui] is mutually exclusive with each another' ) sys.exit(2) if check == 0: logger.error('Provide at least one option [asix1, asix2, oui]') sys.exit(2) try: nic_mac = netifaces.ifaddresses(nic)[netifaces.AF_LINK][0]['addr'] except: logger.error(f'Not found NIC {nic} or MAC address of NIC {nic}') sys.exit(10) in_oui = asix1 or asix2 or oui oui = [int(c, base=16) for c in in_oui.split(sep) ] if type(in_oui) == str else (in_oui or []) mac = [int(c, base=16) for c in nic_mac.split(':') ] if type(nic_mac) == str else (nic_mac or []) mac = oui + mac[len(oui):6] if len(mac) != 6: logger.error( f'Combination between OUI and MAC is invalid [{in_oui},{nic_mac}]') sys.exit(10) logger.success(sep.join(byte_fmt % b for b in mac))
def check_db(): """Check if database exists""" # get all databases all_dbs_list = InfluxService.db_client.get_list_database() # check if current database exists and if return warning message if InfluxService.cnf.INFLUX_DB not in [ str(x['name']) for x in all_dbs_list ]: try: app_logger.warning("Database {0} does not exist".format( InfluxService.cnf.INFLUX_DB)) except exceptions.InfluxDBClientError as e: app_logger.error(str(e)) except exceptions.InfluxDBServerError as e1: app_logger.error(str(e1)) else: try: app_logger.info("Using db {0}".format( InfluxService.cnf.INFLUX_DB)) InfluxService.db_client.switch_database( InfluxService.cnf.INFLUX_DB) except exceptions.InfluxDBClientError as e: app_logger.error(str(e)) except exceptions.InfluxDBServerError as e1: app_logger.error(str(e1))
def install_dnsmasq(self, auto_install: bool = False): if not auto_install: logger.error( 'dnsmasq is not yet installed. Please install [dnsmasq] depends on your distro' ) sys.exit(ErrorCode.MISSING_REQUIREMENT) logger.info('Try to install [dnsmasq]...') pm = self.pm if not pm: logger.error( 'Unknown package manager. Please install [dnsmasq] by yourself' ) sys.exit(ErrorCode.MISSING_REQUIREMENT) pm.install('dnsmasq') self._dns_resolver(self.dns_resolver.probe())
def do_connect(self, account: str, log_lvl: int = logger.INFO): if not account: logger.error(f'VPN account is not correct') sys.exit(ErrorCode.INVALID_ARGUMENT) acc = self.storage.find(account) if not acc: logger.error(f'Not found VPN account') sys.exit(ErrorCode.VPN_ACCOUNT_NOT_FOUND) logger.log(log_lvl, f'Connect VPN account [{account}]...') self.storage.create_or_update(acc, _connect=True) self.exec_command(['AccountConnect'], params=account) self.lease_vpn_service(is_enable=acc.is_default, is_restart=acc.is_default, is_lease_ip=not acc.is_default, account=acc.account)
def __pcm2float(npy_array, _type='float64'): """Convert <npy_array> from pcm to float. Default conversion type is 'float64'. """ if npy_array.dtype.kind != 'i': log.error( "\'__pcm2float\' takes an array of integers, forcing conversion to int16" ) npy_array = npy_array.astype('int16') info = np.iinfo(npy_array.dtype) amp = 2**(info.bits - 1) offset = info.min + amp npy_array = (npy_array - offset) / amp return npy_array.clip(-1., 1.).astype(_type)
def _handle_other_errors(self, failure, invoker): """ If execution reaches this point, we're probably dealing with an actual code issue. We'll print the traceback out for the user, then fall through to the logging error handler. """ invoker.emit_to( 'ERROR: A critical error has occurred. Please notify the staff.' ) invoker.emit_to(failure.getTraceback()) logger.error('Command handler encountered an error') logger.error('Invoker: %s' % invoker.get_appearance_name(None, force_admin_view=True)) # Fall through to the last (and final) logging error handler so the # console can get a copy of the traceback. failure.trap()
def shape(data): """Shape <data> to fit input of neural network.""" log.debug("Shaping data") if data.ndim != 2: log.error("\'shape\' expects a two-dimensional array : (n_samples, sample_len)") return data _dtype = 'int{0}'.format(tml.value('bit_depth', section='audio')) if data.dtype != _dtype: log.warning("\'shape\' expects an {0} array".format(_dtype)) data = data.astype('float64') for i in range(data.shape[0]): data[i] = __pcm2float(data[i].astype(_dtype)) return data.reshape(*data.shape, 1)
def unshape(data): """Unshape <data> output of neural network.""" log.debug("Unshaping data") if data.ndim != 3: log.error("\'unshape\' expects a three-dimensional array : (n_samples, sample_len, n_channels)") return data data = data.reshape(*data.shape[:-1]) if data.dtype.kind != 'f': log.warning("\'unshape\' expects a float array") data = data.astype('float64') for i in range(data.shape[0]): data[i] = __float2pcm(data[i]) return data.astype('int{0}'.format(tml.value('bit_depth', section='audio')))
def setup(self, vpn_service: str, origin_resolv_conf: Path, vpn_resolv_conf: Path, vpn_nameserver_hook_conf: Path): if not self._available: logger.error('[dnsmasq] is not yet installed or is corrupted') sys.exit(ErrorCode.MISSING_REQUIREMENT) logger.info('Setup DNS resolver[dnsmasq]...') dnsmasq_vpn_cfg = self._dnsmasq_vpn_cfg(vpn_service) runtime_resolv_cfg = self.adapt_dnsmasq(origin_resolv_conf, vpn_service) dnsmasq_opts = { '{{DNS_RESOLVED_FILE}}': self.__build_dnsmasq_conf('resolv-file', runtime_resolv_cfg), '{{PORT}}': self.__build_dnsmasq_conf('port', self.dnsmasq_options().get('port', None)), '{{CACHE_SIZE}}': self.__build_dnsmasq_conf( 'cache-size', self.dnsmasq_options().get('cache_size', None)) } logger.debug( f'Add [dnsmasq] config for {vpn_service}[{dnsmasq_vpn_cfg}]...') FileHelper.copy(self.resource_dir.joinpath(self.DNSMASQ_CONFIG_TMPL), dnsmasq_vpn_cfg, force=True) FileHelper.replace_in_file(dnsmasq_vpn_cfg, dnsmasq_opts, backup='') FileHelper.chmod(dnsmasq_vpn_cfg, mode=0o0644) logger.debug( f'Symlink [dnsmasq] VPN nameserver runtime configuration [{vpn_nameserver_hook_conf}]...' ) FileHelper.create_symlink(vpn_nameserver_hook_conf, self._dnsmasq_vpn_hook_cfg, force=True) logger.info(f'Generate System DNS config file from VPN service...') FileHelper.write_file(vpn_resolv_conf, self.__dnsmasq_resolv(vpn_service), mode=0o0644) FileHelper.create_symlink(vpn_resolv_conf, DNSResolver.DNS_SYSTEM_FILE, force=True) self.service.enable(self.config.identity)
def create_config(self, vpn_service: str, auto_connman_dhcp: bool): if self.is_connman(): FileHelper.write_file(self.connman_dhcp, str(auto_connman_dhcp)) return if not FileHelper.is_readable(self.origin_resolv_cfg): logger.info( f'Backup System DNS config file to [{self.origin_resolv_cfg}]...' ) FileHelper.backup(DNSResolver.DNS_SYSTEM_FILE, self.origin_resolv_cfg, remove=False) if not FileHelper.is_readable(self.origin_resolv_cfg): logger.error( f'Not found origin DNS config file [{self.origin_resolv_cfg}]') sys.exit(ErrorCode.FILE_CORRUPTED) if not FileHelper.is_readable(self.vpn_hook_cfg): FileHelper.touch(self.vpn_hook_cfg, 0o0644) self._resolver().setup(vpn_service, self.origin_resolv_cfg, self.vpn_resolv_cfg, self.vpn_hook_cfg) self._resolver().restart(_all=True)
def __generate(quantity, seq, rand, output, overwrite, asix1, asix2, oui, uaa, multicast, byte_fmt, sep): check = len(list(filter(lambda x: x, [asix1, asix2, oui]))) if check > 1: logger.error( 'Option [asix1, asix2, oui] is mutually exclusive with each another' ) sys.exit(2) if seq and rand: logger.error( 'Option [seq, rand] is mutually exclusive with each another') sys.exit(2) oui = asix1 or asix2 or oui oui = [int(c, base=16) for c in oui.split(sep)] if type(oui) == str else (oui or []) seq = [0 for _ in range(5 - len(oui))] + [-1] if seq is None else [ int(c, base=16) for c in seq.split(sep) ] gen = generate_random(oui, quantity, uaa, multicast) if rand else generate_sequence( oui, seq, quantity) out((sep.join(byte_fmt % b for b in each) for each in gen), overwrite, output)
def exec_command(command: str, shell=False, silent=False, log_lvl=logger.DEBUG) -> str: logger.decrease(log_lvl, "Execute command: %s", command) list_cmd = command.split(" | ") if not shell else [command] length = len(list_cmd) prev = None for idx, cmd in enumerate(list_cmd, 1): logger.trace("\tsub_command::%s::%s", cmd, prev) kwargs = {} if EnvHelper.is_py3_5() else {"encoding": "utf-8"} complete = subprocess.run(cmd.split() if not shell else cmd, input=prev, env=TWEAK_ENV, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) ret = complete.returncode lvl = (logger.TRACE if idx < length else logger.DEBUG) if ret == 0 or silent else logger.ERROR try: prev = SystemHelper.__handle_command_result( complete, silent, lvl) except RuntimeError as _: if not silent: logger.error('Failed when executing command %s', cmd) sys.exit(ret) finally: ret_val = ("0. Actual: %s" % ret) if silent and ret != 0 else ret logger.decrease(log_lvl, "%sReturn code: %s", "\t" if idx < length else "", ret_val) if prev: logger.log(log_lvl, "\t%s", prev) return prev
def parseOptions(argv): # Intialize defaults inputFile = None try: options, args = getopt.getopt(argv, "hi:", ["inputFile="]) except getopt.GetoptError as e: logger.error(str(e)) sys.exit(1) for option, arg in options: if option == "-h": print(getCliHelpText()) sys.exit() elif option in ("-i", "--inputFile"): inputFile = arg if not inputFile: raise Exception('-i/--inputFile is required') logger.debug("Input file: {}".format(inputFile)) return (inputFile)
def mse(x, y): len_x, len_y = x.shape[0], y.shape[0] if len_x == len_y: return sum((x - y)**2) / len_x log.error("Can't compute MSE, arrays have different lengths: {0} and {1}".format(len_x, len_y))
def wrapper(*args, **kwargs): if os.getuid() != 0: logger.error("You need root privileges to run this script") sys.exit(100) return func(*args, **kwargs)
import sys, json, MySQLdb from src.utils import logger, getCliHelpText, parseOptions, parseYamlFile from src.data_gen import DataGen if __name__ == "__main__": try: inputFile = parseOptions(sys.argv[1:]) except Exception as e: logger.error(str(e)) sys.exit(1) inputConfig = parseYamlFile(inputFile) if inputConfig["engine"] == "mysql": from src.schema_builders.mysql import MysqlSchemaBuilder as SchemaBuilder from src.writers.mysql import MysqlWriter as Writer from src.contexts.mysql import MysqlContext as Context conn = MySQLdb.connect( inputConfig["host"], inputConfig["user"], inputConfig["password"], inputConfig["database"], inputConfig["port"] ) ctx = Context(conn, inputConfig) else: logger.error("Engine - {} not supported".format(inputConfig["engine"])) sys.exit(1)
def main(): if args.download: # Download all files downloader = Downloader() all_committees = downloader.committees_urls_list() logger.info("Starting to process %i committees" % len(all_committees)) for idx, committee_url in enumerate([ c for c in all_committees ]): # if 'foreign-affairs-committee' in c]): downloader.capture_committee_documents(committee_url) logger.info('Completed committee %i / %i' % (idx, len(all_committees))) pass if args.parse: # Parse all HTML files, store to JSON, and summarise in XLSX html_filenames = glob.glob( os.path.expanduser('~/projects_data/parliament-text/*.html')) df = pd.DataFrame(columns=[ 'members', 'witnesses', 'speakers_dict', 'Q&A', 'plain_text' ], index=[]) for i, html_filename in enumerate(html_filenames[100:200]): with open(html_filename, 'r') as f: html_text = f.read() logger.info('%i / %i: %s' % (i, len(html_filenames), html_filename)) trscrpt = transcript(html_text, { 'status': 'read html from directory', 'html_file_location': f.name }, html_filename=html_filename) trscrpt.process_raw_html() key_data, hyperlink = trscrpt.key_data_summary() df.loc[hyperlink] = key_data xlsx_filename = 'summary.xlsx' key_data_to_xlsx(df, xlsx_filename) if False: # Parse selected files key_html_documents = [ '45429.html', '74822.html' # 'http://data.parliament.uk/writtenevidence/committeeevidence.svc/evidencedocument/treasury-committee/monetary-policy-forward-guidance/oral/3245.html' ] for d in key_html_documents: with open( os.path.join( os.path.expanduser('~/projects_data/parliament-text'), d), 'r') as f: html_text = f.read() trscrpt = transcript(html_text, {'status': 'debugging case'}, html_filename=d) trscrpt.process_raw_html() if args.diagnostic: # generate diagnostic spreadsheet summarising success of parsing all_summaries = [] json_filenames = glob.glob( os.path.expanduser('~/projects_data/parliament-text/*.json')) # df_analysis = pd.DataFrame(columns=['members','witnesses', # 'speakers_dict','Q&A','plain_text'], # index=[]) for i, json_filename in enumerate(json_filenames): with open(json_filename, 'r') as f: json_text = f.read() logger.info('%i / %i: %s' % (i, len(json_filenames), json_filename)) transcript_summary = analyse_json(json_text) all_summaries.append(transcript_summary) # key_data, hyperlink = trscrpt.key_data_summary() # df.loc[hyperlink] = key_data df_all_speaker_stats = pd.concat(all_summaries, ignore_index=True) speaker_data_to_xlsx(df_all_speaker_stats, 'speaker_summary.xlsx') elif args.analyse: # a quick chart to show how we might use the data df_all_speaker_stats = pd.read_excel( 'speaker_summary.xlsx', sheet_name='speaker_stats', # skiprows=3, header=0) df_all_speaker_stats['title'] = df_all_speaker_stats[ 'witness_name'].str.extract( '(MP|Professor|Rt Hon|Lord QC|Dr |Brigadier|Commodore|Colonel|General|Marshal|Sir|Reverend|Rev\.|Rt Rev)', expand=False).str.strip() df_all_speaker_stats.loc[df_all_speaker_stats.title.isna(), 'title'] = 'other' df_all_speaker_stats.title = df_all_speaker_stats.title.str.replace( re.compile('(Rev\.|Rt Rev)'), 'Reverend') df_all_speaker_stats.title = df_all_speaker_stats.title.str.replace( re.compile('(Brigadier|Commodore|Colonel|General|Marshal)'), '[military]') df_all_speaker_stats.title = pd.Categorical(df_all_speaker_stats.title) df_all_speaker_stats.title.value_counts() df_speakers_agg = df_all_speaker_stats[[ 'title', 'gunning_fog' ]].groupby(['title']).agg(['mean', 'count']).reset_index() df_speakers_agg.columns = [ '_'.join(col).strip() for col in df_speakers_agg.columns.values ] df_speakers_agg.to_csv('test.csv') import matplotlib.pyplot as plt import seaborn as sns ax = sns.barplot( x="syllable_count_mean", y="witness_name_", data=df_speakers_agg[df_speakers_agg['syllable_count_count'] > 10]) fig = ax.get_figure() fig.savefig('./img/barplot.png') ax2 = sns.barplot(x="title", y="gunning_fog", data=df_all_speaker_stats) plt.ylim(15, 18) fig = ax2.get_figure() fig.savefig('./img/barplot_fog_by_title.png') else: logger.error('No options selected')