def df_to_output(args, df): # set up write options config_dict = config_lib.get_config() header = False if 'noheader' in args.output_options else True index = True if 'index' in args.output_options else False # define valid output types valid_outputs = [ 'table', 'csv', 'json', 'html', ] # get the output type output_type_set = set(args.output_options).intersection(set(valid_outputs)) if output_type_set: output_type = list(output_type_set)[0] else: output_type = config_dict['io_output_type'] # set up a mapping between output type and writer function writer_for = { 'csv': csv_writer, 'table': table_writer, 'html': html_writer, 'json': json_writer, } na_rep = get_nan_rep(args, config_dict) # call writer in try block to gracefully handle closed pipes try: writer_for[output_type](df, header, index, na_rep) except IOError: pass
def df_to_output(args, df): # set up write options config_dict = config_lib.get_config() header = False if 'noheader' in args.output_options else True index = True if 'index' in args.output_options else False # define valid output types valid_outputs = [ 'table', 'csv', 'html', ] # get the output type output_type_set = set(args.output_options).intersection(set(valid_outputs)) if output_type_set: output_type = list(output_type_set)[0] else: output_type = config_dict['io_output_type'] # set up a mapping between output type and writer function writer_for = { 'csv': csv_writer, 'table': table_writer, 'html': html_writer, } na_rep = get_nan_rep(args, config_dict) # call writer in try block to gracefully handle closed pipes try: writer_for[output_type](df, header, index, na_rep) except IOError: pass
def df_from_input(args, in_file=None): # set up proper state for reading input config_dict = config_lib.get_config() in_file = sys.stdin if in_file is None else in_file sep = get_separator(args, config_dict) header, names = get_header_names(args) # read the input data try: df = pd.read_csv(in_file, sep=sep, header=header, names=names, encoding=ENCODING, low_memory=False) except ValueError: sys.stderr.write('\n{} received no input\n'.format( os.path.basename(sys.argv[0]))) sys.exit(1) # if no names and no neader, create column names if ('noheader' in args.input_options) and (not names): names = ['c{}'.format(nn) for nn, cc in enumerate(df.columns)] df.columns = pd.Index(names) return df
def _df_from_input_csv(args, in_file=None): # set up proper state for reading input config_dict = config_lib.get_config() in_file = sys.stdin if in_file is None else in_file sep = get_separator(args, config_dict) header, names = get_header_names(args) # read the input data try: # for some reason tsv doesn't allow for low_memory option if sep == r'\t': df = pd.read_csv(in_file, sep=sep, header=header, names=names, encoding=ENCODING, engine='python') else: df = pd.read_csv(in_file, sep=sep, header=header, names=names, encoding=ENCODING, low_memory=False) except ValueError: sys.stderr.write('\n{} received no input\n'.format( os.path.basename(sys.argv[0]))) sys.exit(1) # if no names and no neader, create column names if ('noheader' in args.input_options) and (not names): names = ['c{}'.format(nn) for nn, cc in enumerate(df.columns)] df.columns = pd.Index(names) return df
def check_for_modules(module_list): # make sure module_list only contains recognized modules unnamed_modules = set(module_list) - set(CMD_DICT.keys()) unnamed_modules = unnamed_modules - {'datetime', 're'} if unnamed_modules: msg = '\n\nThese modules unrecognized by check_for_modules(): ' msg += '{}\n'.format(unnamed_modules) raise ValueError(msg) # try using configured backend ignoring errors so they'll be caught later if set(module_list).intersection({'matplotlib', 'pylab', 'seaborn'}): CONFIG = config_lib.get_config() try: import matplotlib if matplotlib.get_backend() != CONFIG['plot_backend']: matplotlib.use(CONFIG['plot_backend']) except ImportError: pass # initialize an error message msg = '' # try importing all the required mojkdules for module in sorted(module_list): try: importlib.import_module(module) except ImportError: # add to error message for each bad module msg = msg if msg else HEADER msg += '-' * 60 + '\n' msg += "Missing module '{}'. To install use: \n".format(module) msg += " {}\n\n".format(CMD_DICT[module]) sys.stdout.write(msg + '\n') raise
def test_get_config_non_existent_file(self): """ get_config() creates config file when it doesn't exist """ if os.path.isfile(config_lib.CONFIG_FILE_NAME): os.system('rm {}'.format(config_lib.CONFIG_FILE_NAME)) config = config_lib.get_config() self.assertEqual(config_lib.DEFAULT_DICT, config)
def main(): # --- read in the current configuration default_dict = config_lib.get_config() msg = textwrap.dedent( """ Sets the default IO and plotting behavior for the pandashells toolset by creating/modifying a hidden configuration file (~/.pandashells) ----------------------------------------------------------------------- Examples: * Show the current configuration p.config * Set the configuration to "factory defaults" p.config --force_defaults * Set default input/output to assume white-space delimited columns with no headers. p.config --io_input_header noheader --io_input_type table p.config --io_output_header noheader --io_output_type table ----------------------------------------------------------------------- """ ) # --- populate the arg parser with current configuration parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=msg) parser.add_argument('--force_defaults', action='store_true', dest='force_defaults', help='Force to default settings') for tup in config_lib.CONFIG_OPTS: msg = 'opts: {}'.format(str(tup[1])) parser.add_argument('--%s' % tup[0], nargs=1, type=str, dest=tup[0], metavar='', default=[default_dict[tup[0]]], choices=tup[1], help=msg) # --- parse arguments args = parser.parse_args() # --- set the arguments to the current value of the arg parser config_dict = {t[0]: t[1][0] for t in args.__dict__.items() if not t[0] in ['force_defaults']} if args.force_defaults: cmd = 'rm {} 2>/dev/null'.format(config_lib.CONFIG_FILE_NAME) os.system(cmd) config_dict = config_lib.DEFAULT_DICT config_lib.set_config(config_dict) sys.stdout.write('\n Current Config\n') sys.stdout.write(' ' + '-' * 40 + '\n') for k in sorted(config_dict.keys()): if k not in ['--force_defaults']: sys.stdout.write( ' {: <20} {}\n'.format(k + ':', repr(str(config_dict[k]))))
def set_backend(): if BACKEND_COUNTER['use_called'] == 0: try: import matplotlib config = config_lib.get_config() matplotlib.use(config['plot_backend']) BACKEND_COUNTER.update({'use_called': 1}) except ImportError: # pragma nocover pass
def set_backend(): with warnings.catch_warnings(): if BACKEND_COUNTER['use_called'] == 0: try: warnings.filterwarnings('ignore') import matplotlib config = config_lib.get_config() matplotlib.use(config['plot_backend']) BACKEND_COUNTER.update({'use_called': 1}) warnings.resetwarnings() except ImportError: # pragma nocover pass
def test_get_config_existing_file(self): """ get_config() reads existing file """ if os.path.isfile(config_lib.CONFIG_FILE_NAME): os.system('rm {}'.format(config_lib.CONFIG_FILE_NAME)) test_config = {'name': 'Bill'} with open(config_lib.CONFIG_FILE_NAME, 'w') as f: f.write(json.dumps(test_config)) config = config_lib.get_config() self.assertEqual(test_config, config)
def add_args(parser, *args): """Adds argument blocks to the arg parser :type parser: argparse instance :param parser: The argarse instance to use in adding arguments Additinional arguments are the names of argument blocks to add """ config_dict = config_lib.get_config() _check_for_recognized_args(*args) _io_in_adder(parser, config_dict, *args) _io_out_adder(parser, config_dict, *args) _decorating_adder(parser, *args) _xy_adder(parser, *args)
def test_io_out_adder_active(self): """ _io_out_adder() adds proper arguments """ # set up mock parser parser = MagicMock() group = MagicMock() group.add_argument = MagicMock() parser.add_argument_group = MagicMock(return_value=group) # create expected call signature io_opt_list = [ 'csv', 'table', 'html', 'header', 'noheader', 'index', 'noindex' ] # define the current defaults default_for_output = ['csv', 'header', 'noindex'] msg = 'Must be one of {}'.format(repr(io_opt_list)) call_obj1 = call('-o', '--output_options', nargs='+', type=str, dest='output_options', metavar='option', default=default_for_output, help=msg) msg = ('Replace NaNs with this string. ' 'A string containing \'nan\' will set na_rep to numpy NaN. ' 'Current default is {}').format( repr(str(config_lib.get_config()['io_output_na_rep']))) call_obj2 = call('--output_na_rep', nargs=1, type=str, dest='io_output_na_rep', help=msg) # run the code under test args = ['io_out'] config_dict = { 'io_output_type': 'csv', 'io_output_header': 'header', 'io_output_index': 'noindex', 'io_output_na_rep': 'nan' } arg_lib._io_out_adder(parser, config_dict, *args) # make sure proper calls were made self.assertEqual(group.add_argument.call_args_list[0], call_obj1) self.assertEqual(group.add_argument.call_args_list[1], call_obj2)
def test_io_out_adder_active(self): """ _io_out_adder() adds proper arguments """ # set up mock parser parser = MagicMock() group = MagicMock() group.add_argument = MagicMock() parser.add_argument_group = MagicMock(return_value=group) # create expected call signature io_opt_list = ['csv', 'table', 'json', 'html', 'header', 'noheader', 'index', 'noindex'] # define the current defaults default_for_output = ['csv', 'header', 'noindex'] msg = 'Must be one of {}'.format(repr(io_opt_list)) call_obj1 = call( '-o', '--output_options', nargs='+', type=str, dest='output_options', metavar='option', default=default_for_output, help=msg) msg = ( 'Replace NaNs with this string. ' 'A string containing \'nan\' will set na_rep to numpy NaN. ' 'Current default is {}' ).format(repr(str(config_lib.get_config()['io_output_na_rep']))) call_obj2 = call( '--output_na_rep', nargs=1, type=str, dest='io_output_na_rep', help=msg) # run the code under test args = ['io_out'] config_dict = { 'io_output_type': 'csv', 'io_output_header': 'header', 'io_output_index': 'noindex', 'io_output_na_rep': 'nan' } arg_lib._io_out_adder(parser, config_dict, *args) # make sure proper calls were made self.assertEqual(group.add_argument.call_args_list[0], call_obj1) self.assertEqual(group.add_argument.call_args_list[1], call_obj2)
def main(): # --- read in the current configuration default_dict = config_lib.get_config() msg = textwrap.dedent(""" Sets the default IO and plotting behavior for the pandashells toolset by creating/modifying a hidden configuration file (~/.pandashells) ----------------------------------------------------------------------- Examples: * Show the current configuration p.config * Set the configuration to "factory defaults" p.config --force_defaults * Set default input/output to assume white-space delimited columns with no headers. p.config --io_input_header noheader --io_input_type table p.config --io_output_header noheader --io_output_type table ----------------------------------------------------------------------- """) # --- populate the arg parser with current configuration parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=msg) parser.add_argument('--force_defaults', action='store_true', dest='force_defaults', help='Force to default settings') for tup in config_lib.CONFIG_OPTS: msg = 'opts: {}'.format(str(tup[1])) parser.add_argument('--%s' % tup[0], nargs=1, type=str, dest=tup[0], metavar='', default=[default_dict[tup[0]]], choices=tup[1], help=msg) # --- parse arguments args = parser.parse_args() # --- set the arguments to the current value of the arg parser config_dict = { t[0]: t[1][0] for t in args.__dict__.items() if not t[0] in ['force_defaults'] } if args.force_defaults: cmd = 'rm {} 2>/dev/null'.format(config_lib.CONFIG_FILE_NAME) os.system(cmd) config_dict = config_lib.DEFAULT_DICT config_lib.set_config(config_dict) sys.stdout.write('\n Current Config\n') sys.stdout.write(' ' + '-' * 40 + '\n') for k in sorted(config_dict.keys()): if k not in ['--force_defaults']: sys.stdout.write(' {: <20} {}\n'.format(k + ':', repr(str( config_dict[k]))))