def df_to_output(args, df):
# set up write options
config_dict = config_lib.get_config()
header = False if 'noheader' in args.output_options else True
index = True if 'index' in args.output_options else False
# define valid output types
valid_outputs = [
'table',
'csv',
'json',
'html',
]
# get the output type
output_type_set = set(args.output_options).intersection(set(valid_outputs))
if output_type_set:
output_type = list(output_type_set)[0]
else:
output_type = config_dict['io_output_type']
# set up a mapping between output type and writer function
writer_for = {
'csv': csv_writer,
'table': table_writer,
'html': html_writer,
'json': json_writer,
}
na_rep = get_nan_rep(args, config_dict)
# call writer in try block to gracefully handle closed pipes
try:
writer_for[output_type](df, header, index, na_rep)
except IOError:
pass
def df_to_output(args, df):
# set up write options
config_dict = config_lib.get_config()
header = False if 'noheader' in args.output_options else True
index = True if 'index' in args.output_options else False
# define valid output types
valid_outputs = [
'table',
'csv',
'html',
]
# get the output type
output_type_set = set(args.output_options).intersection(set(valid_outputs))
if output_type_set:
output_type = list(output_type_set)[0]
else:
output_type = config_dict['io_output_type']
# set up a mapping between output type and writer function
writer_for = {
'csv': csv_writer,
'table': table_writer,
'html': html_writer,
}
na_rep = get_nan_rep(args, config_dict)
# call writer in try block to gracefully handle closed pipes
try:
writer_for[output_type](df, header, index, na_rep)
except IOError:
pass
def df_from_input(args, in_file=None):
# set up proper state for reading input
config_dict = config_lib.get_config()
in_file = sys.stdin if in_file is None else in_file
sep = get_separator(args, config_dict)
header, names = get_header_names(args)
# read the input data
try:
df = pd.read_csv(in_file,
sep=sep,
header=header,
names=names,
encoding=ENCODING,
low_memory=False)
except ValueError:
sys.stderr.write('\n{} received no input\n'.format(
os.path.basename(sys.argv[0])))
sys.exit(1)
# if no names and no neader, create column names
if ('noheader' in args.input_options) and (not names):
names = ['c{}'.format(nn) for nn, cc in enumerate(df.columns)]
df.columns = pd.Index(names)
return df
def _df_from_input_csv(args, in_file=None):
# set up proper state for reading input
config_dict = config_lib.get_config()
in_file = sys.stdin if in_file is None else in_file
sep = get_separator(args, config_dict)
header, names = get_header_names(args)
# read the input data
try:
# for some reason tsv doesn't allow for low_memory option
if sep == r'\t':
df = pd.read_csv(in_file, sep=sep, header=header, names=names,
encoding=ENCODING, engine='python')
else:
df = pd.read_csv(in_file, sep=sep, header=header, names=names,
encoding=ENCODING, low_memory=False)
except ValueError:
sys.stderr.write('\n{} received no input\n'.format(
os.path.basename(sys.argv[0])))
sys.exit(1)
# if no names and no neader, create column names
if ('noheader' in args.input_options) and (not names):
names = ['c{}'.format(nn) for nn, cc in enumerate(df.columns)]
df.columns = pd.Index(names)
return df
def check_for_modules(module_list):
# make sure module_list only contains recognized modules
unnamed_modules = set(module_list) - set(CMD_DICT.keys())
unnamed_modules = unnamed_modules - {'datetime', 're'}
if unnamed_modules:
msg = '\n\nThese modules unrecognized by check_for_modules(): '
msg += '{}\n'.format(unnamed_modules)
raise ValueError(msg)
# try using configured backend ignoring errors so they'll be caught later
if set(module_list).intersection({'matplotlib', 'pylab', 'seaborn'}):
CONFIG = config_lib.get_config()
try:
import matplotlib
if matplotlib.get_backend() != CONFIG['plot_backend']:
matplotlib.use(CONFIG['plot_backend'])
except ImportError:
pass
# initialize an error message
msg = ''
# try importing all the required mojkdules
for module in sorted(module_list):
try:
importlib.import_module(module)
except ImportError:
# add to error message for each bad module
msg = msg if msg else HEADER
msg += '-' * 60 + '\n'
msg += "Missing module '{}'. To install use: \n".format(module)
msg += " {}\n\n".format(CMD_DICT[module])
sys.stdout.write(msg + '\n')
raise
def test_get_config_non_existent_file(self):
"""
get_config() creates config file when it doesn't exist
"""
if os.path.isfile(config_lib.CONFIG_FILE_NAME):
os.system('rm {}'.format(config_lib.CONFIG_FILE_NAME))
config = config_lib.get_config()
self.assertEqual(config_lib.DEFAULT_DICT, config)
def test_get_config_non_existent_file(self):
"""
get_config() creates config file when it doesn't exist
"""
if os.path.isfile(config_lib.CONFIG_FILE_NAME):
os.system('rm {}'.format(config_lib.CONFIG_FILE_NAME))
config = config_lib.get_config()
self.assertEqual(config_lib.DEFAULT_DICT, config)
def main():
# --- read in the current configuration
default_dict = config_lib.get_config()
msg = textwrap.dedent(
"""
Sets the default IO and plotting behavior for the pandashells toolset
by creating/modifying a hidden configuration file (~/.pandashells)
-----------------------------------------------------------------------
Examples:
* Show the current configuration
p.config
* Set the configuration to "factory defaults"
p.config --force_defaults
* Set default input/output to assume white-space delimited columns
with no headers.
p.config --io_input_header noheader --io_input_type table
p.config --io_output_header noheader --io_output_type table
-----------------------------------------------------------------------
"""
)
# --- populate the arg parser with current configuration
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter, description=msg)
parser.add_argument('--force_defaults', action='store_true',
dest='force_defaults',
help='Force to default settings')
for tup in config_lib.CONFIG_OPTS:
msg = 'opts: {}'.format(str(tup[1]))
parser.add_argument('--%s' % tup[0], nargs=1, type=str,
dest=tup[0], metavar='',
default=[default_dict[tup[0]]],
choices=tup[1], help=msg)
# --- parse arguments
args = parser.parse_args()
# --- set the arguments to the current value of the arg parser
config_dict = {t[0]: t[1][0] for t in args.__dict__.items()
if not t[0] in ['force_defaults']}
if args.force_defaults:
cmd = 'rm {} 2>/dev/null'.format(config_lib.CONFIG_FILE_NAME)
os.system(cmd)
config_dict = config_lib.DEFAULT_DICT
config_lib.set_config(config_dict)
sys.stdout.write('\n Current Config\n')
sys.stdout.write(' ' + '-' * 40 + '\n')
for k in sorted(config_dict.keys()):
if k not in ['--force_defaults']:
sys.stdout.write(
' {: <20} {}\n'.format(k + ':', repr(str(config_dict[k]))))
def set_backend():
if BACKEND_COUNTER['use_called'] == 0:
try:
import matplotlib
config = config_lib.get_config()
matplotlib.use(config['plot_backend'])
BACKEND_COUNTER.update({'use_called': 1})
except ImportError: # pragma nocover
pass
def set_backend():
with warnings.catch_warnings():
if BACKEND_COUNTER['use_called'] == 0:
try:
warnings.filterwarnings('ignore')
import matplotlib
config = config_lib.get_config()
matplotlib.use(config['plot_backend'])
BACKEND_COUNTER.update({'use_called': 1})
warnings.resetwarnings()
except ImportError: # pragma nocover
pass
def test_get_config_existing_file(self):
"""
get_config() reads existing file
"""
if os.path.isfile(config_lib.CONFIG_FILE_NAME):
os.system('rm {}'.format(config_lib.CONFIG_FILE_NAME))
test_config = {'name': 'Bill'}
with open(config_lib.CONFIG_FILE_NAME, 'w') as f:
f.write(json.dumps(test_config))
config = config_lib.get_config()
self.assertEqual(test_config, config)
def test_get_config_existing_file(self):
"""
get_config() reads existing file
"""
if os.path.isfile(config_lib.CONFIG_FILE_NAME):
os.system('rm {}'.format(config_lib.CONFIG_FILE_NAME))
test_config = {'name': 'Bill'}
with open(config_lib.CONFIG_FILE_NAME, 'w') as f:
f.write(json.dumps(test_config))
config = config_lib.get_config()
self.assertEqual(test_config, config)
def add_args(parser, *args):
"""Adds argument blocks to the arg parser
:type parser: argparse instance
:param parser: The argarse instance to use in adding arguments
Additinional arguments are the names of argument blocks to add
"""
config_dict = config_lib.get_config()
_check_for_recognized_args(*args)
_io_in_adder(parser, config_dict, *args)
_io_out_adder(parser, config_dict, *args)
_decorating_adder(parser, *args)
_xy_adder(parser, *args)
def add_args(parser, *args):
"""Adds argument blocks to the arg parser
:type parser: argparse instance
:param parser: The argarse instance to use in adding arguments
Additinional arguments are the names of argument blocks to add
"""
config_dict = config_lib.get_config()
_check_for_recognized_args(*args)
_io_in_adder(parser, config_dict, *args)
_io_out_adder(parser, config_dict, *args)
_decorating_adder(parser, *args)
_xy_adder(parser, *args)
def test_io_out_adder_active(self):
"""
_io_out_adder() adds proper arguments
"""
# set up mock parser
parser = MagicMock()
group = MagicMock()
group.add_argument = MagicMock()
parser.add_argument_group = MagicMock(return_value=group)
# create expected call signature
io_opt_list = [
'csv', 'table', 'html', 'header', 'noheader', 'index', 'noindex'
]
# define the current defaults
default_for_output = ['csv', 'header', 'noindex']
msg = 'Must be one of {}'.format(repr(io_opt_list))
call_obj1 = call('-o',
'--output_options',
nargs='+',
type=str,
dest='output_options',
metavar='option',
default=default_for_output,
help=msg)
msg = ('Replace NaNs with this string. '
'A string containing \'nan\' will set na_rep to numpy NaN. '
'Current default is {}').format(
repr(str(config_lib.get_config()['io_output_na_rep'])))
call_obj2 = call('--output_na_rep',
nargs=1,
type=str,
dest='io_output_na_rep',
help=msg)
# run the code under test
args = ['io_out']
config_dict = {
'io_output_type': 'csv',
'io_output_header': 'header',
'io_output_index': 'noindex',
'io_output_na_rep': 'nan'
}
arg_lib._io_out_adder(parser, config_dict, *args)
# make sure proper calls were made
self.assertEqual(group.add_argument.call_args_list[0], call_obj1)
self.assertEqual(group.add_argument.call_args_list[1], call_obj2)
def test_io_out_adder_active(self):
"""
_io_out_adder() adds proper arguments
"""
# set up mock parser
parser = MagicMock()
group = MagicMock()
group.add_argument = MagicMock()
parser.add_argument_group = MagicMock(return_value=group)
# create expected call signature
io_opt_list = ['csv', 'table', 'json', 'html',
'header', 'noheader', 'index', 'noindex']
# define the current defaults
default_for_output = ['csv', 'header', 'noindex']
msg = 'Must be one of {}'.format(repr(io_opt_list))
call_obj1 = call(
'-o', '--output_options', nargs='+',
type=str, dest='output_options', metavar='option',
default=default_for_output, help=msg)
msg = (
'Replace NaNs with this string. '
'A string containing \'nan\' will set na_rep to numpy NaN. '
'Current default is {}'
).format(repr(str(config_lib.get_config()['io_output_na_rep'])))
call_obj2 = call(
'--output_na_rep', nargs=1, type=str, dest='io_output_na_rep',
help=msg)
# run the code under test
args = ['io_out']
config_dict = {
'io_output_type': 'csv',
'io_output_header': 'header',
'io_output_index': 'noindex',
'io_output_na_rep': 'nan'
}
arg_lib._io_out_adder(parser, config_dict, *args)
# make sure proper calls were made
self.assertEqual(group.add_argument.call_args_list[0], call_obj1)
self.assertEqual(group.add_argument.call_args_list[1], call_obj2)
def main():
# --- read in the current configuration
default_dict = config_lib.get_config()
msg = textwrap.dedent("""
Sets the default IO and plotting behavior for the pandashells toolset
by creating/modifying a hidden configuration file (~/.pandashells)
-----------------------------------------------------------------------
Examples:
* Show the current configuration
p.config
* Set the configuration to "factory defaults"
p.config --force_defaults
* Set default input/output to assume white-space delimited columns
with no headers.
p.config --io_input_header noheader --io_input_type table
p.config --io_output_header noheader --io_output_type table
-----------------------------------------------------------------------
""")
# --- populate the arg parser with current configuration
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter, description=msg)
parser.add_argument('--force_defaults',
action='store_true',
dest='force_defaults',
help='Force to default settings')
for tup in config_lib.CONFIG_OPTS:
msg = 'opts: {}'.format(str(tup[1]))
parser.add_argument('--%s' % tup[0],
nargs=1,
type=str,
dest=tup[0],
metavar='',
default=[default_dict[tup[0]]],
choices=tup[1],
help=msg)
# --- parse arguments
args = parser.parse_args()
# --- set the arguments to the current value of the arg parser
config_dict = {
t[0]: t[1][0]
for t in args.__dict__.items() if not t[0] in ['force_defaults']
}
if args.force_defaults:
cmd = 'rm {} 2>/dev/null'.format(config_lib.CONFIG_FILE_NAME)
os.system(cmd)
config_dict = config_lib.DEFAULT_DICT
config_lib.set_config(config_dict)
sys.stdout.write('\n Current Config\n')
sys.stdout.write(' ' + '-' * 40 + '\n')
for k in sorted(config_dict.keys()):
if k not in ['--force_defaults']:
sys.stdout.write(' {: <20} {}\n'.format(k + ':',
repr(str(
config_dict[k]))))
