def test_use_case_csv_json(self):
"""Simple use case using core tools for reading,
computing and writing"""
raised = False
try:
data = dr.DataReader(data=data_paths['csv'], f_format='csv').read()
table = st.StashTable(data=data, table='data')
table.save()
subset = table.get(key='gender',
value='Female',
operator='==',
reindex=True)
dw.write_json(data=subset,
file='subset.json',
allow_nan=False,
sort_keys=False)
dw.write_json(data=subset,
file='subset.json',
allow_nan=True,
sort_keys=True)
table.un_stash()
os.remove('subset.json')
except:
raised = True
self.assertIs(raised, False)
def test_write_json_indent(self):
"""Write to json with indent"""
data = dr.DataReader(f_format='csv',
data=cwd + '/test/data/basic_csv.csv') \
.read(delim=',')
dw.write_json(data=data, file='tmp.json', allow_nan=True, indent=4)
self.assertIs(os.path.isfile('tmp.json'), True)
os.remove('tmp.json')
def test_write_yaml(self):
"""Write to yaml"""
data = dr.DataReader(f_format='csv',
data=cwd + '/test/data/basic_csv.csv') \
.read(delim=',')
dw.write_yaml(data=data, file='tmp.yaml')
self.assertIs(os.path.isfile('tmp.yaml'), True)
os.remove('tmp.yaml')
def test_write_json_not_allow_nan(self):
"""Write to json not allowing nan"""
data = dr.DataReader(f_format='csv',
data=cwd + '/test/data/basic_csv.csv') \
.read(delim=',')
dw.write_json(data=data, file='tmp.json', allow_nan=False)
self.assertIs(os.path.isfile('tmp.json'), True)
os.remove('tmp.json')
def test_write_csv_delim_2(self):
"""Write to csv with delim ;"""
data = dr.DataReader(f_format='csv',
data=cwd + '/test/data/basic_csv.csv') \
.read(delim=',')
dw.write_csv(data=data, file='tmp.csv', delimiter=';')
self.assertIs(os.path.isfile('tmp.csv'), True)
os.remove('tmp.csv')
def analyse_stash():
"""example of using analysis on stashed table"""
data = dr.DataReader(data='test/data/json_array.json.gz').read()
table = StashTable(data=data, table='basic')
table.save()
table_ana = Analyse(
table='basic') # stashed table name needs to be specified
# print(table_ana.max(col='id'))
# print(table_ana.min(col='id'))
# print(table_ana.mean(col='id'))
# print(table_ana.mode(col='id'))
# print(table_ana.median(col='id'))
table.un_stash()
def standard_stash():
"""example of a normal data read, stash and write"""
data = dr.DataReader(data='test/data/json_array.json.gz',
f_format='json.gz').read()
dw.write_csv(data=data, file='main.csv')
table = StashTable(data=data, table='basic')
table.save()
col_names = table.col_names()
first_row = table.row(index=0)
first_col = table.col(key='id')
subset = table.get(key='gender',
value='Female',
operator='==',
reindex=True)
dw.write_csv(data=subset, file='subset.csv')
table.un_stash()
def test_use_case_yaml_csv(self):
"""Simple use case using core tools for reading,
computing and writing"""
raised = False
try:
data = dr.DataReader(data=data_paths['yaml'],
f_format='yaml').read()
table = st.StashTable(data=data, table='data')
table.save()
subset = table.get(key='gender',
value='Female',
operator='==',
reindex=True)
dw.write_csv(data=subset, file='subset.csv', header=True)
dw.write_csv(data=subset,
file='subset.csv',
header=True,
quoting='all')
dw.write_csv(data=subset,
file='subset.csv',
header=True,
quoting='minimal')
dw.write_csv(data=subset,
file='subset.csv',
header=True,
quoting='nonumeric')
table.un_stash()
os.remove('subset.csv')
except:
raised = True
self.assertIs(raised, False)
def get_data():
"""Get mock data for stash"""
_dir = cwd + '/test/data/basic_csv.csv'
return dr.DataReader(f_format='csv', data=_dir).read(delim=',')
def read_write():
"""normal read and write"""
data = dr.DataReader(data='test/data/basic_csv.csv.gz').read()
dw.write_csv(data=data, file='test.csv', threads=10)
