#!/usr/bin/env/python
"""
null_value_filter.py -- Replace "NULL" with empty string
"""
__author__ = "Michael Conlon"
__copyright__ = "Copyright 2015 (c), Michael Conlon"
__license__ = "New BSD License"
__version__ = "0.01"
from vivopump import read_csv_fp, write_csv_fp
import shelve
import sys
data_in = read_csv_fp(sys.stdin)
data_out = {}
null_count = 0
for row, data in data_in.items():
new_data =dict(data)
for name, val in new_data.items():
if val == "NULL":
new_data[name] = ""
null_count += 1
data_out[row] = new_data
print >>sys.stderr, "NULL values replaced", null_count
write_csv_fp(sys.stdout, data_out)
#!/usr/bin/env/python
"""
manage_columns_filter.py -- add needed columns, remove unused columns
"""
__author__ = "Michael Conlon"
__copyright__ = "Copyright 2015 (c) Michael Conlon"
__license__ = "New BSD License"
__version__ = "0.01"
from vivopump import read_csv_fp, write_csv_fp, improve_jobcode_description
import sys
data_in = read_csv_fp(sys.stdin)
var_names = data_in[
data_in.keys()[1]].keys() # create a list of var_names from the first row
print >> sys.stderr, "Columns in", var_names
data_out = {}
for row, data in data_in.items():
new_data = dict(data)
# Add these columns
new_data['remove'] = ''
new_data['uri'] = ''
new_data['title'] = improve_jobcode_description(
new_data['JOBCODE_DESCRIPTION'])
new_data['hr_title'] = new_data['JOBCODE_DESCRIPTION']
# Delete these columns
def test_read_csv_fp(self):
fp = open("data/minimal.txt", 'rU')
data = read_csv_fp(fp, delimiter='|')
fp.close()
data_string = "{1: {u'overview': u'None', u'uri': u'http://vivo.ufl.edu/individual/n7023304'}}"
self.assertEqual(data_string, str(data))
