def test_fw_to_dict(self):
"""
Pass in a line and receive dictionary.
"""
fw_config = deepcopy(SAMPLE_CONFIG)
fw_obj = FixedWidth(fw_config)
fw_obj.line = (
"Michael Smith "
"032vegetarian 40.7128 -74.0059-100 98.6201701011.000 \r\n"
)
values = fw_obj.data
self.assertEqual(values["first_name"], "Michael")
self.assertEqual(values["last_name"], "Smith")
self.assertEqual(values["age"], 32)
self.assertEqual(values["meal"], "vegetarian")
self.assertEqual(values["latitude"], Decimal('40.7128'))
self.assertEqual(values["longitude"], Decimal('-74.0059'))
self.assertEqual(values["elevation"], -100)
self.assertEqual(values["temperature"], Decimal('98.6'))
self.assertEqual(values["decimal_precision"], Decimal('1.000'))
self.assertEqual(values["date"],
datetime.datetime.strptime('20170101', '%Y%m%d'))
self.assertEqual(values["none_date"], None)
def test_required_is_none(self):
"""
Pass in a None value and raise exception.
"""
fw_config = deepcopy(SAMPLE_CONFIG)
fw_obj = FixedWidth(fw_config)
fw_obj.update(
last_name="Smith", first_name="Michael",
age=32, meal="vegetarian", latitude=Decimal('40.7128'),
longitude=Decimal('-74.0059'), elevation=None,
)
self.assertRaises(Exception, fw_obj.validate)
def main():
st.header("App for converting csv to fixed-width and other way around")
st.sidebar.subheader("Upload config file [.csv]")
st.sidebar.write("Config file should be same for both ways!")
config_file_upload = st.sidebar.file_uploader("Choose a file")
if config_file_upload is not None:
selection = st.sidebar.selectbox(
"What would you like to convert?",
("CSV->FixedWidth", "FixedWidth->CSV"))
config_file = read_csv_file(config_file_upload)
CONFIG = create_config(config_file)
# Reads config for FixedWidth lib
fw_config = deepcopy(CONFIG)
fw_obj = FixedWidth(fw_config)
if selection == 'CSV->FixedWidth':
st.subheader("Upload CSV file")
else:
st.subheader("Upload FixedWidth file")
data_file = st.file_uploader("Upload a data file")
if data_file is not None:
if selection == "CSV->FixedWidth":
data = read_csv_file(data_file)
lines = ""
for i, row in enumerate(data):
row = clean_row(row)
fw_obj.update(**row)
fw_string = fw_obj.line
lines += fw_string
f = open("yourfile.main", "w")
f.write(lines)
f.close()
st.markdown(
get_binary_file_downloader_html("yourfile.main",
"my_file.main"),
unsafe_allow_html=True,
)
else:
rows = []
data = read_fw_file(data_file)
for i, row in enumerate(data):
fw_obj.line = row
values = fw_obj.data
rows.append(values)
write_csv(rows)
st.markdown(
get_binary_file_downloader_html("yourfile.csv",
"my_file.csv"),
unsafe_allow_html=True,
)
def test_fw_contains_empty_value(self):
"""
Pass in a line with empty value and test that default gets set.
"""
fw_config = deepcopy(SAMPLE_CONFIG)
fw_obj = FixedWidth(fw_config)
fw_obj.line = (
"Michael Smith "
"032vegetarian 40.7128 -74.0059-100 98.620170101 \r\n"
)
self.assertEqual(fw_obj.data["decimal_precision"], Decimal(1))
def test_optional_is_none(self):
"""
Pass in a None value and raise exception.
"""
fw_config = deepcopy(SAMPLE_CONFIG)
fw_obj = FixedWidth(fw_config)
fw_obj.update(
last_name="Smith", first_name="Michael",
age=32, meal="vegetarian", latitude=Decimal('40.7128'),
longitude=Decimal('-74.0059'), elevation=-100, decimal_precision=None,
)
good = (
"Michael Smith "
"032vegetarian 40.7128 -74.0059-100 98.6201701011.000 \r\n"
)
self.assertEqual(fw_obj.line, good)
def test_basic(self):
"""
Test a simple, valid example.
"""
fw_config = deepcopy(SAMPLE_CONFIG)
fw_obj = FixedWidth(fw_config)
fw_obj.update(
last_name="Smith", first_name="Michael",
age=32, meal="vegetarian", latitude=Decimal('40.7128'),
longitude=Decimal('-74.0059'), elevation=-100, decimal_precision=Decimal('1.0001'),
)
fw_string = fw_obj.line
good = (
"Michael Smith "
"032vegetarian 40.7128 -74.0059-100 98.6201701011.001 \r\n"
)
self.assertEqual(fw_string, good)
def getDailyMax(fil):
# Get meta info for this station
fw = FixedWidth(META_CONFIG)
# Get number of observations
nday = (sum(1 for line in open(fil,'rb')) - 1)/2
# read in observations
f = open(fil, 'rb')
header = f.readline()
meta = fw._string_to_dict(header)
fw = FixedWidth(TIDE_CONFIG)
cols = ['Gauge','Name','Lat','Lon','Method','Datum','Offset','Year','Month','Day','Max_Tide']
dfOut = pd.DataFrame(columns=cols)
charOut = np.empty((nday,len(cols)),dtype=object)
lat,lon = fmtll(meta['lat'],meta['lon'])
date_last = -9999
nobs_last = -9999
i = 0
for line in f:
parts = fw._string_to_dict(line)
lvls = [int(parts[str(ii)]) for ii in range(1, 13)]
if meta['datum'] == 'X':
lvls = [9999 for jj in lvls] # these data are not referenced to a datum
date = parts['year']+parts['month']+parts['day']
if date == date_last:
this_max = np.max(lvls)
if this_max == 9999:
this_max = np.nan
if (nobs_last + sum([lvls[ii]!=9999 for ii in range(len(lvls))])) < 12:
dayMax = -999
else:
dayMax = np.max([this_max,max_last])
if np.isnan(dayMax): dayMax = -999
charOut[i,:] = [meta['number']+meta['version'],meta['name'],"{0:.2f}".format(lat),"{0:.2f}".format(lon),meta['method'], \
meta['datum'], meta['offset'],parts['year'],parts['month'],parts['day'],str(dayMax)]
i+=1
else:
nobs_last = sum([lvls[ii]!=9999 for ii in range(len(lvls))])
max_last = np.max(lvls)
if max_last == 9999:
max_last = np.nan
date_last = date
#return dfOut,int(str(date)[0:4]),meta
return charOut,int(str(date)[0:4]),meta
def test_update(self):
"""
Test FixedWidth.update()
"""
fw_config = deepcopy(SAMPLE_CONFIG)
fw_obj = FixedWidth(fw_config)
fw_obj.update(
last_name="Smith", first_name="Michael",
age=32, meal="vegetarian", latitude=Decimal('40.7128'),
longitude=Decimal('-74.0059'), elevation=-100, decimal_precision=1,
)
#change a value
fw_obj.update(meal="Paleo")
self.assertEqual(fw_obj.data["meal"], "Paleo")
#nothing else should have changed
self.assertEqual(fw_obj.data["first_name"], "Michael")
def getMonMean(fil):
print "Opening file: %s" %fil
fw = FixedWidth(META_CONFIG_MON)
# determine how many years we will read
f = open(fil, 'rb')
header = f.readline()
all_years = [int(line[0:12].strip()) for line in f]
# read in observations
f = open(fil, 'rb')
header = f.readline()
meta = fw._string_to_dict(header)
nlin = (max(all_years) - min(all_years) + 1)*12
charOut = np.empty((nlin,7),dtype=object)
lat,lon = fmtll(meta['lat'],meta['lon'])
i = 0
nmiss = 0
_sum = 0.
this_mon = 1
cnt = 0
for ii,line in enumerate(f):
line = line.split()
mon = int(line[1])
if mon == this_mon:
year = line[0]
if line[3] == '9999' or meta['datum'] == 'X':
nmiss += 1
else:
_sum = _sum + float(line[3])
cnt += 1
else:
# The monthly values are calculated from the daily data with a
# simple average of all the daily values in a month. If seven or
# fewer values are missing, the monthly value is calculated.
if (nmiss <= 7):
avg = _sum / float(cnt)
else:
avg = -999
charOut[i,:] = [meta['number']+meta['version'],meta['name'],"{0:.2f}".format(lat),"{0:.2f}".format(lon), \
year,str(this_mon).zfill(2),str(int(avg))]
nmiss = 0
_sum = 0.
cnt = 0
this_mon = mon
i += 1
if (nmiss <= 7):
avg = _sum / float(cnt)
else:
avg = -999
charOut[i,:] = [meta['number']+meta['version'],meta['name'],"{0:.2f}".format(lat),"{0:.2f}".format(lon), \
year,str(this_mon).zfill(2),str(int(avg))]
return charOut,meta