def test_fw_to_dict(self): """ Pass in a line and receive dictionary. """ fw_config = deepcopy(SAMPLE_CONFIG) fw_obj = FixedWidth(fw_config) fw_obj.line = ( "Michael Smith " "032vegetarian 40.7128 -74.0059-100 98.6201701011.000 \r\n" ) values = fw_obj.data self.assertEqual(values["first_name"], "Michael") self.assertEqual(values["last_name"], "Smith") self.assertEqual(values["age"], 32) self.assertEqual(values["meal"], "vegetarian") self.assertEqual(values["latitude"], Decimal('40.7128')) self.assertEqual(values["longitude"], Decimal('-74.0059')) self.assertEqual(values["elevation"], -100) self.assertEqual(values["temperature"], Decimal('98.6')) self.assertEqual(values["decimal_precision"], Decimal('1.000')) self.assertEqual(values["date"], datetime.datetime.strptime('20170101', '%Y%m%d')) self.assertEqual(values["none_date"], None)
def test_required_is_none(self): """ Pass in a None value and raise exception. """ fw_config = deepcopy(SAMPLE_CONFIG) fw_obj = FixedWidth(fw_config) fw_obj.update( last_name="Smith", first_name="Michael", age=32, meal="vegetarian", latitude=Decimal('40.7128'), longitude=Decimal('-74.0059'), elevation=None, ) self.assertRaises(Exception, fw_obj.validate)
def main(): st.header("App for converting csv to fixed-width and other way around") st.sidebar.subheader("Upload config file [.csv]") st.sidebar.write("Config file should be same for both ways!") config_file_upload = st.sidebar.file_uploader("Choose a file") if config_file_upload is not None: selection = st.sidebar.selectbox( "What would you like to convert?", ("CSV->FixedWidth", "FixedWidth->CSV")) config_file = read_csv_file(config_file_upload) CONFIG = create_config(config_file) # Reads config for FixedWidth lib fw_config = deepcopy(CONFIG) fw_obj = FixedWidth(fw_config) if selection == 'CSV->FixedWidth': st.subheader("Upload CSV file") else: st.subheader("Upload FixedWidth file") data_file = st.file_uploader("Upload a data file") if data_file is not None: if selection == "CSV->FixedWidth": data = read_csv_file(data_file) lines = "" for i, row in enumerate(data): row = clean_row(row) fw_obj.update(**row) fw_string = fw_obj.line lines += fw_string f = open("yourfile.main", "w") f.write(lines) f.close() st.markdown( get_binary_file_downloader_html("yourfile.main", "my_file.main"), unsafe_allow_html=True, ) else: rows = [] data = read_fw_file(data_file) for i, row in enumerate(data): fw_obj.line = row values = fw_obj.data rows.append(values) write_csv(rows) st.markdown( get_binary_file_downloader_html("yourfile.csv", "my_file.csv"), unsafe_allow_html=True, )
def test_fw_contains_empty_value(self): """ Pass in a line with empty value and test that default gets set. """ fw_config = deepcopy(SAMPLE_CONFIG) fw_obj = FixedWidth(fw_config) fw_obj.line = ( "Michael Smith " "032vegetarian 40.7128 -74.0059-100 98.620170101 \r\n" ) self.assertEqual(fw_obj.data["decimal_precision"], Decimal(1))
def test_optional_is_none(self): """ Pass in a None value and raise exception. """ fw_config = deepcopy(SAMPLE_CONFIG) fw_obj = FixedWidth(fw_config) fw_obj.update( last_name="Smith", first_name="Michael", age=32, meal="vegetarian", latitude=Decimal('40.7128'), longitude=Decimal('-74.0059'), elevation=-100, decimal_precision=None, ) good = ( "Michael Smith " "032vegetarian 40.7128 -74.0059-100 98.6201701011.000 \r\n" ) self.assertEqual(fw_obj.line, good)
def test_basic(self): """ Test a simple, valid example. """ fw_config = deepcopy(SAMPLE_CONFIG) fw_obj = FixedWidth(fw_config) fw_obj.update( last_name="Smith", first_name="Michael", age=32, meal="vegetarian", latitude=Decimal('40.7128'), longitude=Decimal('-74.0059'), elevation=-100, decimal_precision=Decimal('1.0001'), ) fw_string = fw_obj.line good = ( "Michael Smith " "032vegetarian 40.7128 -74.0059-100 98.6201701011.001 \r\n" ) self.assertEqual(fw_string, good)
def getDailyMax(fil): # Get meta info for this station fw = FixedWidth(META_CONFIG) # Get number of observations nday = (sum(1 for line in open(fil,'rb')) - 1)/2 # read in observations f = open(fil, 'rb') header = f.readline() meta = fw._string_to_dict(header) fw = FixedWidth(TIDE_CONFIG) cols = ['Gauge','Name','Lat','Lon','Method','Datum','Offset','Year','Month','Day','Max_Tide'] dfOut = pd.DataFrame(columns=cols) charOut = np.empty((nday,len(cols)),dtype=object) lat,lon = fmtll(meta['lat'],meta['lon']) date_last = -9999 nobs_last = -9999 i = 0 for line in f: parts = fw._string_to_dict(line) lvls = [int(parts[str(ii)]) for ii in range(1, 13)] if meta['datum'] == 'X': lvls = [9999 for jj in lvls] # these data are not referenced to a datum date = parts['year']+parts['month']+parts['day'] if date == date_last: this_max = np.max(lvls) if this_max == 9999: this_max = np.nan if (nobs_last + sum([lvls[ii]!=9999 for ii in range(len(lvls))])) < 12: dayMax = -999 else: dayMax = np.max([this_max,max_last]) if np.isnan(dayMax): dayMax = -999 charOut[i,:] = [meta['number']+meta['version'],meta['name'],"{0:.2f}".format(lat),"{0:.2f}".format(lon),meta['method'], \ meta['datum'], meta['offset'],parts['year'],parts['month'],parts['day'],str(dayMax)] i+=1 else: nobs_last = sum([lvls[ii]!=9999 for ii in range(len(lvls))]) max_last = np.max(lvls) if max_last == 9999: max_last = np.nan date_last = date #return dfOut,int(str(date)[0:4]),meta return charOut,int(str(date)[0:4]),meta
def test_update(self): """ Test FixedWidth.update() """ fw_config = deepcopy(SAMPLE_CONFIG) fw_obj = FixedWidth(fw_config) fw_obj.update( last_name="Smith", first_name="Michael", age=32, meal="vegetarian", latitude=Decimal('40.7128'), longitude=Decimal('-74.0059'), elevation=-100, decimal_precision=1, ) #change a value fw_obj.update(meal="Paleo") self.assertEqual(fw_obj.data["meal"], "Paleo") #nothing else should have changed self.assertEqual(fw_obj.data["first_name"], "Michael")
def getMonMean(fil): print "Opening file: %s" %fil fw = FixedWidth(META_CONFIG_MON) # determine how many years we will read f = open(fil, 'rb') header = f.readline() all_years = [int(line[0:12].strip()) for line in f] # read in observations f = open(fil, 'rb') header = f.readline() meta = fw._string_to_dict(header) nlin = (max(all_years) - min(all_years) + 1)*12 charOut = np.empty((nlin,7),dtype=object) lat,lon = fmtll(meta['lat'],meta['lon']) i = 0 nmiss = 0 _sum = 0. this_mon = 1 cnt = 0 for ii,line in enumerate(f): line = line.split() mon = int(line[1]) if mon == this_mon: year = line[0] if line[3] == '9999' or meta['datum'] == 'X': nmiss += 1 else: _sum = _sum + float(line[3]) cnt += 1 else: # The monthly values are calculated from the daily data with a # simple average of all the daily values in a month. If seven or # fewer values are missing, the monthly value is calculated. if (nmiss <= 7): avg = _sum / float(cnt) else: avg = -999 charOut[i,:] = [meta['number']+meta['version'],meta['name'],"{0:.2f}".format(lat),"{0:.2f}".format(lon), \ year,str(this_mon).zfill(2),str(int(avg))] nmiss = 0 _sum = 0. cnt = 0 this_mon = mon i += 1 if (nmiss <= 7): avg = _sum / float(cnt) else: avg = -999 charOut[i,:] = [meta['number']+meta['version'],meta['name'],"{0:.2f}".format(lat),"{0:.2f}".format(lon), \ year,str(this_mon).zfill(2),str(int(avg))] return charOut,meta