def read(self):
'''Load the data from in_filename'''
skip_index = find_first(self.in_filename, '^[0-9]{4},[0-9]{4}$') - 1
df = pd.read_table(self.in_filename,
skiprows=skip_index,
header=None,
engine='c',
sep=',',
names=('a', 'b'))
self.pressure_data = np.array([
uc.USGS_PROTOTYPE_V_TO_DBAR(np.float64(x))
for x in df[df.b.isnull() == False].a
])
self.temperature_data = [
uc.USGS_PROTOTYPE_V_TO_C(np.float64(x))
for x in df[df.b.isnull() == False].b
]
with open(self.in_filename, 'r') as wavelog:
for x in wavelog:
# second arg has extra space that is unnecessary
if re.match('^[0-9]{4}.[0-9]{2}.[0-9]{2}', x):
start_ms = uc.datestring_to_ms(x, self.date_format_string)
self.utc_millisecond_data = uc.generate_ms(
start_ms, len(self.pressure_data), self.frequency)
break
def read(self):
'''load the data from in_filename
only parse the initial datetime = much faster
'''
self.get_serial()
skip_index = find_first(self.in_filename, 'Date and Time,Seconds')
data = pd.read_table(self.in_filename,
skiprows=skip_index,
header=None,
engine='c',
sep=',',
usecols=(0, 1, 2, 3))
self.data_start = uc.datestring_to_ms(data[0][1],
self.date_format_string,
self.tz_info,
self.daylight_savings)
self.data_start2 = uc.datestring_to_ms(data[0][2],
self.date_format_string,
self.tz_info,
self.daylight_savings)
self.frequency = 1 / ((self.data_start2 - self.data_start) / 1000)
self.utc_millisecond_data = uc.generate_ms(self.data_start,
len(data[0]),
self.frequency)
self.pressure_data = data[3].values * uc.PSI_TO_DBAR
def wind_data(file_name, mode='netCDF'):
#every fifteen minutes maybe not necessary
frequency = 1 / 900
series_length = 1440
time = unit_conversion.generate_ms(1404647999870, series_length, frequency)
wind_direction = get_rand_circular_data(series_length, 15, 360)
wind_speed = get_rand_discrete_data(series_length, 2, 5, 0)
if mode == 'netCDF':
ds = Dataset(file_name, 'w', format="NETCDF4_CLASSIC")
ds.createDimension('time', len(time))
time_var = ds.createVariable('time', 'f8', ('time'))
time_var[:] = time
wind_speed_var = ds.createVariable('wind_speed', 'f8', ('time'))
wind_speed_var[:] = wind_speed
wind_direction_var = ds.createVariable('wind_direction', 'f8',
('time'))
wind_direction_var[:] = wind_direction
ds.close()
else:
excelFile = pd.DataFrame({
'Time': time,
'Wind Speed in m/s': wind_speed,
'Wind Direction in degrees': wind_direction,
})
excelFile.to_csv(path_or_buf=file_name)
print('total:', len(time), len(wind_direction), len(wind_speed))
def wind_data(file_name, mode='netCDF'):
#every fifteen minutes maybe not necessary
frequency = 1/900
series_length = 1440
time = unit_conversion.generate_ms(1404647999870, series_length, frequency)
wind_direction = get_rand_circular_data(series_length, 15, 360)
wind_speed = get_rand_discrete_data(series_length, 2, 5, 0)
if mode == 'netCDF':
ds = Dataset(file_name, 'w', format="NETCDF4_CLASSIC")
ds.createDimension('time',len(time))
time_var = ds.createVariable('time','f8',('time'))
time_var[:] = time
wind_speed_var = ds.createVariable('wind_speed','f8',('time'))
wind_speed_var[:] = wind_speed
wind_direction_var = ds.createVariable('wind_direction','f8',('time'))
wind_direction_var[:] = wind_direction
ds.close()
else:
excelFile = pd.DataFrame({'Time': time,
'Wind Speed in m/s': wind_speed,
'Wind Direction in degrees': wind_direction,
})
excelFile.to_csv(path_or_buf= file_name)
print('total:', len(time), len(wind_direction), len(wind_speed))
def read(self):
'''load the data from in_filename
only parse the initial datetime = much faster
'''
self.get_serial()
skip_index = find_first(self.in_filename, '^ID') - 1
# for skipping lines in case there is calibration header data
df = pd.read_table(self.in_filename, skiprows=skip_index + 1, header=None,
engine='c', sep=',', usecols=[3, 4, 5])
try:
self.data_start = uc.datestring_to_ms(df[3][3][1:],
self.date_format_string, self.tz_info, self.daylight_savings)
second_stamp = uc.datestring_to_ms(df[3][4][1:],
self.date_format_string, self.tz_info, self.daylight_savings)
self.frequency = 1000 / (second_stamp - self.data_start)
self.pressure_data = df[5].values * uc.PSI_TO_DBAR
start_ms = uc.datestring_to_ms('%s' % df[3][0][1:], self.date_format_string, self.tz_info, self.daylight_savings)
except:
self.data_start = uc.datestring_to_ms(df[3][3][1:],
self.date_format_string2, self.tz_info, self.daylight_savings)
second_stamp = uc.datestring_to_ms(df[3][4][1:],
self.date_format_string2, self.tz_info, self.daylight_savings)
self.frequency = 1000 / (second_stamp - self.data_start)
self.pressure_data = df[5].values * uc.PSI_TO_DBAR
start_ms = uc.datestring_to_ms('%s' % df[3][0][1:], self.date_format_string2, self.tz_info, self.daylight_savings)
self.utc_millisecond_data = uc.generate_ms(start_ms, df.shape[0], self.frequency)
def read(self):
'''load the data from in_filename
only parse the initial datetime = much faster
'''
self.get_serial()
second = False
skip_index = find_first(self.in_filename, '"#"')
if skip_index == None:
skip_index = find_first(self.in_filename, '#')
second = True
df = pd.read_table(self.in_filename,
skiprows=skip_index,
header=None,
engine='c',
sep=',',
usecols=(1, 2))
df = df.dropna()
try:
first_stamp = uc.datestring_to_ms(df.values[0][0],
self.date_format_string,
self.tz_info,
self.daylight_savings)
second_stamp = uc.datestring_to_ms(df.values[1][0],
self.date_format_string,
self.tz_info,
self.daylight_savings)
except:
first_stamp = uc.datestring_to_ms(df.values[0][0],
self.date_format_string2,
self.tz_info,
self.daylight_savings)
second_stamp = uc.datestring_to_ms(df.values[1][0],
self.date_format_string2,
self.tz_info,
self.daylight_savings)
self.frequency = 1000 / (second_stamp - first_stamp)
try:
start_ms = uc.datestring_to_ms(df[1][0], self.date_format_string,
self.tz_info, self.daylight_savings)
except:
start_ms = uc.datestring_to_ms(df[1][0], self.date_format_string2,
self.tz_info, self.daylight_savings)
self.utc_millisecond_data = uc.generate_ms(start_ms, df.shape[0],
self.frequency)
# if self.daylight_savings == True:
# self.utc_millisecond_data = [x - 3600000 for x in self.utc_millisecond_data]
self.pressure_data = df[2].values * uc.PSI_TO_DBAR
def read(self):
'''load the data from in_filename
only parse the initial datetime = much faster
'''
skip_index = find_first(self.in_filename, '^[0-9]{2}-[A-Z]{1}[a-z]{2,8}-[0-9]{4}')
df = pd.read_csv(self.in_filename, skiprows=skip_index, delim_whitespace=True,
header=None, engine='c', usecols=[0, 1, 2])
self.datestart = uc.datestring_to_ms('%s %s' % (df[0][0], df[1][0]), self.date_format_string)
self.utc_millisecond_data = uc.generate_ms(self.datestart, df.shape[0] - 1,
self.frequency)
self.pressure_data = np.array([x for x in df[2][:-1]])
def read(self):
'''load the data from in_filename
only parse the initial datetime = much faster
'''
self.get_serial()
skip_index = find_first(self.in_filename, '^ID') - 1
# for skipping lines in case there is calibration header data
df = pd.read_table(self.in_filename,
skiprows=skip_index + 1,
header=None,
engine='c',
sep=',',
usecols=[3, 4, 5])
try:
self.data_start = uc.datestring_to_ms(df[3][3][1:],
self.date_format_string,
self.tz_info,
self.daylight_savings)
second_stamp = uc.datestring_to_ms(df[3][4][1:],
self.date_format_string,
self.tz_info,
self.daylight_savings)
self.frequency = 1000 / (second_stamp - self.data_start)
self.pressure_data = df[5].values * uc.PSI_TO_DBAR
start_ms = uc.datestring_to_ms('%s' % df[3][0][1:],
self.date_format_string,
self.tz_info, self.daylight_savings)
except:
self.data_start = uc.datestring_to_ms(df[3][3][1:],
self.date_format_string2,
self.tz_info,
self.daylight_savings)
second_stamp = uc.datestring_to_ms(df[3][4][1:],
self.date_format_string2,
self.tz_info,
self.daylight_savings)
self.frequency = 1000 / (second_stamp - self.data_start)
self.pressure_data = df[5].values * uc.PSI_TO_DBAR
start_ms = uc.datestring_to_ms('%s' % df[3][0][1:],
self.date_format_string2,
self.tz_info, self.daylight_savings)
self.utc_millisecond_data = uc.generate_ms(start_ms, df.shape[0],
self.frequency)
def quick_dirty_wind_data(in_file_name, out_file_name):
df = pd.read_csv(in_file_name, header=None)
#generate 6 minute utc millisecond data
#millisecond is a time stamp for Fri Jan 22 2016 23:00:00
time = unit_conversion.generate_ms(1453503600000.0, 1213, 1/360)
wind_direction = df[6]
wind_speed = df[8]
ds = Dataset(out_file_name, 'w', format="NETCDF4_CLASSIC")
ds.createDimension('time',len(time))
time_var = ds.createVariable('time','f8',('time'))
time_var[:] = time
wind_speed_var = ds.createVariable('wind_speed','f8',('time'))
wind_speed_var[:] = wind_speed.values
wind_direction_var = ds.createVariable('wind_direction','f8',('time'))
wind_direction_var[:] = wind_direction.values
ds.close()
def quick_dirty_wind_data(in_file_name, out_file_name):
df = pd.read_csv(in_file_name, header=None)
#generate 6 minute utc millisecond data
#millisecond is a time stamp for Fri Jan 22 2016 23:00:00
time = unit_conversion.generate_ms(1453503600000.0, 1213, 1 / 360)
wind_direction = df[6]
wind_speed = df[8]
ds = Dataset(out_file_name, 'w', format="NETCDF4_CLASSIC")
ds.createDimension('time', len(time))
time_var = ds.createVariable('time', 'f8', ('time'))
time_var[:] = time
wind_speed_var = ds.createVariable('wind_speed', 'f8', ('time'))
wind_speed_var[:] = wind_speed.values
wind_direction_var = ds.createVariable('wind_direction', 'f8', ('time'))
wind_direction_var[:] = wind_direction.values
ds.close()
def read(self):
'''load the data from in_filename
only parse the initial datetime = much faster
'''
skip_index = find_first(self.in_filename,
'^[0-9]{2}-[A-Z]{1}[a-z]{2,8}-[0-9]{4}')
df = pd.read_csv(self.in_filename,
skiprows=skip_index,
delim_whitespace=True,
header=None,
engine='c',
usecols=[0, 1, 2])
self.datestart = uc.datestring_to_ms('%s %s' % (df[0][0], df[1][0]),
self.date_format_string)
self.utc_millisecond_data = uc.generate_ms(self.datestart,
df.shape[0] - 1,
self.frequency)
self.pressure_data = np.array([x for x in df[2][:-1]])
def read(self):
'''load the data from in_filename
only parse the initial datetime = much faster
'''
self.get_serial()
skip_index = find_first(self.in_filename, 'Date and Time,Seconds')
data = pd.read_table(self.in_filename, skiprows=skip_index, header=None,
engine='c', sep=',', usecols=(0,1,2,3))
self.data_start = uc.datestring_to_ms(data[0][1], self.date_format_string,
self.tz_info, self.daylight_savings)
self.data_start2 = uc.datestring_to_ms(data[0][2], self.date_format_string,
self.tz_info, self.daylight_savings)
self.frequency = 1 / ((self.data_start2 - self.data_start) / 1000)
self.utc_millisecond_data = uc.generate_ms(self.data_start, len(data[0]),
self.frequency)
self.pressure_data = data[3].values * uc.PSI_TO_DBAR
def read(self):
'''Load the data from in_filename'''
skip_index = find_first(self.in_filename, '^[0-9]{4},[0-9]{4}$') - 1
df = pd.read_table(self.in_filename, skiprows=skip_index, header=None,
engine='c', sep=',', names=('a', 'b'))
self.pressure_data = np.array([
uc.USGS_PROTOTYPE_V_TO_DBAR(np.float64(x))
for x in df[df.b.isnull() == False].a])
self.temperature_data = [
uc.USGS_PROTOTYPE_V_TO_C(np.float64(x))
for x in df[df.b.isnull() == False].b]
with open(self.in_filename, 'r') as wavelog:
for x in wavelog:
# second arg has extra space that is unnecessary
if re.match('^[0-9]{4}.[0-9]{2}.[0-9]{2}', x):
start_ms = uc.datestring_to_ms(x, self.date_format_string)
self.utc_millisecond_data = uc.generate_ms(start_ms,
len(self.pressure_data),
self.frequency)
break
def read(self):
'''load the data from in_filename
only parse the initial datetime = much faster
'''
self.get_serial()
second = False
skip_index = find_first(self.in_filename, '"#"')
if skip_index == None:
skip_index = find_first(self.in_filename, '#')
second = True
df = pd.read_table(self.in_filename, skiprows=skip_index, header=None,
engine='c', sep=',', usecols=(1, 2))
df = df.dropna()
try:
first_stamp = uc.datestring_to_ms(df.values[0][0], self.date_format_string, self.tz_info, self.daylight_savings)
second_stamp = uc.datestring_to_ms(df.values[1][0], self.date_format_string, self.tz_info, self.daylight_savings)
except:
first_stamp = uc.datestring_to_ms(df.values[0][0], self.date_format_string2, self.tz_info, self.daylight_savings)
second_stamp = uc.datestring_to_ms(df.values[1][0], self.date_format_string2, self.tz_info, self.daylight_savings)
self.frequency = 1000 / (second_stamp - first_stamp)
try:
start_ms = uc.datestring_to_ms(df[1][0], self.date_format_string, self.tz_info, self.daylight_savings)
except:
start_ms = uc.datestring_to_ms(df[1][0], self.date_format_string2, self.tz_info, self.daylight_savings)
self.utc_millisecond_data = uc.generate_ms(start_ms, df.shape[0], self.frequency)
# if self.daylight_savings == True:
# self.utc_millisecond_data = [x - 3600000 for x in self.utc_millisecond_data]
self.pressure_data = df[2].values * uc.PSI_TO_DBAR
def change_netCDFTime(in_file_name, out_file_name, start_ms):
shutil.copy(in_file_name, out_file_name)
time_len = len(nc.get_time(out_file_name))
new_time = unit_conversion.generate_ms(start_ms, time_len, 1/900)
nc.set_variable_data(out_file_name, 'time', new_time)
def change_netCDFTime(in_file_name, out_file_name, start_ms):
shutil.copy(in_file_name, out_file_name)
time_len = len(nc.get_time(out_file_name))
new_time = unit_conversion.generate_ms(start_ms, time_len, 1 / 900)
nc.set_variable_data(out_file_name, 'time', new_time)