def __init__(self,
rld_dir='',
out_dir='',
site_filter='',
encryption_pass='',
token='',
header_type='standard',
export_type='meas',
export_format='csv_zipped',
**kwargs):
if check_platform() == 'win32':
self.platform = 'win32'
self.folder_split = '\\'
self.rld_dir = windows_folder_path(rld_dir)
self.out_dir = windows_folder_path(out_dir)
else:
self.platform = 'linux'
self.folder_split = '/'
self.rld_dir = linux_folder_path(rld_dir)
self.out_dir = linux_folder_path(out_dir)
self.encryption_pass = encryption_pass
self.export_format = export_format
self.export_type = export_type
self.site_filter = site_filter
self.header_type = header_type
self.token = token
#import nrgApiUrl, token as tk
self.NrgUrl = nrgApiUrl
if len(tk) > 10 and len(self.token) < 10:
self.token = tk
if self.token == '':
print(
'\n\nA valid token is required to use the nrg_convert_api.\nPlease contact [email protected] for an API token'
)
def _copy_txt_file(self):
"""copy TXT file from self.ScaledData to self.out_dir"""
try:
txt_file_name = os.path.basename(self._filename)[:-4] + '.txt'
txt_file_path = os.path.join(self.ScaledData,txt_file_name)
out_path = self.file_path_joiner.join([self.out_dir,txt_file_name])
except:
print("could not do the needful")
if self.platform == 'linux':
out_path = linux_folder_path(self.out_dir) + txt_file_name
txt_file_path = ''.join([self.wine_folder, '/NRG/ScaledData/',txt_file_name])
try:
shutil.copy(txt_file_path, out_path)
try:
os.remove(txt_file_path)
except:
print("{0} remains in {1}".format(txt_file_name, self.ScaledData))
except:
import traceback
print(traceback.format_exc())
print("Unable to copy {0} to {1}".format(txt_file_name,self.out_dir))
def __init__(self, rwd_dir='', out_dir='', filename='', encryption_pin='',
sdr_path=r'C:/NRG/SymDR/SDR.exe',
convert_type='meas', site_filter='',
wine_folder='~/.wine/drive_c/', **kwargs):
if encryption_pin != '':
self.command_switch = '/z'
else:
self.command_switch = '/q'
self.encryption_pin = encryption_pin
self.sdr_path = windows_folder_path(sdr_path)[:-1]
self.root_folder = "\\".join(self.sdr_path.split('\\')[:-2])
self.RawData = self.root_folder + '\\RawData\\'
self.ScaledData = self.root_folder + '\\ScaledData\\'
self.site_filter = site_filter
self.rwd_dir = windows_folder_path(rwd_dir) # rwd_dir must be in Windows format, even if using Wine
self.platform = check_platform()
if self.platform == 'win32':
self.out_dir = windows_folder_path(out_dir)
self.file_path_joiner = '\\'
pass
else:
self.out_dir = linux_folder_path(out_dir)
self.check_sdr()
self.file_path_joiner = '/'
if filename != '':
self.filename = filename
self.single_file()
self.wine_folder = wine_folder
def list_files(self):
"""
get list of files in rwd_dir
"""
self.dir_paths = []
self.rwd_file_list = []
if self.platform == 'win32':
walk_path = self.rwd_dir
else:
walk_path = linux_folder_path(self.rwd_dir)
for dirpath, subdirs, files in os.walk(walk_path):
self.dir_paths.append(dirpath)
for x in files:
if x.startswith(self.site_filter) and x.lower().endswith('.rwd'):
self.rwd_file_list.append(x)
def __init__(self, rwd_dir='', out_dir='', filename='', encryption_pin='',
sdr_path=r'C:/NRG/SymDR/SDR.exe',
convert_type='meas', file_filter='',
wine_folder='~/.wine/drive_c/',
use_site_file=False, raw_mode=False, progress_bar=True, show_result=True, **kwargs):
if encryption_pin != '':
self.command_switch = '/z' # noqueue with pin
else:
self.command_switch = '/q' # noqueue (logger params)
if use_site_file == True:
self.command_switch = '/s' # silent (site file params)
if raw_mode == True:
self.command_switch = '/r' # silent (site file params)
self.filename = filename
self.progress_bar = progress_bar
self.encryption_pin = encryption_pin
self.sdr_path = windows_folder_path(sdr_path)[:-1]
self.root_folder = "\\".join(self.sdr_path.split('\\')[:-2])
self.RawData = self.root_folder + '\\RawData\\'
self.ScaledData = self.root_folder + '\\ScaledData\\'
self.file_filter = file_filter
if 'site_filter' in kwargs and file_filter == '':
self.file_filter = kwargs.get('site_filter')
self.rwd_dir = windows_folder_path(rwd_dir) # rwd_dir must be in Windows format, even if using Wine
self.show_result = show_result
self.platform = check_platform()
self.wine_folder = wine_folder
self.check_sdr()
if self.platform == 'win32':
self.out_dir = windows_folder_path(out_dir)
self.file_path_joiner = '\\'
else:
self.out_dir = linux_folder_path(out_dir)
self.file_path_joiner = '/'
if self.filename:
self.counter = 1
self.rwd_dir = os.path.dirname(self.filename)
self.file_filter = os.path.basename(self.filename)
self.convert()
def copy_txt_file(self):
"""
copy TXT file from self.ScaledData to self.out_dir
"""
try:
txt_file_name = self._filename.split('\\')[-1][:-4] + '.txt'
txt_file_path = "\\".join([self.ScaledData,txt_file_name])
out_path = self.file_path_joiner.join([self.out_dir,txt_file_name])
except:
print("could not do the needful")
if self.platform == 'linux':
out_path = linux_folder_path(self.out_dir) + txt_file_name
txt_file_path = ''.join([self.wine_folder, 'NRG/ScaledData/',txt_file_name])
try:
shutil.copy(txt_file_path, out_path)
try:
os.remove(txt_file_path)
except:
print("{0} remains in {1}".format(txt_file_name, self.ScaledData))
except:
print("Unable to copy {0} to {1}".format(txt_file_name,self.out_dir))
def concat_txt(self,
output_txt=False,
txt_dir='',
out_file='',
file_type='meas',
header='standard',
site_filter='',
**kwargs):
"""
Will concatenate all text files in the txt_dir that match
the site_filter argument. Note these are both blank by default.
"""
self.site_filter = site_filter
self.file_type = file_type
if check_platform() == 'win32':
self.txt_dir = windows_folder_path(txt_dir)
else:
self.txt_dir = linux_folder_path(txt_dir)
first_file = True
files = sorted(glob(self.txt_dir + '*.txt'))
for f in files:
if self.site_filter in f and self.file_type in f:
print("Adding {0} ...\t\t".format(f), end="", flush=True)
if first_file == True:
first_file = False
try:
base = sympro_txt_read(f)
print("[OK]")
pass
except IndexError:
print('Only standard SymPRO headertypes accepted')
break
else:
file_path = f
try:
s = sympro_txt_read(file_path)
base.data = base.data.append(s.data, sort=False)
print("[OK]")
except:
print("[FAILED]")
print("could not concat {0}".format(file_path))
pass
else:
pass
if output_txt == True:
if out_file == "":
out_file = datetime.today().strftime(
"%Y-%m-%d") + "_SymPRO.txt"
base.data.to_csv(txt_dir + out_file, sep=',', index=False)
self.out_file = out_file
try:
self.ch_info = s.ch_info
self.ch_list = s.ch_list
self.data = base.data.drop_duplicates(subset=['Timestamp'],
keep='first')
self.head = s.head
self.site_info = s.site_info
except UnboundLocalError:
print("No files match to contatenate.")
return None
self.ch_info = s.ch_info
self.ch_list = s.ch_list
self.data = base.data.drop_duplicates(subset=['Timestamp'],
keep='first')
self.head = s.head
self.site_info = s.site_info
def concat(self, output_txt=False, out_file='', file_filter=''):
"""
combine exported rwd files (in txt format)
"""
self.file_filter = file_filter
if check_platform() == 'win32':
self.txt_dir = windows_folder_path(self.txt_dir)
else:
self.txt_dir = linux_folder_path(self.txt_dir)
first_file = True
files = sorted(glob(self.txt_dir + '*.txt'))
self.file_count = len(files)
self.pad = len(str(self.file_count)) + 1
# probably create dataframe of any channel changes
## presence of log file indicates sensor change? maybe?
self.counter = 1
for f in files:
if self.file_filter in f:
print("Adding {0}/{1} {2} ... ".format(
str(self.counter).rjust(self.pad),
str(self.file_count).ljust(self.pad), f),
end="",
flush=True)
if first_file == True:
first_file = False
try:
base = read_text_data(filename=f,
data_type=self.data_type,
file_filter=self.file_filter,
file_ext=self.file_ext,
sep=self.sep)
print("[OK]")
pass
except IndexError:
print('Only standard headertypes accepted')
break
else:
file_path = f
try:
s = read_text_data(filename=f,
data_type=self.data_type,
file_filter=self.file_filter,
file_ext=self.file_ext,
sep=self.sep)
base.data = base.data.append(s.data, sort=False)
print("[OK]")
except:
print("[FAILED]")
print("could not concat {0}".format(file_path))
pass
else:
pass
self.counter += 1
if output_txt == True:
if out_file == "":
out_file = datetime.today().strftime(
"%Y-%m-%d") + "_SymPRO.txt"
base.data.to_csv(txt_dir + out_file, sep=',', index=False)
self.out_file = out_file
try:
self.ch_info = s.ch_info
self.ch_list = s.ch_list
self.data = base.data.drop_duplicates(
subset=[self.header_sections['data_header']], keep='first')
#self.head = s.head
self.site_info = s.site_info
except UnboundLocalError:
print("No files match to contatenate.")
return None
self.ch_info = s.ch_info
self.ch_list = s.ch_list
self.data = base.data.drop_duplicates(
subset=[self.header_sections['data_header']], keep='first')
#self.head = s.head
self.site_info = s.site_info
def concat_txt(self,
txt_dir='',
output_txt=False,
out_file='',
file_filter='',
file_filter2='',
start_date='1970-01-01',
end_date='2150-12-31',
progress_bar=True):
"""
"""
from glob import glob
self.txt_dir = txt_dir
self.output_txt = output_txt
self.out_file = out_file
self.file_filter = file_filter
self.file_filter2 = file_filter2
self.start_date = start_date
self.end_date = end_date
if check_platform() == 'win32':
self.txt_dir = windows_folder_path(txt_dir)
else:
self.txt_dir = linux_folder_path(txt_dir)
first_file = True
files = [
f for f in sorted(glob(self.txt_dir + "*"))\
if self.file_filter in f and self.file_filter2 in f\
and date_check(self.start_date, self.end_date, f)
]
self.file_count = len(files)
self.pad = len(str(self.file_count))
self.counter = 1
self.start_time = datetime.now()
for f in files:
if self.file_filter in f and self.file_filter2 in f:
if progress_bar:
draw_progress_bar(self.counter, self.file_count,
self.start_time)
else:
print("Adding {0}/{1} ... {2} ".format(
str(self.counter).rjust(self.pad),
str(self.file_count).ljust(self.pad), f),
end="",
flush=True)
if first_file == True:
first_file = False
try:
base = spidar_data_read(f)
if progress_bar == False: print("[OK]")
pass
except IndexError:
print('Only standard Spidar headertypes accepted')
break
else:
file_path = f
try:
s = spidar_data_read(file_path)
base.data = base.data.append(s.data, sort=False)
if progress_bar == False: print("[OK]")
except Exception as e:
if progress_bar == False: print("[FAILED]")
print("could not concat {0}".format(file_path))
print(e)
pass
else:
pass
self.counter += 1
if out_file != "":
self.out_file = out_file
if output_txt == True:
base.data.to_csv(txt_dir + out_file, sep=',', index=False)
try:
self.base = base
self.heights = base.heights
self.data = base.data.drop_duplicates(subset=['Timestamp'],
keep='first')
self.data.reset_index(drop=True, inplace=True)
except Exception as e:
print("No files match to contatenate.")
print(e)
return None
def concat(self, output_txt=False, out_file='', file_filter='', filter2='', progress_bar=True):
"""combine exported rwd files (in txt format)
parameters
----------
output_txt : bool
set to True to save a concatenated text file
out_file : str
filepath, absolute or relative
file_filter : str
filter2 : str
progress_bar : bool
"""
self.file_filter = file_filter
if self.filter2 == '':
self.filter2 = filter2
if check_platform() == 'win32':
self.txt_dir = windows_folder_path(self.txt_dir)
else:
self.txt_dir = linux_folder_path(self.txt_dir)
first_file = True
files = sorted(glob(self.txt_dir + '*.txt'))
self.file_count = len(files)
self.pad = len(str(self.file_count)) + 1
self.counter = 1
self.start_time = datetime.now()
for f in files:
if self.file_filter in f and self.filter2 in f:
if progress_bar:
draw_progress_bar(self.counter, self.file_count, self.start_time)
else:
print("Adding {0}/{1} {2} ... ".format(str(self.counter).rjust(self.pad),str(self.file_count).ljust(self.pad),f), end="", flush=True)
if first_file == True:
first_file = False
try:
base = read_text_data(filename=f,data_type=self.data_type,
file_filter=self.file_filter, file_ext=self.file_ext,
sep=self.sep)
if progress_bar != True: print("[OK]")
pass
except IndexError:
print('Only standard headertypes accepted')
break
else:
file_path = f
try:
s = read_text_data(filename=f,data_type=self.data_type,
file_filter=self.file_filter, file_ext=self.file_ext,
sep=self.sep)
base.data = base.data.append(s.data, sort=False)
if progress_bar != True: print("[OK]")
except:
if progress_bar != True: print("[FAILED]")
print("could not concat {0}".format(file_path))
pass
else:
pass
self.counter += 1
if output_txt == True:
if out_file == "":
out_file = f"{self.data_type}_" + datetime.today().strftime("%Y-%m-%d") + ".txt"
base.data.to_csv(out_file, sep=',', index=False)
self.out_file = out_file
try:
self.ch_info = s.ch_info
self.ch_list = s.ch_list
self.data = base.data.drop_duplicates(subset=[self.header_sections['data_header']], keep='first')
self.head = s.head
self.site_info = s.site_info
self.filename = s.filename
self.site_number = self.filename.split("\\")[-1][:4]
self.format_rwd_site_data()
except UnboundLocalError:
print("No files match to contatenate.")
return None
def concat_txt(self,
txt_dir='',
file_type='meas',
file_filter='',
filter2='',
start_date='1970-01-01',
end_date='2150-12-31',
ch_details=False,
output_txt=False,
out_file='',
progress_bar=True,
**kwargs):
"""Will concatenate all text files in the txt_dir
files must match the site_filter argument. Note these are both blank by default.
Parameters
----------
txt_dir : str
directory holding txt files
file_type : str
type of export (meas, event, comm, sample, etc...)
file_filter : str
text filter for txt files, like site number, etc.
filter2 : str
secondary text filter
start_date : str
for filtering files to concat based on date "YYYY-mm-dd"
end_date : str
for filtering files to concat based on date "YYYY-mm-dd"
ch_details : bool
show additional info in ch_info dataframe
output_txt : bool
create a txt output of data df
out_file : str
filename to write data dataframe too if output_txt = True
progress_bar : bool
show bar on concat [True] or list of files [False]
Returns
---------
ch_info : obj
pandas dataframe of ch_list (below) pulled out of file with sympro_txt_read.arrange_ch_info()
ch_list : list
list of channel info; can be converted to json w/ import json ... json.dumps(fut.ch_info)
data : obj
pandas dataframe of all data
head : obj
lines at the top of the txt file..., used when rebuilding timeshifted files
site_info : obj
pandas dataframe of site information
logger_sn : str
ipack_sn : str
logger_type : str
ipack_type : str
latitude : float
longitude : float
elevation : int
site_number : str
site_description : str
start_date : str
txt_file_names : list
list of files included in concatenation
Examples
--------
Read files into nrgpy reader object
>>> import nrgpy
>>> reader = nrgpy.sympro_txt_read()
>>> reader.concat_txt(
txt_dir='/path/to/txt/files/',
file_filter='123456', # site 123456
start_date='2020-01-01',
end_date='2020-01-31',
)
Time elapsed: 2 s | 33 / 33 [=============================================] 100%
Queue processed
>>> reader.logger_sn
'820600019'
>>> reader.ch_info
Bearing: Channel: Description: Effective Date: Height: Offset: Scale Factor: Serial Number: Type: Units:
0 50.00 1 NRG S1 2020-01-31 00:00:00 33.00 0.13900 0.09350 94120000059 Anemometer m/s
1 230.00 2 NRG S1 2020-01-31 00:00:00 0.00 0.13900 0.09350 94120000058 Anemometer m/s
2 50.00 3 NRG S1 2020-01-31 00:00:00 22.00 0.13900 0.09350 94120000057 Anemometer m/s
3 230.00 4 NRG 40C Anem 2020-01-31 00:00:00 22.00 0.35000 0.76500 179500324860 Anemometer m/s
4 50.00 5 NRG 40C Anem 2020-01-31 00:00:00 12.00 0.35000 0.76500 179500324859 Anemometer m/s
5 230.00 6 NRG S1 2020-01-31 00:00:00 12.00 0.13900 0.09350 94120000056 Anemometer m/s
6 320.00 13 NRG 200M Vane 2020-01-31 00:00:00 32.00 -1.46020 147.91100 10700000125 Vane Deg
7 320.00 14 NRG 200M Vane 2020-01-31 00:00:00 21.00 -1.46020 147.91100 10700000124 Vane Deg
8 0.00 15 NRG T60 Temp 2020-01-31 00:00:00 34.00 -40.85550 44.74360 9400000705 Analog C
9 0.00 16 NRG T60 Temp 2020-01-31 00:00:00 2.00 -40.85550 44.74360 9400000xxx Analog C
10 0.00 17 NRG RH5X Humi 2020-01-31 00:00:00 0.00 0.00000 20.00000 NaN Analog %RH
11 0.00 20 NRG BP60 Baro 2020-01-31 00:00:00 0.00 495.27700 243.91400 NaN Analog hPa
12 0.00 21 NRG BP60 Baro 2020-01-31 00:00:00 2.00 495.04400 244.23900 9396FT1937 Analog hPa
"""
if 'site_filter' in kwargs and file_filter == '':
self.file_filter = kwargs.get('site_filter')
else:
self.file_filter = file_filter
self.ch_details = ch_details
self.start_date = start_date
self.end_date = end_date
self.filter2 = filter2
self.file_type = file_type
self.txt_file_names = []
if check_platform() == 'win32':
self.txt_dir = windows_folder_path(txt_dir)
else:
self.txt_dir = linux_folder_path(txt_dir)
first_file = True
files = [
f for f in sorted(glob(self.txt_dir + '*.txt'))\
if self.file_filter in f and self.filter2 in f\
and date_check(self.start_date, self.end_date, f)
]
self.file_count = len(files)
self.pad = len(str(self.file_count))
self.counter = 1
self.start_time = datetime.now()
for f in files:
if self.file_filter in f and self.file_type in f and self.filter2 in f:
if progress_bar:
draw_progress_bar(self.counter, self.file_count,
self.start_time)
else:
print("Adding {0}/{1} ... {2} ... ".format(
str(self.counter).rjust(self.pad),
str(self.file_count).ljust(self.pad),
os.path.basename(f)),
end="",
flush=True)
if first_file == True:
first_file = False
try:
base = sympro_txt_read(f)
if progress_bar != True: print("[OK]")
self.txt_file_names.append(os.path.basename(f))
except IndexError:
print('Only standard SymPRO headertypes accepted')
break
else:
file_path = f
try:
s = sympro_txt_read(file_path,
ch_details=self.ch_details)
base.data = base.data.append(s.data, sort=False)
if progress_bar != True: print("[OK]")
self.txt_file_names.append(os.path.basename(f))
except:
if progress_bar != True: print("[FAILED]")
print("could not concat {0}".format(
os.path.basename(file_path)))
pass
else:
pass
self.counter += 1
if out_file != "":
self.out_file = out_file
if output_txt == True:
base.data.to_csv(os.path.join(txt_dir, out_file),
sep=',',
index=False)
try:
self.ch_info = s.ch_info
self.ch_list = s.ch_list
self.array = s.array
self.data = base.data.drop_duplicates(subset=['Timestamp'],
keep='first')
self.data.reset_index(drop=True, inplace=True)
self.head = s.head
self.site_info = s.site_info
self.format_site_data()
print("\n")
except UnboundLocalError:
print("No files match to contatenate.")
return None
def __init__(self, rld_dir='', out_dir='', filename='', site_filter='',
filter2 = '', start_date='1970-01-01', end_date='2150-12-31',
client_id='', client_secret='', token='',
encryption_pass='', header_type='standard', nec_file='',
export_type='meas', export_format='csv_zipped', **kwargs):
if check_platform() == 'win32':
self.platform = 'win32'
self.folder_split = '\\'
self.rld_dir = windows_folder_path(rld_dir)
self.out_dir = windows_folder_path(out_dir)
else:
self.platform = 'linux'
self.folder_split = '/'
self.rld_dir = linux_folder_path(rld_dir)
self.out_dir = linux_folder_path(out_dir)
self.encryption_pass = encryption_pass
self.export_format = export_format
self.export_type = export_type
self.site_filter = site_filter
if 'file_filter' in kwargs and site_filter == '':
self.file_filter = kwargs.get('file_filter')
self.site_filter = self.file_filter
self.filter2 = filter2
self.start_date = start_date
self.end_date = end_date
self.header_type = header_type
self.nec_file = nec_file
self.token = token
self.client_id = client_id
self.client_secret = client_secret
affirm_directory(self.out_dir)
if self.client_id and self.client_secret:
self.session_token, self.session_start_time = maintain_session_token(client_id=self.client_id, client_secret=self.client_secret)
if self.session_token:
self.convert_url = ConvertServiceUrl
else:
print("Unable to get session token for conversion")
else:
self.convert_url = nrgApiUrl
print("[Deprecation warning]\n------------------------------------------------------")
print(" NRGPy Convert API will require a client_id and \n client_secret starting March 15, 2020.")
print(" Please contact [email protected] or visit \n https://services.nrgsystems.com to sign up.")
print("------------------------------------------------------\n")
if len(tk) > 10 and len(self.token) < 10:
self.token = tk
if (not self.token) and (not self.client_id) and (not self.client_secret):
print('[Access error] Valid credentials are required.\nPlease contact [email protected] or visit \nhttps://services.nrgsystems.com for API access')
if filename != '':
self.pad = 1
self.counter = 1
self.raw_count = 1
self.progress_bar=False
self.start_time = datetime.now()
self.single_file(filename)
def concat_txt(self,
output_txt=False,
txt_dir='',
out_file='',
file_type='meas',
header='standard',
file_filter='',
**kwargs):
"""
Will concatenate all text files in the txt_dir that match
the site_filter argument. Note these are both blank by default.
"""
if 'ch_details' in kwargs:
self.ch_details = kwargs.get('ch_details')
if 'site_filter' in kwargs and file_filter == '':
self.file_filter = kwargs.get('site_filter')
else:
self.file_filter = file_filter
self.file_type = file_type
if check_platform() == 'win32':
self.txt_dir = windows_folder_path(txt_dir)
else:
self.txt_dir = linux_folder_path(txt_dir)
first_file = True
files = [
f for f in sorted(glob(self.txt_dir + '*.txt'))
if self.file_filter in f
]
self.file_count = len(files)
self.pad = len(str(self.file_count))
self.counter = 1
for f in files:
if self.file_filter in f and self.file_type in f:
print("Adding {0}/{1} ... {2}".format(
str(self.counter).rjust(self.pad),
str(self.file_count).ljust(self.pad), f),
end="",
flush=True)
if first_file == True:
first_file = False
try:
base = sympro_txt_read(f)
print("[OK]")
pass
except IndexError:
print('Only standard SymPRO headertypes accepted')
break
else:
file_path = f
try:
s = sympro_txt_read(file_path,
ch_details=self.ch_details)
base.data = base.data.append(s.data, sort=False)
print("[OK]")
except:
print("[FAILED]")
print("could not concat {0}".format(file_path))
pass
else:
pass
self.counter += 1
if out_file != "":
self.out_file = out_file
if output_txt == True:
base.data.to_csv(txt_dir + out_file, sep=',', index=False)
try:
self.ch_info = s.ch_info
self.ch_list = s.ch_list
self.array = s.array
self.data = base.data.drop_duplicates(subset=['Timestamp'],
keep='first')
self.data.reset_index(drop=True, inplace=True)
self.head = s.head
self.site_info = s.site_info
except UnboundLocalError:
print("No files match to contatenate.")
return None
