def make_mat_files(start_time, end_time='Now', data_folder='/user/slops/data/SPS_DATA/OP_DATA/multiQ/',
SPSuser=None, filename_converted='multiq_converted.txt'):
if type(start_time) is str:
start_tstamp_unix = th.localtime2unixstamp(start_time)
else:
start_tstamp_unix = start_time
if end_time == 'Now':
end_tstamp_unix = time.mktime(time.localtime())
elif type(end_time) is str:
end_tstamp_unix = th.localtime2unixstamp(end_time)
else:
end_tstamp_unix = end_time
try:
with open(filename_converted, 'r') as fid:
list_converted = fid.read().split('\n')
except IOError:
list_converted = []
list_date_strings = th.date_strings_interval(start_tstamp_unix, end_tstamp_unix)
sdds_folder_list = []
for date_string in list_date_strings:
sdds_folder_list.extend(glob.glob(data_folder + date_string))
for sdds_folder in sdds_folder_list:
print '\nConverting data in folder: %s\n'%sdds_folder
file_list = os.listdir(sdds_folder)
for filename in file_list:
# print filename
time_string = ' '.join(filename.split('.')[0].split('_')[-2:])
tstamp_filename = int(time.mktime(datetime.datetime.strptime(time_string, "%d%b%y %H-%M-%S").timetuple()))
if not(tstamp_filename > start_tstamp_unix and tstamp_filename < end_tstamp_unix):
continue
if SPSuser != None:
user_filename = filename.split('_')[1]
if user_filename != SPSuser:
continue
if filename in list_converted:
continue
try:
complete_path = sdds_folder + '/' + filename
print complete_path
sdds_to_file(complete_path)
with open(filename_converted, 'a+') as fid:
fid.write(filename+'\n')
except Exception as err:
print 'Skipped:'
print complete_path
print 'Got exception:'
print err
def make_mat_files(start_time, end_time='Now', data_folder='/user/slops/data/SPS_DATA/OP_DATA/QMETER/',
SPSuser=None, filename_converted='qmeter_converted.txt'):
if type(start_time) is str:
start_tstamp_unix = th.localtime2unixstamp(start_time)
else:
start_tstamp_unix = start_time
if end_time == 'Now':
end_tstamp_unix = time.mktime(time.localtime())
elif type(end_time) is str:
end_tstamp_unix = th.localtime2unixstamp(end_time)
else:
end_tstamp_unix = end_time
try:
with open(filename_converted, 'r') as fid:
list_converted = fid.read().split('\n')
except IOError:
list_converted = []
list_date_strings = th.date_strings_interval(start_tstamp_unix, end_tstamp_unix)
sdds_folder_list = []
for date_string in list_date_strings:
sdds_folder_list.extend(glob.glob(data_folder + date_string + '/SPSBQSB2@Acquisition/'))
for sdds_folder in sdds_folder_list:
print '\nConverting data in folder: %s\n'%sdds_folder
file_list = os.listdir(sdds_folder)
for filename in file_list:
tstamp_filename = int(float(filename.split('@')[-2]) / 1e9)
if not(tstamp_filename > start_tstamp_unix and tstamp_filename < end_tstamp_unix):
continue
if SPSuser != None:
user_filename = filename.split('.')[-2]
if user_filename != SPSuser:
continue
if filename in list_converted:
continue
try:
complete_path = sdds_folder + filename
print complete_path
sdds_to_file(complete_path)
with open(filename_converted, 'a+') as fid:
fid.write(filename+'\n')
except Exception as err:
print 'Skipped:'
print complete_path
print 'Got exception:'
print err
def make_mat_files(start_time, end_time='Now', data_folder='/user/slops/data/SPS_DATA/OP_DATA/RF-COUNTER/',
device_name=None, SPSuser=None, filename_converted='freq_converted.txt'):
if type(start_time) is str:
start_tstamp_unix = th.localtime2unixstamp(start_time)
else:
start_tstamp_unix = start_time
if end_time == 'Now':
end_tstamp_unix = time.mktime(time.localtime())
elif type(end_time) is str:
end_tstamp_unix = th.localtime2unixstamp(end_time)
else:
end_tstamp_unix = end_time
try:
with open(filename_converted, 'r') as fid:
list_converted = fid.read().split('\n')
except IOError:
list_converted = []
list_date_strings = th.date_strings_interval(start_tstamp_unix, end_tstamp_unix)
sdds_folder_list = []
for date_string in list_date_strings:
sdds_folder_list.extend(glob.glob(data_folder + date_string + '/DiagnosticCounter@MeasFrequency/'))
for sdds_folder in sdds_folder_list:
print '\nConverting data in folder: %s\n'%sdds_folder
file_list = os.listdir(sdds_folder)
for filename in file_list:
tstamp_filename = int(float(filename.split('@')[-2]) / 1e9)
if not(tstamp_filename > start_tstamp_unix and tstamp_filename < end_tstamp_unix):
continue
if SPSuser != None:
user_filename = filename.split('.')[-3]
if user_filename != SPSuser:
continue
if filename in list_converted:
continue
try:
complete_path = sdds_folder + filename
print complete_path
sdds_to_file(complete_path, SPSuser, tstamp_filename)
with open(filename_converted, 'a+') as fid:
fid.write(filename+'\n')
except Exception as err:
print 'Skipped:'
print complete_path
print 'Got exception:'
print err
def make_mat_files(start_time,
end_time='Now',
data_folder='/user/slops/data/SPS_DATA/OP_DATA/multiQ/',
SPSuser=None,
filename_converted='multiq_converted.txt'):
if type(start_time) is str:
start_tstamp_unix = th.localtime2unixstamp(start_time)
else:
start_tstamp_unix = start_time
if end_time == 'Now':
end_tstamp_unix = time.mktime(time.localtime())
elif type(end_time) is str:
end_tstamp_unix = th.localtime2unixstamp(end_time)
else:
end_tstamp_unix = end_time
try:
with open(filename_converted, 'r') as fid:
list_converted = fid.read().split('\n')
except IOError:
list_converted = []
list_date_strings = th.date_strings_interval(start_tstamp_unix,
end_tstamp_unix)
sdds_folder_list = []
for date_string in list_date_strings:
sdds_folder_list.extend(glob.glob(data_folder + date_string))
for sdds_folder in sdds_folder_list:
print '\nConverting data in folder: %s\n' % sdds_folder
file_list = os.listdir(sdds_folder)
for filename in file_list:
# print filename
time_string = ' '.join(filename.split('.')[0].split('_')[-2:])
tstamp_filename = int(
time.mktime(
datetime.datetime.strptime(time_string,
"%d%b%y %H-%M-%S").timetuple()))
if not (tstamp_filename > start_tstamp_unix
and tstamp_filename < end_tstamp_unix):
continue
if SPSuser != None:
user_filename = filename.split('_')[1]
if user_filename != SPSuser:
continue
if filename in list_converted:
continue
try:
complete_path = sdds_folder + '/' + filename
print complete_path
sdds_to_file(complete_path)
with open(filename_converted, 'a+') as fid:
fid.write(filename + '\n')
except Exception as err:
print 'Skipped:'
print complete_path
print 'Got exception:'
print err
def make_mat_files(start_time,
end_time='Now',
data_folder='/user/slops/data/SPS_DATA/OP_DATA/SPS_BCT/',
device_name=None,
SPSuser=None,
filename_converted='bct_converted.txt'):
if type(start_time) is str:
start_tstamp_unix = th.localtime2unixstamp(start_time)
else:
start_tstamp_unix = start_time
if end_time == 'Now':
end_tstamp_unix = time.mktime(time.localtime())
elif type(end_time) is str:
end_tstamp_unix = th.localtime2unixstamp(end_time)
else:
end_tstamp_unix = end_time
try:
with open(filename_converted, 'r') as fid:
list_converted = fid.read().split('\n')
except IOError:
list_converted = []
list_date_strings = th.date_strings_interval(start_tstamp_unix,
end_tstamp_unix)
sdds_folder_list = []
for date_string in list_date_strings:
if device_name == None:
sdds_folder_list.extend(
glob.glob(data_folder + date_string +
'/SPS.BCTDC.*@Acquisition/'))
else:
device_path = glob.glob(data_folder + date_string +
'/SPS.BCTDC.%s*@Acquisition/' %
device_name)
if len(device_path) == 0:
print 'No data for device %s on %s' % (device_name,
date_string)
else:
sdds_folder_list.extend(device_path)
for sdds_folder in sdds_folder_list:
print '\nConverting data in folder: %s\n' % sdds_folder
file_list = os.listdir(sdds_folder)
for filename in file_list:
tstamp_filename = int(float(filename.split('@')[-2]) / 1e9)
if not (tstamp_filename > start_tstamp_unix
and tstamp_filename < end_tstamp_unix):
continue
if SPSuser != None:
user_filename = filename.split('.')[-2]
if user_filename != SPSuser:
continue
if filename in list_converted:
continue
complete_path = sdds_folder + filename
print complete_path
sdds_to_file(complete_path)
with open(filename_converted, 'a+') as fid:
fid.write(filename + '\n')
try:
complete_path = sdds_folder + filename
print complete_path
sdds_to_file(complete_path)
with open(filename_converted, 'a+') as fid:
fid.write(filename + '\n')
except Exception as err:
print 'Skipped:'
print complete_path
print 'Got exception:'
print err