def print_parameter(self):
if not self.initialized:
utility.print_error(f'PRM NOT initialized')
raise Exception
utility.print_info(f'PRM print parameter')
for key, val in param_list.items():
utility.print_info(f'PRM key = {key + ",":18} ' + f'val = {val}')
def __init__(self):
utility.print_debug('CAL __init__')
param_man = param_manager.ParamManager()
calc_map = param_man.get('field_map')
min_dist = 1e10
with open(calc_map, 'r') as f:
for line in f:
if '#' in line:
continue
# utility.print_info(f'CAL {line}')
columns = line.split()
p = (numpy.round(float(columns[0]) * 1e3),
numpy.round(float(columns[2]) * 1e3),
numpy.round(-float(columns[1]) * 1e3))
__class__.field_map[p] = [
float(columns[3]),
float(columns[5]), -float(columns[4])
]
p = numpy.array([
float(columns[0]) * 1e3,
float(columns[2]) * 1e3, -float(columns[1]) * 1e3
])
d = numpy.linalg.norm(p - ref_pos)
if d < min_dist:
min_dist = d
min_pos = p
ref_calc = [
float(columns[3]),
float(columns[4]),
float(columns[5])
]
utility.print_info('CAL initialized')
def __init__(self):
utility.print_debug('FLD __init__')
param_man = param_manager.ParamManager()
data_path = param_man.get('data_path')
center_x = float(param_man.get('center_x'))
center_y = float(param_man.get('center_y'))
center_z = float(param_man.get('center_z'))
self.field_map = dict()
self.bscale = 1.0
process_time_array = []
for fname in sorted(os.listdir(data_path)):
fname = os.path.join(data_path, fname)
if not os.path.isfile(fname):
continue
if ('2019051' in fname or '2019052' in fname or '20190530' in fname
or '20190531_02' in fname or '20190531_03' in fname
or '20190531_04' in fname or '20190531_05' in fname
or '20190531_061407' in fname):
#'20190530_021341' in fname):
continue
mtime = os.stat(fname).st_mtime
if mtime < 1557990000:
continue
prev_time = None
with open(fname) as f:
utility.print_info(f'FLD read {fname}')
for line in f:
columns = line.split()
if '#' in line or len(columns) < 10:
continue
ix = int(
numpy.round(float(columns[3]) - center_x) / 10 + 26)
iy = int(
numpy.round(float(columns[4]) - center_y) / 10 + 26)
iz = int(
numpy.round(float(columns[5]) - center_z) / 10 + 28)
istep = int(float(columns[2]))
if abs(float(columns[10])) < 0.5:
continue
# key = (ix, iy, iz)
key = (ix, iy, istep)
curr_time = self.__get_timestamp(columns)
columns.append(curr_time)
if key in self.field_map.keys() and False:
print('exist', self.field_map[key].data, columns)
else:
# self.field_map[ix, iy, iz] = FieldElement(columns)
self.field_map[ix, iy, istep] = FieldElement(columns)
#print("2 step " + str(columns[2]))
if prev_time is not None:
process_time = curr_time - prev_time
process_time_array.append(process_time)
prev_time = curr_time
speed = numpy.mean(process_time_array)
# utility.print_info(f'FLD average speed = {speed:.3f} s')
self.calculate_maxwell_equation()
def prepare_training_dataset():
train_df = utility.read_csv(
train_csv_file
) # Only need to read from once (~9 minutes). Then write to an HDF file and use that afterwards
# train_df = read_hdf(train_hdf_file, hdf_key)
utility.print_info(train_df)
utility.write_to_hdf(train_df, train_hdf_file, hdf_key,
5) # Only need to write once
trunc_train_df = utility.truncate_dataset(train_df, train_start_row,
train_end_row)
utility.write_to_hdf(trunc_train_df, truncated_train_hdf_file, hdf_key, 5)
def dowload_compile_and_install_libiconv():
try:
cmd = "axel https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.15.tar.gz\n \
tar -xvf libiconv-1.15.tar.gz\n \
cd libiconv-1.15\n \
sudo ./configure --prefix=/usr/local\n \
sudo make\n \
sudo make install\n \
cd -\n \
sudo rm libiconv* -rf\n \
sudo ldconfig"
ret = call(cmd, shell=True)
if (ret != 0):
exit_with_msg('install libiconv')
except KeyboardInterrupt:
print_info('Keyboard interrupt by user')
sys.exit(1)
def login(session, url, username, password, debug=False):
user_data = {
"csrfmiddlewaretoken": get_csrf_token(session, url),
"username": username,
"password": password,
"next": "/admin/"
}
r = session.post(url, headers=user_headers, data=user_data)
ret = search_element(
r,
'<a href="/admin/qcloud_app/server_device_verify_tb/add/" class="addlink">'
)
if (ret != True and debug):
ret = search_element(r, '<p class="errornote">')
if (ret):
print_info('Maybe username or password error')
return ret
def link_hiredis(ext_dir):
try:
if (path.isfile('/usr/lib/x86_64-linux-gnu/libhiredis.so.0.13')):
return
if (path.isfile('/usr/lib/x86_64-linux-gnu/libhiredis.so.0.10')):
ret = ln_hiredis()
if (ret != 0):
exit_with_msg('Link hiredis')
else:
cmd = 'sudo cp %slibhiredis.so.0.10 /usr/lib/x86_64-linux-gnu' % ext_dir
ret = call(cmd, shell=True)
if (ret != 0):
exit_with_msg('cp libhiredis.so')
else:
ret = ln_hiredis()
if (ret != 0):
exit_with_msg('Link hiredis')
except KeyboardInterrupt:
print_info('Keyboard interrupt by user')
sys.exit(1)
def create_and_exec_install_shell(ext_object):
try:
cmd = '#!/bin/bash\nsudo apt-get update\n'
for package_name in ext_object['install_packages']:
cmd += 'sudo apt-get install -y %s\n' % package_name
install_shell = open('./install.sh', 'w+')
cmd += '\n'
install_shell.write(cmd)
install_shell.close()
ret = call('sudo chmod a+x install.sh', shell=True)
if (ret != 0):
exit_with_msg('chmod for install')
ret = call('./install.sh', shell=True)
if (ret != 0):
exit_with_msg('exec install.sh')
ret = call('sudo rm -f install.sh', shell=True)
if (ret != 0):
exit_with_msg('rm install.sh')
except KeyboardInterrupt:
print_info('Keyboard interrupt by user - when install shell')
sys.exit(1)
except IOError:
print_error('IO Error - when install shell')
sys.exit(1)
def initialize(self, param_file):
self.param_file = param_file
self.param_list = dict()
utility.print_info(f'PRM read {self.param_file}')
if os.path.isfile(self.param_file):
self.param_list['param_file'] = self.param_file
with open(self.param_file, 'r') as f:
for line in f:
columns = line.split()
if len(columns) < 2 or '#' in columns[0]:
continue
self.param_list[columns[0]] = columns[1]
utility.print_info(f'PRM key = {columns[0] + ",":18} ' +
f'val = {columns[1]}')
utility.print_info(f'PRM initialized')
self.initialized = True
else:
utility.print_error(f'PRM NOT initialized')
self.initialized = False
if (ret != 0):
exit_with_msg('Link hiredis')
except KeyboardInterrupt:
print_info('Keyboard interrupt by user')
sys.exit(1)
if __name__ == "__main__":
isCalc = False
isCalc = parseArg(sys.argv[1:])
ext_dir = '../ext/'
assert_ext(ext_dir + 'ext.json')
ext_object = load(open(ext_dir + 'ext.json', 'r'))
create_and_exec_install_shell(ext_object)
session = Session()
ret = login(session, ext_object['cloud_login_url'], ext_object['cloud_username'],
ext_object['cloud_password'], debug=ext_object['debug_mode'])
if (ret):
print_info('Login success')
ret = add_server(
session, ext_object['cloud_add_server_url'], ext_dir, debug=ext_object['debug_mode'])
if (ret):
print_info('Add server success')
else:
exit_with_msg('Add server fail')
else:
exit_with_msg('Login fail')
if (isCalc == False):
link_hiredis(ext_dir)
dowload_compile_and_install_libiconv()
def __init__(self):
utility.print_debug('FLD __init__')
param_man = param_manager.ParamManager()
data_path = param_man.get('data_path')
center_x = float(param_man.get('center_x'))
center_y = float(param_man.get('center_y'))
center_z = float(param_man.get('center_z'))
self.field_map = dict()
self.bscale = 1.0
process_time_array = []
for fname in sorted(os.listdir(data_path)):
fname = os.path.join(data_path, fname)
mtime = os.stat(fname).st_mtime
if not os.path.isfile(fname):
continue
if mtime < 1558083682:
continue
# if ('20190520_061450' in fname or
# '20190520_063540' in fname or
# '20190527_060958' in fname or
# '20190527_061237' in fname or
# '20190530_021341' in fname):
# if '20190530_021341' not in fname:
# if '20190531_183253' not in fname:
# if ('20190531_191821' not in fname and
# '20190531_193258' not in fname and
# '20190601_020337' not in fname):
# if ('20190603_112806' not in fname and
# '20190603_121624' not in fname):
''' main data '''
if not use_nmr and (
'201906' in fname or '20190520_061450' in fname
or '20190520_063540' in fname or '20190522_071913' in fname
or '20190527_060958' in fname or '20190527_061237' in fname
or '20190530_021341' in fname or '20190531_054932' in fname
or '20190531_054932' in fname or '20190531_06' in fname or
'20190531_1' in fname or '20190531_2' in fname or False):
continue
if (use_nmr and '20190603_112806' not in fname
and '20190603_121624' not in fname):
continue
prev_time = None
with open(fname) as f:
utility.print_info(f'FLD read {fname}')
for iline, line in enumerate(f):
columns = line.split()
if use_nmr:
if '#' in line or len(columns) < 14:
continue
else:
if '#' in line or len(columns) < 13:
continue
ix = int(
numpy.round(float(columns[3]) - center_x) / 10 + 26)
if use_nmr:
ix = len(self.field_map)
iy = int(
numpy.round(float(columns[4]) - center_y) / 10 + 26)
iz = int(
numpy.round(float(columns[5]) - center_z) / 10 + 28)
# if abs(float(columns[10])) < 0.5:
# continue
key = (ix, iy, iz)
curr_time = self.__get_timestamp(columns)
columns.append(curr_time)
if key in self.field_map.keys() and False:
print('exist', self.field_map[key].data, columns)
else:
self.field_map[ix, iy, iz] = FieldElement(columns)
if prev_time is not None:
process_time = curr_time - prev_time
process_time_array.append(process_time)
prev_time = curr_time
speed = numpy.mean(process_time_array)
utility.print_info(f'FLD average speed = {speed:.3f} s')
self.calculate_maxwell_equation()
if not os.path.exists(params.results_dir):
os.makedirs(params.results_dir)
# ## Import truncated Data from hdf files
# ### 3 sets: train, validation, test. Print basic stats about them and plot them
# In[9]:
train_df = utility.read_hdf(truncated_train_hdf_file, hdf_key)
valid_df = utility.read_hdf(validation_hdf_file, hdf_key)
test_df = utility.read_hdf(test_hdf_file, hdf_key)
# In[10]:
utility.print_info(train_df)
if do_plot_series:
utility.plot_series(
train_df, "Training series", params.results_dir
) # This is time and memory consuming. Do not run this cell unless necessary
# In[11]:
utility.print_info(valid_df)
if do_plot_series:
utility.plot_series(
valid_df, "Validation series", params.results_dir
) # This is time and memory consuming. Do not run this cell unless necessary