def testExportPlane(self):
# Create an empty temp folder to use for temporary model files.
try:
os.mkdir(Util.path_conversion("tests/temp"))
except OSError:
pass
pass
# The file path we will use.
file_path = Util.path_conversion("tests/temp/plane.dat")
# Import the model.
mesh = ModelShipper.load_stl_model(Util.path_conversion("tests/test_models/plane.stl"))
self.assertNotEqual(mesh, False)
model = LDrawModel(mesh)
"""# Export the model.
ModelShipper.save_ldraw_file_model(file_path, model)
# Read the file.
with open(file_path, 'r') as file:
file_data = file.read()
# The final file data should look like this.
self.assertEqual(
file_data,
"0 LScan auto generated part plane\n0 Name: plane.dat\n0 Author: Rando\n0 !LICENSE Redistributable under CCAL version 2.0 : see CAreadme.txt\n3 4 -0.5 0.0 0.5 -0.5 0.0 -0.5 0.5 0.0 -0.5\n3 4 0.5 0.0 -0.5 0.5 0.0 0.5 -0.5 0.0 0.5\n")
"""
# Cleanup.
if os.path.exists(file_path):
os.remove(file_path)
os.rmdir(Util.path_conversion("tests/temp/"))
def test_rmdir(self):
temp_test_dir = Util.path_conversion("tests/temp/temp_test")
Util.mkdir(temp_test_dir)
self.assertTrue(Util.is_dir(temp_test_dir))
Util.rmdir(temp_test_dir)
self.assertFalse(Util.is_dir(temp_test_dir))
Util.rmdir(Util.path_conversion("tests/temp"))
def answer(article_path, questions_path):
u = Util()
a = Answer(Article(u.load_txt_article(article_path)))
q = u.open_utf_file(questions_path)
q = q.splitlines()
for question in q:
print(a.answer(question))
def run_predict_cv(self) -> None:
"""クロスバリデーションで学習した各foldのモデルの平均により、テストデータの予測を行う
あらかじめrun_train_cvを実行しておく必要がある
"""
logger.info(f'{self.run_name} - start prediction cv')
test_x = RunnerLeaveOneOut.load_x_test(self.features)
preds = []
# 各foldのモデルで予測を行う
for i_fold in range(self.n_fold):
logger.info(f'{self.run_name} - start prediction fold:{i_fold}')
model = self.build_model(i_fold)
model.load_model(self.features)
pred = model.predict(test_x)
preds.append(pred)
logger.info(f'{self.run_name} - end prediction fold:{i_fold}')
# 予測の平均値を出力する
pred_avg = np.mean(preds, axis=0)
# 予測結果の保存
Util.dump(pred_avg, f'../model/pred/{self.run_name}-test.pkl')
logger.info(f'{self.run_name} - end prediction cv')
def on_launch(request_id, session):
# Welcome the User to the App
util = Util()
return Util.build_response(
util, "",
Util.build_speechlet_response(util, '', "Welcome to Studio", "",
False))
def create_settings(filename: str):
"""Generate initial settings file based on current working directory.
:param filename:
:return:
"""
# default stl directory
default_stl_dir = Util.path_conversion("assets/models/")
# default part name
default_part_name = "untitled.dat"
# default part name directory
default_part_dir = Util.path_conversion("assets/parts/")
# default author
default_author = "First Last"
# default license
default_license = "Redistributable under CCAL version 2.0 : see CAreadme.txt"
# default Log directory
default_log_dir = Util.path_conversion(str(Path.home()) + "/Documents")
default_settings = {
"stl_dir": default_stl_dir,
"part_name": default_part_name,
"part_dir": default_part_dir,
"author": default_author,
"license": default_license,
"log_dir": default_log_dir
}
file_path = Util.path_conversion(f"assets/settings/{filename}")
try:
with open(file_path, "w") as file:
json.dump(default_settings, file, indent=4)
except FileNotFoundError as ferr:
print(ferr)
def execute(self):
"""
desc: 主程序
"""
host = Util.get_host()
port = Util.get_conf()[0]
self.logger.info("host is: %s" % host)
return self.get_return({"online": 1, "host": host, "port": port})
def save_model(self, feature):
model_path = os.path.join(f'../model/model/{feature}',
f'{self.run_fold_name}.h5')
scaler_path = os.path.join(f'../model/model/{feature}',
f'{self.run_fold_name}-scaler.pkl')
os.makedirs(os.path.dirname(model_path), exist_ok=True)
self.model.save(model_path)
Util.dump(self.scaler, scaler_path)
def add_model(self, model):
model_id = model['id']
name = model['name']
metrics = model['metrics']
self._ml_models_list[model_id] = {'name': name, 'metrics': metrics}
Util.write_to_json(self._db_filename, self._ml_models_list)
def text_ctrl_input_on_kill_focus(self, event):
"""Get the path for STL input file from user typing into TextCtrl element.
:param event:
:return:
"""
prev_text = self.stl_path_text
self.stl_path_text = self.stl_path_input.GetValue()
self.stl_path_input.SetValue(MetadataPanel.reduce_text_path(self.stl_path_input.GetValue()))
if prev_text != self.stl_path_text:
# Check file path validity
if Util.is_file(self.stl_path_text):
if self.stl_path_text.endswith('.stl'):
# Check if this .stl is valid
mesh = ModelShipper.load_stl_model(self.stl_path_text)
if mesh:
# Load in LDraw object to input model
ModelShipper.input_model = LDrawModel(mesh)
self.stl_dir = Util.get_parent(self.stl_path_text) # Only the dir
SettingsManager.save_settings("stl_dir", self.stl_dir)
self.stl_path_isvalid = True
UIDriver.fire_event(
UserEvent(UserEventType.INPUT_MODEL_READY,
LogMessage(LogType.INFORMATION,
"Input file loaded from: '" +
self.stl_path_text + "'.")))
else:
self.stl_path_isvalid = False
UIDriver.fire_event(
UserEvent(UserEventType.LOG_INFO,
LogMessage(LogType.ERROR,
"The input file '" +
self.stl_path_text +
"' is not a valid STL file.")))
else:
self.stl_path_isvalid = False
UIDriver.fire_event(
UserEvent(UserEventType.LOG_INFO,
LogMessage(LogType.ERROR,
"Input file must have .stl extension.")))
else:
self.stl_path_isvalid = False
if len(self.stl_path_text) <=0:
log_msg = "Input filepath cannot be blank."
else:
log_msg = "The path '" + self.stl_path_text + "' could not be found."
UIDriver.fire_event(
UserEvent(UserEventType.LOG_INFO,
LogMessage(LogType.ERROR, log_msg)))
self.check_input()
event.Skip()
def start_callout(self, phone_num: str, intent: str) -> str:
# Stars a callout camapaign
self.valid_input([self.api_key, phone_num])
print("Starting Callout")
data = requests.post(self.build_url_v2(self.api_key, phone_num))
print(data)
util = Util()
return Util.build_response(
util, "",
Util.build_speechlet_response(util, intent,
"Starting Campaign" + '', "", True))
def notify_agent():
"""
desc: 启动时，告知agent的ip:port
"""
host = Util.get_host()
port = Util.get_conf()[0]
notify_url = Util.get_conf()[1]
payload = {
"machine_ip": host,
"machine_port": port,
}
print payload
requests.post(notify_url, params=payload)
def run_train_cv(self, lr_curve_train_sizes: Optional[List[int]]=None) -> None:
"""クロスバリデーションでの学習・評価を行う
学習・評価とともに、各foldのモデルの保存、スコアのログ出力についても行う
"""
logger.info(f'{self.run_name} - start training cv')
scores = []
va_idxes = []
preds = []
lr_curve = not (lr_curve_train_sizes is None)
if lr_curve:
train_scores = np.empty(( 0, len(lr_curve_train_sizes) ) , float)
valid_scores = np.empty(( 0, len(lr_curve_train_sizes) ) , float)
# 各foldで学習を行う
for i_fold in range(self.n_fold):
# 学習を行う
logger.info(f'{self.run_name} fold {i_fold} - start training')
if lr_curve:
model, va_idx, va_pred, score, tr_score, val_score = self.train_fold_lr(i_fold, lr_curve_train_sizes)
train_scores = np.append(train_scores, tr_score.reshape(1, len(lr_curve_train_sizes)) ,axis=0)
valid_scores = np.append(valid_scores, val_score.reshape(1, len(lr_curve_train_sizes)),axis=0)
else:
model, va_idx, va_pred, score = self.train_fold(i_fold)
logger.info(f'{self.run_name} fold {i_fold} - end training - score {score}')
# モデルを保存する
model.save_model(self.features)
# 結果を保持する
va_idxes.append(va_idx)
scores.append(score)
preds.append(va_pred)
# 各foldの結果をまとめる
va_idxes = np.concatenate(va_idxes)
order = np.argsort(va_idxes)
preds = np.concatenate(preds, axis=0)
preds = preds[order]
logger.info(f'{self.run_name} - end training cv - score {np.mean(scores)}')
# 予測結果の保存
Util.dump(preds, f'../model/pred/{self.features}/{self.run_name}-train.pkl')
# 評価結果の保存
logger.result_scores(self.run_name, scores)
if lr_curve:
self._plot_lr_curve(lr_curve_train_sizes, train_scores, valid_scores)
def __init__(self):
Util.createTables(usersTable) # 테이블 자동생성
self.dbClass = Helper()
self.invaildUserAuthResponse = {
"statusCode": 500,
"error": "Bad Request",
"message": "잘못된 접근입니다."
}
self.failUserAuthResponse = {
"statusCode": 500,
"error": "Bad Request",
"message": "유저 정보를 찾을 수 없습니다."
}
self.dbClass = Helper()
def load_model(self, feature):
model_path = os.path.join(f'../model/model/{feature}',
f'{self.run_fold_name}.h5')
scaler_path = os.path.join(f'../model/model/{feature}',
f'{self.run_fold_name}-scaler.pkl')
self.model = load_model(model_path)
self.scaler = Util.load(scaler_path)
def testAuthor1(self):
authorsIn = [
"Rafed Muhammad Yasir", "Moumita Asad", "Aquib Azmain",
"Eusha Kadir", "Kishan Kumar Ganguly"
]
authorsOut = ["Yasir", "Asad", "Azmain", "Kadir", "Ganguly"]
self.assertEqual(Util.lastNamify(authorsIn), authorsOut)
def build_mesh_triangulation_data(file_path):
mesh = Mesh.from_file(Util.path_conversion(file_path))
triangles = MeshTriangulation.get_mesh_triangles(mesh)
normal_groups = MeshTriangulation.make_normal_groups(triangles)
normal_groups_copy = copy.deepcopy(normal_groups)
faces = MeshTriangulation.make_face_groups_loop(normal_groups_copy)
face_boundaries, normals = MeshTriangulation.make_face_boundaries(
faces)
simple_boundaries = MeshTriangulation.make_simple_boundaries(
face_boundaries)
separate_boundaries = MeshTriangulation.split_boundaries(
simple_boundaries)
ordered_separate_boundaries = MeshTriangulation.find_outside_boundary(
separate_boundaries)
triangulated_faces = MeshTriangulation.buckets_to_dicts(
ordered_separate_boundaries)
mesh_dict = {
"input_mesh": mesh,
"triangles": triangles,
"normal_groups": normal_groups,
"faces": faces,
"separate_boundaries": separate_boundaries,
"ordered_separate_boundaries": ordered_separate_boundaries,
"triangulated_faces": triangulated_faces
}
return mesh_dict
def run_workflow(self, workflow_id: str, intent: str) -> str:
# Runs the workflow that matches the given id
self.retrieve_token()
self.valid_input([self.token, workflow_id])
print("Starting Workflow")
# Run Workflow
payload = {'token': self.token, 'workflow_id': workflow_id}
data = requests.post(self.build_url('workflow', 'run'), data=payload)
print(data)
print(json.loads(data.text)['result'])
util = Util()
return Util.build_response(
util, "",
Util.build_speechlet_response(
util, intent,
"Initializing Workflow" + json.loads(data.text)['result'], "",
True))
def run_predict_all(self) -> None:
"""学習データすべてで学習したモデルにより、テストデータの予測を行う
あらかじめrun_train_allを実行しておく必要がある
"""
logger.info(f'{self.run_name} - start prediction all')
test_x = RunnerLeaveOneOut.load_x_test(self.features)
# 学習データ全てで学習したモデルで予測を行う
i_fold = 'all'
model = self.build_model(i_fold)
model.load_model(self.features)
pred = model.predict(test_x)
# 予測結果の保存
Util.dump(pred, f'../model/pred/{self.run_name}-test.pkl')
logger.info(f'{self.run_name} - end prediction all')
def load_settings(self):
"""Load settings values into memory on startup.
"""
# If settings file doesnt exist
if not Util.is_file(SettingsManager.file_path):
# If directory doesnt exist
if not Util.is_dir(SettingsManager.settings_path):
Util.mkdir(SettingsManager.settings_path)
# Create user settings with default
SettingsManager.create_settings(SettingsManager.filename)
with open(SettingsManager.file_path, "r") as file:
file_settings = json.load(file)
self.stl_dir = file_settings["stl_dir"]
self.part_name = file_settings["part_name"]
self.part_dir = file_settings["part_dir"]
self.author_default = file_settings["author"]
self.license_default = file_settings["license"]
def browse_output(self, event):
"""Browse for a valid output file path
:param event:
:return:
"""
UIDriver.fire_event(UserEvent(
UserEventType.RENDERING_CANVAS_DISABLE,
LogMessage(LogType.IGNORE, "")))
dat_wildcard = "*.dat"
dialog = wx.FileDialog(self, "Choose a location for the LDraw file",
defaultDir=self.part_dir,
style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT,
wildcard=dat_wildcard)
dialog.SetFilename(self.part_name)
if dialog.ShowModal() == wx.ID_OK:
pathname = dialog.GetPath()
if self.out_file != pathname:
# Check if part name ends with .dat, if not append that
if not pathname.endswith('.dat'):
pathname = pathname + '.dat'
self.out_file = pathname # Full path
self.part_dir = Util.get_parent(pathname) # Only the dir
self.part_name = Util.get_filename(pathname) # Only filename
self.ldraw_name_isvalid = True
SettingsManager.save_settings("part_dir", self.part_dir)
SettingsManager.save_settings("part_name", self.part_name)
self.ldraw_name_input.SetValue(self.out_file)
self.ldraw_name_input.SetValue(MetadataPanel.reduce_text_path(self.ldraw_name_input.GetValue()))
self.check_input()
UIDriver.fire_event(
UserEvent(UserEventType.LOG_INFO,
LogMessage(LogType.INFORMATION,
"Output file will be saved as: '" +
self.out_file + "'.")))
UIDriver.fire_event(UserEvent(
UserEventType.RENDERING_CANVAS_ENABLE,
LogMessage(LogType.IGNORE, "")))
dialog.Destroy()
def __init__(self):
self.dbClass = Helper(init=True)
self.utilClass = Util()
self.paymentClass = ManagePayment()
self.s3 = self.utilClass.getBotoClient('s3')
self.invaildUserAuthResponse = {
"statusCode": 500,
"error": "Bad Request",
"message": "잘못된 접근입니다."
}
self.failUserAuthResponse = {
"statusCode": 500,
"error": "Bad Request",
"message": "유저 정보를 찾을 수 없습니다."
}
self.sendEmailAuthResponse = {
"statusCode": 200,
"message": "고객님의 메일로 링크를 보내드렸습니다. 메일발송까지 5-10분 정도 소요될 수 있습니다."
}
def browse_input(self, event):
"""Browse for a valid STL input file.
:param event:
:return:
"""
UIDriver.fire_event(UserEvent(
UserEventType.RENDERING_CANVAS_DISABLE,
LogMessage(LogType.IGNORE, "")))
stl_wildcard = "*.stl"
dialog = wx.FileDialog(self, "Choose a STL file",
defaultDir=self.stl_dir, wildcard=stl_wildcard,
style=wx.FD_OPEN
| wx.FD_FILE_MUST_EXIST)
if dialog.ShowModal() == wx.ID_OK:
filename = dialog.GetPath()
# Check for file existing
# If valid, pass to worker thread who will check data
if self.stl_path_text != filename:
self.stl_path_input.SetValue(filename)
self.stl_path_input.SetValue(MetadataPanel.reduce_text_path(self.stl_path_input.GetValue()))
# Only update stuff if selection changed
# Check if this .stl is valid
mesh = ModelShipper.load_stl_model(filename)
if mesh:
# Load in LDraw object to input model
ModelShipper.input_model = LDrawModel(mesh)
self.stl_dir = Util.get_parent(filename) # Only the dir
self.stl_path_text = filename # The whole path to file
self.stl_path_isvalid = True
SettingsManager.save_settings("stl_dir", self.stl_dir)
UIDriver.fire_event(
UserEvent(UserEventType.INPUT_MODEL_READY,
LogMessage(LogType.INFORMATION,
"Input file loaded from: '" +
self.stl_path_text + "'.")))
else:
self.stl_path_isvalid = False
UIDriver.fire_event(
UserEvent(UserEventType.LOG_INFO,
LogMessage(LogType.ERROR,
"The input file '" +
filename +
"' is not a valid STL file.")))
self.check_input()
UIDriver.fire_event(UserEvent(
UserEventType.RENDERING_CANVAS_ENABLE,
LogMessage(LogType.IGNORE, "")))
dialog.Destroy()
def train_test_split_matrix(self, split=0.2, use_saved=True):
"""
if use_saved is true, the pickle files of train and test are loaded. If not, we split the data into train and
test datasets.
:param split: the test proportion of the data; has to be a float between 0 and 1
:param use_saved: if true, we use the local pickled copies of the data
"""
if use_saved:
self.train = Util.load_pickle_object(
self.config.get_train_data_loc())
self.test = Util.load_pickle_object(
self.config.get_test_data_loc())
else:
matrix_copy = self.matrix.copy(deep=True)
split_point = int(matrix_copy.shape[1] * split)
# here we shuffle the columns to prevent overfitting
shuffled_columns = matrix_copy.columns.tolist()
shuffle(shuffled_columns)
matrix_copy = matrix_copy[shuffled_columns]
train_matrix = matrix_copy.copy(deep=True)
train_matrix.iloc[:, split_point:] = 0
test_matrix = matrix_copy.copy(deep=True)
test_matrix.iloc[:, :split_point] = 0
Util.pickle_object(self.config.get_train_data_loc(), train_matrix)
Util.pickle_object(self.config.get_test_data_loc(), test_matrix)
self.train = train_matrix
self.test = test_matrix
def test_(self):
test_message = "test input model message"
input_model = ModelShipper.load_stl_model(
Util.path_conversion("tests/test_models/plane.stl"))
model_message = InputModelMessage(LogType.INFORMATION, test_message,
input_model)
self.assertEqual(model_message.get_message(), test_message)
self.assertEqual(model_message.get_message_type(), LogType.INFORMATION)
self.assertIsNotNone(model_message.get_timestamp())
self.assertIsNotNone(model_message.get_model())
def testImportPlane(self):
# Load the model from the assets folder.
mesh = ModelShipper.load_stl_model(Util.path_conversion("tests/test_models/plane.stl"))
self.assertNotEqual(mesh, False)
# First triangle facet.
self.assertEqual(mesh.v0[0], Vector3([0.5, 0., -0.5]))
self.assertEqual(mesh.v1[0], Vector3([-0.5, 0., -0.5]))
self.assertEqual(mesh.v2[0], Vector3([-0.5, 0., 0.5]))
# Second triangle facet.
self.assertEqual(mesh.v0[1], Vector3([-0.5, 0., 0.5]))
self.assertEqual(mesh.v1[1], Vector3([0.5, 0., 0.5]))
self.assertEqual(mesh.v2[1], Vector3([0.5, 0., -0.5]))
def list_all_scripts(self, intent: str) -> str:
# Lists all the available scripts
self.retrieve_token()
self.valid_input([self.token])
print("Listing Scripts")
# List Scripts
payload = {'token': self.token}
data = requests.post(self.build_url('script', 'list-all'),
data=payload)
print(data)
util = Util()
try:
data = json.loads(data.text)['result']['error']['message']
print(data)
return Util.build_response(
util, "",
Util.build_speechlet_response(util, intent, data, "", True))
except Exception:
print("Failed")
return Util.build_response(
util, "",
Util.build_speechlet_response(util, intent, "Failed", "",
True))
def update_workflow(self, workflow_id: str, name: str, status: str,
intent: str) -> str:
# updates the workflow given the name and status arguments
self.retrieve_token()
self.valid_input([self.token, workflow_id, name, status])
print("Updating Workflow")
# Update Workflow
payload = {
'token': self.token,
'workflow_id': workflow_id,
'name': name,
'status': status
}
data = requests.post(self.build_url('workflow', 'update'),
data=payload)
print(data)
print(json.loads(data.text)['result']['message'])
util = Util()
return Util.build_response(
util, "",
Util.build_speechlet_response(
util, intent, "Updating Workflow" +
json.loads(data.text)['result']['message'], "", True))
def prepare(self):
"""
desc: prepare：解压入参数据，初始化环境变量，目录等
"""
home = Util.get_conf()[2]
self.decouple_data()
self.logger.info("action is: %s" % self.action)
self.logger.info("desc is: %s" % self.desc)
self.logger.info("task_id is: %s" % self.task_id)
self.logger.info("params is %s" % json.dumps(self.params))
self.logger.info("import env variables in .bashrc.")
subprocess.Popen("source %s/.bashrc" % home, shell=True)
self.logger.info('init dirs.')
os.system("/bin/sh init_task.sh %s" % self.task_id)
def answer(self, question, return_score=False):
"""
Answer the given question.
Args:
question: Question string
Returns:
Answer to question as string
"""
u = Util()
question_embedding = u.embeddings([question])[0]
sentences_list = []
for paragraph in self.article.sentences:
paragraph_text = [s.text for s in paragraph]
sentences_list += paragraph_text
sentences_embeddings = u.embeddings(sentences_list)
distances = []
for i, embedding in enumerate(sentences_embeddings):
diffs = np.inner(question_embedding, embedding)
dist = diffs
distances.append((dist, sentences_list[i]))
distances.sort(key=lambda x: x[0], reverse=True)
most_similar_sentence = distances[0][1]
most_similar_score = distances[0][0]
if return_score:
return (most_similar_sentence, most_similar_score)
return most_similar_sentence