def get():
global DB_POOL
with operator_lock:
if not hasattr(DB_POOL, 'lock'):
DB_POOL.lock = threading.Lock()
if getattr(LOCKED, 'locked', False):
# 在同一线程中是不应该重复请求连接的
raise RuntimeError("同一线程[%s]中不应该重复申请连接，代码有错误，请查找" % _thread.get_ident())
DB_POOL.lock.acquire() # 锁定当前请求，直到连接可用
LOCKED.locked = True
if getattr(DB_POOL, 'conn', None) is None:
DB_POOL.conn = None
DB_POOL.use_times = 0 # 为了避免创建数据库连接失败导致的put异常
try:
DB_POOL.conn = db2api.connect(database=configs.DBNAME, user=configs.USER,
password=configs.PWD, host=configs.HOST, port=configs.PORT)
DB_POOL.use_times = USE_TIMES or 10 # 若未设置，则按10次
except:
# 创建连接失败，应恢复线程状态
put()
import traceback2
traceback2.print_exc()
# raise
else:
# 超过28800s不使用数据库连接，mysql会报错，可采用ping重新连接的方式应用
if DB_TYPE == 'mysql':
DB_POOL.conn.ping(reconnect=True)
DB_POOL.use_times -= 1
return DB_POOL.conn
def print_cleanup_error(context, cleanup_func, exception):
cleanup_func_name = getattr(cleanup_func, "__name__", None)
if not cleanup_func_name:
cleanup_func_name = "%r" % cleanup_func
print(u"CLEANUP-ERROR in %s: %s: %s" %
(cleanup_func_name, exception.__class__.__name__, exception))
traceback.print_exc(file=sys.stdout)
def main(self):
# get command line arguments
args = self.parser.parse_args()
# process argument
if args.loglevel is not None:
loglevel = getattr(logging, args.loglevel.upper(), None)
logging.basicConfig(format=LOG_FORMAT, level=loglevel)
self.config_file = args.config_file
log.info(f'Reading config file: {self.config_file}')
self.config_df = pd.read_csv(self.config_file)
self.report_df = pd.DataFrame()
self.report_dir = args.reportdir
self.report_file = self.get_report_file_name()
for index, row in self.config_df.iterrows():
try:
iteration = self.program(index, row, args=args)
iteration.execute()
self.record_success(index, iteration.get_report_dict())
except Exception as e:
traceback2.print_exc(file=sys.stdout)
log.error(str(e))
self.record_failure(index, str(e))
finally:
log.info(f'Finished iteration: {index}')
self.config_df.to_csv(self.config_file, index=False)
def yolo_detect(original_image):
cfg_file = './cfg/yolov3.cfg'
weight_file = './weights/yolov3.weights'
namesfile = 'data/coco.names'
try:
m = Darknet(cfg_file)
m.load_weights(weight_file)
class_names = load_class_names(namesfile)
resized_image = cv2.resize(original_image, (m.width, m.height))
nms_thresh = 0.6
iou_thresh = 0.4
yolo4_prediction = "Test yolo4_prediction"
eff_prediction = "Test eff_prediction"
boxes = detect_objects(m, resized_image, iou_thresh, nms_thresh)
url = plot_boxes(original_image, boxes, class_names, plot_labels = True)
objects = print_objects(boxes, class_names)
yolo4_prediction = run_yolo4(original_image)
yolo4_prediction = json.dumps(yolo4_prediction)
eff_prediction = run_effiecientNet(original_image)
eff_prediction = json.dumps(eff_prediction)
print("*****************")
print(yolo4_prediction)
print("*****************")
print("*****************")
print(eff_prediction)
print("*****************")
return objects, url, yolo4_prediction, eff_prediction
except Exception:
print("********yolo_detect**********")
traceback.print_exc()
print("********yolo_detect**********")
return "Failed"
def print_cleanup_error(context, cleanup_func, exception):
cleanup_func_name = getattr(cleanup_func, "__name__", None)
if not cleanup_func_name:
cleanup_func_name = "%r" % cleanup_func
print(u"CLEANUP-ERROR in %s: %s: %s" %
(cleanup_func_name, exception.__class__.__name__, exception))
traceback.print_exc(file=sys.stdout)
def sharpeAndCml(source, riskFree, symbols):
ma = dx.market_environment('ma', dt.date(2010, 1, 1))
ma.add_list('symbols', symbols)
ma.add_constant('source', findSource(source))
# ma.add_constant('final date', dt.date(2014, 3, 1))
ma.add_constant('final date', dt.datetime.now())
retVal = '{\n'
retVal += '"EfficientPortfolios":'
try:
port = mvp.MeanVariancePortfolio(source + '_stocks', ma)
effFrontier = port.get_efficient_frontier_bl(
st.config["efficient_frontier"]["points_number"])
retVal += effFrontier.toJson()
retVal += ',\n'
try:
cpl = port.get_capital_market_line_bl_1(effFrontier.vols,
effFrontier.rets,
riskless_asset=riskFree)
retVal += '"CML":' + cpl.toJson()
except Exception as e:
retVal += '"CML": {{"error":"{}"}}'.format(
str(e).replace('"', '\''))
traceback.print_exc()
except Exception as e:
retVal += '{{"error":"{}"}}'.format(str(e).replace('"', '\''))
traceback.print_exc()
retVal += "\n}"
print(retVal)
return retVal
def connect(self):
print('connecting....')
try:
from urlparse import urlparse
except ImportError:
from urllib.parse import urlparse
result = QInputDialog.getText(self, 'Connect To Host', 'Host Address',
QLineEdit.Normal)
if not result[1]:
return
try:
host = str(result[0].toUtf8())
except AttributeError:
host = str(result[0])
# traceback2.print_exc()
try:
if urlparse(host).hostname:
self.ftp.connect(host=urlparse(host).hostname,
port=21,
timeout=10)
else:
self.ftp.connect(host=host, port=21, timeout=10)
self.login()
except:
message = QMessageBox.information(self, '地址错误', '输入地址无响应，请重新输入')
self.connect()
traceback2.print_exc()
def check_traceback_format(self, cleanup_func=None):
try:
if issubclass(six.binary_type, six.string_types):
# Python 2.6 or other platform where the interpreter
# is likely going to be spitting out bytes, which will
# then fail with io.StringIO(), so we skip the cross-
# checks with the C API there. Note that _testcapi
# is included in (at least) Ubuntu CPython packages, which
# makes the import check less effective than desired.
raise ImportError
from _testcapi import traceback_print
except ImportError:
traceback_print = None
try:
self.some_exception()
except KeyError:
type_, value, tb = sys.exc_info()
if cleanup_func is not None:
# Clear the inner frames, not this one
cleanup_func(tb.tb_next)
traceback_fmt = u('Traceback (most recent call last):\n') + \
u('').join(traceback.format_tb(tb))
if traceback_print is not None:
file_ = StringIO()
traceback_print(tb, file_)
python_fmt = file_.getvalue()
# Call all _tb and _exc functions
with captured_output("stderr") as tbstderr:
traceback.print_tb(tb)
tbfile = StringIO()
traceback.print_tb(tb, file=tbfile)
with captured_output("stderr") as excstderr:
traceback.print_exc()
excfmt = traceback.format_exc()
excfile = StringIO()
traceback.print_exc(file=excfile)
else:
self.fail("unable to create test traceback string")
# Make sure that Python and the traceback module format the same thing
if traceback_print is not None:
self.assertEqual(traceback_fmt, python_fmt)
# Now verify the _tb func output
self.assertEqual(tbstderr.getvalue(), tbfile.getvalue())
# Now verify the _exc func output
self.assertEqual(excstderr.getvalue(), excfile.getvalue())
self.assertEqual(excfmt, excfile.getvalue())
# Make sure that the traceback is properly indented.
tb_lines = traceback_fmt.splitlines()
self.assertEqual(len(tb_lines), 5)
banner = tb_lines[0]
location, source_line = tb_lines[-2:]
self.assertTrue(banner.startswith('Traceback'))
self.assertTrue(location.startswith(' File'))
self.assertTrue(source_line.startswith(' raise'))
def check_traceback_format(self, cleanup_func=None):
try:
if issubclass(six.binary_type, six.string_types):
# Python 2.6 or other platform where the interpreter
# is likely going to be spitting out bytes, which will
# then fail with io.StringIO(), so we skip the cross-
# checks with the C API there. Note that _testcapi
# is included in (at least) Ubuntu CPython packages, which
# makes the import check less effective than desired.
raise ImportError
from _testcapi import traceback_print
except ImportError:
traceback_print = None
try:
self.some_exception()
except KeyError:
type_, value, tb = sys.exc_info()
if cleanup_func is not None:
# Clear the inner frames, not this one
cleanup_func(tb.tb_next)
traceback_fmt = u('Traceback (most recent call last):\n') + \
u('').join(traceback.format_tb(tb))
if traceback_print is not None:
file_ = StringIO()
traceback_print(tb, file_)
python_fmt = file_.getvalue()
# Call all _tb and _exc functions
with captured_output("stderr") as tbstderr:
traceback.print_tb(tb)
tbfile = StringIO()
traceback.print_tb(tb, file=tbfile)
with captured_output("stderr") as excstderr:
traceback.print_exc()
excfmt = traceback.format_exc()
excfile = StringIO()
traceback.print_exc(file=excfile)
else:
self.fail("unable to create test traceback string")
# Make sure that Python and the traceback module format the same thing
if traceback_print is not None:
self.assertEqual(traceback_fmt, python_fmt)
# Now verify the _tb func output
self.assertEqual(tbstderr.getvalue(), tbfile.getvalue())
# Now verify the _exc func output
self.assertEqual(excstderr.getvalue(), excfile.getvalue())
self.assertEqual(excfmt, excfile.getvalue())
# Make sure that the traceback is properly indented.
tb_lines = traceback_fmt.splitlines()
self.assertEqual(len(tb_lines), 5)
banner = tb_lines[0]
location, source_line = tb_lines[-2:]
self.assertTrue(banner.startswith('Traceback'))
self.assertTrue(location.startswith(' File'))
self.assertTrue(source_line.startswith(' raise'))
def run_effiecientNet(image_path):
""" Test that the model is able to predict an Egyptian cat with a confidence level higher than 50% """
try:
EfficientNetB7Wrapper.load_model()
predictions = EfficientNetB7Wrapper.predict(image_path)
return predictions
# return "Success"
except Exception:
print("******************")
traceback.print_exc()
print("******************")
return "Failed"
def ClassifyImages(DiskPath,imdir):
try:
# Get Files
print('------------------------Image Classification Start------------------------')
#print(' Classify images'+ imdir)
filelist = getListOfFiles(imdir)
#filelist = glob.glob(os.path.join(imdir, '*.png'))
filelist.sort()
UnsupervisedClassification(DiskPath,filelist)
except Exception as e:
print('Error occurred in Classification:::.', e)
traceback.print_exc()
def run(self, fit: bool = True, get_report: bool = True):
"""
Fit and predict model
:param train: call fit function on model. Set to false when using pre-trained/CV models. Default is True
:return: report dictionary, y_predict
"""
log.info('#' * 40)
log.info(f'Running model: {str(self)}')
log.info('#' * 40)
try:
if fit:
self.report.start_timer(Keys.TRAIN_TIME_MIN)
self.model = self._fit_model()
self.report.end_timer(Keys.TRAIN_TIME_MIN)
model_filename = self._get_model_filename()
# model_filename = f'{self.model_dir}/{self.description}.jbl'
self.report.record(Keys.MODEL_FILE, model_filename)
self.report.start_timer(Keys.MODEL_SAVE_TIME_MIN)
self._save_model(model_filename)
# with open(model_filename, 'wb') as file:
# joblib.dump(self.model, model_filename)
self.report.end_timer(Keys.MODEL_SAVE_TIME_MIN)
self.report.start_timer(Keys.PREDICT_TIME_MIN)
# self.y_predict = self.model.predict(self.x_test)
self.y_predict = self._predict()
self.report.end_timer(Keys.PREDICT_TIME_MIN)
self.report.record(Keys.STATUS, Status.SUCCESS)
except Exception as e:
traceback2.print_exc(file=sys.stdout)
log.error(str(e))
self.report.record(Keys.STATUS, Status.FAILED)
self.report.record(Keys.MESSAGE, str(e))
finally:
self.report.record(Keys.STATUS_DATE,
datetime.now().strftime(TIME_FORMAT))
log.info(f'Finished running model: {str(self)}')
if get_report:
return self.y_predict, self.get_report_dict()
return self.y_predict
def run(self, fit: bool = True, report: bool = True):
"""
Fit and predict model
:param train: call fit function on model. Set to false when using pre-trained/CV models. Default is True
:return: report dictionary, y_predict
"""
log.info('#' * 20)
log.info(f'Running model: {str(self)}')
log.info('#' * 20)
try:
if fit:
self.report.start_timer(Keys.TRAIN_TIME_MIN)
self.model = self.model.fit(self.x_train, self.y_train)
self.report.end_timer(Keys.TRAIN_TIME_MIN)
# TODO: add logic for CV's
model_filename = f'{MODEL_DIR}/{self.description}.jbl'
self.report.record(Keys.MODEL_FILE, model_filename)
self.report.start_timer(Keys.MODEL_SAVE_TIME_MIN)
with open(model_filename, 'wb') as file:
joblib.dump(self.model, model_filename)
self.report.end_timer(Keys.MODEL_SAVE_TIME_MIN)
self.report.start_timer(Keys.PREDICT_TIME_MIN)
self.y_predict = self.model.predict(self.x_test)
self.report.end_timer(Keys.PREDICT_TIME_MIN)
self.report.record(Keys.STATUS, Status.SUCCESS)
except Exception as e:
traceback2.print_exc(file=sys.stdout)
log.error(str(e))
self.report.record(Keys.STATUS, Status.FAILED)
self.report.record(Keys.MESSAGE, str(e))
finally:
self.report.record(Keys.STATUS_DATE,
datetime.now().strftime(TIME_FORMAT))
log.info(f'Finished running model: {str(self)}')
# TODO: refactor old notebooks and swith ordering
if report:
return self.get_report_dict(), self.y_predict
return self.y_predict
def LogException(extraText='',
channel='general',
toConsole=1,
toLogServer=1,
toAlertSvc=None,
toMsgWindow=1,
exctype=None,
exc=None,
tb=None,
severity=None,
show_locals=1):
"""Log an exception to the log server. this code prevents recursion so a exception in the exception processing wont crash."""
global logExceptionLevel
if logExceptionLevel > 0:
return
_tmpctx = blue.pyos.taskletTimer.EnterTasklet('Logging::LogException')
logExceptionLevel += 1
if not exctype:
exctype, exc, tb = sys.exc_info()
try:
try:
_LogException((exctype, exc, tb), extraText, channel, toConsole,
toLogServer, toAlertSvc, toMsgWindow, severity,
show_locals)
return
except Exception:
try:
traceback2.print_exc(show_locals=3)
stream = LogChannelStream(GetChannel('general'))
traceback2.print_exc(show_locals=3, file=stream)
stream.close()
except Exception:
pass
try:
traceback2.print_exception(exctype,
exc,
tb,
file=sys.stdout,
show_locals=show_locals)
stream = LogChannelStream(GetChannel(channel), ERR)
print >> stream, 'retrying traceback log, got an error in _LogException'
traceback2.print_exception(exctype,
exc,
tb,
file=stream,
show_locals=show_locals)
stream.close()
except Exception:
try:
traceback2.print_exc(show_locals=3)
except:
pass
finally:
del tb
logExceptionLevel -= 1
blue.pyos.taskletTimer.ReturnFromTasklet(_tmpctx)
def LogException(extraText = '', channel = 'general', toConsole = 1, toLogServer = 1, toAlertSvc = None, toMsgWindow = 1, exctype = None, exc = None, tb = None, severity = None, show_locals = 1):
global logExceptionLevel
if logExceptionLevel > 0:
return
_tmpctx = blue.pyos.taskletTimer.EnterTasklet('logmodule::LogException')
logExceptionLevel += 1
if not exctype:
exctype, exc, tb = sys.exc_info()
try:
try:
_LogException((exctype, exc, tb), extraText, channel, toConsole, toLogServer, toAlertSvc, toMsgWindow, severity, show_locals)
return
except Exception:
try:
traceback2.print_exc(show_locals=3)
stream = LogChannelStream(GetChannel('general'))
traceback2.print_exc(show_locals=3, file=stream)
stream.close()
except Exception:
pass
try:
traceback2.print_exception(exctype, exc, tb, file=sys.stdout, show_locals=show_locals)
stream = LogChannelStream(GetChannel(channel), ERR)
print >> stream, 'retrying traceback log, got an error in _LogException'
traceback2.print_exception(exctype, exc, tb, file=stream, show_locals=show_locals)
stream.close()
except Exception:
try:
traceback2.print_exc(show_locals=3)
except:
pass
finally:
del tb
logExceptionLevel -= 1
blue.pyos.taskletTimer.ReturnFromTasklet(_tmpctx)
if save_start and save_end:
config_df.loc[index, f"save_time_{lda}"] = \
round((save_end - save_start).total_seconds() / 60, 2)
elif save_start:
config_df.loc[index, f"save_time_{lda}"] = \
round((datetime.now() - save_start).total_seconds() / 60, 2)
if lda_start and lda_end:
config_df.loc[index, f"lda_time_{lda}"] = \
round((lda_end - lda_start).total_seconds() / 60, 2)
elif lda_start:
config_df.loc[index, f"lda_time_{lda}"] = \
round((datetime.now() - lda_start).total_seconds() / 60, 2)
except Exception as e:
traceback2.print_exc(file=sys.stderr)
config_df.loc[index, "status"] = "failed"
config_df.loc[index, "message"] = str(e)
finally:
if read_infile_start and read_infile_end:
config_df.loc[index, "read_time"] = \
round((read_infile_end - read_infile_start).total_seconds() / 60, 2)
elif read_infile_start:
config_df.loc[index, "read_time"] = \
round((datetime.now() - read_infile_start).total_seconds() / 60, 2)
# write status of each round
config_df.to_csv(args.config_file, index=False)
def upload(self):
item = self.Local_Filelist.currentItem()
filesize = int(item.text(1))
try:
if item.text(0).split('.')[-1] not in ['jpg', 'png']:
self.messageBox.message_sig.emit('格式不正确，请选择图片')
return
except:
traceback2.print_exc()
#加密
try:
# print('encrypting....')
# encrypt_key = QInputDialog.getText(self, 'Encrypt', 'Encrypt Key', QLineEdit.Normal)
self.inputMessage.show_sig.emit(1)
Encrykey = self.inputMessage.message
while Encrykey == '':
Encrykey = self.inputMessage.message
self.inputMessage.show_sig.emit(0)
if not Encrykey[1]:
return
except:
traceback2.print_exc()
try:
srcfile = os.path.join(self.local_pwd, str(item.text(0).toUtf8()))
dstfile = os.path.join(self.pwd, str(item.text(0).toUtf8()))
dstfile = dstfile.replace('\\', '/')
except AttributeError:
try:
srcfile = os.path.join(self.local_pwd, str(item.text(0)))
srcfileEncry = RRBE.RRBE_upload(srcfile, 0.1, Encrykey[0])
filename = os.path.basename(srcfileEncry)
dstfile = os.path.join(self.pwd, filename)
dstfile = dstfile.replace('\\', '/')
except:
traceback2.print_exc()
try:
self.progressDialog.show_sig.emit(1)
origin_len = len(self.progressDialog.widgetlist)
self.progressDialog.addProgress_sig.emit('upload',
'Upload ' + srcfile,
filesize)
while origin_len == len(self.progressDialog.widgetlist):
pass
pb = self.progressDialog.widgetlist[
len(self.progressDialog.widgetlist) - 1]
def callback(data):
pb.updateProgress.emit(data)
file = open(srcfileEncry, 'rb')
fp = FTP()
fp.connect(host=self.ftp.host,
port=self.ftp.port,
timeout=self.ftp.timeout)
fp.login(user=self.ftp.user, passwd=self.ftp.passwd)
fp.set_pasv(0)
fp.cwd(self.pwd)
resp = fp.storbinary(cmd='STOR ' + dstfile,
fp=file,
callback=callback)
print(resp)
file.close()
#fp.storbinary(cmd='STOR '+srcfileEncry, fp=file, callback=callback)
# time.sleep(1)
try:
if os.path.exists(srcfileEncry):
os.remove(srcfileEncry)
except:
traceback2.print_exc()
except:
# message = QMessageBox.information(self,'无权限','对不起，您没有此操作的权限')
print('对不起，您没有此操作的权限')
traceback2.print_exc()
def local_right_menu(self, pos):
item = self.Local_Filelist.currentItem()
# item.setFlags(Qt.ItemIsEditable)
menu = QMenu(self.Local_Filelist)
refresh = menu.addAction("刷新")
mkdir = menu.addAction("新建文件夹")
newfile = menu.addAction("新建文件")
rename = menu.addAction("重命名")
remove = menu.addAction("删除文件")
decrypt = menu.addAction("解密图像")
reinstate = menu.addAction("还原图像")
extract = menu.addAction("提取信息")
action = menu.exec_(self.Local_Filelist.mapToGlobal(pos))
# refresh
if action == refresh:
self.updateLocalFileList()
return
# mkdir
elif action == mkdir:
try:
dir_name = QInputDialog.getText(self, '创建文件夹', '请输出文件夹名称',
QLineEdit.Normal)
if not dir_name[1]:
return
# os.mkdir(os.path.join(self.local_pwd, dir_name[0]))
os.mkdir(self.local_pwd + './' + dir_name[0])
self.updateLocalFileList()
self._set_current_item('local', dir_name[0])
except FileExistsError:
message = QMessageBox.information(self, '文件夹已存在',
'文件夹名称已存在，请修改文件名称后再创建')
# remove
elif action == remove:
topCount = self.Local_Filelist.topLevelItemCount()
for i in range(topCount):
item_chosen = self.Local_Filelist.topLevelItem(i)
if (item_chosen == item):
break
import shutil
pathname = os.path.join(self.local_pwd, str(item.text(0)))
# pathname = pathname.replace('\\', '/')
if (os.path.isdir(pathname)):
shutil.rmtree(pathname)
else:
os.remove(pathname)
self.updateLocalFileList()
self.Local_Filelist.setCurrentItem(
self.Local_Filelist.topLevelItem(i))
# rename
elif action == rename:
rename = QInputDialog.getText(self, '重命名', '请输出文件名',
QLineEdit.Normal)
if not rename[1]:
return
pathname = os.path.join(self.local_pwd, str(item.text(0)))
# pathname = pathname.replace('\\', '/')
os.rename(pathname, os.path.join(self.local_pwd, str(rename[0])))
self.updateLocalFileList()
self._set_current_item('local', rename[0])
# new file
elif action == newfile:
file_name = QInputDialog.getText(self, '创建文件', '请输出文件名',
QLineEdit.Normal)
if not file_name[1]:
return
try:
open(self.local_pwd + './' + file_name[0], mode='x')
import webbrowser
webbrowser.open(self.local_pwd + './' + file_name[0])
self.updateLocalFileList()
self._set_current_item('local', file_name[0])
except FileExistsError:
message = QMessageBox.information(self, '文件已存在',
'文件名已存在，请重新创建')
# 解密图像
elif action == decrypt:
pathname = os.path.join(self.local_pwd, str(item.text(0)))
try:
if item.text(0).split('.')[-1] not in ['jpg', 'png']:
message = QMessageBox.information(self, '错误', '请选择图片')
return
decrypt = QInputDialog.getText(self, '解密图像', '请输入解密密钥',
QLineEdit.Normal)
if not decrypt[1]:
return
RRBE.RRBE_decrypt(pathname, 0.1, decrypt[0])
except:
traceback2.print_exc()
return
message = QMessageBox.information(self, '解密', '解密完成')
# 恢复图像
elif action == reinstate:
pathname = os.path.join(self.local_pwd, str(item.text(0)))
if item.text(0).split('.')[-1] not in ['jpg', 'png']:
message = QMessageBox.information(self, '错误', '请选择图片')
return
RRBE.RRBE_recovery(pathname, 0.1)
QMessageBox.information(self, '恢复', '恢复成功')
# 提取信息
elif action == extract:
try:
pathname = os.path.join(self.local_pwd, str(item.text(0)))
if item.text(0).split('.')[-1] not in ['jpg', 'png']:
message = QMessageBox.information(self, '错误', '请选择图片')
return
extract = QInputDialog.getText(self, '提取信息', '请输入数据密钥',
QLineEdit.Normal)
if not extract[1]:
return
message = RRBE.RRBE_extract(pathname, 0.1, extract[0])
QMessageBox.information(self, '提取', message)
except Exception as e:
QMessageBox.information(self, '错误', str(e))
traceback2.print_exc()
else:
return
