def createCsvFile(self, writer): self._init_xml() try: # ヘッダを書き込み headerFilePath = os.path.join(current_app.root_path, self.csvDir, self.headerFileName) with open(headerFilePath, 'r', encoding='utf_8_sig') as f: l_strip = [s.strip() for s in f.readlines()] writer.writerow(l_strip) # データリストを書き込み if self.dataList is not None: for data in self.dataList: rowData = [] for col in self.xmlReader.getColumnList(): if hasattr(data, col): rowData.append(getattr(data, col)) else: rowData.append('') writer.writerow(rowData) except Exception as e: tb = sys.exc_info()[2] StrUtil.print_error("createCsvFile. error_msg:{}".format( str(e.with_traceback(tb))))
def _check_date(year, month, day): try: newDataStr = "%04d/%02d/%02d" % (int(year), int(month), int(day)) newDate = datetime.datetime.strptime(newDataStr, "%Y/%m/%d") return True except Exception as e: tb = sys.exc_info()[2] StrUtil.print_error("_check_date error_msg:{}".format(str(e.with_traceback(tb)))) return False
def unzip_file(uf, unzipDirPath, file_name): df = os.path.join(unzipDirPath, str(file_name)) try: decompressedFile = gzip.open(uf, 'rb') if not os.path.isdir(unzipDirPath): os.makedirs(unzipDirPath) openDf = open(df, 'wb') openDf.write(decompressedFile.read()) decompressedFile.close() openDf.close() return df except Exception as e: StrUtil.print_error('unzip_file file_path:{}'.format(str(df))) return None
def check_input_form_data_by_db2(param_prop): err_msgs = [] try: if 'form' not in param_prop \ or 'table_name' not in param_prop \ or 'col_prop' not in param_prop: err_msgs.append(Const.INVALID_PARAM_ERR_MSG) param_prop['err_msgs'] = err_msgs return col_prop = param_prop['col_prop'] if 'cname' not in col_prop \ or 'input_field' not in col_prop \ or 'db_field' not in col_prop: err_msgs.append(Const.INVALID_PARAM_ERR_MSG) param_prop['err_msgs'] = err_msgs return form = param_prop['form'] user_tab_columns = DbUtil.get_user_tab_columns_hash( param_prop['table_name']) for idx in range(0, len(col_prop['cname'])): input_field = col_prop['input_field'][idx] value = str(form.__dict__[input_field].data) db_field = col_prop['db_field'][idx] # 必須チェック if 'nullable' in user_tab_columns[db_field]: if user_tab_columns[db_field][ 'nullable'] == 'N' and not value: err_msgs.append( Const.REQUIRED_MSG.format(col_prop['cname'][idx])) # 桁数チェック if 'data_type' in user_tab_columns[ db_field] and 'data_length' in user_tab_columns[ db_field]: data_type = user_tab_columns[db_field]['data_type'] data_length = user_tab_columns[db_field]['data_length'] if data_type == 'VARCHAR2' or data_type == 'CHAR': if StrUtil.lenb(value) > int(data_length): err_msgs.append( Const.LENGTH_OVER_MSG.format( col_prop['cname'][idx], str(data_length))) elif data_type == 'NUMBER': if re.search(',', str(data_length)): t = value t = re.sub(r'[^\.]', r'', t) if len(t) > 1 or re.search('[^0-9^\.]', value): err_msgs.append( Const.NUMERICAL_VALUE_REQUIRED_MSG.format( col_prop['cname'][idx])) else: if re.search('[^0-9]', value): err_msgs.append( Const.INTEGER_VALUE_REQUIRED_MSG.format( col_prop['cname'][idx])) elif data_type == 'DATE': if DateUtil.check_date_format(value, Const.DATE_FORMAT) != 0: err_msgs.append( Const.AVAILABLE_DATE_REQUIRED_MSG.format( col_prop['cname'][idx], value)) param_prop['err_msgs'] = err_msgs except Exception as e: tb = sys.exc_info()[2] param_prop['err_msgs'] = str(e.with_traceback(tb)) StrUtil.print_error( 'check_input_form_data_by_db error_msg:{}'.format( str(e.with_traceback(tb))))
def check_input_form_data_by_prop(param_prop): err_msgs = [] try: if 'form' not in param_prop: err_msgs.append(Const.INVALID_PARAM_ERR_MSG) param_prop['err_msgs'].extend(err_msgs) return form = param_prop['form'] for pro in param_prop['pro_list']: property_type = pro.get("property_type") if "KEYWORD" == property_type: continue col_name = pro.get("db_column_name").lower() value = form.__dict__[col_name].data if col_name.startswith("num_"): if len(value) > 0: value = float(value) else: value = '' # 必須チェック if pro.get("nullable") == 'FALSE' and not value: err_msgs.append( Const.REQUIRED_MSG.format(pro.get("property_name"))) continue if not value: continue # 数字チェック if "NUMBER" == property_type: if NumUtil.is_number_data(str(value)) != 1: err_msgs.append( Const.NUMERICAL_VALUE_REQUIRED_MSG.format( pro.get("property_name"))) else: num_prop = {'sign_ref': '', 'i_ref': '', 'f_ref': ''} NumUtil.split_number(str(value), num_prop) if (len(num_prop['i_ref']) + len(num_prop['f_ref'])) > int( pro.get("i_len")): err_msgs.append( Const.INTEGRAL_PART_OUT_OF_RANGE_MSG.format( pro.get("property_name"), str(pro.get("i_len") - pro.get("f_len")))) if len(num_prop['f_ref']) > int(pro.get("f_len")): err_msgs.append( Const.FRACTIONAL_PART_OUT_OF_RANGE_MSG.format( pro.get("property_name"), str(pro.get("f_len")))) # 日付チェック elif 'DATE' == property_type: if DateUtil.check_date_format(value, Const.DATE_FORMAT) != 0: err_msgs.append( Const.AVAILABLE_DATE_REQUIRED_MSG.format( pro.get("property_name"), value)) # 文字列チェック elif 'TEXT' == property_type or 'TEXT_MULTILINE' == property_type: # 桁数チェック if pro.get("data_size"): if StrUtil.lenb(value) > int(pro.get("data_size")): err_msgs.append( Const.LENGTH_OVER_MSG.format( pro.get("property_name"), str(pro.get("data_size")))) # バリデータチェック(正式表現) re_cond = pro.get('validate_rule') if re_cond and len(value) > 0: try: if not re.search(re_cond, value): err_msgs.append( pro.get('validate_err_msg').replace( '<#DATA#>', value)) except Exception as e: tb = sys.exc_info()[2] StrUtil.print_error( 'check_input_form_data_by_prop validate_rule:{} error_msg:{}' .format(re_cond, str(e.with_traceback(tb)))) param_prop['err_msgs'].extend(err_msgs) except Exception as e: tb = sys.exc_info()[2] param_prop['err_msgs'].extend(str(e.with_traceback(tb))) StrUtil.print_error( 'check_input_form_data_by_prop error_msg:{}'.format( str(e.with_traceback(tb))))
def check_input_form_data_by_db(param_prop): err_msgs = [] try: if 'table_name' not in param_prop \ or 'col_prop' not in param_prop: err_msgs.append(Const.INVALID_PARAM_ERR_MSG) param_prop['err_msgs'].extend(err_msgs) return col_prop = param_prop['col_prop'] if 'cname' not in col_prop \ or 'input_value' not in col_prop \ or 'db_field' not in col_prop: err_msgs.append(Const.INVALID_PARAM_ERR_MSG) param_prop['err_msgs'].extend(err_msgs) return user_tab_columns = DbUtil.get_user_tab_columns_hash( param_prop['table_name']) for idx in range(0, len(col_prop['cname'])): value = col_prop['input_value'][idx] db_field = col_prop['db_field'][idx] # 必須チェック if 'nullable' in user_tab_columns[db_field]: if user_tab_columns[db_field][ 'nullable'] == 'N' and not value: err_msgs.append( Const.REQUIRED_MSG.format(col_prop['cname'][idx])) continue if not value: continue if 'data_type' in user_tab_columns[ db_field] and 'data_length' in user_tab_columns[ db_field]: data_type = user_tab_columns[db_field]['data_type'] data_length = user_tab_columns[db_field]['data_length'] # 文字列チェック if data_type == 'VARCHAR2' or data_type == 'CHAR': # 桁数チェック if StrUtil.lenb(value) > int(data_length): err_msgs.append( Const.LENGTH_OVER_MSG.format( col_prop['cname'][idx], str(data_length))) # 数字チェック elif data_type == 'NUMBER': """ if re.search(',', str(data_length)): t = value t = re.sub(r'[^\.]', r'', t) if len(t) > 1 or re.search('[^0-9^\.]', value): err_msgs.append( Const.NUMERICAL_VALUE_REQUIRED_MSG.format(col_prop['cname'][idx])) else: if re.search('[^0-9]', value): err_msgs.append( Const.INTEGER_VALUE_REQUIRED_MSG.format(col_prop['cname'][idx])) """ if NumUtil.is_number_data(str(value)) != 1: err_msgs.append( Const.NUMERICAL_VALUE_REQUIRED_MSG.format( col_prop['cname'][idx])) else: num_prop = { 'sign_ref': '', 'i_ref': '', 'f_ref': '' } NumUtil.split_number(str(value), num_prop) if 'data_precision' in user_tab_columns[db_field] \ and user_tab_columns[db_field]['data_precision'] is not None: if len(num_prop['i_ref']) > int( user_tab_columns[db_field] ['data_precision']): err_msgs.append( Const.INTEGRAL_PART_OUT_OF_RANGE_MSG. format( col_prop['cname'][idx], str(user_tab_columns[db_field] ['data_precision']))) if 'data_scale' in user_tab_columns[db_field] \ and user_tab_columns[db_field]['data_scale'] is not None: if len(num_prop['f_ref']) > int( user_tab_columns[db_field] ['data_scale']): err_msgs.append( Const.FRACTIONAL_PART_OUT_OF_RANGE_MSG. format( col_prop['cname'][idx], str(user_tab_columns[db_field] ['data_scale']))) # 日付チェック elif data_type == 'DATE': if DateUtil.check_date_format(value, Const.DATE_FORMAT) != 0: err_msgs.append( Const.AVAILABLE_DATE_REQUIRED_MSG.format( col_prop['cname'][idx], value)) # 文字列「CLOB」チェック elif data_type == 'CLOB': if len(value) > 10 * 1024: err_msgs.append( Const.LENGTH_OVER_MSG.format( col_prop['cname'][idx], '10,000')) param_prop['err_msgs'].extend(err_msgs) except Exception as e: tb = sys.exc_info()[2] param_prop['err_msgs'].extend(str(e.with_traceback(tb))) StrUtil.print_error( 'check_input_form_data_by_db error_msg:{}'.format( str(e.with_traceback(tb))))
def save_privs_dept(func, request): if len(func) == 0: return render_template('error/404.html') res, msg = {}, "OK" form = PrivsDeptForm() err_msgs = [] isSaveError = False db_id = request.form["db_id"] if func == Const.UPDATE_PRIVS_DEPT: old_div_cd = request.form["old_div_cd"] old_dept_cd = request.form["old_dept_cd"] old_emp_type_cd = request.form["old_emp_type_cd"] old_working_type_cd = request.form["old_working_type_cd"] old_privs_type = request.form["old_privs_type"] corp_cd = request.form["corp_cd"] div_cd = request.form["div_cd"] dept_cd = request.form["dept_cd"] emp_type_cd = request.form["emp_type_cd"] working_type_cd = request.form["working_type_cd"] privs_type = request.form["privs_type"] cmsDbPrivsDept = CmsDbPrivsDept() cmsDbCodeMaster = CmsDbCodeMaster() OPERATION_NOTE = "CORP_CD={}, DIV_CD={}, DEPT_CD={}, EMP_TYPE_CD={}, WORKING_TYPE_ID={}, PRIVS_TYPE={}" # 保存処理(新規、編集) if func == Const.ADD_PRIVS_DEPT or func == Const.UPDATE_PRIVS_DEPT: if func == Const.ADD_PRIVS_DEPT: isCorpCdExist = cmsDbCodeMaster.checkCorpCdExist(corp_cd) if not isCorpCdExist: err_msgs.append( Const.DATA_NOT_EXIST_ERR_MSG.replace("%s", "Corp Cd")) isSaveError = True privsDept = cmsDbPrivsDept.getPrivsDept(db_id, corp_cd, div_cd, dept_cd, emp_type_cd, working_type_cd, privs_type) if privsDept and privsDept.corp_cd == corp_cd: err_msgs.append(Const.DATA_EXIST_ERR_MSG) isSaveError = True else: privsDept = cmsDbPrivsDept.getPrivsDept(db_id, corp_cd, div_cd, dept_cd, emp_type_cd, working_type_cd, privs_type) # 登録しようとするデータが存在すれば(自分自身以外)、更新できないよう if privsDept and \ (privsDept.div_cd != old_div_cd or privsDept.dept_cd != old_dept_cd or privsDept.emp_type_cd != old_emp_type_cd or privsDept.working_type_cd != old_working_type_cd): err_msgs.append(Const.DATA_EXIST_ERR_MSG) isSaveError = True if not isSaveError: # 入力チェックする cname = [ "Corp Cd", "Div Cd", "Dept Cd", "Emp Type", "Working Type", "Privs Type", ] input_value = [ corp_cd, div_cd, dept_cd, emp_type_cd, working_type_cd, privs_type, ] db_field = [ "MANAGEMENT_CORP_CD", "DIV_CD", "DEPT_CD", "EMP_TYPE_CD", "WORKING_TYPE_CD", "PRIVS_TYPE", ] col_prop = { 'cname': cname, 'input_value': input_value, 'db_field': db_field } param_prop = { 'err_msgs': [], 'table_name': 'CMS_DB_PRIVS_DEPT', 'form': form, 'col_prop': col_prop } DbUtil.check_input_form_data_by_db(param_prop) if len(param_prop['err_msgs']) > 0: err_msgs = param_prop['err_msgs'] isSaveError = True if request.method == 'POST' and not isSaveError: if form.validate_on_submit() == False: StrUtil.print_debug("validate error.") else: try: if func == Const.ADD_PRIVS_DEPT: addPrivsDept = CmsDbPrivsDept(db_id, corp_cd, div_cd, dept_cd, emp_type_cd, working_type_cd, privs_type) cmsDbPrivsDept.addPrivsDept(addPrivsDept, current_user.get_id()) # Privs Dept登録を記録する pkgCmsLog = PkgCmsLog() pkgCmsLog.saveOperationLog( current_user.get_id(), db_id, operation_cd=Const.OPERATION_CD_ADD_PRIVS_DEPT, object_id=None, object_type=None, note=OPERATION_NOTE.format(corp_cd, div_cd, dept_cd, emp_type_cd, working_type_cd, privs_type)) db.session.commit() else: cmsDbPrivsDept.uptPrivsDept( db_id, corp_cd, div_cd, dept_cd, emp_type_cd, working_type_cd, old_div_cd, old_dept_cd, old_emp_type_cd, old_working_type_cd, old_privs_type, current_user.get_id()) # Privs Dept変更を記録する pkgCmsLog = PkgCmsLog() pkgCmsLog.saveOperationLog( current_user.tuid, db_id, operation_cd=Const.OPERATION_CD_UPDATE_PRIVS_DEPT, object_id=None, object_type=None, note=OPERATION_NOTE.format(corp_cd, div_cd, dept_cd, emp_type_cd, working_type_cd, privs_type)) db.session.commit() except Exception as e: db.session.rollback() tb = sys.exc_info()[2] StrUtil.print_error( "Database save failed. error_msg:{}".format( str(e.with_traceback(tb)))) err_msgs.append('Database save failed.') # 削除処理 elif func == Const.DELETE_PRIVS_DEPT: try: cmsDbPrivsDept.delPrivsDept(db_id, corp_cd, div_cd, dept_cd, emp_type_cd, working_type_cd, privs_type, current_user.get_id()) # Privs Dept削除を記録する pkgCmsLog = PkgCmsLog() pkgCmsLog.saveOperationLog( current_user.tuid, db_id, operation_cd=Const.OPERATION_CD_DELETE_PRIVS_DEPT, object_id=None, object_type=None, note=OPERATION_NOTE.format(corp_cd, div_cd, dept_cd, emp_type_cd, working_type_cd, privs_type)) db.session.commit() except Exception as e: db.session.rollback() tb = sys.exc_info()[2] StrUtil.print_error("Database save failed. error_msg:{}".format( str(e.with_traceback(tb)))) err_msgs.append('Database delete failed.') res = {**res, **{"err_msgs": err_msgs}} return Response(json.dumps(res))
def save_privs_user(func, request): if len(func) == 0: return render_template('error/404.html') res, msg = {}, "OK" form = PrivsUserForm() err_msgs = [] isSaveError = False db_id = request.form["db_id"] if func == Const.UPDATE_PRIVS_USER: old_corp_cd = request.form["old_corp_cd"] old_dept_cd = request.form["old_dept_cd"] old_privs_type = request.form["old_privs_type"] corp_cd = request.form["corp_cd"] dept_cd = request.form["dept_cd"] tuid = request.form["user_id"] privs_type = request.form["privs_type"] cmsDbPrivsUser = CmsDbPrivsUser() # 保存処理(新規、編集) if func == Const.ADD_PRIVS_USER or func == Const.UPDATE_PRIVS_USER: if func == Const.ADD_PRIVS_USER: privsUser = cmsDbPrivsUser.getPrivsUser(db_id, corp_cd, dept_cd, tuid, privs_type) if privsUser and privsUser.tuid == tuid: err_msgs.append(Const.DATA_EXIST_ERR_MSG) isSaveError = True user_info = User.getUserInfo(tuid) if not user_info: err_msgs.append(Const.USER_ID_NOT_EXIST_ERR_MSG) isSaveError = True else: privsUser = cmsDbPrivsUser.getPrivsUser(db_id, corp_cd, dept_cd, tuid, privs_type) # 登録しようとするデータが存在すれば(自分自身以外)、更新できないよう if privsUser and \ (privsUser.corp_cd != old_corp_cd or privsUser.dept_cd != old_dept_cd): err_msgs.append(Const.DATA_EXIST_ERR_MSG) isSaveError = True if not isSaveError: # 入力チェックする cname = [ "Corp Cd", "Department", "User Id", "Privs Type", ] input_value = [ corp_cd, dept_cd, tuid, privs_type, ] db_field = [ "MANAGEMENT_CORP_CD", "DEPT_CD", "TUID", "PRIVS_TYPE", ] col_prop = { 'cname': cname, 'input_value': input_value, 'db_field': db_field } param_prop = { 'err_msgs': [], 'table_name': 'CMS_DB_PRIVS_USER', 'form': form, 'col_prop': col_prop } DbUtil.check_input_form_data_by_db(param_prop) if len(param_prop['err_msgs']) > 0: err_msgs = param_prop['err_msgs'] isSaveError = True if request.method == 'POST' and not isSaveError: # form = DatabaseForm(request.form) if form.validate_on_submit() == False: StrUtil.print_debug("validate error.") else: try: if func == Const.ADD_PRIVS_USER: addPrivsUser = CmsDbPrivsUser(db_id, corp_cd, dept_cd, tuid, privs_type) cmsDbPrivsUser.addPrivsUser(addPrivsUser, tuid) # Privs User登録を記録する pkgCmsLog = PkgCmsLog() pkgCmsLog.saveOperationLog( current_user.tuid, db_id, operation_cd=Const.OPERATION_CD_ADD_PRIVS_USER, object_id=None, object_type=None, note=tuid) db.session.commit() else: uptPrivsUser = cmsDbPrivsUser.uptPrivsUser( db_id, old_corp_cd, old_dept_cd, tuid, old_privs_type, corp_cd, dept_cd, privs_type, current_user.get_id()) # Privs User変更を記録する pkgCmsLog = PkgCmsLog() pkgCmsLog.saveOperationLog( current_user.tuid, db_id, operation_cd=Const.OPERATION_CD_UPDATE_PRIVS_USER, object_id=None, object_type=None, note=tuid) db.session.commit() except Exception as e: db.session.rollback() tb = sys.exc_info()[2] StrUtil.print_error( "Database save failed. error_msg:{}".format( str(e.with_traceback(tb)))) err_msgs.append('Database save failed.') # 削除処理 elif func == Const.DELETE_PRIVS_USER: try: cmsDbPrivsUser.delPrivsUser(db_id, corp_cd, dept_cd, tuid, privs_type, current_user.get_id()) # Privs User削除を記録する pkgCmsLog = PkgCmsLog() pkgCmsLog.saveOperationLog( current_user.tuid, db_id, operation_cd=Const.OPERATION_CD_DELETE_PRIVS_USER, object_id=None, object_type=None, note=tuid) db.session.commit() except Exception as e: db.session.rollback() tb = sys.exc_info()[2] StrUtil.print_error("Database save failed. error_msg:{}".format( str(e.with_traceback(tb)))) err_msgs.append('Database delete failed.') res = {**res, **{"err_msgs": err_msgs}} return Response(json.dumps(res))
def getCtxTitle(self, object_type_id, object_id, file_id, ctx_title_format, ctx_title_rst): ctx_text_arr = [] try: index = 0 object_column_sql = '' file_column_sql = '' from_addition_sql = '' and_addition_sql = '' ctx_title_format_arr = re.findall(r'[A-Za-z_0-9]+', ctx_title_format) for _ctx_title_format in ctx_title_format_arr: if index > 0: if file_id is not None and _ctx_title_format in 'FILE_NAME': file_column_sql += ', ' else: object_column_sql += ', ' if file_id is not None and _ctx_title_format in 'FILE_NAME': file_column_sql += 'F.{} COL_{}'.format( _ctx_title_format, index) from_addition_sql = ', CMS_FILE F' and_addition_sql = 'AND O.OBJECT_ID = F.PARENT_OBJECT_ID' and_addition_sql = Const.CONTACT_FORMAT.format( and_addition_sql, 'AND F.FILE_ID = {}'.format(file_id)) and_addition_sql = Const.CONTACT_FORMAT.format( and_addition_sql, 'AND F.IS_DELETED = 0') else: object_column_sql += 'O.{} COL_{}'.format( _ctx_title_format, index) index += 1 selectSql = ''' SELECT {object_column_sql} {file_column_sql} FROM CMS_OBJECT O {from_addition_sql} WHERE O.IS_DELETED = 0 AND O.OBJECT_ID = :object_id {and_addition_sql} ''' selectSql = selectSql.format(object_column_sql=object_column_sql, file_column_sql=file_column_sql, from_addition_sql=from_addition_sql, and_addition_sql=and_addition_sql) t = text(selectSql) rst = db.session.execute(t, {'object_id': object_id}).first() if rst is not None: for idx in range(0, index): # StrUtil.print_debug('rst idx:[{}]'.format(str(rst.items()[idx][1]))) # ctx_text_arr.append(StrUtil.get_safe_string(rst.items()[idx][1])) ctx_title_format = ctx_title_format.replace( '<#' + ctx_title_format_arr[idx] + '#>', StrUtil.get_safe_string(rst.items()[idx][1])) # 全文検索の対象にするテキストはcms_object_property.ctx_flg = 1で設定されている属性にする if object_type_id: index = 0 object_column_sql = '' cmsObjectProperty = CmsObjectProperty() for ctx_db_obj in cmsObjectProperty.getCtxObjectDbNames( object_type_id): if index > 0: object_column_sql += ', ' object_column_sql += 'O.{} COL_{}'.format( ctx_db_obj.db_column_name, index) index += 1 selectSql = ''' SELECT {object_column_sql} FROM CMS_OBJECT O WHERE O.IS_DELETED = 0 AND O.OBJECT_ID = :object_id ''' selectSql = selectSql.format( object_column_sql=object_column_sql) t = text(selectSql) rst = db.session.execute(t, {'object_id': object_id}).first() if rst is not None: for idx in range(0, index): ctx_text_val = StrUtil.get_safe_string( rst.items()[idx][1]) if ctx_text_val: ctx_text_arr.append(ctx_text_val) except Exception as e: db.session.rollback() tb = sys.exc_info()[2] StrUtil.print_error("getCtxTitle info:{} error_msg:{}".format( 'object_id:{} file_id:{} ctx_title_format:{}'.format( object_id, file_id, ctx_title_format), str(e.with_traceback(tb)))) ctx_title_rst['CTX_TITLE'] = ctx_title_format ctx_title_rst['CTX_TEXT'] = "\n".join(ctx_text_arr) ctx_title_rst['CTX_ERROR_FLG'] = 1 ctx_title_rst['CTX_ERROR_LOG'] = str(e.with_traceback(tb)) return ctx_title_rst['CTX_TITLE'] = ctx_title_format ctx_title_rst['CTX_TEXT'] = "\n".join(ctx_text_arr) ctx_title_rst['CTX_ERROR_FLG'] = 0 ctx_title_rst['CTX_ERROR_LOG'] = ''
def optimize_ctx(self, app): try: # データベースオブジェクトを取得する db_list = CmsDb.getCmsDbList() if db_list is None: return False row_num = str(StrUtil.get_safe_config(app, 'CTX_MAX_OBJECT_CNT')) for db_info in db_list: StrUtil.print_debug("optimize_ctx db_id=[{}] begin.".format( str(db_info.db_id))) cms_object = CmsObject() for object_info in cms_object.getCtxObjectList( db_info.db_id, row_num): cmsCtxData = CmsCtxData() # cms_ctx_dataからレコード削除 (updateされた場合の対応) cmsCtxData.delCmsCtxData(object_info.object_id, db_info.db_id) # タイトルテキスト 例:<#IDX_TEXT_001#> : <#IDX_TEXT_002#> ctx_title_rst = { 'CTX_TITLE': '', 'CTX_TEXT': '', 'CTX_ERROR_FLG': 0, 'CTX_ERROR_LOG': '' } cms_object.getCtxTitle(object_info.object_type_id, object_info.object_id, None, object_info.ctx_title_format, ctx_title_rst) # cms_ctx_dataに登録する情報を設定する cmsCtxData.db_id = db_info.db_id cmsCtxData.object_id = object_info.object_id cmsCtxData.object_updated_at = object_info.updated_at cmsCtxData.ctx_title = ctx_title_rst['CTX_TITLE'] cmsCtxData.ctx_text = ctx_title_rst['CTX_TEXT'] cmsCtxData.ctx_error_log = ctx_title_rst['CTX_ERROR_LOG'] cmsCtxData.data_type = Const.DATA_TYPE_OBJECT url = Const.URL_FORMAT.format( str(StrUtil.get_safe_config( app, 'CMS_SYS_URL')).strip('/') + '/property', 'func={}&db_id={}&id={}&object_id={}'.format( 'show_property', db_info.db_id, object_info.parent_folder_id, object_info.object_id)) cmsCtxData.ctx_url = url cmsCtxData.ctx_error_flg = ctx_title_rst['CTX_ERROR_FLG'] # cms_ctx_dataに登録する cmsCtxData.addCmsCtxData(cmsCtxData) # INDEXに登録したら、 cms_object.ctx_indexed_flg=1にする cms_object.ctxUpdObject(object_info.object_id, 1) # cms_object_property, cms_file_typeからINDEX対象の属性やファイルを特定 cms_file = CmsFile() for file_info in cms_file.get_ctx_file_list( object_info.object_id): if not ctx_allowed_file(file_info.file_name): continue cmsCtxData = CmsCtxData() # ctx_text = ctx_text_format.format(ctx_text, # file_info.file_name + ":" # + os.path.join(file_info.dir_name, # file_info.c_file_name)) StrUtil.print_debug( 'ctx_file file_info=[file_name={}; file_path={}]'. format( file_info.file_name, os.path.join(file_info.dir_name, file_info.c_file_name))) # CTX_TITLE_FOTMATの取得 cmsFileType = CmsFileType() fileTypeInfo = cmsFileType.getFileTypeInfo( file_info.file_type_id) # タイトルテキスト 例:<#IDX_TEXT_001#> : <#IDX_TEXT_002#> (<#FILE_NAME#>) ctx_title_rst = { 'CTX_TITLE': '', 'CTX_TEXT': '', 'CTX_ERROR_FLG': 0, 'CTX_ERROR_LOG': '' } cms_object.getCtxTitle(object_info.object_type_id, object_info.object_id, file_info.file_id, fileTypeInfo.ctx_title_format, ctx_title_rst) ctx_text_rst = { 'CTX_TEXT': '', 'CTX_ERROR_FLG·': 0, 'CTX_ERROR_LOG': '' } CtxUtil._get_ctx_text(app, file_info, ctx_text_rst) # URL url = '' # テキスト ctx_text = Const.CONTACT_FORMAT.format( ctx_title_rst['CTX_TEXT'], ctx_text_rst['CTX_TEXT']) # エラーメッセージ ctx_error_log = ctx_title_rst['CTX_ERROR_LOG'] if len(ctx_error_log) != 0: ctx_error_log += '\n' ctx_error_log += ctx_text_rst['CTX_ERROR_LOG'] # cms_ctx_dataに登録する情報を設定する cmsCtxData.db_id = db_info.db_id cmsCtxData.object_id = object_info.object_id cmsCtxData.object_updated_at = object_info.updated_at cmsCtxData.ctx_title = ctx_title_rst['CTX_TITLE'] cmsCtxData.ctx_text = ctx_text cmsCtxData.ctx_error_log = StrUtil.truncate( ctx_error_log, 4000) cmsCtxData.data_type = Const.DATA_TYPE_FILE if ctx_text_rst['CTX_ERROR_FLG'] == 0: url = Const.URL_FORMAT.format( str(StrUtil.get_safe_config( app, 'CMS_SYS_URL')).strip('/') + '/download_file', 'db_id={}&file_id={}'.format( db_info.db_id, file_info.file_id)) cmsCtxData.ctx_url = url cmsCtxData.ctx_error_flg = ctx_text_rst[ 'CTX_ERROR_FLG'] # cms_ctx_dataに登録する cmsCtxData.addCmsCtxData(cmsCtxData) cms_file = CmsFile(object_info.object_id) # INDEXに登録したら、 cms_file.ctx_indexed_flg=1にする cms_file.setCtxIndexedFlg(file_info.file_id, 1) # DBごと処理後にtmpフォルダを空にする CtxUtil._tmp_file_remove(app) StrUtil.print_debug('optimize_ctx db_id=[{}] end.'.format( str(db_info.db_id))) db.session.commit() except Exception as e: db.session.rollback() tb = sys.exc_info()[2] StrUtil.print_error('optimize_ctx error_msg:{}'.format( str(e.with_traceback(tb)))) CtxUtil._tmp_file_remove(app)
def _get_ctx_text(app, file_info, ctx_text_rst): file_path = os.path.join(file_info.dir_name, file_info.c_file_name) if not os.path.isfile(file_path): ctx_text_rst['CTX_TEXT'] = '' ctx_text_rst['CTX_ERROR_FLG'] = 1 ctx_text_rst['CTX_ERROR_LOG'] = Const.NO_FILE_FOUND_MSG.format( file_path) return # 解凍パスを取得する unzip_dir_path = str(StrUtil.get_safe_config(app, 'UNZIP_DIR_PATH_CTX')) # ファイルを解凍する tmp_file_name = Const.FILE_EXTENSION_FORMAT.format( file_info.file_id, file_info.file_name.rsplit('.', 1)[1].lower()) unzip_file_path = FileUtil.unzip_file(file_path, unzip_dir_path, tmp_file_name) # ファイル存在しないか解凍失敗の場合 if unzip_file_path is None: ctx_text_rst['CTX_TEXT'] = '' ctx_text_rst['CTX_ERROR_FLG'] = 1 ctx_text_rst['CTX_ERROR_LOG'] = Const.UNZIP_ERROR_MSG.format( unzip_file_path) return try: # 設定ファイルからJAVA_LIBを取得 java_lib = str(StrUtil.get_safe_config(app, 'JAVA_LIB')) # PDFファイルのテキスト抽出 cmd = '{java_bin} -jar {java_lib} -t {file_path}'.format( **{ 'java_bin': '/usr/java/jdk1.8.0_40/bin/java', 'java_lib': os.path.join(java_lib, 'tika-app-1.23.jar'), 'file_path': unzip_file_path, }) completed_process = subprocess.run(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) # 子プロセスが異常終了の場合 if completed_process.returncode != 0: ctx_text_rst['CTX_TEXT'] = '' ctx_text_rst['CTX_ERROR_FLG'] = 1 ctx_text_rst[ 'CTX_ERROR_LOG'] = completed_process.stderr.decode('utf-8') return ctx_text_rst['CTX_TEXT'] = completed_process.stdout.decode('utf-8') ctx_text_rst['CTX_ERROR_FLG'] = 0 ctx_text_rst['CTX_ERROR_LOG'] = '' except Exception as e: tb = sys.exc_info()[2] StrUtil.print_error("subprocess.run info:{} error_msg:{}".format( 'file_name:{} file_path:{}/{}'.format(file_info.file_name, file_info.dir_name, file_info.c_file_name), str(e.with_traceback(tb)))) ctx_text_rst['CTX_TEXT'] = '' ctx_text_rst['CTX_ERROR_FLG'] = 1 ctx_text_rst['CTX_ERROR_LOG'] = Const.CONTACT_FORMAT.format( cmd, Const.CMD_ERROR_MSG.format(str(e.with_traceback(tb))))