def check_is_ddl_sql(sql):
word_list = sql.split()
if len(word_list) < 1:
raise MyError('sql is empty, sql="{0}"'.format(sql))
key_word = word_list[0].lower()
if 'create' != key_word and 'alter' != key_word:
raise MyError('sql must be ddl, key_word="{0}", sql="{1}"'.format(key_word, sql))
def check_is_query_sql(sql):
word_list = sql.split()
if len(word_list) < 1:
raise MyError('sql is empty, sql="{0}"'.format(sql))
key_word = word_list[0].lower()
if 'select' != key_word and 'show' != key_word and 'desc' != key_word:
raise MyError('sql must be query, key_word="{0}", sql="{1}"'.format(key_word, sql))
def change_tenant(self, tenant_id):
self._cursor.exec_sql("alter system change tenant tenant_id = {0}".format(tenant_id))
(desc, results) = self._query_cursor.exec_query("select effective_tenant_id()")
if (1 != len(results) or 1 != len(results[0])):
raise MyError("results cnt not match")
elif (tenant_id != results[0][0]):
raise MyError("change tenant failed, effective_tenant_id:{0}, tenant_id:{1}"
.format(results[0][0], tenant_id))
def check_is_update_sql(sql):
word_list = sql.split()
if len(word_list) < 1:
raise MyError('sql is empty, sql="{0}"'.format(sql))
key_word = word_list[0].lower()
if 'insert' != key_word and 'update' != key_word and 'replace' != key_word and 'set' != key_word and 'delete' != key_word:
# 还有类似这种：select @current_ts := now()
if not (len(word_list) >= 3 and 'select' == word_list[0].lower()\
and word_list[1].lower().startswith('@') and ':=' == word_list[2].lower()):
raise MyError('sql must be update, key_word="{0}", sql="{1}"'.format(key_word, sql))
def change_tenant(cur, tenant_id):
# change tenant
sql = "alter system change tenant tenant_id = {0};".format(tenant_id)
logging.info(sql);
cur.execute(sql);
# check
sql = "select effective_tenant_id();"
cur.execute(sql)
result = cur.fetchall()
if (1 != len(result) or 1 != len(result[0])):
raise MyError("invalid result cnt")
elif (tenant_id != result[0][0]):
raise MyError("effective_tenant_id:{0} , tenant_id:{1}".format(result[0][0], tenant_id))
def fetch_observer_version(query_cur):
(desc, results) = query_cur.exec_query("""select distinct value from __all_virtual_sys_parameter_stat where name='min_observer_version'""")
if len(results) != 1:
raise MyError('query results count is not 1')
else:
logging.info('get observer version success, version = {0}'.format(results[0][0]))
return results[0][0]
def gen_update_sys_var_history_sql_for_tenant(dml_cur, tenant_id, sys_var):
try:
actual_tenant_id = get_actual_tenant_id(tenant_id)
(desc, results) = dml_cur.exec_query("""select schema_version from oceanbase.__all_sys_variable_history
where tenant_id = {0} and name = '{1}'
order by schema_version desc limit 1"""
.format(actual_tenant_id, sys_var["name"]))
schema_version = results[0][0]
(desc, results) = dml_cur.exec_query("""select value from __all_sys_variable where tenant_id={0} and name='{1}' limit 1"""
.format(actual_tenant_id, sys_var["name"]))
res_len = len(results)
if res_len != 1:
logging.error('fail to get value from __all_sys_variable, result count:'+ str(res_len))
raise MyError('fail to get value from __all_sys_variable')
value = results[0][0]
min_val = sys_var["min_val"] if "min_val" in sys_var.keys() else ''
max_val = sys_var["max_val"] if "max_val" in sys_var.keys() else ''
replace_sql = """replace into oceanbase.__all_sys_variable_history(
tenant_id,
zone,
name,
schema_version,
is_deleted,
data_type,
value,
info,
flags,
min_val,
max_val)
values(%d, '', '%s', %d, 0, %d, '%s', '%s', %d, '%s', '%s')
"""%(actual_tenant_id, sys_var["name"], schema_version, sys_var["data_type"], value, sys_var["info"], sys_var["flags"], min_val, max_val)
return replace_sql
except Exception, e:
logging.exception('fail to gen replace sys var history sql')
raise e
def exec_sys_vars_upgrade_dml(cur, tenant_id_list):
if len(tenant_id_list) <= 0:
logging.error('distinct tenant id count is <= 0, tenant_id_count: %d', len(tenant_id_list))
raise MyError('invalid arg')
dml_cur = DMLCursor(cur)
# 操作前先dump出oceanbase.__all_sys_variable表的所有数据
my_utils.query_and_dump_results(dml_cur, """select * from oceanbase.__all_virtual_sys_variable""")
# 操作前先dump出oceanbase.__all_sys_variable_history表的所有数据
my_utils.query_and_dump_results(dml_cur, """select * from oceanbase.__all_virtual_sys_variable_history""")
for i in range(0, len(tenant_id_list)):
tenant_id = tenant_id_list[i]
# calc diff
(update_sys_var_list, update_sys_var_ori_list, add_sys_var_list) = calc_diff_sys_var(cur, tenant_id)
logging.info('update system variables list: [%s]', ', '.join(str(sv) for sv in update_sys_var_list))
logging.info('update system variables original list: [%s]', ', '.join(str(sv) for sv in update_sys_var_ori_list))
logging.info('add system variables list: [%s]', ', '.join(str(sv) for sv in add_sys_var_list))
# update
change_tenant(cur, tenant_id)
update_sys_vars_for_tenant(dml_cur, tenant_id, update_sys_var_list)
add_sys_vars_for_tenant(dml_cur, tenant_id, add_sys_var_list)
special_update_sys_vars_for_tenant(dml_cur, tenant_id, add_sys_var_list, 'nls_date_format', 'YYYY-MM-DD HH24:MI:SS');
special_update_sys_vars_for_tenant(dml_cur, tenant_id, add_sys_var_list, 'nls_timestamp_format', 'YYYY-MM-DD HH24:MI:SS.FF');
special_update_sys_vars_for_tenant(dml_cur, tenant_id, add_sys_var_list, 'nls_timestamp_tz_format', 'YYYY-MM-DD HH24:MI:SS.FF TZR TZD');
# reset
sys_tenant_id = 1
change_tenant(cur, sys_tenant_id)
def get_sys_vars_upgrade_dmls_str(cur, query_cur, tenant_id_list, update_sys_var_list, add_sys_var_list):
ret_str = ''
if len(tenant_id_list) <= 0:
logging.error('distinct tenant id count is <= 0, tenant_id_count: %d', len(tenant_id_list))
raise MyError('invalid arg')
for i in range(0, len(tenant_id_list)):
tenant_id = tenant_id_list[i]
change_tenant(cur, tenant_id)
if i > 0:
ret_str += '\n'
ret_str += gen_sys_var_update_sqls_for_tenant(query_cur, tenant_id, update_sys_var_list)
if ret_str != '' and len(add_sys_var_list) > 0:
ret_str += '\n'
for i in range(0, len(tenant_id_list)):
tenant_id = tenant_id_list[i]
change_tenant(cur, tenant_id)
if i > 0:
ret_str += '\n'
ret_str += gen_sys_var_add_sqls_for_tenant(query_cur, tenant_id, add_sys_var_list)
if ret_str != '' and gen_sys_var_special_update_sqls_for_tenant(tenant_id_list[0]) != '':
ret_str += '\n'
for i in range(0, len(tenant_id_list)):
tenant_id = tenant_id_list[i]
change_tenant(cur, tenant_id)
if i > 0:
ret_str += '\n'
ret_str += gen_sys_var_special_update_sqls_for_tenant(tenant_id)
sys_tenant_id= 1
change_tenant(cur, sys_tenant_id)
return ret_str
def special_update_sys_vars_for_tenant(dml_cur, tenant_id, add_sys_var_list, sys_var_name, sys_var_value):
try:
sys_var = None
for i in range(0, len(add_sys_var_list)):
if (sys_var_name == add_sys_var_list[i]["name"]):
sys_var = add_sys_var_list[i]
break;
if None == sys_var:
logging.info('%s is not new, no need special update again', sys_var_name)
return
sys_var["value"] = sys_var_value;
update_sql = gen_update_sys_var_value_sql_for_tenant(tenant_id, sys_var)
rowcount = dml_cur.exec_update(update_sql)
if 1 != rowcount:
# 以history为准，考虑可重入，此处不校验__all_sys_variable的更新结果
logging.info('sys var not change, just skip, sql: %s, tenant_id: %d', update_sql, tenant_id)
else:
logging.info('succeed to update sys var for tenant, sql: %s, tenant_id: %d', update_sql, tenant_id)
#replace update sys var to __all_sys_variable_history
replace_sql = gen_update_sys_var_history_sql_for_tenant(dml_cur, tenant_id, sys_var)
rowcount = dml_cur.exec_update(replace_sql)
if 1 != rowcount and 2 != rowcount:
logging.error('fail to replace sysvar, replace_sql:%s'%replace_sql)
raise MyError('fail to repalce sysvar')
else:
logging.info('succeed to replace sys var history for tenant, sql: %s, tenant_id: %d', replace_sql, tenant_id)
except Exception, e:
logging.exception('fail to add for tenant, tenant_id: %d', tenant_id)
raise e
def smote(feature_list, new_sample_num, resampling_size):
"""
SMOTEによるリサンプリング
指定された数のサンプルを作成し、返す
なお、無作為抽出のみ実装している
@param feature_list: あるクラスの特徴量のリスト リストの中身はarrayになっている
@param new_sample_num: 作成するサンプル数
@param resampling_size: リサンプリングに使う数
@return 新しいサンプルのリスト list型 配列の中身はarray型
bootstrapと異なり、duplicationやrandomオブジェクトはなし、選択はsmoteライブラリ内で行われるので
"""
# 与えられたサンプル数がリサンプリング手法での抽出数に満たない場合
if len(feature_list) < resampling_size:
raise MyError(
"Sample Size is smaller than the number of resampling size.")
# 一旦array型に変換
feature_array = np.asarray(feature_list, dtype=np.float32)
# SMOTEを実行
percentage_new_sample = int(
round(float(new_sample_num) / len(feature_array) * 100))
new_feature_array = ext_smote.SMOTE(feature_array, percentage_new_sample,
resampling_size)
# 生成された各サンプルをリストに追加して行く
new_feature = []
for i in range(new_sample_num):
new_feature.append(new_feature_array[i])
return new_feature
def results_to_str(desc, results):
ret_str = ''
max_width_list = []
for col_desc in desc:
max_width_list.append(len(str(col_desc[0])))
col_count = len(max_width_list)
for result in results:
if col_count != len(result):
raise MyError(
'column count is not equal, desc column count: {0}, data column count: {1}'
.format(col_count, len(result)))
for i in range(0, col_count):
result_col_width = len(str(result[i]))
if max_width_list[i] < result_col_width:
max_width_list[i] = result_col_width
# 打印列名
for i in range(0, col_count):
if i > 0:
ret_str += ' ' # 空四格
ret_str += str(desc[i][0])
# 补足空白
for j in range(0, max_width_list[i] - len(str(desc[i][0]))):
ret_str += ' '
# 打印数据
for result in results:
ret_str += '\n' # 先换行
for i in range(0, col_count):
if i > 0:
ret_str += ' ' # 空四格
ret_str += str(result[i])
# 补足空白
for j in range(0, max_width_list[i] - len(str(result[i]))):
ret_str += ' '
return ret_str
def actions_cls_compare(x, y):
diff = x.get_seq_num() - y.get_seq_num()
if 0 == diff:
raise MyError('seq num is equal')
elif diff < 0:
return -1
else:
return 1
def assign_piece(self, piece, row, col):
"""This method places a piece from GoPiece to a specific position on
the game board. It returns an error if the specified position is
invalid or if the position is already taken.
"""
row = int(row)
col = int(col)
row_index = row - 1
col_index = col - 1
if row > self.__board_size or row < 1:
raise MyError('Invalid position.')
elif col > self.__board_size or col < 1:
raise MyError('Invalid position.')
elif self.__go_board[row_index][col_index] != " - ":
raise MyError('Position is occupied.')
else:
self.__go_board[row_index][col_index]=piece
def deal_with_local_opts():
global g_opts
if has_no_local_opts():
raise MyError('no local options, can not deal with local options')
else:
for opt in g_opts:
if opt.is_local_opt() and opt.has_value():
deal_with_local_opt(opt)
# 只处理一个
return
def bootstrap(feature_list, new_sample_num, resampling_size, randobj, duplication=True):
"""
ブートストラップによるリサンプリング
指定された数のサンプルを作成し、返す
なお、無作為抽出のみ実装している
@param feature_list: 特徴量のリスト 配列の中身はarrayになっている
@param new_sample_num: 作成するサンプル数
@param resampling_size: リサンプリングに使う数
@param randobj: 乱数モジュール
@param duplication: 親の重複を許すか否か（defaultはTrue）
@return 新しいサンプルのリスト list型 配列の中身はarray型
"""
# 与えられたサンプル数がリサンプリング手法での抽出数に満たない場合
if len(feature_list) < resampling_size:
raise MyError("Sample Size is smaller than the number of resampling size.")
# 無作為抽出によるサンプルの生成
new_feature_list = []
index_list = range(len(feature_list))
if duplication:
for s in range(new_sample_num):
randobj.shuffle(index_list)
group_index_list = index_list[:resampling_size]
#print group_index_list
new_feature = np.average(feature_list[group_index_list], axis=0) # 行間で平均 @UndefinedVariable
#print new_feature
new_feature_list.append(list(new_feature))
else:
# 全ての組合せを生成
combi = list(itertools.combinations(index_list, resampling_size))
# 組合せの数が生成数に足りない場合
if len(combi) < new_sample_num:
raise MyError("Sample Combination Size is smaller than the number of resampling size.")
# ランダマイズ
randobj.shuffle(combi)
# 必要なサンプル数を選択し、特徴量を平均
for s in range(new_sample_num):
new_feature = np.average([feature_list[i] for i in combi[s]], axis=0)
new_feature_list.append(list(new_feature))
return new_feature_list
def __init__(self, color='black'):
"""This method has a private attribute named 'color' (default value is
'black') which must have a value of either 'black' or 'white'. The
method creates a Gomoku piece, and raises an error if the color is not
'black' or 'white'.
"""
if str(color).lower().strip() != 'black' and \
str(color).lower().strip() != 'white':
raise MyError('Wrong color.')
else:
self.__color = str(color).lower().strip()
def __init__(self,
short_name,
long_name,
is_with_param,
is_local_opt,
default_value=None):
if short_name in Option.__g_short_name_set:
raise MyError(
'duplicate option short name: {0}'.format(short_name))
elif long_name in Option.__g_long_name_set:
raise MyError('duplicate option long name: {0}'.format(long_name))
Option.__g_short_name_set.add(short_name)
Option.__g_long_name_set.add(long_name)
self.__short_name = short_name
self.__long_name = long_name
self.__is_with_param = is_with_param
self.__is_local_opt = is_local_opt
self.__has_value = False
if None != default_value:
self.set_value(default_value)
def main():
"""The main function of the program creates an instance of the Gomoku
class. It then lets the two players take turns placing pieces on the game
board until a player wins or until a player quits by entering "q". The
board is printed and the board is checked for a winner after each player
takes their turn. Any errors are dealt with by this function by displaying
an error message and reprompting the player to place their piece."""
board = Gomoku()
print(board)
play = input('Input a row then column separated by a comma (q to quit): ')
while play.lower() != 'q':
play_list = play.strip().split(',')
while play.lower() != 'q':
try:
if len(play_list) != 2:
raise MyError('Incorrect input.')
row = int(play_list[0].strip())
col = int(play_list[1].strip())
break
except ValueError:
print('Incorrect input.')
print('Try again.')
print(board)
play = input('Input a row then column separated by\
a comma (q to quit): ')
play_list = play.strip().split(',')
except MyError as error_message:
print(error_message)
print('Try again.')
print(board)
play = input('Input a row then column separated by a\
comma (q to quit): ')
play_list = play.strip().split(',')
try:
current_player = board.get_current_player()
piece = GoPiece(current_player)
board.assign_piece(piece,row,col)
if board.current_player_is_winner():
print(board)
print('{} wins!'.format(current_player))
return
else:
board.switch_current_player()
except MyError as error_message:
print('{:s}\nTry again.'.format(str(error_message)))
print(board)
play = input('Input a row then column separated \
by a comma (q to quit): ')
def __init__(self, board_size=15, win_count=5, current_player='black'):
"""This method has four private attributes. board_size (default value
is 15) represents the size of the game board. win_count (default value
is 5) is the number of pieces in a row needed to win the game.
current_player (default value is 'black') is the color of the current
player. go_board is a list of row lists representing the game board.
"""
if str(current_player).lower().strip() != 'black' and \
str(current_player).lower().strip() != 'white':
raise MyError('Wrong color.')
else:
self.__current_player = str(current_player).lower().strip()
self.__win_count = int(win_count)
self.__board_size = int(board_size)
self.__go_board = [[' - ' for j in range(self.__board_size)] \
for i in range(self.__board_size)]
def do_upgrade_by_argv(argv):
upgrade_params = UpgradeParams()
opts.change_opt_defult_value('log-file', upgrade_params.log_filename)
opts.parse_options(argv)
if not opts.has_no_local_opts():
opts.deal_with_local_opts()
else:
opts.check_db_client_opts()
log_filename = opts.get_opt_log_file()
upgrade_params.log_filename = log_filename
# 日志配置放在这里是为了前面的操作不要覆盖掉日志文件
config_logging_module(upgrade_params.log_filename)
try:
host = opts.get_opt_host()
port = int(opts.get_opt_port())
user = opts.get_opt_user()
password = opts.get_opt_password()
cmd_module_str = opts.get_opt_module()
module_set = set([])
all_module_set = run_modules.get_all_module_set()
cmd_module_list = cmd_module_str.split(',')
for cmd_module in cmd_module_list:
if run_modules.ALL_MODULE == cmd_module:
module_set = module_set | all_module_set
elif cmd_module in all_module_set:
module_set.add(cmd_module)
else:
raise MyError('invalid module: {0}'.format(cmd_module))
logging.info('parameters from cmd: host=\"%s\", port=%s, user=\"%s\", password=\"%s\", module=\"%s\", log-file=\"%s\"',\
host, port, user, password, module_set, log_filename)
do_upgrade(host, port, user, password, module_set, upgrade_params)
except mysql.connector.Error, e:
logging.exception('mysql connctor error')
logging.exception('run error, maybe you can reference ' +
upgrade_params.rollback_sql_filename +
' to rollback it')
raise e
except Exception, e:
logging.exception('normal error')
logging.exception('run error, maybe you can reference ' +
upgrade_params.rollback_sql_filename +
' to rollback it')
raise e
def do_add_recovery_status_to_all_zone(conn, cur):
try:
logging.info('add recovery status row to __all_zone for each zone')
zones = []
recovery_status = []
# pre-check, may skip
check_updated_sql = "select * from oceanbase.__all_zone where zone !='' AND name='recovery_status'"
cur.execute(check_updated_sql)
recovery_status = cur.fetchall()
if 0 < len(recovery_status):
logging.info(
'[recovery_status] row already exists, no need to add')
# get zones
if 0 >= len(recovery_status):
all_zone_sql = "select distinct(zone) zone from oceanbase.__all_zone where zone !=''"
cur.execute(all_zone_sql)
zone_results = cur.fetchall()
for r in zone_results:
zones.append("('" + r[0] +
"', 'recovery_status', 0, 'NORMAL')")
# add rows
if 0 < len(zones):
upgrade_sql = "insert into oceanbase.__all_zone(zone, name, value, info) values " + ','.join(
zones)
logging.info(upgrade_sql)
cur.execute(upgrade_sql)
conn.commit()
# check result
if 0 < len(zones):
cur.execute(check_updated_sql)
check_results = cur.fetchall()
if len(check_results) != len(zones):
raise MyError(
'fail insert [recovery_status] row into __all_zone')
except Exception, e:
logging.exception('do_add_recovery_status_to_all_zone error')
raise e
def add_sys_vars_for_tenant(dml_cur, tenant_id, add_sys_var_list):
try:
for i in range(0, len(add_sys_var_list)):
sys_var = add_sys_var_list[i]
add_sql = gen_add_sys_var_sql_for_tenant(tenant_id, sys_var)
rowcount = dml_cur.exec_update(add_sql)
if 1 != rowcount:
# 以history为准，考虑可重入，此处不校验__all_sys_variable的更新结果
logging.info('sys var not change, just skip, sql: %s, tenant_id: %d', update_sql, tenant_id)
else:
logging.info('succeed to insert sys var for tenant, sql: %s, tenant_id: %d', add_sql, tenant_id)
replace_sql = gen_replace_sys_var_history_sql_for_tenant(dml_cur, tenant_id, sys_var)
rowcount = dml_cur.exec_update(replace_sql)
if 1 != rowcount:
logging.error('fail to replace system variable history, sql:%s'%replace_sql)
raise MyError('fail to replace system variable history')
else:
logging.info('succeed to replace sys var for tenant, sql: %s, tenant_id: %d', replace_sql, tenant_id)
except Exception, e:
logging.exception('fail to add for tenant, tenant_id: %d', tenant_id)
raise e
def update_sys_vars_for_tenant(dml_cur, tenant_id, update_sys_var_list):
try:
for i in range(0, len(update_sys_var_list)):
sys_var = update_sys_var_list[i]
update_sql = gen_update_sys_var_sql_for_tenant(tenant_id, sys_var)
rowcount = dml_cur.exec_update(update_sql)
if 1 != rowcount:
# 以history为准，考虑可重入，此处不校验__all_sys_variable的更新结果
logging.info('sys var not change, just skip, sql: %s, tenant_id: %d', update_sql, tenant_id)
else:
logging.info('succeed to update sys var for tenant, sql: %s, tenant_id: %d', update_sql, tenant_id)
#replace update sys var to __all_sys_variable_history
replace_sql = gen_update_sys_var_history_sql_for_tenant(dml_cur, tenant_id, sys_var)
rowcount = dml_cur.exec_update(replace_sql)
if 1 != rowcount and 2 != rowcount:
logging.error('fail to replace sysvar, replace_sql:%s'%replace_sql)
raise MyError('fail to repalce sysvar')
else:
logging.info('succeed to replace sys var history for tenant, sql: %s, tenant_id: %d', replace_sql, tenant_id)
except Exception, e:
logging.exception('fail to update for tenant, tenant_id: %d', tenant_id)
raise e
def optimizeParameter(multi_svm_model,
kernel_type,
feature_array,
label_array,
_fold=5,
_costs=None,
_gammas=None):
"""
Grid Searchによるパラメータ最適化
@param multi_svm_model: SVMの多クラス用モデル
@param kernel_type: カーネル種別
@param feature_array: 特徴量
@param label_array: ラベル
@keyword _fold: フォールド数
@keyword _costs: コストのリスト(default = 2^-5, 2^-3, ..., 2^13, 2^15)
@param _gammas: ガンマのリスト(default = 2^3, 2^1, ..., 2^-13, 2^-15)
@return: 最適化済コスト値、ガンマ値
コストとガンマはlibSVMを参照した
-log2c {begin,end,step | "null"} : set the range of c (default -5,15,2)
begin,end,step -- c_range = 2^{begin,...,begin+k*step,...,end}
"null" -- do not grid with c
-log2g {begin,end,step | "null"} : set the range of g (default 3,-15,-2)
begin,end,step -- g_range = 2^{begin,...,begin+k*step,...,end}
"null" -- do not grid with g
"""
# クラスの適合性Check
if type(multi_svm_model) != OneVsRestClassifier:
raise MyError(
"SVM model's type is invalid. It must be OneVsRestClassifier.")
# コストとガンマのdefault値の設定
if _costs == None:
_costs = [2**i for i in range(-5, 17, 2)]
if _gammas == None:
_gammas = [2**i for i in range(3, -17, -2)]
# 線形の場合はガンマ値を0にしておく
if kernel_type == "linear":
_gammas = [1.0]
best_c = _costs[0]
best_gamma = _gammas[0]
best_acc = 0
for c in _costs:
for g in _gammas:
# コスト値とガンマ値のセット
# 多クラス分類器(OneVsRestClassifier)にセットしているSVCに、estimator変数でアクセスする
multi_svm_model.estimator.set_params(C=c, gamma=g)
# クロスバリデーション（N-fold数分の識別率が返ってくる）
acc_list = skcr.cross_val_score(multi_svm_model,
feature_array,
label_array,
cv=_fold)
acc = np.average(acc_list)
# 最大の識別率を確保
if acc > best_acc:
best_c = c
best_g = g
best_acc = acc
# print multi_svm_model.estimator.get_params()
# print multi_svm_model.estimators_
return best_c, best_g
def check_db_client_opts():
global g_opts
for opt in g_opts:
if not opt.is_local_opt() and not opt.has_value():
raise MyError('option "-{0}" has not been specified, maybe you should run "{1} --help" for help'\
.format(opt.get_short_name(), sys.argv[0]))
def do_upgrade(my_host, my_port, my_user, my_passwd, my_module_set,
upgrade_params):
try:
conn = mysql.connector.connect(user=my_user,
password=my_passwd,
host=my_host,
port=my_port,
database='oceanbase',
raise_on_warnings=True)
cur = conn.cursor(buffered=True)
try:
query_cur = actions.QueryCursor(cur)
# 开始升级前的检查
check_before_upgrade(query_cur, upgrade_params)
# get min_observer_version
version = actions.fetch_observer_version(query_cur)
need_check_standby_cluster = cmp(version, '2.2.40') >= 0
# 获取租户id列表
tenant_id_list = actions.fetch_tenant_ids(query_cur)
if len(tenant_id_list) <= 0:
logging.error(
'distinct tenant id count is <= 0, tenant_id_count: %d',
len(tenant_id_list))
raise MyError('no tenant id')
logging.info(
'there has %s distinct tenant ids: [%s]', len(tenant_id_list),
','.join(str(tenant_id) for tenant_id in tenant_id_list))
# 计算需要添加或更新的系统变量
conn.commit()
conn.autocommit = True
(update_sys_var_list, update_sys_var_ori_list,
add_sys_var_list) = upgrade_sys_vars.calc_diff_sys_var(
cur, tenant_id_list[0])
dump_sql_to_file(cur, query_cur, upgrade_params.sql_dump_filename,
tenant_id_list, update_sys_var_list,
add_sys_var_list)
conn.autocommit = False
conn.commit()
logging.info('update system variables list: [%s]',
', '.join(str(sv) for sv in update_sys_var_list))
logging.info('update system variables original list: [%s]',
', '.join(str(sv) for sv in update_sys_var_ori_list))
logging.info('add system variables list: [%s]',
', '.join(str(sv) for sv in add_sys_var_list))
logging.info(
'================succeed to dump sql to file: {0}==============='
.format(upgrade_params.sql_dump_filename))
if run_modules.MODULE_DDL in my_module_set:
logging.info('================begin to run ddl===============')
conn.autocommit = True
normal_ddl_actions_pre.do_normal_ddl_actions(cur)
logging.info(
'================succeed to run ddl===============')
conn.autocommit = False
if run_modules.MODULE_NORMAL_DML in my_module_set:
logging.info(
'================begin to run normal dml===============')
normal_dml_actions_pre.do_normal_dml_actions(cur)
logging.info(
'================succeed to run normal dml===============')
conn.commit()
actions.refresh_commit_sql_list()
logging.info(
'================succeed to commit dml===============')
if run_modules.MODULE_EACH_TENANT_DML in my_module_set:
logging.info(
'================begin to run each tenant dml==============='
)
conn.autocommit = True
each_tenant_dml_actions_pre.do_each_tenant_dml_actions(
cur, tenant_id_list)
conn.autocommit = False
logging.info(
'================succeed to run each tenant dml==============='
)
# 更新系统变量
if run_modules.MODULE_SYSTEM_VARIABLE_DML in my_module_set:
logging.info(
'================begin to run system variable dml==============='
)
conn.autocommit = True
upgrade_sys_vars.exec_sys_vars_upgrade_dml(cur, tenant_id_list)
conn.autocommit = False
logging.info(
'================succeed to run system variable dml==============='
)
if run_modules.MODULE_SPECIAL_ACTION in my_module_set:
logging.info(
'================begin to run special action==============='
)
conn.autocommit = True
special_upgrade_action_pre.do_special_upgrade(
conn, cur, tenant_id_list, my_user, my_passwd)
conn.autocommit = False
actions.refresh_commit_sql_list()
logging.info(
'================succeed to commit special action==============='
)
except Exception, e:
logging.exception('run error')
raise e
finally:
# 打印统计信息
print_stats()
# 将回滚sql写到文件中
actions.dump_rollback_sql_to_file(
upgrade_params.rollback_sql_filename)
cur.close()
conn.close()