def batch_consumers(fi_id, generator, q, batch_limit=PER_THREAD_BATCH_LIMIT):
logger.info("Starting API batching for FI {}".format(fi_id))
# iterate over each consumer
num_batches = 0
for batch in generator:
api_batch = []
num_batches += 1
# TODO: Modify generator SQL to name fields instead of iterating over them all
for row in batch:
api_batch.append(map_row_to_marketingcloud_api(row, fi_id))
# send to MC when we have gathered enough for batch limit
while True:
try:
q.put({
"batch_num": num_batches,
"fi_id": fi_id,
"data": api_batch
}, False)
break
except Full:
sleep(1)
except Exception:
logger.exception('Failed to put data on the worker queue')
logger.debug("Queuing batch {} for FI {} from thread {}".format(num_batches, fi_id,
threading.current_thread().getName()))
logger.info(f"Completed delivery of marketing cloud messages for FI {fi_id}, {num_batches} were delivered")
def delete_thirdlevel(self):
try:
condition_list = list()
module = self.comboBox_module.currentText()
condition_list.clear()
condition_list.append("module='" + module + "'")
module_id = DBManager().query("modules", "module_id", condition_list)[0]
condition_list.clear()
try:
# 获取当前选中的对象list，currentItems是当前点过的选项而不是选中的
delete_data_object = self.listWidget_thirdlevel.selectedItems()
delete_data_list = list()
for i in delete_data_object:
delete_data_list.append(i.text())
except:
delete_data_object = None
if delete_data_object is not None:
for i in delete_data_list:
logger.debug(i)
thirdlevel_data = "thirdlevel_element='" + i + "'"
logger.debug(thirdlevel_data)
condition_list.clear()
condition_list.append(thirdlevel_data)
condition_list.append("module_id='" + str(module_id) + "'")
logger.debug(condition_list)
DBManager().delete("thirdlevel", condition_list)
self.listWidget_thirdlevel_handle()
except:
logger.exception("发现错误：")
def sublevel_help(self):
try:
self.help_widget = QtWidgets.QWidget()
self.window_help = Ui_Form_Help()
self.window_help.setupUi(self.help_widget)
except:
logger.exception("发现错误:")
def listWidget_casemodel_handle(self):
try:
self.listWidget_caseModel.clear()
condition_List = list()
module = self.comboBox_module.currentText()
if module is not "":
condition_List.clear()
condition_List.append("module='" + module + "'")
module_id = DBManager().query("modules", "module_id", condition_List)[0]
condition_List.clear()
condition_List.append("module_id='" + str(module_id) + "'")
logger.debug("=====================================================================")
logger.debug(condition_List)
toplevel_element_list = DBManager().query("casemodel", "toplevel_element", condition_List)
logger.debug(toplevel_element_list)
sublevel_element_list = DBManager().query("casemodel", "sublevel_element", condition_List)
thirdlevel_element_list = DBManager().query("casemodel", "thirdlevel_element", condition_List)
casemodel_list = list()
for j in range(len(toplevel_element_list)):
casemodel_list.append(
toplevel_element_list[j] + "-" + sublevel_element_list[j] + ":" + thirdlevel_element_list[j])
logger.debug(casemodel_list)
for k in casemodel_list:
self.listWidget_caseModel.addItem(k)
except:
logger.exception("发现错误：")
def time_stamp(cls):
try:
# 格式化
time_format1 = '%Y%m%d%H%M%S'
return time.strftime(time_format1)
except Exception as e:
logger.error(e)
logger.exception(u"捕获到错误如下:")
def ytdyccrm_cancel_func(self):
try:
self.__data_dict["dt"] = self.__time_stamp
sign_str = str(self.__data_dict["dt"]) + self.__data_dict["method"] + str(2.0) + self.__secretkey_value
sign = Md5.md5(sign_str)
self.__data_dict["sign"] = sign
return self.__data_dict
except Exception as e:
logger.error(e)
logger.exception(u"捕获到错误如下:")
def __init__(self, interface_code, data_dict, secretkey_value):
try:
# 生成时间戳
self.__time_stamp = TimeStamp.time_stamp()
self.__data_dict = data_dict
self.__secretkey_value = secretkey_value
self.__interface_code = interface_code
except Exception as e:
logger.error(e)
logger.exception(u"捕获到错误如下:")
def processing(self):
try:
processing_dict = dict()
# 字典生成key和value的对应关系，value为方法对象而不是执行方法
processing_dict["ytdyccrmcancel"] = self.ytdyccrm_cancel_func
if self.__interface_code in processing_dict:
return processing_dict[self.__interface_code]()
else:
return 0
except Exception as e:
logger.error(e)
logger.exception(u"捕获到错误如下:")
def urldecoded(self, data):
try:
data_list_first = data.split("&")
data_list_second = list()
for tmp in data_list_first:
data_list_second.append(tmp.split("="))
result = dict()
for i in data_list_second:
result[i[0]] = i[1]
return result
except Exception as e:
logger.error(e)
logger.exception(u"捕获到错误如下:")
def listWidget_sublevel_handle(self, comboBox_current_data):
try:
self.listWidget_sublevel.clear()
condition_List = list()
condition_List.append("toplevel_element='" + comboBox_current_data + "'")
toplevel_id = DBManager().query("toplevel", "toplevel_id", condition_List)
condition_List.clear()
condition_List.append("toplevel_id=" + toplevel_id[0])
sublevel_element = DBManager().query("sublevel", "sublevel_element", condition_List)
for i in sublevel_element:
self.listWidget_sublevel.addItem(i)
self.listWidget_sublevel.setCurrentRow(0)
except:
logger.exception("发现错误：")
def clear_all(self):
try:
listWidget_count = self.listWidget_caseModel.count()
module = self.comboBox_module.currentText()
condition_list = list()
condition_list.clear()
condition_list.append("module='" + module + "'")
module_id = DBManager().query("modules", "module_id", condition_list)[0]
condition_list.clear()
condition_list.append("module_id='" + module_id + "'")
while (listWidget_count != 0):
DBManager().delete("casemodel", condition_list)
listWidget_count = listWidget_count - 1
self.listWidget_caseModel.clear()
except:
logger.exception("发现错误：")
def add_to_casemodel(self):
try:
module = self.comboBox_module.currentText()
condition_List = list()
condition_List.clear()
condition_List.append("module='" + module + "'")
module_id = DBManager().query("modules", "module_id", condition_List)[0]
toplevel_element = self.comboBox_toplevel.currentText()
items_sublevel = self.listWidget_sublevel.selectedItems()
items_thirdlevel = self.listWidget_thirdlevel.selectedItems()
for i_sublevel in items_sublevel:
sublevel_element = i_sublevel.text()
condition_List.clear()
condition_List.append("sublevel_element='" + i_sublevel.text() + "'")
sublevel_id = DBManager().query("sublevel", "sublevel_id", condition_List)[0]
# 使用字典记录即将添加的值，并调用公用方法添加数据
for i_thirdlevel in items_thirdlevel:
data_dict = dict()
condition_List.clear()
condition_List.append("sublevel_id='" + sublevel_id + "'")
logger.debug(sublevel_id)
condition_List.append("module_id='" + str(module_id) + "'")
condition_List.append("thirdlevel_element='" + i_thirdlevel.text() + "'")
logger.debug(condition_List)
thirdlevel_id = DBManager().query("thirdlevel", "thirdlevel_id", condition_List)[0]
condition_List.clear()
condition_List.append("module_id='" + str(module_id) + "'")
condition_List.append("thirdlevel_element='" + i_thirdlevel.text() + "'")
casemodel_thirdlevel_id = DBManager().query("casemodel", "thirdlevel_id", condition_List)
if thirdlevel_id not in casemodel_thirdlevel_id:
data_dict["thirdlevel_id"] = thirdlevel_id
thirdlevel_element = i_thirdlevel.text()
data_dict["thirdlevel_element"] = thirdlevel_element
data_dict["sublevel_id"] = sublevel_id
data_dict["sublevel_element"] = sublevel_element
condition_List.clear()
condition_List.append("toplevel_element='" + toplevel_element + "'")
toplevel_id = DBManager().query("toplevel", "toplevel_id", condition_List)[0]
data_dict["toplevel_id"] = toplevel_id
data_dict["toplevel_element"] = toplevel_element
data_dict["module_id"] = module_id
logger.debug(data_dict)
DBManager().insert_data("casemodel", data_dict)
self.listWidget_casemodel_handle()
except:
logger.exception("发现错误：")
def to_excel(self):
try:
# toplevel_id在sublevel表中个数索引从1开始
toplevel_count_list = list()
condition_list = list()
module = self.comboBox_module.currentText()
condition_list.clear()
condition_list.append("module='" + module + "'")
module_id = DBManager().query("modules", "module_id", condition_list)[0]
logger.debug(module_id)
logger.debug(condition_list)
for i in range(4):
i = i + 1
condition_list.clear()
condition_list.append("toplevel_id='" + str(i) + "'")
toplevel_count_list.append(len(DBManager().query("sublevel", "sublevel_elemnt", condition_list)))
TEMPLATE_FILE = "./templates/测试建模模板.xlsx"
SHEET_COUNT = 4
real_index_list = list()
condition_list.clear()
condition_list.append("module_id='" + str(module_id) + "'")
toplevel_id = DBManager().query("casemodel", "toplevel_id", condition_list)
# sublevel_id数据索引从1开始
sublevel_id = DBManager().query("casemodel", "sublevel_id", condition_list)
for j in sublevel_id:
real_index = int(j)
toplevel_count_index = 0
# 由于sublevel_id数据索引和sublevel表中toplevel_id个数索引均从1开始
# 故两者之差大于0时认为当前sublevel_element为下一个顶级元素的次级元素，需要做差取真实表格索引
while ((real_index - toplevel_count_list[toplevel_count_index]) > 0):
real_index = real_index - toplevel_count_list[toplevel_count_index]
toplevel_count_index = toplevel_count_index + 1
real_index_list.append(real_index)
thirdlevel_element = DBManager().query("casemodel", "thirdlevel_element", condition_list)
data_list = list()
data_list.append(toplevel_id)
data_list.append(real_index_list)
data_list.append(thirdlevel_element)
logger.debug(data_list)
OutputWithTemplate().output_with_excel(TEMPLATE_FILE, SHEET_COUNT, data_list)
except:
logger.exception("发现错误:")
def add_thirdlevel(self):
try:
data_dict = dict()
condition_list = list()
module = self.comboBox_module.currentText()
condition_list.clear()
condition_list.append("module='" + module + "'")
module_id = DBManager().query("modules", "module_id", condition_list)[0]
thirdlevel_element = self.textEdit_thirdlevel.toPlainText()
# 由于casemodel界面展示时包含“:”和“-”，所以在处理包含这两个符号的选项时会出错，这个弹提示不允许这个的数据输入
if ":" in thirdlevel_element or "：" in thirdlevel_element or "-" in thirdlevel_element:
msg_box = QtWidgets.QMessageBox(QtWidgets.QMessageBox.Warning, "错误输入提示", "不允许输入冒号或横线！")
msg_box.exec_()
else:
items_sublevel = self.listWidget_sublevel.selectedItems()
for i_sublevel in items_sublevel:
sublevel_element = i_sublevel.text()
condition_list.clear()
condition_list.append("sublevel_element='" + i_sublevel.text() + "'")
sublevel_id = DBManager().query("sublevel", "sublevel_id", condition_list)[0]
condition_list.clear()
condition_list.append("module_id='" + module_id + "'")
module_id_thirdlevel = DBManager().query("thirdlevel", "thirdlevel_id", condition_list)
module_id_thirdlevel_toint = list()
for i in module_id_thirdlevel:
module_id_thirdlevel_toint.append(int(i))
if len(module_id_thirdlevel) == 0:
thirdlevel_id = str(1)
else:
thirdlevel_id = str(max(module_id_thirdlevel_toint) + 1)
logger.debug(thirdlevel_id)
condition_list.clear()
if thirdlevel_element is not "":
data_dict["thirdlevel_element"] = thirdlevel_element
data_dict["sublevel_id"] = sublevel_id
data_dict["module_id"] = module_id
data_dict["thirdlevel_id"] = thirdlevel_id
DBManager().insert_data("thirdlevel", data_dict)
self.textEdit_thirdlevel.clear()
self.listWidget_thirdlevel_handle()
except:
logger.exception("发现错误：")
def delete_selection(self):
try:
condition_list = list()
module = self.comboBox_module.currentText()
condition_list.clear()
condition_list.append("module='" + module + "'")
module_id = DBManager().query("modules", "module_id", condition_list)[0]
condition_list.clear()
try:
delete_data = self.listWidget_caseModel.currentItem().text()
except:
delete_data = ""
if delete_data is not "":
thirdlevel_data = "thirdlevel_element='" + delete_data.split(":")[1] + "'"
condition_list.append(thirdlevel_data)
condition_list.append("module_id='" + str(module_id) + "'")
logger.debug(condition_list)
DBManager().delete("casemodel", condition_list)
self.listWidget_casemodel_handle()
except:
logger.exception("发现错误：")
def listWidget_thirdlevel_handle(self):
try:
# 获取列表中选中的值
self.listWidget_thirdlevel.clear()
condition_List = list()
module = self.comboBox_module.currentText()
condition_List.clear()
condition_List.append("module='" + module + "'")
module_id = DBManager().query("modules", "module_id", condition_List)[0]
thirdlevel_widget = self.listWidget_sublevel.currentItem()
row_data = thirdlevel_widget.text()
condition_List.clear()
condition_List.append("sublevel_element='" + row_data + "'")
sublevel_id = DBManager().query("sublevel", "sublevel_id", condition_List)
condition_List.clear()
condition_List.append("sublevel_id='" + sublevel_id[0] + "'")
condition_List.append("module_id='" + str(module_id) + "'")
sublevel_element = DBManager().query("thirdlevel", "thirdlevel_element", condition_List)
for i in sublevel_element:
self.listWidget_thirdlevel.addItem(i)
except:
logger.exception("发现错误：")
def output_with_excel(self, template_file, sheet_count, data_list):
'''
:param template_file: 模板文件路径
:param sheet_count: 模板文件容sheet的个数，比如5个sheet页通过这个参数可以指定就用3个
:param data_list: 数据list，其中包含sheet页list，表头list，数据list
本方法中均使用list进行数据操作，由于模板中第一行为标题，故插入数据的行数索引从1开始
'''
try:
data = xlrd.open_workbook(template_file)
sheet_name = data.sheet_names()
sheet_list = list()
# 将模板文件sheet对象存入sheet_list列表
for i in range(sheet_count):
sheet_list.append(data.sheets()[i])
template_data_list = list()
# 将模板文件每个sheet的数据list存入tempLate_data_list列表
for i_sheet in sheet_list:
template_data_list.append(i_sheet.row_values(0))
output_file = xlwt.Workbook()
# 字体设定
font = xlwt.Font()
font.bold = True
# 边框设定
borders = xlwt.Borders()
borders.left = 1
borders.right = 1
borders.top = 1
borders.bottom = 1
# 背景色设定
backcolor = xlwt.Pattern()
backcolor.pattern = xlwt.Pattern.SOLID_PATTERN
backcolor.pattern_fore_colour = 5
style = xlwt.XFStyle()
style.font = font
style.borders = borders
style.pattern = backcolor
output_file_sheet_list = list()
# 输出文件添加sheet并给予标题,对象存入output_file_sheet_list列表
for j in range(sheet_count):
output_file_sheet_list.append(output_file.add_sheet(sheet_name[j], cell_overwrite_ok=True))
# j_sheet_num为sheet页的索引
for j_sheet_num in range(len(sheet_list)):
# j_sheet_row_index为模板文件template_dataList的索引
for j_sheet_row_index in range(len(template_data_list[j_sheet_num])):
output_file_sheet_list[j_sheet_num].write(0, j_sheet_row_index,
template_data_list[j_sheet_num][j_sheet_row_index], style)
# 数据格式为{"sheet页数":{"列数":[当前添加的数据1,当前添加的数据2]}}
filter_dict = dict()
logger.debug("++++++++++++++++++++++++++++++")
logger.debug(data_list)
# 以追加方式添加数据
for data_num in range(len(data_list[0])):
logger.debug(data_num)
logger.debug(data_list)
# 当前sheet的索引
data_sheet_num = int(data_list[0][data_num]) - 1
data_row_num = 1
data_col_num = int(data_list[1][data_num]) - 1
logger.debug("===================================")
logger.debug(data_sheet_num)
logger.debug(data_col_num)
# 判断当前sheet页数是否在filter_dict中，即当前sheet是否曾经出现过
if str(data_sheet_num) not in filter_dict:
# 当前sheet页数不存在filter_dict,添加对应data_sheet_num字典
filter_dict[str(data_sheet_num)] = {}
# 增加对应列数的数据list
filter_dict[str(data_sheet_num)][str(data_col_num)] = list()
# 由于sheet页数第一次出现，故数据也一定第一次出现，所以之列列表中添加值
filter_dict[str(data_sheet_num)][str(data_col_num)].append(data_col_num)
else:
# 当前sheet页数存在filter_dict中，判断当前列数是否在列数字典中，即当前列数是否出现过
if str(data_col_num) not in filter_dict[str(data_sheet_num)]:
# 当前列数未出现过,添加列表
filter_dict[str(data_sheet_num)][str(data_col_num)] = list()
# 向列数字典列表插数据，但插入数据的当前行数为初始值，即1
filter_dict[str(data_sheet_num)][str(data_col_num)].append(data_col_num)
else:
# 当前列数出现过。向列数字典列表插数据，插入数据的当前行数为总出现的次数
filter_dict[str(data_sheet_num)][str(data_col_num)].append(data_col_num)
data_row_num = len(filter_dict[str(data_sheet_num)][str(data_col_num)])
logger.debug(filter_dict)
data_preview_add = data_list[2][data_num]
# 向excel中插入数据
output_file_sheet_list[data_sheet_num].write(data_row_num, data_col_num, data_preview_add)
output_file.save(str(TimeStamp.time_stamp()) + ".xls")
except:
logger.exception("发现错误")