Exemple #1
0
def batch_consumers(fi_id, generator, q, batch_limit=PER_THREAD_BATCH_LIMIT):
    logger.info("Starting API batching for FI {}".format(fi_id))

    # iterate over each consumer
    num_batches = 0

    for batch in generator:
        api_batch = []
        num_batches += 1
        # TODO: Modify generator SQL to name fields instead of iterating over them all
        for row in batch:
            api_batch.append(map_row_to_marketingcloud_api(row, fi_id))

        # send to MC when we have gathered enough for batch limit

        while True:
            try:
                q.put({
                    "batch_num": num_batches,
                    "fi_id": fi_id,
                    "data": api_batch
                }, False)
                break
            except Full:
                sleep(1)
            except Exception:
                logger.exception('Failed to put data on the worker queue')
            
        logger.debug("Queuing batch {} for FI {} from thread {}".format(num_batches, fi_id,
                                                                       threading.current_thread().getName()))

    logger.info(f"Completed delivery of marketing cloud messages for FI {fi_id}, {num_batches} were delivered")
Exemple #2
0
 def delete_thirdlevel(self):
     try:
         condition_list = list()
         module = self.comboBox_module.currentText()
         condition_list.clear()
         condition_list.append("module='" + module + "'")
         module_id = DBManager().query("modules", "module_id", condition_list)[0]
         condition_list.clear()
         try:
             # 获取当前选中的对象list,currentItems是当前点过的选项而不是选中的
             delete_data_object = self.listWidget_thirdlevel.selectedItems()
             delete_data_list = list()
             for i in delete_data_object:
                 delete_data_list.append(i.text())
         except:
             delete_data_object = None
         if delete_data_object is not None:
             for i in delete_data_list:
                 logger.debug(i)
                 thirdlevel_data = "thirdlevel_element='" + i + "'"
                 logger.debug(thirdlevel_data)
                 condition_list.clear()
                 condition_list.append(thirdlevel_data)
                 condition_list.append("module_id='" + str(module_id) + "'")
                 logger.debug(condition_list)
                 DBManager().delete("thirdlevel", condition_list)
                 self.listWidget_thirdlevel_handle()
     except:
         logger.exception("发现错误:")
Exemple #3
0
 def sublevel_help(self):
     try:
         self.help_widget = QtWidgets.QWidget()
         self.window_help = Ui_Form_Help()
         self.window_help.setupUi(self.help_widget)
     except:
         logger.exception("发现错误:")
Exemple #4
0
 def listWidget_casemodel_handle(self):
     try:
         self.listWidget_caseModel.clear()
         condition_List = list()
         module = self.comboBox_module.currentText()
         if module is not "":
             condition_List.clear()
             condition_List.append("module='" + module + "'")
             module_id = DBManager().query("modules", "module_id", condition_List)[0]
             condition_List.clear()
             condition_List.append("module_id='" + str(module_id) + "'")
             logger.debug("=====================================================================")
             logger.debug(condition_List)
             toplevel_element_list = DBManager().query("casemodel", "toplevel_element", condition_List)
             logger.debug(toplevel_element_list)
             sublevel_element_list = DBManager().query("casemodel", "sublevel_element", condition_List)
             thirdlevel_element_list = DBManager().query("casemodel", "thirdlevel_element", condition_List)
             casemodel_list = list()
             for j in range(len(toplevel_element_list)):
                 casemodel_list.append(
                     toplevel_element_list[j] + "-" + sublevel_element_list[j] + ":" + thirdlevel_element_list[j])
             logger.debug(casemodel_list)
             for k in casemodel_list:
                 self.listWidget_caseModel.addItem(k)
     except:
         logger.exception("发现错误:")
Exemple #5
0
 def time_stamp(cls):
     try:
         # 格式化
         time_format1 = '%Y%m%d%H%M%S'
         return time.strftime(time_format1)
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
 def ytdyccrm_cancel_func(self):
     try:
         self.__data_dict["dt"] = self.__time_stamp
         sign_str = str(self.__data_dict["dt"]) + self.__data_dict["method"] + str(2.0) + self.__secretkey_value
         sign = Md5.md5(sign_str)
         self.__data_dict["sign"] = sign
         return self.__data_dict
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
 def __init__(self, interface_code, data_dict, secretkey_value):
     try:
         # 生成时间戳
         self.__time_stamp = TimeStamp.time_stamp()
         self.__data_dict = data_dict
         self.__secretkey_value = secretkey_value
         self.__interface_code = interface_code
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
 def processing(self):
     try:
         processing_dict = dict()
         # 字典生成key和value的对应关系,value为方法对象而不是执行方法
         processing_dict["ytdyccrmcancel"] = self.ytdyccrm_cancel_func
         if self.__interface_code in processing_dict:
             return processing_dict[self.__interface_code]()
         else:
             return 0
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
 def urldecoded(self, data):
     try:
         data_list_first = data.split("&")
         data_list_second = list()
         for tmp in data_list_first:
             data_list_second.append(tmp.split("="))
             result = dict()
         for i in data_list_second:
             result[i[0]] = i[1]
         return result
     except Exception as e:
         logger.error(e)
         logger.exception(u"捕获到错误如下:")
Exemple #10
0
 def listWidget_sublevel_handle(self, comboBox_current_data):
     try:
         self.listWidget_sublevel.clear()
         condition_List = list()
         condition_List.append("toplevel_element='" + comboBox_current_data + "'")
         toplevel_id = DBManager().query("toplevel", "toplevel_id", condition_List)
         condition_List.clear()
         condition_List.append("toplevel_id=" + toplevel_id[0])
         sublevel_element = DBManager().query("sublevel", "sublevel_element", condition_List)
         for i in sublevel_element:
             self.listWidget_sublevel.addItem(i)
         self.listWidget_sublevel.setCurrentRow(0)
     except:
         logger.exception("发现错误:")
Exemple #11
0
 def clear_all(self):
     try:
         listWidget_count = self.listWidget_caseModel.count()
         module = self.comboBox_module.currentText()
         condition_list = list()
         condition_list.clear()
         condition_list.append("module='" + module + "'")
         module_id = DBManager().query("modules", "module_id", condition_list)[0]
         condition_list.clear()
         condition_list.append("module_id='" + module_id + "'")
         while (listWidget_count != 0):
             DBManager().delete("casemodel", condition_list)
             listWidget_count = listWidget_count - 1
         self.listWidget_caseModel.clear()
     except:
         logger.exception("发现错误:")
Exemple #12
0
    def add_to_casemodel(self):
        try:
            module = self.comboBox_module.currentText()
            condition_List = list()
            condition_List.clear()
            condition_List.append("module='" + module + "'")
            module_id = DBManager().query("modules", "module_id", condition_List)[0]
            toplevel_element = self.comboBox_toplevel.currentText()
            items_sublevel = self.listWidget_sublevel.selectedItems()
            items_thirdlevel = self.listWidget_thirdlevel.selectedItems()
            for i_sublevel in items_sublevel:
                sublevel_element = i_sublevel.text()
                condition_List.clear()
                condition_List.append("sublevel_element='" + i_sublevel.text() + "'")
                sublevel_id = DBManager().query("sublevel", "sublevel_id", condition_List)[0]

            # 使用字典记录即将添加的值,并调用公用方法添加数据
            for i_thirdlevel in items_thirdlevel:
                data_dict = dict()
                condition_List.clear()
                condition_List.append("sublevel_id='" + sublevel_id + "'")
                logger.debug(sublevel_id)
                condition_List.append("module_id='" + str(module_id) + "'")
                condition_List.append("thirdlevel_element='" + i_thirdlevel.text() + "'")
                logger.debug(condition_List)
                thirdlevel_id = DBManager().query("thirdlevel", "thirdlevel_id", condition_List)[0]
                condition_List.clear()
                condition_List.append("module_id='" + str(module_id) + "'")
                condition_List.append("thirdlevel_element='" + i_thirdlevel.text() + "'")
                casemodel_thirdlevel_id = DBManager().query("casemodel", "thirdlevel_id", condition_List)
                if thirdlevel_id not in casemodel_thirdlevel_id:
                    data_dict["thirdlevel_id"] = thirdlevel_id
                    thirdlevel_element = i_thirdlevel.text()
                    data_dict["thirdlevel_element"] = thirdlevel_element
                    data_dict["sublevel_id"] = sublevel_id
                    data_dict["sublevel_element"] = sublevel_element
                    condition_List.clear()
                    condition_List.append("toplevel_element='" + toplevel_element + "'")
                    toplevel_id = DBManager().query("toplevel", "toplevel_id", condition_List)[0]
                    data_dict["toplevel_id"] = toplevel_id
                    data_dict["toplevel_element"] = toplevel_element
                    data_dict["module_id"] = module_id
                    logger.debug(data_dict)
                    DBManager().insert_data("casemodel", data_dict)
            self.listWidget_casemodel_handle()
        except:
            logger.exception("发现错误:")
Exemple #13
0
 def to_excel(self):
     try:
         # toplevel_id在sublevel表中个数索引从1开始
         toplevel_count_list = list()
         condition_list = list()
         module = self.comboBox_module.currentText()
         condition_list.clear()
         condition_list.append("module='" + module + "'")
         module_id = DBManager().query("modules", "module_id", condition_list)[0]
         logger.debug(module_id)
         logger.debug(condition_list)
         for i in range(4):
             i = i + 1
             condition_list.clear()
             condition_list.append("toplevel_id='" + str(i) + "'")
             toplevel_count_list.append(len(DBManager().query("sublevel", "sublevel_elemnt", condition_list)))
         TEMPLATE_FILE = "./templates/测试建模模板.xlsx"
         SHEET_COUNT = 4
         real_index_list = list()
         condition_list.clear()
         condition_list.append("module_id='" + str(module_id) + "'")
         toplevel_id = DBManager().query("casemodel", "toplevel_id", condition_list)
         # sublevel_id数据索引从1开始
         sublevel_id = DBManager().query("casemodel", "sublevel_id", condition_list)
         for j in sublevel_id:
             real_index = int(j)
             toplevel_count_index = 0
             # 由于sublevel_id数据索引和sublevel表中toplevel_id个数索引均从1开始
             # 故两者之差大于0时认为当前sublevel_element为下一个顶级元素的次级元素,需要做差取真实表格索引
             while ((real_index - toplevel_count_list[toplevel_count_index]) > 0):
                 real_index = real_index - toplevel_count_list[toplevel_count_index]
                 toplevel_count_index = toplevel_count_index + 1
             real_index_list.append(real_index)
         thirdlevel_element = DBManager().query("casemodel", "thirdlevel_element", condition_list)
         data_list = list()
         data_list.append(toplevel_id)
         data_list.append(real_index_list)
         data_list.append(thirdlevel_element)
         logger.debug(data_list)
         OutputWithTemplate().output_with_excel(TEMPLATE_FILE, SHEET_COUNT, data_list)
     except:
         logger.exception("发现错误:")
Exemple #14
0
 def add_thirdlevel(self):
     try:
         data_dict = dict()
         condition_list = list()
         module = self.comboBox_module.currentText()
         condition_list.clear()
         condition_list.append("module='" + module + "'")
         module_id = DBManager().query("modules", "module_id", condition_list)[0]
         thirdlevel_element = self.textEdit_thirdlevel.toPlainText()
         # 由于casemodel界面展示时包含“:”和“-”,所以在处理包含这两个符号的选项时会出错,这个弹提示不允许这个的数据输入
         if ":" in thirdlevel_element or ":" in thirdlevel_element or "-" in thirdlevel_element:
             msg_box = QtWidgets.QMessageBox(QtWidgets.QMessageBox.Warning, "错误输入提示", "不允许输入冒号或横线!")
             msg_box.exec_()
         else:
             items_sublevel = self.listWidget_sublevel.selectedItems()
             for i_sublevel in items_sublevel:
                 sublevel_element = i_sublevel.text()
                 condition_list.clear()
                 condition_list.append("sublevel_element='" + i_sublevel.text() + "'")
                 sublevel_id = DBManager().query("sublevel", "sublevel_id", condition_list)[0]
             condition_list.clear()
             condition_list.append("module_id='" + module_id + "'")
             module_id_thirdlevel = DBManager().query("thirdlevel", "thirdlevel_id", condition_list)
             module_id_thirdlevel_toint = list()
             for i in module_id_thirdlevel:
                 module_id_thirdlevel_toint.append(int(i))
             if len(module_id_thirdlevel) == 0:
                 thirdlevel_id = str(1)
             else:
                 thirdlevel_id = str(max(module_id_thirdlevel_toint) + 1)
             logger.debug(thirdlevel_id)
             condition_list.clear()
             if thirdlevel_element is not "":
                 data_dict["thirdlevel_element"] = thirdlevel_element
             data_dict["sublevel_id"] = sublevel_id
             data_dict["module_id"] = module_id
             data_dict["thirdlevel_id"] = thirdlevel_id
             DBManager().insert_data("thirdlevel", data_dict)
             self.textEdit_thirdlevel.clear()
             self.listWidget_thirdlevel_handle()
     except:
         logger.exception("发现错误:")
Exemple #15
0
 def delete_selection(self):
     try:
         condition_list = list()
         module = self.comboBox_module.currentText()
         condition_list.clear()
         condition_list.append("module='" + module + "'")
         module_id = DBManager().query("modules", "module_id", condition_list)[0]
         condition_list.clear()
         try:
             delete_data = self.listWidget_caseModel.currentItem().text()
         except:
             delete_data = ""
         if delete_data is not "":
             thirdlevel_data = "thirdlevel_element='" + delete_data.split(":")[1] + "'"
             condition_list.append(thirdlevel_data)
             condition_list.append("module_id='" + str(module_id) + "'")
             logger.debug(condition_list)
             DBManager().delete("casemodel", condition_list)
             self.listWidget_casemodel_handle()
     except:
         logger.exception("发现错误:")
Exemple #16
0
 def listWidget_thirdlevel_handle(self):
     try:
         # 获取列表中选中的值
         self.listWidget_thirdlevel.clear()
         condition_List = list()
         module = self.comboBox_module.currentText()
         condition_List.clear()
         condition_List.append("module='" + module + "'")
         module_id = DBManager().query("modules", "module_id", condition_List)[0]
         thirdlevel_widget = self.listWidget_sublevel.currentItem()
         row_data = thirdlevel_widget.text()
         condition_List.clear()
         condition_List.append("sublevel_element='" + row_data + "'")
         sublevel_id = DBManager().query("sublevel", "sublevel_id", condition_List)
         condition_List.clear()
         condition_List.append("sublevel_id='" + sublevel_id[0] + "'")
         condition_List.append("module_id='" + str(module_id) + "'")
         sublevel_element = DBManager().query("thirdlevel", "thirdlevel_element", condition_List)
         for i in sublevel_element:
             self.listWidget_thirdlevel.addItem(i)
     except:
         logger.exception("发现错误:")
    def output_with_excel(self, template_file, sheet_count, data_list):
        '''
        :param template_file: 模板文件路径
        :param sheet_count: 模板文件容sheet的个数,比如5个sheet页通过这个参数可以指定就用3个
        :param data_list: 数据list,其中包含sheet页list,表头list,数据list
        本方法中均使用list进行数据操作,由于模板中第一行为标题,故插入数据的行数索引从1开始
        '''
        try:
            data = xlrd.open_workbook(template_file)
            sheet_name = data.sheet_names()
            sheet_list = list()
            # 将模板文件sheet对象存入sheet_list列表
            for i in range(sheet_count):
                sheet_list.append(data.sheets()[i])
            template_data_list = list()
            # 将模板文件每个sheet的数据list存入tempLate_data_list列表
            for i_sheet in sheet_list:
                template_data_list.append(i_sheet.row_values(0))
            output_file = xlwt.Workbook()
            # 字体设定
            font = xlwt.Font()
            font.bold = True
            # 边框设定
            borders = xlwt.Borders()
            borders.left = 1
            borders.right = 1
            borders.top = 1
            borders.bottom = 1
            # 背景色设定
            backcolor = xlwt.Pattern()
            backcolor.pattern = xlwt.Pattern.SOLID_PATTERN
            backcolor.pattern_fore_colour = 5
            style = xlwt.XFStyle()
            style.font = font
            style.borders = borders
            style.pattern = backcolor
            output_file_sheet_list = list()
            # 输出文件添加sheet并给予标题,对象存入output_file_sheet_list列表
            for j in range(sheet_count):
                output_file_sheet_list.append(output_file.add_sheet(sheet_name[j], cell_overwrite_ok=True))
            # j_sheet_num为sheet页的索引
            for j_sheet_num in range(len(sheet_list)):
                # j_sheet_row_index为模板文件template_dataList的索引
                for j_sheet_row_index in range(len(template_data_list[j_sheet_num])):
                    output_file_sheet_list[j_sheet_num].write(0, j_sheet_row_index,
                                                              template_data_list[j_sheet_num][j_sheet_row_index], style)

            # 数据格式为{"sheet页数":{"列数":[当前添加的数据1,当前添加的数据2]}}
            filter_dict = dict()
            logger.debug("++++++++++++++++++++++++++++++")
            logger.debug(data_list)
            # 以追加方式添加数据
            for data_num in range(len(data_list[0])):
                logger.debug(data_num)
                logger.debug(data_list)
                # 当前sheet的索引
                data_sheet_num = int(data_list[0][data_num]) - 1
                data_row_num = 1
                data_col_num = int(data_list[1][data_num]) - 1
                logger.debug("===================================")
                logger.debug(data_sheet_num)
                logger.debug(data_col_num)
                # 判断当前sheet页数是否在filter_dict中,即当前sheet是否曾经出现过
                if str(data_sheet_num) not in filter_dict:
                    # 当前sheet页数不存在filter_dict,添加对应data_sheet_num字典
                    filter_dict[str(data_sheet_num)] = {}
                    # 增加对应列数的数据list
                    filter_dict[str(data_sheet_num)][str(data_col_num)] = list()
                    # 由于sheet页数第一次出现,故数据也一定第一次出现,所以之列列表中添加值
                    filter_dict[str(data_sheet_num)][str(data_col_num)].append(data_col_num)
                else:
                    # 当前sheet页数存在filter_dict中,判断当前列数是否在列数字典中,即当前列数是否出现过
                    if str(data_col_num) not in filter_dict[str(data_sheet_num)]:
                        # 当前列数未出现过,添加列表
                        filter_dict[str(data_sheet_num)][str(data_col_num)] = list()
                        # 向列数字典列表插数据,但插入数据的当前行数为初始值,即1
                        filter_dict[str(data_sheet_num)][str(data_col_num)].append(data_col_num)
                    else:
                        # 当前列数出现过。向列数字典列表插数据,插入数据的当前行数为总出现的次数
                        filter_dict[str(data_sheet_num)][str(data_col_num)].append(data_col_num)
                        data_row_num = len(filter_dict[str(data_sheet_num)][str(data_col_num)])
                logger.debug(filter_dict)
                data_preview_add = data_list[2][data_num]
                # 向excel中插入数据
                output_file_sheet_list[data_sheet_num].write(data_row_num, data_col_num, data_preview_add)

            output_file.save(str(TimeStamp.time_stamp()) + ".xls")


        except:
            logger.exception("发现错误")