def __init__(self): self.rows = xlsx_to_rows(GLOBAL_SPLIT_RULES_FILE) self.data = { row[0]: { "latest": "", "files": [], "exception": "" } for row in self.rows }
def __enter__(self): self.file_name = PRODUCT_FILTER_RESULT_FILE() if os.path.exists(self.file_name): self.rows = xlsx_to_rows(self.file_name) for row in self.rows[1:]: self.data[f"{row[0]}|{row[1]}"] = { "file_type": row[2], "customer": row[3] } return self
def __enter__(self): self.file_name = SPLIT_RESULT_FILE() if os.path.exists(self.file_name): rows = xlsx_to_rows(self.file_name) for row in rows: self.data[row[0]] = { "latest": row[5], "files": [i for i in row[6].split(", ") if i], "exception": row[7] } return self
def clean(): rows = xlsx_to_rows(GLOBAL_SPLIT_RULES_FILE) for row in rows[1:]: mkdir(os.path.join(row[0], "history")) for filename in os.listdir(row[0]): if filename.startswith("done"): shutil.move(os.path.join(row[0], filename), os.path.join(row[0], "history", filename)) for filename in os.listdir(os.path.join(row[0], "history")): now = time.time() if now - os.stat(os.path.join( row[0], "history", filename)).st_ctime > 30 * 24 * 60 * 60: os.remove(os.path.join(row[0], "history", filename))
def load_product_rules(rules_file, origin_rules=None): if origin_rules: origin_rules = deepcopy(origin_rules) else: origin_rules = {} rows = xlsx_to_rows(rules_file)[1:] for row in rows: origin_rules[f"{row[0]}-{row[1]}"] = { "name": row[2], "size": row[3], "unit": row[4], "multi": int(row[6]) if row[5] == "是" else 1 } if row[7] == "是" else {} return origin_rules
class F0Worker(WorkerBase): """ 文件从待分发到F0,只执行文件移动和备份 """ RULES = {} rows = xlsx_to_rows(GLOBAL_SPLIT_RULES_FILE)[1:] for row in rows: if not row[0]: break RULES[row[0]] = { "targets": [i for i in row[1].split("|")], "sign": row[2], "sep": row[3], "pos": row[4] } def __init__(self, input_file): super().__init__(input_file) def new_file_name(self, customer): md5_str = get_md5(self.input_file) file_path, file_name = os.path.split(self.input_file) return f"F0_" \ f"{file_name.split('_')[0]}_" \ f"{file_name.split('_')[1]}_" \ f"{customer}_" \ f"{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}_" \ f"{md5_str}.xlsx" def get_backup_file(self): return os.path.join( os.path.split(self.input_file)[0], "history", f"done{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}_{get_md5(self.input_file)}_" f"{os.path.split(self.input_file)[1]}") def real_process(self): print(f"F0: {self.input_file}") if not self.data: return False file_path, file_name = os.path.split(self.input_file) if file_name.split(self.RULES[file_path]["sep"])[int(self.RULES[file_path]["pos"])] \ == self.RULES[file_path]["sign"]: self.output_files = [] for target in self.RULES[file_path]["targets"]: new_file_name = self.new_file_name(target.split(os.sep)[-1]) self.output_files.append(os.path.join(target, new_file_name)) return True return False
def transform_add_list(filename, key_fields, value_fields): rows = xlsx_to_rows(filename) field_to_index = {i: index for index, i in enumerate(rows[0])} result = {} for row in rows[1:]: key = "-".join( [row[field_to_index[key]] for key in key_fields.split("&")]) value = [row[field_to_index[key]] for key in value_fields.split("&")] result[key] = value def _wrapper(data_row, data_field_to_index): data_key = "-".join( [data_row[data_field_to_index[k]] for k in key_fields.split("&")]) data_row.extend(result.get(data_key, [])) return data_row return _wrapper
def __init__(self, input_file): self.data = None self.input_file = input_file self.output_files = self.get_output_files() self.backup_file = self.get_backup_file() try: if input_file.endswith(".xls") or input_file.endswith(".xlsx"): self.data = xlsx_to_rows(input_file) elif input_file.endswith(".csv"): process_csv2utf8(self.input_file) with open(self.input_file) as f: reader = csv.reader(f) self.data = [i for i in reader] elif input_file.endswith(".err"): pass else: self.error(f"格式未知,需为xlsx, xls或csv") except Exception as e: self.error(f"解析文件失败:{self.input_file}")
def load_header_rules(rules_file, origin_rules=None): if origin_rules: origin_rules = deepcopy(origin_rules) else: origin_rules = { "I": {"key_titles": {}, "optional_titles": {}}, "P": {"key_titles": {}, "optional_titles": {}}, "S": {"key_titles": {}, "optional_titles": {}} } for sheet_name in origin_rules.keys(): try: rows = xlsx_to_rows(rules_file, sheet_name) except KeyError: raise ValueError(f"规则文件{rules_file}中找不到必要sheet:{sheet_name}") current = "key_titles" for index, i in enumerate(rows[0]): if not i: current = "optional_titles" continue if origin := origin_rules[sheet_name][current].get(i): origin_rules[sheet_name][current][i] = [row[index] for row in rows] + origin else: origin_rules[sheet_name][current][i] = [row[index] for row in rows]
class F3Worker(AdvancedWorkerBase): TARGET_INFO = {} rows = xlsx_to_rows(GLOBAL_TARGET_RULES_FILE) for row in rows[1:]: TARGET_INFO[row[0]] = [row[1], row[0], row[2], row[3], row[4]] RESULTS = {} for customer, file_name in MATCH_RULES_FILES.items(): RESULTS[customer] = {} rows = xlsx_to_rows(file_name) for row in rows[1:]: RESULTS[customer][f"{row[0]}-{row[1]}"] = { "id": row[2], "current": row[3], "name": row[4], "code": row[5], "type": row[6], "province": row[7], "address": row[8], "city": row[9] } def get_output_files(self): return [self.input_file.replace("F2_", "F3_")] def get_backup_file(self): return os.path.join( os.path.split(self.input_file)[0], "status", f"done{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}_{os.path.split(self.input_file)[1]}" ) @classmethod def set_rule(cls, customer): cls.RESULTS[customer] = {} wb = openpyxl.Workbook() ws = wb.active title = [ "originCustomerName", "reference", "别名ID", "当前状态", "customerName", "customerCode", "customerType", "customerProvince", "customerAddress", "customerCity" ] data = [title] for index_i, i in enumerate(data): for index_j, j in enumerate(i): ws.cell(index_i + 1, index_j + 1).value = j wb.save(os.path.join(ORDERED_FILES_DIR, customer, "customer_list.xlsx")) @classmethod def write_back(cls, customer, name, reference): back_file = os.path.join(ORDERED_FILES_DIR, customer, "customer_list.xlsx") to_add = [ name, reference, cls.RESULTS[customer][f"{name}-{reference}"]["id"], cls.RESULTS[customer][f"{name}-{reference}"]["current"], cls.RESULTS[customer][f"{name}-{reference}"]["name"], cls.RESULTS[customer][f"{name}-{reference}"]["code"], cls.RESULTS[customer][f"{name}-{reference}"]["type"], cls.RESULTS[customer][f"{name}-{reference}"]["province"], cls.RESULTS[customer][f"{name}-{reference}"]["address"], cls.RESULTS[customer][f"{name}-{reference}"]["city"] ] wb = openpyxl.load_workbook(back_file) ws = wb.worksheets[0] data = [] for i in ws.rows: if not i[0].value: break data.append([ str(j.value).strip() if j.value is not None else "" for j in i ]) for index, i in enumerate(data): if i[0] == name and i[1] == reference: data[index] = to_add break else: data.append(to_add) for index_i, i in enumerate(data): for index_j, j in enumerate(i): ws.cell(index_i + 1, index_j + 1).value = j wb.save(back_file) @staticmethod def graceful_get(url, data_dict): while True: try: response = requests.get(url, params=data_dict) json_data = response.json() return json_data except Exception as e: print("Connection failed.", e) time.sleep(1) @classmethod def get_from_online(cls, customer, name, reference): print(f"Trying to get {name}-{reference} from server.") if info := cls.RESULTS[customer].get(f"{name}-{reference}"): json_data = cls.graceful_get(GET_RESULT_URL, {"case_id": info["id"]}) if not json_data["data"]: return json_data["message"] else: