def __init__(self):
self.rows = xlsx_to_rows(GLOBAL_SPLIT_RULES_FILE)
self.data = {
row[0]: {
"latest": "",
"files": [],
"exception": ""
}
for row in self.rows
}
def __enter__(self):
self.file_name = PRODUCT_FILTER_RESULT_FILE()
if os.path.exists(self.file_name):
self.rows = xlsx_to_rows(self.file_name)
for row in self.rows[1:]:
self.data[f"{row[0]}|{row[1]}"] = {
"file_type": row[2],
"customer": row[3]
}
return self
def __enter__(self):
self.file_name = SPLIT_RESULT_FILE()
if os.path.exists(self.file_name):
rows = xlsx_to_rows(self.file_name)
for row in rows:
self.data[row[0]] = {
"latest": row[5],
"files": [i for i in row[6].split(", ") if i],
"exception": row[7]
}
return self
def clean():
rows = xlsx_to_rows(GLOBAL_SPLIT_RULES_FILE)
for row in rows[1:]:
mkdir(os.path.join(row[0], "history"))
for filename in os.listdir(row[0]):
if filename.startswith("done"):
shutil.move(os.path.join(row[0], filename),
os.path.join(row[0], "history", filename))
for filename in os.listdir(os.path.join(row[0], "history")):
now = time.time()
if now - os.stat(os.path.join(
row[0], "history", filename)).st_ctime > 30 * 24 * 60 * 60:
os.remove(os.path.join(row[0], "history", filename))
def load_product_rules(rules_file, origin_rules=None):
if origin_rules:
origin_rules = deepcopy(origin_rules)
else:
origin_rules = {}
rows = xlsx_to_rows(rules_file)[1:]
for row in rows:
origin_rules[f"{row[0]}-{row[1]}"] = {
"name": row[2],
"size": row[3],
"unit": row[4],
"multi": int(row[6]) if row[5] == "是" else 1
} if row[7] == "是" else {}
return origin_rules
class F0Worker(WorkerBase):
"""
文件从待分发到F0，只执行文件移动和备份
"""
RULES = {}
rows = xlsx_to_rows(GLOBAL_SPLIT_RULES_FILE)[1:]
for row in rows:
if not row[0]:
break
RULES[row[0]] = {
"targets": [i for i in row[1].split("|")],
"sign": row[2],
"sep": row[3],
"pos": row[4]
}
def __init__(self, input_file):
super().__init__(input_file)
def new_file_name(self, customer):
md5_str = get_md5(self.input_file)
file_path, file_name = os.path.split(self.input_file)
return f"F0_" \
f"{file_name.split('_')[0]}_" \
f"{file_name.split('_')[1]}_" \
f"{customer}_" \
f"{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}_" \
f"{md5_str}.xlsx"
def get_backup_file(self):
return os.path.join(
os.path.split(self.input_file)[0], "history",
f"done{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}_{get_md5(self.input_file)}_"
f"{os.path.split(self.input_file)[1]}")
def real_process(self):
print(f"F0: {self.input_file}")
if not self.data:
return False
file_path, file_name = os.path.split(self.input_file)
if file_name.split(self.RULES[file_path]["sep"])[int(self.RULES[file_path]["pos"])] \
== self.RULES[file_path]["sign"]:
self.output_files = []
for target in self.RULES[file_path]["targets"]:
new_file_name = self.new_file_name(target.split(os.sep)[-1])
self.output_files.append(os.path.join(target, new_file_name))
return True
return False
def transform_add_list(filename, key_fields, value_fields):
rows = xlsx_to_rows(filename)
field_to_index = {i: index for index, i in enumerate(rows[0])}
result = {}
for row in rows[1:]:
key = "-".join(
[row[field_to_index[key]] for key in key_fields.split("&")])
value = [row[field_to_index[key]] for key in value_fields.split("&")]
result[key] = value
def _wrapper(data_row, data_field_to_index):
data_key = "-".join(
[data_row[data_field_to_index[k]] for k in key_fields.split("&")])
data_row.extend(result.get(data_key, []))
return data_row
return _wrapper
def __init__(self, input_file):
self.data = None
self.input_file = input_file
self.output_files = self.get_output_files()
self.backup_file = self.get_backup_file()
try:
if input_file.endswith(".xls") or input_file.endswith(".xlsx"):
self.data = xlsx_to_rows(input_file)
elif input_file.endswith(".csv"):
process_csv2utf8(self.input_file)
with open(self.input_file) as f:
reader = csv.reader(f)
self.data = [i for i in reader]
elif input_file.endswith(".err"):
pass
else:
self.error(f"格式未知，需为xlsx, xls或csv")
except Exception as e:
self.error(f"解析文件失败：{self.input_file}")
def load_header_rules(rules_file, origin_rules=None):
if origin_rules:
origin_rules = deepcopy(origin_rules)
else:
origin_rules = {
"I": {"key_titles": {}, "optional_titles": {}},
"P": {"key_titles": {}, "optional_titles": {}},
"S": {"key_titles": {}, "optional_titles": {}}
}
for sheet_name in origin_rules.keys():
try:
rows = xlsx_to_rows(rules_file, sheet_name)
except KeyError:
raise ValueError(f"规则文件{rules_file}中找不到必要sheet：{sheet_name}")
current = "key_titles"
for index, i in enumerate(rows[0]):
if not i:
current = "optional_titles"
continue
if origin := origin_rules[sheet_name][current].get(i):
origin_rules[sheet_name][current][i] = [row[index] for row in rows] + origin
else:
origin_rules[sheet_name][current][i] = [row[index] for row in rows]
class F3Worker(AdvancedWorkerBase):
TARGET_INFO = {}
rows = xlsx_to_rows(GLOBAL_TARGET_RULES_FILE)
for row in rows[1:]:
TARGET_INFO[row[0]] = [row[1], row[0], row[2], row[3], row[4]]
RESULTS = {}
for customer, file_name in MATCH_RULES_FILES.items():
RESULTS[customer] = {}
rows = xlsx_to_rows(file_name)
for row in rows[1:]:
RESULTS[customer][f"{row[0]}-{row[1]}"] = {
"id": row[2],
"current": row[3],
"name": row[4],
"code": row[5],
"type": row[6],
"province": row[7],
"address": row[8],
"city": row[9]
}
def get_output_files(self):
return [self.input_file.replace("F2_", "F3_")]
def get_backup_file(self):
return os.path.join(
os.path.split(self.input_file)[0], "status",
f"done{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}_{os.path.split(self.input_file)[1]}"
)
@classmethod
def set_rule(cls, customer):
cls.RESULTS[customer] = {}
wb = openpyxl.Workbook()
ws = wb.active
title = [
"originCustomerName", "reference", "别名ID", "当前状态", "customerName",
"customerCode", "customerType", "customerProvince",
"customerAddress", "customerCity"
]
data = [title]
for index_i, i in enumerate(data):
for index_j, j in enumerate(i):
ws.cell(index_i + 1, index_j + 1).value = j
wb.save(os.path.join(ORDERED_FILES_DIR, customer,
"customer_list.xlsx"))
@classmethod
def write_back(cls, customer, name, reference):
back_file = os.path.join(ORDERED_FILES_DIR, customer,
"customer_list.xlsx")
to_add = [
name, reference,
cls.RESULTS[customer][f"{name}-{reference}"]["id"],
cls.RESULTS[customer][f"{name}-{reference}"]["current"],
cls.RESULTS[customer][f"{name}-{reference}"]["name"],
cls.RESULTS[customer][f"{name}-{reference}"]["code"],
cls.RESULTS[customer][f"{name}-{reference}"]["type"],
cls.RESULTS[customer][f"{name}-{reference}"]["province"],
cls.RESULTS[customer][f"{name}-{reference}"]["address"],
cls.RESULTS[customer][f"{name}-{reference}"]["city"]
]
wb = openpyxl.load_workbook(back_file)
ws = wb.worksheets[0]
data = []
for i in ws.rows:
if not i[0].value:
break
data.append([
str(j.value).strip() if j.value is not None else "" for j in i
])
for index, i in enumerate(data):
if i[0] == name and i[1] == reference:
data[index] = to_add
break
else:
data.append(to_add)
for index_i, i in enumerate(data):
for index_j, j in enumerate(i):
ws.cell(index_i + 1, index_j + 1).value = j
wb.save(back_file)
@staticmethod
def graceful_get(url, data_dict):
while True:
try:
response = requests.get(url, params=data_dict)
json_data = response.json()
return json_data
except Exception as e:
print("Connection failed.", e)
time.sleep(1)
@classmethod
def get_from_online(cls, customer, name, reference):
print(f"Trying to get {name}-{reference} from server.")
if info := cls.RESULTS[customer].get(f"{name}-{reference}"):
json_data = cls.graceful_get(GET_RESULT_URL,
{"case_id": info["id"]})
if not json_data["data"]:
return json_data["message"]
else:
