def leerPGNs(self, ficheros, dlTmp):
erroneos = duplicados = importados = 0
allows_fen = self.allows_positions
allows_complete_game = self.allows_complete_game
allows_cero_moves = self.allows_zero_moves
duplicate_check = not self.allows_duplicates
t1 = time.time() - 0.7 # para que empiece enseguida
if self.with_db_stat:
self.db_stat.massive_append_set(True)
def write_logs(fich, pgn):
with open(fich, "at", encoding="utf-8", errors="ignore") as ferr:
ferr.write(pgn)
ferr.write("\n")
si_cols_cambiados = False
select = ",".join('"%s"' % campo for campo in self.li_fields)
select_values = ("?," * len(self.li_fields))[:-1]
sql = "INSERT INTO Games (%s) VALUES (%s);" % (select, select_values)
st_xpv_bloque = set()
li_regs = []
n_regs = 0
conexion = self.conexion
dcabs = self.recuperaConfig("dcabs", drots.copy())
obj_decode = Util.Decode()
decode = obj_decode.decode
for fichero in ficheros:
nomfichero = os.path.basename(fichero)
fich_erroneos = os.path.join(Code.configuracion.carpetaTemporal(), nomfichero[:-3] + "errors.pgn")
fich_duplicados = os.path.join(Code.configuracion.carpetaTemporal(), nomfichero[:-3] + "duplicates.pgn")
dlTmp.pon_titulo(nomfichero)
next_n = random.randint(1000, 2000)
obj_decode.read_file(fichero)
with PGNreader(fichero, self.depthStat()) as fpgn:
bsize = fpgn.size
for n, (body, is_raw, pv, fens, bdCab, bdCablwr, btell) in enumerate(fpgn, 1):
if n == next_n:
if time.time() - t1 > 0.8:
if not dlTmp.actualiza(
erroneos + duplicados + importados,
erroneos,
duplicados,
importados,
btell * 100.0 / bsize,
):
break
t1 = time.time()
next_n = n + random.randint(1000, 2000)
# Sin movimientos
if not pv and not allows_cero_moves:
erroneos += 1
write_logs(fich_erroneos, decode(fpgn.bpgn()))
continue
dCab = {decode(k): decode(v) for k, v in bdCab.items()}
dCablwr = {decode(k): decode(v) for k, v in bdCablwr.items()}
dcabs.update(dCablwr)
xpv = pv_xpv(pv)
fen = dCab.get("FEN", None)
if fen:
if fen == FEN_INITIAL:
del dCab["FEN"]
del dCablwr["fen"]
fen = None
else:
if not allows_fen:
erroneos += 1
write_logs(fich_erroneos, decode(fpgn.bpgn()))
continue
xpv = "|%s|%s" % (fen, xpv)
if not fen:
if not allows_complete_game:
erroneos += 1
write_logs(fich_erroneos, decode(fpgn.bpgn()))
continue
fen = None # por si hay alguno vacio
if duplicate_check:
# Duplicados en el bloque actual
if xpv in st_xpv_bloque:
ok = False
# Duplicados respecto a las grabadas ya
else:
cursor = conexion.execute("SELECT COUNT(*) FROM games WHERE XPV = ?", (xpv,))
ok = cursor.fetchone()[0] == 0
if not ok:
duplicados += 1
write_logs(fich_duplicados, decode(fpgn.bpgn()))
continue
st_xpv_bloque.add(xpv)
for k in dCab:
if not (k in self.li_fields):
# Grabamos lo que hay
if li_regs:
n_regs = 0
conexion.executemany(sql, li_regs)
li_regs = []
st_xpv_bloque = set()
conexion.commit()
if self.with_db_stat:
self.db_stat.massive_append_set(False)
self.db_stat.commit()
self.db_stat.massive_append_set(True)
self.add_column(k)
si_cols_cambiados = True
select_values = ("?," * len(self.li_fields))[:-1]
sql = "insert into Games (%s) values (%s);" % (self.select, select_values)
reg = []
result = "*"
for campo in self.li_fields:
if campo == "XPV":
reg.append(xpv)
elif campo == "_DATA_":
data = None
if not is_raw:
data = memoryview(BODY_SAVE + body)
reg.append(data)
elif campo == "PLYCOUNT":
reg.append(pv.count(" ") + 1)
else:
reg.append(dCab.get(campo))
if campo == "RESULT":
result = dCab.get(campo, "*")
if self.with_db_stat and fen is None and pv:
self.db_stat.append(pv, result)
li_regs.append(reg)
n_regs += 1
importados += 1
if n_regs == 20000:
n_regs = 0
conexion.executemany(sql, li_regs)
li_regs = []
st_xpv_bloque = set()
conexion.commit()
if self.with_db_stat:
self.db_stat.commit()
dlTmp.actualiza(erroneos + duplicados + importados, erroneos, duplicados, importados, 100.00)
dlTmp.ponSaving()
if li_regs:
conexion.executemany(sql, li_regs)
conexion.commit()
if self.with_db_stat:
self.db_stat.massive_append_set(False)
self.db_stat.commit()
conexion.commit()
dlTmp.ponContinuar()
self.guardaConfig("dcabs", dcabs)
return si_cols_cambiados
