def resetfromlog(self):
"""
drop the db and reinsert (and update) all the entries from log to the empty db
"""
if not os.path.exists(self.logfile):
print("The logfile " + self.logfile + " doesn't exist!")
return
# backup the old one
if os.path.exists(self.dbFilePath):
bak = self.dbFilePath + "~"
if os.path.exists(bak):
if input("A backup already exists. Remove it (only Yes is accepted)? ") != "Yes":
return
os.remove(bak)
os.rename(self.dbFilePath, bak)
# read new one
print("This might take a while depending on the log size. Please wait ...")
self.dbConnection = DBConnection(self.dbFilePath)
with self.dbConnection.getSessionContext() as session:
with open(self.logfile, "r") as self.logf:
for ll in self.logf:
if ll.startswith(Register.LOGADD):
dbf = self._dbFileFromLog(ll[len(Register.LOGADD):])
DBFileRegister.insert(session, dbf, commit=False)
elif ll.startswith(Register.LOGUPDATED):
dbf = self._dbFileFromLog(ll[len(Register.LOGUPDATED):])
session.commit()
DBFileRegister.update(session,dbf, setall=True)
# TODO display progress (line, items, date)
session.commit()
print("Done")
def batchimport(self):
"""
store all data stored in the provided files (usually named like sumlog.txt)
Only [MYSUM...] entries are valid.
"""
pgr, pcom, pdir = "", "", "" # previous group, comment and directory
ii = 0 # file no
jj = 0 # successfully imported entries
warn = 0 # number of warnings (duplicities)
failfiles = [] # a list of files that failed to import
allDBFilesToStore = []
with self.dbConnection.getSessionContext() as session:
for ff in self.files:
ii = ii + 1
cdir = os.path.dirname(ff) # directory
if cdir != pdir:
print("Directory [{}]".format(cdir))
pdir = cdir
gr, com = self.getgroupcomment(ff, imp=True)
if gr != pgr or com != pcom:
print("Using group:{} comment:{}".format(gr, com))
pgr, pcom = gr, com
self.printstatus(ii, ff, "")
with open(ff, "r") as fsum:
ll = 0
fail = False
dbFilesToStoreFromImportFile = []
for line in fsum:
ll = ll + 1
self.printstatus(ii, ff, "L" + str(ll))
try:
ms = MySum.fromString(line)
except ValueError:
self.printstatus(ii, ff, "not a MYSUM!")
print()
fail = True
break
self.printstatus(ii, ff, ms.fileName + " L" + str(ll))
dbf = DBFile(
fileName = ms.fileName,
group = gr,
comment = com,
fileSize = ms.fileSize,
md1 = ms.md1,
md5 = ms.md5,
ed2k = ms.ed2k,
)
matchingDBFile = self._determineMatchingDBFile(
session = session,
fileSize = dbf.fileSize,
md1 = dbf.md1,
md5 = dbf.md5,
ed2k = dbf.ed2k,
)
fullMatch = ((matchingDBFile is not None) and
(dbf.fileName == matchingDBFile.fileName) and
(dbf.group == matchingDBFile.group) and
(dbf.comment == matchingDBFile.comment))
if (matchingDBFile):
warn = warn + 1
if (fullMatch):
self.printstatus(ii, ff, "Already registered (full match) as {} L{}".format(matchingDBFile.fileId, ll))
else:
self.printstatus(ii, ff, "Already registered (data match) as {} L{}".format(matchingDBFile.fileId, ll))
print()
continue
jj = jj + 1
dbFilesToStoreFromImportFile.append(dbf)
if fail:
if ll == 1:
self.printstatus(ii, ff, "FAILED")
failfiles.append(ff)
print()
else:
sll = "after " + str(ll) + " lines"
self.printstatus(ii, ff, "FAILED " + sll)
failfiles.append(ff + " (" + sll + ")")
print()
else:
for dbf in dbFilesToStoreFromImportFile:
DBFileRegister.insert(session, dbf, commit=False)
allDBFilesToStore.extend(dbFilesToStoreFromImportFile)
print()
print(self.RULER)
print("About to import {} entries ({} warnings) from {} files out of {}".format(jj, warn, len(self.files) - len(failfiles), len(self.files)))
if len(failfiles) > 0:
print("A list of files that failed:")
for ff in failfiles:
print(" " + ff)
if self.docommit(failfiles):
session.commit()
for storedDBFile in allDBFilesToStore:
self.log(Register.LOGADD + self._formatDBFileForLog(storedDBFile))
print("Done.")
else:
print("Aborted!")
def registercheck(self, register):
"""
register (or check) the given files
"""
pgr, pcom, pdir, psize = "", "", "", 0 # previous group, comment, directory and size
tstart = time.time()
ii = 0
failfiles = []
dbFilesToStore = []
with self.dbConnection.getSessionContext() as session:
for ff in self.files:
ii = ii + 1
fileMightBeRegistered = False
cdir = os.path.dirname(ff) # directory
sff = os.path.basename(ff) # short filename
try:
if cdir != pdir:
print("Directory [{}]".format(cdir))
pdir = cdir
dbf = DBFile(fileName = sff)
if register: # group and comment
gr, com = self.getgroupcomment(ff)
if gr != pgr or com != pcom:
print("Using group:'{}' comment:'{}'".format(gr, com))
pgr, pcom = gr, com
dbf.group, dbf.comment = gr, com
self.printstatus(ii, sff, "Quick")
# stage 1 - silent
ms = MySum(ff)
ms.upgrade(1)
dbf.fileSize = ms.fileSize
dbf.md1 = ms.md1
fileMightBeRegistered = self._determineFileMightBeRegistered(
session = session,
fileSize = dbf.fileSize,
md1 = dbf.md1,
)
if ((not register) and (not fileMightBeRegistered)):
self.printstatus(ii, sff, "FAILED")
failfiles.append(ff)
print()
continue
tt = threading.Thread(target=ms.upgrade,args=[2])
tt.start()
while tt.is_alive():
try:
self.printstatus(ii, sff, self.msgpgs(ms,psize,tstart,fileMightBeRegistered))
time.sleep(0.25)
except KeyboardInterrupt:
ms.requestStop()
tt.join()
raise
tt.join()
psize = psize + ms.fileSize
dbf.md5 = ms.md5
dbf.ed2k = ms.ed2k
matchingDBFile = self._determineMatchingDBFile(
session = session,
fileSize = dbf.fileSize,
md1 = dbf.md1,
md5 = dbf.md5,
ed2k = dbf.ed2k,
)
fullMatch = (
(matchingDBFile is not None) and
(dbf.fileName == matchingDBFile.fileName) and
(dbf.group == matchingDBFile.group) and
(dbf.comment == matchingDBFile.comment)
)
if register:
if (matchingDBFile is None):
DBFileRegister.insert(session, dbf, commit=False)
self.printstatus(ii, sff, "New entry " + str(dbf.fileId))
dbFilesToStore.append(dbf)
else:
if (fullMatch):
self.printstatus(ii, sff, "Already registered (full match) as " + str(matchingDBFile.fileId))
else:
self.printstatus(ii, sff, "Already registered (data match) as " + str(matchingDBFile.fileId))
failfiles.append(ff)
else:
if (matchingDBFile is None):
self.printstatus(ii, sff, "FAILED")
failfiles.append(ff)
else:
stat = "OK"
if (dbf.fileName != matchingDBFile.fileName):
stat = "(as " + matchingDBFile.fileName + ") OK"
stat = "id:" + str(matchingDBFile.fileId) + " " + stat
self.printstatus(ii, sff, stat)
print()
except KeyboardInterrupt:
self.printstatus(ii, sff, "Interrupted")
failfiles.append(ff + " (Interrupted)")
print()
break
print(self.RULER)
if register:
print("About to register {} files out of {}".format(ii-len(failfiles),ii))
else:
print("Passed {} files out of {}.{}".format(ii-len(failfiles), ii, "ALL OK" if not len(failfiles) else "" ))
if len(failfiles) > 0:
print("A list of files that failed:")
for ff in failfiles:
print(" " + ff)
if register:
if len(failfiles) == ii:
print("No files were registered!")
elif self.docommit(failfiles):
session.commit()
for storedDBFile in dbFilesToStore:
self.log(Register.LOGADD + self._formatDBFileForLog(storedDBFile))
print("Done.")
else:
print("Aborted!")
