def resetfromlog(self):
"""
drop the db and reinsert (and update) all the entries from log to the empty db
"""
if not os.path.exists(self.logfile):
print("The logfile " + self.logfile + " doesn't exist!")
return
# backup the old one
if os.path.exists(self.dbFilePath):
bak = self.dbFilePath + "~"
if os.path.exists(bak):
if input("A backup already exists. Remove it (only Yes is accepted)? ") != "Yes":
return
os.remove(bak)
os.rename(self.dbFilePath, bak)
# read new one
print("This might take a while depending on the log size. Please wait ...")
self.dbConnection = DBConnection(self.dbFilePath)
with self.dbConnection.getSessionContext() as session:
with open(self.logfile, "r") as self.logf:
for ll in self.logf:
if ll.startswith(Register.LOGADD):
dbf = self._dbFileFromLog(ll[len(Register.LOGADD):])
DBFileRegister.insert(session, dbf, commit=False)
elif ll.startswith(Register.LOGUPDATED):
dbf = self._dbFileFromLog(ll[len(Register.LOGUPDATED):])
session.commit()
DBFileRegister.update(session,dbf, setall=True)
# TODO display progress (line, items, date)
session.commit()
print("Done")
def setdata(self):
"""
Change some details of the entries given by IDs. IDs are required.
Only filename, comment and group can be changed.
"""
ff = None
if (isinstance(self.files, list)):
if len(self.files) > 1:
print("Please provide just one name or none at all!")
return
elif len(self.files) == 1:
ff = self.files[0]
if (self.fileId is None):
print("Please provide file ID (-i option).")
return
self.fileId = int(self.fileId)
dbf = DBFile(fileId=self.fileId, fileName=ff, group=self.group, comment=self.comment)
self.log(Register.LOGUPDATE + self._formatDBFileForLog(dbf))
with self.dbConnection.getSessionContext() as session:
dbf = DBFileRegister.update(session, dbf)
if not dbf:
print("Error updating the entry!")
else:
self.log(Register.LOGUPDATED + self._formatDBFileForLog(dbf))
def _determineMatchingDBFile(session, fileSize, md1, md5, ed2k):
# DOC {{{
"""Returns the DBFile() that matches the size, the MD5 sum of the first
megabyte, the MD5 sum of the entire file and the ED2K sum of the
specified DBFile() and is already registered in the register.
Parameters
session -- an instance of the SQLAlchemy's Session()
fileSize -- the size of the file to look for in bytes
md1 -- the MD5 sum of the first megabyte of the file
md1 -- the MD5 sum of the entire file
ed2k -- the ED2K sum of the entire file
"""
# }}}
# CODE {{{
dbfs = DBFileRegister.query(
session = session,
fileSize = fileSize,
md1 = md1,
md5 = md5,
ed2k = ed2k,
)
if ((dbfs is None) or (len(dbfs) == 0)):
return None
else:
return dbfs[0]
def query(self):
"""
Query the register for any entry matching the parameters.
Only parts of the entry have to match the given parameters (ilike)
"""
ff = None
if (isinstance(self.files, list)):
if len(self.files) > 1:
print("Please provide just one name or none at all!")
return
elif len(self.files) == 1:
ff = self.files[0]
# create a new query arguments
dbFileQueryArguments = DBFileQueryArguments()
# add a query argument matching the file's ID if it is specified {{{
if (self.fileId is not None):
dbFileQueryArguments.fileId = self.fileId
# }}}
# add a query argument matching the file's name if it is specified {{{
if (ff is not None):
dbFileQueryArguments.fileName = ff
# }}}
# add a query argument matching the file's group if it is specified {{{
if (self.group is not None):
dbFileQueryArguments.group = self.group
# }}}
# add a query argument matching the file's comment if it is specified {{{
if (self.comment is not None):
dbFileQueryArguments.comment = self.comment
# }}}
with self.dbConnection.getSessionContext() as session:
ll = DBFileRegister.query(session, dbFileQueryArguments)
if (ll is None):
print("No record matches the query!")
else:
formatDBFileMethod = self._formatDBFile
if self.queryasmysum:
formatDBFileMethod = self._formatDBFileAsMysum
elif self.queryed2k:
formatDBFileMethod = self._formatDBFileAsED2K
elif self.queryverbose:
formatDBFileMethod = functools.partial(self._formatDBFile, verbose = True)
for dbf in ll:
print(formatDBFileMethod(dbf))
def _determineFileMightBeRegistered(session, fileSize, md1):
# DOC {{{
"""Returns True if there is a registered file of the size and the MD5
sum of the first megabyte specified in the provided DBFile(), False
otherwise.
Parameters
session -- an instance of the SQLAlchemy's Session()
fileSize -- the size of the file to look for in bytes
md1 -- the MD5 sum of the first megabyte of the file
"""
# }}}
# CODE {{{
dbfs = DBFileRegister.query(
session = session,
fileSize = fileSize,
md1 = md1,
)
return (dbfs is not None)
def batchimport(self):
"""
store all data stored in the provided files (usually named like sumlog.txt)
Only [MYSUM...] entries are valid.
"""
pgr, pcom, pdir = "", "", "" # previous group, comment and directory
ii = 0 # file no
jj = 0 # successfully imported entries
warn = 0 # number of warnings (duplicities)
failfiles = [] # a list of files that failed to import
allDBFilesToStore = []
with self.dbConnection.getSessionContext() as session:
for ff in self.files:
ii = ii + 1
cdir = os.path.dirname(ff) # directory
if cdir != pdir:
print("Directory [{}]".format(cdir))
pdir = cdir
gr, com = self.getgroupcomment(ff, imp=True)
if gr != pgr or com != pcom:
print("Using group:{} comment:{}".format(gr, com))
pgr, pcom = gr, com
self.printstatus(ii, ff, "")
with open(ff, "r") as fsum:
ll = 0
fail = False
dbFilesToStoreFromImportFile = []
for line in fsum:
ll = ll + 1
self.printstatus(ii, ff, "L" + str(ll))
try:
ms = MySum.fromString(line)
except ValueError:
self.printstatus(ii, ff, "not a MYSUM!")
print()
fail = True
break
self.printstatus(ii, ff, ms.fileName + " L" + str(ll))
dbf = DBFile(
fileName = ms.fileName,
group = gr,
comment = com,
fileSize = ms.fileSize,
md1 = ms.md1,
md5 = ms.md5,
ed2k = ms.ed2k,
)
matchingDBFile = self._determineMatchingDBFile(
session = session,
fileSize = dbf.fileSize,
md1 = dbf.md1,
md5 = dbf.md5,
ed2k = dbf.ed2k,
)
fullMatch = ((matchingDBFile is not None) and
(dbf.fileName == matchingDBFile.fileName) and
(dbf.group == matchingDBFile.group) and
(dbf.comment == matchingDBFile.comment))
if (matchingDBFile):
warn = warn + 1
if (fullMatch):
self.printstatus(ii, ff, "Already registered (full match) as {} L{}".format(matchingDBFile.fileId, ll))
else:
self.printstatus(ii, ff, "Already registered (data match) as {} L{}".format(matchingDBFile.fileId, ll))
print()
continue
jj = jj + 1
dbFilesToStoreFromImportFile.append(dbf)
if fail:
if ll == 1:
self.printstatus(ii, ff, "FAILED")
failfiles.append(ff)
print()
else:
sll = "after " + str(ll) + " lines"
self.printstatus(ii, ff, "FAILED " + sll)
failfiles.append(ff + " (" + sll + ")")
print()
else:
for dbf in dbFilesToStoreFromImportFile:
DBFileRegister.insert(session, dbf, commit=False)
allDBFilesToStore.extend(dbFilesToStoreFromImportFile)
print()
print(self.RULER)
print("About to import {} entries ({} warnings) from {} files out of {}".format(jj, warn, len(self.files) - len(failfiles), len(self.files)))
if len(failfiles) > 0:
print("A list of files that failed:")
for ff in failfiles:
print(" " + ff)
if self.docommit(failfiles):
session.commit()
for storedDBFile in allDBFilesToStore:
self.log(Register.LOGADD + self._formatDBFileForLog(storedDBFile))
print("Done.")
else:
print("Aborted!")
def registercheck(self, register):
"""
register (or check) the given files
"""
pgr, pcom, pdir, psize = "", "", "", 0 # previous group, comment, directory and size
tstart = time.time()
ii = 0
failfiles = []
dbFilesToStore = []
with self.dbConnection.getSessionContext() as session:
for ff in self.files:
ii = ii + 1
fileMightBeRegistered = False
cdir = os.path.dirname(ff) # directory
sff = os.path.basename(ff) # short filename
try:
if cdir != pdir:
print("Directory [{}]".format(cdir))
pdir = cdir
dbf = DBFile(fileName = sff)
if register: # group and comment
gr, com = self.getgroupcomment(ff)
if gr != pgr or com != pcom:
print("Using group:'{}' comment:'{}'".format(gr, com))
pgr, pcom = gr, com
dbf.group, dbf.comment = gr, com
self.printstatus(ii, sff, "Quick")
# stage 1 - silent
ms = MySum(ff)
ms.upgrade(1)
dbf.fileSize = ms.fileSize
dbf.md1 = ms.md1
fileMightBeRegistered = self._determineFileMightBeRegistered(
session = session,
fileSize = dbf.fileSize,
md1 = dbf.md1,
)
if ((not register) and (not fileMightBeRegistered)):
self.printstatus(ii, sff, "FAILED")
failfiles.append(ff)
print()
continue
tt = threading.Thread(target=ms.upgrade,args=[2])
tt.start()
while tt.is_alive():
try:
self.printstatus(ii, sff, self.msgpgs(ms,psize,tstart,fileMightBeRegistered))
time.sleep(0.25)
except KeyboardInterrupt:
ms.requestStop()
tt.join()
raise
tt.join()
psize = psize + ms.fileSize
dbf.md5 = ms.md5
dbf.ed2k = ms.ed2k
matchingDBFile = self._determineMatchingDBFile(
session = session,
fileSize = dbf.fileSize,
md1 = dbf.md1,
md5 = dbf.md5,
ed2k = dbf.ed2k,
)
fullMatch = (
(matchingDBFile is not None) and
(dbf.fileName == matchingDBFile.fileName) and
(dbf.group == matchingDBFile.group) and
(dbf.comment == matchingDBFile.comment)
)
if register:
if (matchingDBFile is None):
DBFileRegister.insert(session, dbf, commit=False)
self.printstatus(ii, sff, "New entry " + str(dbf.fileId))
dbFilesToStore.append(dbf)
else:
if (fullMatch):
self.printstatus(ii, sff, "Already registered (full match) as " + str(matchingDBFile.fileId))
else:
self.printstatus(ii, sff, "Already registered (data match) as " + str(matchingDBFile.fileId))
failfiles.append(ff)
else:
if (matchingDBFile is None):
self.printstatus(ii, sff, "FAILED")
failfiles.append(ff)
else:
stat = "OK"
if (dbf.fileName != matchingDBFile.fileName):
stat = "(as " + matchingDBFile.fileName + ") OK"
stat = "id:" + str(matchingDBFile.fileId) + " " + stat
self.printstatus(ii, sff, stat)
print()
except KeyboardInterrupt:
self.printstatus(ii, sff, "Interrupted")
failfiles.append(ff + " (Interrupted)")
print()
break
print(self.RULER)
if register:
print("About to register {} files out of {}".format(ii-len(failfiles),ii))
else:
print("Passed {} files out of {}.{}".format(ii-len(failfiles), ii, "ALL OK" if not len(failfiles) else "" ))
if len(failfiles) > 0:
print("A list of files that failed:")
for ff in failfiles:
print(" " + ff)
if register:
if len(failfiles) == ii:
print("No files were registered!")
elif self.docommit(failfiles):
session.commit()
for storedDBFile in dbFilesToStore:
self.log(Register.LOGADD + self._formatDBFileForLog(storedDBFile))
print("Done.")
else:
print("Aborted!")
