def batchimport(self):
"""
store all data stored in the provided files (usually named like sumlog.txt)
Only [MYSUM...] entries are valid.
"""
pgr, pcom, pdir = "", "", "" # previous group, comment and directory
ii = 0 # file no
jj = 0 # successfully imported entries
warn = 0 # number of warnings (duplicities)
failfiles = [] # a list of files that failed to import
allDBFilesToStore = []
with self.dbConnection.getSessionContext() as session:
for ff in self.files:
ii = ii + 1
cdir = os.path.dirname(ff) # directory
if cdir != pdir:
print("Directory [{}]".format(cdir))
pdir = cdir
gr, com = self.getgroupcomment(ff, imp=True)
if gr != pgr or com != pcom:
print("Using group:{} comment:{}".format(gr, com))
pgr, pcom = gr, com
self.printstatus(ii, ff, "")
with open(ff, "r") as fsum:
ll = 0
fail = False
dbFilesToStoreFromImportFile = []
for line in fsum:
ll = ll + 1
self.printstatus(ii, ff, "L" + str(ll))
try:
ms = MySum.fromString(line)
except ValueError:
self.printstatus(ii, ff, "not a MYSUM!")
print()
fail = True
break
self.printstatus(ii, ff, ms.fileName + " L" + str(ll))
dbf = DBFile(
fileName = ms.fileName,
group = gr,
comment = com,
fileSize = ms.fileSize,
md1 = ms.md1,
md5 = ms.md5,
ed2k = ms.ed2k,
)
matchingDBFile = self._determineMatchingDBFile(
session = session,
fileSize = dbf.fileSize,
md1 = dbf.md1,
md5 = dbf.md5,
ed2k = dbf.ed2k,
)
fullMatch = ((matchingDBFile is not None) and
(dbf.fileName == matchingDBFile.fileName) and
(dbf.group == matchingDBFile.group) and
(dbf.comment == matchingDBFile.comment))
if (matchingDBFile):
warn = warn + 1
if (fullMatch):
self.printstatus(ii, ff, "Already registered (full match) as {} L{}".format(matchingDBFile.fileId, ll))
else:
self.printstatus(ii, ff, "Already registered (data match) as {} L{}".format(matchingDBFile.fileId, ll))
print()
continue
jj = jj + 1
dbFilesToStoreFromImportFile.append(dbf)
if fail:
if ll == 1:
self.printstatus(ii, ff, "FAILED")
failfiles.append(ff)
print()
else:
sll = "after " + str(ll) + " lines"
self.printstatus(ii, ff, "FAILED " + sll)
failfiles.append(ff + " (" + sll + ")")
print()
else:
for dbf in dbFilesToStoreFromImportFile:
DBFileRegister.insert(session, dbf, commit=False)
allDBFilesToStore.extend(dbFilesToStoreFromImportFile)
print()
print(self.RULER)
print("About to import {} entries ({} warnings) from {} files out of {}".format(jj, warn, len(self.files) - len(failfiles), len(self.files)))
if len(failfiles) > 0:
print("A list of files that failed:")
for ff in failfiles:
print(" " + ff)
if self.docommit(failfiles):
session.commit()
for storedDBFile in allDBFilesToStore:
self.log(Register.LOGADD + self._formatDBFileForLog(storedDBFile))
print("Done.")
else:
print("Aborted!")
