def updateinfo(options, to_remove_files, to_keep_files, logfile):
to_remove = []
for book in to_remove_files:
to_remove.append(dmBookWrapper(book[BOOK]))
to_keep = []
for book in to_keep_files:
to_keep.append(dmBookWrapper(book[BOOK]))
for field in FIELDS_TO_UPDATE_INFO:
data = None
# Get available data
for book in to_remove:
book_data = getattr(book, field[0])
if options["debug"]:
logfile.write(' rem: ' + book.FileName + ': ' + field[0] +
' = ' + ToString(book_data) + '\n')
if book_data:
data = book_data
break
if not data:
continue
try:
data = field[1](data)
except:
if options["verbose"]:
logfile.write(
'updating... Could not convert data to correct type ' +
field[0] + ' = ' + ToString(data) + '\n')
continue
# Set in missing books
for book in to_keep:
book_data = getattr(book, field[0])
if options["debug"]:
logfile.write(' keep: ' + book.FileName + ': ' + field[0] +
' = ' + ToString(book_data) + '\n')
if not book_data:
if not options["updateinfo"]:
logfile.write('[simulation] ')
logfile.write('updating... ' + book.FileName + ': ' +
field[0] + ' = ' + ToString(data) + '\n')
if options["updateinfo"]:
setattr(book.raw, field[0], data)
def updateinfo(options, to_remove_files, to_keep_files, logfile):
to_remove = []
for book in to_remove_files:
to_remove.append(dmBookWrapper(book[BOOK]))
to_keep = []
for book in to_keep_files:
to_keep.append(dmBookWrapper(book[BOOK]))
for field in FIELDS_TO_UPDATE_INFO:
data = None
# Get available data
for book in to_remove:
book_data = getattr(book, field[0])
if options["debug"]:
logfile.write(' rem: ' + book.FileName + ': ' + field[0] + ' = ' + ToString(book_data) + '\n')
if book_data:
data = book_data
break
if not data:
continue
try:
data = field[1](data)
except:
if options["verbose"]:
logfile.write('updating... Could not convert data to correct type ' + field[0] + ' = ' + ToString(data) + '\n')
continue
# Set in missing books
for book in to_keep:
book_data = getattr(book, field[0])
if options["debug"]:
logfile.write(' keep: ' + book.FileName + ': ' + field[0] + ' = ' + ToString(book_data) + '\n')
if not book_data:
if not options["updateinfo"]:
logfile.write('[simulation] ')
logfile.write('updating... ' + book.FileName + ': ' + field[0] + ' = ' + ToString(data) + '\n')
if options["updateinfo"]:
setattr(book.raw, field[0], data)
def ProcessDuplicates(books, logfile):
#########################################
#
# Getting comics info
comiclist = []
for book in books:
b = dmBookWrapper(book)
# re.sub(r'^0+','',b.Number) -> removes leading 0's
series = b.Series
if b.Volume:
series += ' Vol.' + b.Volume
series += ' ' + b.Format
comiclist.append((cleanupseries(series),re.sub(r'^0+','',b.Number),b.Volume,b.FileName,b.PageCount,b.FileSize/1048576.0,b.ID,b.CVDB_ID,b.FilePath,book.Tags,book.Notes,b.FileFormat,b.ScanInformation,book))
logfile.write('Parsing '+str(len(comiclist))+ ' ecomics\n')
#
#
########################################
#########################################
#
# Setting intial options values
options = {"movefiles":MOVEFILES,
"removefromlib":REMOVEFROMLIB,
"updateinfo":UPDATEINFO,
"verbose":VERBOSE,
"debug":DEBUG,
"sizemargin":SIZEMARGIN,
"coverpages":COVERPAGES,
"c2c_noads_gap":C2C_NOADS_GAP}
########################################
#
# Main Loop
#
try:
###########################################
#
# Load rules file
rules = LoadRules(logfile, options)
#
############################################
############################################
#
# Massage comics to get a list of dupes groups
''' Now we group books looking for dupes! '''
comiclist.sort()
''' begin sorting and sort the list '''
# TODO: I need to cleanup the series names and issues 1/2, 0.5, etc...
# TODO: Also, check for CVDB items first!
cl = {}
''' temp dictionary'''
for key, group in groupby(comiclist, lambda x: x[SERIES]):
cl[key] = list(group)
'''groups by series'''
''' cl is a dictionary that now has 'series' as keys'''
''' we remove series with only one ecomic '''
logfile.write('============= Begining dupes identification ==================================\n\n')
logfile.write('Parsing '+str(len(comiclist))+ ' ecomics\n')
logfile.write('Found '+str(len(cl))+ ' different series\n')
if options["verbose"]:
for series in sorted(cl.keys()):
logfile.write('\t'+series+'\n')
remove = []
for series in cl.keys():
if len(cl[series])==1:
remove.append(series)
for series in remove:
del cl[series]
logfile.write('Found '+str(len(cl))+ ' different series with more than one issue\n')
if options["verbose"]:
for series in sorted(cl.keys()):
logfile.write('\t'+series+'\n')
''' we now regroup each series looking for dupe issues '''
# We need to use a different list or sometimes python duplicates entries
temp_cl = {}
for series in cl.keys():
cl[series].sort()
temp_dict = {}
for key, group in groupby(cl[series], lambda x: x[NUMBER]):
temp_dict[key] = list(group)
temp_cl[series] = temp_dict
cl = temp_cl
''' cleaning issues without dupes '''
remove = []
for series in cl.keys():
for number in cl[series]:
if len(cl[series][number])==1:
remove.append([series,number])
for a in remove:
del cl[a[0]][a[1]]
''' now a second go for series without issues after non-dupe removal '''
remove = []
for i in cl:
if len(cl[i])==0:
remove.append(i)
for i in remove:
del cl[i]
logfile.write('Found '+str(len(cl))+ ' different series with dupes\n')
if options["verbose"]:
for series in sorted(cl.keys()):
logfile.write('\t'+series+'\t('+str(cl[series].keys())+')\n')
''' Now I have them sorted, I convert them to a simple list of lists (groups)...
each item in this list is a list of dupes '''
dupe_groups = []
for i in cl:
for j in cl[i]:
dupe_groups.append(cl[i][j])
logfile.write('Found '+str(len(dupe_groups)) +' groups of dupes, with a total of '+ str(len(reduce(list.__add__, dupe_groups, [])))+ ' ecomics.\n')
if options["verbose"]:
for group in sorted(dupe_groups):
logfile.write('\t'+group[0][SERIES]+' #'+group[0][NUMBER]+'\n')
for comic in group:
logfile.write('\t\t'+comic[FILENAME]+'\n')
dupe_groups.sort()
logfile.write('\n============= End of dupes identification=====================================\n\n\n\n')
logfile.write('============= Beginning dupes processing =====================================\n\n')
del cl
#
##########################################################
#
# Exception handling
#
except NoRulesFileException, ex:
MessageBox.Show('ERROR: '+ str(ex), "ERROR in Rules File", MessageBoxButtons.OK, MessageBoxIcon.Exclamation)
logfile.write('\n\nERROR in Rules File:\n')
traceback.print_exc(None, logfile, False)
return
def ProcessDuplicates(books, logfile):
#########################################
#
# Getting comics info
comiclist = []
for book in books:
b = dmBookWrapper(book)
# re.sub(r'^0+','',b.Number) -> removes leading 0's
series = b.Series
if b.Volume:
series += ' Vol.' + b.Volume
series += ' ' + b.Format
comiclist.append(
(cleanupseries(series), re.sub(r'^0+', '',
b.Number), b.Volume, b.FileName,
b.PageCount, b.FileSize / 1048576.0, b.ID, b.CVDB_ID, b.FilePath,
book.Tags, book.Notes, b.FileFormat, b.ScanInformation, book))
logfile.write('Parsing ' + str(len(comiclist)) + ' ecomics\n')
#
#
########################################
#########################################
#
# Setting intial options values
options = {
"movefiles": MOVEFILES,
"removefromlib": REMOVEFROMLIB,
"updateinfo": UPDATEINFO,
"verbose": VERBOSE,
"debug": DEBUG,
"sizemargin": SIZEMARGIN,
"coverpages": COVERPAGES,
"c2c_noads_gap": C2C_NOADS_GAP
}
########################################
#
# Main Loop
#
try:
###########################################
#
# Load rules file
rules = LoadRules(logfile, options)
#
############################################
############################################
#
# Massage comics to get a list of dupes groups
''' Now we group books looking for dupes! '''
comiclist.sort()
''' begin sorting and sort the list '''
# TODO: I need to cleanup the series names and issues 1/2, 0.5, etc...
# TODO: Also, check for CVDB items first!
cl = {}
''' temp dictionary'''
for key, group in groupby(comiclist, lambda x: x[SERIES]):
cl[key] = list(group)
'''groups by series'''
''' cl is a dictionary that now has 'series' as keys'''
''' we remove series with only one ecomic '''
logfile.write(
'============= Begining dupes identification ==================================\n\n'
)
logfile.write('Parsing ' + str(len(comiclist)) + ' ecomics\n')
logfile.write('Found ' + str(len(cl)) + ' different series\n')
if options["verbose"]:
for series in sorted(cl.keys()):
logfile.write('\t' + series + '\n')
remove = []
for series in cl.keys():
if len(cl[series]) == 1:
remove.append(series)
for series in remove:
del cl[series]
logfile.write('Found ' + str(len(cl)) +
' different series with more than one issue\n')
if options["verbose"]:
for series in sorted(cl.keys()):
logfile.write('\t' + series + '\n')
''' we now regroup each series looking for dupe issues '''
# We need to use a different list or sometimes python duplicates entries
temp_cl = {}
for series in cl.keys():
cl[series].sort()
temp_dict = {}
for key, group in groupby(cl[series], lambda x: x[NUMBER]):
temp_dict[key] = list(group)
temp_cl[series] = temp_dict
cl = temp_cl
''' cleaning issues without dupes '''
remove = []
for series in cl.keys():
for number in cl[series]:
if len(cl[series][number]) == 1:
remove.append([series, number])
for a in remove:
del cl[a[0]][a[1]]
''' now a second go for series without issues after non-dupe removal '''
remove = []
for i in cl:
if len(cl[i]) == 0:
remove.append(i)
for i in remove:
del cl[i]
logfile.write('Found ' + str(len(cl)) +
' different series with dupes\n')
if options["verbose"]:
for series in sorted(cl.keys()):
logfile.write('\t' + series + '\t(' + str(cl[series].keys()) +
')\n')
''' Now I have them sorted, I convert them to a simple list of lists (groups)...
each item in this list is a list of dupes '''
dupe_groups = []
for i in cl:
for j in cl[i]:
dupe_groups.append(cl[i][j])
logfile.write('Found ' + str(len(dupe_groups)) +
' groups of dupes, with a total of ' +
str(len(reduce(list.__add__, dupe_groups, []))) +
' ecomics.\n')
if options["verbose"]:
for group in sorted(dupe_groups):
logfile.write('\t' + group[0][SERIES] + ' #' +
group[0][NUMBER] + '\n')
for comic in group:
logfile.write('\t\t' + comic[FILENAME] + '\n')
dupe_groups.sort()
logfile.write(
'\n============= End of dupes identification=====================================\n\n\n\n'
)
logfile.write(
'============= Beginning dupes processing =====================================\n\n'
)
del cl
#
##########################################################
#
# Exception handling
#
except NoRulesFileException, ex:
MessageBox.Show('ERROR: ' + str(ex), "ERROR in Rules File",
MessageBoxButtons.OK, MessageBoxIcon.Exclamation)
logfile.write('\n\nERROR in Rules File:\n')
traceback.print_exc(None, logfile, False)
return
