def add_group(form): name = form.name.data.strip() have = AlternateTranslatorNames.query.filter( AlternateTranslatorNames.cleanname == nt.prepFilenameForMatching( name)).scalar() if have: flash(gettext('Group already exists!')) return redirect(url_for('renderGroupId', sid=have.group)) else: new = Translators( name=name, changetime=datetime.datetime.now(), changeuser=g.user.id, ) db.session.add(new) db.session.commit() newname = AlternateTranslatorNames( name=name, cleanname=nt.prepFilenameForMatching(name), group=new.id, changetime=datetime.datetime.now(), changeuser=g.user.id) db.session.add(newname) db.session.commit() flash(gettext('Group Created!')) return redirect(url_for('renderGroupId', sid=new.id))
def add_group(form): name = form.name.data.strip() have = AlternateTranslatorNames.query.filter(AlternateTranslatorNames.cleanname==nt.prepFilenameForMatching(name)).scalar() if have: flash(gettext('Group already exists!')) return redirect(url_for('renderGroupId', sid=have.group)) else: new = Translators( name = name, changetime = datetime.datetime.now(), changeuser = g.user.id, ) db.session.add(new) db.session.commit() newname = AlternateTranslatorNames( name = name, cleanname = nt.prepFilenameForMatching(name), group = new.id, changetime = datetime.datetime.now(), changeuser = g.user.id ) db.session.add(newname) db.session.commit() flash(gettext('Group Created!')) return redirect(url_for('renderGroupId', sid=new.id))
def add_series(form): name = form.name.data.strip() stripped = nt.prepFilenameForMatching(name) have = AlternateNames.query.filter(AlternateNames.cleanname==stripped).all() rel_type = form.type.data.strip() if len(have) == 1: flash(gettext('Series exists under a different name!')) return redirect(url_for('renderSeriesId', sid=have[0].series)) elif have: flash(gettext('Have multiple candidate series that look like that name!')) return redirect(url_for('search', title=name)) else: new = Series( title = name, tl_type = rel_type, changetime = datetime.datetime.now(), changeuser = g.user.id, ) db.session.add(new) db.session.commit() # session must be committed before adding alternate names, # or the primary key links will fail. series_tools.updateAltNames(new, [name]) flash(gettext('Series Created!')) # return redirect(url_for('index')) return redirect(url_for('renderSeriesId', sid=new.id))
def title_search(searchterm, page=1): searchtermclean = bleach.clean(searchterm, strip=True) searchterm = nt.prepFilenameForMatching(searchtermclean) if not searchterm: return render_template( 'not-implemented-yet.html', message= 'No search term entered (or search term collapsed down to nothing).' ) similarity = Function('similarity', AlternateNames.cleanname, (searchterm)) query = select([ AlternateNames.series, AlternateNames.cleanname, AlternateNames.name, similarity ], from_obj=[AlternateNames], order_by=desc(similarity)).where( or_( AlternateNames.cleanname.op("%%")(searchterm), AlternateNames.cleanname.like(searchterm + "%%"))).limit(50) results = db.session.execute(query).fetchall() data = collections.OrderedDict() uid = g.user.get_id() for result in results: dbid = result[0] if not dbid in data: data[dbid] = {} data[dbid]['row'] = Series.query.filter(Series.id == dbid).one() if uid: data[dbid]['watch'] = Watches \ .query \ .filter(Watches.series_id==dbid) \ .filter(Watches.user_id==uid) \ .scalar() else: data[dbid]['watch'] = [] data[dbid]['results'] = [] # We only care about relative ordering, and # since we're ordered when we iterate, if we # just append here, things will stay in the correct # order. data[dbid]['results'].append(result) # print(results) # print(data) return render_template( 'text-search.html', results=data, name_key="tag", page_url_prefix='tag-id', searchTarget="Titles", searchValue=searchtermclean, title='Search for \'{name}\''.format(name=searchtermclean))
def updateAltNames(series, altnames, deleteother=True): # print("Alt names:", altnames) altnames = [name.strip() for name in altnames] cleaned = {} for name in altnames: if name.lower().strip(): cleaned[name.lower().strip()] = name havenames = AlternateNames.query.filter(AlternateNames.series==series.id).order_by(AlternateNames.name).all() havenames = {bleach.clean(name.name.lower().strip(), strip=True) : name for name in havenames} for name in cleaned.keys(): if name in havenames: havenames.pop(name) else: newname = AlternateNames( name = cleaned[name], cleanname = nt.prepFilenameForMatching(cleaned[name]), series = series.id, changetime = datetime.datetime.now(), changeuser = getCurrentUserId() ) db.session.add(newname) if deleteother: for key, value in havenames.items(): db.session.delete(value) db.session.commit()
def generate_similarity_query(searchterm, cols=None): searchtermclean = bleach.clean(searchterm, strip=True) searchtermprocessed = nt.prepFilenameForMatching(searchtermclean) similarity = Function('similarity', AlternateNames.cleanname, (searchtermprocessed)) if cols is None: cols = [AlternateNames.series, AlternateNames.cleanname, AlternateNames.name, similarity] if not searchterm: return None, None query = select( cols, from_obj=[AlternateNames], order_by=desc(similarity) ).where( or_( AlternateNames.cleanname.op("%%")(searchtermprocessed), AlternateNames.cleanname.like(searchtermprocessed + "%%") ) ).limit( 50 ) return query, searchtermprocessed
def updateGroupAltNames(group, altnames, delete=True): print("Alt names:", altnames) altnames = [name.strip() for name in altnames] cleaned = {} for name in altnames: if name.lower().strip(): cleaned[name.lower().strip()] = name havenames = AlternateTranslatorNames.query.filter(AlternateTranslatorNames.group==group.id).order_by(AlternateTranslatorNames.name).all() havenames = {name.name.lower().strip() : name for name in havenames} for name in cleaned.keys(): if name in havenames: havenames.pop(name) else: newname = AlternateTranslatorNames( name = cleaned[name], cleanname = nt.prepFilenameForMatching(cleaned[name]), group = group.id, changetime = datetime.datetime.now(), changeuser = getCurrentUserId() ) db.session.add(newname) if delete: for key, value in havenames.items(): db.session.delete(value) db.session.commit()
def updateAltNames(series, altnames, deleteother=True): # print("Alt names:", altnames) altnames = [name.strip() for name in altnames] cleaned = {} for name in altnames: if name.lower().strip(): cleaned[name.lower().strip()] = name havenames = AlternateNames.query.filter(AlternateNames.series==series.id).order_by(AlternateNames.name).all() havenames = {bleach.clean(name.name.lower().strip(), strip=True) : name for name in havenames} for name in cleaned.keys(): if name in havenames: havenames.pop(name) else: have = AlternateNames.query.filter(AlternateNames.series==series.id).filter(AlternateNames.name == cleaned[name]).count() if not have: newname = AlternateNames( name = cleaned[name], cleanname = nt.prepFilenameForMatching(cleaned[name]), series = series.id, changetime = datetime.datetime.now(), changeuser = getCurrentUserId() ) db.session.add(newname) if deleteother: for dummy_key, value in havenames.items(): db.session.delete(value) db.session.commit()
def updateGroupAltNames(group, altnames, deleteother=True): print("Alt names:", altnames) altnames = [name.strip() for name in altnames] cleaned = {} for name in altnames: if name.lower().strip(): cleaned[name.lower().strip()] = name havenames = AlternateTranslatorNames.query.filter(AlternateTranslatorNames.group==group.id).order_by(AlternateTranslatorNames.name).all() havenames = {name.name.lower().strip() : name for name in havenames} for name in cleaned.keys(): if name in havenames: havenames.pop(name) else: newname = AlternateTranslatorNames( name = bleach.clean(cleaned[name], strip=True), cleanname = nt.prepFilenameForMatching(cleaned[name]), group = group.id, changetime = datetime.datetime.now(), changeuser = getCurrentUserId() ) db.session.add(newname) if deleteother: for key, value in havenames.items(): db.session.delete(value) db.session.commit()
def title_search(searchterm, page=1): searchtermclean = bleach.clean(searchterm, strip=True) searchterm = nt.prepFilenameForMatching(searchtermclean) if not searchterm: return render_template( "not-implemented-yet.html", message="No search term entered (or search term collapsed down to nothing)." ) similarity = Function("similarity", AlternateNames.cleanname, (searchterm)) query = ( select( [AlternateNames.series, AlternateNames.cleanname, AlternateNames.name, similarity], from_obj=[AlternateNames], order_by=desc(similarity), ) .where(AlternateNames.cleanname.op("%%")(searchterm)) .limit(50) ) results = db.session.execute(query).fetchall() data = collections.OrderedDict() uid = g.user.get_id() for result in results: dbid = result[0] if not dbid in data: data[dbid] = {} data[dbid]["row"] = Series.query.filter(Series.id == dbid).one() if uid: data[dbid]["watch"] = ( Watches.query.filter(Watches.series_id == dbid).filter(Watches.user_id == uid).scalar() ) else: data[dbid]["watch"] = [] data[dbid]["results"] = [] # We only care about relative ordering, and # since we're ordered when we iterate, if we # just append here, things will stay in the correct # order. data[dbid]["results"].append(result) # print(results) # print(data) return render_template( "text-search.html", results=data, name_key="tag", page_url_prefix="tag-id", searchTarget="Titles", searchValue=searchtermclean, title="Search for '{name}'".format(name=searchtermclean), )
def get_create_group(groupname, changeuser): groupname = groupname[:500] cleanName = nt.prepFilenameForMatching(groupname) # If the group name collapses down to nothing when cleaned, search for it without cleaning. if len(cleanName): have = AlternateTranslatorNames.query.filter( AlternateTranslatorNames.cleanname == cleanName).all() else: have = AlternateTranslatorNames.query.filter( AlternateTranslatorNames.name == groupname).all() if not have: print("Need to create new translator entry for ", groupname) new = Translators(name=groupname, changeuser=changeuser, changetime=datetime.datetime.now()) db.session.add(new) db.session.commit() newalt = AlternateTranslatorNames( group=new.id, name=new.name, cleanname=nt.prepFilenameForMatching(new.name), changetime=datetime.datetime.now(), changeuser=changeuser, ) db.session.add(newalt) db.session.commit() return new else: if len(have) == 1: group = have[0] assert group.group_row is not None, ( "Wat? Row: '%s', '%s', '%s'" % (group.id, group.name, group.group_row)) elif len(have) > 1: group = pick_best_match(have, groupname) else: raise ValueError("Wat for groupname: '%s'" % groupname) row = group.group_row return row
def get_create_group(groupname, changeuser): groupname = groupname[:500] cleanName = nt.prepFilenameForMatching(groupname) # If the group name collapses down to nothing when cleaned, search for it without cleaning. if len(cleanName): have = AlternateTranslatorNames.query.filter(AlternateTranslatorNames.cleanname==cleanName).all() else: have = AlternateTranslatorNames.query.filter(AlternateTranslatorNames.name==groupname).all() if not have: print("Need to create new translator entry for ", groupname) new = Translators( name = groupname, changeuser = changeuser, changetime = datetime.datetime.now() ) db.session.add(new) db.session.commit() newalt = AlternateTranslatorNames( group = new.id, name = new.name, cleanname = nt.prepFilenameForMatching(new.name), changetime = datetime.datetime.now(), changeuser = changeuser, ) db.session.add(newalt) db.session.commit() return new else: if len(have) == 1: group = have[0] assert group.group_row is not None, ("Wat? Row: '%s', '%s', '%s'" % (group.id, group.name, group.group_row)) elif len(have) > 1: group = pick_best_match(have, groupname) else: raise ValueError("Wat for groupname: '%s'" % groupname) row = group.group_row return row
def get_create_series(seriesname, tl_type): # If two series have the same alt-name, pick the one with # the lower database-id have = AlternateNames \ .query \ .filter(AlternateNames.name == seriesname) \ .order_by(AlternateNames.id) \ .limit(1) \ .scalar() if not have: haveS = Series \ .query \ .filter(Series.title == seriesname) \ .limit(1) \ .scalar() if haveS: print("Wat? Item that isn't in the altname table but still exists?") return haveS print("Need to create new series entry for ", seriesname) new = Series( title=seriesname, changeuser = RSS_USER_ID, # Hard coded RSS user ID. Probably a bad idea. changetime = datetime.datetime.now(), tl_type = tl_type, ) db.session.add(new) db.session.flush() newname = AlternateNames( name = seriesname, cleanname = nt.prepFilenameForMatching(seriesname), series = new.id, changetime = datetime.datetime.now(), changeuser = RSS_USER_ID ) db.session.add(newname) db.session.flush() db.session.commit() return new return have.series_row
def create_series(seriesname, tl_type, changeuser, author_name, alt_names = None): if alt_names is None: alt_names = [] alt_names.append(seriesname) print("Need to create new series entry for ", seriesname) new = Series( title=seriesname, changeuser = changeuser, # Hard coded RSS user ID. Probably a bad idea. changetime = datetime.datetime.now(), tl_type = tl_type, ) db.session.add(new) db.session.flush() if author_name: if isinstance(author_name, str): author_name = [author_name, ] series_tools.setAuthorIllust(new, author=author_name) for altname in alt_names: have = AlternateNames \ .query \ .filter(AlternateNames.name == altname) \ .order_by(AlternateNames.id) \ .scalar() if not have: altn_row = AlternateNames( name = altname, cleanname = nt.prepFilenameForMatching(altname), series = new.id, changetime = datetime.datetime.now(), changeuser = changeuser ) db.session.add(altn_row) return new
def add_series(form): name = form.name.data.strip() name = text_tools.fix_string(name, recase=False) stripped = nt.prepFilenameForMatching(name) have = AlternateNames.query.filter( AlternateNames.cleanname == stripped).all() rel_type = form.type.data.strip() if len(have) == 1: flash(gettext('Series exists under a different name!')) return redirect( url_for('renderSeriesIdWithoutSlug', sid=have[0].series)) elif have: flash( gettext( 'Have multiple candidate series that look like that name!')) return redirect(url_for('search', title=name)) else: new = Series( title=name, tl_type=rel_type, changetime=datetime.datetime.now(), changeuser=g.user.id, ) db.session.add(new) db.session.commit() # session must be committed before adding alternate names, # or the primary key links will fail. series_tools.updateAltNames(new, [name]) flash(gettext('Series Created!')) # return redirect(url_for('index')) return redirect(url_for('renderSeriesIdWithoutSlug', sid=new.id))
def get_create_group(groupname): have = Translators.query.filter(Translators.name==groupname).scalar() if not have: print("Need to create new translator entry for ", groupname) new = Translators( name = groupname, changeuser = RSS_USER_ID, changetime = datetime.datetime.now() ) db.session.add(new) db.session.commit() newalt = AlternateTranslatorNames( group = new.id, name = new.name, cleanname = nt.prepFilenameForMatching(new.name), changetime = datetime.datetime.now(), changeuser = RSS_USER_ID, ) db.session.add(newalt) db.session.commit() return new return have
def add_similarity_query(searchterm, query): searchtermclean = bleach.clean(searchterm, strip=True) searchtermprocessed = nt.prepFilenameForMatching(searchtermclean) similarity = Function('similarity', AlternateNames.cleanname, (searchtermprocessed)) if not searchterm: return None, None query = query.filter( Series.id.in_( db.session.query(AlternateNames.series).filter( or_( AlternateNames.cleanname.op("%%")(searchtermprocessed), AlternateNames.cleanname.like(searchtermprocessed + "%%") ) ).order_by( desc(similarity) ) ) ) return query
def get_create_series(seriesname, tl_type, author_name=False): # print("get_create_series(): '%s', '%s', '%s'" % (seriesname, tl_type, author_name)) tries = 0 while 1: try: have = AlternateNames \ .query \ .filter(AlternateNames.name == seriesname) \ .order_by(AlternateNames.id) \ .all() # print("get_create_series for title: '%s'" % seriesname) # print("Altnames matches: ", have) # for item in have: # print((item.series_row.id, item.series_row.title, [tmp.name.lower() for tmp in item.series_row.author])) # print("Want:", author_name) # There's 4 options here: # - Update and have item has author -> # match, fail if match fails. # - Update has author, have does not -> # only allow matches after haves with authors exhausted. # - have has author, update does not -> # Glob onto series anyways. # - Update and have do not have author -> # do best match. # From the perspective of our approach, if we have a name, we try for that, then # look for empty items, finally return none if nothing present. # if we don't have a name, we look for # Try to match any alt-names we have. valid_haves = [tmp for tmp in have if tmp.series_row.tl_type == tl_type] # Try for author match first: if author_name: for item in [tmp for tmp in valid_haves if tmp.series_row.author]: if isinstance(author_name, list): if any([auth_tmp.lower() in [tmp.name.lower() for tmp in item.series_row.author] for auth_tmp in author_name]): # print("AuthorName match!") return item.series_row else: if author_name.lower() in [tmp.name.lower() for tmp in item.series_row.author]: return item.series_row for item in [tmp for tmp in valid_haves if not tmp.series_row.author]: return item.series_row else: # No author specified globs onto first possible match. for item in valid_haves: return item.series_row # print("No match found while filtering by author-name!") haveS = Series \ .query \ .filter(Series.title == seriesname) \ .limit(1) \ .scalar() if haveS and author_name: if isinstance(author_name, str): sName = "{} ({})".format(seriesname, author_name) else: sName = "{} ({})".format(seriesname, ", ".join(author_name)) elif haveS: if haveS.tl_type != tl_type: if tl_type == "oel": st = "OEL" else: st = tl_type.title() sName = "{} ({})".format(seriesname, st) else: # print("Wat? Item that isn't in the altname table but still exists?") return haveS else: sName = seriesname # We've built a new series title by appending the author/tl_type # Now we need to check if that exists too. if sName != seriesname: haveS = Series \ .query \ .filter(Series.title == seriesname) \ .limit(1) \ .scalar() return haveS print("Need to create new series entry for ", seriesname) new = Series( title=sName, changeuser = RSS_USER_ID, # Hard coded RSS user ID. Probably a bad idea. changetime = datetime.datetime.now(), tl_type = tl_type, ) db.session.add(new) db.session.flush() if author_name: if isinstance(author_name, str): author_name = [author_name, ] app.series_tools.setAuthorIllust(new, author=author_name) altn1 = AlternateNames( name = seriesname, cleanname = nt.prepFilenameForMatching(seriesname), series = new.id, changetime = datetime.datetime.now(), changeuser = RSS_USER_ID ) db.session.add(altn1) if sName != seriesname: altn2 = AlternateNames( name = sName, cleanname = nt.prepFilenameForMatching(seriesname), series = new.id, changetime = datetime.datetime.now(), changeuser = RSS_USER_ID ) db.session.add(altn2) db.session.commit() return new except sqlalchemy.exc.IntegrityError: print("Concurrency issue?") print("'%s', '%s', '%s'" % (seriesname, tl_type, author_name)) db.session.rollback() tries += 1 if tries > 3: raise except Exception: print("Error!") raise
def get_create_series(seriesname, tl_type, changeuser, author_name=False): # print("get_create_series(): '%s', '%s', '%s'" % (seriesname, tl_type, author_name)) tries = 0 while 1: try: have = AlternateNames \ .query \ .filter(AlternateNames.name == seriesname) \ .order_by(AlternateNames.id) \ .all() # print("get_create_series for title: '%s'" % seriesname) # print("Altnames matches: ", have) # for item in have: # print((item.series_row.id, item.series_row.title, [tmp.name.lower() for tmp in item.series_row.author])) # print("Want:", author_name) # There's 4 options here: # - Update and have item has author -> # match, fail if match fails. # - Update has author, have does not -> # only allow matches after haves with authors exhausted. # - have has author, update does not -> # Glob onto series anyways. # - Update and have do not have author -> # do best match. # From the perspective of our approach, if we have a name, we try for that, then # look for empty items, finally return none if nothing present. # if we don't have a name, we look for # Try to match any alt-names we have. if not all([tmp.series_row for tmp in have]): db.session.commit() valid_haves = [ tmp for tmp in have if tmp.series_row and tmp.series_row.tl_type == tl_type ] # Try for author match first: if author_name: for item in [ tmp for tmp in valid_haves if tmp.series_row.author ]: if isinstance(author_name, list): if any([ auth_tmp.lower() in [ tmp.name.lower() for tmp in item.series_row.author ] for auth_tmp in author_name ]): # print("AuthorName match!") return item.series_row else: if author_name.lower() in [ tmp.name.lower() for tmp in item.series_row.author ]: return item.series_row for item in [ tmp for tmp in valid_haves if not tmp.series_row.author ]: return item.series_row else: # No author specified globs onto first possible match. for item in valid_haves: return item.series_row # print("No match found while filtering by author-name!") haveS = Series \ .query \ .filter(Series.title == seriesname) \ .limit(1) \ .scalar() if haveS and author_name: if isinstance(author_name, str): sName = "{} ({})".format(seriesname, author_name) else: sName = "{} ({})".format(seriesname, ", ".join(author_name)) elif haveS: if haveS.tl_type != tl_type: if tl_type == "oel": st = "OEL" else: st = tl_type.title() sName = "{} ({})".format(seriesname, st) else: # print("Wat? Item that isn't in the altname table but still exists?") return haveS else: sName = seriesname # We've built a new series title by appending the author/tl_type # Now we need to check if that exists too. if sName != seriesname: haveS = Series \ .query \ .filter(Series.title == seriesname) \ .limit(1) \ .scalar() return haveS print("Need to create new series entry for ", seriesname) new = Series( title=sName, changeuser= changeuser, # Hard coded RSS user ID. Probably a bad idea. changetime=datetime.datetime.now(), tl_type=tl_type, ) db.session.add(new) db.session.flush() if author_name: if isinstance(author_name, str): author_name = [ author_name, ] series_tools.setAuthorIllust(new, author=author_name) altn1 = AlternateNames( name=seriesname, cleanname=nt.prepFilenameForMatching(seriesname), series=new.id, changetime=datetime.datetime.now(), changeuser=changeuser) db.session.add(altn1) if sName != seriesname: altn2 = AlternateNames( name=sName, cleanname=nt.prepFilenameForMatching(seriesname), series=new.id, changetime=datetime.datetime.now(), changeuser=changeuser) db.session.add(altn2) db.session.commit() return new except sqlalchemy.exc.IntegrityError: print("Concurrency issue?") print("'%s', '%s', '%s'" % (seriesname, tl_type, author_name)) db.session.rollback() tries += 1 if tries > 3: raise except Exception: print("Error!") raise