def insert_parsed_release(item): assert 'tl_type' in item assert 'srcname' in item assert 'series' in item if "nu_release" in item: update_id = NU_SRC_USER_ID else: update_id = RSS_USER_ID item = text_tools.fix_dict(item, recase=False) if item["tl_type"] not in ['oel', 'translated']: raise ValueError("Invalid TL Type '%s'! Wat?" % item["tl_type"]) group = get_create_group(item['srcname'], update_id) assert group is not None if 'loose_match' in item and item['loose_match']: series = get_create_series(item['series'], item["tl_type"], update_id) check_insert_release(item, group, series, update_id, loose_match=True) else: if 'match_author' in item and item['match_author']: series = get_create_series(item['series'], item["tl_type"], update_id, item['author']) else: series = get_create_series(item['series'], item["tl_type"], update_id) check_insert_release(item, group, series, update_id)
def insert_parsed_release(item): assert 'tl_type' in item assert 'srcname' in item assert 'series' in item if "nu_release" in item: update_id = NU_SRC_USER_ID else: update_id = RSS_USER_ID item = text_tools.fix_dict(item, recase=False) if item["tl_type"] not in ['oel', 'translated']: raise ValueError("Invalid TL Type '%s'! Wat?" % item["tl_type"]) group = get_create_group(item['srcname'], update_id) assert group is not None kwargs = { "input_series_name" : item['series'], "tl_type" : item["tl_type"], "changeuser" : update_id, } if 'match_author' in item and item['match_author']: kwargs['author_name_list'] = item['author'] series = get_create_series(**kwargs) prefix_match = item.get('prefix_match', False) loose_match = item.get('loose_match', False) check_insert_release(item, group, series, update_id, loose_match=loose_match, prefix_match=prefix_match)
def update_series_info(item): # print("update_series_info", item) assert 'title' in item assert 'author' in item assert 'tags' in item assert 'desc' in item assert 'tl_type' in item item = text_tools.fix_dict(item, recase=False) print("Series info update message for '%s'!" % item['title']) if not 'update_only' in item: item['update_only'] = False if not 'alt_titles' in item: item['alt_titles'] = [] item['alt_titles'].append(item['title']) if item['update_only']: series = get_series_from_any(item['alt_titles'], item["tl_type"], item['author']) else: series = get_create_series(item['title'], item["tl_type"], RSS_USER_ID, item['author']) # Break if the tl type has changed, something is probably mismatched if series.tl_type != item['tl_type']: print("WARNING! TlType mismatch? Wat?") print("Series:", series) print("###################################") print(series.title) print("-----------------------------------") print(item['title']) print("###################################") print(series.author) print("-----------------------------------") print(item['author']) print("###################################") print(series.description) print("-----------------------------------") print(item['desc']) print("###################################") print(series.tl_type) print("-----------------------------------") print(item['tl_type']) print("###################################") print(series.tags) print("-----------------------------------") print(item['tags']) return changeable = generate_automated_only_change_flags(series) # print(changeable) # print(item) # { # 'license_en': True, # 'orig_lang': True, # 'pub_date': True, # 'title': True, # 'description': False, # 'chapter': True, # 'origin_loc': True, # 'website': True, # 'region': True, # 'tl_type': True, # 'tot_chapter': True, # 'orig_status': True, # 'sort_mode': True, # 'volume': True, # 'demographic': True, # 'tot_volume': True, # 'type': True # } # This only generates a change is it needs to, so we can call it unconditionally. if changeable['title']: series_tools.updateTitle(series, item['title']) if changeable['description'] and 'desc' in item and item['desc']: newd = bleach.clean(item['desc'], strip=True, tags=['p', 'em', 'strong', 'b', 'i', 'a']) if newd != series.description and len(newd.strip()): series.description = newd elif ('desc' in item and item['desc'] and not series.description): newd = bleach.clean(item['desc'], strip=True, tags=['p', 'em', 'strong', 'b', 'i', 'a']) if len(newd.strip()): series.description = newd if 'homepage' in item and item['homepage'] and ( not series.website or (bleach.clean(item['homepage']) != series.website and changeable['website'])): neww = bleach.clean(item['homepage']) if len(neww.strip()): series.website = neww if 'coostate' in item and item['coostate']: if not series.orig_status or (item['coostate'] != series.orig_status and changeable['orig_status']): neww = bleach.clean(item['coostate']) if len(neww.strip()): series.orig_status = neww if 'tl_type' in item and item['tl_type']: if not series.tl_type or (item['tl_type'] != series.tl_type and changeable['tl_type']): neww = bleach.clean(item['tl_type']).strip() if neww and neww in ['western', 'eastern', 'unknown']: series.tl_type = neww # if 'transcomplete' in item and item['transcomplete']: # if not series.tl_complete or (item['transcomplete'] != series.tl_complete and changeable['transcomplete']): # neww = item['transcomplete'].strip() # if neww and neww in ['yes', 'no']: # series.tl_complete = neww if 'licensed' in item and item['licensed']: if not series.license_en or (item['licensed'] != series.license_en and changeable['license_en']): neww = item['licensed'].strip().lower() if neww and neww in ['yes', 'no']: series.license_en = neww == 'yes' if 'author' in item and item['author']: tmp = item['author'] if isinstance(tmp, str): tmp = [ tmp, ] series_tools.setAuthorIllust(series, author=tmp, deleteother=False) if 'illust' in item and item['illust']: tmp = item['illust'] if isinstance(tmp, str): tmp = [ tmp, ] series_tools.setAuthorIllust(series, illust=tmp, deleteother=False) if 'tags' in item and item['tags']: series_tools.updateTags(series, item['tags'], deleteother=False, allow_new=item.get('create_tags', False)) if 'genres' in item and item['genres']: series_tools.updateGenres(series, item['genres'], deleteother=False) if 'alt_titles' in item and item['alt_titles']: series_tools.updateAltNames(series, item['alt_titles'] + [ item['title'], ], deleteother=False) if 'pubnames' in item and item['pubnames']: series_tools.updatePublishers(series, item['pubnames'], deleteother=False) if 'pubdate' in item and item['pubdate']: if not series.pub_date: series.pub_date = datetime.datetime.utcfromtimestamp( item['pubdate']) if 'sourcesite' in item and item['sourcesite']: pass series.changeuser = RSS_USER_ID db.session.flush() db.session.commit()
def insert_raw_item(item): ''' insert item `item` into the feed database. fields in `item`: "title" "contents" "guid" "linkUrl" "feedtype" "published" "updated" "authors" -> "href" -> "name" "tags" ''' # print(item) if not 'srcname' in item: print("No srcname? Old item?") return entry = {} entry['title'] = item.pop('title') entry['contents'] = item.pop('contents', 'N/A') if not isinstance(entry['contents'], str): if isinstance(entry['contents'], list): entry['contents'] = entry['contents'].pop() if 'value' in entry['contents']: entry['contents'] = entry['contents']['value'] else: print(entry['contents']) entry['contents'] = str(entry['contents']) entry['guid'] = item.pop('guid') entry['linkurl'] = item.pop('linkUrl') entry['region'] = item.pop('feedtype') entry['srcname'] = item.pop('srcname') entry['published'] = datetime.datetime.fromtimestamp(item.pop('published')) if 'updated' in item: entry['updated'] = datetime.datetime.fromtimestamp(item.pop('updated')) item = text_tools.fix_dict(item, recase=False) itemrows = Feeds.query.filter(Feeds.guid == entry['guid']).all() if not itemrows: print("New feed item: ", entry['guid']) itemrow = Feeds(**entry) db.session.add(itemrow) db.session.flush() else: itemrow = itemrows[0] for tag in item.pop('tags'): if not FeedTags.query \ .filter(FeedTags.article_id==itemrow.id) \ .filter(FeedTags.tag == tag.strip()).scalar(): newtag = FeedTags(article_id=itemrow.id, tag=tag.strip()) db.session.add(newtag) db.session.flush() for author in item.pop('authors'): if not 'name' in author: continue if not FeedAuthors.query \ .filter(FeedAuthors.article_id==itemrow.id) \ .filter(FeedAuthors.name == author['name'].strip()).scalar(): newtag = FeedAuthors(article_id=itemrow.id, name=author['name'].strip()) db.session.add(newtag) db.session.flush() db.session.commit()
def update_series_info(item): # print("update_series_info", item) assert 'title' in item assert 'author' in item assert 'tags' in item assert 'desc' in item assert 'tl_type' in item item = text_tools.fix_dict(item, recase=False) print("Series info update message for '%s'!" % item['title']) if not 'update_only' in item: item['update_only'] = False if not 'alt_titles' in item: item['alt_titles'] = [] item['alt_titles'].append(item['title']) if item['update_only']: series = get_series_from_any(item['alt_titles'], item["tl_type"], item['author']) else: series = get_create_series( input_series_name = item['title'], tl_type = item["tl_type"], changeuser = RSS_USER_ID, author_name_list = item['author'] ) # Break if the tl type has changed, something is probably mismatched if series.tl_type != item['tl_type']: print("WARNING! TlType mismatch? Wat?") print("Series:", series) print("###################################") print(series.title) print("-----------------------------------") print(item['title']) print("###################################") print(series.author) print("-----------------------------------") print(item['author']) print("###################################") print(series.description) print("-----------------------------------") print(item['desc']) print("###################################") print(series.tl_type) print("-----------------------------------") print(item['tl_type']) print("###################################") print(series.tags) print("-----------------------------------") print(item['tags']) return changeable = generate_automated_only_change_flags(series) # print(changeable) # print(item) # { # 'license_en': True, # 'orig_lang': True, # 'pub_date': True, # 'title': True, # 'description': False, # 'chapter': True, # 'origin_loc': True, # 'website': True, # 'region': True, # 'tl_type': True, # 'tot_chapter': True, # 'orig_status': True, # 'sort_mode': True, # 'volume': True, # 'demographic': True, # 'tot_volume': True, # 'type': True # } # This only generates a change is it needs to, so we can call it unconditionally. if changeable['title']: series_tools.updateTitle(series, item['title']) if changeable['description'] and 'desc' in item and item['desc']: newd = bleach.clean(item['desc'], strip=True, tags = ['p', 'em', 'strong', 'b', 'i', 'a']) if newd != series.description and len(newd.strip()): series.description = newd elif ('desc' in item and item['desc'] and not series.description): newd = bleach.clean(item['desc'], strip=True, tags = ['p', 'em', 'strong', 'b', 'i', 'a']) if len(newd.strip()): series.description = newd if 'homepage' in item and item['homepage'] and ( not series.website or (bleach.clean(item['homepage']) != series.website and changeable['website']) ): neww = bleach.clean(item['homepage']) if len(neww.strip()): series.website = neww if 'coostate' in item and item['coostate']: if not series.orig_status or (item['coostate'] != series.orig_status and changeable['orig_status']): neww = bleach.clean(item['coostate']) if len(neww.strip()): series.orig_status = neww if 'tl_type' in item and item['tl_type']: if not series.tl_type or (item['tl_type'] != series.tl_type and changeable['tl_type']): neww = bleach.clean(item['tl_type']).strip() if neww and neww in ['western', 'eastern', 'unknown']: series.tl_type = neww # if 'transcomplete' in item and item['transcomplete']: # if not series.tl_complete or (item['transcomplete'] != series.tl_complete and changeable['transcomplete']): # neww = item['transcomplete'].strip() # if neww and neww in ['yes', 'no']: # series.tl_complete = neww if 'licensed' in item and item['licensed']: if not series.license_en or (item['licensed'] != series.license_en and changeable['license_en']): neww = item['licensed'].strip().lower() if neww and neww in ['yes', 'no']: series.license_en = neww == 'yes' if 'author' in item and item['author']: tmp = item['author'] if isinstance(tmp, str): tmp = [tmp, ] series_tools.setAuthorIllust(series, author=tmp, deleteother=False) if 'illust' in item and item['illust']: tmp = item['illust'] if isinstance(tmp, str): tmp = [tmp, ] series_tools.setAuthorIllust(series, illust=tmp, deleteother=False) if 'tags' in item and item['tags']: series_tools.updateTags(series, item['tags'], deleteother=False, allow_new=item.get('create_tags', False)) if 'genres' in item and item['genres']: series_tools.updateGenres(series, item['genres'], deleteother=False) if 'alt_titles' in item and item['alt_titles']: series_tools.updateAltNames(series, item['alt_titles']+[item['title'], ], deleteother=False) if 'pubnames' in item and item['pubnames']: series_tools.updatePublishers(series, item['pubnames'], deleteother=False) if 'pubdate' in item and item['pubdate']: if not series.pub_date: series.pub_date = datetime.datetime.utcfromtimestamp(item['pubdate']) if 'sourcesite' in item and item['sourcesite']: pass series.changeuser = RSS_USER_ID db.session.flush() db.session.commit()