def insert_parsed_release(item):
assert 'tl_type' in item
assert 'srcname' in item
assert 'series' in item
if "nu_release" in item:
update_id = NU_SRC_USER_ID
else:
update_id = RSS_USER_ID
item = text_tools.fix_dict(item, recase=False)
if item["tl_type"] not in ['oel', 'translated']:
raise ValueError("Invalid TL Type '%s'! Wat?" % item["tl_type"])
group = get_create_group(item['srcname'], update_id)
assert group is not None
if 'loose_match' in item and item['loose_match']:
series = get_create_series(item['series'], item["tl_type"], update_id)
check_insert_release(item, group, series, update_id, loose_match=True)
else:
if 'match_author' in item and item['match_author']:
series = get_create_series(item['series'], item["tl_type"],
update_id, item['author'])
else:
series = get_create_series(item['series'], item["tl_type"],
update_id)
check_insert_release(item, group, series, update_id)
def insert_parsed_release(item):
assert 'tl_type' in item
assert 'srcname' in item
assert 'series' in item
if "nu_release" in item:
update_id = NU_SRC_USER_ID
else:
update_id = RSS_USER_ID
item = text_tools.fix_dict(item, recase=False)
if item["tl_type"] not in ['oel', 'translated']:
raise ValueError("Invalid TL Type '%s'! Wat?" % item["tl_type"])
group = get_create_group(item['srcname'], update_id)
assert group is not None
kwargs = {
"input_series_name" : item['series'],
"tl_type" : item["tl_type"],
"changeuser" : update_id,
}
if 'match_author' in item and item['match_author']:
kwargs['author_name_list'] = item['author']
series = get_create_series(**kwargs)
prefix_match = item.get('prefix_match', False)
loose_match = item.get('loose_match', False)
check_insert_release(item, group, series, update_id, loose_match=loose_match, prefix_match=prefix_match)
def update_series_info(item):
# print("update_series_info", item)
assert 'title' in item
assert 'author' in item
assert 'tags' in item
assert 'desc' in item
assert 'tl_type' in item
item = text_tools.fix_dict(item, recase=False)
print("Series info update message for '%s'!" % item['title'])
if not 'update_only' in item:
item['update_only'] = False
if not 'alt_titles' in item:
item['alt_titles'] = []
item['alt_titles'].append(item['title'])
if item['update_only']:
series = get_series_from_any(item['alt_titles'], item["tl_type"],
item['author'])
else:
series = get_create_series(item['title'], item["tl_type"], RSS_USER_ID,
item['author'])
# Break if the tl type has changed, something is probably mismatched
if series.tl_type != item['tl_type']:
print("WARNING! TlType mismatch? Wat?")
print("Series:", series)
print("###################################")
print(series.title)
print("-----------------------------------")
print(item['title'])
print("###################################")
print(series.author)
print("-----------------------------------")
print(item['author'])
print("###################################")
print(series.description)
print("-----------------------------------")
print(item['desc'])
print("###################################")
print(series.tl_type)
print("-----------------------------------")
print(item['tl_type'])
print("###################################")
print(series.tags)
print("-----------------------------------")
print(item['tags'])
return
changeable = generate_automated_only_change_flags(series)
# print(changeable)
# print(item)
# {
# 'license_en': True,
# 'orig_lang': True,
# 'pub_date': True,
# 'title': True,
# 'description': False,
# 'chapter': True,
# 'origin_loc': True,
# 'website': True,
# 'region': True,
# 'tl_type': True,
# 'tot_chapter': True,
# 'orig_status': True,
# 'sort_mode': True,
# 'volume': True,
# 'demographic': True,
# 'tot_volume': True,
# 'type': True
# }
# This only generates a change is it needs to, so we can call it unconditionally.
if changeable['title']:
series_tools.updateTitle(series, item['title'])
if changeable['description'] and 'desc' in item and item['desc']:
newd = bleach.clean(item['desc'],
strip=True,
tags=['p', 'em', 'strong', 'b', 'i', 'a'])
if newd != series.description and len(newd.strip()):
series.description = newd
elif ('desc' in item and item['desc'] and not series.description):
newd = bleach.clean(item['desc'],
strip=True,
tags=['p', 'em', 'strong', 'b', 'i', 'a'])
if len(newd.strip()):
series.description = newd
if 'homepage' in item and item['homepage'] and (
not series.website or
(bleach.clean(item['homepage']) != series.website
and changeable['website'])):
neww = bleach.clean(item['homepage'])
if len(neww.strip()):
series.website = neww
if 'coostate' in item and item['coostate']:
if not series.orig_status or (item['coostate'] != series.orig_status
and changeable['orig_status']):
neww = bleach.clean(item['coostate'])
if len(neww.strip()):
series.orig_status = neww
if 'tl_type' in item and item['tl_type']:
if not series.tl_type or (item['tl_type'] != series.tl_type
and changeable['tl_type']):
neww = bleach.clean(item['tl_type']).strip()
if neww and neww in ['western', 'eastern', 'unknown']:
series.tl_type = neww
# if 'transcomplete' in item and item['transcomplete']:
# if not series.tl_complete or (item['transcomplete'] != series.tl_complete and changeable['transcomplete']):
# neww = item['transcomplete'].strip()
# if neww and neww in ['yes', 'no']:
# series.tl_complete = neww
if 'licensed' in item and item['licensed']:
if not series.license_en or (item['licensed'] != series.license_en
and changeable['license_en']):
neww = item['licensed'].strip().lower()
if neww and neww in ['yes', 'no']:
series.license_en = neww == 'yes'
if 'author' in item and item['author']:
tmp = item['author']
if isinstance(tmp, str):
tmp = [
tmp,
]
series_tools.setAuthorIllust(series, author=tmp, deleteother=False)
if 'illust' in item and item['illust']:
tmp = item['illust']
if isinstance(tmp, str):
tmp = [
tmp,
]
series_tools.setAuthorIllust(series, illust=tmp, deleteother=False)
if 'tags' in item and item['tags']:
series_tools.updateTags(series,
item['tags'],
deleteother=False,
allow_new=item.get('create_tags', False))
if 'genres' in item and item['genres']:
series_tools.updateGenres(series, item['genres'], deleteother=False)
if 'alt_titles' in item and item['alt_titles']:
series_tools.updateAltNames(series,
item['alt_titles'] + [
item['title'],
],
deleteother=False)
if 'pubnames' in item and item['pubnames']:
series_tools.updatePublishers(series,
item['pubnames'],
deleteother=False)
if 'pubdate' in item and item['pubdate']:
if not series.pub_date:
series.pub_date = datetime.datetime.utcfromtimestamp(
item['pubdate'])
if 'sourcesite' in item and item['sourcesite']:
pass
series.changeuser = RSS_USER_ID
db.session.flush()
db.session.commit()
def insert_raw_item(item):
'''
insert item `item` into the feed database.
fields in `item`:
"title"
"contents"
"guid"
"linkUrl"
"feedtype"
"published"
"updated"
"authors"
-> "href"
-> "name"
"tags"
'''
# print(item)
if not 'srcname' in item:
print("No srcname? Old item?")
return
entry = {}
entry['title'] = item.pop('title')
entry['contents'] = item.pop('contents', 'N/A')
if not isinstance(entry['contents'], str):
if isinstance(entry['contents'], list):
entry['contents'] = entry['contents'].pop()
if 'value' in entry['contents']:
entry['contents'] = entry['contents']['value']
else:
print(entry['contents'])
entry['contents'] = str(entry['contents'])
entry['guid'] = item.pop('guid')
entry['linkurl'] = item.pop('linkUrl')
entry['region'] = item.pop('feedtype')
entry['srcname'] = item.pop('srcname')
entry['published'] = datetime.datetime.fromtimestamp(item.pop('published'))
if 'updated' in item:
entry['updated'] = datetime.datetime.fromtimestamp(item.pop('updated'))
item = text_tools.fix_dict(item, recase=False)
itemrows = Feeds.query.filter(Feeds.guid == entry['guid']).all()
if not itemrows:
print("New feed item: ", entry['guid'])
itemrow = Feeds(**entry)
db.session.add(itemrow)
db.session.flush()
else:
itemrow = itemrows[0]
for tag in item.pop('tags'):
if not FeedTags.query \
.filter(FeedTags.article_id==itemrow.id) \
.filter(FeedTags.tag == tag.strip()).scalar():
newtag = FeedTags(article_id=itemrow.id, tag=tag.strip())
db.session.add(newtag)
db.session.flush()
for author in item.pop('authors'):
if not 'name' in author:
continue
if not FeedAuthors.query \
.filter(FeedAuthors.article_id==itemrow.id) \
.filter(FeedAuthors.name == author['name'].strip()).scalar():
newtag = FeedAuthors(article_id=itemrow.id,
name=author['name'].strip())
db.session.add(newtag)
db.session.flush()
db.session.commit()
def update_series_info(item):
# print("update_series_info", item)
assert 'title' in item
assert 'author' in item
assert 'tags' in item
assert 'desc' in item
assert 'tl_type' in item
item = text_tools.fix_dict(item, recase=False)
print("Series info update message for '%s'!" % item['title'])
if not 'update_only' in item:
item['update_only'] = False
if not 'alt_titles' in item:
item['alt_titles'] = []
item['alt_titles'].append(item['title'])
if item['update_only']:
series = get_series_from_any(item['alt_titles'], item["tl_type"], item['author'])
else:
series = get_create_series(
input_series_name = item['title'],
tl_type = item["tl_type"],
changeuser = RSS_USER_ID,
author_name_list = item['author']
)
# Break if the tl type has changed, something is probably mismatched
if series.tl_type != item['tl_type']:
print("WARNING! TlType mismatch? Wat?")
print("Series:", series)
print("###################################")
print(series.title)
print("-----------------------------------")
print(item['title'])
print("###################################")
print(series.author)
print("-----------------------------------")
print(item['author'])
print("###################################")
print(series.description)
print("-----------------------------------")
print(item['desc'])
print("###################################")
print(series.tl_type)
print("-----------------------------------")
print(item['tl_type'])
print("###################################")
print(series.tags)
print("-----------------------------------")
print(item['tags'])
return
changeable = generate_automated_only_change_flags(series)
# print(changeable)
# print(item)
# {
# 'license_en': True,
# 'orig_lang': True,
# 'pub_date': True,
# 'title': True,
# 'description': False,
# 'chapter': True,
# 'origin_loc': True,
# 'website': True,
# 'region': True,
# 'tl_type': True,
# 'tot_chapter': True,
# 'orig_status': True,
# 'sort_mode': True,
# 'volume': True,
# 'demographic': True,
# 'tot_volume': True,
# 'type': True
# }
# This only generates a change is it needs to, so we can call it unconditionally.
if changeable['title']:
series_tools.updateTitle(series, item['title'])
if changeable['description'] and 'desc' in item and item['desc']:
newd = bleach.clean(item['desc'], strip=True, tags = ['p', 'em', 'strong', 'b', 'i', 'a'])
if newd != series.description and len(newd.strip()):
series.description = newd
elif ('desc' in item and item['desc'] and not series.description):
newd = bleach.clean(item['desc'], strip=True, tags = ['p', 'em', 'strong', 'b', 'i', 'a'])
if len(newd.strip()):
series.description = newd
if 'homepage' in item and item['homepage'] and (
not series.website or
(bleach.clean(item['homepage']) != series.website and changeable['website'])
):
neww = bleach.clean(item['homepage'])
if len(neww.strip()):
series.website = neww
if 'coostate' in item and item['coostate']:
if not series.orig_status or (item['coostate'] != series.orig_status and changeable['orig_status']):
neww = bleach.clean(item['coostate'])
if len(neww.strip()):
series.orig_status = neww
if 'tl_type' in item and item['tl_type']:
if not series.tl_type or (item['tl_type'] != series.tl_type and changeable['tl_type']):
neww = bleach.clean(item['tl_type']).strip()
if neww and neww in ['western', 'eastern', 'unknown']:
series.tl_type = neww
# if 'transcomplete' in item and item['transcomplete']:
# if not series.tl_complete or (item['transcomplete'] != series.tl_complete and changeable['transcomplete']):
# neww = item['transcomplete'].strip()
# if neww and neww in ['yes', 'no']:
# series.tl_complete = neww
if 'licensed' in item and item['licensed']:
if not series.license_en or (item['licensed'] != series.license_en and changeable['license_en']):
neww = item['licensed'].strip().lower()
if neww and neww in ['yes', 'no']:
series.license_en = neww == 'yes'
if 'author' in item and item['author']:
tmp = item['author']
if isinstance(tmp, str):
tmp = [tmp, ]
series_tools.setAuthorIllust(series, author=tmp, deleteother=False)
if 'illust' in item and item['illust']:
tmp = item['illust']
if isinstance(tmp, str):
tmp = [tmp, ]
series_tools.setAuthorIllust(series, illust=tmp, deleteother=False)
if 'tags' in item and item['tags']:
series_tools.updateTags(series, item['tags'], deleteother=False, allow_new=item.get('create_tags', False))
if 'genres' in item and item['genres']:
series_tools.updateGenres(series, item['genres'], deleteother=False)
if 'alt_titles' in item and item['alt_titles']:
series_tools.updateAltNames(series, item['alt_titles']+[item['title'], ], deleteother=False)
if 'pubnames' in item and item['pubnames']:
series_tools.updatePublishers(series, item['pubnames'], deleteother=False)
if 'pubdate' in item and item['pubdate']:
if not series.pub_date:
series.pub_date = datetime.datetime.utcfromtimestamp(item['pubdate'])
if 'sourcesite' in item and item['sourcesite']:
pass
series.changeuser = RSS_USER_ID
db.session.flush()
db.session.commit()
def insert_raw_item(item):
'''
insert item `item` into the feed database.
fields in `item`:
"title"
"contents"
"guid"
"linkUrl"
"feedtype"
"published"
"updated"
"authors"
-> "href"
-> "name"
"tags"
'''
# print(item)
if not 'srcname' in item:
print("No srcname? Old item?")
return
entry = {}
entry['title'] = item.pop('title')
entry['contents'] = item.pop('contents', 'N/A')
if not isinstance(entry['contents'], str):
if isinstance(entry['contents'], list):
entry['contents'] = entry['contents'].pop()
if 'value' in entry['contents']:
entry['contents'] = entry['contents']['value']
else:
print(entry['contents'])
entry['contents'] = str(entry['contents'])
entry['guid'] = item.pop('guid')
entry['linkurl'] = item.pop('linkUrl')
entry['region'] = item.pop('feedtype')
entry['srcname'] = item.pop('srcname')
entry['published'] = datetime.datetime.fromtimestamp(item.pop('published'))
if 'updated' in item:
entry['updated'] = datetime.datetime.fromtimestamp(item.pop('updated'))
item = text_tools.fix_dict(item, recase=False)
itemrows = Feeds.query.filter(Feeds.guid == entry['guid']).all()
if not itemrows:
print("New feed item: ", entry['guid'])
itemrow = Feeds(**entry)
db.session.add(itemrow)
db.session.flush()
else:
itemrow = itemrows[0]
for tag in item.pop('tags'):
if not FeedTags.query \
.filter(FeedTags.article_id==itemrow.id) \
.filter(FeedTags.tag == tag.strip()).scalar():
newtag = FeedTags(article_id=itemrow.id, tag=tag.strip())
db.session.add(newtag)
db.session.flush()
for author in item.pop('authors'):
if not 'name' in author:
continue
if not FeedAuthors.query \
.filter(FeedAuthors.article_id==itemrow.id) \
.filter(FeedAuthors.name == author['name'].strip()).scalar():
newtag = FeedAuthors(article_id=itemrow.id, name=author['name'].strip())
db.session.add(newtag)
db.session.flush()
db.session.commit()