def import_tags(self):
self.stdout.write(self.style.MIGRATE_LABEL("Importing Tags"))
progress = ProgressBar(widgets=[Percentage(), Bar()],
maxval=len(self.tags)).start()
for i, tag in enumerate(self.tags):
objTag = Tag(id=tag.get('id'),
name=tag.get('name'),
slug=tag.get('slug'))
objTag.save()
progress.update(i)
progress.finish()
def create_thread(request):
tuple = get_tags(request.POST['title'])
# Create and save new thread
t = Thread(author=request.POST['author'], title=tuple[0], professor_viewable=(request.POST.has_key('professor_viewable')))
t.save()
for ttag in tuple[1]:
tag = Tag(tag = ttag)
tag.save()
t.tags.add(tag)
t.save()
# Create and save new post
p = Post(author=request.POST['author'], content=request.POST['content'], thread=t)
p.save()
return HttpResponseRedirect(reverse('posts.views.index'))
def add_post():
form = PostForm(request.form)
if form.validate():
post = Post(bleach.clean(form.content.data), current_user.id, datetime.datetime.utcnow())
words = form.content.data.split()
for tag in Tag.parse_tags(words):
db_session.add(tag)
post.tags.append(tag)
db_session.add(post)
db_session.commit()
flash("Successful!")
else:
flash("Something went wrong. Make sure that your tweet's length is less than 140 characters", 'error')
return redirect(url_for('main.hello'))
def newtag(request, post_id):
if request.method == 'POST':
if request.POST['body']:
tag = Tag()
tag.body = request.POST['body']
tag.post = get_object_or_404(Post, id=post_id)
tag.author = request.user
tag.save()
return redirect(reverse('posts:postdetails', args=[post_id]))
else:
return redirect(reverse('posts:postdetails', args=[post_id]))
def p_detail(request, post_id):
tmp = Post.objects.get(id=post_id)
if request.method == 'POST':
# foreign key에 대한 정보가 없어서 에러가 발생했던 것이다
# tag 객체를 만들어서 foreign 키인 post로 찾은 post 객체를 전달해주고
# 그 객체를 TagForm의 instance로 추가시켜서 tag_form을 완성하면 에러가 나지 않는다
tag = Tag(post=tmp)
tag_form = TagForm(request.POST, instance=tag)
if tag_form.is_valid():
tag_form.save()
return redirect('posts:detail', post_id)
else:
tag_form = TagForm()
context = {
'post': tmp,
'tag_form': tag_form
}
return render(request, 'detail.html', context)
def setUp(self):
self.tag_a = Tag(name='a')
self.tag_b = Tag(name='b')
self.tag_c = Tag(name='c')
self.tag_d = Tag(name='d')
self.tag_e = Tag(name='e')
self.tag_a.save()
self.tag_b.save()
self.tag_c.save()
self.tag_d.save()
self.tag_e.save()
self.category_1 = Category(name='cat 1')
self.category_2 = Category(name='cat 2')
self.category_3 = Category(name='cat 3')
self.category_1.save()
self.category_2.save()
self.category_3.save()
self.post_1 = Post(pk=1, title='Markdown Syntax', slug='markdown-syntax',
content="Here's some markdown syntax.",
published=True)
self.post_2 = Post(pk=2, title='First Post', slug='first-post',
content='This is my first post.',
published=True)
self.post_3 = Post(pk=3, title='Django Unit Testing', slug='django-unit-testing',
content="To test django with unit testing, do this...",
published=True)
self.post_4 = Post(pk=4, title='Depressed', slug='depressed',
content="I'm depressed...",
published=False)
# this determines time_cr
self.post_3.save()
self.post_1.save()
self.post_2.save()
self.post_4.save()
self.post_1.tags.add(self.tag_a)
self.post_1.tags.add(self.tag_b)
self.post_1.tags.add(self.tag_c)
self.post_1.categories.add(self.category_1)
self.post_1.categories.add(self.category_2)
self.post_2.tags.add(self.tag_b)
self.post_2.tags.add(self.tag_d)
self.post_2.tags.add(self.tag_e)
self.post_2.categories.add(self.category_2)
self.post_2.categories.add(self.category_3)
self.post_3.tags.add(self.tag_b)
self.post_3.tags.add(self.tag_d)
self.post_3.tags.add(self.tag_e)
self.post_4.categories.add(self.category_3)
# this determines time_lu
self.post_4.save()
self.post_1.save()
self.post_3.save()
self.post_2.save()
self.client = Client()
class view_post_tests(TestCase):
def setUp(self):
self.tag_a = Tag(name='a')
self.tag_b = Tag(name='b')
self.tag_c = Tag(name='c')
self.tag_d = Tag(name='d')
self.tag_e = Tag(name='e')
self.tag_a.save()
self.tag_b.save()
self.tag_c.save()
self.tag_d.save()
self.tag_e.save()
self.category_1 = Category(name='cat 1')
self.category_2 = Category(name='cat 2')
self.category_3 = Category(name='cat 3')
self.category_1.save()
self.category_2.save()
self.category_3.save()
self.post_1 = Post(pk=1, title='Markdown Syntax', slug='markdown-syntax',
content="Here's some markdown syntax.",
published=True)
self.post_2 = Post(pk=2, title='First Post', slug='first-post',
content='This is my first post.',
published=True)
self.post_3 = Post(pk=3, title='Django Unit Testing', slug='django-unit-testing',
content="To test django with unit testing, do this...",
published=True)
self.post_4 = Post(pk=4, title='Depressed', slug='depressed',
content="I'm depressed...",
published=False)
# this determines time_cr
self.post_3.save()
self.post_1.save()
self.post_2.save()
self.post_4.save()
self.post_1.tags.add(self.tag_a)
self.post_1.tags.add(self.tag_b)
self.post_1.tags.add(self.tag_c)
self.post_1.categories.add(self.category_1)
self.post_1.categories.add(self.category_2)
self.post_2.tags.add(self.tag_b)
self.post_2.tags.add(self.tag_d)
self.post_2.tags.add(self.tag_e)
self.post_2.categories.add(self.category_2)
self.post_2.categories.add(self.category_3)
self.post_3.tags.add(self.tag_b)
self.post_3.tags.add(self.tag_d)
self.post_3.tags.add(self.tag_e)
self.post_4.categories.add(self.category_3)
# this determines time_lu
self.post_4.save()
self.post_1.save()
self.post_3.save()
self.post_2.save()
self.client = Client()
def test_view_single_posts_by_id(self):
# public post
response = self.client.get('/posts/2/')
self.assertIn('This is my first post', response.content)
# private post
response = self.client.get('/posts/4/')
self.assertIn("The post you're looking for does not exist. ", response.content)
# non-existent post
response = self.client.get('/posts/100/')
self.assertIn("The post you're looking for does not exist. ", response.content)
def test_view_single_posts_by_slug(self):
# public post
response = self.client.get('/posts/first-post/')
self.assertIn('This is my first post', response.content)
# private post
response = self.client.get('/posts/depressed/')
self.assertIn("The post you're looking for does not exist. ", response.content)
# non-existent post
response = self.client.get('/posts/not-exist/')
self.assertIn("The post you're looking for does not exist. ", response.content)
def test_view_single_posts_by_date_slug(self):
pass
# # public post
# response = self.client.get('/posts/2014/04/20/first-post/')
# self.assertIn('This is my first post', response.content)
# # private post
# response = self.client.get('/posts/????/')
# self.assertIn("The post you're looking for does not exist. ", response.content)
# # non-existent post: slug does not exist
# response = self.client.get('/posts/????/')
# self.assertIn("The post you're looking for does not exist. ", response.content)
# # non-existent post: date/slug mismatch
# response = self.client.get('/posts/????/')
# self.assertIn("The post you're looking for does not exist. ", response.content)
def test_view_all_posts(self):
'''
private posts should not show
public posts are listed in chronolgical order of creation time
'''
# redirects properly
response = self.client.get('/posts/')
self.assertRedirects(response, '/')
# posts appear in correct order
response = self.client.get('/')
post_1_position = response.content.find("Here's some markdown syntax.")
post_2_position = response.content.find('This is my first post.')
post_3_position = response.content.find("To test django with unit testing")
self.assertGreater(post_1_position, 0)
self.assertGreater(post_2_position, 0)
self.assertGreater(post_3_position, 0)
self.assertLess(post_2_position, post_1_position)
self.assertLess(post_1_position, post_3_position)
# private post doesn't appear
post_4_position = response.content.find("I'm depressed")
self.assertEqual(post_4_position, -1)
def test_get_absolute_url(self):
# time below is already Apr 30 in UTC time (local time is Central Time, UTC-6/5s)
self.post_2.time_cr = datetime(2014, 4, 29, 20, 13, 19, 0, get_default_timezone())
the_absolute_url = self.post_2.get_absolute_url()
# make sure permalink doesn't roll over to Apr 30
self.assertEqual(the_absolute_url, '/posts/2014/04/29/first-post/')
def test_add_tag(self):
t = Tag(name='c++')
t.save()
j = Tag.objects.get(name='c++')
self.assertEqual(j.name, 'c++')
def handle(self, *args, **options):
with open("ingredients.json", "r", encoding="utf-8") as fh:
data = json.load(fh)
for i in data:
dimension = i["dimension"]
title = i["title"]
if not Ingredients.objects.filter(title=title).first():
if dimension in [
"по вкусу",
"стакан",
"кусок",
"горсть",
"банка",
"тушка",
"пакет",
]:
dimension = "г"
ingredient = Ingredients(title=title, dimension=dimension)
ingredient.save()
if not Tag.objects.filter(name="Завтрак").first():
tag_breakfast = Tag(name="Завтрак", slug="breakfast", color="orange")
tag_breakfast.save()
if not Tag.objects.filter(name="Обед").first():
tag_lunch = Tag(name="Обед", slug="lunch", color="green")
tag_lunch.save()
if not Tag.objects.filter(name="Ужин").first():
tag_dinner = Tag(name="Ужин", slug="dinner", color="purple")
tag_dinner.save()
def handle(self, *args, **options):
ua = '[adjective][species] (explore621) {}'.format(options['username'])
started = datetime.now()
started = started.replace(tzinfo=pytz.UTC)
ingested = 0
total_new = 0
total_updated = 0
last_id = 0
tags_added = {}
sources_added = {}
artists_added = {}
self.stdout.write('Refreshing data from e621 starting at {}'.format(
started.isoformat()))
self.stdout.write(
'Fetching {} pages worth of posts at {} per page'.format(
options['pages'], options['per_page']))
for i in range(1, options['pages'] + 1):
time.sleep(1)
self.stdout.write('--- Fetching page {}'.format(i))
r = requests.get(
'https://e621.net/post/index.json',
{'limit': options['per_page'], 'page': i},
headers={'user-agent': ua})
if r.status_code != 200:
self.stdout.write(
self.style.NOTICE(' got {} on page {}, skipping'.format(
r.status_code, i)))
continue
tags_added = {}
sources_added = {}
artists_added = {}
updated = 0
new = 0
for record in r.json():
if record['id'] > last_id:
last_id = record['id']
created_at = pytz.utc.localize(
datetime.fromtimestamp(record['created_at']['s']))
try:
post = Post.objects.get(source_id=record['id'])
post.source_id = record['id']
post.description = record['description']
post.created_at = created_at
post.creator_id = record['creator_id']
post.author = record['author']
post.change = record['change']
post.source = record['source']
post.score = record['score']
post.fav_count = record['fav_count']
post.md5 = record['md5']
post.file_size = record['file_size']
post.file_url = record['file_url']
post.width = record['width']
post.height = record['height']
post.file_ext = record['file_ext']
post.preview_url = record['preview_url']
post.preview_width = record['preview_width']
post.preview_height = record['preview_height']
post.sample_url = record['sample_url']
post.sample_width = record['sample_width']
post.sample_height = record['sample_height']
post.rating = record['rating']
post.status = record['status']
post.has_comments = record['has_comments']
post.has_notes = record['has_notes']
post.has_children = record['has_children']
post.children = record['children']
post.parent_id = record['parent_id']
updated += 1
except Post.DoesNotExist:
post = Post(
source_id=record['id'],
description=record['description'],
created_at=created_at,
creator_id=record['creator_id'],
author=record['author'],
change=record['change'],
source=record['source'],
score=record['score'],
fav_count=record['fav_count'],
md5=record['md5'],
file_size=record['file_size'],
file_url=record['file_url'],
width=record['width'],
height=record['height'],
file_ext=record['file_ext'],
preview_url=record['preview_url'],
preview_width=record['preview_width'],
preview_height=record['preview_height'],
sample_url=record['sample_url'],
sample_width=record['sample_width'],
sample_height=record['sample_height'],
rating=record['rating'],
status=record['status'],
has_comments=record['has_comments'],
has_notes=record['has_notes'],
has_children=record['has_children'],
children=record['children'],
parent_id=record['parent_id'])
new += 1
record_tags = record['tags'].split(' ')
tags = []
for record_tag in record_tags:
if record_tag in tags_added:
tag = tags_added[record_tag]
else:
try:
tag = Tag.objects.get(tag=record_tag)
except Tag.DoesNotExist:
tag = Tag(tag=record_tag)
tag.save()
tags_added[record_tag] = tag
tags.append(tag)
artists = []
for record_artist in record.get('artist', []):
if record_artist in artists_added:
artist = artists_added[record_artist]
else:
try:
artist = Artist.objects.get(name=record_artist)
except Artist.DoesNotExist:
artist = Artist(name=record_artist)
artist.save()
artists_added[record_artist] = artist
artists.append(artist)
sources = []
for record_source in record.get('sources', []):
if record_source in sources_added:
source = sources_added[record_source]
else:
try:
source = Source.objects.get(url=record_source)
except Source.DoesNotExist:
source = Source(url=record_source)
source.save()
sources_added[record_source] = source
sources.append(source)
try:
post.save()
except IntegrityError:
continue
post.artists.set(artists)
post.sources.set(sources)
post.tags.set(tags)
ingested += 1
total_new += new
total_updated += updated
self.stdout.write(
self.style.SUCCESS(
' processed page {}; {} new, {} updated'.format(
i, new, updated)))
self.stdout.write(
self.style.SUCCESS('{} posts ingested ({} new - {} updated)'.format(
ingested, total_new, total_updated)))
tags_fixed = 0
fixed_tags = []
self.stdout.write('Fixing typeless tags')
for tag in Tag.objects.filter(tag_type=-1):
self.stdout.write('--- Fixing tag {}'.format(tag.tag))
r = requests.get(
'https://e621.net/tag/show.json',
params={'name': tag.tag},
headers={'user-agent': '[adjective][species]'})
if 'type' not in r.json():
self.stdout.write(
self.style.NOTICE(' not fixing {}'.format(tag.tag)))
continue
tag.tag_type = r.json()['type']
tag.save()
tags_fixed += 1
fixed_tags.append(tag.tag)
self.stdout.write(
self.style.SUCCESS(' fixed {} ({})'.format(
tag.tag, tag.get_tag_type_display())))
time.sleep(0.7)
self.stdout.write(
self.style.SUCCESS('{} tags fixed'.format(tags_fixed)))
empty = Tag.objects.annotate(Count('post')).filter(post__count=0)
tags_deleted = 0
deleted_tags = []
self.stdout.write('Deleting empty tags')
for tag in empty:
self.stdout.write(
self.style.NOTICE('--- deleting {}'.format(tag.tag)))
deleted_tags.append(tag.tag)
tag.delete()
tags_deleted += 1
self.stdout.write(
self.style.SUCCESS('{} empty tags deleted'.format(tags_deleted)))
self.stdout.write('Deleting empty sources')
sources_deleted = len([
s.delete() for s in
Source.objects.annotate(Count('post')).filter(post__count=0)])
self.stdout.write(
self.style.NOTICE('--- {} sources deleted'.format(
sources_deleted)))
self.stdout.write('Deleting empty artists')
artists_deleted = len([
a.delete() for a in
Artist.objects.annotate(Count('post')).filter(post__count=0)])
self.stdout.write(
self.style.NOTICE('--- {} artists deleted'.format(
artists_deleted)))
log = IngestLog(
started=started,
records_ingested=ingested,
new=total_new,
updated=total_updated,
last_id=last_id,
fixed_tags=' '.join(fixed_tags),
deleted_tags=' '.join(deleted_tags),
sources_deleted=sources_deleted,
artists_deleted=artists_deleted)
log.save()
self.stdout.write(
self.style.SUCCESS('Finished refreshing in {}'.format(
str(log.finished - log.started))))
def handle(self, *args, **options):
self.stdout.write('Importing WordPress posts')
wp_table_prefix = options['wp-table-prefix']
utc = pytz.utc
my_conv = {
FIELD_TYPE.LONG: int,
FIELD_TYPE.INT24: int,
FIELD_TYPE.SHORT: int,
FIELD_TYPE.TINY: int,
FIELD_TYPE.LONGLONG: int,
FIELD_TYPE.DATETIME: str,
}
try:
db = MySQLdb.connect(host=options['host'],
user=options['user'],
passwd=options['passwd'],
db=options['db'],
conv=my_conv,
use_unicode=True,
charset='utf8mb4')
except Exception as e:
self.stderr.write('Unable to connect to database')
self.stderr.write(str(e))
return
cur = db.cursor()
cur.execute('SET NAMES utf8mb4')
cur.execute("SET CHARACTER SET utf8mb4")
cur.execute("SET character_set_connection=utf8mb4")
wpTagMap = dict()
wpCategoryMap = dict()
posts = list()
# GET TAGS
self.stdout.write('Migrating Tags')
cur.execute(f"""SELECT COUNT(*)
FROM {wp_table_prefix}_terms AS terms
INNER JOIN {wp_table_prefix}_term_taxonomy AS taxonomy
ON terms.term_id = taxonomy.term_id
INNER JOIN {wp_table_prefix}_term_relationships AS relationships
ON relationships.`term_taxonomy_id` = taxonomy.`term_taxonomy_id`
INNER JOIN {wp_table_prefix}_posts AS posts
ON relationships.`object_id` = posts.`ID`
WHERE taxonomy.taxonomy = 'post_tag'
AND posts.post_type = 'post'""")
row = cur.fetchone()
self.stdout.write(f"{row[0]} tags to migrate")
cur.execute(f"""SELECT
terms.term_id,
terms.name,
terms.slug,
posts.ID
FROM {wp_table_prefix}_terms AS terms
INNER JOIN {wp_table_prefix}_term_taxonomy AS taxonomy
ON terms.term_id = taxonomy.term_id
INNER JOIN {wp_table_prefix}_term_relationships AS relationships
ON relationships.`term_taxonomy_id` = taxonomy.`term_taxonomy_id`
INNER JOIN {wp_table_prefix}_posts AS posts
ON relationships.`object_id` = posts.`ID`
WHERE taxonomy.taxonomy = 'post_tag'
AND posts.post_type = 'post'""")
rows = cur.fetchall()
savedTags = dict()
for row in rows:
tag = Tag(name=row[1], slug=row[2])
if row[0] in savedTags:
tag = savedTags[row[0]]
else:
tag.save()
savedTags[row[0]] = tag
if row[3] not in wpTagMap:
wpTagMap[row[3]] = list()
wpTagMap[row[3]].append(tag)
self.stdout.write(f"{len(wpTagMap)} tags migrated")
# GET CATEGORIES
self.stdout.write('Migrating Categories')
cur.execute(f"""SELECT COUNT(*)
FROM {wp_table_prefix}_terms AS terms
INNER JOIN {wp_table_prefix}_term_taxonomy AS taxonomy
ON terms.term_id = taxonomy.term_id
INNER JOIN {wp_table_prefix}_term_relationships AS relationships
ON relationships.`term_taxonomy_id` = taxonomy.`term_taxonomy_id`
INNER JOIN {wp_table_prefix}_posts AS posts
ON relationships.`object_id` = posts.`ID`
WHERE taxonomy.taxonomy = 'category'
AND posts.post_type = 'post'""")
row = cur.fetchone()
self.stdout.write(f"{row[0]} categories to migrate")
cur.execute(f"""SELECT
terms.term_id,
terms.name,
terms.slug,
posts.ID
FROM {wp_table_prefix}_terms AS terms
INNER JOIN {wp_table_prefix}_term_taxonomy AS taxonomy
ON terms.term_id = taxonomy.term_id
INNER JOIN {wp_table_prefix}_term_relationships AS relationships
ON relationships.`term_taxonomy_id` = taxonomy.`term_taxonomy_id`
INNER JOIN {wp_table_prefix}_posts AS posts
ON relationships.`object_id` = posts.`ID`
WHERE taxonomy.taxonomy = 'category'
AND posts.post_type = 'post'""")
rows = cur.fetchall()
savedCategories = dict()
for row in rows:
category = Category(name=row[1], slug=row[2])
if row[0] in savedCategories:
category = savedCategories[row[0]]
else:
category.save()
savedCategories[row[0]] = category
wpCategoryMap[row[3]] = category
self.stdout.write(f"{len(wpCategoryMap)} categories migrated")
# POSTS
self.stdout.write('Migrating Posts')
cur.execute(
f"SELECT COUNT(*) FROM {wp_table_prefix}_posts WHERE post_type = 'post'"
)
row = cur.fetchone()
self.stdout.write(f"{row[0]} posts to migrate")
# we need a default Author
author = Profile.objects.all()[:1].get()
self.stdout.write(f'Default author is {author.name}')
cur.execute(
f"SELECT ID, post_date_gmt, post_content, post_title, post_excerpt, post_status, post_name, post_modified_gmt FROM {wp_table_prefix}_posts WHERE post_type = 'post'"
)
rows = cur.fetchall()
for row in rows:
if not row[6]:
self.stdout.write(f'{row[3]} has no slug. Skipping.')
continue
published = row[1]
if published == '0000-00-00 00:00:00':
published = None
else:
published = utc.localize(datetime.datetime.strptime(
published, "%Y-%m-%d %H:%M:%S"),
is_dst=None)
updated = row[7]
if updated == '0000-00-00 00:00:00':
updated = None
else:
updated = utc.localize(datetime.datetime.strptime(
updated, "%Y-%m-%d %H:%M:%S"),
is_dst=None)
post = Post(published=published,
content=row[2],
title=row[3],
summary=row[4],
is_published=row[5] == 'publish',
slug=row[6],
updated=updated,
author=author)
try:
post.save()
except Exception as e:
self.stderr.write(str(e))
self.stderr.write(post.name)
raise e
posts.append({'post': post, 'wpId': row[0]})
self.stdout.write(f"{len(posts)} posts migrated")
# Attaching Tags and Categories
self.stdout.write("Attaching tags and categories")
for post in posts:
if post['wpId'] in wpCategoryMap:
post['post'].category = wpCategoryMap[post['wpId']]
else:
self.stdout.write(f"No category for {post['post'].title}")
if post['wpId'] in wpTagMap:
post['post'].tags.set(wpTagMap[post['wpId']])
else:
self.stdout.write(f"No tags for {post['post'].title}")
post['post'].save()
self.stdout.write("Tags and categories attached")
cur.close()
db.close()
self.stdout.write('WordPress posts imported')
return
def handle(self, *args, **options):
ua = '[adjective][species] (explore621) {}'.format(options['username'])
started = datetime.now()
self.stdout.write(
'Performing a full refresh from e621 starting at {}'.format(
started.isoformat()))
max_id = Post.objects.aggregate(mid=Max('source_id'))['mid']
min_id = Post.objects.aggregate(mid=Min('source_id'))['mid']
total_added = 0
while max_id > min_id:
time.sleep(1)
r = requests.get('https://e621.net/post/index.json', {
'limit': 320,
'before_id': max_id
},
headers={'user-agent': ua})
if r.status_code != 200:
self.stdout.write(
self.style.NOTICE(
' got {} on before_id {}, skipping'.format(
r.status_code, max_id)))
continue
skipped = 0
added = 0
for record in r.json():
max_id = record['id']
created_at = pytz.utc.localize(
datetime.fromtimestamp(record['created_at']['s']))
if Post.objects.filter(source_id=record['id']).count() != 0:
skipped += 1
continue
post = Post(source_id=record['id'],
description=record['description'],
created_at=created_at,
creator_id=record['creator_id'],
author=record['author'],
change=record['change'],
source=record['source'],
score=record['score'],
fav_count=record['fav_count'],
md5=record['md5'],
file_size=record['file_size'],
file_url=record['file_url'],
width=record['width'],
height=record['height'],
file_ext=record['file_ext'],
preview_url=record['preview_url'],
preview_width=record['preview_width'],
preview_height=record['preview_height'],
sample_url=record['sample_url'],
sample_width=record['sample_width'],
sample_height=record['sample_height'],
rating=record['rating'],
status=record['status'],
has_comments=record['has_comments'],
has_notes=record['has_notes'],
has_children=record['has_children'],
children=record['children'],
parent_id=record['parent_id'])
record_tags = record['tags'].split(' ')
tags = []
for record_tag in record_tags:
try:
tag = Tag.objects.get(tag=record_tag)
except Tag.DoesNotExist:
tag = Tag(tag=record_tag)
tag.save()
tags.append(tag)
artists = []
for record_artist in record.get('artist', []):
try:
artist = Artist.objects.get(name=record_artist)
except Artist.DoesNotExist:
artist = Artist(name=record_artist)
artist.save()
artists.append(artist)
sources = []
for record_source in record.get('sources', []):
try:
source = Source.objects.get(url=record_source)
except Source.DoesNotExist:
source = Source(url=record_source)
source.save()
sources.append(source)
try:
post.save()
except IntegrityError:
continue
post.artists.set(artists)
post.sources.set(sources)
post.tags.set(tags)
added += 1
self.stdout.write(
self.style.SUCCESS(
'Ending at {}: {} added / {} skipped'.format(
max_id, added, skipped)))
total_added += added
self.stdout.write('Fixing typeless tags')
for tag in Tag.objects.filter(tag_type=-1):
self.stdout.write('--- Fixing tag {}'.format(tag.tag))
r = requests.get('https://e621.net/tag/show.json',
params={'name': tag.tag},
headers={'user-agent': '[adjective][species]'})
if 'type' not in r.json():
self.stdout.write(
self.style.NOTICE(' not fixing {}'.format(tag.tag)))
continue
tag.tag_type = r.json()['type']
tag.save()
tags_fixed += 1
fixed_tags.append(tag.tag)
self.stdout.write(
self.style.SUCCESS(' fixed {} ({})'.format(
tag.tag, tag.get_tag_type_display())))
time.sleep(0.7)
self.stdout.write(
self.style.SUCCESS('{} tags fixed'.format(tags_fixed)))
self.stdout.write(
self.style.SUCCESS('Finished full refresh in {}'.format(
str(datetime.now() - started))))
