コード例 #1
0
 def import_tags(self):
     self.stdout.write(self.style.MIGRATE_LABEL("Importing Tags"))
     progress = ProgressBar(widgets=[Percentage(), Bar()],
                            maxval=len(self.tags)).start()
     for i, tag in enumerate(self.tags):
         objTag = Tag(id=tag.get('id'),
                      name=tag.get('name'),
                      slug=tag.get('slug'))
         objTag.save()
         progress.update(i)
     progress.finish()
コード例 #2
0
ファイル: views.py プロジェクト: robertsami/COS333
def create_thread(request):
	tuple = get_tags(request.POST['title'])
	# Create and save new thread
	t = Thread(author=request.POST['author'], title=tuple[0], professor_viewable=(request.POST.has_key('professor_viewable')))
	t.save()
	for ttag in tuple[1]:
		tag = Tag(tag = ttag)
		tag.save()
		t.tags.add(tag)
	t.save()
	# Create and save new post
	p = Post(author=request.POST['author'], content=request.POST['content'], thread=t)
	p.save()
	return HttpResponseRedirect(reverse('posts.views.index'))
コード例 #3
0
ファイル: views.py プロジェクト: KarolJanik/sky-twitter
def add_post():
    form = PostForm(request.form)
    if form.validate():
        post = Post(bleach.clean(form.content.data), current_user.id, datetime.datetime.utcnow())
        words = form.content.data.split()
        for tag in Tag.parse_tags(words):
            db_session.add(tag)
            post.tags.append(tag)
        db_session.add(post)
        db_session.commit()
        flash("Successful!")
    else:
        flash("Something went wrong. Make sure that your tweet's length is less than 140 characters", 'error')
    return redirect(url_for('main.hello'))
コード例 #4
0
def newtag(request, post_id):
    if request.method == 'POST':
        if request.POST['body']:
            tag = Tag()
            tag.body = request.POST['body']
            tag.post = get_object_or_404(Post, id=post_id)
            tag.author = request.user

            tag.save()

            return redirect(reverse('posts:postdetails', args=[post_id]))
        else:
            return redirect(reverse('posts:postdetails', args=[post_id]))
コード例 #5
0
def p_detail(request, post_id):
    tmp = Post.objects.get(id=post_id)
    if request.method == 'POST':
        # foreign key에 대한 정보가 없어서 에러가 발생했던 것이다
        # tag 객체를 만들어서 foreign 키인 post로 찾은 post 객체를 전달해주고
        # 그 객체를 TagForm의 instance로 추가시켜서 tag_form을 완성하면 에러가 나지 않는다
        tag = Tag(post=tmp)
        tag_form = TagForm(request.POST, instance=tag)
        if tag_form.is_valid():
            tag_form.save()
            return redirect('posts:detail', post_id)
    else:
        tag_form = TagForm()
        context = {
            'post': tmp,
            'tag_form': tag_form
        }
    return render(request, 'detail.html', context)
コード例 #6
0
ファイル: tests.py プロジェクト: Justontheway/blog.r1ff.com
    def setUp(self):
        self.tag_a = Tag(name='a')
        self.tag_b = Tag(name='b')
        self.tag_c = Tag(name='c')
        self.tag_d = Tag(name='d')
        self.tag_e = Tag(name='e')

        self.tag_a.save()
        self.tag_b.save()
        self.tag_c.save()
        self.tag_d.save()
        self.tag_e.save()

        self.category_1 = Category(name='cat 1')
        self.category_2 = Category(name='cat 2')
        self.category_3 = Category(name='cat 3')

        self.category_1.save()
        self.category_2.save()
        self.category_3.save()

        self.post_1 = Post(pk=1, title='Markdown Syntax', slug='markdown-syntax', 
            content="Here's some markdown syntax.", 
            published=True)
        self.post_2 = Post(pk=2, title='First Post', slug='first-post', 
            content='This is my first post.', 
            published=True)
        self.post_3 = Post(pk=3, title='Django Unit Testing', slug='django-unit-testing', 
            content="To test django with unit testing, do this...", 
            published=True)
        self.post_4 = Post(pk=4, title='Depressed', slug='depressed', 
            content="I'm depressed...", 
            published=False)

        # this determines time_cr
        self.post_3.save()
        self.post_1.save()
        self.post_2.save()
        self.post_4.save()

        self.post_1.tags.add(self.tag_a)
        self.post_1.tags.add(self.tag_b)
        self.post_1.tags.add(self.tag_c)
        self.post_1.categories.add(self.category_1)
        self.post_1.categories.add(self.category_2)

        self.post_2.tags.add(self.tag_b)
        self.post_2.tags.add(self.tag_d)
        self.post_2.tags.add(self.tag_e)
        self.post_2.categories.add(self.category_2)
        self.post_2.categories.add(self.category_3)

        self.post_3.tags.add(self.tag_b)
        self.post_3.tags.add(self.tag_d)
        self.post_3.tags.add(self.tag_e)

        self.post_4.categories.add(self.category_3)

        # this determines time_lu
        self.post_4.save()
        self.post_1.save()
        self.post_3.save()
        self.post_2.save()

        self.client = Client()
コード例 #7
0
ファイル: tests.py プロジェクト: Justontheway/blog.r1ff.com
class view_post_tests(TestCase):
    def setUp(self):
        self.tag_a = Tag(name='a')
        self.tag_b = Tag(name='b')
        self.tag_c = Tag(name='c')
        self.tag_d = Tag(name='d')
        self.tag_e = Tag(name='e')

        self.tag_a.save()
        self.tag_b.save()
        self.tag_c.save()
        self.tag_d.save()
        self.tag_e.save()

        self.category_1 = Category(name='cat 1')
        self.category_2 = Category(name='cat 2')
        self.category_3 = Category(name='cat 3')

        self.category_1.save()
        self.category_2.save()
        self.category_3.save()

        self.post_1 = Post(pk=1, title='Markdown Syntax', slug='markdown-syntax', 
            content="Here's some markdown syntax.", 
            published=True)
        self.post_2 = Post(pk=2, title='First Post', slug='first-post', 
            content='This is my first post.', 
            published=True)
        self.post_3 = Post(pk=3, title='Django Unit Testing', slug='django-unit-testing', 
            content="To test django with unit testing, do this...", 
            published=True)
        self.post_4 = Post(pk=4, title='Depressed', slug='depressed', 
            content="I'm depressed...", 
            published=False)

        # this determines time_cr
        self.post_3.save()
        self.post_1.save()
        self.post_2.save()
        self.post_4.save()

        self.post_1.tags.add(self.tag_a)
        self.post_1.tags.add(self.tag_b)
        self.post_1.tags.add(self.tag_c)
        self.post_1.categories.add(self.category_1)
        self.post_1.categories.add(self.category_2)

        self.post_2.tags.add(self.tag_b)
        self.post_2.tags.add(self.tag_d)
        self.post_2.tags.add(self.tag_e)
        self.post_2.categories.add(self.category_2)
        self.post_2.categories.add(self.category_3)

        self.post_3.tags.add(self.tag_b)
        self.post_3.tags.add(self.tag_d)
        self.post_3.tags.add(self.tag_e)

        self.post_4.categories.add(self.category_3)

        # this determines time_lu
        self.post_4.save()
        self.post_1.save()
        self.post_3.save()
        self.post_2.save()

        self.client = Client()


    def test_view_single_posts_by_id(self):
        # public post
        response = self.client.get('/posts/2/')
        self.assertIn('This is my first post', response.content)

        # private post
        response = self.client.get('/posts/4/')
        self.assertIn("The post you're looking for does not exist. ", response.content)

        # non-existent post
        response = self.client.get('/posts/100/')
        self.assertIn("The post you're looking for does not exist. ", response.content)

    def test_view_single_posts_by_slug(self):
        # public post
        response = self.client.get('/posts/first-post/')
        self.assertIn('This is my first post', response.content)

        # private post
        response = self.client.get('/posts/depressed/')
        self.assertIn("The post you're looking for does not exist. ", response.content)

        # non-existent post
        response = self.client.get('/posts/not-exist/')
        self.assertIn("The post you're looking for does not exist. ", response.content)

    def test_view_single_posts_by_date_slug(self):
        pass
        # # public post
        # response = self.client.get('/posts/2014/04/20/first-post/')
        # self.assertIn('This is my first post', response.content)

        # # private post
        # response = self.client.get('/posts/????/')
        # self.assertIn("The post you're looking for does not exist. ", response.content)

        # # non-existent post: slug does not exist
        # response = self.client.get('/posts/????/')
        # self.assertIn("The post you're looking for does not exist. ", response.content)

        # # non-existent post: date/slug mismatch
        # response = self.client.get('/posts/????/')
        # self.assertIn("The post you're looking for does not exist. ", response.content)

    def test_view_all_posts(self):
        '''
        private posts should not show
        public posts are listed in chronolgical order of creation time
        '''

        # redirects properly
        response = self.client.get('/posts/')
        self.assertRedirects(response, '/')

        # posts appear in correct order
        response = self.client.get('/')
        post_1_position = response.content.find("Here's some markdown syntax.")
        post_2_position = response.content.find('This is my first post.')
        post_3_position = response.content.find("To test django with unit testing")

        self.assertGreater(post_1_position, 0)
        self.assertGreater(post_2_position, 0)
        self.assertGreater(post_3_position, 0)

        self.assertLess(post_2_position, post_1_position)
        self.assertLess(post_1_position, post_3_position)

        # private post doesn't appear
        post_4_position = response.content.find("I'm depressed")
        self.assertEqual(post_4_position, -1)

    def test_get_absolute_url(self):
        # time below is already Apr 30 in UTC time (local time is Central Time, UTC-6/5s)
        self.post_2.time_cr = datetime(2014, 4, 29, 20, 13, 19, 0, get_default_timezone())
        the_absolute_url = self.post_2.get_absolute_url()
        # make sure permalink doesn't roll over to Apr 30
        self.assertEqual(the_absolute_url, '/posts/2014/04/29/first-post/')
コード例 #8
0
 def test_add_tag(self):
     t = Tag(name='c++')
     t.save()
     j = Tag.objects.get(name='c++')
     self.assertEqual(j.name, 'c++')
コード例 #9
0
 def handle(self, *args, **options):
     with open("ingredients.json", "r", encoding="utf-8") as fh:
         data = json.load(fh)
     for i in data:
         dimension = i["dimension"]
         title = i["title"]
         if not Ingredients.objects.filter(title=title).first():
             if dimension in [
                 "по вкусу",
                 "стакан",
                 "кусок",
                 "горсть",
                 "банка",
                 "тушка",
                 "пакет",
             ]:
                 dimension = "г"
             ingredient = Ingredients(title=title, dimension=dimension)
             ingredient.save()
     if not Tag.objects.filter(name="Завтрак").first():
         tag_breakfast = Tag(name="Завтрак", slug="breakfast", color="orange")
         tag_breakfast.save()
     if not Tag.objects.filter(name="Обед").first():
         tag_lunch = Tag(name="Обед", slug="lunch", color="green")
         tag_lunch.save()
     if not Tag.objects.filter(name="Ужин").first():
         tag_dinner = Tag(name="Ужин", slug="dinner", color="purple")
         tag_dinner.save()
コード例 #10
0
 def handle(self, *args, **options):
     ua = '[adjective][species] (explore621) {}'.format(options['username'])
     started = datetime.now()
     started = started.replace(tzinfo=pytz.UTC)
     ingested = 0
     total_new = 0
     total_updated = 0
     last_id = 0
     tags_added = {}
     sources_added = {}
     artists_added = {}
     self.stdout.write('Refreshing data from e621 starting at {}'.format(
         started.isoformat()))
     self.stdout.write(
         'Fetching {} pages worth of posts at {} per page'.format(
             options['pages'], options['per_page']))
     for i in range(1, options['pages'] + 1):
         time.sleep(1)
         self.stdout.write('--- Fetching page {}'.format(i))
         r = requests.get(
             'https://e621.net/post/index.json',
             {'limit': options['per_page'], 'page': i},
             headers={'user-agent': ua})
         if r.status_code != 200:
             self.stdout.write(
                 self.style.NOTICE('    got {} on page {}, skipping'.format(
                     r.status_code, i)))
             continue
         tags_added = {}
         sources_added = {}
         artists_added = {}
         updated = 0
         new = 0
         for record in r.json():
             if record['id'] > last_id:
                 last_id = record['id']
             created_at = pytz.utc.localize(
                 datetime.fromtimestamp(record['created_at']['s']))
             try:
                 post = Post.objects.get(source_id=record['id'])
                 post.source_id = record['id']
                 post.description = record['description']
                 post.created_at = created_at
                 post.creator_id = record['creator_id']
                 post.author = record['author']
                 post.change = record['change']
                 post.source = record['source']
                 post.score = record['score']
                 post.fav_count = record['fav_count']
                 post.md5 = record['md5']
                 post.file_size = record['file_size']
                 post.file_url = record['file_url']
                 post.width = record['width']
                 post.height = record['height']
                 post.file_ext = record['file_ext']
                 post.preview_url = record['preview_url']
                 post.preview_width = record['preview_width']
                 post.preview_height = record['preview_height']
                 post.sample_url = record['sample_url']
                 post.sample_width = record['sample_width']
                 post.sample_height = record['sample_height']
                 post.rating = record['rating']
                 post.status = record['status']
                 post.has_comments = record['has_comments']
                 post.has_notes = record['has_notes']
                 post.has_children = record['has_children']
                 post.children = record['children']
                 post.parent_id = record['parent_id']
                 updated += 1
             except Post.DoesNotExist:
                 post = Post(
                     source_id=record['id'],
                     description=record['description'],
                     created_at=created_at,
                     creator_id=record['creator_id'],
                     author=record['author'],
                     change=record['change'],
                     source=record['source'],
                     score=record['score'],
                     fav_count=record['fav_count'],
                     md5=record['md5'],
                     file_size=record['file_size'],
                     file_url=record['file_url'],
                     width=record['width'],
                     height=record['height'],
                     file_ext=record['file_ext'],
                     preview_url=record['preview_url'],
                     preview_width=record['preview_width'],
                     preview_height=record['preview_height'],
                     sample_url=record['sample_url'],
                     sample_width=record['sample_width'],
                     sample_height=record['sample_height'],
                     rating=record['rating'],
                     status=record['status'],
                     has_comments=record['has_comments'],
                     has_notes=record['has_notes'],
                     has_children=record['has_children'],
                     children=record['children'],
                     parent_id=record['parent_id'])
                 new += 1
             record_tags = record['tags'].split(' ')
             tags = []
             for record_tag in record_tags:
                 if record_tag in tags_added:
                     tag = tags_added[record_tag]
                 else:
                     try:
                         tag = Tag.objects.get(tag=record_tag)
                     except Tag.DoesNotExist:
                         tag = Tag(tag=record_tag)
                         tag.save()
                     tags_added[record_tag] = tag
                 tags.append(tag)
             artists = []
             for record_artist in record.get('artist', []):
                 if record_artist in artists_added:
                     artist = artists_added[record_artist]
                 else:
                     try:
                         artist = Artist.objects.get(name=record_artist)
                     except Artist.DoesNotExist:
                         artist = Artist(name=record_artist)
                         artist.save()
                     artists_added[record_artist] = artist
                 artists.append(artist)
             sources = []
             for record_source in record.get('sources', []):
                 if record_source in sources_added:
                     source = sources_added[record_source]
                 else:
                     try:
                         source = Source.objects.get(url=record_source)
                     except Source.DoesNotExist:
                         source = Source(url=record_source)
                         source.save()
                     sources_added[record_source] = source
                 sources.append(source)
             try:
                 post.save()
             except IntegrityError:
                 continue
             post.artists.set(artists)
             post.sources.set(sources)
             post.tags.set(tags)
             ingested += 1
         total_new += new
         total_updated += updated
         self.stdout.write(
             self.style.SUCCESS(
                 '    processed page {}; {} new, {} updated'.format(
                     i, new, updated)))
     self.stdout.write(
         self.style.SUCCESS('{} posts ingested ({} new - {} updated)'.format(
             ingested, total_new, total_updated)))
     tags_fixed = 0
     fixed_tags = []
     self.stdout.write('Fixing typeless tags')
     for tag in Tag.objects.filter(tag_type=-1):
         self.stdout.write('--- Fixing tag {}'.format(tag.tag))
         r = requests.get(
             'https://e621.net/tag/show.json',
             params={'name': tag.tag},
             headers={'user-agent': '[adjective][species]'})
         if 'type' not in r.json():
             self.stdout.write(
                 self.style.NOTICE('    not fixing {}'.format(tag.tag)))
             continue
         tag.tag_type = r.json()['type']
         tag.save()
         tags_fixed += 1
         fixed_tags.append(tag.tag)
         self.stdout.write(
             self.style.SUCCESS('    fixed {} ({})'.format(
                 tag.tag, tag.get_tag_type_display())))
         time.sleep(0.7)
     self.stdout.write(
         self.style.SUCCESS('{} tags fixed'.format(tags_fixed)))
     empty = Tag.objects.annotate(Count('post')).filter(post__count=0)
     tags_deleted = 0
     deleted_tags = []
     self.stdout.write('Deleting empty tags')
     for tag in empty:
         self.stdout.write(
             self.style.NOTICE('--- deleting {}'.format(tag.tag)))
         deleted_tags.append(tag.tag)
         tag.delete()
         tags_deleted += 1
     self.stdout.write(
         self.style.SUCCESS('{} empty tags deleted'.format(tags_deleted)))
     self.stdout.write('Deleting empty sources')
     sources_deleted = len([
         s.delete() for s in
         Source.objects.annotate(Count('post')).filter(post__count=0)])
     self.stdout.write(
         self.style.NOTICE('--- {} sources deleted'.format(
             sources_deleted)))
     self.stdout.write('Deleting empty artists')
     artists_deleted = len([
         a.delete() for a in
         Artist.objects.annotate(Count('post')).filter(post__count=0)])
     self.stdout.write(
         self.style.NOTICE('--- {} artists deleted'.format(
             artists_deleted)))
     log = IngestLog(
         started=started,
         records_ingested=ingested,
         new=total_new,
         updated=total_updated,
         last_id=last_id,
         fixed_tags=' '.join(fixed_tags),
         deleted_tags=' '.join(deleted_tags),
         sources_deleted=sources_deleted,
         artists_deleted=artists_deleted)
     log.save()
     self.stdout.write(
         self.style.SUCCESS('Finished refreshing in {}'.format(
             str(log.finished - log.started))))
コード例 #11
0
    def handle(self, *args, **options):
        self.stdout.write('Importing WordPress posts')
        wp_table_prefix = options['wp-table-prefix']
        utc = pytz.utc

        my_conv = {
            FIELD_TYPE.LONG: int,
            FIELD_TYPE.INT24: int,
            FIELD_TYPE.SHORT: int,
            FIELD_TYPE.TINY: int,
            FIELD_TYPE.LONGLONG: int,
            FIELD_TYPE.DATETIME: str,
        }

        try:
            db = MySQLdb.connect(host=options['host'],
                                 user=options['user'],
                                 passwd=options['passwd'],
                                 db=options['db'],
                                 conv=my_conv,
                                 use_unicode=True,
                                 charset='utf8mb4')
        except Exception as e:
            self.stderr.write('Unable to connect to database')
            self.stderr.write(str(e))
            return

        cur = db.cursor()

        cur.execute('SET NAMES utf8mb4')
        cur.execute("SET CHARACTER SET utf8mb4")
        cur.execute("SET character_set_connection=utf8mb4")

        wpTagMap = dict()
        wpCategoryMap = dict()
        posts = list()

        # GET TAGS
        self.stdout.write('Migrating Tags')

        cur.execute(f"""SELECT COUNT(*) 
            FROM {wp_table_prefix}_terms AS terms 
            INNER JOIN {wp_table_prefix}_term_taxonomy AS taxonomy 
                ON terms.term_id = taxonomy.term_id 
            INNER JOIN {wp_table_prefix}_term_relationships AS relationships
                ON relationships.`term_taxonomy_id` = taxonomy.`term_taxonomy_id`
            INNER JOIN {wp_table_prefix}_posts AS posts
                ON relationships.`object_id` = posts.`ID`
            WHERE taxonomy.taxonomy = 'post_tag'
                AND posts.post_type = 'post'""")
        row = cur.fetchone()
        self.stdout.write(f"{row[0]} tags to migrate")

        cur.execute(f"""SELECT 
                terms.term_id, 
                terms.name, 
                terms.slug, 
                posts.ID
            FROM {wp_table_prefix}_terms AS terms 
            INNER JOIN {wp_table_prefix}_term_taxonomy AS taxonomy 
                ON terms.term_id = taxonomy.term_id 
            INNER JOIN {wp_table_prefix}_term_relationships AS relationships
                ON relationships.`term_taxonomy_id` = taxonomy.`term_taxonomy_id`
            INNER JOIN {wp_table_prefix}_posts AS posts
                ON relationships.`object_id` = posts.`ID`
            WHERE taxonomy.taxonomy = 'post_tag'
                AND posts.post_type = 'post'""")

        rows = cur.fetchall()

        savedTags = dict()

        for row in rows:
            tag = Tag(name=row[1], slug=row[2])

            if row[0] in savedTags:
                tag = savedTags[row[0]]
            else:
                tag.save()
                savedTags[row[0]] = tag

            if row[3] not in wpTagMap:
                wpTagMap[row[3]] = list()

            wpTagMap[row[3]].append(tag)

        self.stdout.write(f"{len(wpTagMap)} tags migrated")

        # GET CATEGORIES
        self.stdout.write('Migrating Categories')
        cur.execute(f"""SELECT COUNT(*) 
            FROM {wp_table_prefix}_terms AS terms 
            INNER JOIN {wp_table_prefix}_term_taxonomy AS taxonomy 
                ON terms.term_id = taxonomy.term_id 
            INNER JOIN {wp_table_prefix}_term_relationships AS relationships
                ON relationships.`term_taxonomy_id` = taxonomy.`term_taxonomy_id`
            INNER JOIN {wp_table_prefix}_posts AS posts
                ON relationships.`object_id` = posts.`ID`
            WHERE taxonomy.taxonomy = 'category'
                AND posts.post_type = 'post'""")
        row = cur.fetchone()
        self.stdout.write(f"{row[0]} categories to migrate")

        cur.execute(f"""SELECT 
                terms.term_id, 
                terms.name, 
                terms.slug, 
                posts.ID 
            FROM {wp_table_prefix}_terms AS terms 
            INNER JOIN {wp_table_prefix}_term_taxonomy AS taxonomy 
                ON terms.term_id = taxonomy.term_id 
            INNER JOIN {wp_table_prefix}_term_relationships AS relationships
                ON relationships.`term_taxonomy_id` = taxonomy.`term_taxonomy_id`
            INNER JOIN {wp_table_prefix}_posts AS posts
                ON relationships.`object_id` = posts.`ID`
            WHERE taxonomy.taxonomy = 'category'
                AND posts.post_type = 'post'""")

        rows = cur.fetchall()

        savedCategories = dict()

        for row in rows:
            category = Category(name=row[1], slug=row[2])

            if row[0] in savedCategories:
                category = savedCategories[row[0]]
            else:
                category.save()
                savedCategories[row[0]] = category

            wpCategoryMap[row[3]] = category

        self.stdout.write(f"{len(wpCategoryMap)} categories migrated")

        # POSTS
        self.stdout.write('Migrating Posts')
        cur.execute(
            f"SELECT COUNT(*) FROM {wp_table_prefix}_posts WHERE post_type = 'post'"
        )
        row = cur.fetchone()
        self.stdout.write(f"{row[0]} posts to migrate")

        # we need a default Author
        author = Profile.objects.all()[:1].get()
        self.stdout.write(f'Default author is {author.name}')

        cur.execute(
            f"SELECT ID, post_date_gmt, post_content, post_title, post_excerpt, post_status, post_name, post_modified_gmt FROM {wp_table_prefix}_posts WHERE post_type = 'post'"
        )

        rows = cur.fetchall()

        for row in rows:
            if not row[6]:
                self.stdout.write(f'{row[3]} has no slug. Skipping.')
                continue

            published = row[1]
            if published == '0000-00-00 00:00:00':
                published = None
            else:
                published = utc.localize(datetime.datetime.strptime(
                    published, "%Y-%m-%d %H:%M:%S"),
                                         is_dst=None)

            updated = row[7]
            if updated == '0000-00-00 00:00:00':
                updated = None
            else:
                updated = utc.localize(datetime.datetime.strptime(
                    updated, "%Y-%m-%d %H:%M:%S"),
                                       is_dst=None)

            post = Post(published=published,
                        content=row[2],
                        title=row[3],
                        summary=row[4],
                        is_published=row[5] == 'publish',
                        slug=row[6],
                        updated=updated,
                        author=author)

            try:
                post.save()
            except Exception as e:
                self.stderr.write(str(e))
                self.stderr.write(post.name)
                raise e
            posts.append({'post': post, 'wpId': row[0]})

        self.stdout.write(f"{len(posts)} posts migrated")

        # Attaching Tags and Categories
        self.stdout.write("Attaching tags and categories")
        for post in posts:
            if post['wpId'] in wpCategoryMap:
                post['post'].category = wpCategoryMap[post['wpId']]
            else:
                self.stdout.write(f"No category for {post['post'].title}")

            if post['wpId'] in wpTagMap:
                post['post'].tags.set(wpTagMap[post['wpId']])
            else:
                self.stdout.write(f"No tags for {post['post'].title}")

            post['post'].save()

        self.stdout.write("Tags and categories attached")

        cur.close()
        db.close()
        self.stdout.write('WordPress posts imported')
        return
コード例 #12
0
 def handle(self, *args, **options):
     ua = '[adjective][species] (explore621) {}'.format(options['username'])
     started = datetime.now()
     self.stdout.write(
         'Performing a full refresh from e621 starting at {}'.format(
             started.isoformat()))
     max_id = Post.objects.aggregate(mid=Max('source_id'))['mid']
     min_id = Post.objects.aggregate(mid=Min('source_id'))['mid']
     total_added = 0
     while max_id > min_id:
         time.sleep(1)
         r = requests.get('https://e621.net/post/index.json', {
             'limit': 320,
             'before_id': max_id
         },
                          headers={'user-agent': ua})
         if r.status_code != 200:
             self.stdout.write(
                 self.style.NOTICE(
                     '    got {} on before_id {}, skipping'.format(
                         r.status_code, max_id)))
             continue
         skipped = 0
         added = 0
         for record in r.json():
             max_id = record['id']
             created_at = pytz.utc.localize(
                 datetime.fromtimestamp(record['created_at']['s']))
             if Post.objects.filter(source_id=record['id']).count() != 0:
                 skipped += 1
                 continue
             post = Post(source_id=record['id'],
                         description=record['description'],
                         created_at=created_at,
                         creator_id=record['creator_id'],
                         author=record['author'],
                         change=record['change'],
                         source=record['source'],
                         score=record['score'],
                         fav_count=record['fav_count'],
                         md5=record['md5'],
                         file_size=record['file_size'],
                         file_url=record['file_url'],
                         width=record['width'],
                         height=record['height'],
                         file_ext=record['file_ext'],
                         preview_url=record['preview_url'],
                         preview_width=record['preview_width'],
                         preview_height=record['preview_height'],
                         sample_url=record['sample_url'],
                         sample_width=record['sample_width'],
                         sample_height=record['sample_height'],
                         rating=record['rating'],
                         status=record['status'],
                         has_comments=record['has_comments'],
                         has_notes=record['has_notes'],
                         has_children=record['has_children'],
                         children=record['children'],
                         parent_id=record['parent_id'])
             record_tags = record['tags'].split(' ')
             tags = []
             for record_tag in record_tags:
                 try:
                     tag = Tag.objects.get(tag=record_tag)
                 except Tag.DoesNotExist:
                     tag = Tag(tag=record_tag)
                     tag.save()
                 tags.append(tag)
             artists = []
             for record_artist in record.get('artist', []):
                 try:
                     artist = Artist.objects.get(name=record_artist)
                 except Artist.DoesNotExist:
                     artist = Artist(name=record_artist)
                     artist.save()
                 artists.append(artist)
             sources = []
             for record_source in record.get('sources', []):
                 try:
                     source = Source.objects.get(url=record_source)
                 except Source.DoesNotExist:
                     source = Source(url=record_source)
                     source.save()
                 sources.append(source)
             try:
                 post.save()
             except IntegrityError:
                 continue
             post.artists.set(artists)
             post.sources.set(sources)
             post.tags.set(tags)
             added += 1
         self.stdout.write(
             self.style.SUCCESS(
                 'Ending at {}: {} added / {} skipped'.format(
                     max_id, added, skipped)))
         total_added += added
     self.stdout.write('Fixing typeless tags')
     for tag in Tag.objects.filter(tag_type=-1):
         self.stdout.write('--- Fixing tag {}'.format(tag.tag))
         r = requests.get('https://e621.net/tag/show.json',
                          params={'name': tag.tag},
                          headers={'user-agent': '[adjective][species]'})
         if 'type' not in r.json():
             self.stdout.write(
                 self.style.NOTICE('    not fixing {}'.format(tag.tag)))
             continue
         tag.tag_type = r.json()['type']
         tag.save()
         tags_fixed += 1
         fixed_tags.append(tag.tag)
         self.stdout.write(
             self.style.SUCCESS('    fixed {} ({})'.format(
                 tag.tag, tag.get_tag_type_display())))
         time.sleep(0.7)
     self.stdout.write(
         self.style.SUCCESS('{} tags fixed'.format(tags_fixed)))
     self.stdout.write(
         self.style.SUCCESS('Finished full refresh in {}'.format(
             str(datetime.now() - started))))