# encoding: utf-8 from __future__ import print_function from pybooru import Danbooru from pybooru import Moebooru konachan = Moebooru("konachan") kona_tags = konachan.tag_list(order='date') print(konachan.last_call) kona_post = konachan.post_list() print(konachan.last_call) lolibooru = Moebooru("lolibooru") kona_tags = lolibooru.tag_list(order='date') print(lolibooru.last_call) kona_post = lolibooru.post_list() print(lolibooru.last_call) danbooru = Danbooru('danbooru') dan_tags = danbooru.tag_list(order='name') print(danbooru.last_call) dan_post = danbooru.post_list(tags="computer") print(danbooru.last_call)
# encoding: utf-8 from __future__ import print_function from pybooru import Danbooru from pybooru import Moebooru konachan = Moebooru("konachan") kona_tags = konachan.tag_list(order='date') print(konachan.last_call) kona_post = konachan.post_list() print(konachan.last_call) danbooru = Danbooru('danbooru') dan_tags = danbooru.tag_list(order='name') print(danbooru.last_call) dan_post = danbooru.post_list(tags="computer") print(danbooru.last_call)
# -*- coding: utf-8 -*- from __future__ import unicode_literals from pybooru import Moebooru client = Moebooru('Konachan') tags = client.tag_list(order='name') for tag in tags: print("Tag: {0} ----- {1}".format(tag['name'], tag['type']))
class SakugabooruService(object): BASE_URL = SAKUGABOORU_BASE_URL MAX_DEPTH = 11 def __init__(self): self.client = Moebooru(site_url=self.BASE_URL) self.local_cache = dict() self.max_page = 1e9 self.created_tags = list() def _save_post(self, post_dict): post, created = Post.objects.update_or_create( id=post_dict['id'], defaults={ 'source': post_dict['source'], 'file_size': post_dict['file_size'], 'is_shown': post_dict['is_shown_in_index'], 'is_pending': post_dict['status'] == "pending", 'md5': post_dict['md5'], 'ext': post_dict['file_ext'], 'created_at': datetime.fromtimestamp(int(post_dict['created_at']), tz=utc), 'score': post_dict['score'], 'rating': post_dict['rating'], 'sample_url': post_dict['sample_url'], 'sample_file_size': post_dict['sample_file_size'] }) tags = self.update_tags(post_dict['tags'].split(' ')) with transaction.atomic(): post.tags.clear() for tag in tags: post.tags.add(tag) post.uploader = self.update_uploader(post_dict['author'], post.is_pending, post.is_shown) post.save() return post @staticmethod def update_uploader(author, is_pending, is_shown): uploader, created = Uploader.objects.get_or_create(name=author) if is_shown and not (is_pending or uploader.in_whitelist): uploader.in_whitelist = True uploader.save() return uploader def update_tag(self, tag): res = self.client.tag_list(name=tag.name) for tag_dict in res: if tag_dict['name'] == tag.name: if tag.type != tag_dict['type']: tag.type = tag_dict['type'] tag.save() logger.info("Tag[{}] type has been updated.".format( tag.name)) break return tag def update_tags(self, tag_str_list, force_update=False): tags = list() for tag_str in tag_str_list: tag, created = Tag.objects.get_or_create(name=tag_str) if created or force_update: self.created_tags.append(tag) try: self.update_tag(tag) except: logger.exception( "Update tag [] from sakugabooru failed.".format( tag.name)) tags.append(tag) return tags def _get_post_dict_by_api(self, post_id): """ get post_dict by post_id using api and save every accessed page to cache. :param post_id: Integer :return: success: post_dict; failed: None """ limit = 100 post_dict = self.local_cache.get(post_id, None) if post_dict: return post_dict page_stack = list() min_page = 1 page_stack.append(min_page) max_page = self.max_page while page_stack and len(page_stack) <= self.MAX_DEPTH: res = self._get_posts_page(page=page_stack[-1], limit=limit) if not res: last_page = page_stack.pop() if last_page < self.max_page: self.max_page = last_page if last_page < max_page: max_page = last_page if len(page_stack) > 0: page_stack.append((max_page + page_stack[-1]) // 2) continue return None first_id = res[0]["id"] last_id = res[-1]["id"] logger.info("Posts Page[{}]: first[{}], last [{}]".format( page_stack[-1], first_id, last_id)) if post_id > first_id: if page_stack[-1] <= 1: return None next_page = max( page_stack[-1] + (-(post_id - first_id) // limit), (min_page + page_stack[-1]) // 2) max_page = page_stack[-1] if next_page < 1: next_page = 1 page_stack.append(next_page) elif post_id < last_id: next_page = min( page_stack[-1] - (-(last_id - post_id) // limit), (max_page + page_stack[-1]) // 2) min_page = page_stack[-1] if next_page in page_stack: break if next_page > self.max_page: next_page = (self.max_page + page_stack[-1]) // 2 page_stack.append(next_page) else: return self.local_cache.get(post_id) return self.local_cache.get(post_id, None) def update_post(self, post_id): posts = self.update_posts(post_id) if posts: return posts[0] return None def update_posts(self, *post_ids): try: for post_id in post_ids: if post_id not in self.local_cache: if not self._get_post_dict_by_api(post_id): try: post = Post.objects.get(id=post_id) post.is_shown = False post.is_pending = False post.save() except Post.DoesNotExist: pass finally: updated = [] for v in self.local_cache.values(): self._save_post(v) updated.append(str(v['id'])) logger.info(f"Posts[{','.join(updated)}] Updated.") return Post.objects.filter(id__in=post_ids) def _save_cache(self, res): for post_dict in res: self.local_cache[post_dict["id"]] = post_dict @staticmethod def _refresh_is_shown(page_res): if page_res: for post in Post.objects.filter( id__gte=page_res[-1]['id'], id__lte=page_res[0]['id']).exclude( id__in=[x['id'] for x in page_res]): post.is_pending = False post.is_shown = False post.save() def _get_posts_page(self, page=1, limit=100): try: res = self.client.post_list(page=page, limit=limit) self._save_cache(res) self._refresh_is_shown(res) return res except: logger.exception( "Failed to get posts page [{}] from sakugabooru with limit[{}]." .format(page, limit)) return dict() def update_posts_by_page(self, page=1, limit=100, escape_id=None): posts = list() logger.info( "Getting posts page [{}] from sakugabooru with limit[{}].".format( page, limit)) for post_dict in self._get_posts_page(page=page, limit=limit): if escape_id and post_dict["id"] <= escape_id: break post = self._save_post(post_dict) if post: posts.append(post) logger.info(f"Posts[{','.join([str(x.id) for x in posts])}] Updated.") return posts