def mapala_pages(BLOCKCHAIN): url = MAPALA_URL + 'index?blockchain=%s' % BLOCKCHAIN # запрашивает с мапалы только отсутствующие в базе страницы data = requests.get(url).json() for article in data['allModels']: if any(key not in article for key in ['title', 'author', 'body']): continue try: author = author_define(article.get('author'), True) try: article['meta'] = json.loads(article['meta']) except json.decoder.JSONDecodeError: article['meta'] = {} try: tags = [tag.lower() for tag in article.get('meta').get('tags')] except: tags = None position = None if 'coordinates' in article[ 'meta'] and article['meta']['coordinates'] != '': try: position = Point( article['meta']['coordinates'].split(',')[0], article['meta']['coordinates'].split(',')[1]) except: position = None position_text = '' if 'location' in article['meta']: position_text = article['meta']['location'] if Page.objects.filter(author=author).exists(): continue else: page = Page(author_id=author.id, title=article.get('title')) page.created_at = aware_date_mapala(article.get('created_at')) page.updated_at = aware_date_mapala(article.get('updated_at')) page.body = article.get('body') page.image = article.get('meta').get('image') page.parent_permlink = article.get('parent_permlink') try: page.miniature = article['meta']['image'][0] except KeyError: pass if BLOCKCHAIN == 'golos': page.locale = 'ru' else: page.locale = 'en' page.total_payout_value = article.get('total_payout_value') page.total_pending_payout_value = article.get( 'total_pending_payout_value') page.status = 2 page.position = position page.position_text = position_text page.save() create_page_images(page, article) if tags: for tag in tags: page.tags.add(tag_define(tag)) voters = json.loads(article.get('voters')) for v in voters: page.voters.add(author_define(v, True).id) country = article.get('country') city = article.get('city') category = article.get('category') sub_category = article.get('sub_category') for m_t in [ mastertag_define('mapala_places', country), mastertag_define(country, city), mastertag_define('mapala_categories', category), mastertag_define(category, sub_category) ]: if m_t: page.master_tag = m_t page.save() get_comments_for(page, page.permlink) except StopIteration as e: print('Warning: ', e) return True