def process_item(self, item, spider): image = item.get('image_data', None) image_filename = item.get('image_filename', None) current = NewsItem.objects.filter( Q(title=item['title'], datetime=item['datetime']) | Q(source=item['source']), generated=True).first() if current is not None: print(f"{item['title']} - {item['datetime']}\n" f"\t {item['source']} {item.get('image_filename', None)}\n\tCurrent: {current} {current.cover_img}") if current is None: current = NewsItem( title=item['title'], datetime=item['datetime'], source=item['source'], content=item['content'], generated=True, ) else: current.content = item['content'] if image_filename is not None: if image is None: logging.error("Inconsistency. Image name but no image.") return current.cover_img.save(image_filename, File(io.BytesIO(image))) current.gen_summary() current.save() return item
def done(self, request, cleaned_data): ni = NewsItem() ni.topic = cleaned_data['topic'] ni.body = cleaned_data['body'] ni.title = cleaned_data['title'] ni.submitter = request.user ni.save() return HttpResponseRedirect('/news/submitted')
def gen_news(title, category, site, url, chksum, pub_date): ni = NewsItem() ni.site = site ni.title = title ni.url = url ni.pub_date = pub_date ni.category = category ni.chksum = chksum ni.save()
def _convert_to_news_item(self, e): title = e['title'] ingress = e['introduction'] subject = 'miehet-edustus' if e['team'] == 'Miehet Edustus' else 'heimo' if subject == 'miehet-edustus': body = e['content'] + \ f"<p class='author'>// {e['text_by']}</p>" else: if e['team']: body = f"<p class='team'>{e['team']}</p>" + \ e['content'] + \ f"<p class='author'>// {e['text_by']}</p>" else: body = e['content'] + \ f"<p class='author'>// {e['text_by']}</p>" try: publication_date = datetime.datetime.strptime( e['published_at'], '%Y-%m-%dT%H:%M:%S+03:00') except ValueError: publication_date = datetime.datetime.strptime( e['published_at'], '%Y-%m-%dT%H:%M:%S+02:00') except TypeError: publication_date = datetime.datetime(2012, 1, 1, 0, 0, 0) # there are null values.. publication_date = pytz.timezone('Europe/Helsinki').localize( publication_date) if NewsItem.objects.filter(title=title, publication_date=publication_date).exists(): print(f'Already had {title}') return NewsItem.objects.get(title=title, publication_date=publication_date) news_item = NewsItem( title=title, ingress=ingress, subject=subject, body=body, publication_date=publication_date, ) if e['mainimage']: image = requests.get(e['mainimage']) fp = BytesIO() fp.write(image.content) name = urlparse(e['mainimage']).path.split('/')[-1] news_item.image.save(name, File(fp)) news_item.save() print(f'Imported #{news_item.id} {news_item.title}') return news_item
def test_method_sync_staging__with_news(self): # add newssite newspage = NewsPage( site=self.staticsite, index=self.indexpage, created_by=self.system_admin_user, updated_by=self.system_admin_user, ) newspage.save() one_day_ago = timezone.now() - timezone.timedelta(days=1) # update IndexPage.newsitems_template_variablename self.indexpage.newsitems_template_variablename = 'news_items' self.indexpage.save() # create is_publish items publshed_newsitems_count = 15 for i in range(publshed_newsitems_count): newsitem = NewsItem( newspage=newspage, publish_on=one_day_ago, is_published=True, image=self._get_dummy_image_file( sample_image_filename=self.news_image_filename), image_relpath=self.news_image_relpath, created_by=self.system_admin_user, updated_by=self.system_admin_user, ) newsitem.save() transferred_relative_paths = self.staticsite.sync( update_production=False) self.assertTrue(transferred_relative_paths) expected_keys = ( 'stylesheets/plugins/bootstrap3.min.css', 'stylesheets/plugins/drawer.min.css', 'stylesheets/style.css', 'index.html', 'imgs/news/sample-photo.jpeg', 'news/news_0.html', 'news/news_1.html', 'news/news_2.html', ) objects = S3_CLIENT.list_objects( Bucket=self.staging_bucket_name)['Contents'] actual_keys = [obj['Key'] for obj in objects] missing = set(expected_keys) - set(actual_keys) self.assertFalse(missing, f'missing Keys: {missing}')
def handle(self, *args, **options): print "Getting reddit update" r = reddit.Reddit('willcritchlow anythinginterestingbot') submissions = r.get_subreddit('worldnews').get_top('week') for submission in submissions: try: ni = NewsItem.objects.get(slug=submission.id) except: ni = NewsItem() ni.slug = submission.id ni.title = submission.title ni.url = submission.short_link # use the short link to guarantee it will fit in the database ni.comment_url = submission.permalink ni.created = datetime.fromtimestamp(submission.created_utc) ni.score = submission.score try: ni.save() except: print "Error saving %s" % ni.title print "Reddit update complete"
def handle(self, *args, **options): words = WORDS[:] images = build_image_factory() if not len(images): images = [None for i in range(NUM_ITEMS)] while len(images): i = NewsItem() random_name = words[random.randint(0, len(words) - 1)] i.title = text_type(random_name).capitalize() i.slug = "{0}-{1}".format(slugify(i.title), uuid.uuid4()) i.image = images.pop() i.body = text_type(SENTENCES[random.randint(0, len(SENTENCES) - 1)]) i.date_published = radar.random_datetime() try: i.save() words.remove(random_name) except Exception as e: images.append(i.image) logger.debug(e)
def _news(self, title='Test', content='TestContent', enabled=True): news = NewsItem(title=title, content=content, enabled=enabled) news.save() return news
def insert_new_new_to_database(): new_new = get_random_new() haber = NewsItem() haber.title = new_new haber.content = new_new haber.save()