def news(): """Show News Feed""" # Get articles articles = get_articles() # send articles for rendering return render_template("news.html", articles=articles)
if __name__ == '__main__': print("Starting...") # create client es = ElasticsearchClient('10.0.0.35') print("Connected") # set fetching corresponding time window from_ = '2021-02-18T00:00:00.000' to_ = '2021-02-18T23:59:00.000' timeframe = (from_, to_) # get articles print("Fetching articles...") articles = get_articles(es, timeframe) print('Total articles fetched: {}'.format(len(articles))) # transform fetched data to df data = pd.DataFrame(articles).rename(columns={0: 'datetime', 1: 'article'}) data = data.join(data['article'].apply(pd.Series)) cleaned_data = get_scientific(data) network = list() for index, row in cleaned_data.iterrows(): extracted_entities = extract_ne(row['text']) print('\tFor article with title: `{}` found {} entities'.format( row['title'], len(extracted_entities))) for entity in extracted_entities: pair = {
TRAIN_DEVICE = args.device else: TRAIN_DEVICE = DEFAULT_TRAIN_DEVICE torch.manual_seed(SEED) device = torch.device(TRAIN_DEVICE) theme_folders = helpers.get_article_themes(ARTICLE_FOLDER) TEXT = data.Field(sequential=True, tokenize='spacy', batch_first=True) THEME = data.LabelField(batch_first=True, use_vocab=False) fields = {'text': TEXT} for theme in theme_folders: fields[theme] = THEME df = helpers.get_articles(ARTICLE_FOLDER, theme_folders) training_data = helpers.DataFrameDataset(df, fields) train_data, valid_data = training_data.split( split_ratio=0.9, random_state=random.seed(SEED)) #initialize glove embeddings TEXT.build_vocab(train_data, max_size=1000, vectors='glove.6B.200d') train_iter, val_iter = data.BucketIterator.splits( (train_data, valid_data), batch_sizes=(BATCH_SIZE, BATCH_SIZE), device=device, sort_key=lambda x: len(x.text), sort_within_batch=False,
def archives(request, tag=None): return render_to_response(request, 'ladypenh/archives.html', dict(theme_name=helpers.get_theme(helpers.today()), articles=helpers.get_articles(helpers.today(), tag), tags=helpers.get_tags()))