def latestNews():
try:
api = CryptoControlAPI(CRYPTO_NEWS_KEY)
return jsonify(api.getTopNews())
except Exception as e:
print(e)
response = ERR_RESP
return response
def latestCoinNews():
try:
api = CryptoControlAPI(CRYPTO_NEWS_KEY)
res = api.getTopNewsByCoin(request.form['name'])
return jsonify(res)
except Exception as e:
print(e)
response = ERR_RESP
return response
def cryptocontrol_scrape(entity):
with open('../scraping/api_key.json') as f:
api_key = json.load(f)['cryptocontrol']
# connect to CryptoControlAPI
api = CryptoControlAPI(api_key)
# connect to a self-hosted proxy server (to improve performance) that points to cryptocontrol.io
proxyApi = CryptoControlAPI(api_key, "http://cryptocontrol_proxy/api/v1/public")
# retrieve all recent articles from that entity
articles = api.getLatestNewsByCoin(entity)
# create data frame
column_names = ["date_time", "title", "excerpt", "domain", \
"article_url", "image_url", "hotness", "activity_hotness", "source_id"]
df = pd.DataFrame(columns = column_names)
for article in articles:
# retrieve id
source_id = article["_id"]
# retrieve date
date_string = article["publishedAt"][:-5]
date_time = datetime.strptime(date_string, "%Y-%m-%dT%H:%M:%S")
# retrive title and description
title = article["title"]
excerpt = article["description"].strip()
hotness = article["hotness"]
activity_hotness = article["activityHotness"]
# retrieve domain and urls
image_url = article["originalImageUrl"]
domain = article["source"]["name"]
article_url = article["url"]
# add information to dataframe
df = df.append({"date_time": date_time, "title": title, \
"excerpt": excerpt, "article_url": article_url, \
"image_url": image_url, "hotness": hotness, \
"activity_hotness": activity_hotness, \
"source_id": source_id
}, ignore_index=True)
return df
# print(cryptocontrol_scrape("ethereum"))
class cryptonews():
def __init__(self, key):
self.key = key
self.api = CryptoControlAPI(key)
self.proxyApi = CryptoControlAPI(
key, "http://cryptocontrol_proxy/api/v1/public")
def enableSentiment(self):
self.sentiment = True
def top_news(self, lang=None):
return self.api.getTopNews(language=lang)
def news_by_coin(self, coin, lang=None):
return self.api.getTopNewsByCoin(coin=coin, language=lang)
def get_top_tweets(self, coin, lang=None):
return self.api.getTopTweetsByCoin(coin=coin, language=lang)
def get_reddit(self, coin, lang=None):
return self.api.getLatestRedditPostsByCoin(coin=coin, language=lang)
def get_top_feed(self, coin, lang):
return self.api.getTopFeedByCoin(coin=coin, language=lang)
def get_latest(self, coin, lang):
return self.api.getLatestItemsByCoin(coin=coin, language=lang)
def get_coinDetails(self, coin, lang):
return self.api.getCoinDetails(coin=coin, language=lang)
from crypto_news_api import CryptoControlAPI
API_AUTH = 'b8d3767b03073fb38780eb1de50198ea'
# Connect to the CryptoControl API
api = CryptoControlAPI(API_AUTH)
# Connect to a self-hosted proxy server (to improve performance) that points to cryptocontrol.io
proxyApi = CryptoControlAPI(API_AUTH,
"http://cryptocontrol_proxy/api/v1/public")
infosSelect = {
'title': [],
'description': [],
'source': {
'favicon': []
},
'url': []
}
def latest_news(lang="po"):
data = api.getLatestNews(lang)
response = []
for index, article in enumerate(data):
info = {
'id': index + 1,
'title': article['title'],
'description': article['description'],
'date': article['publishedAt'],
'source': {
# optional 1
'''
Unitest:
1. It is easy to modulize and good for maintainance.
2. It is written in a class in the form of class xx_Test(unittest.TestCase), in which multiple functions are tested with the self.assertEqual function.
Two arguments are passed into the function: the expected output and the output returned by the function. Then the unittest module will return 1 out of 3 outputs:
OK, ERROR or FAIL to show the end state.
3. Just import unittest library.
'''
# optional 2
# Connect to the CryptoControl API
with open('section1/task3/api_key.json', mode='r') as key_file:
key2 = json.loads(key_file.read())['key2']
api = CryptoControlAPI(key2)
# Connect to a self-hosted proxy server (to improve performance) that points to cryptocontrol.io
proxyApi = CryptoControlAPI(key2, "http://cryptocontrol_proxy/api/v1/public")
# Get top news
print(pd.DataFrame(api.getTopNews()))\
# returned dataframe can be viewed fully in the csv file, same for the other dataframes
pd.DataFrame(api.getTopNews()).to_csv('section1/task3/topnews.csv', index=False)
# get latest russian news
print(pd.DataFrame(api.getLatestNews("ru")))
# # get top bitcoin news
print(pd.DataFrame(api.getTopNewsByCoin("bitcoin")))
class News:
def __init__(self, api_key):
self.language = 'en'
self.api = CryptoControlAPI(api_key)
def set_language(self, language):
# English (en default)
# Chinese/Mandarin 'cn'
# German 'de'
# Italian 'it'
# Japanese 'jp'
# Korean 'ko'
# Portuguese 'po'
# Russian 'ru'
# Spanish 'es'
self.language = language
def getTopNews(self):
# Get the top news articles.
self.data, self.formated = None, None
try:
self.data = self.api.getTopNews(language=self.language)
self.formated = self.format_news(self.data)
print(self.formated)
except Exception as e:
print(e)
def getLatestNews(self):
# Get the latest news articles.
self.data, self.formated = None, None
try:
self.data = self.api.getLatestNews(language=self.language)
self.formated = self.format_news(self.data)
print(self.formated)
except Exception as e:
print(e)
def getTopNewsByCoin(self, coin):
# Get the top news articles for a specific coin from the CryptoControl API.
self.data, self.formated = None, None
try:
self.data = self.api.getTopNewsByCoin(coin,language=self.language)
self.formated = self.format_news(self.data)
print(self.formated)
except Exception as e:
print(e)
def getTopNewsByCategory(self):
# Get news articles grouped by category.
self.data, self.formated = None, None
try:
self.data = self.api.getTopNewsByCategory(language=self.language)
except Exception as e:
print(e)
return
self.formated = pd.concat([self.format_news(self.data[i]) for i in self.data.keys()])
print(self.formated)
def getLatestNewsByCoin(self, coin):
# Get the latest news articles for a specific coin.
self.data, self.formated = None, None
try:
self.data = self.api.getLatestNewsByCoin(coin, language= self.language)
self.formated = self.format_news(self.data)
print(self.formated)
except Exception as e:
print(e)
def getTopNewsByCoinCategory(self, coin):
# Get news articles grouped by category for a specific coin.
self.data, self.formated = None, None
try:
self.data = self.api.getTopNewsByCoinCategory(coin, language=self.language)
self.formated = self.format_news(self.data)
print(self.formated)
except Exception as e:
print(e)
def getTopTweetsByCoin(self, coin):
# Get top tweets for a particular coin
self.tweets, self.formated = None, None
try:
self.tweets = self.api.getTopTweetsByCoin(coin,language=self.language)
self.formated = self.format_tweets(self.tweets)
print(self.formated)
except Exception as e:
print(e)
def getLatestTweetsByCoin(self, coin):
# Get latest tweets for a particular coin
self.tweets, self.formated = None, None
try:
self.tweets = self.api.getLatestTweetsByCoin(coin,language=self.language)
self.formated = self.format_tweets(self.tweets)
print(self.formated)
except Exception as e:
print(e)
def getTopRedditPostsByCoin(self, coin):
# Get top reddit posts for a particular coin
self.reddit, self.formated = None, None
try:
self.reddit = self.api.getTopRedditPostsByCoin(coin,language=self.language)
self.formated = self.format_reddit(self.reddit)
print(self.formated)
except Exception as e:
print(e)
def getTopFeedByCoin(self, coin):
# Get a combined feed (reddit/tweets/articles) for a particular coin sort by time
self.feed, self.formated = None, None
try:
self.feed = self.api.getTopFeedByCoin(coin,language=self.language)
self.formated = self.format_feed(self.feed)
print(self.formated)
except Exception as e:
print(e)
def getCoinDetails(self, coin):
try:
self.coin = self.api.getCoinDetails(coin,language=self.language)
print(pd.DataFrame.from_dict(self.coin,orient = 'index'))
except Exception as e:
print(e)
def export_csv(self):
# export formated data to csv
self.formated.to_csv('section1/task3/Formated_Data.csv', index = 0)
@staticmethod
def format_news(data):
news = pd.DataFrame(data)
news = news[['primaryCategory', 'coins', 'title',
'url', 'source','publishedAt']]
news['coins'] = [[i['tradingSymbol'] for i in news.coins[j]] for j in range(news.coins.size)]
news['source'] = [i['name'] for i in news.source]
return news
@staticmethod
def format_tweets(data):
tweets = pd.DataFrame(data)
tweets = tweets[['text', 'url', 'publishedAt','retweetCount', 'favoriteCount']]
return tweets
@staticmethod
def format_reddit(data):
reddit = pd.DataFrame(data)
reddit = reddit[['title', 'url', 'subreddit', 'publishedAt' , 'comments', 'upvotes']]
return reddit
def format_feed(self, data):
feed = pd.DataFrame(data)
article = self.format_news([x for x in feed.article.dropna()])
tweet = self.format_tweets([x for x in feed.tweet.dropna()])
reddit = self.format_reddit([x for x in feed.reddit.dropna()])
tweet = tweet.rename(columns={'text':'title'})
df = pd.concat([article,tweet,reddit]).sort_values('publishedAt',ascending=False)
df = df.drop('coins', axis = 1)
# df = df[['title','url','publishedAt']]
return df
def __init__(self, api_key):
self.language = 'en'
self.api = CryptoControlAPI(api_key)
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
from crypto_news_api import CryptoControlAPI
import pandas as pd
# Connect to the CryptoControl API
api = CryptoControlAPI(open("task3_api.txt", "r").read())
# Get top news
top_news = api.getTopNews()
df = pd.DataFrame(top_news)
df = df.drop([
'similarArticles', 'coins', 'source', 'description', 'originalImageUrl',
'url', 'thumbnail'
],
axis=1)
df.to_csv('top_news_data.csv', index=False)
# get latest russian news
ru = api.getLatestNews("ru")
df = pd.DataFrame(ru)
df = df.drop([
def __init__(self, key):
self.key = key
self.api = CryptoControlAPI(key)
self.proxyApi = CryptoControlAPI(
key, "http://cryptocontrol_proxy/api/v1/public")
def save_json(data, filename, save):
if save:
filepath = os.path.join(DATA_DIR, filename)
with open(filepath, 'w') as json_file:
json.dump(data, json_file, indent=4)
print('Data Saved')
with open('../../../api-keys/cryptocontrol-API-Key.json',
mode='r') as key_file:
key = json.loads(key_file.read())['key']
# Connect to the CryptoControl API
api = CryptoControlAPI(key)
# Connect to a self-hosted proxy server (to improve performance) that points to cryptocontrol.io
proxyApi = CryptoControlAPI(key, "http://cryptocontrol_proxy/api/v1/public")
# Get top news
top_news = api.getTopNews(language='cn')
top_news_filename = f'top-news-cn.json'
save_json(top_news, top_news_filename, save=True)
# get latest english news
latest_news = api.getLatestNews('en')
latest_news_filename = f'latest-news-en.json'
save_json(latest_news, latest_news_filename, save=True)
# get top bitcoin news
axis=0)
else:
formated_data = news[article_index]
except BaseException:
raise Exception('Data can not be formated')
return formated_data
with open(os.path.join(path, 'cryptocontrol_api.json'), mode='r') as key_file:
key = json.loads(key_file.read())['key']
# Connect to the CryptoControl API
api = CryptoControlAPI(key)
# Connect to a self-hosted proxy server (to improve performance) that
# points to cryptocontrol.io
proxyApi = CryptoControlAPI(key, "http://cryptocontrol_proxy/api/v1/public")
# Do not Enable the sentiment datapoints unless you're a premium user,
# because raise exception does not returns you the message of status 405
# but returns the message when status is not '200'
# Get top news
top_news = api.getTopNews()
tn = format_df(top_news)
save_j(top_news, 'top_news.json')
save_csv(tn, 'top_news.csv')
### get status
status = api.get_status(tx_hash=TX_HASH)
print(status)
### get receipt status
TX_HASH = '0x513c1ba0bebf66436b5fed86ab668452b7805593c05073eb2d51d3a52f480a76'
api = Transactions(api_key=key)
receipt_status = api.get_tx_receipt_status(tx_hash=TX_HASH)
print(receipt_status)
## 2. Explore CryptoControl data API
from crypto_news_api import CryptoControlAPI
api = CryptoControlAPI("c707fd08bf46e30a653fab28ff6eab1e")
proxyApi = CryptoControlAPI("c707fd08bf46e30a653fab28ff6eab1e", "http://cryptocontrol_proxy/api/v1/public")
# Get top news
tn = api.getTopNews()
formatted_tn = pd.DataFrame.from_dict(tn)
print(formatted_tn.head(5))
# get latest chinese news
cn = api.getLatestNews("cn")
formatted_cn = pd.DataFrame.from_dict(cn)
print(formatted_cn.head(5))
# get top bitcoin news
from urllib.parse import quote, quote_plus
from django.contrib.auth.decorators import login_required
import requests
url = 'https://api.coindesk.com/v1/bpi/currentprice.json'
from crypto_news_api import CryptoControlAPI
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from django.conf import settings
from django.core.mail import EmailMultiAlternatives
from django.template.loader import get_template
from django.core.mail import send_mail
api = CryptoControlAPI("ae09e411009fafddd814a7b142edc0a6")
proxyApi = CryptoControlAPI("ae09e411009fafddd814a7b142edc0a6", "http://cryptocontrol_proxy/api/v1/public")
a=0
total_list = []
def home(request):
x = requests.get('https://api.coindesk.com/v1/bpi/currentprice.json')
data = x.json()
dollarrate = data['bpi']['USD']['rate']
poundrate = data['bpi']['GBP']['rate']
eurorate = data['bpi']['EUR']['rate']
updated = data['time']['updated']
disclaimer = data['disclaimer']
news = api.getTopNewsByCoin("bitcoin")
newslist = []
for i in range(15):
image = news[i]['originalImageUrl']
# %%
# ***Important: still, need to install "crypto-news-api" package from PyPI first!
# first, activate cryptoalgowheel conda environment
# then: /anaconda3/envs/cryptoalgowheel/bin/pip install crypto-news-api
# %%
from crypto_news_api import CryptoControlAPI
import numpy as np
import pandas as pd
import json
from datetime import datetime
# %%
api = CryptoControlAPI("39b252536476db3482bab04acea666bc")
# %% [markdown]
# #### get top news (languages can be changed)
# %%
def top_news(api, language="en"):
topnews = api.getTopNews(language)
topnews = pd.DataFrame(topnews)
topnews.to_csv("./top_news.csv", index=False)
top_news(api)
# %% [markdown]
import json
import requests
from config import cryptocontrol_key
from crypto_news_api import CryptoControlAPI
api = CryptoControlAPI(cryptocontrol_key)
def get_crypto_control_top_news():
res = {}
topNews = api.getTopNews()
index = 0
for entry in topNews:
res[index] = {
"title": entry["title"],
"description": entry["description"],
"url": entry["url"],
"source": entry["sourceDomain"],
"coins": [coin["name"] for coin in entry["coins"]]
}
index += 1
return res
get_crypto_control_top_news()
def get_cryptopanic_latest(ticker):
url = 'https://cryptopanic.com/api/v1/posts/?auth_token=f21d6681cc7f5638d2ac3006377811ff433a4833¤cies={}&public=true'.format(
ticker)
from crypto_news_api import CryptoControlAPI
# Connect to the CryptoControl API
api = CryptoControlAPI("")
# # Connect to a self-hosted proxy server (to improve performance) that points to cryptocontrol.io
# proxyApi = CryptoControlAPI("", "http://cryptocontrol_proxy/api/v1/public")
# # Enable the sentiment datapoints
# api.enableSentiment()
# # Get top news
# print(api.getTopNews())
# # get latest russian news
# print(api.getLatestNews("ru"))
# get top bitcoin news
# print(api.getTopNewsByCoin("bitcoin"))
# # get top EOS tweets
# print(api.getTopTweetsByCoin("eos"))
# # get top Ripple reddit posts
# print(api.getLatestRedditPostsByCoin("ripple"))
# # get reddit/tweets/articles in a single combined feed for NEO
# print(api.getTopFeedByCoin("neo"))
# # get latest reddit/tweets/articles (seperated) for Litecoin
# print(api.getLatestItemsByCoin("litecoin"))
from crypto_news_api import CryptoControlAPI
import requests
import pandas as pd
import json
with open('./key.json') as f:
api_key = json.load(f)
api = CryptoControlAPI(api_key['key'])
proxyApi = CryptoControlAPI(api_key['key'],
"http://cryptocontrol_proxy/api/v1/public")
# Enable the sentiment datapoints
api.enableSentiment()
# All the news data are in News.csv
# Get top news
news = api.getTopNews()
df = pd.DataFrame(data=news)
df['type'] = 'Top News'
df.to_csv("./News.csv", encoding='utf-8', index=0)
# get latest russian news
ru = api.getLatestNews("ru")
dfr = pd.DataFrame(data=ru)
dfr['type'] = 'Russian News'
dfr.to_csv("./News.csv", encoding='utf-8', mode='a', header=0, index=0)
# get top bitcoin news
bitcoin = api.getTopNewsByCoin("bitcoin")
dfb = pd.DataFrame(data=bitcoin)
import preprocessor as p
from fbprophet import Prophet
import plotly.graph_objects as go
import dash
import dash_table as dt
import dash_core_components as dcc
import dash_html_components as html
from fbprophet import Prophet
import warnings
from crypto_news_api import CryptoControlAPI
css = ["https://stackpath.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css"]
app = dash.Dash(__name__, external_stylesheets=css)
warnings.simplefilter(action='ignore')
api_key_news = '2df3a15efa6332e41ba90795b9639d32'
api = CryptoControlAPI(api_key_news)
latestTweets = api.getLatestTweetsByCoin("bitcoin")
tweets = pd.DataFrame(latestTweets)
tweets = tweets[['publishedAt','text']]
tweets['publishedAt'] = pd.to_datetime(tweets['publishedAt'])
list1 = []
for i in tweets['publishedAt']:
i = i.strftime('%d-%m-%Y')
list1.append(i)
tweets['publishedAt'] = list1
tweets['publishedAt'] = pd.to_datetime(tweets['publishedAt'])
tweets.rename(columns = {'publishedAt' : 'Date'}, inplace=True)
tweets['cleaned'] = tweets.text.apply(p.clean)
from crypto_news_api import CryptoControlAPI
import pandas as pd
# Connect to the CryptoControl API
api = CryptoControlAPI("c570bf2c119d13e0cc9eb0b3d69d414d")
# Connect to a self-hosted proxy server (to improve performance) that points to cryptocontrol.io
proxyApi = CryptoControlAPI("c570bf2c119d13e0cc9eb0b3d69d414d",
"http://cryptocontrol_proxy/api/v1/public")
# Get top news
print(pd.DataFrame(api.getTopNews()))
# get latest russian news
print(pd.DataFrame(api.getLatestNews("ru")))
# get top bitcoin news
print(pd.DataFrame(api.getTopNewsByCoin("bitcoin")))
# get top EOS tweets
print(pd.DataFrame(api.getTopTweetsByCoin("eos")))
# get top Ripple reddit posts
print(pd.DataFrame(api.getLatestRedditPostsByCoin("ripple")))
# get reddit/tweets/articles in a single combined feed for NEO
print(api.getTopFeedByCoin("neo"))
# get latest reddit/tweets/articles (seperated) for Litecoin
print(api.getLatestItemsByCoin("litecoin"))
# get details (subreddits, twitter handles, description, links) for ethereum