class CoinMarketCap:
def __init__(self, api_key=None):
self.api = CoinMarketCapAPI(api_key or COIN_MARKET_CAP_API_KEY)
@staticmethod
def filter_essential_data(data, coin, currency):
return {
"source":
"Coin Market Cap",
"symbol":
data[coin]["symbol"],
"name":
data[coin]["name"],
"total_supply":
data[coin]["total_supply"],
"value":
f"{data[coin]['quote'][currency]['price']}",
"currency":
currency,
"percent_change_24h":
data[coin]["quote"][currency]["percent_change_24h"],
"market_cap":
data[coin]["quote"][currency]["market_cap"],
}
def price(self, coin, currency):
result = self.api.cryptocurrency_quotes_latest(symbol=coin,
convert=currency)
return self.filter_essential_data(result.data,
coin=coin,
currency=currency)
class Crypto:
def __init__(self, api_key=None):
self.manager = CoinMarketCapAPI(api_key or COIN_MARKET_CAP_API_KEY)
@staticmethod
def filter_essential_data(data, coin, currency):
return {
'symbol':
data[coin]["symbol"],
'name':
data[coin]["name"],
'total_supply':
data[coin]["total_supply"],
'price':
f"{data[coin]['quote'][currency]['price']} {currency}",
'percent_change_24h':
data[coin]["quote"][currency]["percent_change_24h"],
'market_cap':
data[coin]["quote"][currency]["market_cap"]
}
def price(self, coin, currency):
result = self.manager.cryptocurrency_quotes_latest(symbol=coin,
convert=currency)
return self.filter_essential_data(result.data,
coin=coin,
currency=currency)
def insertNewCryptoData():
# Read secret file
dir_path = os.path.dirname(os.path.realpath(__file__))
with open(dir_path + '/secrets.txt') as f:
secret = f.readlines()
secrets = [x.strip() for x in secret]
client_id = secrets[0]
client_secret = secrets[1]
username = secrets[2]
password = secrets[3]
api_key_1 = secrets[4]
api_key_2 = secrets[5]
api_key_3 = secrets[6]
api_key_4 = secrets[7]
coinmarketcapAPI = [api_key_1, api_key_2, api_key_3, api_key_4]
# Connect to CoinMarketCapAPI
coinmarketcap = CoinMarketCapAPI(coinmarketcapAPI[0])
crypto_listings = coinmarketcap.cryptocurrency_map().data
cur_date = datetime.utcnow()
crypto_list = []
crypto_dict = {}
crypto_slug_dict = {}
crypto_slug_names = {}
crypto_link_count_dict = {}
crypto_link_dict = {}
# exclude certain crypto
crypto_exclude_dict = {
'A': 0,
'CAP': 0,
'TOP': 0,
'OK': 0,
'ANY': 0,
'CAN': 0,
'JST': 0
}
for crypto_name in crypto_listings:
crypto_symbol = crypto_name["symbol"]
crypto_slug = crypto_name["slug"]
if crypto_symbol.isalnum(
) and not crypto_symbol in crypto_exclude_dict:
crypto_list.append(crypto_symbol)
crypto_slug_dict[crypto_slug.lower()] = crypto_symbol
crypto_slug_names[crypto_symbol] = crypto_slug
crypto_dict[crypto_symbol] = 0
crypto_link_count_dict[crypto_symbol] = 0
crypto_link_dict[crypto_symbol] = ['', '']
# Perform web scraping
scrape_period = cur_date - timedelta(hours=6)
# Connect to reddit API
print("Scraping the web...")
reddit = praw.Reddit(client_id='Vht-J5m9WtgOVQ',
client_secret=client_secret,
username=username,
password=password,
user_agent='crypto_scraper')
dir_path = os.path.dirname(os.path.realpath(__file__))
with open(dir_path + '/subreddits.txt') as f:
content = f.readlines()
subreddit_list = [x.strip() for x in content]
# Obtain the string of all relevant posts
post_list = []
for subreddit_name in subreddit_list:
subreddit = reddit.subreddit(subreddit_name)
hot_topics = subreddit.hot(limit=1000)
for submissions in hot_topics:
if datetime.utcfromtimestamp(
submissions.created_utc) >= scrape_period:
post_list.append([
submissions.title, submissions.score, submissions.permalink
])
# Count mentions of each cryptos in posts
for submission in post_list:
topics = submission[0]
upvotes = submission[1]
url = 'https://www.reddit.com/' + str(submission[2])
word_list = topics.split()
for words in word_list:
# strip words to alphanumeric (remove special characters)
if not words.isalnum():
words = ''.join(c for c in words if c.isalnum())
if words in crypto_dict:
crypto_dict[words] += upvotes
if upvotes >= crypto_link_count_dict[words]:
crypto_link_count_dict[words] = upvotes
crypto_link_dict[words] = [url, topics]
elif words.lower() in crypto_slug_dict:
cor_symbol = crypto_slug_dict[words.lower()]
crypto_dict[cor_symbol] += upvotes
if upvotes >= crypto_link_count_dict[cor_symbol]:
crypto_link_count_dict[cor_symbol] = upvotes
crypto_link_dict[cor_symbol] = [url, topics]
# Update the price and market cap for each crypto symbol
print("Inserting Crypto Data...")
i = 0
api_counter = 0
while i < len(crypto_list):
if i + 1000 >= len(crypto_list):
crypto_query_str = crypto_list[i:len(crypto_list)]
i = len(crypto_list)
else:
crypto_query_str = crypto_list[i:i + 1000]
i += 1000
coinmarketcap = CoinMarketCapAPI(coinmarketcapAPI[api_counter])
crypto_quotes = coinmarketcap.cryptocurrency_quotes_latest(
symbol=str(','.join(crypto_query_str))).data
api_counter += 1
if api_counter >= len(coinmarketcapAPI):
api_counter = 0
for crypto_symbol in crypto_query_str:
try:
cur_price = round(
crypto_quotes[crypto_symbol]['quote']['USD']['price'], 9)
cur_marketcap = round(
crypto_quotes[crypto_symbol]['quote']['USD']['market_cap'],
9)
cur_vol = round(
crypto_quotes[crypto_symbol]['quote']['USD']['volume_24h'],
9)
cur_percent_change_24h = round(
crypto_quotes[crypto_symbol]['quote']['USD']
['percent_change_24h'], 5)
cur_percent_change_1h = round(
crypto_quotes[crypto_symbol]['quote']['USD']
['percent_change_1h'], 5)
cur_count = crypto_dict[crypto_symbol]
curCryptoDatabase = CryptoDatabase(
symbol=crypto_symbol,
slug=crypto_slug_names[crypto_symbol],
date=cur_date,
source='reddit',
count=cur_count,
popular_link=crypto_link_dict[crypto_symbol][0],
popular_content=crypto_link_dict[crypto_symbol][1],
price=cur_price,
marketcap=cur_marketcap,
volume_24h=cur_vol,
percent_change_24h=cur_percent_change_24h,
percent_change_1h=cur_percent_change_1h)
# Save database
curCryptoDatabase.save()
except:
print("invalid coin: ", crypto_symbol)
return