def cryptoInfoToDf(varCurrency='lisk',
varFromDate='2018-01-01',
varToDate='2018-12-31'):
# def cryptoInfoToDf (varCurrency = 'lisk', varFromDate = '04-01-2017',
# varToDate = '16-01-2017'):
"""
variables:
varCurrency - string value of currency shortcut
varFromDate - date in format of YYYY-MM-DD
varToDate - date in format of YYYY-MM-DD
returns:
dataframe of cryptocurrency data
created by connecting to Coinmarketcap.com
"""
cryptoDict = {
'lisk': 'LSK',
'bitcoin': 'BTC',
'ripple': 'XRP',
'ethereum': 'ETH',
'litecoin': 'LTC'
}
cryptoName = cryptoDict[varCurrency]
fromDate = datetime.datetime.strptime(varFromDate,
'%Y-%m-%d').strftime('%d-%m-%Y')
toDate = datetime.datetime.strptime(varToDate,
'%Y-%m-%d').strftime('%d-%m-%Y')
# cryptoDict = {
# 'LSK':'lisk',
# 'BTC':'bitcoin',
# 'XRP':'ripple',
# 'ETH':'ethereum',
# 'LTC':'litecoin'
# }
# ? initialise scraper
scraper = CmcScraper(cryptoName, fromDate, toDate)
# ? initialise dataframe for the data
headers, data = scraper.get_data()
df = pd.DataFrame(data, columns=headers)
df['Date'] = pd.to_datetime(df['Date'], format="%d-%m-%Y")
df['Currency'] = '{}'.format(varCurrency)
df.columns = [
'Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Market Cap',
'Currency'
]
return df
def __init__(self, coin_code, coin_id=None, start_date=None, end_date=None, all_time=False):
"""
:param coin_code: coin code of cryptocurrency e.g. btc
:param coin_id: default None, somtimes has more than one coin, like:
ACC has three coins: adcoin, accelerator-network, acchain, with different ids
:param start_date: date since when to scrape data (in the format of dd-mm-yyyy)
:param end_date: date to which scrape the data (in the format of dd-mm-yyyy)
:param all_time: 'True' if need data of all time for respective cryptocurrency
"""
CmcScraper.__init__(self, coin_code, start_date, end_date, all_time)
self.coin_id = coin_id
def gatherHistoricalDataFor(coin, start_date, end_date):
scraper = CmcScraper(coin['symbol'], start_date.strftime("%d-%m-%Y"),
end_date.strftime("%d-%m-%Y"))
# get raw data as list of list
headers, historicaldata = scraper.get_data()
# per ora non mi interessa più salvare il valore di Coin, contiene info che non mi servono nello spread. Ma devo mantenerla per far funzionare il resto e il pregresso già salvato su Google Spreadsheet
headers.insert(0, "Coin")
for historicaldata_row in historicaldata:
historicaldata_row.insert(0, "N/A")
return headers, historicaldata
def GetAllCoinData(day_to_buy):
coins = ScrapeCoinMarketCap()
all_coin_data = []
for coin in coins[:100]:
df = CmcScraper(coin).get_dataframe()
df['day_of_week'] = pd.to_datetime(df['Date']).dt.weekday_name
df['symbol'] = coin
df['prior_date_close'] = df['Close**'].shift(-7)
df['percentChange'] = df['Close**'] / df['Close**'].shift(-7) - 1.0
df = df.drop(labels={'Open*', 'High', 'Low', 'Volume'}, axis=1)
df = df[df['day_of_week'] == day_to_buy]
all_coin_data.append(df)
appended_data = pd.concat(all_coin_data, axis=0)
return appended_data
def get_latest_data(coin_list):
"""Get latest price data for given coin list."""
li = coin_list.split()
ret = dict()
for i, x in enumerate(li):
scr = CmcScraper(x)
try:
header, data = scr.get_data()
ti = list()
pr = list()
for j, xx in enumerate(data):
if xx[4]:
pr.append(xx[4])
ti.append(xx[0])
ret[x] = dict(t=ti, p=pr)
except Exception:
print("Invalid coin symbol: {0}".format(x))
print(i, x)
return ret
def get_history_all():
"""Get all history close data."""
cmc = Market()
listing = [item['symbol'] for item in cmc.listings()['data']]
ret = dict()
for i, x in enumerate(listing):
scr = CmcScraper(x)
try:
header, data = scr.get_data()
ti = list()
pr = list()
for j, xx in enumerate(data):
if xx[4]:
pr.append(xx[4])
ti.append(xx[0])
ret[x] = dict(t=ti, p=pr)
except Exception:
print("Invalid coin symbol: {0}".format(x))
print(i, x)
return ret
def retrieve_data(self, ticker):
# initialise scraper without time interval
scraper = CmcScraper(ticker)
# get raw data as list of list
headers, data = scraper.get_data()
# get data in a json format
xrp_json_data = scraper.get_data("json")
# export the data as csv file
scraper.export("csv", name=ticker)
def update(event):
messagebox.showinfo('info',
"""更新历史数据耗时较久，视网络环境需时20到40分钟，请耐心等待""")
cmc = Market()
listing = [item['symbol'] for item in cmc.listings()['data']]
hist = dict()
step = 100 / len(listing)
for i, x in enumerate(listing):
scr = CmcScraper(x)
try:
header, data = scr.get_data()
ti = list()
pr = list()
for j, xx in enumerate(data):
if xx[4]:
pr.append(xx[4])
ti.append(xx[0])
hist[x] = dict(
t=ti,
p=pr
)
except Exception:
print("Invalid coin symbol: {0}".format(x))
print(i, x)
progress.step(step)
progress.update()
sym2id, id2sym = get_id_table()
outdata = dict(
history=hist,
s2i=sym2id,
i2s=id2sym
)
with open("cmc_history.json", "w") as f:
json.dump(outdata, f)
update_date()
messagebox.showinfo('info', "成功更新历史数据")
def main():
global currencyList
n = 0
curDay = strftime("%d-%m-%Y", gmtime())
l = len(currencyList)
#l = 1
while n < l:
# initialise scraper
scraper = CmcScraper(currencyList[n], '01-01-2000', curDay)
# # get data as list of list
headers, data = scraper.get_data()
# # export the data to csv
scraper.export_csv('history/'+currencyList[n]+".csv")
# # get dataframe for the data
df = scraper.get_dataframe()
x = []
y = []
with open('history/'+currencyList[n]+'.csv', 'r') as f:
sreader = csv.reader(f)
i = 0
for row in sreader:
if i > 0 :
x.append(i)
y.append((float(row[2]) + float(row[3]))/2)
i = i + 1
prediction = predict(y,currencyList[n])
print currencyList[n] + ' => ' + str(prediction)
y.reverse()
data = {}
data["pointStart"] = 1230764400000
data["pointInterval"] = 3600000
data["dataLength"] = len(y)
data["data"] = y
data["prediction"] = prediction
json_data = json.dumps(data)
#print json_data
file = open('public/'+currencyList[n]+'.json', 'w')
file.write(json_data)
file.close()
n = n+1
from cryptocmd import CmcScraper
# initialise scraper
scraper = CmcScraper('XRP', '01-01-2015', '30-10-2018')
# get data as list of list
headers, data = scraper.get_data()
# export the data to csv
scraper.export_csv()
# get dataframe for the data
df = scraper.get_dataframe()
import streamlit as st
from cryptocmd import CmcScraper
import plotly.express as px
st.write("# 비트코인 BTC 데이터")
# CmcScraper 사용시 날짜 입력 주의 : '%d-%m-%Y
scraper = CmcScraper('BTC', '01-01-2021', '07-01-2021')
df = scraper.get_dataframe()
fig_close = px.line(df,
x="Date",
y=["Open", "High", "Low", "Close"],
title="가격")
fig_volume = px.line(df, x="Date", y=["Volume"], title="거래량")
st.plotly_chart(fig_close)
st.plotly_chart(fig_volume)
from cryptocmd import CmcScraper
# initialise scraper without passing time interval
scraper = CmcScraper('XRP')
# data as list of list in a variable
headers, data = scraper.get_data()
# export the data as csv file, you can also pass optional name parameter
scraper.export_csv('xrp_all_time.csv')
# Pandas dataFrame for the same data
df = scraper.get_dataframe()
print(df)
from cryptocmd import CmcScraper
scraper = CmcScraper('bch', '01-01-2012', '14-08-2018')
scraper.export_csv('bchusd.csv')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from cryptocmd import CmcScraper
# Initialise scraper with 'coin code' of cryptocurrency
# If time interval is not passed all time data will be scrapped
scraper = CmcScraper('btc')
# You can pass name for the csv explicitly,
# Else it will be named in format {coin_code}_{start_date}_{end_date}.csv
scraper.export_csv('btc_all_time.csv')
import streamlit as st
from cryptocmd import CmcScraper
import plotly.express as px
st.write('# 비트코인 BTC 데이터')
scrppper = CmcScraper('BTC', '01-01-2021', '07-01-2021') # %d-%m-%Y
df = scrppper.get_dataframe()
fig_close = px.line(df, x='Date', y=['Open', 'High', 'Low', 'Close'], title='가격')
fig_volume = px.line(df, x='Date', y=['Volume'], title='거래량')
st.plotly_chart(fig_close)
st.plotly_chart(fig_volume)
def get_historical_quotes(self) -> pd.DataFrame:
scraper = CmcScraper(self.ticker)
df = scraper.get_dataframe()
return pd.DataFrame({'_id' : 'null', 'Ticker' : self.ticker, 'TimeStampID' :
df['Date'], 'Price_USD' : df['Close'], 'Price_BTC' : 'null',
'MarketCap_USD' : df['Market Cap'], 'Volume24hr_USD' : df['Volume']})
def get_market(coin='BTC'):
scraper = CmcScraper('BTC', '06-03-2018', '30-05-2018')
headers, data = scraper.get_data()
scraper.export_csv(
csv_path='/home/cedric/Documents/UM/Info_mining/twitter_stream/market')
# Following methods are available to get data in multiple formats from https://coinmarketcap.com
from cryptocmd import CmcScraper
# initialise scraper without passing time interval
scraper = CmcScraper("XRP")
# data as list of list in a variable
headers, data = scraper.get_data()
# export the data as csv file, you can also pass optional name parameter
scraper.export_csv("xrp_all_time.csv")
# Pandas dataFrame for the same data
df = scraper.get_dataframe()
from cryptocmd import CmcScraper
# initialise scraper
scraper = CmcScraper('BTC', '15-07-2012', '01-08-2018')
# get data as list of list
headers, data = scraper.get_data()
# export the data to csv
scraper.export_csv("test.csv")
# get dataframe for the data
df = scraper.get_dataframe()
# currency = ["ETH","XRP","BCH","EOS"]
extractedData = []
def plotGraph(df):
for index,x in enumerate(df):
plt.plot(x['Date'],x['close'],label=currency[index])
plt.title("CLOSE")
plt.ylabel('close ($)')
plt.xlabel('Date')
plt.legend()
plt.show()
for x in currency:
scraper = CmcScraper(x, '1-1-2014', '18-6-2018')
df = scraper.get_dataframe()
df["Close**"].replace(0,np.nan,inplace=True)
df["close"] = df["Close**"]
df['returns'] = (df['close'].pct_change() + 1).fillna(1)
df = df.loc[:,['Date','close','returns']]
df = df.dropna()
# plotGraph(df)
print(x)
print(df.head())
df.to_pickle(fileName.format(x))
extractedData.append(df)
plotGraph(extractedData)
from cryptocmd import CmcScraper
# Documentation https://github.com/guptarohit/cryptoCMD
# initialise scraper without time interval
scraper = CmcScraper("BTC")
# get raw data as list of list
headers, data = scraper.get_data()
# get data in a json format
xrp_json_data = scraper.get_data("json")
# export the data as csv file, you can also pass optional `name` parameter
scraper.export("csv", name="xrp_all_time")
# Pandas dataFrame for the same data
df = scraper.get_dataframe()
result = xrp_json_data[0]
print(data[0][1])
#! /usr/bin/env python # -*- coding:utf-8 -*- """ @author : MG @Time : 2018/8/1 8:31 @File : only_test.py @contact : [email protected] @desc : """ from cryptocmd import CmcScraper # initialise scraper without passing time interval coin_code = 'ACC' scraper = CmcScraper(coin_code) # data as list of list in a variable headers, data = scraper.get_data() # export the data as csv file, you can also pass optional name parameter scraper.export_csv('{coin_code}_all_time.csv'.format(coin_code=coin_code)) # Pandas dataFrame for the same data df = scraper.get_dataframe()
def get_historical_data(crypto, first_day, last_day):
scraper = CmcScraper(crypto, first_day, last_day)
historical_data_df = scraper.get_dataframe().set_index("Date")
return historical_data_df
def download_data_scrapper(from_symbol):
scraper = CmcScraper(from_symbol)
data = scraper.get_dataframe()
return data
import matplotlib.pyplot as plt
from google.colab import files
uploaded = files.upload()
import io
data = io.BytesIO(uploaded['Blockchain.csv'])
f= pd.read_csv(data)
print(f)
!pip install cryptocmd
from cryptocmd import CmcScraper
# initialise scraper
scraper1 = CmcScraper('XRP', '08-08-2015', '12-04-2018')
scraper2 = CmcScraper('ETH', '08-08-2015', '12-04-2018')
scraper3 = CmcScraper('DASH', '08-08-2015', '12-04-2018')
scraper4 = CmcScraper('LTC', '08-08-2015', '12-04-2018')
scraper5 = CmcScraper('XLM', '08-08-2015', '12-04-2018')
# get dataframe for the data
df1 = scraper1.get_dataframe()
df2 = scraper2.get_dataframe()
df3 = scraper3.get_dataframe()
df4 = scraper4.get_dataframe()
df5 = scraper5.get_dataframe()
Y = f['BCHAIN-MKPRU'][0:978]
X = pd.concat([df1['Close'], df2['Close'], df3['Close'] ,df4['Close'], df5['Close'],f[0:979]], axis=1)
X= X[1:]
import streamlit as st
from cryptocmd import CmcScraper
import plotly.express as px
from datetime import datetime
st.write('#Cryptocurrency Web App')
st.sidebar.header('Menu')
name = st.sidebar.selectbox('Name', ['BTC', 'ETH', 'USDT'])
start_date = st.sidebar.date_input('start date', datetime(2021, 1, 1))
end_date = st.sidebar.date_input('end date', datetime(2021, 1, 25))
scrapper = CmcScraper(name, start_date.strftime('%d-%m-%Y'),
end_date.strftime('%d-%m-%Y'))
df = scrapper.get_dataframe()
fig_close = px.line(df,
x='Date',
y=['Open', 'High', 'Low', 'Close'],
title='Price')
fig_volume = px.line(df, x='Date', y=['Volume'], title='Volume')
st.plotly_chart(fig_close)
st.plotly_chart(fig_volume)
def updateCoins(coinCodes):
"""Update the H5 Repository with Coin Data
Keyword arguments:
coinList - Set of CoinCodes
"""
global apph5datafile
coinList = []
if coinCodes == '*':
print("You asked to update all tracked coins")
coinList = config.get('Active', 'Coins').split(' ')
else:
coinList = coinCodes
print("You asked to list the last week for coins %s" % (coinList))
print("You asked to update coins %s" % (coinList))
for coin in coinList:
havecointable = True
if not os.path.exists(apph5datafile):
print("Creating Datafile %s" % apph5datafile)
havecointable = False
# if not ('/'+coin.upper()) in storedcoins:
# print("Coin %s not in dataset" % coin.upper())
# havecointable = False
if havecointable:
try:
hdf = read_hdf(apph5datafile, coin)
hdf['date_ranked'] = hdf['Date'].rank(ascending=1)
print "Last element from %s is dated %s" % (
coin, str(hdf.at[0, 'Date'])[:10])
lastdate = datetime.strptime(
str(hdf.at[0, 'Date'])[:10], "%Y-%m-%d")
nextday = timedelta(1)
nextday = lastdate + nextday
delta = (datetime.today() - timedelta(1)) - nextday
print("Difference from yesterday to last update ", delta)
except KeyError:
print("No data found for this Coin")
nextday = (datetime.today() - timedelta(2 * 365))
lastdate = datetime.today() - timedelta(1)
print "Updating data from %s to %s" % (str(nextday), str(lastdate))
startdate = '{:%d-%m-%Y}'.format(nextday)
enddate = '{:%d-%m-%Y}'.format(lastdate)
scraper = CmcScraper(coin, startdate, enddate)
else:
scraper = CmcScraper(coin.upper())
# get data as list of list
try:
headers, data = scraper.get_data()
# get dataframe for the data
df = scraper.get_dataframe()
print df
# df.to_hdf('pluton.h5',coin,append=True,format='t')
df.to_hdf(apph5datafile, coin.upper())
except ValueError:
print("No data found")
return
return min_date, max_date
data_load_state = st.sidebar.text('Loading data...')
min_date, max_date = load_data(selected_ticker)
data_load_state.text('Loading data... done!')
### Select date range
date_range = st.sidebar.selectbox(
"Select the timeframe to train the model on:",
options=["All available data", "Specific date range"])
if date_range == "All available data":
### Initialise scraper without time interval
scraper = CmcScraper(selected_ticker)
elif date_range == "Specific date range":
### Initialise scraper with time interval
start_date = st.sidebar.date_input('Select start date:',
min_value=min_date,
max_value=max_date,
value=min_date)
end_date = st.sidebar.date_input('Select end date:',
min_value=min_date,
max_value=max_date,
value=max_date)
scraper = CmcScraper(selected_ticker, str(start_date.strftime("%d-%m-%Y")),
str(end_date.strftime("%d-%m-%Y")))
def load_data(selected_ticker):
init_scraper = CmcScraper(selected_ticker)
df = init_scraper.get_dataframe()
min_date = pd.to_datetime(min(df['Date']))
max_date = pd.to_datetime(max(df['Date']))
return min_date, max_date
def download(self,coin_data_id):
fetcher = CmcScraper(coin_data_id)#(coin['symbol'])
headers, data = fetcher.get_data()
Controller.prepDataframe(self,headers,data)
import os
#os.system('pip install git+git://github.com/guptarohit/cryptoCMD.git')
from cryptocmd import CmcScraper
crypto_name = os.sys.argv[1]
scraper = CmcScraper(crypto_name)
headers, data = scraper.get_data()
scraper.export_csv(crypto_name+'_all_time.csv')
