def retrieve_data(self, ticker): # initialise scraper without time interval scraper = CmcScraper(ticker) # get raw data as list of list headers, data = scraper.get_data() # get data in a json format xrp_json_data = scraper.get_data("json") # export the data as csv file scraper.export("csv", name=ticker)
def cryptoInfoToDf(varCurrency='lisk', varFromDate='2018-01-01', varToDate='2018-12-31'): # def cryptoInfoToDf (varCurrency = 'lisk', varFromDate = '04-01-2017', # varToDate = '16-01-2017'): """ variables: varCurrency - string value of currency shortcut varFromDate - date in format of YYYY-MM-DD varToDate - date in format of YYYY-MM-DD returns: dataframe of cryptocurrency data created by connecting to Coinmarketcap.com """ cryptoDict = { 'lisk': 'LSK', 'bitcoin': 'BTC', 'ripple': 'XRP', 'ethereum': 'ETH', 'litecoin': 'LTC' } cryptoName = cryptoDict[varCurrency] fromDate = datetime.datetime.strptime(varFromDate, '%Y-%m-%d').strftime('%d-%m-%Y') toDate = datetime.datetime.strptime(varToDate, '%Y-%m-%d').strftime('%d-%m-%Y') # cryptoDict = { # 'LSK':'lisk', # 'BTC':'bitcoin', # 'XRP':'ripple', # 'ETH':'ethereum', # 'LTC':'litecoin' # } # ? initialise scraper scraper = CmcScraper(cryptoName, fromDate, toDate) # ? initialise dataframe for the data headers, data = scraper.get_data() df = pd.DataFrame(data, columns=headers) df['Date'] = pd.to_datetime(df['Date'], format="%d-%m-%Y") df['Currency'] = '{}'.format(varCurrency) df.columns = [ 'Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Market Cap', 'Currency' ] return df
def main(): global currencyList n = 0 curDay = strftime("%d-%m-%Y", gmtime()) l = len(currencyList) #l = 1 while n < l: # initialise scraper scraper = CmcScraper(currencyList[n], '01-01-2000', curDay) # # get data as list of list headers, data = scraper.get_data() # # export the data to csv scraper.export_csv('history/'+currencyList[n]+".csv") # # get dataframe for the data df = scraper.get_dataframe() x = [] y = [] with open('history/'+currencyList[n]+'.csv', 'r') as f: sreader = csv.reader(f) i = 0 for row in sreader: if i > 0 : x.append(i) y.append((float(row[2]) + float(row[3]))/2) i = i + 1 prediction = predict(y,currencyList[n]) print currencyList[n] + ' => ' + str(prediction) y.reverse() data = {} data["pointStart"] = 1230764400000 data["pointInterval"] = 3600000 data["dataLength"] = len(y) data["data"] = y data["prediction"] = prediction json_data = json.dumps(data) #print json_data file = open('public/'+currencyList[n]+'.json', 'w') file.write(json_data) file.close() n = n+1
def gatherHistoricalDataFor(coin, start_date, end_date): scraper = CmcScraper(coin['symbol'], start_date.strftime("%d-%m-%Y"), end_date.strftime("%d-%m-%Y")) # get raw data as list of list headers, historicaldata = scraper.get_data() # per ora non mi interessa più salvare il valore di Coin, contiene info che non mi servono nello spread. Ma devo mantenerla per far funzionare il resto e il pregresso già salvato su Google Spreadsheet headers.insert(0, "Coin") for historicaldata_row in historicaldata: historicaldata_row.insert(0, "N/A") return headers, historicaldata
def GetAllCoinData(day_to_buy): coins = ScrapeCoinMarketCap() all_coin_data = [] for coin in coins[:100]: df = CmcScraper(coin).get_dataframe() df['day_of_week'] = pd.to_datetime(df['Date']).dt.weekday_name df['symbol'] = coin df['prior_date_close'] = df['Close**'].shift(-7) df['percentChange'] = df['Close**'] / df['Close**'].shift(-7) - 1.0 df = df.drop(labels={'Open*', 'High', 'Low', 'Volume'}, axis=1) df = df[df['day_of_week'] == day_to_buy] all_coin_data.append(df) appended_data = pd.concat(all_coin_data, axis=0) return appended_data
def get_latest_data(coin_list): """Get latest price data for given coin list.""" li = coin_list.split() ret = dict() for i, x in enumerate(li): scr = CmcScraper(x) try: header, data = scr.get_data() ti = list() pr = list() for j, xx in enumerate(data): if xx[4]: pr.append(xx[4]) ti.append(xx[0]) ret[x] = dict(t=ti, p=pr) except Exception: print("Invalid coin symbol: {0}".format(x)) print(i, x) return ret
def get_history_all(): """Get all history close data.""" cmc = Market() listing = [item['symbol'] for item in cmc.listings()['data']] ret = dict() for i, x in enumerate(listing): scr = CmcScraper(x) try: header, data = scr.get_data() ti = list() pr = list() for j, xx in enumerate(data): if xx[4]: pr.append(xx[4]) ti.append(xx[0]) ret[x] = dict(t=ti, p=pr) except Exception: print("Invalid coin symbol: {0}".format(x)) print(i, x) return ret
def update(event): messagebox.showinfo('info', """更新历史数据耗时较久,视网络环境需时20到40分钟,请耐心等待""") cmc = Market() listing = [item['symbol'] for item in cmc.listings()['data']] hist = dict() step = 100 / len(listing) for i, x in enumerate(listing): scr = CmcScraper(x) try: header, data = scr.get_data() ti = list() pr = list() for j, xx in enumerate(data): if xx[4]: pr.append(xx[4]) ti.append(xx[0]) hist[x] = dict( t=ti, p=pr ) except Exception: print("Invalid coin symbol: {0}".format(x)) print(i, x) progress.step(step) progress.update() sym2id, id2sym = get_id_table() outdata = dict( history=hist, s2i=sym2id, i2s=id2sym ) with open("cmc_history.json", "w") as f: json.dump(outdata, f) update_date() messagebox.showinfo('info', "成功更新历史数据")
def get_market(coin='BTC'): scraper = CmcScraper('BTC', '06-03-2018', '30-05-2018') headers, data = scraper.get_data() scraper.export_csv( csv_path='/home/cedric/Documents/UM/Info_mining/twitter_stream/market')
from cryptocmd import CmcScraper # initialise scraper scraper = CmcScraper('XRP', '01-01-2015', '30-10-2018') # get data as list of list headers, data = scraper.get_data() # export the data to csv scraper.export_csv() # get dataframe for the data df = scraper.get_dataframe()
def get_historical_data(crypto, first_day, last_day): scraper = CmcScraper(crypto, first_day, last_day) historical_data_df = scraper.get_dataframe().set_index("Date") return historical_data_df
#! /usr/bin/env python # -*- coding:utf-8 -*- """ @author : MG @Time : 2018/8/1 8:31 @File : only_test.py @contact : [email protected] @desc : """ from cryptocmd import CmcScraper # initialise scraper without passing time interval coin_code = 'ACC' scraper = CmcScraper(coin_code) # data as list of list in a variable headers, data = scraper.get_data() # export the data as csv file, you can also pass optional name parameter scraper.export_csv('{coin_code}_all_time.csv'.format(coin_code=coin_code)) # Pandas dataFrame for the same data df = scraper.get_dataframe()
import streamlit as st from cryptocmd import CmcScraper import plotly.express as px st.write("# 비트코인 BTC 데이터") # CmcScraper 사용시 날짜 입력 주의 : '%d-%m-%Y scraper = CmcScraper('BTC', '01-01-2021', '07-01-2021') df = scraper.get_dataframe() fig_close = px.line(df, x="Date", y=["Open", "High", "Low", "Close"], title="가격") fig_volume = px.line(df, x="Date", y=["Volume"], title="거래량") st.plotly_chart(fig_close) st.plotly_chart(fig_volume)
import streamlit as st from cryptocmd import CmcScraper import plotly.express as px st.write('# 비트코인 BTC 데이터') scrppper = CmcScraper('BTC', '01-01-2021', '07-01-2021') # %d-%m-%Y df = scrppper.get_dataframe() fig_close = px.line(df, x='Date', y=['Open', 'High', 'Low', 'Close'], title='가격') fig_volume = px.line(df, x='Date', y=['Volume'], title='거래량') st.plotly_chart(fig_close) st.plotly_chart(fig_volume)
from cryptocmd import CmcScraper # initialise scraper scraper = CmcScraper('XRP', '15-10-2017', '25-10-2017') # get data as list of list headers, data = scraper.get_data() # export the data to csv scraper.export_csv('market-period.csv') # get dataframe for the data df = scraper.get_dataframe() print(df)
# Following methods are available to get data in multiple formats from https://coinmarketcap.com from cryptocmd import CmcScraper # initialise scraper without passing time interval scraper = CmcScraper("XRP") # data as list of list in a variable headers, data = scraper.get_data() # export the data as csv file, you can also pass optional name parameter scraper.export_csv("xrp_all_time.csv") # Pandas dataFrame for the same data df = scraper.get_dataframe()
def updateCoins(coinCodes): """Update the H5 Repository with Coin Data Keyword arguments: coinList - Set of CoinCodes """ global apph5datafile coinList = [] if coinCodes == '*': print("You asked to update all tracked coins") coinList = config.get('Active', 'Coins').split(' ') else: coinList = coinCodes print("You asked to list the last week for coins %s" % (coinList)) print("You asked to update coins %s" % (coinList)) for coin in coinList: havecointable = True if not os.path.exists(apph5datafile): print("Creating Datafile %s" % apph5datafile) havecointable = False # if not ('/'+coin.upper()) in storedcoins: # print("Coin %s not in dataset" % coin.upper()) # havecointable = False if havecointable: try: hdf = read_hdf(apph5datafile, coin) hdf['date_ranked'] = hdf['Date'].rank(ascending=1) print "Last element from %s is dated %s" % ( coin, str(hdf.at[0, 'Date'])[:10]) lastdate = datetime.strptime( str(hdf.at[0, 'Date'])[:10], "%Y-%m-%d") nextday = timedelta(1) nextday = lastdate + nextday delta = (datetime.today() - timedelta(1)) - nextday print("Difference from yesterday to last update ", delta) except KeyError: print("No data found for this Coin") nextday = (datetime.today() - timedelta(2 * 365)) lastdate = datetime.today() - timedelta(1) print "Updating data from %s to %s" % (str(nextday), str(lastdate)) startdate = '{:%d-%m-%Y}'.format(nextday) enddate = '{:%d-%m-%Y}'.format(lastdate) scraper = CmcScraper(coin, startdate, enddate) else: scraper = CmcScraper(coin.upper()) # get data as list of list try: headers, data = scraper.get_data() # get dataframe for the data df = scraper.get_dataframe() print df # df.to_hdf('pluton.h5',coin,append=True,format='t') df.to_hdf(apph5datafile, coin.upper()) except ValueError: print("No data found") return
import os #os.system('pip install git+git://github.com/guptarohit/cryptoCMD.git') from cryptocmd import CmcScraper crypto_name = os.sys.argv[1] scraper = CmcScraper(crypto_name) headers, data = scraper.get_data() scraper.export_csv(crypto_name+'_all_time.csv')
def download(self,coin_data_id): fetcher = CmcScraper(coin_data_id)#(coin['symbol']) headers, data = fetcher.get_data() Controller.prepDataframe(self,headers,data)
from cryptocmd import CmcScraper # initialise scraper scraper = CmcScraper('BTC', '15-07-2012', '01-08-2018') # get data as list of list headers, data = scraper.get_data() # export the data to csv scraper.export_csv("test.csv") # get dataframe for the data df = scraper.get_dataframe()
# currency = ["ETH","XRP","BCH","EOS"] extractedData = [] def plotGraph(df): for index,x in enumerate(df): plt.plot(x['Date'],x['close'],label=currency[index]) plt.title("CLOSE") plt.ylabel('close ($)') plt.xlabel('Date') plt.legend() plt.show() for x in currency: scraper = CmcScraper(x, '1-1-2014', '18-6-2018') df = scraper.get_dataframe() df["Close**"].replace(0,np.nan,inplace=True) df["close"] = df["Close**"] df['returns'] = (df['close'].pct_change() + 1).fillna(1) df = df.loc[:,['Date','close','returns']] df = df.dropna() # plotGraph(df) print(x) print(df.head()) df.to_pickle(fileName.format(x)) extractedData.append(df) plotGraph(extractedData)
from cryptocmd import CmcScraper # Documentation https://github.com/guptarohit/cryptoCMD # initialise scraper without time interval scraper = CmcScraper("BTC") # get raw data as list of list headers, data = scraper.get_data() # get data in a json format xrp_json_data = scraper.get_data("json") # export the data as csv file, you can also pass optional `name` parameter scraper.export("csv", name="xrp_all_time") # Pandas dataFrame for the same data df = scraper.get_dataframe() result = xrp_json_data[0] print(data[0][1])
def get_historical_quotes(self) -> pd.DataFrame: scraper = CmcScraper(self.ticker) df = scraper.get_dataframe() return pd.DataFrame({'_id' : 'null', 'Ticker' : self.ticker, 'TimeStampID' : df['Date'], 'Price_USD' : df['Close'], 'Price_BTC' : 'null', 'MarketCap_USD' : df['Market Cap'], 'Volume24hr_USD' : df['Volume']})
return min_date, max_date data_load_state = st.sidebar.text('Loading data...') min_date, max_date = load_data(selected_ticker) data_load_state.text('Loading data... done!') ### Select date range date_range = st.sidebar.selectbox( "Select the timeframe to train the model on:", options=["All available data", "Specific date range"]) if date_range == "All available data": ### Initialise scraper without time interval scraper = CmcScraper(selected_ticker) elif date_range == "Specific date range": ### Initialise scraper with time interval start_date = st.sidebar.date_input('Select start date:', min_value=min_date, max_value=max_date, value=min_date) end_date = st.sidebar.date_input('Select end date:', min_value=min_date, max_value=max_date, value=max_date) scraper = CmcScraper(selected_ticker, str(start_date.strftime("%d-%m-%Y")), str(end_date.strftime("%d-%m-%Y")))
import streamlit as st from cryptocmd import CmcScraper import plotly.express as px from datetime import datetime st.write('#Cryptocurrency Web App') st.sidebar.header('Menu') name = st.sidebar.selectbox('Name', ['BTC', 'ETH', 'USDT']) start_date = st.sidebar.date_input('start date', datetime(2021, 1, 1)) end_date = st.sidebar.date_input('end date', datetime(2021, 1, 25)) scrapper = CmcScraper(name, start_date.strftime('%d-%m-%Y'), end_date.strftime('%d-%m-%Y')) df = scrapper.get_dataframe() fig_close = px.line(df, x='Date', y=['Open', 'High', 'Low', 'Close'], title='Price') fig_volume = px.line(df, x='Date', y=['Volume'], title='Volume') st.plotly_chart(fig_close) st.plotly_chart(fig_volume)
def load_data(selected_ticker): init_scraper = CmcScraper(selected_ticker) df = init_scraper.get_dataframe() min_date = pd.to_datetime(min(df['Date'])) max_date = pd.to_datetime(max(df['Date'])) return min_date, max_date
def download_data_scrapper(from_symbol): scraper = CmcScraper(from_symbol) data = scraper.get_dataframe() return data
from cryptocmd import CmcScraper # initialise scraper without passing time interval scraper = CmcScraper('XRP') # data as list of list in a variable headers, data = scraper.get_data() # export the data as csv file, you can also pass optional name parameter scraper.export_csv('xrp_all_time.csv') # Pandas dataFrame for the same data df = scraper.get_dataframe() print(df)
import matplotlib.pyplot as plt from google.colab import files uploaded = files.upload() import io data = io.BytesIO(uploaded['Blockchain.csv']) f= pd.read_csv(data) print(f) !pip install cryptocmd from cryptocmd import CmcScraper # initialise scraper scraper1 = CmcScraper('XRP', '08-08-2015', '12-04-2018') scraper2 = CmcScraper('ETH', '08-08-2015', '12-04-2018') scraper3 = CmcScraper('DASH', '08-08-2015', '12-04-2018') scraper4 = CmcScraper('LTC', '08-08-2015', '12-04-2018') scraper5 = CmcScraper('XLM', '08-08-2015', '12-04-2018') # get dataframe for the data df1 = scraper1.get_dataframe() df2 = scraper2.get_dataframe() df3 = scraper3.get_dataframe() df4 = scraper4.get_dataframe() df5 = scraper5.get_dataframe() Y = f['BCHAIN-MKPRU'][0:978] X = pd.concat([df1['Close'], df2['Close'], df3['Close'] ,df4['Close'], df5['Close'],f[0:979]], axis=1) X= X[1:]
from cryptocmd import CmcScraper scraper = CmcScraper('bch', '01-01-2012', '14-08-2018') scraper.export_csv('bchusd.csv')