def retrieve_data(self, ticker): # initialise scraper without time interval scraper = CmcScraper(ticker) # get raw data as list of list headers, data = scraper.get_data() # get data in a json format xrp_json_data = scraper.get_data("json") # export the data as csv file scraper.export("csv", name=ticker)
def main(): global currencyList n = 0 curDay = strftime("%d-%m-%Y", gmtime()) l = len(currencyList) #l = 1 while n < l: # initialise scraper scraper = CmcScraper(currencyList[n], '01-01-2000', curDay) # # get data as list of list headers, data = scraper.get_data() # # export the data to csv scraper.export_csv('history/'+currencyList[n]+".csv") # # get dataframe for the data df = scraper.get_dataframe() x = [] y = [] with open('history/'+currencyList[n]+'.csv', 'r') as f: sreader = csv.reader(f) i = 0 for row in sreader: if i > 0 : x.append(i) y.append((float(row[2]) + float(row[3]))/2) i = i + 1 prediction = predict(y,currencyList[n]) print currencyList[n] + ' => ' + str(prediction) y.reverse() data = {} data["pointStart"] = 1230764400000 data["pointInterval"] = 3600000 data["dataLength"] = len(y) data["data"] = y data["prediction"] = prediction json_data = json.dumps(data) #print json_data file = open('public/'+currencyList[n]+'.json', 'w') file.write(json_data) file.close() n = n+1
def cryptoInfoToDf(varCurrency='lisk', varFromDate='2018-01-01', varToDate='2018-12-31'): # def cryptoInfoToDf (varCurrency = 'lisk', varFromDate = '04-01-2017', # varToDate = '16-01-2017'): """ variables: varCurrency - string value of currency shortcut varFromDate - date in format of YYYY-MM-DD varToDate - date in format of YYYY-MM-DD returns: dataframe of cryptocurrency data created by connecting to Coinmarketcap.com """ cryptoDict = { 'lisk': 'LSK', 'bitcoin': 'BTC', 'ripple': 'XRP', 'ethereum': 'ETH', 'litecoin': 'LTC' } cryptoName = cryptoDict[varCurrency] fromDate = datetime.datetime.strptime(varFromDate, '%Y-%m-%d').strftime('%d-%m-%Y') toDate = datetime.datetime.strptime(varToDate, '%Y-%m-%d').strftime('%d-%m-%Y') # cryptoDict = { # 'LSK':'lisk', # 'BTC':'bitcoin', # 'XRP':'ripple', # 'ETH':'ethereum', # 'LTC':'litecoin' # } # ? initialise scraper scraper = CmcScraper(cryptoName, fromDate, toDate) # ? initialise dataframe for the data headers, data = scraper.get_data() df = pd.DataFrame(data, columns=headers) df['Date'] = pd.to_datetime(df['Date'], format="%d-%m-%Y") df['Currency'] = '{}'.format(varCurrency) df.columns = [ 'Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Market Cap', 'Currency' ] return df
def gatherHistoricalDataFor(coin, start_date, end_date): scraper = CmcScraper(coin['symbol'], start_date.strftime("%d-%m-%Y"), end_date.strftime("%d-%m-%Y")) # get raw data as list of list headers, historicaldata = scraper.get_data() # per ora non mi interessa più salvare il valore di Coin, contiene info che non mi servono nello spread. Ma devo mantenerla per far funzionare il resto e il pregresso già salvato su Google Spreadsheet headers.insert(0, "Coin") for historicaldata_row in historicaldata: historicaldata_row.insert(0, "N/A") return headers, historicaldata
def get_latest_data(coin_list): """Get latest price data for given coin list.""" li = coin_list.split() ret = dict() for i, x in enumerate(li): scr = CmcScraper(x) try: header, data = scr.get_data() ti = list() pr = list() for j, xx in enumerate(data): if xx[4]: pr.append(xx[4]) ti.append(xx[0]) ret[x] = dict(t=ti, p=pr) except Exception: print("Invalid coin symbol: {0}".format(x)) print(i, x) return ret
def get_history_all(): """Get all history close data.""" cmc = Market() listing = [item['symbol'] for item in cmc.listings()['data']] ret = dict() for i, x in enumerate(listing): scr = CmcScraper(x) try: header, data = scr.get_data() ti = list() pr = list() for j, xx in enumerate(data): if xx[4]: pr.append(xx[4]) ti.append(xx[0]) ret[x] = dict(t=ti, p=pr) except Exception: print("Invalid coin symbol: {0}".format(x)) print(i, x) return ret
def update(event): messagebox.showinfo('info', """更新历史数据耗时较久,视网络环境需时20到40分钟,请耐心等待""") cmc = Market() listing = [item['symbol'] for item in cmc.listings()['data']] hist = dict() step = 100 / len(listing) for i, x in enumerate(listing): scr = CmcScraper(x) try: header, data = scr.get_data() ti = list() pr = list() for j, xx in enumerate(data): if xx[4]: pr.append(xx[4]) ti.append(xx[0]) hist[x] = dict( t=ti, p=pr ) except Exception: print("Invalid coin symbol: {0}".format(x)) print(i, x) progress.step(step) progress.update() sym2id, id2sym = get_id_table() outdata = dict( history=hist, s2i=sym2id, i2s=id2sym ) with open("cmc_history.json", "w") as f: json.dump(outdata, f) update_date() messagebox.showinfo('info', "成功更新历史数据")
from cryptocmd import CmcScraper # initialise scraper scraper = CmcScraper('XRP', '01-01-2015', '30-10-2018') # get data as list of list headers, data = scraper.get_data() # export the data to csv scraper.export_csv() # get dataframe for the data df = scraper.get_dataframe()
def updateCoins(coinCodes): """Update the H5 Repository with Coin Data Keyword arguments: coinList - Set of CoinCodes """ global apph5datafile coinList = [] if coinCodes == '*': print("You asked to update all tracked coins") coinList = config.get('Active', 'Coins').split(' ') else: coinList = coinCodes print("You asked to list the last week for coins %s" % (coinList)) print("You asked to update coins %s" % (coinList)) for coin in coinList: havecointable = True if not os.path.exists(apph5datafile): print("Creating Datafile %s" % apph5datafile) havecointable = False # if not ('/'+coin.upper()) in storedcoins: # print("Coin %s not in dataset" % coin.upper()) # havecointable = False if havecointable: try: hdf = read_hdf(apph5datafile, coin) hdf['date_ranked'] = hdf['Date'].rank(ascending=1) print "Last element from %s is dated %s" % ( coin, str(hdf.at[0, 'Date'])[:10]) lastdate = datetime.strptime( str(hdf.at[0, 'Date'])[:10], "%Y-%m-%d") nextday = timedelta(1) nextday = lastdate + nextday delta = (datetime.today() - timedelta(1)) - nextday print("Difference from yesterday to last update ", delta) except KeyError: print("No data found for this Coin") nextday = (datetime.today() - timedelta(2 * 365)) lastdate = datetime.today() - timedelta(1) print "Updating data from %s to %s" % (str(nextday), str(lastdate)) startdate = '{:%d-%m-%Y}'.format(nextday) enddate = '{:%d-%m-%Y}'.format(lastdate) scraper = CmcScraper(coin, startdate, enddate) else: scraper = CmcScraper(coin.upper()) # get data as list of list try: headers, data = scraper.get_data() # get dataframe for the data df = scraper.get_dataframe() print df # df.to_hdf('pluton.h5',coin,append=True,format='t') df.to_hdf(apph5datafile, coin.upper()) except ValueError: print("No data found") return
def download(self,coin_data_id): fetcher = CmcScraper(coin_data_id)#(coin['symbol']) headers, data = fetcher.get_data() Controller.prepDataframe(self,headers,data)
from cryptocmd import CmcScraper # Documentation https://github.com/guptarohit/cryptoCMD # initialise scraper without time interval scraper = CmcScraper("BTC") # get raw data as list of list headers, data = scraper.get_data() # get data in a json format xrp_json_data = scraper.get_data("json") # export the data as csv file, you can also pass optional `name` parameter scraper.export("csv", name="xrp_all_time") # Pandas dataFrame for the same data df = scraper.get_dataframe() result = xrp_json_data[0] print(data[0][1])
def get_market(coin='BTC'): scraper = CmcScraper('BTC', '06-03-2018', '30-05-2018') headers, data = scraper.get_data() scraper.export_csv( csv_path='/home/cedric/Documents/UM/Info_mining/twitter_stream/market')