def retrieve_data(self, ticker):
# initialise scraper without time interval
scraper = CmcScraper(ticker)
# get raw data as list of list
headers, data = scraper.get_data()
# get data in a json format
xrp_json_data = scraper.get_data("json")
# export the data as csv file
scraper.export("csv", name=ticker)
def main():
global currencyList
n = 0
curDay = strftime("%d-%m-%Y", gmtime())
l = len(currencyList)
#l = 1
while n < l:
# initialise scraper
scraper = CmcScraper(currencyList[n], '01-01-2000', curDay)
# # get data as list of list
headers, data = scraper.get_data()
# # export the data to csv
scraper.export_csv('history/'+currencyList[n]+".csv")
# # get dataframe for the data
df = scraper.get_dataframe()
x = []
y = []
with open('history/'+currencyList[n]+'.csv', 'r') as f:
sreader = csv.reader(f)
i = 0
for row in sreader:
if i > 0 :
x.append(i)
y.append((float(row[2]) + float(row[3]))/2)
i = i + 1
prediction = predict(y,currencyList[n])
print currencyList[n] + ' => ' + str(prediction)
y.reverse()
data = {}
data["pointStart"] = 1230764400000
data["pointInterval"] = 3600000
data["dataLength"] = len(y)
data["data"] = y
data["prediction"] = prediction
json_data = json.dumps(data)
#print json_data
file = open('public/'+currencyList[n]+'.json', 'w')
file.write(json_data)
file.close()
n = n+1
def cryptoInfoToDf(varCurrency='lisk',
varFromDate='2018-01-01',
varToDate='2018-12-31'):
# def cryptoInfoToDf (varCurrency = 'lisk', varFromDate = '04-01-2017',
# varToDate = '16-01-2017'):
"""
variables:
varCurrency - string value of currency shortcut
varFromDate - date in format of YYYY-MM-DD
varToDate - date in format of YYYY-MM-DD
returns:
dataframe of cryptocurrency data
created by connecting to Coinmarketcap.com
"""
cryptoDict = {
'lisk': 'LSK',
'bitcoin': 'BTC',
'ripple': 'XRP',
'ethereum': 'ETH',
'litecoin': 'LTC'
}
cryptoName = cryptoDict[varCurrency]
fromDate = datetime.datetime.strptime(varFromDate,
'%Y-%m-%d').strftime('%d-%m-%Y')
toDate = datetime.datetime.strptime(varToDate,
'%Y-%m-%d').strftime('%d-%m-%Y')
# cryptoDict = {
# 'LSK':'lisk',
# 'BTC':'bitcoin',
# 'XRP':'ripple',
# 'ETH':'ethereum',
# 'LTC':'litecoin'
# }
# ? initialise scraper
scraper = CmcScraper(cryptoName, fromDate, toDate)
# ? initialise dataframe for the data
headers, data = scraper.get_data()
df = pd.DataFrame(data, columns=headers)
df['Date'] = pd.to_datetime(df['Date'], format="%d-%m-%Y")
df['Currency'] = '{}'.format(varCurrency)
df.columns = [
'Date', 'Open', 'High', 'Low', 'Close', 'Volume', 'Market Cap',
'Currency'
]
return df
def gatherHistoricalDataFor(coin, start_date, end_date):
scraper = CmcScraper(coin['symbol'], start_date.strftime("%d-%m-%Y"),
end_date.strftime("%d-%m-%Y"))
# get raw data as list of list
headers, historicaldata = scraper.get_data()
# per ora non mi interessa più salvare il valore di Coin, contiene info che non mi servono nello spread. Ma devo mantenerla per far funzionare il resto e il pregresso già salvato su Google Spreadsheet
headers.insert(0, "Coin")
for historicaldata_row in historicaldata:
historicaldata_row.insert(0, "N/A")
return headers, historicaldata
def get_latest_data(coin_list):
"""Get latest price data for given coin list."""
li = coin_list.split()
ret = dict()
for i, x in enumerate(li):
scr = CmcScraper(x)
try:
header, data = scr.get_data()
ti = list()
pr = list()
for j, xx in enumerate(data):
if xx[4]:
pr.append(xx[4])
ti.append(xx[0])
ret[x] = dict(t=ti, p=pr)
except Exception:
print("Invalid coin symbol: {0}".format(x))
print(i, x)
return ret
def get_history_all():
"""Get all history close data."""
cmc = Market()
listing = [item['symbol'] for item in cmc.listings()['data']]
ret = dict()
for i, x in enumerate(listing):
scr = CmcScraper(x)
try:
header, data = scr.get_data()
ti = list()
pr = list()
for j, xx in enumerate(data):
if xx[4]:
pr.append(xx[4])
ti.append(xx[0])
ret[x] = dict(t=ti, p=pr)
except Exception:
print("Invalid coin symbol: {0}".format(x))
print(i, x)
return ret
def update(event):
messagebox.showinfo('info',
"""更新历史数据耗时较久，视网络环境需时20到40分钟，请耐心等待""")
cmc = Market()
listing = [item['symbol'] for item in cmc.listings()['data']]
hist = dict()
step = 100 / len(listing)
for i, x in enumerate(listing):
scr = CmcScraper(x)
try:
header, data = scr.get_data()
ti = list()
pr = list()
for j, xx in enumerate(data):
if xx[4]:
pr.append(xx[4])
ti.append(xx[0])
hist[x] = dict(
t=ti,
p=pr
)
except Exception:
print("Invalid coin symbol: {0}".format(x))
print(i, x)
progress.step(step)
progress.update()
sym2id, id2sym = get_id_table()
outdata = dict(
history=hist,
s2i=sym2id,
i2s=id2sym
)
with open("cmc_history.json", "w") as f:
json.dump(outdata, f)
update_date()
messagebox.showinfo('info', "成功更新历史数据")
from cryptocmd import CmcScraper
# initialise scraper
scraper = CmcScraper('XRP', '01-01-2015', '30-10-2018')
# get data as list of list
headers, data = scraper.get_data()
# export the data to csv
scraper.export_csv()
# get dataframe for the data
df = scraper.get_dataframe()
def updateCoins(coinCodes):
"""Update the H5 Repository with Coin Data
Keyword arguments:
coinList - Set of CoinCodes
"""
global apph5datafile
coinList = []
if coinCodes == '*':
print("You asked to update all tracked coins")
coinList = config.get('Active', 'Coins').split(' ')
else:
coinList = coinCodes
print("You asked to list the last week for coins %s" % (coinList))
print("You asked to update coins %s" % (coinList))
for coin in coinList:
havecointable = True
if not os.path.exists(apph5datafile):
print("Creating Datafile %s" % apph5datafile)
havecointable = False
# if not ('/'+coin.upper()) in storedcoins:
# print("Coin %s not in dataset" % coin.upper())
# havecointable = False
if havecointable:
try:
hdf = read_hdf(apph5datafile, coin)
hdf['date_ranked'] = hdf['Date'].rank(ascending=1)
print "Last element from %s is dated %s" % (
coin, str(hdf.at[0, 'Date'])[:10])
lastdate = datetime.strptime(
str(hdf.at[0, 'Date'])[:10], "%Y-%m-%d")
nextday = timedelta(1)
nextday = lastdate + nextday
delta = (datetime.today() - timedelta(1)) - nextday
print("Difference from yesterday to last update ", delta)
except KeyError:
print("No data found for this Coin")
nextday = (datetime.today() - timedelta(2 * 365))
lastdate = datetime.today() - timedelta(1)
print "Updating data from %s to %s" % (str(nextday), str(lastdate))
startdate = '{:%d-%m-%Y}'.format(nextday)
enddate = '{:%d-%m-%Y}'.format(lastdate)
scraper = CmcScraper(coin, startdate, enddate)
else:
scraper = CmcScraper(coin.upper())
# get data as list of list
try:
headers, data = scraper.get_data()
# get dataframe for the data
df = scraper.get_dataframe()
print df
# df.to_hdf('pluton.h5',coin,append=True,format='t')
df.to_hdf(apph5datafile, coin.upper())
except ValueError:
print("No data found")
return
def download(self,coin_data_id):
fetcher = CmcScraper(coin_data_id)#(coin['symbol'])
headers, data = fetcher.get_data()
Controller.prepDataframe(self,headers,data)
from cryptocmd import CmcScraper
# Documentation https://github.com/guptarohit/cryptoCMD
# initialise scraper without time interval
scraper = CmcScraper("BTC")
# get raw data as list of list
headers, data = scraper.get_data()
# get data in a json format
xrp_json_data = scraper.get_data("json")
# export the data as csv file, you can also pass optional `name` parameter
scraper.export("csv", name="xrp_all_time")
# Pandas dataFrame for the same data
df = scraper.get_dataframe()
result = xrp_json_data[0]
print(data[0][1])
def get_market(coin='BTC'):
scraper = CmcScraper('BTC', '06-03-2018', '30-05-2018')
headers, data = scraper.get_data()
scraper.export_csv(
csv_path='/home/cedric/Documents/UM/Info_mining/twitter_stream/market')
