def get_daily_data_records(self, no_records):
myModelObj = MyModel()
coin_data = myModelObj.get_coin_details()
# print(coin_data)
source_id = 1
print(no_records)
# print(coin_data[0][1])
for coin_name in coin_data:
# print(coin_name[1])
URL = "https://coinmarketcap.com/currencies/" + coin_name[
1] + "/historical-data/"
#URL = "https://coinmarketcap.com/currencies/mithril/historical-data/"
#URL = "https://coinmarketcap.com/currencies/fsfdsf/historical-data/"
coinmarketcapObj = CoinmarketcapModel(URL)
rawData = coinmarketcapObj.get_data()
# print(rawData)
records = []
if rawData:
# print(rawData[0])
j = no_records
if len(rawData) <= j:
i = 0
while i < len(rawData):
records.append(rawData[i])
i = i + 1
else:
i = 0
while i < j:
records.append(rawData[i])
i = i + 1
else:
records = []
print(records)
#exit()
cleanData = []
### cleaing data
for item in records:
# print(item)
# print(item[0])
idate = item[0]
myDate = datetime.strptime(idate, '%b %d, %Y').date()
# print(date.date())
# print(date)
# open = float(item[1])
if item[1] == "-":
open = 0
else:
open = float(item[1].replace(',', ''))
# high = float(item[2])
if item[2] == "-":
high = 0
else:
high = float(item[2].replace(',', ''))
# low = float(item[3])
if item[3] == "-":
low = 0
else:
low = float(item[3].replace(',', ''))
# close = float(item[4])
if item[4] == "-":
close = 0
else:
close = float(item[4].replace(',', ''))
if item[5] == "-":
volume = 0
else:
volume = float(item[5].replace(',', ''))
if item[6] == "-":
marketCap = 0
else:
marketCap = float(item[6].replace(',', ''))
coin_id = coin_name[0]
#######
item.append(coin_id)
item.append(source_id)
item[0] = myDate
item[1] = open
item[2] = high
item[3] = low
item[4] = close
item[5] = volume
item[6] = marketCap
# print(item)
cleanData.append(item)
print(cleanData)
#exit()
myModelObj = MyModel()
myModelObj.insert_coin_price_daily(data=cleanData)
def get_daily_data(self):
myModelObj = MyModel()
coin_data = myModelObj.get_coin_details()
# print(coin_data)
# exit()
# coin_data = [(728260,'EFX', 0, 'effect-ai', None, None, None)]
source_id = 1
# print(coin_data[0][1])
for coin_name in coin_data:
# print(coin_name[1])
coin_id = coin_name[0]
URL = "https://coinmarketcap.com/currencies/" + coin_name[
3] + "/historical-data/"
# URL = "https://coinmarketcap.com/currencies/bitcoin/historical-data/"
# URL = "https://coinmarketcap.com/currencies/quantum-resistant-ledger/historical-data/"
coinmarketcapObj = CoinmarketcapModel(URL)
rawData = coinmarketcapObj.get_data()
# print("raw data = ",rawData)
# exit()
updateData = []
if rawData:
try:
name = rawData[0][7]
print(name)
name = name.replace('Historical data for ', '')
print("name = ", name)
url = rawData[0][8]
print(url)
url = re.sub(r'http[s]?://', '', url)
url = re.sub(r'www[a-z0-9]?\.', '', url)
url = re.sub(r'/.*', '', url)
print("url = ", url)
norm_name = name
print(norm_name)
norm_name = re.sub(r'[\.\-_]*', '', norm_name)
norm_name = re.sub(r'\s*', '', norm_name)
norm_name = re.sub(r'\(.*\)', '', norm_name)
norm_name = norm_name.lower().strip()
print("norm name = ", norm_name)
updateData.append(name)
updateData.append(norm_name)
updateData.append(url)
updateData.append(coin_id)
print(updateData)
except:
pass
firstRecord = []
if rawData:
try:
firstRecord.append(rawData[0])
except:
pass
else:
firstRecord = []
cleanData = []
### cleaing data
for item in firstRecord:
idate = item[0]
myDate = datetime.strptime(idate, '%b %d, %Y').date()
if item[1] == "-":
open = 0
else:
open = float(item[1].replace(',', ''))
# high = float(item[2])
if item[2] == "-":
high = 0
else:
high = float(item[2].replace(',', ''))
# low = float(item[3])
if item[3] == "-":
low = 0
else:
low = float(item[3].replace(',', ''))
# close = float(item[4])
if item[4] == "-":
close = 0
else:
close = float(item[4].replace(',', ''))
if item[5] == "-":
volume = 0
else:
volume = float(item[5].replace(',', ''))
if item[6] == "-":
marketCap = 0
else:
marketCap = float(item[6].replace(',', ''))
#######
try:
del item[7]
del item[7]
except:
pass
item.append(coin_id)
item.append(source_id)
item[0] = myDate
item[1] = open
item[2] = high
item[3] = low
item[4] = close
item[5] = volume
item[6] = marketCap
# print(item)
cleanData.append(item)
print(cleanData)
# exit()
myModelObj = MyModel()
myModelObj.insert_coin_price_daily(data=cleanData)
#
myModelObj1 = MyModel()
myModelObj1.updateCoinDetails(data=updateData)
def get_history_data(self):
end = datetime.today().date()
end = str(end).replace("-", "")
# print(type(end))
# print("end = ",end)
# exit()
myModelObj = MyModel()
coin_data = myModelObj.get_coin_price_daily_records_by_range(start=0,
end=100)
# print(coin_data)
source_id = 1
# coin_data = [[1417,'safecoin',4]]
#print(coin_data[0][1])
# print(coin_data)
print(len(coin_data))
# exit()
for item_coin in coin_data:
coin_id = item_coin[0]
no_of_rows_crypto_database = item_coin[2]
print(type(no_of_rows_crypto_database))
URL = "https://coinmarketcap.com/currencies/" + item_coin[
1] + "/historical-data/?start=20130428&end=" + end
print(URL)
# exit()
# URL = "https://coinmarketcap.com/currencies/slimcoin/historical-data/?start=20130428&end=20180430"
#URL = "https://coinmarketcap.com/currencies/farstcoin/historical-data/?start=20130428&end=20180311"
coinmarketcapObj = CoinmarketcapModel(URL)
rawData = coinmarketcapObj.get_daily_data()
# print(rawData)
# exit()
no_of_records_coinmarketcap = len(rawData)
print("Number of records in coinmarketcap = ",
no_of_records_coinmarketcap)
print("Number of records in crypto database = ",
no_of_rows_crypto_database)
# exit()
if no_of_records_coinmarketcap > no_of_rows_crypto_database:
print(no_of_records_coinmarketcap)
print(coin_id)
# exit()
myModelObj1 = MyModel()
myModelObj1.del_records_coin_price_daily(coin_id=coin_id)
# exit()
cleanData = []
### cleaing data
for item in rawData:
# print(item)
# print(item[0])
idate = item[0]
myDate = datetime.strptime(idate, '%b %d, %Y').date()
# print(date.date())
# print(date)
# open = float(item[1])
if item[1] == "-":
open = 0
else:
open = float(item[1].replace(',', ''))
# high = float(item[2])
if item[2] == "-":
high = 0
else:
high = float(item[2].replace(',', ''))
# low = float(item[3])
if item[3] == "-":
low = 0
else:
low = float(item[3].replace(',', ''))
# close = float(item[4])
if item[4] == "-":
close = 0
else:
close = float(item[4].replace(',', ''))
if item[5] == "-":
volume = 0
else:
volume = float(item[5].replace(',', ''))
if item[6] == "-":
marketCap = 0
else:
marketCap = float(item[6].replace(',', ''))
# coin_id = coin_name[0]
#######
# del item[7]
# del item[7]
item.append(coin_id)
item.append(source_id)
item[0] = myDate
item[1] = open
item[2] = high
item[3] = low
item[4] = close
item[5] = volume
item[6] = marketCap
# print(" item = ",item)
cleanData.append(item)
# print("clean data = ", cleanData)
# exit()
# print(cleanData)
myModelObj2 = MyModel()
myModelObj2.insert_coin_price_daily(data=cleanData)
else:
# print(no_of_rows_crypto_database)
print("History Records are Up To Date")
def get_all_historical_data(self):
end = datetime.today().date()
end = str(end).replace("-", "")
myModelObj = MyModel()
coin_data = myModelObj.get_coin_details()
# print(coin_data)
source_id = 1
for item_coin in coin_data:
coin_id = item_coin[0]
URL = "https://coinmarketcap.com/currencies/" + item_coin[
3] + "/historical-data/?start=20130428&end=" + end
print(URL)
# exit()
coinmarketcapObj = CoinmarketcapModel(URL)
rawData = coinmarketcapObj.get_daily_data()
no_of_records_coinmarketcap = len(rawData)
if no_of_records_coinmarketcap > 0:
# exit()
myModelObj1 = MyModel()
myModelObj1.del_records_coin_price_daily(coin_id=coin_id)
# exit()
cleanData = []
### cleaing data
for item in rawData:
idate = item[0]
myDate = datetime.strptime(idate, '%b %d, %Y').date()
if item[1] == "-":
open = 0
else:
open = float(item[1].replace(',', ''))
# high = float(item[2])
if item[2] == "-":
high = 0
else:
high = float(item[2].replace(',', ''))
# low = float(item[3])
if item[3] == "-":
low = 0
else:
low = float(item[3].replace(',', ''))
# close = float(item[4])
if item[4] == "-":
close = 0
else:
close = float(item[4].replace(',', ''))
if item[5] == "-":
volume = 0
else:
volume = float(item[5].replace(',', ''))
if item[6] == "-":
marketCap = 0
else:
marketCap = float(item[6].replace(',', ''))
# coin_id = coin_name[0]
#######
# del item[7]
# del item[7]
item.append(coin_id)
item.append(source_id)
item[0] = myDate
item[1] = open
item[2] = high
item[3] = low
item[4] = close
item[5] = volume
item[6] = marketCap
# print(" item = ",item)
cleanData.append(item)
# print("clean data = ", cleanData)
# exit()
# print(cleanData)
myModelObj2 = MyModel()
myModelObj2.insert_coin_price_daily(data=cleanData)
else:
# print(no_of_rows_crypto_database)
print("History Records are Up To Date")
