def updateCoinDetails(self): myModelObj = MyModel() coin_data = myModelObj.get_coin_details() # print(coin_data) # coin_data = ["bitcoin","ethrium"] # source_id = 1 # print(coin_data[0][1]) for coin_name in coin_data: # print(coin_name[1]) URL = "https://coinmarketcap.com/currencies/" + coin_name[ 1] + "/historical-data/" # URL = "https://coinmarketcap.com/currencies/bitcoin/historical-data/" # URL = "https://coinmarketcap.com/currencies/quantum-resistant-ledger/historical-data/" coinmarketcapObj = CoinmarketcapModel(URL) rawData = coinmarketcapObj.get_coin_details_for_update() # print("raw data = ", rawData) updateData = [] if rawData: try: name = rawData[0] # print(name) name = name.replace('Historical data for ', '') # print(name) except: name = "" try: url = rawData[1] # print(url) url = re.sub(r'http[s]?://', '', url) url = re.sub(r'www[a-z0-9]?\.', '', url) url = re.sub(r'/.*', '', url) # print(url) except: url = "" if name: norm_name = name # print(norm_name) norm_name = re.sub(r'[\.\-_]*', '', norm_name) norm_name = re.sub(r'\s*', '', norm_name) norm_name = re.sub(r'\(.*\)', '', norm_name) norm_name = norm_name.lower().strip() else: norm_name = "" # print(norm_name) updateData.append(name) updateData.append(norm_name) updateData.append(url) updateData.append(coin_name[1]) print(updateData) myModelObj = MyModel() myModelObj.updateCoinDetails(data=updateData)
def get_daily_data_records(self, no_records): myModelObj = MyModel() coin_data = myModelObj.get_coin_details() # print(coin_data) source_id = 1 print(no_records) # print(coin_data[0][1]) for coin_name in coin_data: # print(coin_name[1]) URL = "https://coinmarketcap.com/currencies/" + coin_name[ 1] + "/historical-data/" #URL = "https://coinmarketcap.com/currencies/mithril/historical-data/" #URL = "https://coinmarketcap.com/currencies/fsfdsf/historical-data/" coinmarketcapObj = CoinmarketcapModel(URL) rawData = coinmarketcapObj.get_data() # print(rawData) records = [] if rawData: # print(rawData[0]) j = no_records if len(rawData) <= j: i = 0 while i < len(rawData): records.append(rawData[i]) i = i + 1 else: i = 0 while i < j: records.append(rawData[i]) i = i + 1 else: records = [] print(records) #exit() cleanData = [] ### cleaing data for item in records: # print(item) # print(item[0]) idate = item[0] myDate = datetime.strptime(idate, '%b %d, %Y').date() # print(date.date()) # print(date) # open = float(item[1]) if item[1] == "-": open = 0 else: open = float(item[1].replace(',', '')) # high = float(item[2]) if item[2] == "-": high = 0 else: high = float(item[2].replace(',', '')) # low = float(item[3]) if item[3] == "-": low = 0 else: low = float(item[3].replace(',', '')) # close = float(item[4]) if item[4] == "-": close = 0 else: close = float(item[4].replace(',', '')) if item[5] == "-": volume = 0 else: volume = float(item[5].replace(',', '')) if item[6] == "-": marketCap = 0 else: marketCap = float(item[6].replace(',', '')) coin_id = coin_name[0] ####### item.append(coin_id) item.append(source_id) item[0] = myDate item[1] = open item[2] = high item[3] = low item[4] = close item[5] = volume item[6] = marketCap # print(item) cleanData.append(item) print(cleanData) #exit() myModelObj = MyModel() myModelObj.insert_coin_price_daily(data=cleanData)
def get_daily_data(self): myModelObj = MyModel() coin_data = myModelObj.get_coin_details() # print(coin_data) # exit() # coin_data = [(728260,'EFX', 0, 'effect-ai', None, None, None)] source_id = 1 # print(coin_data[0][1]) for coin_name in coin_data: # print(coin_name[1]) coin_id = coin_name[0] URL = "https://coinmarketcap.com/currencies/" + coin_name[ 3] + "/historical-data/" # URL = "https://coinmarketcap.com/currencies/bitcoin/historical-data/" # URL = "https://coinmarketcap.com/currencies/quantum-resistant-ledger/historical-data/" coinmarketcapObj = CoinmarketcapModel(URL) rawData = coinmarketcapObj.get_data() # print("raw data = ",rawData) # exit() updateData = [] if rawData: try: name = rawData[0][7] print(name) name = name.replace('Historical data for ', '') print("name = ", name) url = rawData[0][8] print(url) url = re.sub(r'http[s]?://', '', url) url = re.sub(r'www[a-z0-9]?\.', '', url) url = re.sub(r'/.*', '', url) print("url = ", url) norm_name = name print(norm_name) norm_name = re.sub(r'[\.\-_]*', '', norm_name) norm_name = re.sub(r'\s*', '', norm_name) norm_name = re.sub(r'\(.*\)', '', norm_name) norm_name = norm_name.lower().strip() print("norm name = ", norm_name) updateData.append(name) updateData.append(norm_name) updateData.append(url) updateData.append(coin_id) print(updateData) except: pass firstRecord = [] if rawData: try: firstRecord.append(rawData[0]) except: pass else: firstRecord = [] cleanData = [] ### cleaing data for item in firstRecord: idate = item[0] myDate = datetime.strptime(idate, '%b %d, %Y').date() if item[1] == "-": open = 0 else: open = float(item[1].replace(',', '')) # high = float(item[2]) if item[2] == "-": high = 0 else: high = float(item[2].replace(',', '')) # low = float(item[3]) if item[3] == "-": low = 0 else: low = float(item[3].replace(',', '')) # close = float(item[4]) if item[4] == "-": close = 0 else: close = float(item[4].replace(',', '')) if item[5] == "-": volume = 0 else: volume = float(item[5].replace(',', '')) if item[6] == "-": marketCap = 0 else: marketCap = float(item[6].replace(',', '')) ####### try: del item[7] del item[7] except: pass item.append(coin_id) item.append(source_id) item[0] = myDate item[1] = open item[2] = high item[3] = low item[4] = close item[5] = volume item[6] = marketCap # print(item) cleanData.append(item) print(cleanData) # exit() myModelObj = MyModel() myModelObj.insert_coin_price_daily(data=cleanData) # myModelObj1 = MyModel() myModelObj1.updateCoinDetails(data=updateData)
def get_minute_data(self): mymodelObj = MyModel() coin_data = mymodelObj.get_coin_details() #print(coin_data) url = "https://api.coinmarketcap.com/v1/ticker/?limit=0" coinmarketcapObj = CoinmarketcapModel(url=url) jsonData = coinmarketcapObj.get_minute_data() # print(jsonData) # exit() data = [] for i in jsonData: #print(i['id']) coin_name = i['id'] rank = int(i['rank']) if i['price_usd'] == None: close_usd = 0 else: close_usd = float(i['price_usd']) if i['price_btc'] == None: close_btc = 0 else: close_btc = float(i['price_btc']) if i['24h_volume_usd'] == None: volume_usd = 0 else: volume_usd = float(i['24h_volume_usd']) if i['market_cap_usd'] == None: market_cap_usd = 0 else: market_cap_usd = float(i['market_cap_usd']) if i['available_supply'] == None: available_supply = 0 else: available_supply = float(i['available_supply']) if i['total_supply'] == None: total_supply = 0 else: total_supply = float(i['total_supply']) if i['max_supply'] == None: max_supply = 0 else: max_supply = float(i['max_supply']) if i['last_updated'] == None: lastUpdatedTime = 0 # print("i m in if ") else: last_updated = int(i['last_updated']) # print("i m in else ") print(last_updated) # exit() last_updated_utc = datetime.utcfromtimestamp( last_updated).strftime('%Y-%m-%d %H:%M:%S') # last_updated_utc = datetime.utcfromtimestamp(last_updated) print(type(last_updated_utc)) # exit() lastUpdatedTime = datetime.strptime(last_updated_utc, "%Y-%m-%d %H:%M:%S") # lastUpdatedTime = datetime.strptime(last_updated_utc, "%Y-%m-%d") print(lastUpdatedTime) print(type(lastUpdatedTime)) # exit() # print(type(last_updated_utc)) # exit() # print(type(last_updated_utc)) # # tempData = [] # tempData.append(last_updated_utc) # data.append(tempData) # print(data) # exit() for item in coin_data: # print(item[3]) if item[3] == coin_name: #print(j[0]) tempData = [] tempData.append(item[0]) # tempData.append(rank) # tempData.append(close_usd) # tempData.append(close_btc) # tempData.append(volume_usd) # tempData.append(market_cap_usd) tempData.append(available_supply) tempData.append(total_supply) tempData.append(max_supply) tempData.append(lastUpdatedTime) print(tempData) # exit() data.append(tempData) # print(data) # exit() mymodelObj2 = MyModel() mymodelObj2.insert_coin_price_minute(data=data)
def get_all_historical_data(self): end = datetime.today().date() end = str(end).replace("-", "") myModelObj = MyModel() coin_data = myModelObj.get_coin_details() # print(coin_data) source_id = 1 for item_coin in coin_data: coin_id = item_coin[0] URL = "https://coinmarketcap.com/currencies/" + item_coin[ 3] + "/historical-data/?start=20130428&end=" + end print(URL) # exit() coinmarketcapObj = CoinmarketcapModel(URL) rawData = coinmarketcapObj.get_daily_data() no_of_records_coinmarketcap = len(rawData) if no_of_records_coinmarketcap > 0: # exit() myModelObj1 = MyModel() myModelObj1.del_records_coin_price_daily(coin_id=coin_id) # exit() cleanData = [] ### cleaing data for item in rawData: idate = item[0] myDate = datetime.strptime(idate, '%b %d, %Y').date() if item[1] == "-": open = 0 else: open = float(item[1].replace(',', '')) # high = float(item[2]) if item[2] == "-": high = 0 else: high = float(item[2].replace(',', '')) # low = float(item[3]) if item[3] == "-": low = 0 else: low = float(item[3].replace(',', '')) # close = float(item[4]) if item[4] == "-": close = 0 else: close = float(item[4].replace(',', '')) if item[5] == "-": volume = 0 else: volume = float(item[5].replace(',', '')) if item[6] == "-": marketCap = 0 else: marketCap = float(item[6].replace(',', '')) # coin_id = coin_name[0] ####### # del item[7] # del item[7] item.append(coin_id) item.append(source_id) item[0] = myDate item[1] = open item[2] = high item[3] = low item[4] = close item[5] = volume item[6] = marketCap # print(" item = ",item) cleanData.append(item) # print("clean data = ", cleanData) # exit() # print(cleanData) myModelObj2 = MyModel() myModelObj2.insert_coin_price_daily(data=cleanData) else: # print(no_of_rows_crypto_database) print("History Records are Up To Date")