def updateMarketData(self, p_market_list="all"):
if p_market_list == "all":
mkt_list = self.scrapeAllCMCMarkets("currencies")
mkt_list = mkt_list + self.scrapeAllCMCMarkets("assets")
else:
mkt_list = p_market_list
download_total = str(len(mkt_list))
print("Markets to downlaod: " + download_total)
# Loop throuth the Markets
error_list = []
no_data_list = []
mkt_count = 0
for market in mkt_list: #self.markets:
# print("No Data: "+str(len(no_data_list)))
# print(no_data_list)
mkt_count += 1
print()
print(str(mkt_count) + " of " + download_total + ". " +
market) # MIKE [0]
# Default start date if data not previously laoded
load_start_date = self.max_start_date
load_end_date = self.default_end_date
# Get the load dates
file_path = self.fileName(market, "day_1")
file_start_date, file_end_date, load_start_date = self.getLoadDates(
market, load_start_date, file_path)
load_start_date = load_start_date + datetime.timedelta(1, 0)
# Print the dates
print("File: " + file_path)
print("File Start: " + str(file_start_date) + " End: " +
str(file_end_date))
print("Load Start: " + str(load_start_date) + " End: " +
str(load_end_date))
# Scrape the webpage
try:
market_data = self.scrapeHistoricalData(
market, load_start_date, load_end_date)
except urllib.error.HTTPError as err:
print(
"HTTP ERROR: CoinMarketCap::updateMarketData: Probably wrong file name in URL"
+ str(err.args))
error_list = error_list.append(market)
# MIKE to do
# except http.client.BadStatusLine as err:
# print("HTTP BadStatusLine: updateMarketData: "+str(err.args))
# error_list = [market] + error_list
except:
print(
"UNKNOWN ERROR: CoinMarketCap::updateMarketData ... market_data = self.scrapeHistoricalData"
)
print(sys.exc_info()[0])
error_list = market + error_list
else:
# Check for no data message
if market_data[0] == [
'No data was found for the selected time period.'
]:
print("No data found for this market.")
# no_data_list = market + no_data_list
# no_data_list = no_data_list.append(market)
else:
# Format the date column
market_data = self.convertScrapedDateFormat(market_data)
# Print number of rows
print("Number of rows imported: " +
str(len(market_data) - 1))
payload = pd.DataFrame(market_data,
columns=[
"Date", "Open", "High", "Low",
"Close", "Volume", "MC"
])
if len(payload) > 0:
payload.Date = pd.to_datetime(
payload.Date,
format="%Y%m%d") # convert to datetime
# payload.Date2 = payload.Date[0].strftime('%Y-%m-%d %H:%M:%S')
# pd.DatetimeIndex(payload[0]*10**9) # all_markets.json() is class 'dict'
payload = payload.set_index(
'Date') # Set the Timestamp as the indes
payload = payload.iloc[::
-1] # Reverse teh order to time ascending
# print(payload.head(5))
# print(payload.tail(5))
# Save the data
mkt = Market(market, self.source_dir_name, "Day_1",
payload)
mkt.saveMarketData(p_file_path=file_path)
#
# # Save the file to CSV
# my_file = Path(file_path)
# if my_file.is_file():
# # File exists
# with open(file_path, 'a') as myfile:
# wr = csv.writer(myfile)
# # Ignore the first row in the scraped data as it is a repeat
# # Add a empty row to ensure the CSV is written starting on the next row
# # Reverse it so it is date ascending (newest at the end)
# wr.writerows(list(reversed(market_data))[1:])
# # wr.writerows([[]]+list(reversed(market_data))[1:])
# else:
# # No file so create new file
# with open(file_path, 'w') as myfile:
# wr = csv.writer(myfile)
# # Reverse it so it is date ascending (newest at the end)
# wr.writerows(reversed(market_data))
print()
print()
print()
print("Success: " + str(len(mkt_list))) #-len(error_list)))
#-str(len(no_data_list))
print("Error: " + str(len(error_list)))
print(error_list)
print("No Data: NOT IMPLEMENTED ... " + str(len(no_data_list)))
print(no_data_list)
def updateMarketData(self, p_market_list="all"):
if p_market_list == "all":
mkt_list = self.getAllMarkets(
) # TODO .Label for poloniex or cmc? Move to subclass
else:
mkt_list = p_market_list
download_total = str(len(mkt_list))
interval_total = len(self.exchange_intervals)
print("Markets to downlaod: " + download_total +
" ... each market has " + str(interval_total) + " intervals")
# convert market format from / to _
mkt_list = [w.replace('/', '_') for w in mkt_list]
# Loop throuth the Markets
error_list = []
no_data_list = []
mkt_count = 0
for market in mkt_list: #self.markets:
# print("No Data: "+str(len(no_data_list)))
# print(no_data_list)
mkt_count += 1
print()
print(str(mkt_count) + " of " + download_total + ". " + market)
for index, interval in self.exchange_intervals.iterrows():
# Default start date if data not previously laoded
load_start_date = self.max_start_date
load_end_date = self.default_end_date
# Get the load dates
file_path = self.fileName(market, interval.Directory_name)
file_start_date, file_end_date, load_start_date = self.getLoadDates(
market, interval, load_start_date, file_path)
# Print the dates
print()
print("Interval: " + interval.Directory_name + " Seconds: " +
str(interval.Seconds) + " Load start time: " +
str(datetime.now()))
print("File: " + file_path)
print("File Start: " + str(file_start_date) + " End: " +
str(file_end_date))
print("Load Start: " + str(load_start_date) + " End: " +
str(load_end_date))
# Calls the URL
try:
market_data = self.getHistoricalData(
market, interval.Seconds,
load_start_date + timedelta(hours=10),
load_end_date + timedelta(hours=10))
except urllib.error.HTTPError as err:
self.printException(
inspect.currentframe().f_lineno,
"ERROR: HHTP - Probably wrong file name in URL - " +
str(err.args) + " --- " + str(sys.exc_info()[0]))
error_list = error_list.append(market)
except:
self.printException(inspect.currentframe().f_lineno,
"ERROR: unknown -")
error_list = [market] + error_list
else:
# Check for no data message
if len(market_data) == 0:
print("No data found for this market.")
else:
# Format the date column
print("Number of Rows imported: " +
str(len(market_data)))
# Save the data
mkt = Market(market, self.source_dir_name,
interval.Directory_name, market_data)
mkt.saveMarketData(p_file_path=file_path)
# exchange specific post processing
self.post_process_file(market, interval.Directory_name)
print()
print()
print()
print("Success: " + str(len(mkt_list))) #-len(error_list)))
#-str(len(no_data_list))
print("Error: " + str(len(error_list)))
print(error_list)
print("No Data: NOT IMPLEMENTED ... " + str(len(no_data_list)))
print(no_data_list)
