def __init__(self):
"""
:return:
"""
self.model = MyModel()
self.model.setPath("data/file.csv")
def run():
"""Builds model, loads data, trains and evaluates"""
#df = dataloader.DataLoader().load_data(CFG)
#print(df[2])
model = MyModel(CFG)
model.load_data()
model.build()
metrics = pd.DataFrame(model.train())
print(metrics)
def get_coin_details(self):
URL = "https://coinmarketcap.com/all/views/all/"
obj = CoinmarketcapModel(URL)
rawData = obj.get_coin_data()
# print(rawData)
# exit()
#myData = [item[1] for item in data]
cleanData = []
for item in rawData:
#myList = str(item[1]).split("\n")
tempList = []
symbol = item[2]
coin_name = item[10]
coin_name = coin_name.split("id-")[1]
tempList.append(symbol)
tempList.append(coin_name)
cleanData.append(tempList)
print(cleanData)
# exit()
myModelObj = MyModel()
myModelObj.insert_coin_details(data=cleanData)
def updateCoinDetails(self):
myModelObj = MyModel()
coin_data = myModelObj.get_coin_details()
# print(coin_data)
# coin_data = ["bitcoin","ethrium"]
# source_id = 1
# print(coin_data[0][1])
for coin_name in coin_data:
# print(coin_name[1])
URL = "https://coinmarketcap.com/currencies/" + coin_name[
1] + "/historical-data/"
# URL = "https://coinmarketcap.com/currencies/bitcoin/historical-data/"
# URL = "https://coinmarketcap.com/currencies/quantum-resistant-ledger/historical-data/"
coinmarketcapObj = CoinmarketcapModel(URL)
rawData = coinmarketcapObj.get_coin_details_for_update()
# print("raw data = ", rawData)
updateData = []
if rawData:
try:
name = rawData[0]
# print(name)
name = name.replace('Historical data for ', '')
# print(name)
except:
name = ""
try:
url = rawData[1]
# print(url)
url = re.sub(r'http[s]?://', '', url)
url = re.sub(r'www[a-z0-9]?\.', '', url)
url = re.sub(r'/.*', '', url)
# print(url)
except:
url = ""
if name:
norm_name = name
# print(norm_name)
norm_name = re.sub(r'[\.\-_]*', '', norm_name)
norm_name = re.sub(r'\s*', '', norm_name)
norm_name = re.sub(r'\(.*\)', '', norm_name)
norm_name = norm_name.lower().strip()
else:
norm_name = ""
# print(norm_name)
updateData.append(name)
updateData.append(norm_name)
updateData.append(url)
updateData.append(coin_name[1])
print(updateData)
myModelObj = MyModel()
myModelObj.updateCoinDetails(data=updateData)
class Test(object):
def __init__(self):
"""
:return:
"""
self.model = MyModel()
self.model.setPath("data/file.csv")
def test01_save(self):
"""
Save
Fungiert wie @Before
:return:
"""
self.model.saveCSV("a;b;c;\nd;e\nf")
assert 1 == 1 #Es ist kein Fehler beim speichern aufgetreten
def test02_load(self):
"""
Load
:return:
"""
print(self.model.loadCSV())
assert self.model.loadCSV() == "a;b;c;\nd;e\nf\n"
def test03_SaveLoad(self):
"""
:return:
"""
text = "z;x;\nh;i\nu;e;"
self.model.saveCSV(text)
assert self.model.loadCSV() == text+"\n"
def test04_SaveSave(self):
"""
Es ist möglich das file oeffter hinter einander zu beschreibem.
:return:
"""
text = "z;xjzhguz;\nh;i\nu;e;"
self.model.saveCSV(text)
self.model.saveCSV(text)
assert self.model.loadCSV() == text+"\n"
def test05_LoadLoad(self):
"""
Oefter laden
:return:
"""
self.model.loadCSV()
assert self.model.loadCSV() == "z;xjzhguz;\nh;i\nu;e;\n";
def get_daily_data_records(self, no_records):
myModelObj = MyModel()
coin_data = myModelObj.get_coin_details()
# print(coin_data)
source_id = 1
print(no_records)
# print(coin_data[0][1])
for coin_name in coin_data:
# print(coin_name[1])
URL = "https://coinmarketcap.com/currencies/" + coin_name[
1] + "/historical-data/"
#URL = "https://coinmarketcap.com/currencies/mithril/historical-data/"
#URL = "https://coinmarketcap.com/currencies/fsfdsf/historical-data/"
coinmarketcapObj = CoinmarketcapModel(URL)
rawData = coinmarketcapObj.get_data()
# print(rawData)
records = []
if rawData:
# print(rawData[0])
j = no_records
if len(rawData) <= j:
i = 0
while i < len(rawData):
records.append(rawData[i])
i = i + 1
else:
i = 0
while i < j:
records.append(rawData[i])
i = i + 1
else:
records = []
print(records)
#exit()
cleanData = []
### cleaing data
for item in records:
# print(item)
# print(item[0])
idate = item[0]
myDate = datetime.strptime(idate, '%b %d, %Y').date()
# print(date.date())
# print(date)
# open = float(item[1])
if item[1] == "-":
open = 0
else:
open = float(item[1].replace(',', ''))
# high = float(item[2])
if item[2] == "-":
high = 0
else:
high = float(item[2].replace(',', ''))
# low = float(item[3])
if item[3] == "-":
low = 0
else:
low = float(item[3].replace(',', ''))
# close = float(item[4])
if item[4] == "-":
close = 0
else:
close = float(item[4].replace(',', ''))
if item[5] == "-":
volume = 0
else:
volume = float(item[5].replace(',', ''))
if item[6] == "-":
marketCap = 0
else:
marketCap = float(item[6].replace(',', ''))
coin_id = coin_name[0]
#######
item.append(coin_id)
item.append(source_id)
item[0] = myDate
item[1] = open
item[2] = high
item[3] = low
item[4] = close
item[5] = volume
item[6] = marketCap
# print(item)
cleanData.append(item)
print(cleanData)
#exit()
myModelObj = MyModel()
myModelObj.insert_coin_price_daily(data=cleanData)
def get_history_data(self):
end = datetime.today().date()
end = str(end).replace("-", "")
# print(type(end))
# print("end = ",end)
# exit()
myModelObj = MyModel()
coin_data = myModelObj.get_coin_price_daily_records_by_range(start=0,
end=100)
# print(coin_data)
source_id = 1
# coin_data = [[1417,'safecoin',4]]
#print(coin_data[0][1])
# print(coin_data)
print(len(coin_data))
# exit()
for item_coin in coin_data:
coin_id = item_coin[0]
no_of_rows_crypto_database = item_coin[2]
print(type(no_of_rows_crypto_database))
URL = "https://coinmarketcap.com/currencies/" + item_coin[
1] + "/historical-data/?start=20130428&end=" + end
print(URL)
# exit()
# URL = "https://coinmarketcap.com/currencies/slimcoin/historical-data/?start=20130428&end=20180430"
#URL = "https://coinmarketcap.com/currencies/farstcoin/historical-data/?start=20130428&end=20180311"
coinmarketcapObj = CoinmarketcapModel(URL)
rawData = coinmarketcapObj.get_daily_data()
# print(rawData)
# exit()
no_of_records_coinmarketcap = len(rawData)
print("Number of records in coinmarketcap = ",
no_of_records_coinmarketcap)
print("Number of records in crypto database = ",
no_of_rows_crypto_database)
# exit()
if no_of_records_coinmarketcap > no_of_rows_crypto_database:
print(no_of_records_coinmarketcap)
print(coin_id)
# exit()
myModelObj1 = MyModel()
myModelObj1.del_records_coin_price_daily(coin_id=coin_id)
# exit()
cleanData = []
### cleaing data
for item in rawData:
# print(item)
# print(item[0])
idate = item[0]
myDate = datetime.strptime(idate, '%b %d, %Y').date()
# print(date.date())
# print(date)
# open = float(item[1])
if item[1] == "-":
open = 0
else:
open = float(item[1].replace(',', ''))
# high = float(item[2])
if item[2] == "-":
high = 0
else:
high = float(item[2].replace(',', ''))
# low = float(item[3])
if item[3] == "-":
low = 0
else:
low = float(item[3].replace(',', ''))
# close = float(item[4])
if item[4] == "-":
close = 0
else:
close = float(item[4].replace(',', ''))
if item[5] == "-":
volume = 0
else:
volume = float(item[5].replace(',', ''))
if item[6] == "-":
marketCap = 0
else:
marketCap = float(item[6].replace(',', ''))
# coin_id = coin_name[0]
#######
# del item[7]
# del item[7]
item.append(coin_id)
item.append(source_id)
item[0] = myDate
item[1] = open
item[2] = high
item[3] = low
item[4] = close
item[5] = volume
item[6] = marketCap
# print(" item = ",item)
cleanData.append(item)
# print("clean data = ", cleanData)
# exit()
# print(cleanData)
myModelObj2 = MyModel()
myModelObj2.insert_coin_price_daily(data=cleanData)
else:
# print(no_of_rows_crypto_database)
print("History Records are Up To Date")
def get_minute_data(self):
mymodelObj = MyModel()
coin_data = mymodelObj.get_coin_details()
#print(coin_data)
url = "https://api.coinmarketcap.com/v1/ticker/?limit=0"
coinmarketcapObj = CoinmarketcapModel(url=url)
jsonData = coinmarketcapObj.get_minute_data()
# print(jsonData)
# exit()
data = []
for i in jsonData:
#print(i['id'])
coin_name = i['id']
rank = int(i['rank'])
if i['price_usd'] == None:
close_usd = 0
else:
close_usd = float(i['price_usd'])
if i['price_btc'] == None:
close_btc = 0
else:
close_btc = float(i['price_btc'])
if i['24h_volume_usd'] == None:
volume_usd = 0
else:
volume_usd = float(i['24h_volume_usd'])
if i['market_cap_usd'] == None:
market_cap_usd = 0
else:
market_cap_usd = float(i['market_cap_usd'])
if i['available_supply'] == None:
available_supply = 0
else:
available_supply = float(i['available_supply'])
if i['total_supply'] == None:
total_supply = 0
else:
total_supply = float(i['total_supply'])
if i['max_supply'] == None:
max_supply = 0
else:
max_supply = float(i['max_supply'])
if i['last_updated'] == None:
lastUpdatedTime = 0
# print("i m in if ")
else:
last_updated = int(i['last_updated'])
# print("i m in else ")
print(last_updated)
# exit()
last_updated_utc = datetime.utcfromtimestamp(
last_updated).strftime('%Y-%m-%d %H:%M:%S')
# last_updated_utc = datetime.utcfromtimestamp(last_updated)
print(type(last_updated_utc))
# exit()
lastUpdatedTime = datetime.strptime(last_updated_utc,
"%Y-%m-%d %H:%M:%S")
# lastUpdatedTime = datetime.strptime(last_updated_utc, "%Y-%m-%d")
print(lastUpdatedTime)
print(type(lastUpdatedTime))
# exit()
# print(type(last_updated_utc))
# exit()
# print(type(last_updated_utc))
#
# tempData = []
# tempData.append(last_updated_utc)
# data.append(tempData)
# print(data)
# exit()
for item in coin_data:
# print(item[3])
if item[3] == coin_name:
#print(j[0])
tempData = []
tempData.append(item[0])
# tempData.append(rank)
# tempData.append(close_usd)
# tempData.append(close_btc)
# tempData.append(volume_usd)
# tempData.append(market_cap_usd)
tempData.append(available_supply)
tempData.append(total_supply)
tempData.append(max_supply)
tempData.append(lastUpdatedTime)
print(tempData)
# exit()
data.append(tempData)
# print(data)
# exit()
mymodelObj2 = MyModel()
mymodelObj2.insert_coin_price_minute(data=data)
def get_daily_data(self):
myModelObj = MyModel()
coin_data = myModelObj.get_coin_details()
# print(coin_data)
# exit()
# coin_data = [(728260,'EFX', 0, 'effect-ai', None, None, None)]
source_id = 1
# print(coin_data[0][1])
for coin_name in coin_data:
# print(coin_name[1])
coin_id = coin_name[0]
URL = "https://coinmarketcap.com/currencies/" + coin_name[
3] + "/historical-data/"
# URL = "https://coinmarketcap.com/currencies/bitcoin/historical-data/"
# URL = "https://coinmarketcap.com/currencies/quantum-resistant-ledger/historical-data/"
coinmarketcapObj = CoinmarketcapModel(URL)
rawData = coinmarketcapObj.get_data()
# print("raw data = ",rawData)
# exit()
updateData = []
if rawData:
try:
name = rawData[0][7]
print(name)
name = name.replace('Historical data for ', '')
print("name = ", name)
url = rawData[0][8]
print(url)
url = re.sub(r'http[s]?://', '', url)
url = re.sub(r'www[a-z0-9]?\.', '', url)
url = re.sub(r'/.*', '', url)
print("url = ", url)
norm_name = name
print(norm_name)
norm_name = re.sub(r'[\.\-_]*', '', norm_name)
norm_name = re.sub(r'\s*', '', norm_name)
norm_name = re.sub(r'\(.*\)', '', norm_name)
norm_name = norm_name.lower().strip()
print("norm name = ", norm_name)
updateData.append(name)
updateData.append(norm_name)
updateData.append(url)
updateData.append(coin_id)
print(updateData)
except:
pass
firstRecord = []
if rawData:
try:
firstRecord.append(rawData[0])
except:
pass
else:
firstRecord = []
cleanData = []
### cleaing data
for item in firstRecord:
idate = item[0]
myDate = datetime.strptime(idate, '%b %d, %Y').date()
if item[1] == "-":
open = 0
else:
open = float(item[1].replace(',', ''))
# high = float(item[2])
if item[2] == "-":
high = 0
else:
high = float(item[2].replace(',', ''))
# low = float(item[3])
if item[3] == "-":
low = 0
else:
low = float(item[3].replace(',', ''))
# close = float(item[4])
if item[4] == "-":
close = 0
else:
close = float(item[4].replace(',', ''))
if item[5] == "-":
volume = 0
else:
volume = float(item[5].replace(',', ''))
if item[6] == "-":
marketCap = 0
else:
marketCap = float(item[6].replace(',', ''))
#######
try:
del item[7]
del item[7]
except:
pass
item.append(coin_id)
item.append(source_id)
item[0] = myDate
item[1] = open
item[2] = high
item[3] = low
item[4] = close
item[5] = volume
item[6] = marketCap
# print(item)
cleanData.append(item)
print(cleanData)
# exit()
myModelObj = MyModel()
myModelObj.insert_coin_price_daily(data=cleanData)
#
myModelObj1 = MyModel()
myModelObj1.updateCoinDetails(data=updateData)
def get_all_historical_data(self):
end = datetime.today().date()
end = str(end).replace("-", "")
myModelObj = MyModel()
coin_data = myModelObj.get_coin_details()
# print(coin_data)
source_id = 1
for item_coin in coin_data:
coin_id = item_coin[0]
URL = "https://coinmarketcap.com/currencies/" + item_coin[
3] + "/historical-data/?start=20130428&end=" + end
print(URL)
# exit()
coinmarketcapObj = CoinmarketcapModel(URL)
rawData = coinmarketcapObj.get_daily_data()
no_of_records_coinmarketcap = len(rawData)
if no_of_records_coinmarketcap > 0:
# exit()
myModelObj1 = MyModel()
myModelObj1.del_records_coin_price_daily(coin_id=coin_id)
# exit()
cleanData = []
### cleaing data
for item in rawData:
idate = item[0]
myDate = datetime.strptime(idate, '%b %d, %Y').date()
if item[1] == "-":
open = 0
else:
open = float(item[1].replace(',', ''))
# high = float(item[2])
if item[2] == "-":
high = 0
else:
high = float(item[2].replace(',', ''))
# low = float(item[3])
if item[3] == "-":
low = 0
else:
low = float(item[3].replace(',', ''))
# close = float(item[4])
if item[4] == "-":
close = 0
else:
close = float(item[4].replace(',', ''))
if item[5] == "-":
volume = 0
else:
volume = float(item[5].replace(',', ''))
if item[6] == "-":
marketCap = 0
else:
marketCap = float(item[6].replace(',', ''))
# coin_id = coin_name[0]
#######
# del item[7]
# del item[7]
item.append(coin_id)
item.append(source_id)
item[0] = myDate
item[1] = open
item[2] = high
item[3] = low
item[4] = close
item[5] = volume
item[6] = marketCap
# print(" item = ",item)
cleanData.append(item)
# print("clean data = ", cleanData)
# exit()
# print(cleanData)
myModelObj2 = MyModel()
myModelObj2.insert_coin_price_daily(data=cleanData)
else:
# print(no_of_rows_crypto_database)
print("History Records are Up To Date")
def train(self, epochs, dataset, proc_size, num_conv_sets, num_dd_sets):
'''
Runs the whole shebang on init
'''
## Arguments that effect model and training
'''
EPOCHS = 3 ## number of training runs [10 - 15 -20]
IMAGE_PROC_SIZE = 100 ## [50 - 100 - 150]
num_conv_sets = 3 ## [1 - 2 - 3]
num_dd_sets = 1 ## [1 - 2 - 3]
iD = 'Manual_Testing'
'''
self.proc_size = proc_size
PRED_THRESHOLD = .5 ## necessary confidence threshold to predict a label
## setting up the file structure
flo = FileFlow()
train_dir = flo._validateBaseDir(self.train_dir)
labels = flo.getTarLabels(train_dir)
labels = [label for label in labels if label in dataset]
num_inp = len(labels)
self.labels = labels
flo.extractTarData(train_dir, labels)
flo.createFileStructure(train_dir, labels)
flo.seperateDatasets(train_dir, labels)
## creating output dir for results
save_to = self.save_dir
try:
os.mkdir(save_to)
except:
pass
## Preparing data
dg = DatGen()
training_dir = train_dir + 'all/train'
valid_dir = train_dir + 'all/valid'
test_dir = train_dir + 'all/test'
train_gen = dg.generateTrainingData(self.proc_size, training_dir)
valid_gen = dg.generateValidationData(self.proc_size, valid_dir)
test_gen = dg.generateTestData(self.proc_size, test_dir)
## Constructing Model
mml = MyModel()
model = mml.makeMyModel(num_conv_sets, num_dd_sets, num_inp,
self.proc_size)
self.model = [model]
## Training
trpr = TrainingProcess()
history = trpr.trainModel(model, epochs, train_gen, valid_gen)
## model is now trained
trpr.generateVisualTrainingResults(history, save_to, self.id)
## Testing
tepr = TestingProcess()
pred = tepr.runTest(test_gen, model)
tepr.generateTestingResults(labels, test_gen, pred, PRED_THRESHOLD,
save_to, self.id)
## cleaning up
flo.cleanUp(train_dir, labels)
def main_leePincombeWelsh():
f = open(Configuration.get_LOG(), "a+")
f.write(
"\n\n......................Experiment Begin......................\n")
f.write("%s\n" % t.time())
f.close()
Configuration.write_configuration(logfile=Configuration.get_LOG())
data_x_a, data_x_b, data_y, embedding_matrix = generate_data_leePincombeWelsh(
)
print(data_x_a.shape)
print(data_x_a[1:3, :])
print(data_x_b.shape)
print(data_y.shape)
print(embedding_matrix.shape)
print(embedding_matrix[1:3, :])
# exit()
indices = np.arange(data_x_a.shape[0])
np.random.shuffle(indices)
data_x_a = data_x_a[indices]
data_x_b = data_x_b[indices]
data_y = data_y[indices]
nb_validation_samples = int(Configuration.get_VALIDATION_SPLIT() *
data_x_a.shape[0])
x_a_train = data_x_a[:-nb_validation_samples]
x_b_train = data_x_b[:-nb_validation_samples]
y_train = data_y[:-nb_validation_samples]
x_a_test = data_x_a[-nb_validation_samples:]
x_b_test = data_x_b[-nb_validation_samples:]
y_test = data_y[-nb_validation_samples:]
# model = MyModel_New(embedding_matrix)
model = MyModel(inputLenght=Configuration.get_MAX_SEQUENCE_LENGTH(),
weights=embedding_matrix)
model.train(data_x_a=x_a_train,
data_x_b=x_b_train,
data_y=y_train,
batch_size=Configuration.get_BATCH_SIZE(),
episode=Configuration.get_EPIDOSE())
# print(model.predict(x_a_test, x_b_test))
f = open(Configuration.get_LOG(), mode="a+")
cosTrain, pearsonTrain = model.evaluate(
data_x_a=x_a_train,
data_x_b=x_b_train,
data_y=y_train,
batch_size=Configuration.get_BATCH_SIZE())
print("cosine of train data = %f, pearson of train data = %f" %
(cosTrain, pearsonTrain))
f.write("cosine of prediction of train data = %f\n" % cosTrain)
cosTest, pearsonTest = model.evaluate(
data_x_a=x_a_test,
data_x_b=x_b_test,
data_y=y_test,
batch_size=Configuration.get_BATCH_SIZE())
f.write("cosine of prediction of test data = %f\n" % cosTest)
print("cosine of test data = %f, pearson of test data = %f" %
(cosTest, pearsonTest))
f.close()
f = open(Configuration.get_LOG(), mode="a+")
f.write("......................Experiment End......................\n\n")
f.close()
