Esempio n. 1
0
    def __init__(self):
        """

        :return:
        """
        self.model = MyModel()
        self.model.setPath("data/file.csv")
Esempio n. 2
0
def run():
    """Builds model, loads data, trains and evaluates"""
    #df = dataloader.DataLoader().load_data(CFG)
    #print(df[2])
    model = MyModel(CFG)
    model.load_data()
    model.build()
    metrics = pd.DataFrame(model.train())
    print(metrics)
 def get_coin_details(self):
     URL = "https://coinmarketcap.com/all/views/all/"
     obj = CoinmarketcapModel(URL)
     rawData = obj.get_coin_data()
     # print(rawData)
     # exit()
     #myData = [item[1] for item in data]
     cleanData = []
     for item in rawData:
         #myList = str(item[1]).split("\n")
         tempList = []
         symbol = item[2]
         coin_name = item[10]
         coin_name = coin_name.split("id-")[1]
         tempList.append(symbol)
         tempList.append(coin_name)
         cleanData.append(tempList)
     print(cleanData)
     # exit()
     myModelObj = MyModel()
     myModelObj.insert_coin_details(data=cleanData)
    def updateCoinDetails(self):
        myModelObj = MyModel()
        coin_data = myModelObj.get_coin_details()
        # print(coin_data)
        # coin_data = ["bitcoin","ethrium"]
        # source_id = 1
        # print(coin_data[0][1])
        for coin_name in coin_data:
            # print(coin_name[1])
            URL = "https://coinmarketcap.com/currencies/" + coin_name[
                1] + "/historical-data/"
            # URL = "https://coinmarketcap.com/currencies/bitcoin/historical-data/"
            # URL = "https://coinmarketcap.com/currencies/quantum-resistant-ledger/historical-data/"
            coinmarketcapObj = CoinmarketcapModel(URL)
            rawData = coinmarketcapObj.get_coin_details_for_update()
            # print("raw data = ", rawData)
            updateData = []
            if rawData:
                try:
                    name = rawData[0]
                    # print(name)
                    name = name.replace('Historical data for ', '')
                # print(name)
                except:
                    name = ""
                try:

                    url = rawData[1]
                    # print(url)
                    url = re.sub(r'http[s]?://', '', url)
                    url = re.sub(r'www[a-z0-9]?\.', '', url)
                    url = re.sub(r'/.*', '', url)
                    # print(url)
                except:
                    url = ""
                if name:
                    norm_name = name
                    # print(norm_name)
                    norm_name = re.sub(r'[\.\-_]*', '', norm_name)
                    norm_name = re.sub(r'\s*', '', norm_name)
                    norm_name = re.sub(r'\(.*\)', '', norm_name)
                    norm_name = norm_name.lower().strip()
                else:
                    norm_name = ""
                # print(norm_name)

                updateData.append(name)
                updateData.append(norm_name)
                updateData.append(url)
                updateData.append(coin_name[1])
                print(updateData)

            myModelObj = MyModel()
            myModelObj.updateCoinDetails(data=updateData)
Esempio n. 5
0
class Test(object):

    def __init__(self):
        """

        :return:
        """
        self.model = MyModel()
        self.model.setPath("data/file.csv")

    def test01_save(self):
        """
        Save
        Fungiert wie @Before
        :return:
        """
        self.model.saveCSV("a;b;c;\nd;e\nf")
        assert 1 == 1 #Es ist kein Fehler beim speichern aufgetreten

    def test02_load(self):
        """
        Load
        :return:
        """
        print(self.model.loadCSV())
        assert self.model.loadCSV() == "a;b;c;\nd;e\nf\n"


    def test03_SaveLoad(self):
        """

        :return:
        """
        text = "z;x;\nh;i\nu;e;"
        self.model.saveCSV(text)
        assert self.model.loadCSV() == text+"\n"

    def test04_SaveSave(self):
        """
        Es ist möglich das file oeffter hinter einander zu beschreibem.
        :return:
        """
        text = "z;xjzhguz;\nh;i\nu;e;"
        self.model.saveCSV(text)
        self.model.saveCSV(text)
        assert self.model.loadCSV() == text+"\n"

    def test05_LoadLoad(self):
        """
        Oefter laden
        :return:
        """
        self.model.loadCSV()
        assert self.model.loadCSV() == "z;xjzhguz;\nh;i\nu;e;\n";
    def get_daily_data_records(self, no_records):
        myModelObj = MyModel()
        coin_data = myModelObj.get_coin_details()
        # print(coin_data)
        source_id = 1
        print(no_records)
        # print(coin_data[0][1])
        for coin_name in coin_data:
            # print(coin_name[1])
            URL = "https://coinmarketcap.com/currencies/" + coin_name[
                1] + "/historical-data/"
            #URL = "https://coinmarketcap.com/currencies/mithril/historical-data/"
            #URL = "https://coinmarketcap.com/currencies/fsfdsf/historical-data/"
            coinmarketcapObj = CoinmarketcapModel(URL)
            rawData = coinmarketcapObj.get_data()
            # print(rawData)
            records = []
            if rawData:
                # print(rawData[0])
                j = no_records
                if len(rawData) <= j:
                    i = 0
                    while i < len(rawData):
                        records.append(rawData[i])
                        i = i + 1
                else:
                    i = 0
                    while i < j:
                        records.append(rawData[i])
                        i = i + 1
            else:
                records = []

            print(records)
            #exit()
            cleanData = []
            ### cleaing data
            for item in records:
                # print(item)
                # print(item[0])
                idate = item[0]
                myDate = datetime.strptime(idate, '%b %d, %Y').date()
                # print(date.date())

                # print(date)
                # open = float(item[1])
                if item[1] == "-":
                    open = 0
                else:
                    open = float(item[1].replace(',', ''))
                # high = float(item[2])
                if item[2] == "-":
                    high = 0
                else:
                    high = float(item[2].replace(',', ''))
                # low = float(item[3])
                if item[3] == "-":
                    low = 0
                else:
                    low = float(item[3].replace(',', ''))
                # close = float(item[4])
                if item[4] == "-":
                    close = 0
                else:
                    close = float(item[4].replace(',', ''))
                if item[5] == "-":
                    volume = 0
                else:
                    volume = float(item[5].replace(',', ''))
                if item[6] == "-":
                    marketCap = 0
                else:
                    marketCap = float(item[6].replace(',', ''))
                coin_id = coin_name[0]
                #######
                item.append(coin_id)
                item.append(source_id)
                item[0] = myDate
                item[1] = open
                item[2] = high
                item[3] = low
                item[4] = close
                item[5] = volume
                item[6] = marketCap

                # print(item)
                cleanData.append(item)

            print(cleanData)
            #exit()
            myModelObj = MyModel()
            myModelObj.insert_coin_price_daily(data=cleanData)
    def get_history_data(self):
        end = datetime.today().date()
        end = str(end).replace("-", "")
        # print(type(end))
        # print("end = ",end)
        # exit()
        myModelObj = MyModel()
        coin_data = myModelObj.get_coin_price_daily_records_by_range(start=0,
                                                                     end=100)
        # print(coin_data)
        source_id = 1
        # coin_data = [[1417,'safecoin',4]]
        #print(coin_data[0][1])
        # print(coin_data)
        print(len(coin_data))
        # exit()
        for item_coin in coin_data:
            coin_id = item_coin[0]
            no_of_rows_crypto_database = item_coin[2]
            print(type(no_of_rows_crypto_database))
            URL = "https://coinmarketcap.com/currencies/" + item_coin[
                1] + "/historical-data/?start=20130428&end=" + end
            print(URL)
            # exit()
            # URL = "https://coinmarketcap.com/currencies/slimcoin/historical-data/?start=20130428&end=20180430"
            #URL = "https://coinmarketcap.com/currencies/farstcoin/historical-data/?start=20130428&end=20180311"
            coinmarketcapObj = CoinmarketcapModel(URL)
            rawData = coinmarketcapObj.get_daily_data()
            # print(rawData)
            # exit()
            no_of_records_coinmarketcap = len(rawData)
            print("Number of records in coinmarketcap = ",
                  no_of_records_coinmarketcap)
            print("Number of records in crypto database = ",
                  no_of_rows_crypto_database)
            # exit()
            if no_of_records_coinmarketcap > no_of_rows_crypto_database:
                print(no_of_records_coinmarketcap)
                print(coin_id)
                # exit()
                myModelObj1 = MyModel()
                myModelObj1.del_records_coin_price_daily(coin_id=coin_id)
                # exit()
                cleanData = []
                ### cleaing data
                for item in rawData:
                    # print(item)
                    # print(item[0])
                    idate = item[0]
                    myDate = datetime.strptime(idate, '%b %d, %Y').date()
                    # print(date.date())

                    # print(date)
                    # open = float(item[1])
                    if item[1] == "-":
                        open = 0
                    else:
                        open = float(item[1].replace(',', ''))
                    # high = float(item[2])
                    if item[2] == "-":
                        high = 0
                    else:
                        high = float(item[2].replace(',', ''))
                    # low = float(item[3])
                    if item[3] == "-":
                        low = 0
                    else:
                        low = float(item[3].replace(',', ''))
                    # close = float(item[4])
                    if item[4] == "-":
                        close = 0
                    else:
                        close = float(item[4].replace(',', ''))
                    if item[5] == "-":
                        volume = 0
                    else:
                        volume = float(item[5].replace(',', ''))
                    if item[6] == "-":
                        marketCap = 0
                    else:
                        marketCap = float(item[6].replace(',', ''))
                    # coin_id = coin_name[0]
                    #######
                    # del item[7]
                    # del item[7]
                    item.append(coin_id)
                    item.append(source_id)
                    item[0] = myDate
                    item[1] = open
                    item[2] = high
                    item[3] = low
                    item[4] = close
                    item[5] = volume
                    item[6] = marketCap

                    # print(" item  = ",item)
                    cleanData.append(item)
                    # print("clean data = ", cleanData)
                    # exit()
                # print(cleanData)
                myModelObj2 = MyModel()
                myModelObj2.insert_coin_price_daily(data=cleanData)
            else:
                # print(no_of_rows_crypto_database)
                print("History Records are Up To Date")
    def get_minute_data(self):
        mymodelObj = MyModel()
        coin_data = mymodelObj.get_coin_details()
        #print(coin_data)
        url = "https://api.coinmarketcap.com/v1/ticker/?limit=0"
        coinmarketcapObj = CoinmarketcapModel(url=url)
        jsonData = coinmarketcapObj.get_minute_data()
        # print(jsonData)
        # exit()
        data = []
        for i in jsonData:
            #print(i['id'])
            coin_name = i['id']
            rank = int(i['rank'])

            if i['price_usd'] == None:
                close_usd = 0
            else:
                close_usd = float(i['price_usd'])

            if i['price_btc'] == None:
                close_btc = 0
            else:
                close_btc = float(i['price_btc'])

            if i['24h_volume_usd'] == None:
                volume_usd = 0
            else:
                volume_usd = float(i['24h_volume_usd'])

            if i['market_cap_usd'] == None:
                market_cap_usd = 0
            else:
                market_cap_usd = float(i['market_cap_usd'])

            if i['available_supply'] == None:
                available_supply = 0
            else:
                available_supply = float(i['available_supply'])

            if i['total_supply'] == None:
                total_supply = 0
            else:
                total_supply = float(i['total_supply'])

            if i['max_supply'] == None:
                max_supply = 0
            else:
                max_supply = float(i['max_supply'])

            if i['last_updated'] == None:
                lastUpdatedTime = 0
                # print("i m in if ")
            else:
                last_updated = int(i['last_updated'])
                # print("i m in else ")
                print(last_updated)
                # exit()
                last_updated_utc = datetime.utcfromtimestamp(
                    last_updated).strftime('%Y-%m-%d %H:%M:%S')
                # last_updated_utc = datetime.utcfromtimestamp(last_updated)
                print(type(last_updated_utc))
                # exit()
                lastUpdatedTime = datetime.strptime(last_updated_utc,
                                                    "%Y-%m-%d %H:%M:%S")
                # lastUpdatedTime = datetime.strptime(last_updated_utc, "%Y-%m-%d")
                print(lastUpdatedTime)
                print(type(lastUpdatedTime))
                # exit()
                # print(type(last_updated_utc))
                # exit()
                # print(type(last_updated_utc))
                #
                # tempData = []
                # tempData.append(last_updated_utc)
                # data.append(tempData)
                # print(data)
            # exit()

            for item in coin_data:
                # print(item[3])
                if item[3] == coin_name:
                    #print(j[0])
                    tempData = []
                    tempData.append(item[0])
                    # tempData.append(rank)
                    # tempData.append(close_usd)
                    # tempData.append(close_btc)
                    # tempData.append(volume_usd)
                    # tempData.append(market_cap_usd)
                    tempData.append(available_supply)
                    tempData.append(total_supply)
                    tempData.append(max_supply)
                    tempData.append(lastUpdatedTime)
                    print(tempData)
                    # exit()
                    data.append(tempData)

        # print(data)
        # exit()
        mymodelObj2 = MyModel()
        mymodelObj2.insert_coin_price_minute(data=data)
    def get_daily_data(self):
        myModelObj = MyModel()
        coin_data = myModelObj.get_coin_details()
        # print(coin_data)
        # exit()
        # coin_data = [(728260,'EFX', 0, 'effect-ai', None, None, None)]
        source_id = 1
        # print(coin_data[0][1])
        for coin_name in coin_data:
            # print(coin_name[1])
            coin_id = coin_name[0]
            URL = "https://coinmarketcap.com/currencies/" + coin_name[
                3] + "/historical-data/"
            # URL = "https://coinmarketcap.com/currencies/bitcoin/historical-data/"
            # URL = "https://coinmarketcap.com/currencies/quantum-resistant-ledger/historical-data/"
            coinmarketcapObj = CoinmarketcapModel(URL)
            rawData = coinmarketcapObj.get_data()
            # print("raw data = ",rawData)
            # exit()
            updateData = []
            if rawData:
                try:
                    name = rawData[0][7]
                    print(name)
                    name = name.replace('Historical data for ', '')
                    print("name = ", name)
                    url = rawData[0][8]
                    print(url)
                    url = re.sub(r'http[s]?://', '', url)
                    url = re.sub(r'www[a-z0-9]?\.', '', url)
                    url = re.sub(r'/.*', '', url)
                    print("url  = ", url)
                    norm_name = name
                    print(norm_name)
                    norm_name = re.sub(r'[\.\-_]*', '', norm_name)
                    norm_name = re.sub(r'\s*', '', norm_name)
                    norm_name = re.sub(r'\(.*\)', '', norm_name)
                    norm_name = norm_name.lower().strip()
                    print("norm name = ", norm_name)
                    updateData.append(name)
                    updateData.append(norm_name)
                    updateData.append(url)
                    updateData.append(coin_id)
                    print(updateData)
                except:
                    pass
            firstRecord = []
            if rawData:
                try:
                    firstRecord.append(rawData[0])
                except:
                    pass
            else:
                firstRecord = []
            cleanData = []
            ### cleaing data
            for item in firstRecord:
                idate = item[0]
                myDate = datetime.strptime(idate, '%b %d, %Y').date()
                if item[1] == "-":
                    open = 0
                else:
                    open = float(item[1].replace(',', ''))
                # high = float(item[2])
                if item[2] == "-":
                    high = 0
                else:
                    high = float(item[2].replace(',', ''))
                # low = float(item[3])
                if item[3] == "-":
                    low = 0
                else:
                    low = float(item[3].replace(',', ''))
                # close = float(item[4])
                if item[4] == "-":
                    close = 0
                else:
                    close = float(item[4].replace(',', ''))
                if item[5] == "-":
                    volume = 0
                else:
                    volume = float(item[5].replace(',', ''))
                if item[6] == "-":
                    marketCap = 0
                else:
                    marketCap = float(item[6].replace(',', ''))

                #######
                try:
                    del item[7]
                    del item[7]
                except:
                    pass

                item.append(coin_id)
                item.append(source_id)
                item[0] = myDate
                item[1] = open
                item[2] = high
                item[3] = low
                item[4] = close
                item[5] = volume
                item[6] = marketCap

                # print(item)
                cleanData.append(item)

            print(cleanData)
            # exit()
            myModelObj = MyModel()
            myModelObj.insert_coin_price_daily(data=cleanData)
            #
            myModelObj1 = MyModel()
            myModelObj1.updateCoinDetails(data=updateData)
    def get_all_historical_data(self):
        end = datetime.today().date()
        end = str(end).replace("-", "")
        myModelObj = MyModel()
        coin_data = myModelObj.get_coin_details()
        # print(coin_data)
        source_id = 1
        for item_coin in coin_data:
            coin_id = item_coin[0]
            URL = "https://coinmarketcap.com/currencies/" + item_coin[
                3] + "/historical-data/?start=20130428&end=" + end
            print(URL)
            # exit()
            coinmarketcapObj = CoinmarketcapModel(URL)
            rawData = coinmarketcapObj.get_daily_data()
            no_of_records_coinmarketcap = len(rawData)

            if no_of_records_coinmarketcap > 0:
                # exit()
                myModelObj1 = MyModel()
                myModelObj1.del_records_coin_price_daily(coin_id=coin_id)
                # exit()
                cleanData = []
                ### cleaing data
                for item in rawData:
                    idate = item[0]
                    myDate = datetime.strptime(idate, '%b %d, %Y').date()
                    if item[1] == "-":
                        open = 0
                    else:
                        open = float(item[1].replace(',', ''))
                    # high = float(item[2])
                    if item[2] == "-":
                        high = 0
                    else:
                        high = float(item[2].replace(',', ''))
                    # low = float(item[3])
                    if item[3] == "-":
                        low = 0
                    else:
                        low = float(item[3].replace(',', ''))
                    # close = float(item[4])
                    if item[4] == "-":
                        close = 0
                    else:
                        close = float(item[4].replace(',', ''))
                    if item[5] == "-":
                        volume = 0
                    else:
                        volume = float(item[5].replace(',', ''))
                    if item[6] == "-":
                        marketCap = 0
                    else:
                        marketCap = float(item[6].replace(',', ''))
                    # coin_id = coin_name[0]
                    #######
                    # del item[7]
                    # del item[7]
                    item.append(coin_id)
                    item.append(source_id)
                    item[0] = myDate
                    item[1] = open
                    item[2] = high
                    item[3] = low
                    item[4] = close
                    item[5] = volume
                    item[6] = marketCap

                    # print(" item  = ",item)
                    cleanData.append(item)
                    # print("clean data = ", cleanData)
                    # exit()
                # print(cleanData)
                myModelObj2 = MyModel()
                myModelObj2.insert_coin_price_daily(data=cleanData)

            else:

                # print(no_of_rows_crypto_database)
                print("History Records are Up To Date")
    def train(self, epochs, dataset, proc_size, num_conv_sets, num_dd_sets):
        '''
        
        Runs the whole shebang on init
        '''

        ## Arguments that effect model and training
        '''
        EPOCHS = 3   ## number of training runs [10 - 15 -20]
        IMAGE_PROC_SIZE = 100 ##   [50 - 100 - 150]
        
        num_conv_sets = 3 ## [1 - 2 - 3]
        num_dd_sets = 1 ## [1 - 2 - 3]
        
        iD = 'Manual_Testing'
        '''
        self.proc_size = proc_size
        PRED_THRESHOLD = .5  ## necessary confidence threshold to predict a label

        ## setting up the file structure

        flo = FileFlow()

        train_dir = flo._validateBaseDir(self.train_dir)

        labels = flo.getTarLabels(train_dir)
        labels = [label for label in labels if label in dataset]
        num_inp = len(labels)
        self.labels = labels

        flo.extractTarData(train_dir, labels)

        flo.createFileStructure(train_dir, labels)

        flo.seperateDatasets(train_dir, labels)

        ## creating output dir for results

        save_to = self.save_dir
        try:
            os.mkdir(save_to)
        except:
            pass

        ## Preparing data

        dg = DatGen()

        training_dir = train_dir + 'all/train'
        valid_dir = train_dir + 'all/valid'
        test_dir = train_dir + 'all/test'

        train_gen = dg.generateTrainingData(self.proc_size, training_dir)
        valid_gen = dg.generateValidationData(self.proc_size, valid_dir)
        test_gen = dg.generateTestData(self.proc_size, test_dir)

        ## Constructing Model

        mml = MyModel()

        model = mml.makeMyModel(num_conv_sets, num_dd_sets, num_inp,
                                self.proc_size)
        self.model = [model]

        ## Training

        trpr = TrainingProcess()

        history = trpr.trainModel(model, epochs, train_gen, valid_gen)
        ## model is now trained

        trpr.generateVisualTrainingResults(history, save_to, self.id)

        ## Testing

        tepr = TestingProcess()

        pred = tepr.runTest(test_gen, model)

        tepr.generateTestingResults(labels, test_gen, pred, PRED_THRESHOLD,
                                    save_to, self.id)

        ## cleaning up

        flo.cleanUp(train_dir, labels)
Esempio n. 12
0
File: laucher.py Progetto: esgl/GXY
def main_leePincombeWelsh():

    f = open(Configuration.get_LOG(), "a+")
    f.write(
        "\n\n......................Experiment Begin......................\n")
    f.write("%s\n" % t.time())
    f.close()
    Configuration.write_configuration(logfile=Configuration.get_LOG())

    data_x_a, data_x_b, data_y, embedding_matrix = generate_data_leePincombeWelsh(
    )
    print(data_x_a.shape)
    print(data_x_a[1:3, :])
    print(data_x_b.shape)
    print(data_y.shape)
    print(embedding_matrix.shape)
    print(embedding_matrix[1:3, :])
    # exit()

    indices = np.arange(data_x_a.shape[0])
    np.random.shuffle(indices)

    data_x_a = data_x_a[indices]
    data_x_b = data_x_b[indices]
    data_y = data_y[indices]

    nb_validation_samples = int(Configuration.get_VALIDATION_SPLIT() *
                                data_x_a.shape[0])
    x_a_train = data_x_a[:-nb_validation_samples]
    x_b_train = data_x_b[:-nb_validation_samples]
    y_train = data_y[:-nb_validation_samples]

    x_a_test = data_x_a[-nb_validation_samples:]
    x_b_test = data_x_b[-nb_validation_samples:]
    y_test = data_y[-nb_validation_samples:]

    # model = MyModel_New(embedding_matrix)
    model = MyModel(inputLenght=Configuration.get_MAX_SEQUENCE_LENGTH(),
                    weights=embedding_matrix)
    model.train(data_x_a=x_a_train,
                data_x_b=x_b_train,
                data_y=y_train,
                batch_size=Configuration.get_BATCH_SIZE(),
                episode=Configuration.get_EPIDOSE())

    # print(model.predict(x_a_test, x_b_test))
    f = open(Configuration.get_LOG(), mode="a+")
    cosTrain, pearsonTrain = model.evaluate(
        data_x_a=x_a_train,
        data_x_b=x_b_train,
        data_y=y_train,
        batch_size=Configuration.get_BATCH_SIZE())
    print("cosine of train data = %f, pearson of train data = %f" %
          (cosTrain, pearsonTrain))
    f.write("cosine of prediction of train data = %f\n" % cosTrain)

    cosTest, pearsonTest = model.evaluate(
        data_x_a=x_a_test,
        data_x_b=x_b_test,
        data_y=y_test,
        batch_size=Configuration.get_BATCH_SIZE())
    f.write("cosine of prediction of test data = %f\n" % cosTest)
    print("cosine of test data = %f, pearson of test data = %f" %
          (cosTest, pearsonTest))
    f.close()

    f = open(Configuration.get_LOG(), mode="a+")
    f.write("......................Experiment End......................\n\n")
    f.close()