def get_all_airports_binarized(self, clean_data):

        transformed = DatasetCreation.writeCities_Airports(clean_data)
        airports_binarized = []
        for code, events in clean_data.items():

            for date, event in events.items():
                airports_binarized.append(DatasetCreation.getAirportBinarizedRepresentation(transformed, code))

        return np.array(airports_binarized)
Пример #2
0
    def get_all_airports_binarized(self, clean_data):

        transformed = DatasetCreation.writeCities_Airports(clean_data)
        airports_binarized = []
        for code, events in clean_data.items():

            for date, event in events.items():
                airports_binarized.append(
                    DatasetCreation.getAirportBinarizedRepresentation(
                        transformed, code))

        return np.array(airports_binarized)
Пример #3
0
def saveToCSVFile():
    #classes = ["cardiomegaly", "effusion" , "mass", "no finding", "other"]
    classes2 = ["Cardiomegaly", "Effusion", "Mass", "No Finding"]  #, "Other"]

    fileP = os.path.dirname(
        os.path.realpath(__file__)) + os.sep + "DataFrame.csv"
    df = DC.getDatasetFrame("version 1 data" + os.sep + "Images")
    # any(types in classification.lower() for types in classes):
    #df['Finding_Labels'] = np.where("|" in df['Finding_Labels'], "Other", df['Finding_Labels'])
    df = df[df['Finding_Labels'].isin(classes2)]

    # Shuffles the dataframe so that the testdata has some "no finding" data
    df = df.sample(frac=1).reset_index(drop=True)

    test = DC.saveDFToCSV(df, fileP)
Пример #4
0
    def preprocess_airports(self):
        firebase_clean = mapper.get_clean_firebase()
        all_clean = firebase_clean.get_all()

        transformed = DatasetCreation.writeCities_Airports(all_clean)
        firebase_clean.firebase.put("/metadata",
                                    name="airports",
                                    data=transformed)
    def binarize_airport(self, airport_code, airport_status):
        cleaned_data = utils.get_clean_data(airport_status)
        weather_binarized = self.get_weather_array(cleaned_data["weather"])
        airport_binarized = DatasetCreation.getAirportBinarizedRepresentation(self.airports_metadata, airport_code)
        wind = [ cleaned_data["wind_magnitude"] ]
        temp = [cleaned_data["temp"] ]
        visibility = [ cleaned_data["visibility"] ]
        delays = self.get_previous_delays_from_airport(airport_code)
        time_binarized = self.binarize_time(datetime.now())

        return np.concatenate((weather_binarized, wind, time_binarized, delays))
Пример #6
0
def main():
    # Configs
    config = configs.config()

    # Reading the selected data
    DataCSVFrame = pd.read_csv("DataFrame.csv",
                               usecols=["Image_Index", "Finding_Labels"],
                               index_col=False)
    labelsSet = set(DataCSVFrame["Finding_Labels"].values)

    # Dictionary with the label as key and the index in the set as value
    labelsDict = {}
    # Dictionary that is the reverse of the one above, to change back from value to the corresponding label
    labelDictClassify = {}

    # Filling the dictionaries
    for index, label in enumerate(labelsSet):
        labelsDict[label] = index
        labelDictClassify[index] = label

    # Path where all the images are stored
    imgPath = config.getImagePath()
    # Creating the dataset
    xrayDataset = DC.XRayDataset(DataCSVFrame, imgPath, labelsDict)

    #    # Getting the first image from the dataset
    #    imgs, labs = xrayDataset.__getitem__([8307])
    #    print(len(imgs))
    #    print(imgs)

    # Get the device (cpu/gpu) to run the model on
    device = DU.getDevice()

    # Gets the ranges of training and test data
    training, testing = DU.splitTrainTest(xrayDataset, config)

    # Get the train and validation sets
    trainSets, valSets = DU.trainValSets(training, config)

    # Initialize the criterion, optimizer and model
    criterion, optimizer, model = NM.modelInit(device)

    # Get the batchsize
    batchsize = config.getBatchSize()

    # Train the model
    trainedModel = NM.trainNetwork(device, xrayDataset, trainSets, valSets,
                                   config, model, criterion, optimizer,
                                   batchsize)

    # Save the model to be used for testing
    NM.save_model(trainedModel, config.getModelName())
Пример #7
0
    def binarize_airport(self, airport_code, airport_status):
        cleaned_data = utils.get_clean_data(airport_status)
        weather_binarized = self.get_weather_array(cleaned_data["weather"])
        airport_binarized = DatasetCreation.getAirportBinarizedRepresentation(
            self.airports_metadata, airport_code)
        wind = [cleaned_data["wind_magnitude"]]
        temp = [cleaned_data["temp"]]
        visibility = [cleaned_data["visibility"]]
        delays = self.get_previous_delays_from_airport(airport_code)
        time_binarized = self.binarize_time(datetime.now())

        return np.concatenate(
            (weather_binarized, wind, time_binarized, delays))
Пример #8
0
def search_for_track(query):
    sp = dc.spotify_auth()
    result = sp.search(query)

    try:
        track_id = result["tracks"]["items"][0]["id"]

        audio_features = sp.audio_features(tracks=[track_id])
        res = {
            key: audio_features[0][key]
            for key in audio_features[0].keys() & {
                'danceability', 'energy', 'key', 'loudness', 'mode',
                'speechiness', 'acousticness', 'instrumentalness', 'liveness',
                'valence', 'tempo', 'id'
            }
        }

        track = sp.track(res["id"])
        res["name"] = track["name"]
        res["artist"] = track["album"]["artists"][0]["name"]
        # print(res)
        new_track = []
        new_track.append(res)
        new_searched_song = pd.DataFrame(new_track)
        pd.DataFrame(new_searched_song).to_pickle("./new_searched_track1.pkl")

        # to reorder features correctly
        searched_track = new_searched_song[[
            'id', 'name', 'artist', 'energy', 'danceability', 'acousticness',
            'instrumentalness', 'mode', 'liveness', 'key', 'tempo', 'valence',
            'loudness', 'speechiness'
        ]]

        # to get only required features
        features = searched_track.loc[:, 'energy':'speechiness']
        cols_to_standardize = features.columns.tolist()

        # return searched_track[cols_to_standardize]
        return searched_track[cols_to_standardize]
    except:
        return None
Пример #9
0
def main():
    # Configs
    config = configs.config()

    # Reading the selected data
    DataCSVFrame = pd.read_csv("DataFrame.csv",
                               usecols=["Image_Index", "Finding_Labels"],
                               index_col=False)

    # Get the labels
    labelsSet = set(DataCSVFrame["Finding_Labels"].values)

    # Dictionary with the label as key and the index in the set as value
    labelsDict = {}
    # Dictionary that is the reverse of the one above, to change back from value to the corresponding label
    labelDictClassify = {}

    # Filling the dictionaries
    for index, label in enumerate(labelsSet):
        labelsDict[label] = index
        labelDictClassify[index] = label

    # Path where all the images are stored
    imgPath = config.getImagePath()
    # Creating the dataset
    xrayDataset = DC.XRayDataset(DataCSVFrame, imgPath, labelsDict)

    #print(xrayDataset.xrayClassFrame)

    # Get the device (cpu/gpu) to run the model on
    device = DU.getDevice()

    # Gets the ranges of training and test data
    training, testing = DU.splitTrainTest(xrayDataset, config)

    # Initialize the model
    model = models.alexnet(pretrained=False, num_classes=4)
    model.to(device)
    if device == 'cuda':
        model = torch.nn.DataParallel(model)
        cudnn.benchmark = True

    #print(model)

    # Load the trained model
    cwd = os.path.dirname(os.path.realpath(__file__))
    model.load_state_dict(
        torch.load("%s%s%s.pth" % (cwd, os.sep, config.getModelName())))
    model.eval()

    # Testing the model
    wrongLabels, labelsCM, predsCM = NM.testing(xrayDataset, testing, model,
                                                device, labelDictClassify)

    # Confusion Matrix
    CMPlot.plot_confusion_matrix(labelsCM,
                                 predsCM,
                                 list(labelsSet),
                                 normalize=False,
                                 title="Confusion Matrix")

    # Show the matrix
    plt.show()
    def preprocess_airports(self):
        firebase_clean = mapper.get_clean_firebase()
        all_clean = firebase_clean.get_all()

        transformed = DatasetCreation.writeCities_Airports(all_clean)
        firebase_clean.firebase.put("/metadata", name="airports", data=transformed)