def get_all_airports_binarized(self, clean_data): transformed = DatasetCreation.writeCities_Airports(clean_data) airports_binarized = [] for code, events in clean_data.items(): for date, event in events.items(): airports_binarized.append(DatasetCreation.getAirportBinarizedRepresentation(transformed, code)) return np.array(airports_binarized)
def get_all_airports_binarized(self, clean_data): transformed = DatasetCreation.writeCities_Airports(clean_data) airports_binarized = [] for code, events in clean_data.items(): for date, event in events.items(): airports_binarized.append( DatasetCreation.getAirportBinarizedRepresentation( transformed, code)) return np.array(airports_binarized)
def saveToCSVFile(): #classes = ["cardiomegaly", "effusion" , "mass", "no finding", "other"] classes2 = ["Cardiomegaly", "Effusion", "Mass", "No Finding"] #, "Other"] fileP = os.path.dirname( os.path.realpath(__file__)) + os.sep + "DataFrame.csv" df = DC.getDatasetFrame("version 1 data" + os.sep + "Images") # any(types in classification.lower() for types in classes): #df['Finding_Labels'] = np.where("|" in df['Finding_Labels'], "Other", df['Finding_Labels']) df = df[df['Finding_Labels'].isin(classes2)] # Shuffles the dataframe so that the testdata has some "no finding" data df = df.sample(frac=1).reset_index(drop=True) test = DC.saveDFToCSV(df, fileP)
def preprocess_airports(self): firebase_clean = mapper.get_clean_firebase() all_clean = firebase_clean.get_all() transformed = DatasetCreation.writeCities_Airports(all_clean) firebase_clean.firebase.put("/metadata", name="airports", data=transformed)
def binarize_airport(self, airport_code, airport_status): cleaned_data = utils.get_clean_data(airport_status) weather_binarized = self.get_weather_array(cleaned_data["weather"]) airport_binarized = DatasetCreation.getAirportBinarizedRepresentation(self.airports_metadata, airport_code) wind = [ cleaned_data["wind_magnitude"] ] temp = [cleaned_data["temp"] ] visibility = [ cleaned_data["visibility"] ] delays = self.get_previous_delays_from_airport(airport_code) time_binarized = self.binarize_time(datetime.now()) return np.concatenate((weather_binarized, wind, time_binarized, delays))
def main(): # Configs config = configs.config() # Reading the selected data DataCSVFrame = pd.read_csv("DataFrame.csv", usecols=["Image_Index", "Finding_Labels"], index_col=False) labelsSet = set(DataCSVFrame["Finding_Labels"].values) # Dictionary with the label as key and the index in the set as value labelsDict = {} # Dictionary that is the reverse of the one above, to change back from value to the corresponding label labelDictClassify = {} # Filling the dictionaries for index, label in enumerate(labelsSet): labelsDict[label] = index labelDictClassify[index] = label # Path where all the images are stored imgPath = config.getImagePath() # Creating the dataset xrayDataset = DC.XRayDataset(DataCSVFrame, imgPath, labelsDict) # # Getting the first image from the dataset # imgs, labs = xrayDataset.__getitem__([8307]) # print(len(imgs)) # print(imgs) # Get the device (cpu/gpu) to run the model on device = DU.getDevice() # Gets the ranges of training and test data training, testing = DU.splitTrainTest(xrayDataset, config) # Get the train and validation sets trainSets, valSets = DU.trainValSets(training, config) # Initialize the criterion, optimizer and model criterion, optimizer, model = NM.modelInit(device) # Get the batchsize batchsize = config.getBatchSize() # Train the model trainedModel = NM.trainNetwork(device, xrayDataset, trainSets, valSets, config, model, criterion, optimizer, batchsize) # Save the model to be used for testing NM.save_model(trainedModel, config.getModelName())
def binarize_airport(self, airport_code, airport_status): cleaned_data = utils.get_clean_data(airport_status) weather_binarized = self.get_weather_array(cleaned_data["weather"]) airport_binarized = DatasetCreation.getAirportBinarizedRepresentation( self.airports_metadata, airport_code) wind = [cleaned_data["wind_magnitude"]] temp = [cleaned_data["temp"]] visibility = [cleaned_data["visibility"]] delays = self.get_previous_delays_from_airport(airport_code) time_binarized = self.binarize_time(datetime.now()) return np.concatenate( (weather_binarized, wind, time_binarized, delays))
def search_for_track(query): sp = dc.spotify_auth() result = sp.search(query) try: track_id = result["tracks"]["items"][0]["id"] audio_features = sp.audio_features(tracks=[track_id]) res = { key: audio_features[0][key] for key in audio_features[0].keys() & { 'danceability', 'energy', 'key', 'loudness', 'mode', 'speechiness', 'acousticness', 'instrumentalness', 'liveness', 'valence', 'tempo', 'id' } } track = sp.track(res["id"]) res["name"] = track["name"] res["artist"] = track["album"]["artists"][0]["name"] # print(res) new_track = [] new_track.append(res) new_searched_song = pd.DataFrame(new_track) pd.DataFrame(new_searched_song).to_pickle("./new_searched_track1.pkl") # to reorder features correctly searched_track = new_searched_song[[ 'id', 'name', 'artist', 'energy', 'danceability', 'acousticness', 'instrumentalness', 'mode', 'liveness', 'key', 'tempo', 'valence', 'loudness', 'speechiness' ]] # to get only required features features = searched_track.loc[:, 'energy':'speechiness'] cols_to_standardize = features.columns.tolist() # return searched_track[cols_to_standardize] return searched_track[cols_to_standardize] except: return None
def main(): # Configs config = configs.config() # Reading the selected data DataCSVFrame = pd.read_csv("DataFrame.csv", usecols=["Image_Index", "Finding_Labels"], index_col=False) # Get the labels labelsSet = set(DataCSVFrame["Finding_Labels"].values) # Dictionary with the label as key and the index in the set as value labelsDict = {} # Dictionary that is the reverse of the one above, to change back from value to the corresponding label labelDictClassify = {} # Filling the dictionaries for index, label in enumerate(labelsSet): labelsDict[label] = index labelDictClassify[index] = label # Path where all the images are stored imgPath = config.getImagePath() # Creating the dataset xrayDataset = DC.XRayDataset(DataCSVFrame, imgPath, labelsDict) #print(xrayDataset.xrayClassFrame) # Get the device (cpu/gpu) to run the model on device = DU.getDevice() # Gets the ranges of training and test data training, testing = DU.splitTrainTest(xrayDataset, config) # Initialize the model model = models.alexnet(pretrained=False, num_classes=4) model.to(device) if device == 'cuda': model = torch.nn.DataParallel(model) cudnn.benchmark = True #print(model) # Load the trained model cwd = os.path.dirname(os.path.realpath(__file__)) model.load_state_dict( torch.load("%s%s%s.pth" % (cwd, os.sep, config.getModelName()))) model.eval() # Testing the model wrongLabels, labelsCM, predsCM = NM.testing(xrayDataset, testing, model, device, labelDictClassify) # Confusion Matrix CMPlot.plot_confusion_matrix(labelsCM, predsCM, list(labelsSet), normalize=False, title="Confusion Matrix") # Show the matrix plt.show()