def test_create_models(): coeffs_data = open('all_features.txt','r').read().encode('utf-8') pca_data = open('pca_features.txt','r').read().encode('utf-8') (rf, pca) = models.create_models(coeffs_data, pca_data) assert type(rf) == type(RandomForestClassifier()) assert type(pca) == type(decomposition.pca.PCA())
def classify_lightcurves(): dataFileNamesR = [] dataFileNamesG = [] fileDirectoryR = lightcurve_path + 'Rfilter/' fileDirectoryG = lightcurve_path + 'Gfilter/' fileDirectoryClassifiedR = lightcurve_path + 'classified/Rfilter/' fileDirectoryClassifiedG = lightcurve_path + 'classified/Gfilter/' for f in os.listdir(fileDirectoryR): if f.endswith('.dat'): dataFileNamesR.append(f) for f in os.listdir(fileDirectoryG): if f.endswith('.dat'): dataFileNamesG.append(f) dataFileNamesR = natsorted(dataFileNamesR) dataFileNamesG = natsorted(dataFileNamesG) rf, pca = models.create_models(features_path + 'all_features.txt', features_path + 'pca_features.txt') class_results = [] for filename in dataFileNamesR: data = np.loadtxt(fileDirectoryR + filename, usecols=np.arange(0, 3)) mjd = [float(i) for i in data[:, 0]] mag = [float(i) for i in data[:, 1]] magerr = [float(i) for i in data[:, 2]] sosort = np.array([mjd, mag, magerr]).T sosort = sosort[sosort[:, 0].argsort(), ] mjd = sosort[:, 0] mag = sosort[:, 1] magerr = sosort[:, 2] + 0.0001 prediction, ml_pred = microlensing_classifier.predict( mag, magerr, rf, pca)[0:2] #print('filename: ', filename, 'prediction: ', prediction, 'ml_pred = ', ml_pred) result = [filename, prediction, ml_pred] class_results.append(result) ml_pred = str(ml_pred) ml_pred = ml_pred.replace("[", "") ml_pred = ml_pred.replace("]", "") location = fileDirectoryClassifiedR + ml_pred + str(filename) np.savetxt(location, data, fmt='%s') for filename in dataFileNamesG: data = np.loadtxt(fileDirectoryG + filename, usecols=np.arange(0, 3)) mjd = [float(i) for i in data[:, 0]] mag = [float(i) for i in data[:, 1]] magerr = [float(i) for i in data[:, 2]] sosort = np.array([mjd, mag, magerr]).T sosort = sosort[sosort[:, 0].argsort(), ] mjd = sosort[:, 0] mag = sosort[:, 1] magerr = sosort[:, 2] try: prediction, ml_pred = microlensing_classifier.predict( mag, magerr, rf, pca)[0:2] except (ValueError): prediction, ml_pred = "NA", "NA" #print('filename: ', filename, 'prediction: ', prediction, 'ml_pred = ', ml_pred) result = [filename, prediction, ml_pred] class_results.append(result) ml_pred = str(ml_pred) ml_pred = ml_pred.replace("[", "") ml_pred = ml_pred.replace("]", "") location = fileDirectoryClassifiedG + ml_pred + str(filename) np.savetxt(location, data, fmt='%s') targetname = str(filename).replace("R.dat", "") targetname = targetname.replace("G.dat", "") print(targetname) #target = Target.objects.get(name=targetname) #target.save(extras={'Microlensing probability': {'probability': ml_pred, 'timestamp': datetime.datetime.now()}}) return class_results
Created on Fri Jan 13 14:46:19 2017 @author: danielgodinez """ import numpy as np import unittest import sys sys.path.append('../../') from LIA import microlensing_classifier from LIA import models data = np.loadtxt('ml_event.txt') mag = data[:, 1] magerr = data[:, 2] rf, pca = models.create_models('all_features.txt', 'pca_features.txt') class Test(unittest.TestCase): """ Unittest to ensure the classifier is working correctly. """ def test_predict(value): value.assertEqual( microlensing_classifier.predict(mag, magerr, rf, pca)[0], 'ML', "Classifier failed, predicted class is not correct.") def test_probability_prediction(value): pred = microlensing_classifier.predict(mag, magerr, rf, pca)[1] value.assertTrue( pred >= 0.4 and pred <= 0.6,
mask = (time>1) & (exptime[dico_time[:,1]]>200) #& (mag>0) & (pscale>0) & (np.median(mag)<19) & (Illumination<1.0) #& (fwhm[dico_time[:,1]]<6) #& (distance>40) #& (pscale>0.8) & (pscale<1.2) except: #Moon,Distance,Illumination = get_Moon(time,ra=267.83589537,dec=-30.060817819) mask = (time>0) #& (mag>0) & (pscale>0) & (fwhm[dico_time[:,1]]<6) if len(time[mask])>5: times.append(np.sort(time[mask])) ### Comment this when done once #n_class = 1000 #training_set.create(times, min_mag=10, max_mag=20, noise=None, cv_n1=2,n_class=n_class,test=False) model = models.create_models('all_features.txt', model='rf') ### ploting TSNE if needed #pcapca = np.loadtxt('pca_features.txt',dtype=str) #vis_data = tsne(n_components=2).fit_transform(pcapca[:,2:].astype(float)) #vis_x = vis_data[:,0] #vis_y = vis_data[:,1] #colors = ['red', 'green', 'blue', 'yellow', 'orange', 'purple', 'black'] #fig, ax = plt.subplots() #for i,object_class in enumerate(['Variable', 'Constant', 'CV', 'ML']): # x=vis_x[i*n_class:(i+1)*n_class] # y=vis_y[i*n_class:(i+1)*n_class] # ax.scatter(x, y, c=colors[i], label=object_class)
# daniels code from __future__ import division from main_script import * #from tsne_python import tsne as TSNE import numpy as np import matplotlib.pyplot as plt from sklearn.manifold import TSNE as tsne from LIA import extract_features, models LIA_directory = '/home/jclark/examples/' pca_model = models.create_models( str(LIA_directory) + 'all_features.txt', str(LIA_directory) + 'pca_features.txt')[1] db = '/home/jclark/examples/ROME-FIELD-16_phot.db' import pdb pdb.set_trace() data = np.loadtxt('pca_features.txt', usecols=np.arange(1, 45)) #data = np.loadtxt('all_features.txt',usecols=np.arange(2,49)) x_data = np.asarray(data).astype('float64') y_data = [1] * 500 + [2] * 500 + [3] * 500 + [4] * 500 + [5] * 10 + [6] * 5 + [ 7 ] * 2 #V,C,ML,CV for VAR in [107961]: #for xxx,VAR in enumerate([74609, 74609]): mag, magerr = [[], []] try: for tel_choice in [1, 2]: mag, magerr = np.append([mag, magerr], extract_lightcurve( VAR,