def train(continueTraining=True):
# load train, truth data
loadData(imgSize=IMAGE_SIZE)
# create model
if IMAGE_SIZE == 128:
model = createModel_128(inputShape=(IMAGE_SIZE, IMAGE_SIZE, 3),
numClasses=10)
else:
model = createModel_64(inputShape=(IMAGE_SIZE, IMAGE_SIZE, 3),
numClasses=10)
if continueTraining == True:
try:
model.load_weights('weight_{0}.hd5'.format(IMAGE_SIZE))
except:
print('Weight load exception.')
# train
for epoch in range(NUM_EPOCH):
print('EPOCH={0} : '.format(epoch), end='')
x_train, y_true = getBatch(numOfBatch=BATCH_SIZE)
loss = model.train_on_batch(x_train, y_true)
print('{0:>6.4f} {1:>6.4f}'.format(loss[0], loss[1]), end='\r')
if (epoch % NUM_WEIGHTSAVE_PERIOD) == 0:
model.save('model_{0}.hd5'.format(IMAGE_SIZE))
model.save_weights('weight_{0}.hd5'.format(IMAGE_SIZE))
def setUp(self):
self.data = data.Data(1, live=False)
print("Loading Data:", DATA_FILE_NAME)
data.loadData(DATA_FILE_NAME)
self.backtester = backtest.Backtest(cash=10000,
data=self.data,
minBrokerFee=1.00,
perShareFee=0.0075,
simple=True,
stateObj=True)
def fit(parameters):
# Load the training data
D_train = data.loadData(parameters["training_fasta"])
# Iterate through the fasta file
for fasta in (os.listdir(parameters["k_mers_path"])):
# Get the k-mers of the actual file
K = kmers.loadKmers(parameters["k_mers_path"] + "/" + fasta)
# Generate the samples matrix (X_train) and the target values (y_train)
X_train, y_train = matrix.generateSamplesTargets(
D_train, K, parameters["k"])
# Instantiate a linear svm classifier
clf = SVC(kernel='linear',
C=1,
probability=True,
random_state=0,
cache_size=1000)
# Fit the classifier
clf.fit(X_train, y_train)
# Get index of the separator
index = fasta.index(".")
# Get he filename
file_name = fasta[0:index]
# Save the model
joblib.dump(clf, parameters["model_path"] + "/" + file_name + ".pkl")
# Information message
print("Model: " + file_name + ".pkl saved at: " +
parameters["model_path"])
def p1_Ridge():
data = loadData()
df_x = np.array(data.loc[:, 'b3356':'b4703'])
df_y = np.array(data.loc[:, 'GrowthRate'])
coef = []
errors = []
alphas = [0.0001, 0.001, 0.1, 1, 10, 100, 1000, 100000]
for i in range(len(alphas)):
ridge_model = linear_model.Ridge(alpha=alphas[i])
ridge_model.fit(df_x, df_y)
scores = cross_validate(ridge_model,
df_x,
df_y,
cv=10,
scoring='neg_mean_squared_error')
errors.append(np.mean(-scores['test_score']))
coef.append(ridge_model.coef_)
errors = np.array(errors)
minIndex = np.argmin(errors)
model_ridge = linear_model.Ridge(alpha=alphas[minIndex])
model_ridge.fit(df_x, df_y)
feature_nozeros = np.count_nonzero(model_ridge.coef_)
print('Method:Ridge Regression')
print('Number of features that have non-zero coefficients:',
feature_nozeros)
print('10-fold Cross Validation MSE:', errors)
print('Optimal Lambda Value:', alphas[minIndex])
def p1_Lasso():
data = loadData()
df_x = np.array(data.loc[:, 'b3356':'b4703'])
df_y = np.array(data.loc[:, 'GrowthRate'])
coef = []
errors = []
alphas = [1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2]
for i in range(len(alphas)):
Lasso_model = linear_model.Lasso(alpha=alphas[i], tol=0.001)
Lasso_model.fit(df_x, df_y)
scores = cross_validate(Lasso_model,
df_x,
df_y,
cv=10,
scoring='neg_mean_squared_error')
errors.append(np.mean(-scores['test_score']))
coef.append(Lasso_model.coef_)
errors = np.array(errors)
minIndex = np.argmin(errors)
model_lasso = linear_model.Lasso(alpha=alphas[minIndex])
model_lasso.fit(df_x, df_y)
feature_zeros = np.count_nonzero(model_lasso.coef_)
print('Method:Lasso Regression')
print('Number of features that have Non-zero coefficients:', feature_zeros)
print('10-fold Cross Validation MSE:', errors)
print('Optimal Lambda Value:', alphas[minIndex])
def identifyPerfectMatch(parameters):
# Display information
print("\nIndentify perfect matches...")
# Initialize the results list
Results = {}
# Get the discriminative motifs
Kmers = kmers.loadKmers(str(parameters["k_mers_path"]))
# Get the sequence dataset
Data = data.loadData(str(parameters["training_fasta"]))
# Add the reference sequence
Data = data.loadReferenceSequence(
Data, str(parameters["refence_sequence_genbank"]))
# Iterate through the k-mers
for kmer in Kmers:
# Display the current motif
print("Signature: " + kmer)
# Get the current k-mer
query = kmer
# Check if there is perfect pairwise alignment of the current kmer with each sequence using parallelization
informations = Parallel(n_jobs=-1)(
delayed(perfectLocalPairwiseSequenceAlignment)(data, query)
for data in Data)
# Save the informations of each sequence according to the current kmer
Results[kmer] = informations
# Return the list of dictionary
return Results
def classify(inputfile='slpdb.mat', epoch_length=30, make_plot=True, save=True, outputfile='classify.txt'):
# Load datas
dataset = loadData(inputfile)
ECG = groupByEpoch(dataset['ECG'], 250, epoch_length)
Resp_in, Resp_out = edr.main(dataset)
# Feature dictionary construction
d_test = dictionaryInitialization()
d_test = dico_construction(ECG, Resp_in, Resp_out, d_test)
Xarr_norm = createNormalizedDataframe(d_test)
# Predecition
model = joblib.load('model.pkl')
pred_res = list()
for j in range(0, len(ECG)):
l = str(model.predict(Xarr_norm[j])[0])
pred_res.append(l)
if make_plot:
# Plot result
plot(pred_res, dataset['labels'], epoch_length)
if save:
# Write prediction into text file
savePredictions(pred_res, outputfile)
def identifyVariations(Results, parameters):
# Display information
print("\nIndentify variations...")
# Get the sequence dataset
Data = data.loadData(str(parameters["training_fasta"]))
# Add the reference sequence
Data = data.loadReferenceSequence(
Data, str(parameters["refence_sequence_genbank"]))
# Iterate through the actual sequences records
for key, records in Results.items():
print("Signature: " + key)
# Get the current k-mer
query = key
# Compute the average position of the k-mers to prune the search for mutated k-mers
positions = []
for record in records:
if record["position"] != None: positions.append(record["position"])
average_position = int(statistics.mean(positions))
# Compute the pairwise alignment of the amplified motif with each sequence to identify mutated motifs using parallelization
informations = Parallel(n_jobs=-1)(
delayed(localPairwiseSequenceAlignmentMutatededMotif)(
records, data, query, average_position) for data in Data)
# Save the updated informations of each sequence according to the current kmer
Results[key] = informations
# Dictionary of mutated motif (s) and associated postion(s)
return Results
def classify(inputfile='slpdb.mat',
epoch_length=30,
make_plot=True,
save=True,
outputfile='classify.txt'):
# Load datas
dataset = loadData(inputfile)
ECG = groupByEpoch(dataset['ECG'], 250, epoch_length)
Resp_in, Resp_out = edr.main(dataset)
# Feature dictionary construction
d_test = dictionaryInitialization()
d_test = dico_construction(ECG, Resp_in, Resp_out, d_test)
Xarr_norm = createNormalizedDataframe(d_test)
# Predecition
model = joblib.load('model.pkl')
pred_res = list()
for j in range(0, len(ECG)):
l = str(model.predict(Xarr_norm[j])[0])
pred_res.append(l)
if make_plot:
# Plot result
plot(pred_res, dataset['labels'], epoch_length)
if save:
# Write prediction into text file
savePredictions(pred_res, outputfile)
def main(): # python neuralNetwork.py <num training data points> alpha = 0.3 layers = [3, 7, 1] iterations = 10 # Generate training data. data.genData(int(sys.argv[1])) train_inputs, train_outputs = data.loadData(layers) # Train network. network = NeuralNetwork(layers) network.train(train_inputs, train_outputs, alpha, iterations) # Generate testing data. data.genData(10000) test_inputs, test_outputs = data.loadData(layers) # Check against testing data. network.numberCorrect(test_inputs, test_outputs)
def make_debug_dict():
train_data = loadData('./sample', True, opt.types, opt.type_filter)
out = codecs.open('./umls_dict_debug.txt', 'w', 'UTF-8')
with codecs.open('/Users/feili/UMLS/2016AA_Snomed_Meddra/META/MRCONSO.RRF',
'r', 'UTF-8') as fp:
for line in fp.readlines():
fields = line.strip().split(u"|")
CUI = fields[0] # Unique identifier for concept
LAT = fields[1] # Language of term
TS = fields[
2] # Term status, P - Preferred for CUI, S - Non-Preferred
LUI = fields[3] # Unique identifier for term
STT = fields[4] # String type, PF-Preferred form of term
SUI = fields[5] # Unique identifier for string
ISPREF = fields[
6] # Atom status - preferred (Y) or not (N) for this string within this concept
AUI = fields[7] # Unique identifier for atom
SAUI = fields[8] # Source asserted atom identifier
SCUI = fields[9] # Source asserted concept identifier
SDUI = fields[10] # Source asserted descriptor identifier
SAB = fields[11] # Abbreviated source name
TTY = fields[
12] # Abbreviation for term type in source vocabulary, for example PN (Metathesaurus Preferred Name) or CD (Clinical Drug).
CODE = fields[13] # Most useful source asserted identifier
STR = fields[14] # String
SRL = fields[
15] # Source restriction level, 0-No additional restrictions; general terms of the license agreement apply, 9-General terms + SNOMED CT Affiliate License in Appendix 2
SUPPRESS = fields[
16] # Suppressible flag. O: All obsolete content, N: None of the above
CVF = fields[
17] # Content View Flag, Bit field used to flag rows included in Content View.
find = False
for document in train_data:
for entity in document.entities:
if CODE in entity.norm_ids:
find = True
break
if find:
out.write(line)
out.close()
def test(model_name):
model = getModel(model_name=model_name)
test_loader = dt.loadData(train=False)
test_loss = 0
correct = 0
print("\nLoad saved model [", model_name, "]...")
if not cfg.pretrained:
model.load_state_dict(
torch.load(cfg.log_path + model_name +
"_cifar10.pt")['model_state_dict'])
model.eval()
print("Done..!")
criterion = nn.CrossEntropyLoss()
print("\nStart to test ", model_name, " ...")
with torch.no_grad():
for data, target in tqdm(test_loader):
data, target = data.to(cfg.device), target.to(cfg.device)
if cfg.convert_to_RGB:
batch_size, channel, width, height = data.size()
data = data.view(batch_size, channel, width,
height).expand(batch_size,
cfg.converted_channel, width,
height)
output = model(data)
test_loss += criterion(output, target).item() # sum up batch loss
pred = output.argmax(
dim=1,
keepdim=True) # get the index of the max log-probability
correct += pred.eq(target.view_as(pred)).sum().item()
test_loss /= len(test_loader.dataset)
print('\tTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\n\n'.
format(test_loss, correct, len(test_loader.dataset),
100. * correct / len(test_loader.dataset)))
def predict(parameters):
# Get the path of the model file
model_path = str(parameters["model_path"])
# Get the path of the k-mers file
k_mers_path = str(parameters["k_mers_path"])
# Get the testing fasta file
file_path = str(parameters["testing_fasta"])
# Get the prediction file path
prediction_path = str(parameters["prediction_path"])
# Get the evaluation mode
evaluation_mode = str(parameters["evaluation_mode"])
# Load the training data
D = data.loadData(file_path)
# Get the set of k-mers
K = kmers.loadKmers(k_mers_path)
# Get the k-mers length
k = len(list(K.keys())[0])
# Generate the samples matrix (X) and the target values (y)
X, y = matrix.generateSamplesTargets(D, K , k)
# Load the classifier
clf = joblib.load(model_path)
# Predict the sequences
y_pred = clf.predict(X)
# If evaluation mode is egal to True
if evaluation_mode == "True":
# If the target values list is empty
if len(y) == 0: print("Evaluation cannot be performed because target values are not given")
# Else display the classification report
else: print("Classification report \n", classification_report(y, y_pred))
# Save the predictions
f = open(prediction_path, "w")
# Write the header
f.write("id,y_pred\n")
# Iterate through the predictions
for i, y in enumerate(y_pred):
# Save the current prediction
f.write(D[i][0] + "," + y + "\n")
# Close the file
f.close()
# Displays a confirmation message
print("Predictions saved at the path:", prediction_path)
def p2(x):
data = loadData()
xtest = x
#data_left = data.loc[~data.index.isin(data_train.index)]
num_iter = 50
predictions = []
for i in range(num_iter):
data_train = resample(data, replace=True, n_samples=round(0.7*194),random_state=i)
ytrain = data_train.loc[:,'GrowthRate']
xtrain = data_train.loc[:,'b3356':'b4703']
Lasso_model = linear_model.Lasso(alpha = 1e-5,tol = 0.001)
Lasso_model.fit(xtrain,ytrain)
predict = np.ndarray.tolist(Lasso_model.predict(xtest))
predictions.append(predict)
return(predictions)
def main(img_path, db_path):
"""
Lance la détection de cratères puis le calcul de l'état de libration.
Affiche le résultat sur une image avec matplotlib
Entrée :
- img_path : chemin vers l'image de Lune
- db_path : chemin vers la base de données de cratères
"""
cratereDB = loadData(db_path)
moon_original = io.imread(img_path)
moon = moon_original.copy()
print("Tailles de l'image : ", moon.shape[:2])
codes, crat_infos, yc, xc, R = detection_crateres(moon)
if len(codes) > 0:
y, x = crat_infos[:, 0], crat_infos[:, 1]
names, lat, lon, _ = getDims(cratereDB, codes)
print("Liste des cratères renseignés :\n", names)
res = trouver_libration(x,
y,
lon,
lat,
xc=xc,
yc=yc,
R=R,
theta=0,
Dphi=0,
Dlambda=0)
# On affiche le résultat
ax = plt.subplot()
moon = draw_final(ax, moon, cratereDB, res, x, y, lon, lat)
io.imshow(moon)
plt.show()
else:
print("Vous n'avez renseigné aucun cratère !")
def fit(parameters):
# Get the parameters
model_path = str(parameters["model_path"])
# Get the path of the k-mers file
k_mers_path = str(parameters["k_mers_path"])
# Get the path of the training fasta file
file_path = str(parameters["training_fasta"])
# Load the training data
D = data.loadData(file_path)
# Get the set of k-mers
K = kmers.loadKmers(k_mers_path)
# Get the k-mers length
k = len(list(K.keys())[0])
# Generate the samples matrix (X) and the target values (y)
X, y = matrix.generateSamplesTargets(D, K , k)
# Instantiate a linear svm classifier
clf = svm()
# Fit the classifier
clf.fit(X, y)
# Save the model
joblib.dump(clf, model_path)
# Displays a confirmation message
print("Model saved at the path:", model_path)
def load_train_set(dictionary_reverse):
documents = loadData(opt.train_file, False, opt.types, opt.type_filter)
train_annotations_dict = {}
for document in documents:
for gold_entity in document.entities:
entity_tokens, _ = preprocess(gold_entity.name, True, True)
if len(entity_tokens) == 0:
continue
entity_key = ""
for token in entity_tokens:
entity_key += token + "_"
if gold_entity.norm_ids[0] in dictionary_reverse:
cui_list = dictionary_reverse[gold_entity.norm_ids[0]]
for cui in cui_list:
setMap(train_annotations_dict, entity_key, cui)
return train_annotations_dict
from sklearn.decomposition import NMF
from data import loadData, saveData
import numpy as np
# 最重要的参数是n_components、alpha、l1_ratio、solver
nmf = NMF(
n_components=128, # k value,默认会保留全部特征
# W H 的初始化方法，'random' | 'nndsvd'(默认) | 'nndsvda' | 'nndsvdar' | 'custom',
init=None,
solver='cd', # 'cd' | 'mu'
# {'frobenius', 'kullback-leibler', 'itakura-saito'},
beta_loss='frobenius',
tol=1e-10, # 停止迭代的极限条件
max_iter=200, # 最大迭代次数
random_state=None,
alpha=0., # 正则化参数
l1_ratio=0., # 正则化参数
verbose=0, # 冗长模式
shuffle=False # 针对"cd solver"
)
trius = loadData(filename='trius.npy')
X = np.abs(trius)
nmf.fit(X)
W = nmf.fit_transform(X)
H = nmf.components_
print('reconstruction_err_', nmf.reconstruction_err_) # 损失函数值
print('n_iter_', nmf.n_iter_) # 实际迭代次数
saveData(W, filename='nmf.npy')
saveData(H, filename='basis.npy')
import numpy as np
from data import loadData, groupByEpoch
from models import createNormalizedDataframe
from feature_construction import dictionaryInitialization, dico_construction
from sklearn.svm import SVC
from sklearn.externals import joblib
from edr import main
# Load datas
ECG = np.array([])
labels = np.array([])
binarylabels = np.array([])
for file in os.listdir("traindata"):
if file.endswith(".mat"):
dataset = loadData('traindata/' + file)
ECG = np.append(ECG, dataset['ECG'])
Resp_in, Resp_out = main(dataset)
labels = np.append(labels, dataset['labels'])
binarylabels = np.append(binarylabels, dataset['binarylabels'])
X = groupByEpoch(ECG)
# Train dictionary construction
d_train = dictionaryInitialization()
d_train = dico_construction(X, Resp_in, Resp_out, d_train)
y = np.array(labels)
Xarr_norm = createNormalizedDataframe(d_train)
Xarr_norm = np.nan_to_num(Xarr_norm).reshape((len(Xarr_norm), 1))
# Compute model and write to file
model = SVC(kernel='linear').fit(Xarr_norm, y)
def main(argv):
global server_thread
print("Starting %s %s..." % (common.app_name, common.app_version))
print(""" __
/ \_
__( )_
_( \_
_( )_
(__________________)
""")
app = QApplication(argv)
network.setup()
filtering.setup()
# Create extension server.
server_thread = extension_server.ExtensionServerThread(QCoreApplication.instance())
# Start DBus loop
if has_dbus:
print("DBus available. Creating main loop...", end=" ")
mainloop = DBusQtMainLoop(set_as_default = True)
dbus.set_default_main_loop(mainloop)
print("done.")
else:
print("DBus unavailable.")
# Create app.
app.setApplicationName(common.app_name)
app.setApplicationVersion(common.app_version)
app.installTranslator(translate.translator)
# We want Nimbus to stay open when the last window is closed,
# so we set this.
app.setQuitOnLastWindowClosed(False)
# If D-Bus is present...
if has_dbus:
print("Creating DBus session bus...", end=" ")
try:
bus = dbus.SessionBus()
except:
print("failed.")
else:
print("done.")
try:
print("Checking for running instances of %s..." % (common.app_name,), end=" ")
proxy = bus.get_object("org.nimbus.%s" % (common.app_name,), "/%s" % common.app_name,)
except:
dbus_present = False
else:
dbus_present = True
print("done.")
# If Nimbus detects the existence of another Nimbus process, it
# will send all the requested URLs to the existing process and
# exit.
if dbus_present:
print("An instance of Nimbus is already running. Passing arguments via DBus.")
for arg in argv[1:]:
proxy.addTab(arg)
if len(argv) < 2:
proxy.addWindow()
return
elif has_dbus:
print("No prior instances found. Continuing on our merry way.")
# Hack together the browser's icon. This needs to be improved.
common.app_icon = common.complete_icon("nimbus")
app.setWindowIcon(common.app_icon)
common.searchEditor = search_manager.SearchEditor()
common.downloadManager = DownloadManager(windowTitle=tr("Downloads"))
common.downloadManager.resize(QSize(480, 320))
common.downloadManager.loadSession()
# Create tray icon.
common.trayIcon = SystemTrayIcon()
common.trayIcon.newWindowRequested.connect(addWindow)
#common.trayIcon.windowReopenRequested.connect(reopenWindow)
common.trayIcon.show()
# Creates a licensing information dialog.
common.licenseDialog = custom_widgets.LicenseDialog()
# Create instance of clear history dialog.
common.chistorydialog = clear_history_dialog.ClearHistoryDialog()
uc = QUrl.fromUserInput(settings.user_css)
websettings = QWebSettings.globalSettings()
websettings.setUserStyleSheetUrl(uc)
websettings.enablePersistentStorage(settings.settings_folder)
websettings.setAttribute(websettings.LocalContentCanAccessRemoteUrls, True)
websettings.setAttribute(websettings.LocalContentCanAccessFileUrls, True)
websettings.setAttribute(websettings.DeveloperExtrasEnabled, True)
try: websettings.setAttribute(websettings.ScrollAnimatorEnabled, True)
except: pass
common.applyWebSettings()
# Set up settings dialog.
settings.settingsDialog = settings_dialog.SettingsDialog()
settings.settingsDialog.setWindowFlags(Qt.Dialog)
closeSettingsDialogAction = QAction(settings.settingsDialog)
closeSettingsDialogAction.setShortcuts(["Esc", "Ctrl+W"])
closeSettingsDialogAction.triggered.connect(settings.settingsDialog.hide)
settings.settingsDialog.addAction(closeSettingsDialogAction)
# Set up clippings manager.
settings.clippingsManager = settings_dialog.ClippingsPanel()
settings.clippingsManager.setWindowFlags(Qt.Dialog)
closeClippingsManagerAction = QAction(settings.clippingsManager)
closeClippingsManagerAction.setShortcuts(["Esc", "Ctrl+W"])
closeClippingsManagerAction.triggered.connect(settings.clippingsManager.hide)
settings.clippingsManager.addAction(closeClippingsManagerAction)
# Create DBus server
if has_dbus:
print("Creating DBus server...", end=" ")
server = DBusServer(bus)
print("done.")
# Load adblock rules.
filtering.adblock_filter_loader.start()
if not os.path.isdir(settings.extensions_folder):
try:
print("Copying extensions...", end=" ")
shutil.copytree(common.extensions_folder,\
settings.extensions_folder)
except:
print("failed.")
else:
print("done.")
if not os.path.isfile(settings.startpage):
try:
print("Copying start page...", end=" ")
shutil.copy2(common.startpage, settings.startpage)
except:
print("failed.")
else:
print("done.")
settings.reload_extensions()
settings.reload_userscripts()
server_thread.setDirectory(settings.extensions_folder)
# Start extension server.
server_thread.start()
# On quit, save settings.
app.aboutToQuit.connect(prepareQuit)
# Load settings.
data.loadData()
# View source dialog.
common.viewSourceDialog = ViewSourceDialogTabber()
#common.viewSourceDialog.show()
# This is a baaad name.
common.sessionSaver = QTimer(QCoreApplication.instance())
common.sessionSaver.timeout.connect(saveSession)
common.sessionSaver.timeout.connect(data.saveData)
if common.portable:
common.sessionSaver.start(50000)
else:
common.sessionSaver.start(30000)
common.desktop = QDesktopWidget()
changeSettings = False
if os.path.isfile(settings.crash_file):
print("Crash file detected.", end="")
if not has_dbus:
print(" With no DBus, %s may already be running." % common.app_name,)
multInstances = QMessageBox.question(None, tr("Hm."), tr("It's not good to run multiple instances of %(app_name)s. Is an instance of %(app_name)s already running?") % {"app_name": common.app_name}, QMessageBox.Yes | QMessageBox.No)
if multInstances == QMessageBox.Yes:
print("%s will now halt." % common.app_name,)
return
else:
print()
clearCache = QMessageBox()
clearCache.setWindowTitle(tr("Ow."))
clearCache.setText(tr("%(app_name)s seems to have crashed during your last session. Fortunately, your tabs were saved up to 30 seconds beforehand. Would you like to restore them?") % {"app_name": common.app_name})
clearCache.addButton(QPushButton(tr("Yes and change &settings")), QMessageBox.YesRole)
clearCache.addButton(QMessageBox.Yes)
clearCache.addButton(QMessageBox.No)
returnValue = clearCache.exec_()
if returnValue == QMessageBox.No:
try: os.remove(settings.session_file)
except: pass
if returnValue == 0:
changeSettings = True
else:
f = open(settings.crash_file, "w")
f.write("")
f.close()
if not "--daemon" in argv and os.path.exists(settings.session_file):
print("Loading previous session...", end=" ")
if changeSettings:
settings.settingsDialog.exec_()
loadSession()
print("done.")
if not "--daemon" in argv and len(argv[1:]) > 0:
# Create instance of MainWindow.
print("Loading the URLs you requested...", end=" ")
if len(browser.windows) > 0:
win = browser.windows[-1]
else:
win = MainWindow(appMode = ("--app" in argv))
# Open URLs from command line.
if len(argv[1:]) > 0:
for arg in argv[1:]:
if "." in arg or ":" in arg:
win.addTab(url=arg)
if win.tabWidget().count() < 1:
win.addTab(url=settings.settings.value("general/Homepage"))
# Show window.
win.show()
print("done.")
elif not "--daemon" in argv and len(argv[1:]) == 0 and len(browser.windows) == 0:
win = MainWindow(appMode = ("--app" in argv))
win.addTab(url=settings.settings.value("general/Homepage"))
win.show()
# Load filtering stuff.
if not os.path.isdir(filtering.hosts_folder):
common.trayIcon.showMessage(tr("Downloading content filters"), ("Ad blocking and host filtering will not work until this completes."))
filtering.update_filters()
else:
filtering.load_host_rules()
# Start app.
print("Kon~!")
sys.exit(app.exec_())
from sklearn.decomposition import PCA from data import loadData import pandas as pd from p4 import p4PR, p4ROC data = loadData() x_set = data.loc[:, 'b3356':'b4703'] pca = PCA(n_components=3) pca.fit(x_set) new_x = pd.DataFrame(pca.transform(x_set)) y1 = data.loc[:, 'Strain'] y2 = data.loc[:, 'Medium'] y3 = data.loc[:, 'Stress'] y4 = data.loc[:, 'GenePerturbed'] # p4PR(new_x,y1) # p4ROC(new_x,y1) # # p4PR(new_x,y2) # p4ROC(new_x,y2) # # p4PR(new_x,y3) # p4ROC(new_x,y3) # # p4PR(new_x,y4) # p4ROC(new_x,y4) # #
def main(argv):
"""This is a documentation string. Write a description of what your code
does here. You should generally put documentation strings ("docstrings")
on all your Python functions."""
#########################
# YOUR CODE GOES HERE #
#########################
scribus.statusMessage(str(scribus.PAPER_A4))
scribus.newDocument(
scribus.PAPER_A4, #paper
(MARGIN, MARGIN, MARGIN, MARGIN), #margins
scribus.LANDSCAPE, #orientation
0, #firstPageNumber
scribus.UNIT_MILLIMETERS, #unit
scribus.PAGE_1, #pagesType
0, #firstPageOrder
0, #numPages
)
scribus.createCharStyle('date1', 'Athelas Bold', 45)
scribus.createCharStyle('month1', 'Athelas Bold', 16)
scribus.createCharStyle('name1', 'Athelas Regular', 16)
scribus.createParagraphStyle('dateL',
alignment=scribus.ALIGN_LEFT,
charstyle='date1')
scribus.createParagraphStyle('monthL',
alignment=scribus.ALIGN_LEFT,
charstyle='month1')
scribus.createParagraphStyle('nameL',
alignment=scribus.ALIGN_LEFT,
charstyle='name1')
scribus.createParagraphStyle('dateR',
alignment=scribus.ALIGN_RIGHT,
charstyle='date1')
scribus.createParagraphStyle('monthR',
alignment=scribus.ALIGN_RIGHT,
charstyle='month1')
scribus.createParagraphStyle('nameR',
alignment=scribus.ALIGN_RIGHT,
charstyle='name1')
masterPage()
d = data.loadData()
"""
We want to make pamphlet (signature) of 5 x A4 papers printed from both sides with 2 weeks on each side
2sided 4imposition refer imposition.py
"""
def pageFn(left, right):
page = scribus.newPage(-1, 'planner')
i = 0
sizeDate = (20, 15)
sizeMonth = (20, 7)
sizeName = (290, 7)
for xend, y in iterDayLines():
try:
tLeftDate = left[i]
posLeftDate = (xstart, y - sizeDate[1])
posLeftMonth = (xstart + sizeDate[0], y - sizeDate[1])
posLeftName = (xstart + sizeDate[0], y - sizeName[1])
tLeftName = ', '.join(d[tLeftDate.month][tLeftDate.day])
objLeftDate = scribus.createText(*posLeftDate + sizeDate)
objLeftMonth = scribus.createText(*posLeftMonth + sizeMonth)
objLeftName = scribus.createText(*posLeftName + sizeName)
scribus.setText(tLeftDate.strftime('%d'), objLeftDate)
scribus.setText(mm(tLeftDate), objLeftMonth)
scribus.setText(tLeftName, objLeftName)
scribus.setStyle('dateL', objLeftDate)
scribus.setStyle('monthL', objLeftMonth)
scribus.setStyle('nameL', objLeftName)
except IndexError:
pass
try:
posRightDate = (xend - sizeDate[0], y - sizeDate[1])
posRightMonth = (xend - sizeDate[0] - sizeMonth[0],
y - sizeDate[1])
posRightName = (xend - sizeDate[0] - sizeName[0],
y - sizeName[1])
tRightDate = right[i]
tRightName = ', '.join(d[tRightDate.month][tRightDate.day])
objRightDate = scribus.createText(*posRightDate + sizeDate)
objRightMonth = scribus.createText(*posRightMonth + sizeMonth)
objRightName = scribus.createText(*posRightName + sizeName)
scribus.setText(tRightDate.strftime('%d'), objRightDate)
scribus.setText(mm(tRightDate), objRightMonth)
scribus.setText(tRightName, objRightName)
scribus.setStyle('dateR', objRightDate)
scribus.setStyle('monthR', objRightMonth)
scribus.setStyle('nameR', objRightName)
except IndexError:
pass
i += 1
dates.forEachWeek(pageFn)
scribus.deletePage(1)
# 'population' : 10,
# 'mutationRate' : 0.0001,
# 'error' : 0.04,
# 'epochs' : 1000
# }
# eann = ann.eann(opt)
# eann.train(train_data, train_result)
# files.saveData(eann, 'eann-all.db')
# files.saveData(eann.w, 'eann-w-all.db')
#init ann
eann = files.loadData('eann-all.db')
eann_w = files.loadData('eann-w-all.db')
opt = {
'architecture' : eann.architecture,
'learningRate' : 9,
'error' : 0.001,
'epochs' : 50,
'batch' : 100
}
nn = ann.ann(opt)
eann_w = np.asarray(eann_w)
for i in range(len(eann_w)):
eann_w[i] = eann_w[i].astype(float)
nn.w = eann_w
nn.train(train_data, train_result)
#init data
train_data = train_data[0:1000]
train_result = train_result[0:1000]
# #init rbm
# rbm = ann.rbm(784, 3, 5, 0.3, 100)
# rbm.train(train_data)
# files.saveData(rbm, 'rbm-1000.db')
rbm = files.loadData('rbm.db')
im = rbm.sim(test_data[0:1])
im = np.array(im[0])
im = im.reshape(28,28)
fig = plt.figure()
plotwindow = fig.add_subplot(111)
plt.imshow(im , cmap='gray')
plt.show()
im = rbm.sim(test_data[0:1])
print np.array_equal(im[0], im[1])
import data as data
import xgboost as xgb
from sklearn.metrics import log_loss
from sklearn.utils import shuffle
from sklearn.preprocessing import StandardScaler
import random
from sklearn.calibration import CalibratedClassifierCV
from sklearn.metrics import roc_auc_score
from sklearn.ensemble import RandomForestClassifier
import getopt
import sys
from XGBoostClassifier import XGBoostClassifier
from tqdm import tqdm
random.seed()
train_x,train_y,valid_x,valid_y,test_x,valid_id,test_id=data.loadData()
def train(model_id,train_x,train_y,valid_x,valid_y,test_x):
train_x,train_y=shuffle(train_x,train_y)
maximum_auc=0.0
random_state=random.randint(0, 1000000)
for i in tqdm(range(1000)):
config={
'base_estimator':'gbtree',
'objective':'multi:softprob',
'metric':'mlogloss',
'num_classes':2,
'learning_rate':random.uniform(0.01,0.15),
'max_depth':20+random.randint(0,10),
'max_samples':random.uniform(0.3,1.0),
""" Created on Sat Mar 21 12:24:16 2015 @author: Paco """ import numpy as np from sklearn.cross_validation import train_test_split from data import loadData from featureConstruction import dictionaryInitialization,dico_construction from models import listModels,createNormalizedDataframe,savePredictions mat_file = 'data_challenge.mat' # Load datas dataset = loadData(mat_file) # Train dictionary construction d_train = dictionaryInitialization() d_train = dico_construction(dataset['X_train'],d_train) labels = np.array(dataset['y_train']) Xarr_norm = createNormalizedDataframe(d_train) # Test our models X_train_train, X_train_test, y_train_train, y_train_test = train_test_split(Xarr_norm,labels,test_size=0.2,random_state=42) X_train_train = np.nan_to_num(X_train_train) X_train_test = np.nan_to_num(X_train_test) # Scores models = listModels()
class probabilityOutput:
def __init__(self):
startData = tableAt(0)
hyperParams = [1 for x in range(100)]
self.distribution = probabilityDistribution(hyperParams, startData)
def readRadioData(self, radio):
radioData = tableAt(radio)
self.distribution.update(radioData)
def printProbabilities(self):
res = str(self.distribution.probability(0))
for i in range(1, 100):
prob = self.distribution.probability(i)
res += ','
res += str(prob)
print(res)
def main():
out = probabilityOutput()
out.readRadioData(0)
out.printProbabilities()
for i in range(1, 303):
radio = int(stdin.readline())
out.readRadioData(radio)
out.printProbabilities()
loadData()
main()
import ann
import data as files
#import the data
matfn=u'mnist/mnist_uint8.mat'
data=sio.loadmat(matfn)
train_data = np.float64(data['train_x']) /255
train_result = np.float64(data['train_y'])
test_data = np.float64(data['test_x']) / 255
test_result = np.float64(data['test_y'])
#load nn
nn = files.loadData('nn.db')
# _w = nn.w
# _w = np.absolute(_w)
# avr = 0
# for i in _w:
# avr += np.mean(i)
# avr = avr/len(_w)
# for i in range(len(_w)):
# for j in range(len(_w[i])):
# for n in range(len(_w[i][j])):
# temp = _w[i][j][n]
# if temp > avr:
# temp = 1
def main(self):
# 데이터를 통한 사전 구성 한다.
char2idx, idx2char, vocabularyLength = data.loadVocabulary()
# 훈련 데이터와 테스트 데이터를 가져온다.
xTrain, yTrain, xTest, yTest = data.loadData()
# 훈련셋 인코딩 만드는 부분이다.
inputTrainEnc, inputTrainEncLength = data.encProcessing(xTrain, char2idx)
# 훈련셋 디코딩 입력 부분 만드는 부분이다.
outputTrainDec, outputTrainDecLength = data.decOutputProcessing(yTrain, char2idx)
# 훈련셋 디코딩 출력 부분 만드는 부분이다.
targetTrainDec = data.decTargetProcessing(yTrain, char2idx)
inputTestEnc, inputTestEncLength = data.encProcessing(xTest,char2idx)
outputTestDec, outputTestDecLength = data.decOutputProcessing(yTest, char2idx)
targetTestDec = data.decTargetProcessing(yTest, char2idx)
# 현재 경로'./'에 현재 경로 하부에
# 체크 포인트를 저장한 디렉토리를 설정한다.
checkPointPath = os.path.join(os.getcwd(), DEFINES.checkPointPath)
# 디렉토리를 만드는 함수이며 두번째 인자 exist_ok가
# True이면 디렉토리가 이미 존재해도 OSError가
# 발생하지 않는다.
# exist_ok가 False이면 이미 존재하면
# OSError가 발생한다.
os.makedirs(checkPointPath, exist_ok=True)
# 에스티메이터 구성한다.
classifier = tf.estimator.Estimator(
model_fn=ml.Model, # 모델 등록한다.
model_dir=DEFINES.checkPointPath, # 체크포인트 위치 등록한다.
params={ # 모델 쪽으로 파라메터 전달한다.
'hiddenSize': DEFINES.hiddenSize, # 가중치 크기 설정한다.
'layerSize': DEFINES.layerSize, # 멀티 레이어 층 개수를 설정한다.
'learningRate': DEFINES.learningRate, # 학습율 설정한다.
'vocabularyLength': vocabularyLength, # 딕셔너리 크기를 설정한다.
'embeddingSize': DEFINES.embeddingSize, # 임베딩 크기를 설정한다.
'embedding': DEFINES.embedding, # 임베딩 사용 유무를 설정한다.
'multilayer': DEFINES.multilayer, # 멀티 레이어 사용 유무를 설정한다.
})
# 학습 실행
print(inputTrainEnc.shape, outputTrainDec.shape, targetTrainDec.shape)
classifier.train(input_fn=lambda:data.trainInputFn(
inputTrainEnc, outputTrainDec, targetTrainDec, DEFINES.batchSize), steps=DEFINES.trainSteps)
evalResult = classifier.evaluate(input_fn=lambda:data.evalInputFn(
inputTestEnc, outputTestDec, targetTestDec, DEFINES.batchSize))
print('\nTest set accuracy: {accuracy:0.3f}\n'.format(**evalResult))
# 테스트용 데이터 만드는 부분이다.
# 인코딩 부분 만든다.
inputPredicEnc, inputPredicEncLength = data.encProcessing(["가끔 궁금해"], char2idx)
# 학습 과정이 아니므로 디코딩 입력은
# 존재하지 않는다.(구조를 맞추기 위해 넣는다.)
outputPredicDec, outputPredicDecLength = data.decOutputProcessing([""], char2idx)
# 학습 과정이 아니므로 디코딩 출력 부분도
# 존재하지 않는다.(구조를 맞추기 위해 넣는다.)
targetPredicDec = data.decTargetProcessing([""], char2idx)
# 예측을 하는 부분이다.
predictions = classifier.predict(
input_fn=lambda:data.evalInputFn(inputPredicEnc, outputPredicDec, targetPredicDec, DEFINES.batchSize))
# 예측한 값을 인지 할 수 있도록
# 텍스트로 변경하는 부분이다.
data.pred2string(predictions, idx2char)
def train(model_name):
# train model
train_loader = dt.loadData(train=True)
model = getModel(model_name = model_name)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=cfg.lr, momentum=cfg.momentum)
scheduler = optim.lr_scheduler.StepLR(optimizer, cfg.lr_decaying_step, gamma = cfg.lr_decaying_value)
saved_model_path = cfg.log_path + model_name + "_cifar10.pt"
if os.path.isfile(saved_model_path):
# load saved checkpoint
checkpoint = torch.load(saved_model_path)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
start_epoch = checkpoint['epoch']
loss = checkpoint['loss']
else:
start_epoch = 1
model.train()
print("\nStart training ", model_name, "...")
for epoch in range(start_epoch, cfg.epochs + 1):
for batch_idx, (data, target) in enumerate(tqdm(train_loader)):
data, target = data.to(cfg.device), target.to(cfg.device)
if cfg.convert_to_RGB:
batch_size, channel, width, height = data.size()
data = data.view(batch_size, channel, width, height).expand(batch_size, cfg.converted_channel, width, height)
optimizer.zero_grad()
output=model(data)
loss = criterion(output, target)
loss.backward()
optimizer.step()
scheduler.step()
if cfg.save_model and (epoch % cfg.save_epoch == 0):
if not(os.path.isdir(cfg.log_path)):
os.makedirs(os.path.join(cfg.log_path))
torch.save({
'epoch':epoch + 1,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss
}, saved_model_path)
print('\tTrain Epoch: {} / {} \t Loss: {:.6f}\n'.format(epoch, cfg.epochs, loss.item()))
print("Done!\n\n")
def test_cmf():
import scipy.io
import cfeval
# matdata1 = scipy.io.loadmat('data1.mat')
# matdata2 = scipy.io.loadmat('data1.mat')
"""
Xtrn = matdata1['Xtrn']
print Xtrn.shape
Xtrn = scipy.sparse.csc_matrix(Xtrn)
Xaux = matdata2['Xtrn'] + matdata2['Xtst']
# Xaux = Xaux.T.tocsc()
Xaux = scipy.sparse.csc_matrix(Xaux.T)
print Xaux.shape
Xtst = matdata1['Xtst']
print Xtst.shape
Xtst = scipy.sparse.csc_matrix(Xtst)
"""
Xs_trn = loadData('./data/meta.txt', './em_66_f2_m11_tr.libsvm')
#Xs_trn = loadData('./data/meta.txt', './data/em_10000_f5_t4_k1_tr.libsvm')
for mat in Xs_trn:
print mat.shape
Xs_tst = loadData('./data/meta.txt', './em_66_f2_m11_te.libsvm')
#Xs_tst = loadData('./data/meta.txt', './data/em_10000_f5_t4_k1_te.libsvm')
for mat in Xs_tst:
print mat.shape
# Xs_trn = [Xtrn, Xaux]
# Xs_tst = [Xtst, None]
# rc_schema = numpy.array([[0, 2], [1, 0]])
rc_schema = numpy.array([[0, 1, 0], [1, 2, 2]]) # rc_schema should be the same order of data matrices
C = 0.1
K = 50 # number of latent factors
alphas = [0.4, 0.4, 0.2]
T = 100 # number of iterations
modes = numpy.zeros(len(Xs_trn), object)
modes[0] = 'densemf'
modes[1] = 'sparsemf'
modes[2] = 'sparsemf'
r0s = [0.1, 0.1, 0.1]
r1s = [0.05, 0.05, 0.05]
[Us, r0s, r1s] = learn(Xs_trn, Xs_tst, rc_schema, r0s, r1s, alphas, modes, K, C, T)
print '******'
print Us[0].shape
print Us[1].shape
print Us[2].shape
print '********'
# Vt = scipy.sparse.csc_matrix(Us[0],dtype=float)
# Ut = scipy.sparse.csc_matrix(Us[1],dtype=float)
Ys_tst = predict(Us, Xs_tst, rc_schema, r0s, r1s, modes)
#tst
X = Xs_tst[0]
Y = Ys_tst[0]
print Y.shape
print Y
print "K: %d, C: %f" % (K, C)
print "alphas: ", alphas
print "rmse: %.4f , mae: %.4f\n" % (cfeval.rmse(X, Y), cfeval.mae(X, Y))
logging.info(d.config)
makedir_and_clear(opt.output)
if opt.cross_validation > 1:
documents = data.load_data_fda(opt.train_file, True, opt.types,
opt.type_filter, True, False)
external_train_data = []
if 'ext_corpus' in d.config:
ext_corpus = d.config['ext_corpus']
for k, v in ext_corpus.items():
if k == 'made' or k == 'cardio':
external_train_data.extend(
data.loadData(v['path'], True, v.get('types'),
v.get('types')))
elif k == 'tac':
external_train_data.extend(
data.load_data_fda(v['path'], True, v.get('types'),
v.get('types'), False, False))
else:
raise RuntimeError("not support external corpus")
logging.info("use {} fold cross validataion".format(
opt.cross_validation))
fold_num = opt.cross_validation
total_doc_num = len(documents)
dev_doc_num = total_doc_num // fold_num
macro_p = 0.0
macro_r = 0.0
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from plot import plotData
from data import loadData
import GradientDescent as gd
import numpy as np
data = loadData('../data/ex1data1.txt')
(m, features) = data.shape
X = data[:, 0]
y = data[:, 1]
# Plotting the data
plotData(X, y)
plt.show()
# Gradient Descent
print("Running Gradient Descent ...")
X = np.append(np.ones((m, 1)), X, 1) # add a column of ones to X
theta = np.zeros((2, 1)) # initialize fitting parameters
# some gradient descent settings
iterations = 1500
alpha = 0.01
# compute and display initial cost (expected: 32.0727338775)
cost = gd.computeCost(X, y, theta)
print("initial cost: ", cost)
def main(argv):
global server_thread
print("Starting %s %s..." % (common.app_name, common.app_version))
print(""" __
/ \_
__( )_
_( \_
_( )_
(__________________)
""")
app = QApplication(argv)
network.setup()
filtering.setup()
# Create extension server.
server_thread = extension_server.ExtensionServerThread(
QCoreApplication.instance())
# Start DBus loop
if has_dbus:
print("DBus available. Creating main loop...", end=" ")
mainloop = DBusQtMainLoop(set_as_default=True)
dbus.set_default_main_loop(mainloop)
print("done.")
else:
print("DBus unavailable.")
# Create app.
app.setApplicationName(common.app_name)
app.setApplicationVersion(common.app_version)
app.installTranslator(translate.translator)
# We want Nimbus to stay open when the last window is closed,
# so we set this.
app.setQuitOnLastWindowClosed(False)
# If D-Bus is present...
if has_dbus:
print("Creating DBus session bus...", end=" ")
try:
bus = dbus.SessionBus()
except:
print("failed.")
else:
print("done.")
try:
print("Checking for running instances of %s..." % (common.app_name, ),
end=" ")
proxy = bus.get_object(
"org.nimbus.%s" % (common.app_name, ),
"/%s" % common.app_name,
)
except:
dbus_present = False
else:
dbus_present = True
print("done.")
# If Nimbus detects the existence of another Nimbus process, it
# will send all the requested URLs to the existing process and
# exit.
if dbus_present:
print(
"An instance of Nimbus is already running. Passing arguments via DBus."
)
for arg in argv[1:]:
proxy.addTab(arg)
if len(argv) < 2:
proxy.addWindow()
return
elif has_dbus:
print("No prior instances found. Continuing on our merry way.")
# Hack together the browser's icon. This needs to be improved.
common.app_icon = common.complete_icon("nimbus")
app.setWindowIcon(common.app_icon)
common.searchEditor = search_manager.SearchEditor()
common.downloadManager = DownloadManager(windowTitle=tr("Downloads"))
common.downloadManager.resize(QSize(480, 320))
common.downloadManager.loadSession()
# Create tray icon.
common.trayIcon = SystemTrayIcon()
common.trayIcon.newWindowRequested.connect(addWindow)
#common.trayIcon.windowReopenRequested.connect(reopenWindow)
common.trayIcon.show()
# Creates a licensing information dialog.
common.licenseDialog = custom_widgets.LicenseDialog()
# Create instance of clear history dialog.
common.chistorydialog = clear_history_dialog.ClearHistoryDialog()
uc = QUrl.fromUserInput(settings.user_css)
websettings = QWebSettings.globalSettings()
websettings.setUserStyleSheetUrl(uc)
websettings.enablePersistentStorage(settings.settings_folder)
websettings.setAttribute(websettings.LocalContentCanAccessRemoteUrls, True)
websettings.setAttribute(websettings.LocalContentCanAccessFileUrls, True)
websettings.setAttribute(websettings.DeveloperExtrasEnabled, True)
try:
websettings.setAttribute(websettings.ScrollAnimatorEnabled, True)
except:
pass
common.applyWebSettings()
# Set up settings dialog.
settings.settingsDialog = settings_dialog.SettingsDialog()
settings.settingsDialog.setWindowFlags(Qt.Dialog)
closeSettingsDialogAction = QAction(settings.settingsDialog)
closeSettingsDialogAction.setShortcuts(["Esc", "Ctrl+W"])
closeSettingsDialogAction.triggered.connect(settings.settingsDialog.hide)
settings.settingsDialog.addAction(closeSettingsDialogAction)
# Set up clippings manager.
settings.clippingsManager = settings_dialog.ClippingsPanel()
settings.clippingsManager.setWindowFlags(Qt.Dialog)
closeClippingsManagerAction = QAction(settings.clippingsManager)
closeClippingsManagerAction.setShortcuts(["Esc", "Ctrl+W"])
closeClippingsManagerAction.triggered.connect(
settings.clippingsManager.hide)
settings.clippingsManager.addAction(closeClippingsManagerAction)
# Create DBus server
if has_dbus:
print("Creating DBus server...", end=" ")
server = DBusServer(bus)
print("done.")
# Load adblock rules.
filtering.adblock_filter_loader.start()
if not os.path.isdir(settings.extensions_folder):
try:
print("Copying extensions...", end=" ")
shutil.copytree(common.extensions_folder,\
settings.extensions_folder)
except:
print("failed.")
else:
print("done.")
if not os.path.isfile(settings.startpage):
try:
print("Copying start page...", end=" ")
shutil.copy2(common.startpage, settings.startpage)
except:
print("failed.")
else:
print("done.")
settings.reload_extensions()
settings.reload_userscripts()
server_thread.setDirectory(settings.extensions_folder)
# Start extension server.
server_thread.start()
# On quit, save settings.
app.aboutToQuit.connect(prepareQuit)
# Load settings.
data.loadData()
# View source dialog.
common.viewSourceDialog = ViewSourceDialogTabber()
#common.viewSourceDialog.show()
# This is a baaad name.
common.sessionSaver = QTimer(QCoreApplication.instance())
common.sessionSaver.timeout.connect(saveSession)
common.sessionSaver.timeout.connect(data.saveData)
if common.portable:
common.sessionSaver.start(50000)
else:
common.sessionSaver.start(30000)
common.desktop = QDesktopWidget()
changeSettings = False
if os.path.isfile(settings.crash_file):
print("Crash file detected.", end="")
if not has_dbus:
print(
" With no DBus, %s may already be running." %
common.app_name, )
multInstances = QMessageBox.question(
None, tr("Hm."),
tr("It's not good to run multiple instances of %(app_name)s. Is an instance of %(app_name)s already running?"
) % {"app_name": common.app_name},
QMessageBox.Yes | QMessageBox.No)
if multInstances == QMessageBox.Yes:
print("%s will now halt." % common.app_name, )
return
else:
print()
clearCache = QMessageBox()
clearCache.setWindowTitle(tr("Ow."))
clearCache.setText(
tr("%(app_name)s seems to have crashed during your last session. Fortunately, your tabs were saved up to 30 seconds beforehand. Would you like to restore them?"
) % {"app_name": common.app_name})
clearCache.addButton(QPushButton(tr("Yes and change &settings")),
QMessageBox.YesRole)
clearCache.addButton(QMessageBox.Yes)
clearCache.addButton(QMessageBox.No)
returnValue = clearCache.exec_()
if returnValue == QMessageBox.No:
try:
os.remove(settings.session_file)
except:
pass
if returnValue == 0:
changeSettings = True
else:
f = open(settings.crash_file, "w")
f.write("")
f.close()
if not "--daemon" in argv and os.path.exists(settings.session_file):
print("Loading previous session...", end=" ")
if changeSettings:
settings.settingsDialog.exec_()
loadSession()
print("done.")
if not "--daemon" in argv and len(argv[1:]) > 0:
# Create instance of MainWindow.
print("Loading the URLs you requested...", end=" ")
if len(browser.windows) > 0:
win = browser.windows[-1]
else:
win = MainWindow(appMode=("--app" in argv))
# Open URLs from command line.
if len(argv[1:]) > 0:
for arg in argv[1:]:
if "." in arg or ":" in arg:
win.addTab(url=arg)
if win.tabWidget().count() < 1:
win.addTab(url=settings.settings.value("general/Homepage"))
# Show window.
win.show()
print("done.")
elif not "--daemon" in argv and len(argv[1:]) == 0 and len(
browser.windows) == 0:
win = MainWindow(appMode=("--app" in argv))
win.addTab(url=settings.settings.value("general/Homepage"))
win.show()
# Load filtering stuff.
if not os.path.isdir(filtering.hosts_folder):
common.trayIcon.showMessage(tr("Downloading content filters"), (
"Ad blocking and host filtering will not work until this completes."
))
filtering.update_filters()
else:
filtering.load_host_rules()
# Start app.
print("Kon~!")
sys.exit(app.exec_())
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from plot import plotData
from data import loadData
import GradientDescent as gd
import numpy as np
data = loadData('../data/ex1data1.txt')
(m, features) = data.shape
X = data[:, 0]
y = data[:, 1]
# Plotting the data
plotData(X, y)
plt.show()
# Gradient Descent
print("Running Gradient Descent ...")
X = np.append(np.ones((m, 1)), X, 1) # add a column of ones to X
theta = np.zeros((2, 1)) # initialize fitting parameters
# some gradient descent settings
iterations = 1500
alpha = 0.01
# compute and display initial cost (expected: 32.0727338775)
cost = gd.computeCost(X, y, theta)
print("initial cost: ", cost)
app = Flask(__name__)
# Conexion con el servidor
vcap_config = os.environ.get('VCAP_SERVICES')
decoded_config = json.loads(vcap_config)
for key, value in decoded_config.iteritems():
if key.startswith('mongodb'):
mongo_creds = decoded_config[key][0]['credentials']
mongo_url = str(mongo_creds['url'])
client = MongoClient(mongo_url)
logging.warn('carga de datos')
# Cargamos los datos
data.loadData(client)
logging.warn('fin de carga de datos')
@app.route('/')
def Welcome():
return app.send_static_file('index.html')
@app.route('/listar')
def listar():
logging.warn('Inicio Consulta')
db = client.db
listado = db.listado
lista = listado.find().sort('nombre',1)
logging.warn('Fin Consulta')
from sklearn.model_selection import train_test_split
from data import loadData, saveData
import tensorflow as tf
import config
seed = config.seed
F = loadData(filename='feature.npy')
labels = loadData(filename='labels.npy')
x_train, x_test, y_train, y_test = train_test_split(F,
labels,
test_size=0.1,
random_state=seed)
# Visualize decoder setting
# Parameters
LR = 0.01
training_epoch = 400
batch_size = 128
display_step = 20
# Network Parameters
n_input = F.shape[1]
# hidden layer settings
n_hidden_1 = 256 # 1st layer num features
n_hidden_2 = 128 # 2nd layer num features
n_hidden_3 = 8 # 3rd layer num features
# tf Graph input (only pictures)
X = tf.placeholder("float", [None, n_input])
Y = tf.placeholder("int64", [
if opt.random_seed != 0:
random.seed(opt.random_seed)
np.random.seed(opt.random_seed)
torch.manual_seed(opt.random_seed)
torch.cuda.manual_seed_all(opt.random_seed)
d = data.Data(opt)
logging.info(d.config)
makedir_and_clear(opt.output)
logging.info("load data ...")
train_data = data.loadData(opt.train_file, True, opt.types, opt.type_filter)
dev_data = data.loadData(opt.dev_file, True, opt.types, opt.type_filter)
if opt.test_file:
test_data = data.loadData(opt.test_file, False, opt.types, opt.type_filter)
else:
test_data = None
logging.info("load dict ...")
UMLS_dict, UMLS_dict_reverse = umls.load_umls_MRCONSO(d.config['norm_dict'])
logging.info("dict concept number {}".format(len(UMLS_dict)))
train(train_data, dev_data, test_data, d, UMLS_dict, UMLS_dict_reverse, opt, None, False)
elif opt.whattodo == 2:
def trainModel():
# Parse args
parser = argparse.ArgumentParser(description='Train the CNN')
parser.add_argument('--expt_dir', default='./logs',
help='save dir for experiment logs')
parser.add_argument('--train', default='./data',
help='path to training set')
parser.add_argument('--val', default='./data',
help='path to validation set')
parser.add_argument('--test', default='./data',
help='path to test set')
parser.add_argument('--save_dir', default='./models',
help='path to save model')
parser.add_argument('--arch', default='models/cnn.json',
help = 'path to model architecture')
parser.add_argument('--model_name', default = 'model',
help = 'name of the model to save logs, weights')
parser.add_argument('--lr', default = 0.001,
help = 'learning rate')
parser.add_argument('--init', default = '1',
help = 'initialization')
parser.add_argument('--batch_size', default = 20,
help = 'batch_size')
args = parser.parse_args()
# Load data
train_path, valid_path, test_path = args.train, args.val, args.test
logs_path = args.expt_dir
model_path, model_name = args.save_dir, args.model_name
model_path = os.path.join(model_path, model_name)
if not os.path.isdir(model_path):
os.mkdir(model_path)
lr, batch_size, init = float(args.lr), int(args.batch_size), int(args.init)
data = loadData(train_path, valid_path, test_path)
train_X, train_Y, valid_X, valid_Y, test_X, test_Y = data['train']['X'], data['train']['Y'],\
data['valid']['X'], data['valid']['Y'],\
data['test']['X'], data['test']['Y'],
# Logging
train_log_name = '{}.train.log'.format(model_name)
valid_log_name = '{}.valid.log'.format(model_name)
train_log = setup_logger('train-log', os.path.join(logs_path, train_log_name))
valid_log = setup_logger('valid-log', os.path.join(logs_path, valid_log_name))
# Train
num_epochs = 500
num_batches = int(float(train_X.shape[0]) / batch_size)
steps = 0
patience = 100
early_stop=0
model = getModel(lr)
loss_history = [np.inf]
for epoch in range(num_epochs):
print 'Epoch {}'.format(epoch)
steps = 0
indices = np.arange(train_X.shape[0])
np.random.shuffle(indices)
train_X, train_Y = train_X[indices], train_Y[indices]
for batch in range(num_batches):
start, end = batch * batch_size, (batch + 1) * batch_size
x, y = Augment(train_X[range(start, end)]).batch, train_Y[range(start, end)]
model.fit(x.reshape((-1, 1, 28, 28)), y, batch_size = batch_size, verbose = 0)
steps += batch_size
if steps % train_X.shape[0] == 0 and steps != 0:
train_loss, train_acc = model.evaluate(train_X.reshape((-1, 1, 28, 28)), train_Y)
train_log.info('Epoch {}, Step {}, Loss: {}, Accuracy: {}, lr: {}'.format(epoch, steps, train_loss, train_acc, lr))
valid_loss, valid_acc = model.evaluate(valid_X.reshape((-1, 1, 28, 28)), valid_Y)
valid_log.info('Epoch {}, Step {}, Loss: {}, Accuracy: {}, lr: {}'.format(epoch, steps, valid_loss, valid_acc, lr))
if valid_loss < min(loss_history):
save_path = os.path.join(model_path, 'model')
model.save(save_path)
early_stop = 0
early_stop += 1
if (early_stop >= patience):
print "No improvement in validation loss for " + str(patience) + " steps - stopping training!"
print("Optimization Finished!")
return 1
loss_history.append(valid_loss)
print("Optimization Finished!")
scatter.set(yticklabels=[])
scatter.fig.suptitle("Scatter plot of all variables w.r.t. " + LABEL +
" " + outlierInfo,
fontsize=40)
scatter.fig.subplots_adjust(top=.9)
plt.savefig(join(
GRAPHS_FOLDER,
"scatter_plot_of_all_variables_wrt_" + LABEL + "_(" + iter + ").png"),
format="png")
if __name__ == "__main__":
if not isdir(DATASETS_FOLDER): makedirs(DATASETS_FOLDER)
if not isdir(GRAPHS_FOLDER): makedirs(GRAPHS_FOLDER)
fetchData(URL, DATASETS_FOLDER, ZIP_FILENAME)
data = loadData(DATASETS_FOLDER, ZIP_FILENAME, FILENAME)
printData(data)
visualizeData(data, "1")
""" From the distributions and heatmaps, we see chlorides has the highest skewness and kurtosis.
Also, other variables like volatile acidity, free sulfur dioxide, density, etc have a lot of outliers. """
""" From the boxplot, we confirm that chlorides have the highest number of outliers,
and all the remaining variables have outliers except alcohol. """
""" From the heatmap, residual sugar and density has correlation = 0.84,
free sulfur dioxide and total sulfur dioxide has correlation = 0.62,
alcohol and density has correlation = -0.78. """
""" From the scatterplot, density seems to have a linear or constant relationship with every other variable.
Also, heatmap suggests same. Hence, we will drop density.
Out of free sulfur dioxide and total sulfur dioxide, we drop free sulfur dioxide since it has lower correlation with quality.
We also drop the lowest correlated variable, citric acid. """
def trainModel():
# Parse args
parser = argparse.ArgumentParser(description='Train the CNN')
parser.add_argument('--expt_dir',
default='./logs',
help='save dir for experiment logs')
parser.add_argument('--train',
default='./data',
help='path to training set')
parser.add_argument('--val',
default='./data',
help='path to validation set')
parser.add_argument('--test', default='./data', help='path to test set')
parser.add_argument('--save_dir',
default='./models',
help='path to save model')
parser.add_argument('--arch',
default='models/cnn.json',
help='path to model architecture')
parser.add_argument('--model_name',
default='model',
help='name of the model to save logs, weights')
parser.add_argument('--lr', default=0.001, help='learning rate')
parser.add_argument('--init', default='1', help='initialization')
parser.add_argument('--batch_size', default=20, help='batch_size')
args = parser.parse_args()
# Load data
train_path, valid_path, test_path = args.train, args.val, args.test
logs_path = args.expt_dir
model_path, model_arch, model_name = args.save_dir, args.arch, args.model_name
model_path = os.path.join(model_path, model_name)
if not os.path.isdir(model_path):
os.mkdir(model_path)
lr, batch_size, init = float(args.lr), int(args.batch_size), int(args.init)
data = loadData(train_path, valid_path, test_path)
train_X, train_Y, valid_X, valid_Y, test_X, test_Y = data['train']['X'], data['train']['Y'],\
data['valid']['X'], data['valid']['Y'],\
data['test']['X'], data['test']['Y'],
# Load architecture
arch = loadArch(model_arch)
# Logging
train_log_name = '{}.train.log'.format(model_name)
valid_log_name = '{}.valid.log'.format(model_name)
train_log = setup_logger('train-log',
os.path.join(logs_path, train_log_name))
valid_log = setup_logger('valid-log',
os.path.join(logs_path, valid_log_name))
# GPU config
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=1.0)
# Train
num_epochs = 100
num_batches = int(float(train_X.shape[0]) / batch_size)
steps = 0
patience = 50
early_stop = 0
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as session:
model = CNN(arch, session, logs_path, init, lr)
loss_history = [np.inf]
for epoch in range(num_epochs):
print 'Epoch {}'.format(epoch)
steps = 0
indices = np.arange(train_X.shape[0])
np.random.shuffle(indices)
train_X, train_Y = train_X[indices], train_Y[indices]
for batch in range(num_batches):
start, end = batch * batch_size, (batch + 1) * batch_size
x, y = Augment(train_X[range(start,
end)]).batch, train_Y[range(
start, end)]
try:
model.step(x, y)
except MemoryError:
print 'Memory error in step'
exit()
steps += batch_size
if steps % train_X.shape[0] == 0 and steps != 0:
try:
train_loss, train_acc = testModel(
model, train_X, train_Y, batch_size)
except MemoryError:
print 'Memory error in test for train'
exit()
train_log.info(
'Epoch {}, Step {}, Loss: {}, Accuracy: {}, lr: {}'.
format(epoch, steps, train_loss, train_acc, model.lr))
try:
valid_loss, valid_acc = testModel(
model, valid_X, valid_Y, batch_size)
except MemoryError:
print 'Memory error in test for valid'
exit()
valid_log.info(
'Epoch {}, Step {}, Loss: {}, Accuracy: {}, lr: {}'.
format(epoch, steps, valid_loss, valid_acc, model.lr))
if valid_loss < min(loss_history):
save_path = os.path.join(model_path, 'model')
model.save(save_path)
early_stop = 0
early_stop += 1
if (early_stop >= patience):
print "No improvement in validation loss for " + str(
patience) + " steps - stopping training!"
print("Optimization Finished!")
return 1
loss_history.append(valid_loss)
print("Optimization Finished!")
from eval.rsquared import rsquaredEval
from eval.mae import maeEval
from eval.nmse import nmseEval
from eval.fb import fbEval
from crossvalidation import crossValidation
from models.model_decisiontree import trainDecisionTree, applyDecisionTree
# parameters
k = 4
dataFile1 = "/media/sf_Google_Drive/transfer/data/data_year.csv"
#outputDir = "/media/sf_Google_Drive/transfer/model_output_year/"
# load the data, both of them
data1 = {}
columns1 = []
loadData(dataFile1, ["location", "year"], data1, columns1)
random.seed(42)
models = []
models.append({
"name": "linear",
"norm": NONORMALIZATION,
"train": trainLinearRegression,
"apply": applyLinearRegression,
"data": data1,
"columns": columns1,
"parameters": {
'intercept': True,
'normalize': True,
"features": columns1
}
#
import pandas as pd
import data as dat
# Use pandas to read in the sentiment lexicon:
df = pd.read_csv('lexicon/sentiment-lexicon.csv')
# If you want to preview:
#df.head()
# Load in the data using the data module:
data = dat.loadData()
# Look up words in the lexicon:
score = 0
word_count = 0
sentiment_tally = { 'neutral': 0, 'negative': 0, 'positive': 0 }
missing = {}
found = {}
for word in data:
_score = 0
# Get the lexicon row which matches our word:
row = df[df.word1 == word]