Python SkyNet示例

示例#1

文件： stage_2_launcher_test.py 项目： cc13ny/galatea

job_name = 'cfa_test'
chunk_size = 500
batch_size = 10

from pylearn2.utils import serial
import SkyNet


SkyNet.set_job_name(job_name)
components = SkyNet.get_dir_path('components')

num_examples = serial.load(components+'/num_examples.pkl')
serial.save(components+'/chunk_size.pkl',chunk_size)
serial.save(components+'/batch_size.pkl',batch_size)

#command = '--mem=2000 '+SkyNet.get_user_dir(tmp = False, force = SkyNet.cluster) + '/galatea/contractive_feature_analysis/mnist_experiment_condor/stage_3_worker.py '+job_name+' "{{'
print "HACK: not asking for mem=2000"
command = SkyNet.get_user_dir(tmp = False, force = SkyNet.cluster) + '/galatea/contractive_feature_analysis/mnist_experiment_condor/stage_3_worker_test.py '+job_name+' "{{'

assert num_examples % chunk_size == 0

for b in xrange(0,num_examples,chunk_size):
    if b != 0:
        command+=','
    command += str(b)
command += '}}"'

SkyNet.launch_job(command)

示例#2

print 'Compiling theano PCA function'
t1 = time.time()
pca_input = T.matrix()
pca_output = pca_model(pca_input)
pca_func = function([pca_input], pca_output)
t2 = time.time()
print(t2 - t1), ' seconds'

print 'Running PCA'
t1 = time.time()
g0 = pca_func(X)
del X
t2 = time.time()
print(t2 - t1), ' seconds'

SkyNet.set_job_name(job_name)
components = SkyNet.get_dir_path('components')
serial.save(components + '/pca_model.pkl', pca_model)

del pca_model
del pca_output
del pca_func
gc.collect()

print 'Computing basis expansion'
t1 = time.time()
g1 = expand.expand(g0)

expanded_dim = g1.shape[1]
t2 = time.time()
print(t2 - t1), ' seconds'

示例#3

文件： stage_1_prepper_no_pca.py 项目： cc13ny/galatea

num_examples, input_dim = X.shape

pca_dim = X.shape[1]
print 'Training PCA with %d dimensions' % pca_dim
t1 = time.time()
pca_model = CovEigPCA(num_components = pca_dim)
pca_model.train(X)
pca_model.W.set_value(N.cast['float32'](N.identity(X.shape[1])))
pca_model.mean.set_value(N.cast['float32'](N.zeros(X.shape[1])))
t2 = time.time()
print (t2-t1),' seconds'


SkyNet.set_job_name(job_name)
components = SkyNet.get_dir_path('components')
serial.save(components+'/pca_model.pkl',pca_model)
serial.save(components+'/dataset.pkl',data)

g0 = X

print 'Computing basis expansion'
t1 = time.time()
g1 = expand.expand(g0)


expanded_dim = g1.shape[1]
t2 = time.time()
print (t2-t1),' seconds'

示例#4

文件： stage_4_finisher.py 项目： cc13ny/galatea

job_name = 'cfa_cos_tanh'

from pylearn2.utils import serial
import SkyNet
import numpy as N
from scipy.linalg import eigh
import time

SkyNet.set_job_name(job_name)
components = SkyNet.get_dir_path('components')

num_examples = serial.load(components+'/num_examples.pkl')
chunk_size = serial.load(components+'/chunk_size.pkl')
batch_size = serial.load(components+'/batch_size.pkl')
expanded_dim = serial.load(components+'/expanded_dim.pkl')
whitened_dim = serial.load(components+'/whitener.pkl').get_weights().shape[1]

instability_matrices = SkyNet.get_dir_path('instability_matrices')
components = SkyNet.get_dir_path('components')

assert num_examples % chunk_size == 0
num_chunks = num_examples / chunk_size

G = N.zeros((whitened_dim, whitened_dim) )

print 'Summing up instability matrices'
for b in xrange(0,num_examples,chunk_size):
    tmp = N.load(instability_matrices+'/instability_matrix_%d.npy' % b)
    G += tmp

示例#5

job_name = 'cfa_cos_tanh'
chunk_size = 500
batch_size = 10

from pylearn2.utils import serial
import SkyNet


SkyNet.set_job_name(job_name)
components = SkyNet.get_dir_path('components')

num_examples = serial.load(components+'/num_examples.pkl')
serial.save(components+'/chunk_size.pkl',chunk_size)
serial.save(components+'/batch_size.pkl',batch_size)

command = '--mem=2000 '+SkyNet.get_user_dir(tmp = False, force = SkyNet.cluster) + '/galatea/contractive_feature_analysis/cos_experiment_condor/stage_3_worker.py '+job_name+' "{{'

assert num_examples % chunk_size == 0

for b in xrange(0,num_examples,chunk_size):
    if b != 0:
        command+=','
    command += str(b)
command += '}}"'

SkyNet.launch_job(command)

示例#6

job_name = 'cfa_olshausen'

from pylearn2.utils import serial
import SkyNet
import numpy as N
from scipy.linalg import eigh
import time

SkyNet.set_job_name(job_name)
components = SkyNet.get_dir_path('components')

num_examples = serial.load(components + '/num_examples.pkl')
chunk_size = serial.load(components + '/chunk_size.pkl')
batch_size = serial.load(components + '/batch_size.pkl')
expanded_dim = serial.load(components + '/expanded_dim.pkl')

instability_matrices = SkyNet.get_dir_path('instability_matrices')
components = SkyNet.get_dir_path('components')

assert num_examples % chunk_size == 0
num_chunks = num_examples / chunk_size

G = N.zeros((expanded_dim, expanded_dim))

print 'Summing up instability matrices'
for b in xrange(0, num_examples, chunk_size):
    tmp = N.load(instability_matrices + '/instability_matrix_%d.npy' % b)
    G += tmp

print 'Finding eigenvectors'
t1 = time.time()

示例#7

文件： condor_launch_4000_1.py 项目： cc13ny/galatea

import numpy as N
import SkyNet

job_name = "recons_srbm_4000_1"

SkyNet.set_job_name(job_name)
configsDir = SkyNet.get_dir_path('configs')
modelsDir = SkyNet.get_dir_path('models')

rng = N.random.RandomState([1,2,3])

command = "--mem=2000 "+SkyNet.get_user_dir(tmp = False, force = SkyNet.cluster) + '/ift6266h11/framework/scripts/train.py '+configsDir+'/config_"{{'

first = True

num_jobs = 100

for i in xrange(num_jobs):

    if not first:
        command += ','
    ""

    first = False

    nhid = 4000
    irange = rng.uniform(.01,.03)
    if rng.uniform(0.,1.) > 0.5:
        learn_beta = 1
        beta_lr_scale = N.exp(rng.uniform(N.log(1e-5),N.log(1e-2)))
        beta = N.exp(rng.uniform(N.log(1.0),N.log(5)))

示例#8

import numpy as N
import SkyNet

job_name = "recons_srbm_2"

SkyNet.set_job_name(job_name)
configsDir = SkyNet.get_dir_path('configs')
modelsDir = SkyNet.get_dir_path('models')

rng = N.random.RandomState([1,2,3])

command = SkyNet.get_user_dir(tmp = False, force = SkyNet.cluster) + '/ift6266h11/framework/scripts/train.py '+configsDir+'/config_"{{'

first = True

num_jobs = 100

for i in xrange(num_jobs):

    if not first:
        command += ','
    ""

    first = False

    nhid = 4000
    irange = rng.uniform(.01,.03)
    if rng.uniform(0.,1.) > 0.5:
        learn_beta = 1
        beta_lr_scale = N.exp(rng.uniform(N.log(1e-4),N.log(1e-2)))
        beta = N.exp(rng.uniform(N.log(1.0),N.log(5)))

示例#9

文件： reference_package_wiskott_labels.py 项目： vd114/galatea

#! /usr/bin/env python

"
Note: this file was used only for dataset creation. It is provided here only as a reference
"
assert False, 'This file is only a reference and should probably not be run or imported.'



import os
import numpy as N
import SkyNet
from scipy import io

base = SkyNet.get_wiskott_path() + '/'

sets = os.listdir(base)
sets = [ x for x in sets if x != 'zips' ]

for set in sets:
    print 'making labels for '+set
    setdir = base + set

    is_fish = (set.find('fish') != -1)


    print '\treading labels'

    configs = io.loadmat(setdir+'/configs/configs.mat')
    print '\tformatting labels'