def timing_triple_cloud():
execfile('picloud_venture_credentials.py')
exp_params = experiment.exp_param_defaults({})
exp_params['intermediate_iter'] = 1
exp_params['max_initial_run_time'] = 30
exp_params['max_burn_time'] = 30
exp_params['max_sample_time'] = 30
exp_params['n_samples'] = 25
print experiment.exp_params_to_str(exp_params)
data = scipy.io.loadmat("../data/irm_synth/irm_synth_20.mat", squeeze_me=True)
observed = list(zip(data['train_i'].flat, data['train_j'].flat, data['train_v'].flat))
missing = list(zip(data['test_i'].flat, data['test_j'].flat, data['test_v'].flat))
data = {'observations' : observed, 'missing' : missing}
model = models.product_IRM
model_params = {'D' : 1, 'alpha' : 1, 'symmetric' : True}
# Timing run
print 'Timing'
job_id = cloud.call(experiment.network_cv_timing_run, data, model, exp_params, model_params, _max_runtime=5, _env=cloud_environment)
time_per_mh_iter = cloud.result(job_id)['time_per_mh_iter']
# Live run
print 'Live'
exp_params['intermediate_iter'] = max(1, int(round(0.9 * exp_params['max_sample_time'] / (exp_params['n_samples'] * time_per_mh_iter))))
job_id = cloud.call(experiment.network_cv_single_run, data, model, exp_params, model_params, _max_runtime=5, _env=cloud_environment)
cloud.join(job_id)
print cloud.result(job_id)
def main():
cloud.setkey(2329, '270cb3cccb9beb65d2f424b24ccbd5a920c5ccef')
try:
fn = raw_input()
f = open(fn)
L = float(f.readline())
line = f.readline()
data = []
while line:
d = map(eval, line.split())
data.append(d)
line = f.readline()
f.close()
n = len(data)
Gs = np.array(data[0:n / 2])
ls = np.array(data[n / 2::])
outstr = ''
outstr += 'submitting cross validation to picloud\n'
cloud.config.max_transmit_data = 12000000
start = time.time()
jid = cloud.call(traintst, Gs, ls, L)
outstr += 'submission time: %s\n' % str(time.time() - start)
start = time.time()
result = cloud.result(jid)
outstr += 'cloud execution time: %s\n' % str(time.time() - start)
outstr += 'misclassification rate: %f\n' % np.mean(result)
outstr += 'standard deviation: %f\n' % np.std(result)
outstr += '--------done---------------------'
print outstr
except:
print 'an error occurred'
def patentComparer(): print 'Started:' print datetime.now() #wordVectorDir = "C:\\Users\\Deepak\\Dropbox\\6.864 Project\\Word Vectors" wordVectorDir = "/Users/aratner/Dropbox/6.864 Project/Word Vectors" # Get patent claims patentData = getPatentData(200) print "Got patent data..." # Obtain the tree from the claim using the rule based parser / machine learnt model trees = [(x,hm_tree(y,3)) for (x,y) in patentData] embeddingTrees = [(x,generateEmbeddingTree(tree, wordVectorDir)) for (x,tree) in trees] #print embeddingTrees #for embeddingTree in embeddingTrees: # embeddingTree[1].draw() print "Generated trees for claims..." # Partition trees into two sets - a training set and a testing set (trainingTrees, testingTrees) = partitionTrees(embeddingTrees) # Run the unfolding recursive auto-encoder on the tree, using picloud #params = train_params(trainingTrees, math.pow(10,-5), 0.01, False) jid = cloud.call(train_params, trainingTrees, math.pow(10,-5), 0.01, True, 60, _type='c2') print 'jid = ' + str(jid) params = cloud.result(jid) save_params(params, '_finalpicloud') print 'RAE training complete, parameters saved, at:' print datetime.now() print "Done..."
def runcode(self, code):
"""Execute a code object.
When an exception occurs, self.showtraceback() is called to
display a traceback. All exceptions are caught except
SystemExit, which is reraised.
A note about KeyboardInterrupt: this exception may occur
elsewhere in this code, and may not always be caught. The
caller should be prepared to deal with it.
"""
try:
job = cloud.call(cloud_run, code, self.locals)
cloud.join(job)
result = cloud.result(job)
self.locals.update(result)
info = cloud.info(job, ['stderr', 'stdout'])[job]
sys.stdout.write(info['stdout'])
sys.stderr.write(info['stderr'])
except SystemExit:
raise
except KeyboardInterrupt:
raise OperationAborted('Interrupted')
except cloud.CloudException, e:
self.showcloudtraceback(e)
def main():
cloud.setkey(2329,'270cb3cccb9beb65d2f424b24ccbd5a920c5ccef')
try:
fn = raw_input()
f = open(fn)
L = float(f.readline())
line = f.readline()
data = []
while line:
d = map(eval,line.split())
data.append(d)
line = f.readline()
f.close()
n = len(data)
Gs = np.array(data[0:n/2])
ls = np.array(data[n/2::])
outstr = ''
outstr += 'submitting cross validation to picloud\n'
cloud.config.max_transmit_data=12000000
start = time.time()
jid = cloud.call(traintst,Gs,ls,L)
outstr += 'submission time: %s\n' %str(time.time()-start)
start = time.time()
result = cloud.result(jid)
outstr += 'cloud execution time: %s\n' %str(time.time()-start)
outstr += 'misclassification rate: %f\n' %np.mean(result)
outstr += 'standard deviation: %f\n' %np.std(result)
outstr += '--------done---------------------'
print outstr
except:
print 'an error occurred'
def geoS(N, S):
job_id = cloud.call(get_LowerTrunc_GeomSeries, N, S, _type='m1')
geomSeries = cloud.result(job_id)
geomSeries = get_LowerTrunc_GeomSeries(N, S)
geomSeries = np.log(geomSeries)
plt.plot(geomSeries, color='m',lw=3,label='Geometric series\nN='+str(N)+' S='+str(S))
print 'geometric series: done'
return
def test_picloud(cloud, n, fast):
begin = time.time()
jid = cloud.call(get_prime, n, _high_cpu=True)
cloud.result(jid)
print n,
print "took ",
print time.time() - begin,
print "seconds"
def test_picloud(cloud, n, fast):
begin = time.time()
jid = cloud.call(get_prime, n, _high_cpu=True)
cloud.result(jid)
print n ,
print "took " ,
print time.time() - begin ,
print "seconds"
def logS(N, S):
job_id = cloud.call(mete.get_mete_rad, S, N, _type='m1')
logSeries = cloud.result(job_id)
#logSeries = mete.get_mete_rad(S, N) # The expected SAD from the random sample
logSeries = np.log(logSeries[0])
plt.plot(logSeries, color='gray', lw=3, label='Log-series\nN='+str(N)+' S='+str(S))
print 'log-series: done'
return
def process_json(json_data, use_cloud, quit_on_first=False, ignore_pickle=False, debug_info=None):
ret = None
if not use_cloud:
ret = _process(json_data, use_cloud, quit_on_first=quit_on_first, ignore_pickle=ignore_pickle, debug_info=debug_info)
else:
ret = cloud.call(_process, json_data, quit_on_first=quit_on_first, use_cloud=True, ignore_pickle=ignore_pickle, _type="f2", _env='pygame_env')
return ret
def generate_items_analytics_main_job(index, itemidlist):
itemdetailjobids = []
filenames = []
if index.token:
getitembestmatchlist = [itemidlist[i:i + index.configuration['item_analytics_calls_per_machine']] for i in range(0, len(itemidlist), index.configuration['item_analytics_calls_per_machine'])]
for sublist in getitembestmatchlist:
itemdetailjobids.append(cloud.call(etl.get_item_analytics, etldriver=index.etldriver, itemid=sublist, selleridkey=index.selleridkey,locale=index.locale, token=index.token, _type="c1", _label="GET ITEMS BESTMATCH"))
filenames.append(make_file_name(itemid=sublist, locale=index.locale, datatype="bestmatch"))
return itemdetailjobids, filenames
def clear(prefix, remote):
if (remote):
k = cloud.call(delete_prefix, prefix)
return cloud.result(k)
else:
path = util.cachePath(prefix)
if (os.path.isdir(path)):
sh.rmtree(path)
elif (os.path.isfile(path)):
os.remove(path)
def report(batch, remote, debug, dependency = []):
params = cache.get("batch/%s/params" % batch, remote)
logging.info("running reporter instance")
if (remote):
k = cloud.call(reporter.report, batch, params, remote, debug, _label = "%s/report" % batch, _depends_on = dependency, _type = 'c1', _max_runtime = 30)
logging.info("k %s" % k)
return k
else:
result = reporter.report(batch, params, remote, debug)
return result
def calc_pi():
"""Incorrect way"""
# offload monte_carlo to the cloud, returns a Job Id
jid = cloud.call(monte_carlo, total_tests, _type='c2')
# block until job is done, and get result
num_in_circle = cloud.result(jid)
pi = (4 * num_in_circle) / float(total_tests)
return pi
def calc_pi():
"""Incorrect way"""
# offload monte_carlo to the cloud, returns a Job Id
jid = cloud.call(monte_carlo, total_tests, _type='c2')
# block until job is done, and get result
num_in_circle = cloud.result(jid)
pi = (4 * num_in_circle) / float(total_tests)
return pi
def branch_out_in_cloud(board, i, j, possibilities, depth):
jids = []
for digit in possibilities:
new_board = copy.deepcopy(board)
new_board.fill(i, j, digit)
jid = cloud.call(solve, new_board, depth)
jids.append(jid)
for result in cloud.iresult(jids):
if result:
return result
return None
def build_index(**kwargs):
sellerdata = kwargs.get('sellerdata')
configuration = kwargs.get('configuration')
etlconfiguration = kwargs.get('etlconfiguration', False)
itemids = kwargs.get('itemids', [])
sellerdata['etl'] = json.loads(sellerdata['etl'])
if not configuration:
from etl.config.drivers.picloud.ebay import picloud_ebay_configuration
configuration = picloud_ebay_configuration
index = BuildLogic(sellerdata, configuration, PiCloudEBayETLDriver(), etlconfiguration)
itemdetailjobid = 0
categoryjobid = cloud.call(generate_categories, index=index, _type="c1", _max_runtime=20, _label="GET GATEGORIES")
findingjobids = cloud.call(generate_finding_calls, index=index, categoryjobid=categoryjobid, _type="c1", _depends_on=categoryjobid, _label="GENERATE FINDING CALLS")
#findingjobids=243103
#get_items_detail_and_analytics_main_job(index=index,findingjobids=findingjobids)
itemdetailjobid = cloud.call(generate_items_detail_and_analytics_main_job, index=index, findingjobids=findingjobids, _type="c1", _depends_on=findingjobids, _label="GENERATE ITEM DETAIL CALLS")
return itemdetailjobid
def create_alternative_worlds(self):
'''
Simulate each of the economies.
'''
T = self.T
I = self.I
print 'Simulating a world with %d countries.' % I
# these are the individual simulation jobs
jobs = cloud.map(simulate_an_economy,self.economies,[T]*I,_label='A world with %d countries.' % I,_type='c2')
# this is a harvester job, collecting the results of each individual simulation
return cloud.call(GDP_collector, jobs, T, T1, T2,_depends_on=jobs,_label='Collecting GDP results.')
def __init__(self, usecloud=False, cloud_env=''):
if usecloud and not have_cloud:
print 'warning: cloud module is not installed, running locally'
usecloud = False
self.usecloud = usecloud
self.cloud_env = cloud_env
if self.usecloud:
print 'uploading dataset to cloud...'
cloud.files.put(DIGITS_FN)
self.preprocess_job = cloud.call(self.preprocess, _env=self.cloud_env)
else:
self._samples, self._labels = self.preprocess()
def __init__(self, usecloud=False, cloud_env=''):
if usecloud and not have_cloud:
print 'warning: cloud module is not installed, running locally'
usecloud = False
self.usecloud = usecloud
self.cloud_env = cloud_env
if self.usecloud:
print 'uploading dataset to cloud...'
cloud.files.put(DIGITS_FN)
self.preprocess_job = cloud.call(self.preprocess,
_env=self.cloud_env)
else:
self._samples, self._labels = self.preprocess()
def f(gens):
k = 7
n= k + 1
effectiveAttrIndices = range(k)
probMutation = 0.004
probMisclassification = 0.20
popSize = 1500
jid = cloud.call(neap_uga, **dict(m=popSize,
n=n,
gens=gens,
probMutation=probMutation,
effectiveAttrIndices=effectiveAttrIndices,
probMisclassification=probMisclassification))
print "Kicked off trial %s" % jid
return jid
def do_network_call(itemidlistlist=[], callback=None):
setcloudkey()
etldriver = PiCloudEBayETLDriver()
filenamelist = []
try:
jobids = []
for sublist in itemidlistlist:
jobid = cloud.call(etl.get_item_details, etldriver=etldriver, itemid=sublist, _type="c1", _label="GET NOTIFICATION ITEM DETAILS")
jobids.append(jobid)
filename = make_file_name(itemid=sublist, datatype="itemdetail")
filenamelist.append(filename)
print jobids
if len(jobids) > 0:
cloud.result(jobids, ignore_errors=False)
except Exception, e:
print e
def timing_run_cloud():
execfile('picloud_venture_credentials.py')
exp_params = experiment.exp_param_defaults({})
exp_params['intermediate_iter'] = 1
exp_params['max_initial_run_time'] = 30
print experiment.exp_params_to_str(exp_params)
data = scipy.io.loadmat("../data/irm_synth/irm_synth_20.mat", squeeze_me=True)
observed = list(zip(data['train_i'].flat, data['train_j'].flat, data['train_v'].flat))
missing = list(zip(data['test_i'].flat, data['test_j'].flat, data['test_v'].flat))
data = {'observations' : observed, 'missing' : missing}
model = models.product_IRM
model_params = {'D' : 1, 'alpha' : 1, 'symmetric' : True}
job_id = cloud.call(experiment.network_cv_timing_run, data, model, exp_params, model_params, _max_runtime=5, _env=cloud_environment)
cloud.join(job_id)
print cloud.result(job_id)
def f(gens):
k = 7
n = k + 1
effectiveAttrIndices = range(k)
probMutation = 0.004
probMisclassification = 0.20
popSize = 1500
jid = cloud.call(
neap_uga,
**dict(m=popSize,
n=n,
gens=gens,
probMutation=probMutation,
effectiveAttrIndices=effectiveAttrIndices,
probMisclassification=probMisclassification))
print "Kicked off trial %s" % jid
return jid
def trainIntermediateRAE():
print 'Started:'
print datetime.now()
#wordVectorDir = "C:\\Users\\Deepak\\Dropbox\\6.864 Project\\Word Vectors"
wordVectorDir = "/Users/aratner/Dropbox/6.864 Project/Word Vectors"
# Get patent claims
patentData = getPatentData(500)
print "Got patent data..."
# Obtain the tree from the claim using the rule based parser / machine learnt model
trees = [(x, hm_tree(y, 3)) for (x, y) in patentData]
totalTrainingdata = []
for treeData in trees:
tree = treeData[1]
#tree.draw()
label = treeData[0]
totalTrainingdata.extend(generatePhraseTree(tree, wordVectorDir))
print "Generated trees for claims..."
# Partition trees into two sets - a training set and a testing set
#(trainingTrees, testingTrees) = partitionTrees(totalTrainingdata)
trainingTrees = totalTrainingdata
testingTrees = totalTrainingdata
#for tree in trainingTrees:
# if (len(tree) > 1):
# tree.draw()
# Run the unfolding recursive auto-encoder on the tree
jid = cloud.call(train_params,
trainingTrees,
math.pow(10, -5),
0.01,
True,
60,
_type='c2')
print 'jid =' + str(jid)
params = cloud.result(jid)
save_params(params, '_finalpicloud_subtrees')
print 'RAE training complete, at:'
print datetime.now()
print "Done..."
def build_bestmatchonly_index(**kwargs):
sellerdata = kwargs.get('sellerdata')
configuration = kwargs.get('configuration')
etlconfiguration = kwargs.get('etlconfiguration', False)
itemids = kwargs.get('itemids', [])
sellerdata['etl'] = json.loads(sellerdata['etl'])
if not configuration:
from etl.config.drivers.picloud.ebay import picloud_ebay_configuration
configuration = picloud_ebay_configuration
index = BuildLogic(sellerdata, configuration, PiCloudEBayETLDriver(), etlconfiguration)
itemidlist = get_sellers_items(index.selleridkey)
itemdetailjobid = cloud.call(generate_items_analytics_main_job, index=index, itemidlist=itemidlist, _type="c1", _label="GENERATE ITEM BESTMATCH ONLY CALLS")
return itemdetailjobid
def calc_pi():
"""Almost correct way"""
num_jobs = 8
tests_per_call = total_tests/num_jobs
# list of job ids for all jobs we're spawning
jids = []
for _ in range(num_jobs):
# call() does not block, so jobs run in parallel
jid = cloud.call(monte_carlo, tests_per_call, _type='c2')
jids.append(jid)
# aggregate the number of darts that land in the circle
# across all jobs that we spawned
num_in_circle = 0
for jid in jids:
num_in_circle += cloud.result(jid)
pi = (4 * num_in_circle) / float(total_tests)
return pi
def trainIntermediateRAE(): print 'Started:' print datetime.now() #wordVectorDir = "C:\\Users\\Deepak\\Dropbox\\6.864 Project\\Word Vectors" wordVectorDir = "/Users/aratner/Dropbox/6.864 Project/Word Vectors" # Get patent claims patentData = getPatentData(500) print "Got patent data..." # Obtain the tree from the claim using the rule based parser / machine learnt model trees = [(x,hm_tree(y,3)) for (x,y) in patentData] totalTrainingdata = [] for treeData in trees: tree = treeData[1] #tree.draw() label = treeData[0] totalTrainingdata.extend(generatePhraseTree(tree, wordVectorDir)) print "Generated trees for claims..." # Partition trees into two sets - a training set and a testing set #(trainingTrees, testingTrees) = partitionTrees(totalTrainingdata) trainingTrees = totalTrainingdata testingTrees = totalTrainingdata #for tree in trainingTrees: # if (len(tree) > 1): # tree.draw() # Run the unfolding recursive auto-encoder on the tree jid = cloud.call(train_params, trainingTrees, math.pow(10,-5), 0.01, True, 60, _type='c2') print 'jid ='+str(jid) params = cloud.result(jid) save_params(params, '_finalpicloud_subtrees') print 'RAE training complete, at:' print datetime.now() print "Done..."
def patentComparer():
print 'Started:'
print datetime.now()
#wordVectorDir = "C:\\Users\\Deepak\\Dropbox\\6.864 Project\\Word Vectors"
wordVectorDir = "/Users/aratner/Dropbox/6.864 Project/Word Vectors"
# Get patent claims
patentData = getPatentData(200)
print "Got patent data..."
# Obtain the tree from the claim using the rule based parser / machine learnt model
trees = [(x, hm_tree(y, 3)) for (x, y) in patentData]
embeddingTrees = [(x, generateEmbeddingTree(tree, wordVectorDir))
for (x, tree) in trees]
#print embeddingTrees
#for embeddingTree in embeddingTrees:
# embeddingTree[1].draw()
print "Generated trees for claims..."
# Partition trees into two sets - a training set and a testing set
(trainingTrees, testingTrees) = partitionTrees(embeddingTrees)
# Run the unfolding recursive auto-encoder on the tree, using picloud
#params = train_params(trainingTrees, math.pow(10,-5), 0.01, False)
jid = cloud.call(train_params,
trainingTrees,
math.pow(10, -5),
0.01,
True,
60,
_type='c2')
print 'jid = ' + str(jid)
params = cloud.result(jid)
save_params(params, '_finalpicloud')
print 'RAE training complete, parameters saved, at:'
print datetime.now()
print "Done..."
def create_alternative_worlds(self):
'''
Simulate each of the economies.
'''
T = self.T
I = self.I
print 'Simulating a world with %d countries.' % I
# these are the individual simulation jobs
jobs = cloud.map(simulate_an_economy,
self.economies, [T] * I,
_label='A world with %d countries.' % I,
_type='c2')
# this is a harvester job, collecting the results of each individual simulation
return cloud.call(GDP_collector,
jobs,
T,
T1,
T2,
_depends_on=jobs,
_label='Collecting GDP results.')
def start_indexing_on_cloud(self, etlconfiguration):
try:
successfullsellerfilenamelist=[]
jobids = []
self.filenamelist = []
self.successsellers = []
mainjobids = []
for seller in self.sellers:
try:
update_sellers_status([seller], dict(phase="INDEXING", progress="STARTED"))
if etlconfiguration == "BESTMATCHONLY":
tempjobids = build_bestmatchonly_index(sellerdata=seller, configuration=self.configuration, etlconfiguration=etlconfiguration)
seller['itemdetailjobid'] = tempjobids
mainjobids.append(tempjobids)
else:
jobids.append(cloud.call(build_index, sellerdata=seller, configuration=self.configuration, etlconfiguration=etlconfiguration, _label="SELLER"))
seller['jobid'] = jobids[len(jobids) - 1]
self.successsellers.append(seller)
except Exception, e:
logger.exception(str(e))
update_sellers_status([seller], dict(phase="INDEXING", progress="ERROR"))
finally:
pass
def fetch_results(self, iters=None, via_remote=False, run_mode='local'):
"""
Returns the result of the job that has already been run as a :py:class:`History` object. Typically you would call :py:meth:`run` first, then call :py:meth:`fetch_results` to get the resutlts. The method has various methods to control how much of the job is returned, to avoid excessive memory usage and data transfer between the cloud and local machine.
:param iters: If *iters* is an iterable, returns only the iterations of the chain in *iters*. If *iters* is a scalar, return every *iters* state (the stride). If None, returns all states.
:param via_remote: If *True*, executes the state filtering on the cloud before transferring the data to the local machine. If false, filter the state on the local machine.
:param run_mode: Controls whether to search for the results on the local macine or on the cloud. Can be *local* or *cloud*.
:return: A :py:class:`History` object that contains a filtered version the states of the Markov chain visited when this job ran.
"""
def f():
if run_mode == 'cloud':
cloud.join([self.job_id])
store = storage.CloudStore()
else:
store = storage.LocalStore()
full_history = store[self.params]
partial_history = History()
if iters is None:
partial_history.states = full_history.states
else:
if isinstance(iters, int): #iters interpreted as stride
iter_set = range(0, len(full_history.states), iters)
else:
iter_set = iters
partial_history.states = [
state for state in full_history.states
if state.iter in iter_set
]
partial_history.job = self
partial_history.summary = full_history.summary
return partial_history
if via_remote:
job_id = cloud.call(f, _env=picloud_env)
return cloud.result(job_id)
else:
return f()
def run_test():
HowManyTimesIsTricky = 0
for sample_number in range(NumberOfSamples):
sample = sample_once(0)
if sample is True:
HowManyTimesIsTricky = HowManyTimesIsTricky + 1
print "** Time: " + str()
return "Empirical prob.: " + str(
float(HowManyTimesIsTricky) /
NumberOfSamples) + " versus the theoretical =appr.= 0.850498339"
start_time = time.time()
print "Without mapping:"
jid = cloud.call(run_test, _env=cloud_environment, _type='c1')
result = cloud.result(jid)
print "** Time: " + str(time.time() - start_time)
print "** " + result
print ""
print ""
print ""
print "With mapping:"
start_time = time.time()
jids = cloud.map(sample_once,
range(NumberOfSamples),
_env=cloud_environment,
_type='c1')
result = cloud.result(jids)
import cloud
import xmlrpclib
def matrix_multiply_test():
import numpy as np
from time import time
x = np.zeros((1000, 1000))
t_start = time()
np.dot(x, x)
t_elapsed = time() - t_start
s = xmlrpclib.ServerProxy('http://localhost:8001')
s.add(2, 3)
return t_elapsed
print "local computation time:", matrix_multiply_test()
from time import time
for i in xrange(1):
jid = cloud.call(matrix_multiply_test, _type='c2')
t_start_req = time()
print "cloud computation time:", cloud.result(jid)
print "total request time:", time() - t_start_req
def main(): jid = cloud.call(search_yellow(), type='m1') cloud.result(jid)
def test_exception3():
'''Raise TypeError since cloud.call called with 1 invalid argument'''
jid = cloud.call("asdf")
def test_multiply():
jid = cloud.call(lambda: 3*3)
answer = cloud.result(jid)
assert answer == 9
randomword3 = randomword(5)
if engdict.check(randomword3) == True:
randomkey3 = randomword3 + str(random.randint(0, 99))
elif engdict.check(randomword3) == False:
englist = engdict.suggest(randomword3)
if len(englist) > 0:
randomkey3 = englist[0] + str(random.randint(0, 99))
else:
randomkey3 = randomword3 + str(random.randint(0, 99))
if 'randomkey0' and 'randomkey3' and 'randomkey1' in locals():
whasher0 = hashlib.new("md5")
whasher0.update(randomkey0)
whasher3 = hashlib.new("md5")
whasher3.update(randomkey3)
whasher1 = hashlib.new("md5")
whasher1.update(randomkey1)
print(randomkey0 + " + " + str(whasher0.hexdigest()) + "\n")
print(randomkey3 + " + " + str(whasher3.hexdigest()) + "\n")
print(randomkey1 + " + " + str(whasher1.hexdigest()) + "\n")
fileb.write(randomkey0 + " + " + str(whasher0.hexdigest()) + "\n")
fileb.write(randomkey3 + " + " + str(whasher3.hexdigest()) + "\n")
fileb.write(randomkey1 + " + " + str(whasher1.hexdigest()) + "\n")
jid = cloud.call(randomword) # square(3) evaluated on PiCloud
cloud.result(jid)
print('Value added to cloud')
print('Password added')
mainroutine()
def run(self, run_mode='local', use_cache=False):
"""
Runs the job, storing all the states of the Markov chain in a datastore.
:param run_mode: A string that controls how and where the job is run. Currently has two allowable values:
local
Runs the job locally and stores the data locally. Useful for debugging.
cloud
Runs the job on picloud and stores the data in a picloud bucket.
:param use_cache: If *True* and this job has already been run at a previous time, return the results of that job. If *False*, rerun the job.
:return: If run_mode is 'cloud', returns the picloud job id in a non-blocking way. If run_mode is 'local', does not return anything and will not return until the job is completed.
:raise:
"""
chain = self.chain
data = self.data
if run_mode == 'cloud':
store = storage.CloudStore()
else:
store = storage.LocalStore()
# Calculating the key outside of the cloud is necessary since the hashing functions on the cloud may not
# agree with the local hashing functions (might be a 32-bit vs 64-bit python issue).
self.key = store.hash_key(self.params)
def f():
if run_mode == "cloud":
store = storage.CloudStore()
elif run_mode == "local":
store = storage.LocalStore()
else:
raise BaseException("Run mode %r not recognized" % run_mode)
store.auto_hash = False
if use_cache and (self.key in store):
logger.debug("Cache hit")
return
logger.debug("Cache miss")
logger.debug("Running job")
data._load_data()
chain.data = data.train_data
chain.data_source = data
ioff()
states = chain._run()
ion()
logger.debug('Chain completed')
history = History()
history.states = states
history.job = self
logger.debug("Summarizing chain")
history.summary = chain.summarize(history)
logger.debug("Chain summarized")
logger.debug("Job params: %r" % (self.params, ))
store[self.key] = history
store.close()
if run_mode == 'local':
return f()
elif run_mode == 'cloud':
job_id = cloud.call(f, _env=picloud_env)
self.job_id = job_id
return job_id
def run_experiment(experiment,
roleouts,
episodes,
in_cloud=False,
dynProfile=None):
""" Runs the given experiment and returns the results.
"""
def run():
if dynProfile is None:
maxsteps = len(experiment.profile) # episode length
else:
maxsteps = dynProfile.shape[1]
na = len(experiment.agents)
ni = roleouts * episodes * maxsteps
all_action = zeros((na, 0))
all_reward = zeros((na, 0))
epsilon = zeros((na, ni)) # exploration rate
# Converts to action vector in percentage markup values.
vmarkup = vectorize(get_markup)
for roleout in range(roleouts):
if dynProfile is not None:
# Apply new load profile before each roleout (week).
i = roleout * episodes # index of first profile value
experiment.profile = dynProfile[i:i + episodes, :]
# print "PROFILE:", experiment.profile, episodes
experiment.doEpisodes(
episodes) # number of samples per learning step
nei = episodes * maxsteps # num interactions per role
epi_action = zeros((0, nei))
epi_reward = zeros((0, nei))
for i, (task, agent) in \
enumerate(zip(experiment.tasks, experiment.agents)):
action = copy(agent.history["action"])
reward = copy(agent.history["reward"])
for j in range(nei):
if isinstance(agent.learner, DirectSearchLearner):
action[j, :] = task.denormalize(action[j, :])
k = nei * roleout
epsilon[i, k:k + nei] = agent.learner.explorer.sigma[0]
elif isinstance(agent.learner, ValueBasedLearner):
action[j, :] = vmarkup(action[j, :], task)
k = nei * roleout
epsilon[i, k:k + nei] = agent.learner.explorer.epsilon
else:
action = vmarkup(action, task)
# FIXME: Only stores action[0] for all interactions.
epi_action = c_[epi_action.T, action[:, 0].flatten()].T
epi_reward = c_[epi_reward.T, reward.flatten()].T
if hasattr(agent, "module"):
print "PARAMS:", agent.module.params
agent.learn()
agent.reset()
all_action = c_[all_action, epi_action]
all_reward = c_[all_reward, epi_reward]
return all_action, all_reward, epsilon
if in_cloud:
import cloud
job_id = cloud.call(run, _high_cpu=False)
result = cloud.result(job_id)
all_action, all_reward, epsilon = result
else:
all_action, all_reward, epsilon = run()
return all_action, all_reward, epsilon
def step_fitness(self):
"""
Run fitness tests for the current generation, and evolve the next generation.
"""
# Firstly, render code for all the genomes in the current population. Each genome owns its own
# simulation object, because we want to interleave the simulations, running D_ROUNDS of simulation
# rounds for all genomes, and killing off the weakest until BROOD_SIZE genomes remain.
if self.next_population:
self.population = copy.deepcopy(self.next_population)
self.next_population = None
for genome in self.population:
code = genome.render(debug = self.DEBUG)
genome.code_hash = md5.md5(code).hexdigest()
genome.agent_name = 'agent_' + genome.code_hash
genome.agent_path = 'agents/rendered/' + genome.agent_name + '.py'
f = open(genome.agent_path, 'w')
f.write(code)
f.close()
genome.agent_module = __import__('agents.rendered.'+genome.agent_name, fromlist=['*'])
genome.simulation = simulate.Simulate(**self.sim_parameters)
genome.simulation.agent_move = genome.agent_module.move
genome.simulation.agent_observe_who = genome.agent_module.observe_who
jobs = {}
def job_callback(job):
jobs[job].simulation = cloud.result(job)
logger.debug('Job %d completed with fitness %.2f.' % (job, 1.0*jobs[job].simulation.total_payoff / jobs[job].simulation.round))
def job_error(job):
logger.debug('Job %d terminated with an error.' % job)
while len(self.population) > self.BROOD_SIZE:
if self.single_thread:
for genome in self.population:
try:
genome.simulation.run(N_rounds = self.D_ROUNDS, return_self = True)
except:
e = sys.exc_info()
logger.debug('----------------------------------------------------------------------')
logger.debug(traceback.format_exc())
logger.debug("State graph:")
logger.debug(pprint.pformat(genome.state))
else:
for genome in self.population:
jobs[cloud.call(genome.simulation.run, N_rounds = self.D_ROUNDS, return_self = True,
_callback = [job_callback], _callback_on_error = [job_error], _fast_serialization = 0,
_type='c1')] = genome
done = False
while not done:
done = True
try:
cloud.join(jobs.keys())
except cloud.CloudException:
done = False
e = sys.exc_info()
logger.debug("More information on Job %d's unexpected termination:" % e[1].jid)
logger.debug("State graph:")
logger.debug(pprint.pformat(jobs[e[1].jid].state))
jobs.pop(e[1].jid)
self.population.sort(reverse=True, key=lambda genome: 1.0 * genome.simulation.total_payoff)
self.population = [genome for genome in self.population
if genome.simulation.total_payoff >= self.PERFORMANCE_THRESHOLD]
logger.debug([1.0 * genome.simulation.total_payoff / genome.simulation.round for genome in self.population])
new_N = int(round(len(self.population) * (1. - self.DECIMATION_PERCENT)))
if new_N < self.BROOD_SIZE:
new_N = self.BROOD_SIZE
# Let the fittest survive
self.population = self.population[0:new_N]
import cloud
def square(x):
return x * x
jid = cloud.call(square, 9)
print 'Squaring 9 (9*) has been pushed to the cloud with job id %s' % jid
res = cloud.result(jid)
print 'Result is %s' % res
def upload():
cloud.files.put("data/case6ww.pkl")
#upload()
def pf():
cloud.files.get("case6ww.pkl", "case6ww.pkl")
# Data files format if recognised according to file extension.
case = pylon.Case.load("case6ww.pkl")
# Pass the case to the solver and solve.
# sol = pylon.NewtonPF(case, iter_max=20).solve()
# sol = pylon.FastDecoupledPF(case, method=pylon.XB).solve()
sol = pylon.OPF(case, dc=False).solve()
return sol
job_id = cloud.call(pf, _high_cpu=False)
solution = cloud.result(job_id)
if solution["converged"]:
print "Completed in %.3fs." % solution["elapsed"]
else:
print "Failed!"
# recall the first segment -- let's work on that segment valid_segments[0] # <codecell> # look at how long it takes to run locally %time segment_stats(valid_segments[0], None) # <codecell> # here's how to run it on PiCloud # Prerequisite: http://docs.picloud.com/primer.html <--- READ THIS AND STUDY TO REFRESH YOUR MEMORY import cloud jid = cloud.call(segment_stats, '1346823845675', None, _env='/rdhyee/Working_with_Open_Data') # <codecell> # pull up status -- refresh until done cloud.status(jid) # <codecell> # this will block until job is done or errors out cloud.join(jid) # <codecell> # get your result
import cloud
def add(x, y):
return x + y
jid = cloud.call(add, 1, 2)
answer = cloud.result(jid)
print answer
config = yaml.load(open("conf/conf.yaml"))
experiment = "w%d-%d-p%d" % (config["NUMBER_OF_WORKERS"], config["RETURN_RATE"]
* 100, config["POPULATION_SIZE"])
experiment_id = experiment + "-%d" % round(time.time(), 0)
datafile = open(experiment_id + ".dat", "a")
conf_out = open("conf/" + experiment_id + ".yaml", "w")
yaml.dump(config, conf_out)
conf_out.close()
for i in range(1):
start = time.time()
init_job = cloud.call(ppeaks.initialize,
config=config,
_type=config["WORKER_TYPE"],
_env="deap")
tInitialize = time.time() - start
print i, tInitialize
params = [(w, config) for w in range(config["NUMBER_OF_WORKERS"])]
jids = cloud.map(ppeaks.work,
params,
_type=config["WORKER_TYPE"],
_depends_on=init_job)
results_list = cloud.result(jids)
tTotal = time.time() - start
totals = "%d,%0.2f,%0.2f" % (i, round(tTotal, 2), round(tInitialize, 2))
print totals
datafile.write(totals + '\n')
import cloud
import xmlrpclib
def matrix_multiply_test():
import numpy as np
from time import time
x = np.zeros((1000,1000))
t_start = time()
np.dot(x,x)
t_elapsed = time() - t_start
s = xmlrpclib.ServerProxy('http://localhost:8001')
s.add(2,3)
return t_elapsed
print "local computation time:", matrix_multiply_test()
from time import time
for i in xrange(1):
jid = cloud.call(matrix_multiply_test, _type='c2')
t_start_req = time()
print "cloud computation time:", cloud.result(jid)
print "total request time:", time() - t_start_req
Output is a new object in bucket with 'thumb_' prepended."""
thumbnail_filename = 'thumb_' + key_name
# download object to filesystem
cloud.bucket.get(key_name)
img = Image.open(key_name)
img.thumbnail((100,100), Image.ANTIALIAS)
# save the image to the filesystem
img.save(thumbnail_filename, 'JPEG')
# store the image file in your bucket
cloud.bucket.put(thumbnail_filename)
if __name__ == '__main__':
# put face.jpg into your bucket
cloud.bucket.put('face.jpg')
# run thumbnail() on the cloud
jid = cloud.call(thumbnail, 'face.jpg')
# wait for job to finish
cloud.join(jid)
# download image
cloud.bucket.get('thumb_face.jpg')
def test_exception2():
'''Raise TypeError since cloud.call called without arguments'''
jid = cloud.call()
varlist
), ' : ', var_to_vary, V, ' tau:', REStime, ' OptTau:', optTau,
avg_vals = []
if use_picloud == 'n' and var_to_vary == 'Volume':
avg_vals = c1dc.hydrobide(V, REStime, Rcons, Pcons, propQ, time,
dormancy, mean, std, lgp, maint)
elif use_picloud == 'y' and var_to_vary == 'Volume':
job_id = cloud.call(c1dc.hydrobide,
V,
REStime,
Rcons,
Pcons,
propQ,
time,
dormancy,
mean,
std,
lgp,
maint,
_type='m1')
avg_vals = cloud.result(job_id)
if len(avg_vals) == 1:
print avg_vals[0]
elif len(avg_vals) > 1:
REStimes3.append(REStime)
avgN, avgR, avgTO, avgQ, avgS, avgCT, avgEvar, avgH, RADinfo, avgab, avgMaxU, burnIn, lag, avgP = avg_vals
def test_exception4():
'''Raise TypeError since cloud.call called with 2 invalid arguments'''
jid = cloud.call("asdf","sadf")
for s in range(0, S):
coef_old[:, s] = Vf[s].getCoeffs()
Nmax = 100
def solveOnCloud(Vf, c_policy, xprime_policy):
diff = [0] * Nmax
for i in range(0, Nmax):
Vf, c_policy, xprime_policy = bellman.iterateBellmanOnCloud(
Vf, c_policy, xprime_policy, Para, nCloud=5)
for s_ in range(0, S):
diff[i] = max(diff[i],
np.max(np.abs(coef_old[:, s_] - Vf[s_].getCoeffs())))
coef_old[:, s_] = Vf[s_].getCoeffs()
return Vf, c_policy, xprime_policy, diff
jid = cloud.call(solveOnCloud,
Vf,
c_policy,
xprime_policy,
_env="gspy_env",
_type='m1')
Vf, c_policy, xprime_policy, diff = cloud.result(jid)
#Now fit accurate Policy functions
nx = max(min(Para.nx * 10, 1000), 1000)
xgrid = np.linspace(Para.xmin, Para.xmax, nx)
#c_policy,xprime_policy = bellman.fitNewPolicies(xgrid,Vf,c_policy,xprime_policy,Para)