Пример #1
0
def timing_triple_cloud():
    execfile('picloud_venture_credentials.py')
    exp_params = experiment.exp_param_defaults({})
    exp_params['intermediate_iter'] = 1
    exp_params['max_initial_run_time'] = 30
    exp_params['max_burn_time'] = 30
    exp_params['max_sample_time'] = 30
    exp_params['n_samples'] = 25
    print experiment.exp_params_to_str(exp_params)
    
    data = scipy.io.loadmat("../data/irm_synth/irm_synth_20.mat", squeeze_me=True)
    observed = list(zip(data['train_i'].flat, data['train_j'].flat, data['train_v'].flat))
    missing  = list(zip(data['test_i'].flat,  data['test_j'].flat,  data['test_v'].flat))
    data = {'observations' : observed, 'missing' : missing}
    
    model = models.product_IRM
    model_params = {'D' : 1, 'alpha' : 1, 'symmetric' : True}
    
    # Timing run
    print 'Timing'
    job_id = cloud.call(experiment.network_cv_timing_run, data, model, exp_params, model_params, _max_runtime=5, _env=cloud_environment)
    time_per_mh_iter = cloud.result(job_id)['time_per_mh_iter']
    
    # Live run
    print 'Live'
    exp_params['intermediate_iter'] = max(1, int(round(0.9 * exp_params['max_sample_time'] / (exp_params['n_samples'] * time_per_mh_iter))))
    job_id = cloud.call(experiment.network_cv_single_run, data, model, exp_params, model_params, _max_runtime=5, _env=cloud_environment)
    cloud.join(job_id)
    print cloud.result(job_id)
Пример #2
0
def main():
    cloud.setkey(2329, '270cb3cccb9beb65d2f424b24ccbd5a920c5ccef')
    try:
        fn = raw_input()
        f = open(fn)
        L = float(f.readline())
        line = f.readline()
        data = []
        while line:
            d = map(eval, line.split())
            data.append(d)
            line = f.readline()
        f.close()
        n = len(data)
        Gs = np.array(data[0:n / 2])
        ls = np.array(data[n / 2::])
        outstr = ''
        outstr += 'submitting cross validation to picloud\n'
        cloud.config.max_transmit_data = 12000000
        start = time.time()
        jid = cloud.call(traintst, Gs, ls, L)
        outstr += 'submission time: %s\n' % str(time.time() - start)
        start = time.time()
        result = cloud.result(jid)
        outstr += 'cloud execution time: %s\n' % str(time.time() - start)
        outstr += 'misclassification rate: %f\n' % np.mean(result)
        outstr += 'standard deviation:     %f\n' % np.std(result)
        outstr += '--------done---------------------'
        print outstr
    except:
        print 'an error occurred'
Пример #3
0
def patentComparer():
  print 'Started:'
  print datetime.now()
  #wordVectorDir = "C:\\Users\\Deepak\\Dropbox\\6.864 Project\\Word Vectors"
  wordVectorDir = "/Users/aratner/Dropbox/6.864 Project/Word Vectors"

  # Get patent claims
  patentData = getPatentData(200)
  print "Got patent data..."

  # Obtain the tree from the claim using the rule based parser / machine learnt model
  trees = [(x,hm_tree(y,3)) for (x,y) in patentData]
  embeddingTrees = [(x,generateEmbeddingTree(tree, wordVectorDir)) for (x,tree) in trees]

  #print embeddingTrees
  #for embeddingTree in embeddingTrees:
  #  embeddingTree[1].draw()

  print "Generated trees for claims..."

  # Partition trees into two sets - a training set and a testing set
  (trainingTrees, testingTrees) = partitionTrees(embeddingTrees)

  # Run the unfolding recursive auto-encoder on the tree, using picloud
  #params = train_params(trainingTrees, math.pow(10,-5), 0.01, False)
  jid = cloud.call(train_params, trainingTrees, math.pow(10,-5), 0.01, True, 60, _type='c2')
  print 'jid = ' + str(jid)
  params = cloud.result(jid)
  save_params(params, '_finalpicloud')
  print 'RAE training complete, parameters saved, at:'
  print datetime.now()

  print "Done..."
Пример #4
0
    def runcode(self, code):
        """Execute a code object.

        When an exception occurs, self.showtraceback() is called to
        display a traceback.  All exceptions are caught except
        SystemExit, which is reraised.

        A note about KeyboardInterrupt: this exception may occur
        elsewhere in this code, and may not always be caught.  The
        caller should be prepared to deal with it.

        """
        try:
            job = cloud.call(cloud_run, code, self.locals)
            cloud.join(job)
            result = cloud.result(job)
            self.locals.update(result) 
            info = cloud.info(job, ['stderr', 'stdout'])[job]
            sys.stdout.write(info['stdout'])
            sys.stderr.write(info['stderr'])
        except SystemExit:
            raise
        except KeyboardInterrupt:
            raise OperationAborted('Interrupted')
        except cloud.CloudException, e:
            self.showcloudtraceback(e)
Пример #5
0
def main():
    cloud.setkey(2329,'270cb3cccb9beb65d2f424b24ccbd5a920c5ccef')   
    try:
        fn = raw_input()
        f = open(fn)
        L = float(f.readline())           
        line = f.readline()        
        data = []
        while line:
            d = map(eval,line.split())
            data.append(d)
            line = f.readline()         
        f.close()
        n = len(data)            
        Gs = np.array(data[0:n/2])
        ls = np.array(data[n/2::])
        outstr = ''
        outstr += 'submitting cross validation to picloud\n'
        cloud.config.max_transmit_data=12000000    
        start = time.time()
        jid = cloud.call(traintst,Gs,ls,L)  
        outstr += 'submission time: %s\n' %str(time.time()-start)
        start = time.time()    
        result = cloud.result(jid) 
        outstr += 'cloud execution time: %s\n' %str(time.time()-start)      
        outstr += 'misclassification rate: %f\n' %np.mean(result)
        outstr += 'standard deviation:     %f\n' %np.std(result)         
        outstr += '--------done---------------------'  
        print outstr
    except:
        print 'an error occurred'     
Пример #6
0
def geoS(N, S):
    job_id = cloud.call(get_LowerTrunc_GeomSeries, N, S, _type='m1')
    geomSeries = cloud.result(job_id)   
    geomSeries = get_LowerTrunc_GeomSeries(N, S)
    geomSeries = np.log(geomSeries)
    plt.plot(geomSeries, color='m',lw=3,label='Geometric series\nN='+str(N)+' S='+str(S))
    print 'geometric series: done'
    return
Пример #7
0
def test_picloud(cloud, n, fast):
    begin = time.time()
    jid = cloud.call(get_prime, n, _high_cpu=True)
    cloud.result(jid)
    print n,
    print "took ",
    print time.time() - begin,
    print "seconds"
Пример #8
0
def test_picloud(cloud, n, fast):
    begin = time.time()
    jid = cloud.call(get_prime, n, _high_cpu=True)
    cloud.result(jid)
    print n , 
    print "took " ,
    print time.time() - begin , 
    print "seconds"
Пример #9
0
def logS(N, S):
    job_id = cloud.call(mete.get_mete_rad, S, N, _type='m1')
    logSeries = cloud.result(job_id)   
    #logSeries = mete.get_mete_rad(S, N) # The expected SAD from the random sample
    logSeries = np.log(logSeries[0])
    plt.plot(logSeries, color='gray', lw=3, label='Log-series\nN='+str(N)+' S='+str(S))
    print 'log-series: done'
    return
Пример #10
0
def process_json(json_data, use_cloud, quit_on_first=False, ignore_pickle=False, debug_info=None):
    ret = None
    if not use_cloud:
        ret = _process(json_data, use_cloud, quit_on_first=quit_on_first, ignore_pickle=ignore_pickle, debug_info=debug_info)
    else:
        ret = cloud.call(_process, json_data, quit_on_first=quit_on_first, use_cloud=True, ignore_pickle=ignore_pickle, _type="f2", _env='pygame_env')

    return ret
Пример #11
0
def generate_items_analytics_main_job(index, itemidlist):
    itemdetailjobids = []
    filenames = []
    if index.token:
        getitembestmatchlist = [itemidlist[i:i + index.configuration['item_analytics_calls_per_machine']] for i in range(0, len(itemidlist), index.configuration['item_analytics_calls_per_machine'])]
        for sublist in getitembestmatchlist:
            itemdetailjobids.append(cloud.call(etl.get_item_analytics, etldriver=index.etldriver, itemid=sublist, selleridkey=index.selleridkey,locale=index.locale, token=index.token, _type="c1", _label="GET ITEMS BESTMATCH"))
            filenames.append(make_file_name(itemid=sublist, locale=index.locale, datatype="bestmatch"))
    return itemdetailjobids, filenames
Пример #12
0
def clear(prefix, remote):
    if (remote):
        k = cloud.call(delete_prefix, prefix)
        return cloud.result(k)
    else:
        path = util.cachePath(prefix)
        if (os.path.isdir(path)):
            sh.rmtree(path)
        elif (os.path.isfile(path)):
            os.remove(path)
Пример #13
0
def report(batch, remote, debug, dependency = []):
    params = cache.get("batch/%s/params" % batch, remote)
    logging.info("running reporter instance")
    if (remote):
        k = cloud.call(reporter.report, batch, params, remote, debug, _label = "%s/report" % batch, _depends_on = dependency, _type = 'c1', _max_runtime = 30)
        logging.info("k %s" % k)
        return k
    else:
        result = reporter.report(batch, params, remote, debug)
        return result
Пример #14
0
def calc_pi():
    """Incorrect way"""

    # offload monte_carlo to the cloud, returns a Job Id
    jid = cloud.call(monte_carlo, total_tests, _type='c2')

    # block until job is done, and get result
    num_in_circle = cloud.result(jid)

    pi = (4 * num_in_circle) / float(total_tests)
    return pi
Пример #15
0
def calc_pi():
    """Incorrect way"""

    # offload monte_carlo to the cloud, returns a Job Id
    jid = cloud.call(monte_carlo, total_tests, _type='c2')

    # block until job is done, and get result
    num_in_circle = cloud.result(jid)

    pi = (4 * num_in_circle) / float(total_tests)
    return pi
Пример #16
0
def branch_out_in_cloud(board, i, j, possibilities, depth):
    jids = []
    for digit in possibilities:
        new_board = copy.deepcopy(board)
        new_board.fill(i, j, digit)
        jid = cloud.call(solve, new_board, depth)
        jids.append(jid)
    for result in cloud.iresult(jids):
        if result:
            return result
    return None
Пример #17
0
def build_index(**kwargs):
    sellerdata = kwargs.get('sellerdata')
    configuration = kwargs.get('configuration')
    etlconfiguration = kwargs.get('etlconfiguration', False)
    itemids = kwargs.get('itemids', [])
    sellerdata['etl'] = json.loads(sellerdata['etl'])
    
    if not configuration:
        from etl.config.drivers.picloud.ebay import picloud_ebay_configuration
        configuration = picloud_ebay_configuration

    index = BuildLogic(sellerdata, configuration, PiCloudEBayETLDriver(), etlconfiguration)
    itemdetailjobid = 0
    
    categoryjobid = cloud.call(generate_categories, index=index, _type="c1", _max_runtime=20, _label="GET GATEGORIES")
    findingjobids = cloud.call(generate_finding_calls, index=index, categoryjobid=categoryjobid, _type="c1", _depends_on=categoryjobid, _label="GENERATE FINDING CALLS")
    #findingjobids=243103
    #get_items_detail_and_analytics_main_job(index=index,findingjobids=findingjobids)
    itemdetailjobid = cloud.call(generate_items_detail_and_analytics_main_job, index=index, findingjobids=findingjobids, _type="c1", _depends_on=findingjobids, _label="GENERATE ITEM DETAIL CALLS")
    
    return itemdetailjobid
    def create_alternative_worlds(self):
        ''' 
        Simulate each of the economies. 
        '''
        T = self.T
        I = self.I

        print 'Simulating a world with %d countries.' % I

        # these are the individual simulation jobs
        jobs = cloud.map(simulate_an_economy,self.economies,[T]*I,_label='A world with %d countries.' % I,_type='c2')
        # this is a harvester job, collecting the results of each individual simulation
        return cloud.call(GDP_collector, jobs, T, T1, T2,_depends_on=jobs,_label='Collecting GDP results.')
Пример #19
0
    def __init__(self, usecloud=False, cloud_env=''):
        if usecloud and not have_cloud:
            print 'warning: cloud module is not installed, running locally'
            usecloud = False
        self.usecloud = usecloud
        self.cloud_env = cloud_env

        if self.usecloud:
            print 'uploading dataset to cloud...'
            cloud.files.put(DIGITS_FN)
            self.preprocess_job = cloud.call(self.preprocess, _env=self.cloud_env)
        else:
            self._samples, self._labels = self.preprocess()
Пример #20
0
    def __init__(self, usecloud=False, cloud_env=''):
        if usecloud and not have_cloud:
            print 'warning: cloud module is not installed, running locally'
            usecloud = False
        self.usecloud = usecloud
        self.cloud_env = cloud_env

        if self.usecloud:
            print 'uploading dataset to cloud...'
            cloud.files.put(DIGITS_FN)
            self.preprocess_job = cloud.call(self.preprocess,
                                             _env=self.cloud_env)
        else:
            self._samples, self._labels = self.preprocess()
Пример #21
0
def f(gens):
        k = 7
        n= k + 1
        effectiveAttrIndices = range(k)
        probMutation = 0.004
        probMisclassification = 0.20
        popSize = 1500
        jid = cloud.call(neap_uga, **dict(m=popSize,
                       n=n,
                       gens=gens,
                       probMutation=probMutation,
                       effectiveAttrIndices=effectiveAttrIndices,
                       probMisclassification=probMisclassification))
        print "Kicked off trial %s" % jid
        return jid
Пример #22
0
def do_network_call(itemidlistlist=[], callback=None):
    setcloudkey()
    etldriver = PiCloudEBayETLDriver()
    filenamelist = []
    try:
        jobids = []
        for sublist in itemidlistlist:
            jobid = cloud.call(etl.get_item_details, etldriver=etldriver, itemid=sublist, _type="c1", _label="GET NOTIFICATION ITEM DETAILS")
            jobids.append(jobid)
            filename = make_file_name(itemid=sublist, datatype="itemdetail")
            filenamelist.append(filename)
        print jobids
        if len(jobids) > 0:
            cloud.result(jobids, ignore_errors=False)
    except Exception, e:
        print e
Пример #23
0
def timing_run_cloud():
    execfile('picloud_venture_credentials.py')
    exp_params = experiment.exp_param_defaults({})
    exp_params['intermediate_iter'] = 1
    exp_params['max_initial_run_time'] = 30
    print experiment.exp_params_to_str(exp_params)
    
    data = scipy.io.loadmat("../data/irm_synth/irm_synth_20.mat", squeeze_me=True)
    observed = list(zip(data['train_i'].flat, data['train_j'].flat, data['train_v'].flat))
    missing  = list(zip(data['test_i'].flat,  data['test_j'].flat,  data['test_v'].flat))
    data = {'observations' : observed, 'missing' : missing}
    
    model = models.product_IRM
    model_params = {'D' : 1, 'alpha' : 1, 'symmetric' : True}
    job_id = cloud.call(experiment.network_cv_timing_run, data, model, exp_params, model_params, _max_runtime=5, _env=cloud_environment)
    cloud.join(job_id)
    print cloud.result(job_id)
Пример #24
0
def f(gens):
    k = 7
    n = k + 1
    effectiveAttrIndices = range(k)
    probMutation = 0.004
    probMisclassification = 0.20
    popSize = 1500
    jid = cloud.call(
        neap_uga,
        **dict(m=popSize,
               n=n,
               gens=gens,
               probMutation=probMutation,
               effectiveAttrIndices=effectiveAttrIndices,
               probMisclassification=probMisclassification))
    print "Kicked off trial %s" % jid
    return jid
Пример #25
0
def trainIntermediateRAE():
    print 'Started:'
    print datetime.now()
    #wordVectorDir = "C:\\Users\\Deepak\\Dropbox\\6.864 Project\\Word Vectors"
    wordVectorDir = "/Users/aratner/Dropbox/6.864 Project/Word Vectors"

    # Get patent claims
    patentData = getPatentData(500)
    print "Got patent data..."

    # Obtain the tree from the claim using the rule based parser / machine learnt model
    trees = [(x, hm_tree(y, 3)) for (x, y) in patentData]
    totalTrainingdata = []
    for treeData in trees:
        tree = treeData[1]
        #tree.draw()
        label = treeData[0]
        totalTrainingdata.extend(generatePhraseTree(tree, wordVectorDir))

    print "Generated trees for claims..."

    # Partition trees into two sets - a training set and a testing set
    #(trainingTrees, testingTrees) = partitionTrees(totalTrainingdata)
    trainingTrees = totalTrainingdata
    testingTrees = totalTrainingdata

    #for tree in trainingTrees:
    # 	if (len(tree) > 1):
    #		tree.draw()

    # Run the unfolding recursive auto-encoder on the tree
    jid = cloud.call(train_params,
                     trainingTrees,
                     math.pow(10, -5),
                     0.01,
                     True,
                     60,
                     _type='c2')
    print 'jid =' + str(jid)
    params = cloud.result(jid)
    save_params(params, '_finalpicloud_subtrees')
    print 'RAE training complete, at:'
    print datetime.now()

    print "Done..."
Пример #26
0
def build_bestmatchonly_index(**kwargs):
    sellerdata = kwargs.get('sellerdata')
    configuration = kwargs.get('configuration')
    etlconfiguration = kwargs.get('etlconfiguration', False)
    itemids = kwargs.get('itemids', [])
    sellerdata['etl'] = json.loads(sellerdata['etl'])
    
    if not configuration:
        from etl.config.drivers.picloud.ebay import picloud_ebay_configuration
        configuration = picloud_ebay_configuration

    index = BuildLogic(sellerdata, configuration, PiCloudEBayETLDriver(), etlconfiguration)

    itemidlist = get_sellers_items(index.selleridkey)
    
    itemdetailjobid = cloud.call(generate_items_analytics_main_job, index=index, itemidlist=itemidlist, _type="c1", _label="GENERATE ITEM BESTMATCH ONLY CALLS")
    
    return itemdetailjobid
Пример #27
0
def calc_pi():
    """Almost correct way"""

    num_jobs = 8
    tests_per_call = total_tests/num_jobs

    # list of job ids for all jobs we're spawning
    jids = []
    for _ in range(num_jobs):
        # call() does not block, so jobs run in parallel
        jid = cloud.call(monte_carlo, tests_per_call, _type='c2')
        jids.append(jid)

    # aggregate the number of darts that land in the circle
    # across all jobs that we spawned
    num_in_circle = 0
    for jid in jids:
        num_in_circle += cloud.result(jid)

    pi = (4 * num_in_circle) / float(total_tests)
    return pi
Пример #28
0
def trainIntermediateRAE():
  print 'Started:'
  print datetime.now()
  #wordVectorDir = "C:\\Users\\Deepak\\Dropbox\\6.864 Project\\Word Vectors"
  wordVectorDir = "/Users/aratner/Dropbox/6.864 Project/Word Vectors"

  # Get patent claims
  patentData = getPatentData(500)
  print "Got patent data..."

  # Obtain the tree from the claim using the rule based parser / machine learnt model
  trees = [(x,hm_tree(y,3)) for (x,y) in patentData]
  totalTrainingdata = []
  for treeData in trees:
  	tree = treeData[1]
  	#tree.draw()
  	label = treeData[0]
  	totalTrainingdata.extend(generatePhraseTree(tree, wordVectorDir))

  print "Generated trees for claims..."

  # Partition trees into two sets - a training set and a testing set
  #(trainingTrees, testingTrees) = partitionTrees(totalTrainingdata)
  trainingTrees = totalTrainingdata
  testingTrees = totalTrainingdata

  #for tree in trainingTrees:
  # 	if (len(tree) > 1):
  #		tree.draw()

  # Run the unfolding recursive auto-encoder on the tree
  jid = cloud.call(train_params, trainingTrees, math.pow(10,-5), 0.01, True, 60, _type='c2')
  print 'jid ='+str(jid)
  params = cloud.result(jid)
  save_params(params, '_finalpicloud_subtrees')
  print 'RAE training complete, at:'
  print datetime.now()


  print "Done..."
Пример #29
0
def patentComparer():
    print 'Started:'
    print datetime.now()
    #wordVectorDir = "C:\\Users\\Deepak\\Dropbox\\6.864 Project\\Word Vectors"
    wordVectorDir = "/Users/aratner/Dropbox/6.864 Project/Word Vectors"

    # Get patent claims
    patentData = getPatentData(200)
    print "Got patent data..."

    # Obtain the tree from the claim using the rule based parser / machine learnt model
    trees = [(x, hm_tree(y, 3)) for (x, y) in patentData]
    embeddingTrees = [(x, generateEmbeddingTree(tree, wordVectorDir))
                      for (x, tree) in trees]

    #print embeddingTrees
    #for embeddingTree in embeddingTrees:
    #  embeddingTree[1].draw()

    print "Generated trees for claims..."

    # Partition trees into two sets - a training set and a testing set
    (trainingTrees, testingTrees) = partitionTrees(embeddingTrees)

    # Run the unfolding recursive auto-encoder on the tree, using picloud
    #params = train_params(trainingTrees, math.pow(10,-5), 0.01, False)
    jid = cloud.call(train_params,
                     trainingTrees,
                     math.pow(10, -5),
                     0.01,
                     True,
                     60,
                     _type='c2')
    print 'jid = ' + str(jid)
    params = cloud.result(jid)
    save_params(params, '_finalpicloud')
    print 'RAE training complete, parameters saved, at:'
    print datetime.now()

    print "Done..."
    def create_alternative_worlds(self):
        ''' 
        Simulate each of the economies. 
        '''
        T = self.T
        I = self.I

        print 'Simulating a world with %d countries.' % I

        # these are the individual simulation jobs
        jobs = cloud.map(simulate_an_economy,
                         self.economies, [T] * I,
                         _label='A world with %d countries.' % I,
                         _type='c2')
        # this is a harvester job, collecting the results of each individual simulation
        return cloud.call(GDP_collector,
                          jobs,
                          T,
                          T1,
                          T2,
                          _depends_on=jobs,
                          _label='Collecting GDP results.')
Пример #31
0
 def start_indexing_on_cloud(self, etlconfiguration):
     try:
         successfullsellerfilenamelist=[]
         jobids = []
         self.filenamelist = []
         self.successsellers = []
         mainjobids = []
         for seller in self.sellers:
             try:
                 update_sellers_status([seller], dict(phase="INDEXING", progress="STARTED"))
                 if etlconfiguration == "BESTMATCHONLY":
                     tempjobids = build_bestmatchonly_index(sellerdata=seller, configuration=self.configuration, etlconfiguration=etlconfiguration)
                     seller['itemdetailjobid'] = tempjobids
                     mainjobids.append(tempjobids)
                 else:
                     jobids.append(cloud.call(build_index, sellerdata=seller, configuration=self.configuration, etlconfiguration=etlconfiguration, _label="SELLER"))
                     seller['jobid'] = jobids[len(jobids) - 1]
                 self.successsellers.append(seller)
             except Exception, e:
                 logger.exception(str(e))
                 update_sellers_status([seller], dict(phase="INDEXING", progress="ERROR"))
             finally:
                 pass
Пример #32
0
    def fetch_results(self, iters=None, via_remote=False, run_mode='local'):
        """
        Returns the result of the job that has already been run as a :py:class:`History` object. Typically you would call :py:meth:`run` first, then call :py:meth:`fetch_results` to get the resutlts. The method has various methods to control how much of the job is returned, to avoid excessive memory usage and data transfer between the cloud and local machine.

        :param iters: If *iters* is an iterable, returns only the iterations of the chain in *iters*. If *iters* is a scalar, return every *iters* state (the stride). If None, returns all states.
        :param via_remote: If *True*, executes the state filtering on the cloud before transferring the data to the local machine. If false, filter the state on the local machine.
        :param run_mode: Controls whether to search for the results on the local macine or on the cloud. Can be *local* or *cloud*.
        :return: A :py:class:`History` object that contains a filtered version the states of the Markov chain visited when this job ran.
        """
        def f():
            if run_mode == 'cloud':
                cloud.join([self.job_id])
                store = storage.CloudStore()
            else:
                store = storage.LocalStore()
            full_history = store[self.params]
            partial_history = History()
            if iters is None:
                partial_history.states = full_history.states
            else:
                if isinstance(iters, int):  #iters interpreted as stride
                    iter_set = range(0, len(full_history.states), iters)
                else:
                    iter_set = iters
                partial_history.states = [
                    state for state in full_history.states
                    if state.iter in iter_set
                ]
            partial_history.job = self
            partial_history.summary = full_history.summary
            return partial_history

        if via_remote:
            job_id = cloud.call(f, _env=picloud_env)
            return cloud.result(job_id)
        else:
            return f()
Пример #33
0
def run_test():
    HowManyTimesIsTricky = 0
    for sample_number in range(NumberOfSamples):
        sample = sample_once(0)
        if sample is True:
            HowManyTimesIsTricky = HowManyTimesIsTricky + 1
    print "** Time: " + str()
    return "Empirical prob.: " + str(
        float(HowManyTimesIsTricky) /
        NumberOfSamples) + " versus the theoretical =appr.= 0.850498339"


start_time = time.time()
print "Without mapping:"
jid = cloud.call(run_test, _env=cloud_environment, _type='c1')
result = cloud.result(jid)
print "** Time: " + str(time.time() - start_time)
print "** " + result

print ""
print ""
print ""

print "With mapping:"
start_time = time.time()
jids = cloud.map(sample_once,
                 range(NumberOfSamples),
                 _env=cloud_environment,
                 _type='c1')
result = cloud.result(jids)
Пример #34
0
import cloud
import xmlrpclib


def matrix_multiply_test():
    import numpy as np
    from time import time
    x = np.zeros((1000, 1000))
    t_start = time()
    np.dot(x, x)
    t_elapsed = time() - t_start

    s = xmlrpclib.ServerProxy('http://localhost:8001')
    s.add(2, 3)

    return t_elapsed


print "local computation time:", matrix_multiply_test()

from time import time

for i in xrange(1):
    jid = cloud.call(matrix_multiply_test, _type='c2')
    t_start_req = time()
    print "cloud computation time:", cloud.result(jid)
    print "total request time:", time() - t_start_req
Пример #35
0
def main():
	jid = cloud.call(search_yellow(), type='m1')
	cloud.result(jid)
Пример #36
0
def test_exception3():
    '''Raise TypeError since cloud.call called with 1 invalid argument'''
    jid = cloud.call("asdf")
Пример #37
0
def test_multiply():
    jid = cloud.call(lambda: 3*3)
    answer = cloud.result(jid)
    assert answer == 9
Пример #38
0
        randomword3 = randomword(5)
        if engdict.check(randomword3) == True:
            randomkey3 = randomword3 + str(random.randint(0, 99))
        elif engdict.check(randomword3) == False:
            englist = engdict.suggest(randomword3)
            if len(englist) > 0:
                randomkey3 = englist[0] + str(random.randint(0, 99))
            else:
                randomkey3 = randomword3 + str(random.randint(0, 99))

        if 'randomkey0' and 'randomkey3' and 'randomkey1' in locals():
            whasher0 = hashlib.new("md5")
            whasher0.update(randomkey0)
            whasher3 = hashlib.new("md5")
            whasher3.update(randomkey3)
            whasher1 = hashlib.new("md5")
            whasher1.update(randomkey1)
            print(randomkey0 + " + " + str(whasher0.hexdigest()) + "\n")
            print(randomkey3 + " + " + str(whasher3.hexdigest()) + "\n")
            print(randomkey1 + " + " + str(whasher1.hexdigest()) + "\n")
            fileb.write(randomkey0 + " + " + str(whasher0.hexdigest()) + "\n")
            fileb.write(randomkey3 + " + " + str(whasher3.hexdigest()) + "\n")
            fileb.write(randomkey1 + " + " + str(whasher1.hexdigest()) + "\n")


jid = cloud.call(randomword)  # square(3) evaluated on PiCloud
cloud.result(jid)
print('Value added to cloud')
print('Password added')
mainroutine()
Пример #39
0
    def run(self, run_mode='local', use_cache=False):
        """
        Runs the job, storing all the states of the Markov chain in a datastore.

        :param run_mode: A string that controls how and where the job is run. Currently has two allowable values:

         local
          Runs the job locally and stores the data locally. Useful for debugging.

         cloud
           Runs the job on picloud and stores the data in a picloud bucket.


        :param use_cache: If *True* and this job has already been run at a previous time, return the results of that job. If *False*, rerun the job.
        :return: If run_mode is 'cloud', returns the picloud job id in a non-blocking way. If run_mode is 'local', does not return anything and will not return until the job is completed.
        :raise:
        """
        chain = self.chain
        data = self.data
        if run_mode == 'cloud':
            store = storage.CloudStore()
        else:
            store = storage.LocalStore()
            # Calculating the key outside of the cloud is necessary since the hashing functions on the cloud may not
        # agree with the local hashing functions (might be a 32-bit vs 64-bit python issue).
        self.key = store.hash_key(self.params)

        def f():
            if run_mode == "cloud":
                store = storage.CloudStore()
            elif run_mode == "local":
                store = storage.LocalStore()
            else:
                raise BaseException("Run mode %r not recognized" % run_mode)
            store.auto_hash = False
            if use_cache and (self.key in store):
                logger.debug("Cache hit")
                return
            logger.debug("Cache miss")
            logger.debug("Running job")
            data._load_data()
            chain.data = data.train_data
            chain.data_source = data
            ioff()
            states = chain._run()
            ion()
            logger.debug('Chain completed')
            history = History()
            history.states = states
            history.job = self
            logger.debug("Summarizing chain")
            history.summary = chain.summarize(history)
            logger.debug("Chain summarized")
            logger.debug("Job params: %r" % (self.params, ))
            store[self.key] = history
            store.close()

        if run_mode == 'local':
            return f()
        elif run_mode == 'cloud':
            job_id = cloud.call(f, _env=picloud_env)
            self.job_id = job_id
            return job_id
Пример #40
0
def run_experiment(experiment,
                   roleouts,
                   episodes,
                   in_cloud=False,
                   dynProfile=None):
    """ Runs the given experiment and returns the results.
    """
    def run():
        if dynProfile is None:
            maxsteps = len(experiment.profile)  # episode length
        else:
            maxsteps = dynProfile.shape[1]
        na = len(experiment.agents)
        ni = roleouts * episodes * maxsteps

        all_action = zeros((na, 0))
        all_reward = zeros((na, 0))
        epsilon = zeros((na, ni))  # exploration rate

        # Converts to action vector in percentage markup values.
        vmarkup = vectorize(get_markup)

        for roleout in range(roleouts):
            if dynProfile is not None:
                # Apply new load profile before each roleout (week).
                i = roleout * episodes  # index of first profile value
                experiment.profile = dynProfile[i:i + episodes, :]


#            print "PROFILE:", experiment.profile, episodes

            experiment.doEpisodes(
                episodes)  # number of samples per learning step

            nei = episodes * maxsteps  # num interactions per role
            epi_action = zeros((0, nei))
            epi_reward = zeros((0, nei))

            for i, (task, agent) in \
            enumerate(zip(experiment.tasks, experiment.agents)):
                action = copy(agent.history["action"])
                reward = copy(agent.history["reward"])

                for j in range(nei):
                    if isinstance(agent.learner, DirectSearchLearner):
                        action[j, :] = task.denormalize(action[j, :])
                        k = nei * roleout
                        epsilon[i, k:k + nei] = agent.learner.explorer.sigma[0]
                    elif isinstance(agent.learner, ValueBasedLearner):
                        action[j, :] = vmarkup(action[j, :], task)
                        k = nei * roleout
                        epsilon[i, k:k + nei] = agent.learner.explorer.epsilon
                    else:
                        action = vmarkup(action, task)

                # FIXME: Only stores action[0] for all interactions.
                epi_action = c_[epi_action.T, action[:, 0].flatten()].T
                epi_reward = c_[epi_reward.T, reward.flatten()].T

                if hasattr(agent, "module"):
                    print "PARAMS:", agent.module.params

                agent.learn()
                agent.reset()

            all_action = c_[all_action, epi_action]
            all_reward = c_[all_reward, epi_reward]

        return all_action, all_reward, epsilon

    if in_cloud:
        import cloud
        job_id = cloud.call(run, _high_cpu=False)
        result = cloud.result(job_id)
        all_action, all_reward, epsilon = result
    else:
        all_action, all_reward, epsilon = run()

    return all_action, all_reward, epsilon
Пример #41
0
    def step_fitness(self):
        """
        Run fitness tests for the current generation, and evolve the next generation.
        """

        # Firstly, render code for all the genomes in the current population. Each genome owns its own
        # simulation object, because we want to interleave the simulations, running D_ROUNDS of simulation
        # rounds for all genomes, and killing off the weakest until BROOD_SIZE genomes remain.

        if self.next_population:
            self.population = copy.deepcopy(self.next_population)
            self.next_population = None

        for genome in self.population:
            code = genome.render(debug = self.DEBUG)
            genome.code_hash = md5.md5(code).hexdigest()
            genome.agent_name = 'agent_' + genome.code_hash
            genome.agent_path = 'agents/rendered/' + genome.agent_name + '.py'
            f = open(genome.agent_path, 'w')
            f.write(code)
            f.close()
            genome.agent_module = __import__('agents.rendered.'+genome.agent_name, fromlist=['*'])
            genome.simulation = simulate.Simulate(**self.sim_parameters)
            genome.simulation.agent_move = genome.agent_module.move
            genome.simulation.agent_observe_who = genome.agent_module.observe_who

        jobs = {}

        def job_callback(job):
            jobs[job].simulation = cloud.result(job)
            logger.debug('Job %d completed with fitness %.2f.' % (job, 1.0*jobs[job].simulation.total_payoff / jobs[job].simulation.round))
        
        def job_error(job):
            logger.debug('Job %d terminated with an error.' % job)
       
        while len(self.population) > self.BROOD_SIZE:

            if self.single_thread:
                for genome in self.population:
                    try:
                        genome.simulation.run(N_rounds = self.D_ROUNDS, return_self = True)
                    except:
                        e = sys.exc_info()
                        logger.debug('----------------------------------------------------------------------')
                        logger.debug(traceback.format_exc())
                        logger.debug("State graph:")
                        logger.debug(pprint.pformat(genome.state))                
            else:
                for genome in self.population:

                    jobs[cloud.call(genome.simulation.run, N_rounds = self.D_ROUNDS, return_self = True, 
                        _callback = [job_callback], _callback_on_error = [job_error], _fast_serialization = 0,
                        _type='c1')] = genome
                
                done = False
                while not done:
                    done = True
                    try:
                        cloud.join(jobs.keys())
                    except cloud.CloudException:
                        done = False
                        e = sys.exc_info()
                        logger.debug("More information on Job %d's unexpected termination:" % e[1].jid)
                        logger.debug("State graph:")
                        logger.debug(pprint.pformat(jobs[e[1].jid].state))
                        jobs.pop(e[1].jid)
            
            self.population.sort(reverse=True, key=lambda genome: 1.0 * genome.simulation.total_payoff)

            self.population = [genome for genome in self.population 
                               if genome.simulation.total_payoff >= self.PERFORMANCE_THRESHOLD]

            logger.debug([1.0 * genome.simulation.total_payoff / genome.simulation.round for genome in self.population])

            new_N = int(round(len(self.population) * (1. - self.DECIMATION_PERCENT)))
            if new_N < self.BROOD_SIZE:
                new_N = self.BROOD_SIZE
            
            # Let the fittest survive
            self.population = self.population[0:new_N]
Пример #42
0
import cloud


def square(x):
    return x * x


jid = cloud.call(square, 9)
print 'Squaring 9 (9*) has been pushed to the cloud with job id %s' % jid

res = cloud.result(jid)
print 'Result is %s' % res
Пример #43
0
def upload():
    cloud.files.put("data/case6ww.pkl")


#upload()


def pf():
    cloud.files.get("case6ww.pkl", "case6ww.pkl")

    # Data files format if recognised according to file extension.
    case = pylon.Case.load("case6ww.pkl")

    # Pass the case to the solver and solve.
    #    sol = pylon.NewtonPF(case, iter_max=20).solve()
    #    sol = pylon.FastDecoupledPF(case, method=pylon.XB).solve()
    sol = pylon.OPF(case, dc=False).solve()

    return sol


job_id = cloud.call(pf, _high_cpu=False)

solution = cloud.result(job_id)

if solution["converged"]:
    print "Completed in %.3fs." % solution["elapsed"]
else:
    print "Failed!"
# recall the first segment -- let's work on that segment
valid_segments[0]

# <codecell>

# look at how long it takes to run locally

%time segment_stats(valid_segments[0], None)

# <codecell>

# here's how to run it on PiCloud
# Prerequisite:  http://docs.picloud.com/primer.html <--- READ THIS AND STUDY TO REFRESH YOUR MEMORY

import cloud
jid = cloud.call(segment_stats, '1346823845675', None, _env='/rdhyee/Working_with_Open_Data')

# <codecell>

# pull up status -- refresh until done
cloud.status(jid)

# <codecell>

# this will block until job is done or errors out

cloud.join(jid)

# <codecell>

# get your result
Пример #45
0
import cloud

def add(x, y):
    return x + y
jid = cloud.call(add, 1, 2) 
answer = cloud.result(jid)
print answer
Пример #46
0
config = yaml.load(open("conf/conf.yaml"))

experiment = "w%d-%d-p%d" % (config["NUMBER_OF_WORKERS"], config["RETURN_RATE"]
                             * 100, config["POPULATION_SIZE"])
experiment_id = experiment + "-%d" % round(time.time(), 0)

datafile = open(experiment_id + ".dat", "a")
conf_out = open("conf/" + experiment_id + ".yaml", "w")
yaml.dump(config, conf_out)
conf_out.close()

for i in range(1):
    start = time.time()
    init_job = cloud.call(ppeaks.initialize,
                          config=config,
                          _type=config["WORKER_TYPE"],
                          _env="deap")
    tInitialize = time.time() - start
    print i, tInitialize
    params = [(w, config) for w in range(config["NUMBER_OF_WORKERS"])]

    jids = cloud.map(ppeaks.work,
                     params,
                     _type=config["WORKER_TYPE"],
                     _depends_on=init_job)
    results_list = cloud.result(jids)

    tTotal = time.time() - start
    totals = "%d,%0.2f,%0.2f" % (i, round(tTotal, 2), round(tInitialize, 2))
    print totals
    datafile.write(totals + '\n')
Пример #47
0
import cloud
import xmlrpclib


def matrix_multiply_test():
    import numpy as np
    from time import time
    x = np.zeros((1000,1000))
    t_start = time()
    np.dot(x,x)
    t_elapsed = time() - t_start

    s = xmlrpclib.ServerProxy('http://localhost:8001')
    s.add(2,3)

    return t_elapsed
    

print "local computation time:", matrix_multiply_test()

from time import time    


for i in xrange(1):
    jid = cloud.call(matrix_multiply_test, _type='c2')
    t_start_req = time()
    print "cloud computation time:", cloud.result(jid)    
    print "total request time:", time() - t_start_req
Пример #48
0
    Output is a new object in bucket with 'thumb_' prepended."""

    thumbnail_filename = 'thumb_' + key_name

    # download object to filesystem
    cloud.bucket.get(key_name)

    img = Image.open(key_name)
    img.thumbnail((100,100), Image.ANTIALIAS)

    # save the image to the filesystem
    img.save(thumbnail_filename, 'JPEG')

    # store the image file in your bucket
    cloud.bucket.put(thumbnail_filename)


if __name__ == '__main__':

    # put face.jpg into your bucket
    cloud.bucket.put('face.jpg')

    # run thumbnail() on the cloud
    jid = cloud.call(thumbnail, 'face.jpg')

    # wait for job to finish
    cloud.join(jid)

    # download image
    cloud.bucket.get('thumb_face.jpg')
Пример #49
0
def test_exception2():
    '''Raise TypeError since cloud.call called without arguments'''
    jid = cloud.call()
Пример #50
0
            varlist
        ), ' : ', var_to_vary, V, ' tau:', REStime, ' OptTau:', optTau,
        avg_vals = []

        if use_picloud == 'n' and var_to_vary == 'Volume':
            avg_vals = c1dc.hydrobide(V, REStime, Rcons, Pcons, propQ, time,
                                      dormancy, mean, std, lgp, maint)

        elif use_picloud == 'y' and var_to_vary == 'Volume':
            job_id = cloud.call(c1dc.hydrobide,
                                V,
                                REStime,
                                Rcons,
                                Pcons,
                                propQ,
                                time,
                                dormancy,
                                mean,
                                std,
                                lgp,
                                maint,
                                _type='m1')
            avg_vals = cloud.result(job_id)

        if len(avg_vals) == 1:
            print avg_vals[0]

        elif len(avg_vals) > 1:
            REStimes3.append(REStime)

            avgN, avgR, avgTO, avgQ, avgS, avgCT, avgEvar, avgH, RADinfo, avgab, avgMaxU, burnIn, lag, avgP = avg_vals
Пример #51
0
def test_exception4():
    '''Raise TypeError since cloud.call called with 2 invalid arguments'''
    jid = cloud.call("asdf","sadf")
Пример #52
0
for s in range(0, S):
    coef_old[:, s] = Vf[s].getCoeffs()

Nmax = 100


def solveOnCloud(Vf, c_policy, xprime_policy):
    diff = [0] * Nmax
    for i in range(0, Nmax):
        Vf, c_policy, xprime_policy = bellman.iterateBellmanOnCloud(
            Vf, c_policy, xprime_policy, Para, nCloud=5)
        for s_ in range(0, S):
            diff[i] = max(diff[i],
                          np.max(np.abs(coef_old[:, s_] - Vf[s_].getCoeffs())))
            coef_old[:, s_] = Vf[s_].getCoeffs()
    return Vf, c_policy, xprime_policy, diff


jid = cloud.call(solveOnCloud,
                 Vf,
                 c_policy,
                 xprime_policy,
                 _env="gspy_env",
                 _type='m1')
Vf, c_policy, xprime_policy, diff = cloud.result(jid)

#Now fit accurate Policy functions
nx = max(min(Para.nx * 10, 1000), 1000)
xgrid = np.linspace(Para.xmin, Para.xmax, nx)
#c_policy,xprime_policy = bellman.fitNewPolicies(xgrid,Vf,c_policy,xprime_policy,Para)