def predict_from_image_batch(self, mnist_batch, index):
t0 = time.time()
connection_pool = ConnectionPool(size=self.CONNECTION_POOL_SIZE,
host=HBaseManager.HOST,
port=HBaseManager.PORT)
hbase_manager = HBaseManager(connection_pool)
process_pool = Pool(self.POOL_SIZE)
n = len(mnist_batch)
indexs = list(range(n))
extract_process = process_pool.starmap_async(self.extract_keys,
zip(mnist_batch, indexs))
extracted_keys = extract_process.get()
predict_hash_args = zip(extracted_keys, indexs)
predictions = [
self.predict_hash_values(keys, hbase_manager, i)
for keys, i in predict_hash_args
]
process_pool.close()
t1 = time.time()
print("Mnist Batch {} predicted in: {} Seconds, For Node: {}".format(
str(index), str(t1 - t0), self.__str__()))
return predictions
def train_batch(self, mnist_batch, index):
'''
:type mnist_batch: list of tuple
:type deviate: boolean
:rtype: None
'''
t0 = time.time()
connection_pool = ConnectionPool(size=self.CONNECTION_POOL_SIZE, host=HBaseManager.HOST, port=HBaseManager.PORT)
hbase_manager = HBaseManager(connection_pool)
process_pool = Pool(self.POOL_SIZE)
thread_pool = ThreadPool(self.POOL_SIZE)
n = len(mnist_batch)
numbers, mnist_images = MnistHelper.extract_numbers_images(mnist_batch)
mnist_images = [mnist_obs[MnistModel.PREDICTOR_INDEX] for mnist_obs in mnist_batch]
indexs = list(range(n))
extract_process = process_pool.starmap_async(self.extract_keys, zip(mnist_images, indexs))
extracted_keys = extract_process.get()
store_hash_args = zip(extracted_keys, numbers, indexs)
[self.store_hash_values(k, n, hbase_manager, i) for k, n, i in store_hash_args]
process_pool.close()
thread_pool.close()
t1 = time.time()
print("Time taken to train batch {} : {} Seconds".format(str(index),str(t1 - t0)))
def test_connection_pool():
from thriftpy2.thrift import TException
def run():
name = threading.current_thread().name
print("Thread %s starting" % name)
def inner_function():
# Nested connection requests must return the same connection
with pool.connection() as another_connection:
assert connection is another_connection
# Fake an exception once in a while
if random.random() < .25:
print("Introducing random failure")
connection.transport.close()
raise TException("Fake transport exception")
for i in range(50):
with pool.connection() as connection:
connection.tables()
try:
inner_function()
except TException:
# This error should have been picked up by the
# connection pool, and the connection should have
# been replaced by a fresh one
pass
connection.tables()
print("Thread %s done" % name)
N_THREADS = 10
pool = ConnectionPool(size=3, **connection_kwargs)
threads = [threading.Thread(target=run) for i in range(N_THREADS)]
for t in threads:
t.start()
while threads:
for t in threads:
t.join(timeout=.1)
# filter out finished threads
threads = [t for t in threads if t.is_alive()]
print("%d threads still alive" % len(threads))
def test_pool_exhaustion():
pool = ConnectionPool(size=1, **connection_kwargs)
def run():
with assert_raises(NoConnectionsAvailable):
with pool.connection(timeout=.1) as connection:
connection.tables()
with pool.connection():
# At this point the only connection is assigned to this thread,
# so another thread cannot obtain a connection at this point.
t = threading.Thread(target=run)
t.start()
t.join()
def test_connection_pool():
def run():
name = threading.current_thread().name
print "Thread %s starting" % name
def inner_function():
# Nested connection requests must return the same connection
with pool.connection() as another_connection:
assert connection is another_connection
for i in xrange(100):
with pool.connection() as connection:
connection.tables()
# Fake an exception once in a while
if random.random() < .001:
connection._tainted = True
inner_function()
print "Thread %s done" % name
N_THREADS = 50
pool = ConnectionPool(size=3, **connection_kwargs)
threads = [threading.Thread(target=run) for i in xrange(N_THREADS)]
for t in threads:
t.start()
while threads:
for t in threads:
t.join(timeout=.1)
# filter out finished threads
threads = [t for t in threads if t.is_alive()]
print "%d threads still alive" % len(threads)
def test_connection_pool_construction():
with assert_raises(TypeError):
ConnectionPool(size='abc')
with assert_raises(ValueError):
ConnectionPool(size=0)
def setup(self):
HBaseManager(ConnectionPool(size=1, host=HBaseManager.HOST, port=HBaseManager.PORT)).create_table(
table_name=self.TABLE_NAME, delete=True)
from django.http import HttpResponse
from happybase import Connection, ConnectionPool
from collections import Counter
import random
import itertools
import pandas as pd
import numpy as np
from django.shortcuts import render_to_response
from desktop.lib.django_util import render
import datetime, json
from os.path import abspath, split, join
import strategy
# conn = Connection('192.168.2.41')
pool = ConnectionPool(size=5, host='192.168.2.41')
'''
user_t = conn.table('haodou')
keys_t = conn.table('tag_search_keys')
id2tokens_t = conn.table('id2tokens') #id-标签对应表
favs_t = conn.table('favs_by_time') #key: time, column: token, value: [w1, w2, w3, ...]
user_tags_t = conn.table('haodou_user_tags') # 用户标签库
goods_t = conn.table('haodou_goods')
td_w_t = conn.table('td_w') # tag_date_weight
ud_w_t = conn.table('ud_w') # user_date_weight
'''
current_tag = ''
country_index = [u'中国', u'加拿大', u'美国', u'日本', u'澳大利亚']
token2tag = {