def setUp(self):
#get redis connection
#TODO: able to override with env vars
self.url_cache = redis_collections.Dict(
key='test-url-cache-key-delete-me')
self.hash_cache = redis_collections.Dict(
key='test-hash-cache-key-delete-me')
self.url_cache['https://example.edu'] = {
'If-None-Match': 'nice etag',
'If-Modified-Since': 'since test val'
}
def run(architecture_file, model_file, num_batches=10, pattern="centralModel-*"):
print "evaluating Q values..."
redisInstance = Redis(host='localhost', port=6379, db=0)
model_keys = redisInstance.keys(pattern)
results = {}
net = BaristaNet(architecture_file, model_file, None)
replay_dataset = ReplayDataset("temp-q-converge-dset.hdf5",
net.state[0].shape,
dset_size=1000,
overwrite=True)
net.add_dataset(replay_dataset)
game = SnakeGame()
preprocessor = generate_preprocessor(net.state.shape[2:], gray_scale)
exp_gain = ExpGain(net, ['w', 'a', 's', 'd'], preprocessor, game.cpu_play,
replay_dataset, game.encode_state())
print "Generating new experiences..."
for _ in xrange(100):
exp_gain.generate_experience(1e5)
print "Done"
for key in model_keys:
print "Evaluating model:", key
model = dict(redisC.Dict(key=key, redis=redisInstance))
q_avg = evaluate_model(net, model, num_batches)
results[key] = q_avg
for key in sorted(results.keys()):
print key.ljust(25) + "%0.4f" % results[key]
def saveSnapshot(snapshot_name, model):
print "saving snapshot"
snapshot = redisC.Dict(redis=redisInstance, key=snapshot_name)
for key in model:
snapshot[key] = model[key]
filename = snapshot_name + str(datetime.now())
with open(filename, 'wb') as f:
pickle.dump(dict(model), f)
print "[SNAPSHOT] Model snapshot saved:", snapshot_name
def __init__(self,
cdb=None,
url_couchdb=None,
couchdb_name=None,
couch_view=COUCHDB_VIEW,
bucket_bases=BUCKET_BASES,
object_auth=None,
get_if_object=False,
ignore_content_type=False,
url_cache=None,
hash_cache=None,
harvested_object_cache=None):
self._config = config()
if cdb:
self._couchdb = cdb
else:
if not url_couchdb:
url_couchdb = self._config['couchdb_url']
self._couchdb = get_couchdb(url=url_couchdb, dbname=couchdb_name)
self._bucket_bases = bucket_bases
self._view = couch_view
# auth is a tuple of username, password
self._auth = object_auth
self.get_if_object = get_if_object # if object field exists, get
self.ignore_content_type = ignore_content_type # Don't check content-type in headers
self._redis = Redis(
host=self._config['redis_host'],
port=self._config['redis_port'],
password=self._config['redis_password'],
socket_connect_timeout=self._config['redis_connect_timeout'])
self._url_cache = url_cache if url_cache is not None else \
redis_collections.Dict(key='ucldc-image-url-cache',
redis=self._redis)
self._hash_cache = hash_cache if hash_cache is not None else \
redis_collections.Dict(key='ucldc-image-hash-cache',
redis=self._redis)
self._object_cache = harvested_object_cache if harvested_object_cache \
else \
redis_collections.Dict(
key='ucldc:harvester:harvested-images',
redis=self._redis)
def initParams(solver_filename, reset=True):
global redisInstance
redisInstance = Redis(host='localhost', port=6379, db=0)
averageReward = redisC.Dict(redis=redisInstance, key="averageReward")
if reset:
# Remove all previously saved snapshots from redis
for name in redisInstance.keys(MODEL_NAME + "-*"):
snapshot = redisC.Dict(redis=redisInstance, key=name)
snapshot.clear()
centralModel.clear()
rmsprop.clear()
adagrad.clear()
averageReward.clear()
redisInstance.set("iteration", 0)
# Instantiate model parameters according to initialization
# scheme specified in .prototxt file
solver = SGDSolver(solver_filename, )
for name in solver.net.params:
if is_tracked_param(name):
parameters = solver.net.params[name]
init = []
for i in range(len(parameters)):
init.append(np.array(parameters[i].data, dtype='float32'))
centralModel[name] = init
print
print "[Redis Collection]: Initialized the following parameters:"
for key in centralModel.keys():
print " - " + key + ' (%d parameters)' % len(centralModel[key])
else:
solver = SGDSolver(solver_filename, )
for name in solver.net.params:
parameters = solver.net.params[name]
assert (name in centralModel.keys()
and len(centralModel[name]) == len(parameters) and
"Model in Redis database does not match specified solver.")
def __init__(self, app, schedule=None, max_interval=None,
Publisher=None, lazy=False, sync_every_tasks=None, **kwargs):
#self.schedule_filename = kwargs.get('schedule_filename')
redis_uri = app.conf.CELERY_SCHEDULE_BACKEND
# set redis manager
self.manager = RedisManager(redis_uri)
#keys = self.manager.inspect(pattern='*', debug=False)
self._prefix = app.conf.CELERY_REDIS_SCHEDULER_KEY_PREFIX
self._schedule = redis_collections.Dict(key=self._prefix, redis=self.manager.conn)
Scheduler.__init__(self, app, schedule=schedule,
max_interval=max_interval, Publisher=Publisher,
lazy=lazy, sync_every_tasks=sync_every_tasks, **kwargs)
def __init__(s, g, gv_output_file, step):
s.g, s.gv_output_file = g, gv_output_file
s.step = step
s.frame_templates = redis_collections.Dict(
redis=strict_redis_connection)
'''one time script to populate redis with harvested image object data'''
from harvester.config import config
from harvester.couchdb_init import get_couchdb
from harvester.couchdb_pager import couchdb_pager
from redis import Redis
import redis_collections
_config = config()
_redis = Redis(host=_config['redis_host'],
port=_config['redis_port'],
password=_config['redis_password'],
socket_connect_timeout=_config['redis_connect_timeout'])
object_cache = redis_collections.Dict(key='ucldc:harvester:harvested-images',
redis=_redis)
_couchdb = get_couchdb(url=_config['couchdb_url'], dbname='ucldc')
v = couchdb_pager(_couchdb, include_docs='true')
for r in v:
doc = r.doc
if 'object' in doc:
did = doc['_id']
if 'object_dimensions' not in doc:
print "NO DIMS for {} -- not caching".format(did)
else:
object_cache[did] = [doc['object'], doc['object_dimensions']]
print "OBJECT CACHE : {} === {}".format(did, object_cache[did])
from log import get_logger
MB = 1024.0 * 1024.0
LOCAL_DIR = path.dirname(path.abspath(__file__))
PID_FILE = path.join(LOCAL_DIR, 'speedtest.pid')
DB_FILE = path.join(LOCAL_DIR, 'speedtest.db')
PORT = 8002
MIN_DATETIME = datetime(2016, 1, 1)
logger = get_logger('speedtest')
is_daemon = False
redis = redislite.StrictRedis(DB_FILE, serverconfig={'port': PORT})
data = redis_collections.List(redis=redis, key='speed')
settings = redis_collections.Dict(redis=redis, key='settings')
def get_splitted_output(cmd):
return subprocess.check_output(cmd).decode().replace('\n', '').replace(
'\r', '').split(',')
def get_speed():
headers = get_splitted_output(['speedtest', '--csv-header'])
try:
speed = get_splitted_output(['speedtest', '--csv'])
except Exception as ex:
if not is_daemon:
print(ex)
logger.error(ex)
def __init__(self, options, *args, **kwargs):
super(StorageObject, self).__init__(options, *args, **kwargs)
self.redis = redis.Redis(host=self.options.redis_host,
port=self.options.redis_port)
self.storage = redis_collections.Dict(key=self.options.redis_key,
redis=self.redis)
def saveAverageReward():
print "saving all rewards to disk"
averageReward = redisC.Dict(redis=redisInstance, key='averageReward')
filename = "averageReward" + str(datetime.now())
with open(filename, 'wb') as f:
pickle.dump(dict(averageReward), f)
def work(identification, graph_name, _range_start, _range_end, redis_fn):
global redis_connection, strict_redis_connection, sparql_server, current_step, range_start, range_end, ss, just_unbound_bindings, frames_done_count, frame_templates, bnode_strings
range_start, range_end = _range_start, _range_end
sparql_server = sparql.SPARQLServer(sparql_uri)
redis_connection = redislite.Redis(redis_fn)
strict_redis_connection = redislite.StrictRedis(redis_fn)
frame_templates = redis_collections.Dict(key='frames',
redis=strict_redis_connection,
writeback=True)
bnode_strings = redis_collections.Dict(key='bnodes',
redis=strict_redis_connection,
writeback=True)
raw = defaultdict(list)
#todo limit queries with range_start, then uncomment this
#if range_start != 0:
# raw = redis_load('checkpoint'+str(range_start - 1))
raw['frames'] += list(
query(('frame', 'parent', 'is_for_rule', 'step_finished',
'step_created'), """WHERE
{
""" + frame_query() + """
OPTIONAL {?frame kbdbg:has_parent ?parent}.
?frame kbdbg:is_for_rule ?is_for_rule.
}"""))
raw['bnodes'] += list(
query(('bnode', 'frame', 'items', 'step_created', 'step_finished'),
"""WHERE
{
?bnode kbdbg:has_items ?items.
?bnode kbdbg:has_parent ?frame.
GRAPH ?g_created {?bnode rdf:type kbdbg:bnode}.
""" + step_magic('_created') + """
OPTIONAL {
GRAPH ?g_finished{?frame kbdbg:is_finished true}.
""" + step_bind('_finished') + """
}
}"""))
raw['results'] += list(
query(('uri', 'value', 'step_unbound'), """WHERE {GRAPH ?g_created
{
?uri rdf:type kbdbg:result.
?uri rdf:value ?value.
}.""" + step_magic('_created') + """
OPTIONAL {GRAPH ?g_unbound {?uri kbdbg:was_ubound true}.}.""" +
step_bind('_unbound') + '}'))
raw['bindings'] += list(
query(('x', 'source', 'target', 'source_frame', 'target_frame',
'source_is_bnode', 'target_is_bnode', 'source_term_idx',
'target_term_idx', 'source_is_in_head', 'target_is_in_head',
'source_arg_idx', 'target_arg_idx', 'stepbinding_unbound',
'stepbinding_failed', 'stepbinding_created'), """WHERE
{
GRAPH ?gbinding_created {?x rdf:type kbdbg:binding.}.
""" + step_magic('binding_created') + """
OPTIONAL {GRAPH ?gbinding_unbound {?x kbdbg:was_unbound true}.
""" + step_bind('binding_unbound') + """
}.
OPTIONAL {GRAPH ?gbinding_failed {?x kbdbg:failed true}.
""" + step_bind('binding_failed') + """
}.
?x kbdbg:has_source ?source.
?x kbdbg:has_target ?target.
?source kbdbg:has_frame ?source_frame.
?target kbdbg:has_frame ?target_frame.
OPTIONAL {?source kbdbg:is_bnode ?source_is_bnode.}.
OPTIONAL {?target kbdbg:is_bnode ?target_is_bnode.}.
?source kbdbg:term_idx ?source_term_idx.
?target kbdbg:term_idx ?target_term_idx.
OPTIONAL {?source kbdbg:is_in_head ?source_is_in_head.}.
OPTIONAL {?target kbdbg:is_in_head ?target_is_in_head.}.
OPTIONAL {?source kbdbg:arg_idx ?source_arg_idx.}.
OPTIONAL {?target kbdbg:arg_idx ?target_arg_idx.}.
}"""))
current_step = range_end
redis_save('checkpoint' + str(range_end), filter_out_irrelevant_stuff(raw))
current_step = '666'
last_bindings = raw['bindings'][:]
for i in range(range_start, range_end + 1):
current_step = i
ss = '[' + str(current_step) + ']'
info('work ' + ss)
state = filter_out_irrelevant_stuff(raw)
if len(state['frames']) == 0:
info('no frames.' + ss)
continue
#wont work with euler-style steps
#if last_bindings == state['bindings']:
# return 'end'
#todo make emitter save data to output, not to file
e = Emitter()
e.gv("digraph frame" + str(current_step) +
"{ ") #splines=ortho;#gv("pack=true")
e.do_frames(state['frames'])
e.do_bnodes(state['bnodes'])
e.do_results(state['results'])
e.do_bindings(state['bindings'])
e.gv("}")
info('}..' + ss)
args = (ss, identification + '_' + str(current_step).zfill(7) + '.gv',
e.output)
while len(graphviz_futures) > 10000:
info('sleeping')
time.sleep(10)
check_futures2(graphviz_futures)
graphviz_futures.append(graphviz_pool.submit(output, *args))
check_futures2(graphviz_futures)
#print_stats()
redis_connection._cleanup()
strict_redis_connection._cleanup()