def __init__(self, data, var, prefix, Lambda):
global numeric_inverse
dsize = get_batch_size(data)
# TODO: try adding regularizer later
cov_op = data @ t(data) / dsize
if regularize_covariances:
#ii = u.Identity(cov_op.shape[0])
#regularizer = Lambda*ii
regularizer = u.cachedGpuIdentityRegularizer(cov_op.shape[0],
Lambda=u.args.Lambda)
cov_op = cov_op + regularizer
cov_name = "%s_cov_%s" %(prefix, var.op.name)
svd_name = "%s_svd_%s" %(prefix, var.op.name)
# hack
# pp = u.args.kfac_polyak_factor
pp = 1
if pp<1:
self.cov = u.get_variable(name=cov_name, initializer=ii)
else:
self.cov = u.get_variable(name=cov_name, initializer=cov_op)
self.svd = u.SvdWrapper(target=self.cov, name=svd_name,
do_inverses=(inverse_method=='inverse'))
# self.cov_update_op = self.cov.initializer
if pp<1:
self.cov_update_op = self.cov.assign(cov_op*(1-pp)+self.cov*pp).op
else:
self.cov_update_op = self.cov.assign(cov_op).op
def init_var(val, name, is_global=False):
"""Helper to create variables with numpy or TF initial values."""
if isinstance(val, tf.Tensor):
var = u.get_variable(name=name, initializer=val, reuse=is_global)
else:
val = np.array(val)
assert u.is_numeric(val), "Unknown type"
holder = tf.placeholder(dtype,
shape=val.shape,
name=name + "_holder")
var = u.get_variable(name=name,
initializer=holder,
reuse=is_global)
init_dict[holder] = val
if is_global:
global_vars.append(var)
else:
local_vars.append(var)
return var
def main():
logging.basicConfig(level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s")
logging.info("Starting Sync Users Job")
# connect cluster
logging.info("Connecting to target cluster")
db = util.connect_memsql()
logging.info("Connected to target cluster")
# inputs
expected_users_list = util.must_get_env("JWT_USERS_LIST")
expected_users = set()
if expected_users_list != '':
expected_users = set(expected_users_list.split(","))
logging.info("expected list of jwt users: {}".format(expected_users))
expected_permissions_list = util.must_get_env("PERMISSIONS")
expected_permissions = set()
if expected_permissions_list != '':
expected_permissions = set(expected_permissions_list.split(","))
logging.info(
"expected list of permissions: {}".format(expected_permissions))
# get current list of JWT users
actual_users = util.list_jwt_users(db)
logging.info("current list of jwt users: {}".format(actual_users))
# create expected user if it does not exist
# do not require ssl if cluster is using http proxy port
http_proxy_port = util.get_variable(db, "http_proxy_port")
for expected_user in expected_users:
if expected_user not in actual_users:
if int(http_proxy_port) > 0:
logging.info("Creating JWT user {}".format(expected_user))
util.create_jwt_user(db, expected_user)
else:
logging.info("Creating JWT user {} with REQUIRE SSL".format(
expected_user))
util.create_jwt_user_ssl(db, expected_user)
# drop users that are not expected
for actual_user in actual_users:
if actual_user not in expected_users:
logging.info("Dropping JWT user {}".format(actual_user))
util.drop_user(db, actual_user)
# grant all expected users permissions
for expected_user in expected_users:
util.grant_user_permissions(db, expected_user,
expected_permissions_list, "*.*")
logging.info("Done!")
def init_var(val, name, is_global=False):
"""Helper to create variables with numpy or TF initial values."""
if isinstance(val, tf.Tensor):
var = u.get_variable(name=name, initializer=val, reuse=is_global)
else:
val = np.array(val)
assert u.is_numeric(val), "Non-numeric type."
var_struct = u.get_var(name=name, initializer=val, reuse=is_global)
holder = var_struct.val_
init_dict[holder] = val
var = var_struct.var
if is_global:
global_vars.append(var)
else:
local_vars.append(var)
return var
def init_var(val, name, is_global=False):
"""Helper to create variables with numpy or TF initial values."""
if isinstance(val, tf.Tensor):
var = u.get_variable(name=name, initializer=val, reuse=is_global)
else:
val = np.array(val)
assert u.is_numeric(val), "Non-numeric type."
var_struct = u.get_var(name=name, initializer=val, reuse=is_global)
holder = var_struct.val_
init_dict[holder] = val
var = var_struct.var
if is_global:
global_vars.append(var)
else:
local_vars.append(var)
return var