def add_comment(thread_uid, author, text):
"""Add comment to specefied thread.
:param thread_uid: Thread unique ID.
:param author: Author of comment.
:param text: Comment text.
"""
# Setup new UID for comment
comment_uid = uid()
# Store comment metadata and rewrite data about last comment for thread
with content.pipeline() as pipe:
pipe.hmset(build_key(COMMENT_KEY, thread_uid, comment_uid), {
'author': author,
'text': text,
'timestamp': time.time(),
})
pipe.hmset(build_key(THREAD_KEY, thread_uid), {
'last_comment_uid': comment_uid,
})
pipe.execute()
# Put comment to comments list and incr comments counter for thread
with links.pipeline() as pipe:
pipe.rpush(build_key(THREAD_COMMENTS_KEY, thread_uid), comment_uid)
pipe.incr(build_key(THREAD_COUNTER_KEY, thread_uid))
pipe.execute()
# Comment added, everything is alright
return True
async def test_users(self, aiohttp_client):
"""
Test user management: /api/users
:param aiohttp_client:
:return:
"""
client = await aiohttp_client(await app_factory())
# check JWT authorization
credentials = await self.auth_admin(aiohttp_client)
access_token = credentials['token']
headers = {'Authorization': f'Bearer {access_token}'}
test_user = '******'
test_user_edited = 'test_user_edited'
# adding a user
resp = await client.post('/api/users', json={'username': test_user, 'password': uid(6)}, headers=headers)
assert resp.status == 200
# editing user data
resp = await client.put(f'/api/users/{test_user}', json={'password': uid(6)}, headers=headers)
assert resp.status == 200
resp = await client.put(f'/api/users/{test_user}', json={'username': test_user_edited}, headers=headers)
assert resp.status == 200
# deleting a user
resp = await client.delete(f'/api/users/{test_user_edited}', headers=headers)
assert resp.status == 200
def read_zeropoint(zeropoint, blacklist=None):
""" Read the zeropoints and apply the blacklist. """
ext = os.path.splitext(zeropoint)[-1]
if ext == '.csv':
#zp = np.recfromcsv(zeropoint)
zp = pd.read_csv(zeropoint).to_records(index=False)
zp.dtype.names = map(str.upper, zp.dtype.names)
elif ext == '.fits':
zp = fitsio.read(zeropoint, ext=1, columns=ZPCOLS)
else:
msg = "Unrecognized zeropoint extension: %s" % ext
raise Exception(msg)
# If necessary, change the sign of MAG_ZERO
if (zp['MAG_ZERO'] < 0).sum() / float(len(zp)) > 0.90:
msg = "Switching sign of MAG_ZERO."
logger.info(msg)
zp['MAG_ZERO'] *= -1
# reject CCDs with MAG_ZERO < 0
sel_zero = (zp['MAG_ZERO'] >= 0)
if (~sel_zero).sum():
msg = "Found %i CCDs with negative zeropoints." % (~sel_zero).sum()
logger.warning(msg)
# reject CCDs with QSLR_FLAGS > 0
sel_flags = (zp['ZP_FLAG'] == 0)
if (~sel_flags).sum():
msg = "Found %i CCDs with bad ZP flags." % (~sel_flags).sum()
logger.warning(msg)
sel = sel_zero & sel_flags
zp = zp[sel]
if blacklist is None:
return zp
else:
bl = read_blacklist(blacklist)
# Unique value from EXPNUM, CCDNUM
zpval = utils.uid(zp['EXPNUM'], zp['CCDNUM'])
blval = utils.uid(bl['EXPNUM'], bl['CCDNUM'])
idx = np.in1d(zpval, blval)
return zp[~idx]
async def test_users(self, aiohttp_client):
"""Test user management: /api/users
:param aiohttp_client:
:return:
"""
client = await aiohttp_client(await app_factory())
# check JWT authorization
credentials = await self.get_admin_credentials(aiohttp_client)
access_token = credentials["token"]
headers = {"Authorization": f"Bearer {access_token}"}
test_user = "******"
test_user_edited = "test_user_edited"
# adding a user
resp = await client.post(
"/api/users",
json={
"username": test_user,
"password": uid(6)
},
headers=headers,
)
assert resp.status == 200
# editing user data
resp = await client.put(f"/api/users/{test_user}",
json={"password": uid(6)},
headers=headers)
assert resp.status == 200
resp = await client.put(
f"/api/users/{test_user}",
json={"username": test_user_edited},
headers=headers,
)
assert resp.status == 200
# deleting a user
resp = await client.delete(f"/api/users/{test_user_edited}",
headers=headers)
assert resp.status == 200
def submitFace(res):
#validate data
if not res.is_ajax() or not res.method == 'POST':
return bad
try:
json = js.loads(res.body)
except:
return bad
if (json == None or type(json) != dict or not 'gender' in json
or not 'parameters' in json or not 'choices' in json
or type(json['choices']) != dict):
return bad
gender = 'f' if json['gender'] == 'f' else 'm'
pcount = autoencoder.models[autoencoder.default]['bottleneck']
arr = json['parameters']
if type(json['parameters']) != list or len(arr) != pcount:
return bad
try:
arr = np.nan_to_num(np.array(arr).astype(np.float64).clip(-20, 20))
except:
return bad
choices = json['choices']
for key, choice in models.choices.items():
if key == 'g': continue # ignore gender choice
if not key in choices:
return bad
b = True
choice_ = choices[key]
for option in choice['options']:
if option[0] == choice_:
b = False
break
if b: return bad
#save data
uid = utils.uid()
img = autoencoder.decode(autoencoder.default, gender, arr)
cv2.imwrite('faces/' + uid + '.jpg', cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
face = models.Face(uid=uid,
gender=gender,
parameters=js.dumps(list(arr)),
poster_gender=choices['pg'],
poster_sexuality=choices['ps'],
poster_race=choices['pr'],
poster_country=choices['pc'])
face.save()
return HttpResponse(js.dumps('ok'), content_type='application/json')
def get_test_trees():
t1 = tl.Tree()
r1 = uid()
t1.create_node(tag=-1, identifier=r1)
t1.create_node(identifier=1, parent=r1)
t1.create_node(identifier=2, parent=r1)
t2 = tl.Tree()
r2 = uid()
t2.create_node(tag=-1, identifier=r2)
t2.create_node(identifier=3, parent=r2)
t2.create_node(identifier=4, parent=r2)
t2.create_node(identifier=5, parent=r2)
print("T1:")
print(t1)
print("T2:")
print(t2)
return t1, t2
async def test_query_save(self, aiohttp_client):
"""
Test query with db registering and saving results to disk: /api/queries
:param aiohttp_client:
:return:
"""
client = await aiohttp_client(await app_factory())
# authorize
credentials = await self.auth_admin(aiohttp_client)
access_token = credentials['token']
headers = {'Authorization': access_token}
collection = 'ZTF_alerts'
# test query with book-keeping
qu = {"query_type": "find_one",
"query": {
"catalog": collection,
"filter": {},
},
"kwargs": {"save": True, "_id": uid(32)}
}
# print(qu)
resp = await client.post('/api/queries', json=qu, headers=headers, timeout=5)
assert resp.status == 200
result = await resp.json()
# print(result)
assert result['status'] == 'success'
# todo: test getting 'task' and 'result'
# remove enqueued query
resp = await client.delete(f'/api/queries/{result["query_id"]}', headers=headers, timeout=5)
assert resp.status == 200
result = await resp.json()
assert result['status'] == 'success'
def start_thread(author, subject, comment=None):
"""Start new thread with or without comment.
:param author: Thread author.
:param subject: Thread subject.
:param comment: Text for first comment if any.
"""
# Setup new UID for the Thread and add it to Links and Content storages
thread_uid = uid()
content.hmset(build_key(THREAD_KEY, thread_uid), {
'author': author,
'subject': subject,
'timestamp': time.time(),
})
links.lpush(build_key(THREADS_KEY), thread_uid)
# Add comment, but only if it not empty
if comment:
add_comment(thread_uid, author, comment)
# Everything is OK
return True
def find_approx(G: nx.Graph) -> Tuple[tl.Tree, float]:
n = len(G.nodes())
# Initialize C
C = []
for v in G.nodes():
# create a tree with a single root and a leaf.
T = tl.Tree()
root = uid()
T.create_node(-1, root)
T.create_node(v, v, parent=root)
C.append(T)
# fill out the initial costs (lower diagonal only). (4 for each pair)
costs = np.zeros(
(n, n, 4)) # len(C) (tree 1) x len(C) (tree 2) x 4 (merged tree)
for j in range(n):
for k in range(j):
# only consider a merge if that pair would add an edge to the graph.
# otherwise, it is an uninformative merge.
if share_any_edges(G, C[j], C[k]):
# join and collapse are symmetric.
join_cost = our_cost(G, join(C[j], C[k]))
collapse_cost = our_cost(G, collapse(C[j], C[k]))
# b/c absorb is asymmetric, must do both ways for absorb.
absorb_into_j_cost = our_cost(G, absorb(C[j], C[k]))
absorb_into_k_cost = our_cost(G, absorb(C[k], C[j]))
costs[j, k, 0] = join_cost
costs[j, k, 1] = collapse_cost
costs[j, k, 2] = absorb_into_j_cost
costs[j, k, 3] = absorb_into_k_cost
# if dont share an edge, don't consider the pair.
else:
costs[j, k, 0] = 1e15
costs[j, k, 1] = 1e15
costs[j, k, 2] = 1e15
costs[j, k, 3] = 1e15
# Main Loop.
for i in range(1, n):
# debug the trees at this step:
print(f"Step {i}")
for tree in C:
print(tree)
# length of costs array currently.
c_len = costs.shape[0]
# add absurdly high cost to upper diagonal to not consider these.
costs = costs + (1e20 * np.triu(np.ones((c_len, c_len)))).reshape(
(c_len, c_len, 1))
# find cost-minimizing pair and merge type
min_pair_index_flat = np.argmin(
costs) # This gives index of flattened.
# recover indices.
j_min = min_pair_index_flat // costs.shape[2] // c_len
k_min = min_pair_index_flat // costs.shape[2] % c_len
l_min = min_pair_index_flat % costs.shape[2]
# print(f"min: {j_min}, {k_min}, {l_min} = {costs[j_min,k_min,l_min]}")
# get the resulting minimizing tree.
if l_min == 0:
new_tree = join(C[j_min], C[k_min])
elif l_min == 1:
new_tree = collapse(C[j_min], C[k_min])
elif l_min == 2:
new_tree = absorb(C[j_min], C[k_min])
else:
new_tree = absorb(C[k_min], C[j_min])
# get the new set of trees
C = [tree for ti, tree in enumerate(C) if ti not in [j_min, k_min]]
C.append(new_tree)
# recalculate the new costs (again, only need lower diagonal)
# add in new row
costs = np.append(costs,
np.zeros((1, costs.shape[1], costs.shape[2])),
axis=0)
costs = np.append(costs,
np.zeros((costs.shape[0], 1, costs.shape[2])),
axis=1)
# get rid of old rows.
costs = np.delete(costs, (j_min, k_min), axis=0)
costs = np.delete(costs, (j_min, k_min), axis=1)
# fill in new row
j = -1
for k in range(costs.shape[1] - 1):
if share_any_edges(G, C[j], C[k]):
join_cost = our_cost(G, join(C[j], C[k]))
collapse_cost = our_cost(G, collapse(C[j], C[k]))
# b/c absorb is asymmetric, must do both ways for absorb.
absorb_into_j_cost = our_cost(G, absorb(C[j], C[k]))
absorb_into_k_cost = our_cost(G, absorb(C[k], C[j]))
costs[j, k, 0] = join_cost
costs[j, k, 1] = collapse_cost
costs[j, k, 2] = absorb_into_j_cost
costs[j, k, 3] = absorb_into_k_cost
else:
costs[j, k, 0] = 1e15
costs[j, k, 1] = 1e15
costs[j, k, 2] = 1e15
costs[j, k, 3] = 1e15
print(f"min k: {l_min}")
assert len(C) == 1
return C[0], our_cost(G, C[0])
qslrfile = 'y2n_y1a1_qslr_v6.fits'
if not os.path.exists(qslrfile):
query = download.qslr_query()
download.download(qslrfile, query, section='dessci')
print "Loading %s..." % qslrfile
qslr = fitsio.read(qslrfile)
names = [HPX]
values = [ang2pix(nside, qslr['RA_MEAN'], qslr['DEC_MEAN'])]
qslr = recfn.append_fields(qslr,
names,
values,
usemask=False,
asrecarray=True)
qslr = qslr[qslr['QSLR_FLAG'] == 0]
gcm_uid = uid(gcm['EXPNUM'], gcm['CCDNUM'])
qslr_uid = uid(qslr['EXPNUM'], qslr['CCDNUM'])
gcm = gcm[np.in1d(gcm_uid, qslr_uid)]
qslr = qslr[np.in1d(qslr_uid, gcm_uid)]
gcm = gcm[np.lexsort((gcm['CCDNUM'], gcm['EXPNUM']))]
qslr = qslr[np.lexsort((qslr['CCDNUM'], qslr['EXPNUM']))]
if (gcm['EXPNUM']!=qslr['EXPNUM']).any() \
or (gcm['CCDNUM']!=qslr['CCDNUM']).any():
msg = "GCM and qSLR do not match"
raise Exception(msg)
gcm = gcm[np.lexsort((gcm['CCDNUM'], gcm['EXPNUM']))]
qslr = qslr[np.lexsort((qslr['CCDNUM'], qslr['EXPNUM']))]
def train(self,
x,
y,
batch_size,
out_folder,
output_checkpoint_inputs_word2vec=None):
if not self.model_compiled:
raise Exception("The model must be compiled first.")
train_uid = utils.uid()
description = "{train_uid} batch={batch_size} cfg_index={cfg_idx}"\
.format(train_uid=train_uid, batch_size=batch_size, cfg_idx=self.cfg_idx)
print("Training: {}".format(description))
# callbacks
callbacks = []
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0.02,
patience=20,
verbose=1,
mode='min',
restore_best_weights=True)
callbacks.append(early_stopping)
check_nan = CheckNanLoss('val_loss')
callbacks.append(check_nan)
tensor_board_log_dir = Path(out_folder, "tensorboard", description)
tensor_board_log_dir.mkdir(parents=True, exist_ok=True)
tensor_board_writer = tf_summary.FileWriter(str(tensor_board_log_dir),
K.get_session().graph)
if output_checkpoint_inputs_word2vec is not None:
output_checkpoint = OutputCheckpoint(
tensor_board_writer=tensor_board_writer,
val_data=(x[:4], y[:4]),
test_data_input_word2vec=output_checkpoint_inputs_word2vec,
print_every=30)
callbacks.append(output_checkpoint)
# last because close the writer on training end
tensor_board = TensorBoard2(writer=tensor_board_writer,
batch_size=batch_size)
callbacks.append(tensor_board)
x_train, x_test, y_train, y_test = sklearn.model_selection.train_test_split(
x, y, train_size=0.8, shuffle=True)
# fit
self.model.fit(
x=x_train,
y=y_train,
epochs=300,
batch_size=batch_size,
verbose=1,
shuffle=True,
callbacks=callbacks,
validation_data=(x_test, y_test),
)
# save
weights_path = Path(out_folder, "weights", "{}.h5".format(description))
weights_path.parent.mkdir(parents=True, exist_ok=True)
from cryptography import fernet
from utils import uid
if __name__ == "__main__":
secrets = dict()
fernet_key = fernet.Fernet.generate_key().decode()
aiohttp_secret_key = uid(32)
jwt_secret_key = uid(32)
for key in ("server", "misc"):
if key not in secrets:
secrets[key] = dict()
secrets["server"]["SECRET_KEY"] = aiohttp_secret_key
secrets["server"]["JWT_SECRET_KEY"] = jwt_secret_key
secrets["misc"]["fernet_key"] = fernet_key
def __init__(self, log_dir):
pathlib.Path(log_dir).mkdir(parents=True, exist_ok=True)
self.log_file = open(os.path.join(log_dir, "{}.txt".format(utils.uid())), "a")
self.stdout = sys.stdout
