def make_new(string_key, hash_method="sha256", text_encoding="utf-8"):
encryption_hash, encryption_hash_salt = utils.hash_string(
string_key, hash_method=hash_method, text_encoding="utf-8")
storage_hash, storage_hash_salt = utils.hash_string(
string_key, hash_method=hash_method, text_encoding="utf-8")
iv = None
data_length = -1
crypto_data = HashBasedCryptoData(from_code=True)
for data_part_name in [
"encryption_hash", "encryption_hash_salt", "storage_hash",
"storage_hash_salt", "iv", "data_length"
]:
setattr(crypto_data, data_part_name, locals()[data_part_name])
crypto_data.iv_is_from_deserializer = False
return crypto_data
async def get(self, _, uid, rhash, answer):
try:
user = await Users.get_by_id(int(uid))
uhash = hash_string(user.name + str(user.id) + user.email)
if not user.accepted:
logging.error('{} was trying to hack us'.format(user.email))
return json({'msg': 'Nice try! But nope.'})
if user.confirmation != 'noans':
return json({
'msg':
'Sorry, it is not possible to change your mind now'
})
if uhash == rhash:
if answer == 'yes':
user.confirmation = 'ack'
await user.update()
return json({'msg': 'Widzimy się w Sobotę 23.09.2017!'})
elif answer == 'no':
user.confirmation = 'rej_user'
await user.update()
return json({'msg': 'Szkoda, że się już nie zobaczymy'})
else:
return json({'msg': 'wrong hash'})
except:
logging.exception('AbsenceConfirmation')
return json({'msg': 'wrong data'})
async def post(self):
global _users
try:
req = self.req.json
user = await Users.get_first('email', req.get('email', ''))
if not user:
return json({'msg': 'User not found'}, status=404)
if not user.active:
return json({'msg': 'User not active'}, status=404)
if hash_string(req.get('password', 'x')) == user.password:
session_uuid = await user.get_session_uuid()
_users[session_uuid] = user
return json({
'success': True,
'admin': user.admin,
'mentor': user.mentor,
'name': user.name,
'email': user.email,
'surname': user.surname,
'lang': user.lang,
'organiser': user.organiser,
'id': user.id,
'session_uuid': session_uuid,
'confirmation': user.confirmation,
'gdpr': user.gdpr,
})
else:
return self.user_error
except DoesNotExist:
return self.user_error
except:
logging.exception('err authentication.post')
return json({'msg': 'internal error'}, status=500)
async def post(self, request, current_user):
try:
req = request.json
if hash_string(req.get('password', 'x')) == current_user.password:
if req['new_password'] == req['new_password_2']:
await current_user.set_password(req['new_password'])
await current_user.update()
return json({
"success": True,
"msg": "You have Successfully changed password"
})
return json({
"success": False,
"msg": "You provided different new passwords"
})
return json({
"success": False,
"msg": "You provided wrong old password"
})
except:
logging.exception('authentication.post')
return json({
'msg': 'internal error sorry please let us now',
"success": False
})
async def post(self, request, current_user):
try:
req = request.json
validation_outcome = await current_user.validate_password(
req['new_password'])
if not validation_outcome['success']:
return json(validation_outcome)
if hash_string(req.get('password', 'x')) == current_user.password:
if req['new_password'] == req['new_password_2']:
await current_user.set_password(req['new_password'])
await current_user.update()
return json({
"success":
True,
"msg":
"You have successfully changed your password"
})
return json({
"success": False,
"msg": "You provided different new password"
})
return json({
"success": False,
"msg": "You provided wrong old password"
})
except:
logging.exception('authentication.post')
return json({
'msg': 'Sorry, internal error. Please let us now!',
"success": False
})
def spoof_cert ( server_cert, ca_cert, key ): # x509 Objects not strings CN = server_cert.get_subject().commonName #print ( CN ) #CERTFILE = "%s_cert.pem" % CN CERTFILE = hash_string ( CN ) cert_dir = "certificates/" temp = cert_dir + CERTFILE # set issuer of server_cert with subject of ca_cert #print ( ca_cert.get_subject() ) server_cert.set_issuer( ca_cert.get_subject() ) #print ( server_cert.get_issuer() ) server_cert.set_pubkey( ca_cert.get_pubkey() ) # sign the certificate using ca_cert key #server_cert.sign ( ca_cert.get_pubkey(), 'sha256' ) #key = crypto.load_privatekey ( crypto.FILETYPE_PEM, ca_cert ) server_cert.sign ( key, 'sha256' ) # dump new certificate in directory open(join(cert_dir, CERTFILE), "wb").write( crypto.dump_certificate(crypto.FILETYPE_PEM, server_cert)) return CERTFILE
async def post(self, request):
global _users
try:
req = request.json
user = await Users.get_first('email', req.get('email', ''))
if not user:
return json({'msg': 'User not found'}, status=404)
if not user.active:
return json({'msg': 'User not active'}, status=404)
if hash_string(req.get('password', 'x')) == user.password:
user.session_uuid = str(uuid4()).replace('-', '')
user.last_login = datetime.utcnow()
await user.update()
_users[user.session_uuid] = user
return json({
'success': True,
'admin': user.admin,
'mentor': user.mentor,
'name': user.name,
'email': user.email,
'surname': user.surname,
'lang': user.lang,
'organiser': user.organiser,
'id': user.id,
'session_uuid': user.session_uuid,
'confirmation': user.confirmation
})
else:
return self.user_error
except DoesNotExist:
return self.user_error
except:
logging.exception('err authentication.post')
return json({'msg': 'internal error'}, status=500)
def store_alerts(self, alert):
pre_hash_string = alert["mash_expression"] + alert["active_since"]
hash_key = utils.hash_string(pre_hash_string)
alert["hash_key"] = hash_key
new_entry = TSAlertsJSON(id=hash_key, json=alert, active_until=None)
new_entry.put()
memcache.set(hash_key, alert)
async def post(self, request, current_user):
req = request.json
link = 'https://{}/api/workshopabsence/'.format(request.host)
if req['email_type'] == 'EmailCustom':
users = await Users.get_by_many_field_value(**req['recipients'])
subject = req['subject']
text = req['text'].format()
await send_email(
recipients=[u.email for u in users],
text=text,
subject=subject
)
elif req['email_type'] == 'EmailTooLate':
users = await Users.get_by_many_field_value(**req['recipients'])
for user in users:
user.confirmation = 'rej_time'
await user.update()
email_data = {
"name": user.name
}
subject = req['subject']
text = req['text'].format(**email_data)
await send_email(
recipients=[user.email],
text=text,
subject=subject
)
await asyncio.sleep(0.05)
elif req['email_type'] == "per_user":
users = await Users.get_by_many_field_value(**req['recipients'])
for user in users:
uhash = hash_string(user.name + str(user.id) + user.email)
email_data = {
"link_yes": link + str(user.id) + '/' + uhash + '/' + 'yes',
"link_no": link + str(user.id) + '/' + uhash + '/' + 'no',
"name": user.name,
"what_can_you_bring": user.what_can_you_bring
}
subject = req['subject']
text = req['text'].format(email_data)
await send_email(
recipients=user.email,
text=text,
subject=subject
)
else:
users = await Users.get_by_many_field_value(**req['recipients'])
recip = []
for user in users:
recip.append(user.email)
subject = req['subject']
text = req['text']
await send_email(
recipients=recip,
text=text,
subject=subject
)
return json({'success': True, 'count': len(users)})
def enqueue_song(self, song_hash):
self._last_hb_ts = time.time() * MICROSECONDS
content = utils.unserialize_response(request.get_data())
command_epoch = content['command_epoch']
master_post_hash = content['hashed_post_playlist']
master_current_song = content['current_song']
failover_resp = utils.format_rpc_response(False, ENQUEUE, {}, \
msg='Replica in recovery mode', \
command_epoch=command_epoch)
if self._in_recovery:
return utils.serialize_response(failover_resp)
print "In Enqueue"
print "Enqueue: " + str(self._current_song)
replica_pre_hash = utils.hash_string(pickle.dumps(
self._playlist_queue))
if replica_pre_hash == master_post_hash and self._current_song == master_current_song:
print "Already Performed Operation in Enqueue"
repeat_resp = utils.format_rpc_response(False, ENQUEUE, {}, \
msg='Already performed operation', \
command_epoch=command_epoch)
return utils.serialize_response(repeat_resp)
# Do enqueue, check for failover mode
song_not_exist = not os.path.exists(utils.get_music_path(song_hash))
self._playlist_queue.append(song_hash)
replica_post_hash = utils.hash_string(
pickle.dumps(self._playlist_queue))
inconsistent_queue = master_post_hash != replica_post_hash or \
master_current_song != self._current_song
print "queue hashes match: " + str(
master_post_hash == replica_post_hash)
print "current song matches: " + str(
master_current_song) + " == " + str(self._current_song)
replica_failover = song_not_exist or inconsistent_queue
if replica_failover:
self._in_recovery = True
return utils.serialize_response(failover_resp)
master_time = content['time']
self._master_timestamp = master_time
resp = utils.format_rpc_response(True, ENQUEUE, {'enqueued': True}, \
command_epoch=command_epoch)
print str(resp)
return utils.serialize_response(resp)
def check_string_key(self,
string_key,
hash_method="sha256",
text_encoding="utf-8"):
storage_hash_with_given_key, _storage_hash_salt = utils.hash_string(
string_key,
salt=self.storage_hash_salt,
hash_method=hash_method,
text_encoding=text_encoding)
if storage_hash_with_given_key != self.storage_hash:
raise InvalidKeyError(string_key)
def dequeue_song(self):
self._last_hb_ts = time.time() * MICROSECONDS
content = utils.unserialize_response(request.get_data())
command_epoch = content['command_epoch']
master_post_hash = content['hashed_post_playlist']
master_current_song = content['current_song']
failover_resp = utils.format_rpc_response(False, DEQUEUE, {}, \
msg='Replica in recovery mode', \
command_epoch=command_epoch)
if self._in_recovery:
return utils.serialize_response(failover_resp)
print "In Dequeue"
replica_pre_hash = utils.hash_string(pickle.dumps(
self._playlist_queue))
if replica_pre_hash == master_post_hash and self._current_song == master_current_song and self._current_song != None:
repeat_resp = utils.format_rpc_response(False, DEQUEUE, {}, \
msg='Already performed operation', \
command_epoch=command_epoch)
return utils.serialize_response(repeat_resp)
# Check for length 0 queue
if len(self._playlist_queue) == 0:
self._current_song = None
else:
self._current_song = self._playlist_queue.popleft()
replica_post_hash = utils.hash_string(
pickle.dumps(self._playlist_queue))
if (replica_post_hash != master_post_hash
or self._current_song != master_current_song):
self._in_recovery = True
return utils.serialize_response(failover_resp)
master_time = content['time']
self._master_timestamp = master_time
resp = utils.format_rpc_response(True, DEQUEUE, {}, \
msg='Successfully dequeued', \
command_epoch=command_epoch)
print "end of dequeue: current song for replica: " + str(
self._current_song)
return utils.serialize_response(resp)
def forward(self, return_status=True, play=True):
song = None
success_response = utils.format_client_response(\
True, FORWARD, {}, \
client_req_id=self._client_req_id)
# No song in future and no song currently playing, nothing to do.
if len(self._playlist_queue) == 0 and self._current_song == None:
if return_status:
self._status_queue.put(success_response)
return
# After a forward command we are always at the start of a song
self._current_offset = 0
# No songs to play anymore
if len(self._playlist_queue) == 0:
print "forward: no songs in playlist"
self._current_song = None
with open(PLAYLIST_STATE_FILE, 'w') as f:
data = utils.format_playlist_state(self._playlist_queue,
self._current_song)
f.write(data)
# Pop out a song to play
else:
print "forward: popping song"
self._current_song = self._playlist_queue.popleft()
with open(PLAYLIST_STATE_FILE, 'w') as f:
data = utils.format_playlist_state(self._playlist_queue,
self._current_song)
f.write(data)
hashed_post_playlist = utils.hash_string(
pickle.dumps(self._playlist_queue))
# Synchronizes dequeue operation across all replicas (for master recovery)
rpc_data = {'hashed_post_playlist': hashed_post_playlist, \
'current_song' : self._current_song, \
'time': time.time() }
# Try indefinitely until we get at least f+1 responses
# Guaranteed RPC won't add to queue since new command_epoch prevents
# Holding mutexes just in case
self.exponential_backoff(rpc_data, DEQUEUE, \
DEQUEUE_URL, \
REPLICA_ACK_TIMEOUT)
# Start playing the next song
# (if current_song == None then will just stop playing music)
if play:
self.play(False)
if return_status:
self._status_queue.put(success_response)
def enqueue_song(song_path):
load_song(song_path)
assert(os.path.exists(song_path))
with open(song_path, 'r') as f:
song_bytes = f.read()
song_hash = utils.hash_string(song_bytes)
url = get_url(ENQUEUE) + "/" + song_hash
try:
r = urllib2.urlopen(url)
except Exception:
print "Error in Enqueue Song"
else:
master_response = utils.unserialize_response(r.read())
if master_response['success'] == True:
print song_path + ' has been enqueued'
else:
print song_path + ' cannot be enqueued'
print master_response['client_req_id']
def enqueue_song(song_path):
load_song(song_path)
assert (os.path.exists(song_path))
with open(song_path, 'r') as f:
song_bytes = f.read()
song_hash = utils.hash_string(song_bytes)
url = get_url(ENQUEUE) + "/" + song_hash
try:
r = urllib2.urlopen(url)
except Exception:
print "Error in Enqueue Song"
else:
master_response = utils.unserialize_response(r.read())
if master_response['success'] == True:
print song_path + ' has been enqueued'
else:
print song_path + ' cannot be enqueued'
print master_response['client_req_id']
async def post(self, request, current_user):
data = request.json
mail_data = data['mail']
users = await Users.get_by_many_field_value(
**data['recipients']['conditions'])
if mail_data['per_user']:
for user in users:
if mail_data['email_type'] == 'EmailTooLate':
user.confirmation = 'rej_time'
await user.update()
email_data = {"name": user.name}
else:
link = 'https://{}/api/event/absence/{}/{}/'.format(
request.host, str(user.id),
hash_string(user.name + str(user.id) + user.email))
email_data = {
"link_yes": link + 'yes',
"link_no": link + 'no',
"name": user.name,
"what_can_you_bring": user.what_can_you_bring
}
subject = mail_data['subject']
text = mail_data['text'].format(**email_data)
await send_email(recipients=[user.email],
text=text,
subject=subject)
await asyncio.sleep(0.03)
else:
subject = mail_data['subject']
text = mail_data['text']
for x in range(len(users) // 50):
await send_email(
recipients=[u.email for u in users[x * 50:(x + 1) * 50]],
text=text,
subject=subject)
await asyncio.sleep(0.03)
return json({
'success':
True,
'msg':
"Send: {} e-mail to {}".format(len(users),
data['recipients']['name'])
})
def use_existing(string_key,
serialized_format,
hash_method="sha256",
delimiter=':',
text_encoding="utf-8"):
crypto_data = HashBasedCryptoDataBidirSerializer.deserialize(
serialized_format,
delimiter=delimiter,
text_encoding=text_encoding)
crypto_data.check_string_key(string_key,
hash_method=hash_method,
text_encoding=text_encoding)
# We now know that the given key is valid! generate the encryption hash using the given key and stored salt
crypto_data.encryption_hash, _salt = utils.hash_string(
string_key,
salt=crypto_data.encryption_hash_salt,
hash_method=hash_method,
text_encoding=text_encoding)
crypto_data.iv_is_from_deserializer = True # Helps keep the programmer from accidentally overwriting the IV by deserializing and then calling encrypt before decrypt
return crypto_data
def enqueue_song(self, params):
song_hash = params['song_hash']
self.load_song(params)
self._playlist_queue.append(song_hash)
with open(PLAYLIST_STATE_FILE, 'w') as f:
data = utils.format_playlist_state(self._playlist_queue,
self._current_song)
f.write(data)
hashed_post_playlist = utils.hash_string(
pickle.dumps(self._playlist_queue))
rpc_data = {'current_song': self._current_song, \
'hashed_post_playlist': hashed_post_playlist, \
'time': time.time() }
self.exponential_backoff(rpc_data, ENQUEUE, \
ENQUEUE_URL + '/' + song_hash, \
REPLICA_ACK_TIMEOUT)
self._status_queue.put(utils.format_client_response(\
True, ENQUEUE, {}, \
client_req_id=self._client_req_id))
def enqueue(song_file):
global playlist
playlist.append(song_file)
return 'songthing'
song_path = MUSIC_DIR + song_file
load_song(song_path)
assert(os.path.exists(song_path))
with open(song_path, 'r') as f:
song_bytes = f.read()
song_hash = utils.hash_string(song_bytes)
url = get_url(ENQUEUE) + "/" + song_hash
try:
r = urllib2.urlopen(url)
except Exception:
print "Error in Enqueue Song"
else:
master_response = utils.unserialize_response(r.read())
if master_response['success'] == True:
print song_path + ' has been enqueued'
playlist.append(song_file)
else:
print song_path + ' cannot be enqueued'
print master_response['client_req_id']
return 'enqueue'
def enqueue(song_file):
global playlist
playlist.append(song_file)
return 'songthing'
song_path = MUSIC_DIR + song_file
load_song(song_path)
assert (os.path.exists(song_path))
with open(song_path, 'r') as f:
song_bytes = f.read()
song_hash = utils.hash_string(song_bytes)
url = get_url(ENQUEUE) + "/" + song_hash
try:
r = urllib2.urlopen(url)
except Exception:
print "Error in Enqueue Song"
else:
master_response = utils.unserialize_response(r.read())
if master_response['success'] == True:
print song_path + ' has been enqueued'
playlist.append(song_file)
else:
print song_path + ' cannot be enqueued'
print master_response['client_req_id']
return 'enqueue'
def get_block(self, block_hash):
for block in self.blocks:
if hash_string(str(block)) == block_hash:
return block
return None
def main(args):
####################
# Arguments
gpu = args.gpu
model_name = args.model
initial_tree_sampling = args.initial_tree_sampling
path_config = args.config
data_augmentation = args.data_augmentation
trial_name = args.name
actiontype = args.actiontype
max_epoch = args.max_epoch
dev_size = args.dev_size
# Check
assert actiontype in ["train", "evaluate"]
if actiontype == "train":
assert max_epoch > 0
assert len(initial_tree_sampling.split("_")) == 3
for type_ in initial_tree_sampling.split("_"):
assert type_ in ["X", "BU", "TD", "RB", "LB", "RB2"]
assert initial_tree_sampling.split("_")[2] != "X"
assert initial_tree_sampling.split("_")[1] != "RB2"
assert initial_tree_sampling.split("_")[2] != "RB2"
if trial_name is None or trial_name == "None":
trial_name = utils.get_current_time()
####################
# Path setting
config = utils.Config(path_config)
basename = "%s.%s.%s.aug_%s.%s" \
% (model_name,
initial_tree_sampling,
utils.get_basename_without_ext(path_config),
data_augmentation,
trial_name)
if actiontype == "train":
path_log = os.path.join(config.getpath("results"),
basename + ".training.log")
elif actiontype == "evaluate":
path_log = os.path.join(config.getpath("results"),
basename + ".evaluation.log")
path_train = os.path.join(config.getpath("results"),
basename + ".training.jsonl")
path_valid = os.path.join(config.getpath("results"),
basename + ".validation.jsonl")
path_snapshot = os.path.join(config.getpath("results"),
basename + ".model")
path_pred = os.path.join(config.getpath("results"),
basename + ".evaluation.ctrees")
path_eval = os.path.join(config.getpath("results"),
basename + ".evaluation.json")
utils.set_logger(path_log)
####################
# Random seed
random_seed = trial_name
random_seed = utils.hash_string(random_seed)
random.seed(random_seed)
np.random.seed(random_seed)
cuda.cupy.random.seed(random_seed)
####################
# Log so far
utils.writelog("gpu=%d" % gpu)
utils.writelog("model_name=%s" % model_name)
utils.writelog("initial_tree_sampling=%s" % initial_tree_sampling)
utils.writelog("path_config=%s" % path_config)
utils.writelog("data_augmentation=%s" % data_augmentation)
utils.writelog("trial_name=%s" % trial_name)
utils.writelog("actiontype=%s" % actiontype)
utils.writelog("max_epoch=%s" % max_epoch)
utils.writelog("dev_size=%s" % dev_size)
utils.writelog("path_log=%s" % path_log)
utils.writelog("path_train=%s" % path_train)
utils.writelog("path_valid=%s" % path_valid)
utils.writelog("path_snapshot=%s" % path_snapshot)
utils.writelog("path_pred=%s" % path_pred)
utils.writelog("path_eval=%s" % path_eval)
utils.writelog("random_seed=%d" % random_seed)
####################
# Data preparation
begin_time = time.time()
train_dataset = dataloader.read_rstdt("train",
relation_level="coarse-grained",
with_root=False)
test_dataset = dataloader.read_rstdt("test",
relation_level="coarse-grained",
with_root=False)
vocab_word = utils.read_vocab(
os.path.join(config.getpath("data"), "rstdt-vocab", "words.vocab.txt"))
vocab_postag = utils.read_vocab(
os.path.join(config.getpath("data"), "rstdt-vocab",
"postags.vocab.txt"))
vocab_deprel = utils.read_vocab(
os.path.join(config.getpath("data"), "rstdt-vocab",
"deprels.vocab.txt"))
if data_augmentation:
external_train_dataset = dataloader.read_ptbwsj_wo_rstdt(
with_root=False)
# Remove documents with only one leaf node
external_train_dataset = utils.filter_dataset(
external_train_dataset,
condition=lambda data: len(data.edu_ids) > 1)
end_time = time.time()
utils.writelog("Loaded the corpus. %f [sec.]" % (end_time - begin_time))
####################
# Hyper parameters
word_dim = config.getint("word_dim")
postag_dim = config.getint("postag_dim")
deprel_dim = config.getint("deprel_dim")
lstm_dim = config.getint("lstm_dim")
mlp_dim = config.getint("mlp_dim")
n_init_epochs = config.getint("n_init_epochs")
negative_size = config.getint("negative_size")
batch_size = config.getint("batch_size")
weight_decay = config.getfloat("weight_decay")
gradient_clipping = config.getfloat("gradient_clipping")
optimizer_name = config.getstr("optimizer_name")
utils.writelog("word_dim=%d" % word_dim)
utils.writelog("postag_dim=%d" % postag_dim)
utils.writelog("deprel_dim=%d" % deprel_dim)
utils.writelog("lstm_dim=%d" % lstm_dim)
utils.writelog("mlp_dim=%d" % mlp_dim)
utils.writelog("n_init_epochs=%d" % n_init_epochs)
utils.writelog("negative_size=%d" % negative_size)
utils.writelog("batch_size=%d" % batch_size)
utils.writelog("weight_decay=%f" % weight_decay)
utils.writelog("gradient_clipping=%f" % gradient_clipping)
utils.writelog("optimizer_name=%s" % optimizer_name)
####################
# Model preparation
cuda.get_device(gpu).use()
# Initialize a model
utils.mkdir(os.path.join(config.getpath("data"), "caches"))
path_embed = config.getpath("pretrained_word_embeddings")
path_caches = os.path.join(
config.getpath("data"), "caches",
"cached." + os.path.basename(path_embed) + ".npy")
if os.path.exists(path_caches):
utils.writelog("Loading cached word embeddings ...")
initialW = np.load(path_caches)
else:
initialW = utils.read_word_embedding_matrix(path=path_embed,
dim=word_dim,
vocab=vocab_word,
scale=0.0)
np.save(path_caches, initialW)
if model_name == "spanbasedmodel":
# Span-based model w/ template features
template_feature_extractor = models.TemplateFeatureExtractor(
dataset=train_dataset)
utils.writelog("Template feature size=%d" %
template_feature_extractor.feature_size)
if actiontype == "train":
for template in template_feature_extractor.templates:
dim = template_feature_extractor.template2dim[template]
utils.writelog("Template feature #%s %s" % (dim, template))
model = models.SpanBasedModel(
vocab_word=vocab_word,
vocab_postag=vocab_postag,
vocab_deprel=vocab_deprel,
word_dim=word_dim,
postag_dim=postag_dim,
deprel_dim=deprel_dim,
lstm_dim=lstm_dim,
mlp_dim=mlp_dim,
initialW=initialW,
template_feature_extractor=template_feature_extractor)
elif model_name == "spanbasedmodel2":
# Span-based model w/o template features
model = models.SpanBasedModel2(vocab_word=vocab_word,
vocab_postag=vocab_postag,
vocab_deprel=vocab_deprel,
word_dim=word_dim,
postag_dim=postag_dim,
deprel_dim=deprel_dim,
lstm_dim=lstm_dim,
mlp_dim=mlp_dim,
initialW=initialW)
else:
raise ValueError("Invalid model_name=%s" % model_name)
utils.writelog("Initialized the model ``%s''" % model_name)
# Load pre-trained parameters
if actiontype != "train":
serializers.load_npz(path_snapshot, model)
utils.writelog("Loaded trained parameters from %s" % path_snapshot)
model.to_gpu(gpu)
####################
# Decoder preparation
decoder = decoders.IncrementalCKYDecoder()
####################
# Initializer preparation
sampler = treesamplers.TreeSampler(initial_tree_sampling.split("_"))
####################
# Training / evaluation
if actiontype == "train":
with chainer.using_config("train", True):
if dev_size > 0:
# Training with cross validation
train_dataset, dev_dataset = utils.split_dataset(
dataset=train_dataset, n_dev=dev_size, seed=None)
with open(
os.path.join(config.getpath("results"),
basename + ".valid_gold.ctrees"),
"w") as f:
for data in dev_dataset:
f.write("%s\n" % " ".join(data.nary_sexp))
else:
# Training with the full training set
dev_dataset = None
if data_augmentation:
train_dataset = np.concatenate(
[train_dataset, external_train_dataset], axis=0)
train(model=model,
decoder=decoder,
sampler=sampler,
max_epoch=max_epoch,
n_init_epochs=n_init_epochs,
negative_size=negative_size,
batch_size=batch_size,
weight_decay=weight_decay,
gradient_clipping=gradient_clipping,
optimizer_name=optimizer_name,
train_dataset=train_dataset,
dev_dataset=dev_dataset,
path_train=path_train,
path_valid=path_valid,
path_snapshot=path_snapshot,
path_pred=os.path.join(config.getpath("results"),
basename + ".valid_pred.ctrees"),
path_gold=os.path.join(config.getpath("results"),
basename + ".valid_gold.ctrees"))
elif actiontype == "evaluate":
with chainer.using_config("train", False), chainer.no_backprop_mode():
# Test
parse(model=model,
decoder=decoder,
dataset=test_dataset,
path_pred=path_pred)
scores = metrics.rst_parseval(
pred_path=path_pred,
gold_path=os.path.join(config.getpath("data"), "rstdt", "wsj",
"test", "gold.labeled.nary.ctrees"))
old_scores = metrics.old_rst_parseval(
pred_path=path_pred,
gold_path=os.path.join(config.getpath("data"), "rstdt", "wsj",
"test", "gold.labeled.nary.ctrees"))
out = {
"Morey2018": {
"Unlabeled Precision": scores["S"]["Precision"] * 100.0,
"Precision_info": scores["S"]["Precision_info"],
"Unlabeled Recall": scores["S"]["Recall"] * 100.0,
"Recall_info": scores["S"]["Recall_info"],
"Micro F1": scores["S"]["Micro F1"] * 100.0
},
"Marcu2000": {
"Unlabeled Precision":
old_scores["S"]["Precision"] * 100.0,
"Precision_info": old_scores["S"]["Precision_info"],
"Unlabeled Recall": old_scores["S"]["Recall"] * 100.0,
"Recall_info": old_scores["S"]["Recall_info"],
"Micro F1": old_scores["S"]["Micro F1"] * 100.0
}
}
utils.write_json(path_eval, out)
utils.writelog(utils.pretty_format_dict(out))
utils.writelog("Done: %s" % basename)
def election_reconnect_or_fail(self):
# first notify election service that we are eligible to cast votes
# after 0.5 seconds if master hasnt been elected, try to get votes ourselves
# finally, if we fail to get votes after 1 second, assume we are partitioned
# and go into failure mode
my_time = time.time()
print "writing playlist state to file: " + str(my_time)
with open(PLAYLIST_STATE_FILE, 'w') as f:
data = utils.format_playlist_state(self._parent._playlist_queue, self._parent._current_song, \
self._parent._master_term, self._parent._master_timestamp)
f.write(data)
print "notifying election service of failure"
replica_url = 'http://' + self._parent._ip + ":" + VOTE_PORT + FAIL_URL
r = RPC(self, VOTE, url=replica_url, ip=self._parent._ip, data={})
r.start()
time.sleep(0.5)
if (time.time()*MICROSECONDS - self._parent._last_hb_ts) > self._timeout_threshold:
print "failed to elect new master, requesting votes"
queue_hash = utils.hash_string(pickle.dumps(self._parent._playlist_queue))
status_update_dict = { "request_votes" : True, "queue_hash" : queue_hash, \
"current_song": self._parent._current_song, "term":self._parent._master_term,\
"timestamp" : self._parent._master_timestamp}
self._parent._state_queue.put(status_update_dict)
print "waiting for response"
for i in range(0, 4):
try:
resp = self._parent._response_queue.get(True, 0.3)
print "got response: " + str(resp)
if "success" in resp:
self.get_new_timeout_threshold()
my_time = time.time()
print "won election, starting master process : " + str(my_time)
os.system("./master.py -r " + str(self._parent._master_term + 1))
return
else:
continue
except Queue.Empty:
continue
else:
replica_url = 'http://' + self._parent._ip + ":" + VOTE_PORT + UNFAIL_URL
r = RPC(self, VOTE, url=replica_url, ip=self._parent._ip, data={})
r.start()
# wait for others to try to get elected again
for i in range (0, 20):
time.sleep(0.25)
# check last time for new master
if (time.time()*MICROSECONDS - self._parent._last_hb_ts) <= self._timeout_threshold:
my_time = time.time()
print "someone won election: " + str(my_time)
replica_url = 'http://' + self._parent._ip + ":" + VOTE_PORT + UNFAIL_URL
r = RPC(self, VOTE, url=replica_url, ip=self._parent._ip, data={})
r.start()
status_update_dict = { "reset_election" : True }
self._parent._state_queue.put(status_update_dict)
return
# if we get here, we should assume we have failed and enter failure mode
print "FAILED: going into recovery mode"
self.get_new_timeout_threshold()
self._parent._in_recovery = True
pygame.mixer.music.stop()
def test_hash_string():
expected_hash = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
assert hash_string('test') == expected_hash
async def create(self):
self.password = hash_string(self.password)
return await super().create()
async def set_password(self, password):
self.password = hash_string(password)
def main(args):
####################
# Arguments
tree_sampling = args.tree_sampling # NOTE
trial_name = args.name
# Check
assert len(tree_sampling.split("_")) == 3
for type_ in tree_sampling.split("_"):
assert type_ in ["X", "BU", "TD", "RB", "LB", "RB2"]
assert tree_sampling.split("_")[2] != "X"
assert tree_sampling.split("_")[1] != "RB2"
assert tree_sampling.split("_")[2] != "RB2"
if trial_name is None or trial_name == "None":
trial_name = utils.get_current_time()
####################
# Path setting
config = utils.Config()
basename = "%s.%s" \
% (tree_sampling,
trial_name)
utils.mkdir(os.path.join(config.getpath("results"), "baselines"))
path_log = os.path.join(config.getpath("results"), "baselines",
basename + ".evaluation.log")
path_pred = os.path.join(config.getpath("results"), "baselines",
basename + ".evaluation.ctrees")
path_eval = os.path.join(config.getpath("results"), "baselines",
basename + ".evaluation.json")
utils.set_logger(path_log)
####################
# Random seed
random_seed = trial_name
random_seed = utils.hash_string(random_seed)
np.random.seed(random_seed)
cuda.cupy.random.seed(random_seed)
####################
# Log so far
utils.writelog("tree_sampling=%s" % tree_sampling)
utils.writelog("trial_name=%s" % trial_name)
utils.writelog("path_log=%s" % path_log)
utils.writelog("path_pred=%s" % path_pred)
utils.writelog("path_eval=%s" % path_eval)
utils.writelog("random_seed=%d" % random_seed)
####################
# Data preparation
begin_time = time.time()
test_databatch = dataloader.read_rstdt("test",
relation_level="coarse-grained",
with_root=False)
end_time = time.time()
utils.writelog("Loaded the corpus. %f [sec.]" % (end_time - begin_time))
####################
# Tree-sampler preparation
sampler = treesamplers.TreeSampler(tree_sampling.split("_")) # NOTE
with chainer.using_config("train", False), chainer.no_backprop_mode():
parse(sampler=sampler, databatch=test_databatch, path_pred=path_pred)
scores = rst_parseval.evaluate(
pred_path=path_pred,
gold_path=os.path.join(config.getpath("data"), "rstdt", "renamed",
"test.labeled.nary.ctrees"))
old_scores = old_rst_parseval.evaluate(
pred_path=path_pred,
gold_path=os.path.join(config.getpath("data"), "rstdt", "renamed",
"test.labeled.nary.ctrees"))
out = {
"Morey2018": {
"Unlabeled Precision": scores["S"]["Precision"] * 100.0,
"Precision_info": scores["S"]["Precision_info"],
"Unlabeled Recall": scores["S"]["Recall"] * 100.0,
"Recall_info": scores["S"]["Recall_info"],
"Micro F1": scores["S"]["Micro F1"] * 100.0
},
"Marcu2000": {
"Unlabeled Precision": old_scores["S"]["Precision"] * 100.0,
"Precision_info": old_scores["S"]["Precision_info"],
"Unlabeled Recall": old_scores["S"]["Recall"] * 100.0,
"Recall_info": old_scores["S"]["Recall_info"],
"Micro F1": old_scores["S"]["Micro F1"] * 100.0
}
}
utils.write_json(path_eval, out)
utils.writelog(utils.pretty_format_dict(out))
utils.writelog("Done.")
