def __init__(self, server, url="", username="", password=""):
Instance.__init__(self,server)
self.instanceTypeName = "skarphed"
self.displayName = "Skarphed"
self.state = self.STATE_OFFLINE
self.scv_loggedin = self.SCV_LOCKED
self.load = self.LOADED_NONE
self.data = {}
self.url = url
self.username = username
self.password = password
self.publickey = None
self.repo_url = ""
self.users = None
self.template = None
self.roles = None
self.modules = None
self.sites = None
self.operationManager = None
self.operationDaemon = None
self.maintenance_mode = True
self.rendermode = None
self.cssPropertySet = None
def __init__(self, server, url="", username="", password=""):
Instance.__init__(self,server)
self.instanceTypeName = "skarphed_repo"
self.displayName = "Skarphed Repository"
self.state = self.STATE_OFFLINE
self.loggedin = self.SCV_LOCKED
self.load = self.LOADED_NONE
self.data = {'modules':[], 'developers':[], 'templates':[]}
self.url = url
self.username = username
self.password = password
self.newPassword = None
def __init__(self, server, url="", username="", password=""):
Instance.__init__(self, server)
self.instanceTypeName = "skarphed_repo"
self.displayName = "Skarphed Repository"
self.state = self.STATE_OFFLINE
self.loggedin = self.SCV_LOCKED
self.load = self.LOADED_NONE
self.data = {'modules': [], 'developers': [], 'templates': []}
self.url = url
self.username = username
self.password = password
self.newPassword = None
def __init__(self, kd_train_file_p, kd_train_file_h, ca_train_file_p, ca_train_file_h, kd_dev_file_p,
kd_dev_file_h, ca_dev_file_p, ca_dev_file_h, config):
self.kd_p_train_insts = []
self.kd_h_train_insts = []
self.ca_p_train_insts = []
self.ca_h_train_insts = []
self.kd_p_dev_insts = []
self.kd_h_dev_insts = []
self.ca_p_dev_insts = []
self.ca_h_dev_insts = []
self.config = config
self.words = []
self.word_alphabet = None
self.char_alphabet = {}
# self.adjunct_roles, self.core_roles = self.split_srl_labels(srl_labels, config.include_c_v)
# self.srl_labels_inv = [""] + self.adjunct_roles + self.core_roles
# self.srl_labels = {l: i for i, l in enumerate(self.srl_labels_inv)}
self.srl_labels_dic = {'': 0}
srl_label_dic_index = 1
# build char dict
with open(config.char_vocab_path, 'r', encoding='utf8') as f:
self.char_alphabet[unk_key] = 0
self.char_alphabet[padding_key] = 1
for idx, char in enumerate(f.readlines()):
char = char.strip()
if char not in self.char_alphabet:
self.char_alphabet[char] = idx+1
# read json into inst
for idx, path in enumerate([kd_train_file_p, kd_train_file_h, ca_train_file_p, ca_train_file_h, kd_dev_file_p,
kd_dev_file_h, ca_dev_file_p, ca_dev_file_h]):
with open(path, 'r', encoding='utf8') as f:
doc_num = 0
sent_num = 0
info_list = []
for line in f.readlines():
line = line.strip()
if len(line) == 0 and len(info_list) != 0:
srl, sentence = self.build_inst(info_list)
info_list = []
doc_num += 1
sent_num += 1
char_list = []
for word in sentence:
char_list.append(list(word))
gold_predicates = self.get_all_predicates(srl)
# label
pre_starts, pre_ends, arg_starts, arg_ends, arg_labels = self.tensorize_srl_relations(srl)
srl_rels = {}
for r in srl:
pred_id = str([r[0], r[1]])
if pred_id not in srl_rels:
srl_rels[pred_id] = []
srl_rels[pred_id].append((r[2], r[3], r[4]))
if idx == 0:
for arg_label in arg_labels:
if arg_label not in self.srl_labels_dic:
self.srl_labels_dic[arg_label] = srl_label_dic_index
srl_label_dic_index += 1
inst = Instance(sentence, srl, char_list, gold_predicates, pre_starts, pre_ends, arg_starts, arg_ends, arg_labels, srl_rels)
inst.default()
self.words.extend(sentence)
if idx == 0:
self.kd_p_train_insts.append(inst)
elif idx == 1:
self.kd_h_train_insts.append(inst)
elif idx == 2:
self.ca_p_train_insts.append(inst)
elif idx == 3:
self.ca_h_train_insts.append(inst)
elif idx == 4:
self.kd_p_dev_insts.append(inst)
elif idx == 5:
self.kd_h_dev_insts.append(inst)
elif idx == 6:
self.ca_p_dev_insts.append(inst)
elif idx == 7:
self.ca_h_dev_insts.append(inst)
else:
info_list.append(line)
if len(info_list) != 0:
srl, sentence = self.build_inst(info_list)
info_list = []
doc_num += 1
sent_num += 1
char_list = []
for word in sentence:
char_list.append(list(word))
gold_predicates = self.get_all_predicates(srl)
# label
pre_starts, pre_ends, arg_starts, arg_ends, arg_labels = self.tensorize_srl_relations(srl)
srl_rels = {}
for r in srl:
pred_id = str([r[0], r[1]])
if pred_id not in srl_rels:
srl_rels[pred_id] = []
srl_rels[pred_id].append((r[2], r[3], r[4]))
if idx == 0:
for arg_label in arg_labels:
if arg_label not in self.srl_labels_dic:
self.srl_labels_dic[arg_label] = srl_label_dic_index
srl_label_dic_index += 1
inst = Instance(sentence, srl, char_list, gold_predicates, pre_starts, pre_ends, arg_starts,
arg_ends, arg_labels, srl_rels)
inst.default()
self.words.extend(sentence)
if idx == 0:
self.kd_p_train_insts.append(inst)
elif idx == 1:
self.kd_h_train_insts.append(inst)
elif idx == 2:
self.ca_p_train_insts.append(inst)
elif idx == 3:
self.ca_h_train_insts.append(inst)
elif idx == 4:
self.kd_p_dev_insts.append(inst)
elif idx == 5:
self.kd_h_dev_insts.append(inst)
elif idx == 6:
self.ca_p_dev_insts.append(inst)
elif idx == 7:
self.ca_h_dev_insts.append(inst)
print('{} has {} docs, {} sentences.'.format(path, doc_num, sent_num))
if idx == 7:
self.dict = self.buildDictionary(self.words)
self.word_alphabet = self.build_vab(self.dict, need_pad_unk=True)
if idx == 7:
self.wordChar2Id(self.kd_p_train_insts)
self.wordChar2Id(self.kd_h_train_insts)
self.wordChar2Id(self.ca_p_train_insts)
self.wordChar2Id(self.ca_h_train_insts)
self.wordChar2Id(self.kd_p_dev_insts)
self.wordChar2Id(self.kd_h_dev_insts)
self.wordChar2Id(self.ca_p_dev_insts)
self.wordChar2Id(self.ca_h_dev_insts)
# self.save_inst_sentence(self.p_train_insts, 'orl/train.json')
# self.save_inst_sentence(self.h_train_insts, 'orl/train.json')
# self.save_inst_sentence(self.p_dev_insts, 'orl/dev.json')
# self.save_inst_sentence(self.h_dev_insts, 'orl/dev.json')
# self.save_inst_sentence(self.p_test_insts, 'orl/test.json')
# self.save_inst_sentence(self.h_test_insts, 'orl/test.json')
print('word dictionary size:{} label size:{}'.format(len(self.word_alphabet.id2string), len(self.srl_labels_dic)))
