def readInstances(self, path, maxInst = -1):
insts = []
r = open(path, encoding='utf8')
info = []
inst = Instance()
for line in r.readlines():
line = line.strip()
if line == "" and len(inst.m_char) != 0:
if (maxInst == -1) or (maxInst > len(insts)):
inst.m_word = get_words(inst.m_char, inst.m_label)
insts.append(inst)
else:
return insts
inst = Instance()
else:
info = line.split(" ")
if len(info) != 3:
print("error format")
uni_char = unicodedata.normalize('NFKC', info[0])
inst.m_char.append(uni_char)
bi_char = unicodedata.normalize('NFKC', info[1][4:])
inst.m_bichar.append(bi_char)
inst.m_label.append(info[2])
r.close()
if len(inst.m_char) != 0:
insts.append(inst)
return insts
def build(self, image, mask_result):
result = []
# check channels to be 4 (b, g, r, alpha)
height, width, channels = image.shape
if channels != 4:
temp = np.full((height, width, 4), 255, dtype=np.uint8)
temp[:, :, :3] = image
image = temp
for mask in mask_result:
root = Group()
root.add_node(Instance(image, "background", (0, 0)))
for i in range(len(mask["class_ids"])):
if mask["scores"][i] < self.threshold:
continue
top, left, down, right = mask["rois"][i]
class_name = self.to_class_name(mask["class_ids"][i])
splitted_image = self.masking(
image[top:down, left:right, :],
mask["masks"][top:down, left:right, i])
image[top:down, left:right, :] = self.masking(image[top:down, left:right, :], \
np.logical_not(mask["masks"][top:down, left:right, i]))
root.add_node(Instance(splitted_image, class_name,
(left, top)))
result.append(root)
return result
def generate_date(self):
self.instances = []
for country in self.countries:
if country == 'Not_Stamp':
files = glob.glob(self.dataset_folder + os.path.sep + country + os.path.sep + "*.*")
for file in files:
self.instances.append(Instance(file, country.ljust(15), '-1000'))
else:
for year in self.years:
files = glob.glob(self.dataset_folder + os.path.sep + country + os.path.sep + year + os.path.sep + "*.*")
for file in files:
self.instances.append(Instance(file, country.ljust(15), year))
print('Total # of instances:' + str(len(self.instances)))
print('Shuffling instances')
rr = range(len(self.instances))
np.random.shuffle(rr)
self.instances = np.take(self.instances, rr, axis=0)
print('Generating training instances')
self.training_instances = []
for i in xrange((len(self.instances) * 2) / 3):
self.training_instances.append(self.instances[i])
print('Generating testing instances')
self.testing_instances = []
for i in range((len(self.instances) * 2) / 3, len(self.instances)):
self.testing_instances.append(self.instances[i])
print('Done.')
def readInstances(self, path, maxInst=-1):
insts = []
r = open(path, encoding='utf8')
info = []
for line in r.readlines():
line = line.strip()
if line == '':
for idx in range(1, len(info)):
inst = Instance()
post = self.clean_str(info[0])
inst.post = post.split(" ")
response = self.clean_str(info[idx])
inst.response = response.split(" ")
if maxInst == -1 or (len(insts) < maxInst):
insts.append(inst)
info = []
else:
info.append(line)
if len(info) != 0:
for idx in range(1, len(info)):
inst = Instance()
inst.post = info[0].split(" ")
inst.response = info[idx].split(" ")
if maxInst == -1 or (len(insts) < maxInst):
insts.append(inst)
r.close()
return insts
def test_compute_label(self):
v1 = {"features": (3, 4), "label_array": (0, 0, 0, 0, 0, 1, 0, 0, 0)}
ins = Instance(v1['features'], v1['label_array'])
self.assertEqual(ins.label, 5)
v1 = {"features": (3, 4), "label": 7}
ins = Instance(v1['features'], label=v1['label'])
self.assertEqual(ins.label, 7)
def generate_candidates(text):
doc = nlp(text)
instances = []
all_mentions = []
mentions_per_sentence = []
for sentence in doc.sents:
mentions = []
# add entities from NER
for ent in sentence.ents:
start = ent.start
end = ent.end
if ent[0].text.lower() in STOP_WORDS or ent[0].text.lower(
) in TITLES:
start += 1
if ent[-1].text == "." and ent.end == sentence.end:
end -= 1
if end > start:
mention = Span(doc, start, end)
mention._.fused_type = [ent.label_]
mentions.append(mention)
# add pronouns as mentions
for word in sentence:
if word.tag_ in PRONOUN_TAGS or word.text.lower() in all_pronouns:
mention = Span(doc, word.i, word.i + 1)
if mention.text.lower() in inv_pronoun_map:
mention._.fused_type = inv_pronoun_map[
mention.text.lower()]
mention._.is_pronoun = True
mentions.append(mention)
if USE_NOUN_CHUNKS:
for chunk in sentence.noun_chunks:
root = chunk.root
if root.ent_type_ or root.tag_ in PRONOUN_TAGS: # we already have it
continue
mentions.append(chunk)
mentions_per_sentence.append(mentions)
all_mentions.extend(mentions)
for i, mentions in enumerate(mentions_per_sentence):
for (subject_entity,
object_entity) in itertools.product(mentions, mentions):
instances.append(Instance(subject_entity, object_entity))
if i > 0:
prev_sentence_mentions = mentions_per_sentence[i - 1]
for (subject_entity,
object_entity) in itertools.product(mentions,
prev_sentence_mentions):
instances.append(Instance(subject_entity, object_entity))
for (subject_entity,
object_entity) in itertools.product(prev_sentence_mentions,
mentions):
instances.append(Instance(subject_entity, object_entity))
if USE_ENTITY_LINKER:
entitylinker.link(doc, all_mentions)
if USE_BERT:
bert_wrapper.run(doc)
return instances
def test_cosine_dist(self):
v1 = Instance((5, 0, 3, 0, 2, 0, 0, 2, 0, 0))
v2 = Instance((3, 0, 2, 0, 1, 1, 0, 1, 0, 1))
self.assertEqual(nn.cosine_dist(v1, v2), 0.06439851429360033)
v1 = Instance((3, 4))
v2 = Instance((6, 8))
self.assertEqual(nn.cosine_dist(v1, v2), 0)
def run():
( train, undef, test ) = data.load_data_wrapper()
log( "Trainning set : %8d instances", ( len(train) ) )
log( "Testing set : %8d instances", ( len(test) ) )
txt = "Converting %s set to Instance objects"
log( txt, ("training") )
train = [ Instance( t[0], label_array=t[1] ) for t in train ]
log( txt, ("test") )
test = [ Instance( t[0], label=t[1] ) for t in test ]
instance = 1
start_compare = time.time()
""" Iterate through testing set """
test_subset = test[0:10]
for i in test_subset:
log("Instance %d", ( instance ))
""" Find the closest pair from training set
"""
closest_pair = train[0]
max_dist = cosine_dist( i, closest_pair )
for j in train[1:1000]:
dist = cosine_dist( i, j )
if( dist < max_dist ):
max_dist = dist
closest_pair = j
if( dist == 0 ):
break
i.predicted_label = closest_pair.label
log(">>> %d, actual : %s , predict : %s", ( instance, test[0].label, test[0].predicted_label) )
instance+=1
end_compare = time.time()
""" Compute confusion_matrix, accuracy and prediction and recall for each label
"""
log("----- Confusion Matrix -----")
matrix = confusion_matrix( test_subset )
log("%s", ( pandas.DataFrame( matrix ) ) )
log("Accuracy : %0.2f", ( accuracy(matrix) ) )
for i in range(NUM_LABEL):
log("Label %d : precision: %.2f \t recall: %.2f",
( i, precision( matrix, i ), recall( matrix, i ) )
)
log("----------------")
log("Time spent : %.0f sec", ( end_compare - start_compare ) )
def test_round_trip():
path = Path("tests/input/h=03_t=005_s=011_m=06.json")
text = path.read_text()
data = json.loads(text)
instance_1 = Instance(data)
instance_2 = Instance(path)
json_1 = instance_1.get_json()
print(json_1)
json_2 = instance_2.get_json()
print(json_2)
assert json_1 == json_2
assert json_1 == text
def test_read_file_ok(self):
graph = Instance()
graph.read_file(NodeTest.GRAPH_1_TEST_PTH)
self.assertEqual(100, graph.get_total_nodes())
self.assertEqual(2266, graph.get_total_edges())
self.assertEqual(100, len(graph.get_nodes()), 100)
self.assertTrue(GraphUtils.are_adjacent(graph.get_node(0), graph.get_node(1)))
def __create_instances(self, dep_id, common_section, config_sections):
emailid = common_section.emailid
instance_user_name = common_section.instance_user_name
# Launch instances
for section in config_sections:
section_name = section.section_name
count = int(section.count)
tag = section.tag
machinetype = section.machinetype
disk1image = section.disk1image
disk2image = section.disk2image
disk1type = section.disk1type
disk2type = section.disk2type
disk1size = section.disk1size
disk2size = section.disk2size
for num in range(count):
instance_name = emailid + "-" + dep_id + "-" + section_name + "-" + tag + "-" + str(
num)
disk1 = Disk(instance_name + "-d1", disk1image, disk1size,
disk1type)
disk2 = Disk(instance_name + "-d2", disk2image, disk2size,
disk2type)
disk1.create()
disk2.create()
disk_list = [disk1, disk2]
instance = Instance(instance_name, disk_list, machinetype,
instance_user_name)
instance.create(common_section)
if section_name == "master":
self.master_instances.append(instance)
else:
self.slave_instances.append(instance)
self.instances.append(instance)
def test_find_clique_by_neighbors_2_OK(self):
graph = Instance()
graph.read_file(SolutionGreedyNeighborsCliqueTest.GRAPH_2_TEST)
solution = SolutionGreedyNeighbors(graph, 'test_graph_type_1')
clique = solution.find_clique_by_neighbors()
print(clique)
self.assertEqual([3, 11, 17, 21, 25, 28, 32, 36, 38, 39, 66, 72], clique)
def test_apply_ls(self):
solution = {16, 18, 19, 20, 21, 23}
graph = Instance()
file = SolutionGraspTests.GRAPH_TEST
graph.read_file(file)
instace_sol = SolutionGrasp()
instace_sol.apply_ls(graph, solution)
def makeinstance(instid: str):
a = instid.split()
assert len(a) == 4, f"wrong instance key {instid}"
d = BENCH[a[0]]
dstart = f"{(d['D0'] + int(a[3]) - 1):02d}" + d['H0']
dend = f"{(d['D0'] + int(a[3])):02d}" + d['H0']
return Instance(d['ntk'], PROFILE[a[1]], dstart, dend, STEPLENGTH[a[2]])
def _parse_data(data, isMalicious):
i = 0 # multipurpose counter
inst = Instance()
flag = False
if data:
for item in data:
for category, element_tuple in item.iteritems():
for name, content in element_tuple:
if content and isinstance(name, str):
for element in content:
if (name is 'activities' and not flag):
inst.setName('.'.join(
element.split('.')[:-1])[:100])
flag = True
if (name is 'fingerprint'
and 'MD5' in element): # MD5
inst.setMD5(element[5:])
if (name is 'permissions'): # permissions
inst.addPermission(element)
i += 1
if (name is 'classes_list'
or name is 'internal_classes_list'
or name is 'external_classes_list'
or name is 'internal_packages_list' or
name is 'external_packages_list'): # API
inst.addApi(element[:100])
#log.info("%s has %d API and %d permissions" % (str(inst.getMD5()),
# len(inst.getAPIlist()), len(inst.getPermissions())))
if isMalicious:
inst.setMalicious()
return inst, '%s has %d API and %d permissions' % (str(
inst.getMD5()), len(inst.getAPIlist()), len(inst.getPermissions()))
def load_from_csv(filename):
with open(filename + ".csv", "r") as csvfile:
csv_reader = csv.DictReader(csvfile)
for row in csv_reader:
inst = Instance(row)
data.append(inst)
return data
def add_instance(self, app_name, cpu, ram):
vm_id = self.resource_allocator.create_vm(cpu, ram)
if vm_id is None:
return False
self.app_name2instances.setdefault(app_name, []).append(
Instance(vm_id, cpu, ram))
return True
def load_data(filename):
print('[1] Loading Data ....')
book = xlrd.open_workbook(filename)
input_sheet = book.sheet_by_name("student-mat")
num_rows = input_sheet.nrows
num_cols = input_sheet.ncols
instances = []
attributes = []
for row in range(1, 2):
for col in range(0, num_cols - 1):
attributes.append(input_sheet.col_values(col)[0])
instances = []
for row in range(1, num_rows):
values = []
for col in range(0, num_cols - 1):
values.append(input_sheet.cell(row, col).value)
str_label = input_sheet.cell(row, num_cols - 1).value
if (input_sheet.cell(row, num_cols - 1).value == 'yes'):
label = 1
else:
label = 0
feature_dict = {}
for i in range(len(attributes)):
feature_dict[attributes[i]] = str(values[i])
i = Instance(feature_dict, label)
instances.append(i)
return instances
def test_find_max_clique_1_OK(self):
graph = Instance()
graph.read_file(SolutionGreedyMaxCliqueTest.GRAPH_1_TEST)
solution = SolutionGreedy(graph, 'test_graph_greedy_simple_1')
max_clique = solution.find_max_clique()
print(max_clique)
self.assertEqual([1, 2, 3, 5], max_clique)
def test_find_clique_by_neighbors_1_OK(self):
graph = Instance()
graph.read_file(SolutionGreedyNeighborsCliqueTest.GRAPH_1_TEST)
solution = SolutionGreedyNeighbors(graph, 'test_graph_greedy_simple_1')
clique = solution.find_clique_by_neighbors()
print(clique)
self.assertEqual([1, 2, 3, 5], clique)
def get_by_arg(self, arg, module_name, dictionary=False):
instance = Instance()
instance.__class__.__name__ = module_name
def ret(x): return '"'+x+'"' \
if type(x).__name__ == "str" \
or type(x).__name__ == "unicode" \
else str(x)
ids = ""
for kk in arg.keys():
ids += kk + \
"=" +\
ret(arg[kk]) + \
" AND "
ids = ids[:len(ids) - 5]
query = "SELECT * " + \
", ".join(instance.to_dict().keys()) + \
" FROM " + \
module_name + \
" WHERE " + \
ids
return self.execute_query(query,
module_name,
one_result=True,
dictionary=dictionary)
def main():
for i in range(3,11):
result = 'compare_ex2/results/result_2ma_' + str(i)
data_creator.filename = 'compare_ex2/data/compare_2ma_' + str(i) + '.txt'
data_creator.jobs = i
data_creator.run()
data_parser.filename = data_creator.filename
jobs, machines, tasks, neh_prio = data_parser.get_instance_parameters()
instance = Instance(str(i), machines, jobs, tasks, neh_prio)
instance.print_info()
start = time.time()
instance.generate_best_cmax()
end = time.time()
bruteforce_time = end - start
instance.save_results(data_parser.filename, 'bruteforce', result + '_bruteforce.json')
start = time.time()
instance.johnsons_algorithm()
end = time.time()
johnson_time = end - start
instance.save_results(data_parser.filename, 'johnson', result + '_johnson.json')
start = time.time()
instance.neh()
end = time.time()
neh_time = end - start
instance.save_results(data_parser.filename, 'neh', result + '_neh.json')
print("INFO:\tBruteforce: " + str(bruteforce_time) + "\n\tJohnson: " + str(johnson_time) + "\n\tNeh: " + str(neh_time))
times = {}
times['filename'] = data_parser.filename
times['bruteforce_time'] = str(bruteforce_time)
times['johnson_time'] = str(johnson_time)
times['neh_time'] = str(neh_time)
times_json = json.dumps(times)
filename = 'compare_ex2/results/result_2ma_' + str(i) + '_times.json'
with open (filename, 'w+') as f:
f.write(times_json)
def __init__(self, handle):
super(Record, self).__init__(handle)
self.props.enable_fullscreen_mode = False
Instance(self)
self.add_events(gtk.gdk.VISIBILITY_NOTIFY_MASK)
self.connect("visibility-notify-event", self._visibility_changed)
#the main classes
self.model = Model(self)
self.ui_init()
#CSCL
self.connect("shared", self._shared_cb)
if self.get_shared_activity():
#have you joined or shared this activity yourself?
if self.get_shared():
self._joined_cb(self)
else:
self.connect("joined", self._joined_cb)
# Realize the video view widget so that it knows its own window XID
self._media_view.realize_video()
# Changing to the first toolbar kicks off the rest of the setup
if self.model.get_has_camera():
self.model.change_mode(constants.MODE_PHOTO)
else:
self.model.change_mode(constants.MODE_AUDIO)
def fetch_instances(self):
self.instances = []
path = 'studies/{}/series/{}/instances'.format(self.study.uid,
self.uid)
for ins_dict in self.pacs._make_request(path):
self.instances.append(
Instance(pacs=self.pacs, series=self, input_dict=ins_dict))
def test_solution_OK(self):
graph = Instance()
graph.read_file(SolutionTest.GRAPH_SIMPLE_1_TEST_PTH)
solution = Solution(graph, 'test-graph-simple-1')
cliques = solution.get_solution_max_cliques()
print(cliques)
expected_sol = [{0, 1, 2}]
self.assertEqual(expected_sol, cliques)
def run2():
""" Run with high service level.
"""
ins = Instance(conf2)
ins.sale = shared_sale
ins.evaluate_point_prediction()
ins.run()
ins.plot()
def run1():
""" Run with restricted service level.
"""
ins = Instance(conf1)
ins.sale = shared_sale
ins.evaluate_point_prediction()
ins.run()
ins.plot()
def get_map(self, num=50):
map = []
for x1 in np.linspace(-1.0, 1.0, num):
for x2 in np.linspace(-1.0, 1.0, num):
p = Instance([x1, x2], None)
p.y = self.clasify(p)
map.append(p)
return map
def test_find_max_clique_2_OK(self):
graph = Instance()
graph.read_file(SolutionGreedyMaxCliqueTest.GRAPH_2_TEST)
solution = SolutionGreedy(graph, 'test_graph_type_1')
max_clique = solution.find_max_clique()
print(max_clique)
self.assertEqual([3, 11, 17, 21, 25, 28, 32, 36, 38, 39, 66, 72],
max_clique)
def get_instance(self, id):
url = self.api_endpoint + 'services/{0}/{1}/instances/{2}'.format(
self.namespace, self.name, id)
r = requests.get(url, headers=self.headers)
util.check_response(r)
data = json.loads(r.text)
instance = Instance(service=self, uuid=data['uuid'], details=r.text)
return instance
