def test_send_sticker(client, catch_event, compare, sticker):
with catch_event("onMessage") as x:
mid = client.send(Message(sticker=sticker))
assert compare(x, mid=mid)
assert subset(vars(x.res["message_object"]), uid=mid, author=client.uid)
assert subset(vars(x.res["message_object"].sticker), uid=sticker.uid)
def test_update_poll_vote(client1, group, catch_event, poll_data):
event, poll, options = poll_data
new_vote_ids = [o.uid for o in options[0:len(options):2] if not o.vote]
re_vote_ids = [o.uid for o in options[0:len(options):2] if o.vote]
new_options = [random_hex(), random_hex()]
with catch_event("onPollVoted") as x:
client1.updatePollVote(
event["poll"].uid,
option_ids=new_vote_ids + re_vote_ids,
new_options=new_options,
)
assert subset(
x.res,
author_id=client1.uid,
thread_id=group["id"],
thread_type=ThreadType.GROUP,
)
assert subset(vars(x.res["poll"]),
title=poll.title,
options_count=len(options + new_options))
for o in new_vote_ids:
assert o in x.res["added_options"]
assert len(x.res["added_options"]) == len(new_vote_ids) + len(new_options)
assert set(x.res["removed_options"]) == set(
o.uid for o in options if o.vote and o.uid not in re_vote_ids)
def test_remove_from_and_add_to_group(client1, client2, group, catch_event):
# Test both methods, while ensuring that the user gets added to the group
try:
with catch_event("onPersonRemoved") as x:
client1.removeUserFromGroup(client2.uid, group["id"])
assert subset(
x.res, removed_id=client2.uid, author_id=client1.uid, thread_id=group["id"]
)
finally:
with catch_event("onPeopleAdded") as x:
client1.addUsersToGroup(client2.uid, group["id"])
assert subset(
x.res, added_ids=[client2.uid], author_id=client1.uid, thread_id=group["id"]
)
def test_remove_from_and_add_admins_to_group(client1, client2, group, catch_event):
# Test both methods, while ensuring that the user gets added as group admin
try:
with catch_event("onAdminRemoved") as x:
client1.removeGroupAdmins(client2.uid, group["id"])
assert subset(
x.res, removed_id=client2.uid, author_id=client1.uid, thread_id=group["id"]
)
finally:
with catch_event("onAdminAdded") as x:
client1.addGroupAdmins(client2.uid, group["id"])
assert subset(
x.res, added_id=client2.uid, author_id=client1.uid, thread_id=group["id"]
)
def test_create_poll(client1, group, catch_event, poll_data):
event, poll, _ = poll_data
assert subset(event, author_id=client1.id, thread=group)
assert subset(vars(event["poll"]),
title=poll.title,
options_count=len(poll.options))
for recv_option in event[
"poll"].options: # The recieved options may not be the full list
(old_option, ) = list(
filter(lambda o: o.text == recv_option.text, poll.options))
voters = [client1.id] if old_option.vote else []
assert subset(vars(recv_option),
voters=voters,
votes_count=len(voters),
vote=False)
def test_fetch_message_emoji(client, emoji, emoji_size):
mid = client.sendEmoji(emoji, emoji_size)
message, = client.fetchThreadMessages(limit=1)
assert subset(
vars(message), uid=mid, author=client.uid, text=emoji, emoji_size=emoji_size
)
def test_fetch_message_info_emoji(client, thread, emoji, emoji_size):
mid = client.sendEmoji(emoji, emoji_size)
message = client.fetchMessageInfo(mid, thread_id=thread["id"])
assert subset(
vars(message), uid=mid, author=client.uid, text=emoji, emoji_size=emoji_size
)
def test_change_image_remote(client1, group, catch_event):
url = "https://github.com/carpedm20/fbchat/raw/master/tests/image.png"
with catch_event("onImageChange") as x:
image_id = client1.changeGroupImageRemote(url, group["id"])
assert subset(
x.res, new_image=image_id, author_id=client1.uid, thread_id=group["id"]
)
def test_change_image_local(client1, group, catch_event):
url = path.join(path.dirname(__file__), "resources", "image.png")
with catch_event("onImageChange") as x:
image_id = client1.changeGroupImageLocal(url, group["id"])
assert subset(
x.res, new_image=image_id, author_id=client1.uid, thread_id=group["id"]
)
def test_fetch_plan_info(client, catch_event, plan_data):
event, plan = plan_data
fetched_plan = client.fetchPlanInfo(plan.uid)
assert subset(vars(fetched_plan),
time=plan.time,
title=plan.title,
author_id=int(client.uid))
def id3(attributes, targetAttribute, dataset):
currentAttributes = attributes.copy()
currentDataSet = dataset.copy()
targetValues = utils.valuesOf(targetAttribute, currentDataSet)
default = utils.majorityValue(targetAttribute, currentDataSet)
currentEntropy = utils.entropy(targetAttribute, currentDataSet)
# If attributes is empty, return the majority value
if len(currentAttributes) == 0:
return default
# if all examples belong to same class
if currentEntropy == 0.0:
[value] = targetValues
return value
bestAttribute = utils.selectBestAttribute(currentAttributes,
targetAttribute, currentDataSet)
tree = {bestAttribute: {}}
bestAttributeValues = utils.valuesOf(bestAttribute, currentDataSet)
currentAttributes.remove(bestAttribute)
for value in bestAttributeValues:
subset = utils.subset(bestAttribute, value, currentDataSet)
subtree = id3(currentAttributes, targetAttribute, subset)
tree[bestAttribute][value] = subtree
return tree
def test_on_plan_ended(client, thread, catch_event, compare):
with catch_event("onPlanEnded") as x:
client.createPlan(Plan(int(time()) + 120, "Wait for ending"))
x.wait(180)
assert subset(x.res,
thread_id=client.uid
if thread["type"] == ThreadType.USER else thread["id"],
thread_type=thread["type"])
def test_delete_messages(client):
text1 = "This message will stay"
text2 = "This message will be removed"
mid1 = client.send(Message(text=text1))
mid2 = client.send(Message(text=text2))
client.delete_messages(mid2)
(message,) = client.fetch_thread_messages(limit=1)
assert subset(vars(message), uid=mid1, author=client.uid, text=text1)
def test_send_images(client, catch_event, compare, method_name, url):
text = "An image sent with {}".format(method_name)
with catch_event("onMessage") as x:
mid = getattr(client, method_name)(url, Message(text))
assert compare(x, mid=mid, message=text)
assert subset(vars(x.res["message_object"]), uid=mid, author=client.uid, text=text)
assert x.res["message_object"].attachments[0]
def test_send_text(client, catch_event, compare, text):
with catch_event("on_message") as x:
mid = client.send(Message(text=text))
assert compare(x, mid=mid, message=text)
assert subset(vars(x.res["message_object"]),
uid=mid,
author=client.uid,
text=text)
def test_on_plan_ended(client, thread, catch_event, compare):
with catch_event("on_plan_ended") as x:
client.create_plan(PlanData(int(time()) + 120, "Wait for ending"))
x.wait(180)
assert subset(
x.res,
thread_id=client.id if thread["type"] is None else thread["id"],
thread_type=thread["type"],
)
def simu_z_bzx(self): snp = subset(100) sample = sample(5000) f = f = uniform(0.01, 0.5) z = binomial(2, f, 100) b_zx = normal(0, np.sqrt(1/(2*f*(1-f)))) z_bzx = np.mean(z*b_zx) return z_bzx
def test_fetch_message_emoji(client, emoji, emoji_size):
mid = client.send_emoji(emoji, emoji_size)
(message, ) = client.fetch_thread_messages(limit=1)
assert subset(vars(message),
id=mid,
author=client.id,
text=emoji,
emoji_size=emoji_size)
def test_send_mentions(client, catch_event, compare, message_with_mentions):
with catch_event("onMessage") as x:
mid = client.send(message_with_mentions)
assert compare(x, mid=mid, message=message_with_mentions.text)
assert subset(vars(x.res["message_object"]), uid=mid, author=client.uid, text=message_with_mentions.text)
# The mentions are not ordered by offset
for m in x.res["message_object"].mentions:
assert vars(m) in [vars(x) for x in message_with_mentions.mentions]
def test_fetch_message_mentions(client, thread, message_with_mentions):
mid = client.send(message_with_mentions)
message, = client.fetchThreadMessages(limit=1)
assert subset(
vars(message), uid=mid, author=client.uid, text=message_with_mentions.text
)
# The mentions are not ordered by offset
for m in message.mentions:
assert vars(m) in [vars(x) for x in message_with_mentions.mentions]
def test_fetch_message_info_mentions(client, thread, message_with_mentions):
mid = client.send(message_with_mentions)
message = client.fetchMessageInfo(mid, thread_id=thread["id"])
assert subset(
vars(message), uid=mid, author=client.uid, text=message_with_mentions.text
)
# The mentions are not ordered by offset
for m in message.mentions:
assert vars(m) in [vars(x) for x in message_with_mentions.mentions]
def test_fetch_message_mentions(client):
text = "This is a test of fetchThreadMessages"
mentions = [Mention(client.uid, offset=10, length=4)]
mid = client.send(Message(text, mentions=mentions))
message, = client.fetchThreadMessages(limit=1)
assert subset(vars(message), uid=mid, author=client.uid, text=text)
for i, m in enumerate(mentions):
assert vars(message.mentions[i]) == vars(m)
def test_change_approval_mode(client1, group, catch_event, require_admin_approval):
with catch_event("onApprovalModeChange") as x:
client1.changeGroupApprovalMode(require_admin_approval, group["id"])
assert subset(
x.res,
approval_mode=require_admin_approval,
author_id=client1.uid,
thread_id=group["id"],
)
def test_change_title(client1, group, catch_event):
title = random_hex()
with catch_event("onTitleChange") as x:
client1.changeThreadTitle(title, group["id"], thread_type=ThreadType.GROUP)
assert subset(
x.res,
author_id=client1.uid,
new_title=title,
thread_id=group["id"],
thread_type=ThreadType.GROUP,
)
def test_edit_plan(client, thread, catch_event, compare, plan_data):
event, plan = plan_data
new_plan = Plan(plan.time + 100, random_hex())
with catch_event("onPlanEdited") as x:
client.editPlan(plan, new_plan)
assert compare(x)
assert subset(
vars(x.res["plan"]),
time=new_plan.time,
title=new_plan.title,
author_id=client.uid,
)
def test_send_remote_files(client, catch_event, compare):
files = ["image.png", "data.json"]
text = "Files sent from remote"
with catch_event("onMessage") as x:
mid = client.sendRemoteFiles(
["https://github.com/carpedm20/fbchat/raw/master/tests/{}".format(f) for f in files],
message=Message(text),
)
assert compare(x, mid=mid, message=text)
assert subset(vars(x.res["message_object"]), uid=mid, author=client.uid, text=text)
assert len(x.res["message_object"].attachments) == len(files)
def test_send_local_files(client, catch_event, compare):
files = ["image.png", "image.jpg", "image.gif", "file.json", "file.txt", "audio.mp3", "video.mp4"]
text = "Files sent locally"
with catch_event("onMessage") as x:
mid = client.sendLocalFiles(
[path.join(path.dirname(__file__), "resources", f) for f in files],
message=Message(text),
)
assert compare(x, mid=mid, message=text)
assert subset(vars(x.res["message_object"]), uid=mid, author=client.uid, text=text)
assert len(x.res["message_object"].attachments) == len(files)
def test_send_emoji(client, catch_event, compare, emoji, emoji_size):
with catch_event("onMessage") as x:
mid = client.sendEmoji(emoji, emoji_size)
assert compare(x, mid=mid, message=emoji)
assert subset(
vars(x.res["message_object"]),
uid=mid,
author=client.uid,
text=emoji,
emoji_size=emoji_size,
)
def test_change_plan_participation(client, thread, catch_event, compare,
plan_data, take_part):
event, plan = plan_data
with catch_event("onPlanParticipation") as x:
client.changePlanParticipation(plan, take_part=take_part)
assert compare(x, take_part=take_part)
assert subset(
vars(x.res["plan"]),
time=plan.time,
title=plan.title,
author_id=client.uid,
going=[client.uid] if take_part else [],
declined=[client.uid] if not take_part else [],
)
def get_information_gain(examples, feature_index, schema, entropy_of_set, is_nominal):
"""
:param examples: list of examples
@type entropy_of_set: float
:return: float
"""
gain = entropy_of_set
feature = utils.get_example_values_for_feature(examples, schema, feature_index, is_nominal)
feature_counts = feature
for feature_value, count in feature_counts.items():
if count == 0:
continue
gain -= count/len(examples) * get_entropy(utils.subset(examples, feature_index, feature_value, is_nominal), schema, feature_index, is_nominal)
return gain
def plan_data(request, client, user, thread, catch_event, compare):
with catch_event("onPlanCreated") as x:
client.createPlan(request.param, thread["id"])
assert compare(x)
assert subset(
vars(x.res["plan"]),
time=request.param.time,
title=request.param.title,
author_id=client.uid,
going=[client.uid],
declined=[],
)
plan_id = x.res["plan"]
assert user["id"] in x.res["plan"].invited
request.param.uid = x.res["plan"].uid
yield x.res, request.param
with catch_event("onPlanDeleted") as x:
client.deletePlan(plan_id)
assert compare(x)
def _mimic_iterator_unbuffered(config, vocab):
if config.training or config.force_trainset:
splits = config.training_splits
else:
splits = config.testing_splits
random.shuffle(splits)
for split in splits:
notes_file = pjoin(config.data_path, 'notes_%02d.pk' % (split,))
if os.path.isfile(notes_file):
print 'Loading data split', split
with open(notes_file, 'rb') as f:
data = pickle.load(f)
raw_data = []
if config.conditional:
raw_aux_data = collections.defaultdict(list)
for note in data:
values = {}
(text, values['gender'], values['has_dod'], values['has_icu_stay'], \
values['admission_type'], values['diagnoses'], values['procedures'], \
values['labs'], values['prescriptions']) = note
if len(text) > 1:
raw_data.append(text)
if config.conditional:
for (feat, dims) in config.mimic_embeddings.items():
if dims > 0:
raw_aux_data[feat].append(values[feat])
print 'Loaded data split', split
batch_len = ((len(raw_data) - 1) // config.batch_size) + 1
pad_count = (batch_len * config.batch_size) - len(raw_data)
indices = range(batch_len)
random.shuffle(indices)
if not config.struct_only:
raw_data = [[] for _ in range(pad_count)] + raw_data
grouped_raw_data = [group for group in utils.grouper(config.batch_size, raw_data)]
grouped_raw_data = [grouped_raw_data[i] for i in indices]
raw_data = [note for group in grouped_raw_data for note in group]
if config.conditional:
for k, v in raw_aux_data.items():
if k in config.fixed_len_features:
v = [0 for _ in range(pad_count)] + v
else:
v = [[] for _ in range(pad_count)] + v
ls = [group for group in utils.grouper(config.batch_size, v)]
ls = [ls[i] for i in indices]
raw_aux_data[k] = [data for group in ls for data in group]
for batch in xrange(batch_len):
batch_data = raw_data[config.batch_size * batch : config.batch_size * (batch + 1)]
if config.conditional:
batch_aux_data = {}
for (feat, vals) in raw_aux_data.items():
batch_aux_data[feat] = vals[config.batch_size * batch : \
config.batch_size * (batch + 1)]
if config.recurrent:
max_note_len = max(len(note) for note in batch_data)
epoch_size = ((max_note_len - 2) // config.num_steps) + 1
data = np.zeros([config.batch_size, epoch_size * config.num_steps + 1],
dtype=np.int32)
else:
min_note_len = min(len(note) for note in batch_data)
data = np.zeros([config.batch_size, min_note_len], dtype=np.int32)
epoch_size = min_note_len - config.num_steps # this can become negative!
if epoch_size <= 0:
continue
if config.recurrent:
mask = np.zeros([config.batch_size, epoch_size * config.num_steps + 1],
dtype=np.float32)
for i, iter_data in enumerate(batch_data):
if config.recurrent:
data[i, 0:len(iter_data)] = iter_data
mask[i, 0:len(iter_data)] = 1.0
else:
data[i, 0:min_note_len] = iter_data[:min_note_len]
aux_data = {}
aux_data_len = {}
if config.conditional:
for feat, vals in batch_aux_data.items():
if feat in config.fixed_len_features:
max_struct_len = 1
else:
max_struct_len = max(len(v) for v in vals)
aux_data[feat] = np.zeros([config.batch_size, max_struct_len],
dtype=np.int32)
aux_data_len[feat] = np.zeros([config.batch_size], dtype=np.int32)
for i, iter_data in enumerate(vals):
if feat in config.fixed_len_features:
aux_data[feat][i, 0] = iter_data
aux_data_len[feat][i] = 1
else:
aux_data[feat][i, 0:len(iter_data)] = iter_data
aux_data_len[feat][i] = len(iter_data)
new_batch = True
epochs = range(epoch_size)
if not config.recurrent:
if config.struct_only:
samples = 1
else:
samples = config.samples_per_note
epochs = [e for e in utils.subset(epochs, samples)]
for i in epochs:
if config.recurrent:
x = data[:, i*config.num_steps:(i+1)*config.num_steps]
y = data[:, i*config.num_steps+1:(i+1)*config.num_steps+1]
m = mask[:, i*config.num_steps+1:(i+1)*config.num_steps+1]
else:
x = np.concatenate([data[:, i:(i+int(config.num_steps/2))],
data[:, (i+1+int(config.num_steps/2)):i+1+config.num_steps]], 1)
y = data[:, i+int(config.num_steps/2)]
m = None
yield (x, y, m, aux_data, aux_data_len, new_batch)
new_batch = False
def _generate_tree(self, examples, feature_indices, depth, node):
"""
Generates a Decision Tree using the ID3 algorithm.
:param examples: numpy array of `Example`s
:param feature_indices:
:param depth:
:param parent:
:return:
"""
if len(examples) == 0:
raise Exception("No examples provided. ID3 failed.")
root = None
if depth == 0:
root = DecisionTreeNode(depth=depth)
else:
root = node
if self.max_depth > 0 and depth == self.max_depth:
root.label = utils.most_common_value(examples)
return root
if utils.is_homogeneous(examples, positive=True): # test if all examples for class label are positive
root.label = True
return root
elif utils.is_homogeneous(examples, positive=False):
root.label = False
return root
if not feature_indices:
root.label = utils.most_common_value(examples)
return root
root.feature_index, test_threshold = get_best_feature_index_and_value(examples, self.schema, feature_indices)
is_nominal = self.schema[root.feature_index].type == 'NOMINAL'
if is_nominal:
root.feature_values = self.schema[root.feature_index].values
else:
root.is_nominal = False
root.feature_values = [test_threshold, test_threshold]
for feature_value_index in range(len(root.feature_values)):
child = DecisionTreeNode(parent=root, depth=depth+1)
root.add_child(child)
if feature_value_index == 0:
comparison_operator = '<'
else:
comparison_operator = '>'
examples_matching_feature_value = utils.subset(examples, root.feature_index, root.feature_values[feature_value_index], is_nominal, comparison_operator=comparison_operator)
if not examples_matching_feature_value:
child.label=utils.most_common_value(examples)
else:
features_without_best_classifier = [index for index in feature_indices if index != root.feature_index]
self._generate_tree(
examples=examples_matching_feature_value,
feature_indices=features_without_best_classifier,
depth=depth + 1,
node=child,
)
return root