def clarify_dataset(considered_items_sorted, attributes_to_consider_entities,
items_id):
from enumerator.enumerator_attribute_complex import init_attributes_complex, create_index_complex
from enumerator.enumerator_attribute_themes2 import tree_leafs, tree_remove
attributes = init_attributes_complex(considered_items_sorted,
attributes_to_consider_entities)
index_all = create_index_complex(considered_items_sorted, attributes)
mapping = {}
mapping_obj = []
considered_items_sorted_clarified = []
for i, o in enumerate(index_all):
t_o = tuple(
frozenset(o[a['name']]) if a['type'] == 'themes' else o[a['name']]
for a in attributes)
#raw_input('...')
if t_o not in mapping:
mapping[t_o] = set()
mapping_obj.append(t_o)
considered_items_sorted_clarified.append(
considered_items_sorted[i])
mapping[t_o] |= {i}
#print t_o,len(mapping)
#print len(mapping), len(index_all)
mapping_from_clarified_to_not_clarified = {
i: mapping[mapping_obj[i]]
for i in range(len(mapping_obj))
}
for i in mapping_from_clarified_to_not_clarified:
print i, mapping_from_clarified_to_not_clarified[i]
raw_input('****')
raw_input(('....'))
# def process_outcome_dataset_with_time(itemsfile,usersfile,outcomesfile,
# time_attr=None,time_format="%d/%m/%y",numeric_attrs=[],array_attrs=[],outcome_attrs=None,method_aggregation_outcome='VECTOR_VALUES',
# delimiter='\t'): #everything must be ordered by dateid
# items,items_header=readCSVwithHeader(itemsfile,numberHeader=numeric_attrs,arrayHeader=array_attrs,delimiter=delimiter)
# users,users_header=readCSVwithHeader(usersfile,numberHeader=numeric_attrs,arrayHeader=array_attrs,delimiter=delimiter)
# outcomes,outcomes_header=readCSVwithHeader(outcomesfile,numberHeader=numeric_attrs,arrayHeader=array_attrs,delimiter=delimiter)
# items_id=items_header[0]
# users_id=users_header[0]
# outcome_attrs=outcome_attrs if outcome_attrs is not None else [outcomes_header[2]]
# position_attr=outcome_attrs[0]
# outcomes_processed=outcome_representation_in_reviews(outcomes,position_attr,outcome_attrs,method_aggregation_outcome)
# if time_attr is not None:
# for item in items:
# item[time_attr]=from_string_to_date(item[time_attr],time_format)
# items=sorted(items,key=lambda x:x[time_attr])
# map_items_to_sorted_id={v[items_id]:i for i,v in enumerate(items)}
# items_metadata={}
# for item in items:
# item[items_id]=map_items_to_sorted_id[item[items_id]]
# items_metadata[item[items_id]]=item
# for row in outcomes_processed:
# row[items_id]=map_items_to_sorted_id[row[items_id]]
# else:
# items_metadata={row[items_id]:row for row in items}
# users_metadata={row[users_id]:row for row in users}
# all_users_to_items_outcomes={}
# for row in outcomes_processed:
# v_id_rev=row[items_id]
# u_id_rev=row[users_id]
# pos_rev=row[position_attr]
# if u_id_rev not in all_users_to_items_outcomes:
# all_users_to_items_outcomes[u_id_rev]={}
# all_users_to_items_outcomes[u_id_rev][v_id_rev]=pos_rev
# return items_metadata,users_metadata,all_users_to_items_outcomes,items_id,users_id
def process_outcome_dataset(itemsfile,
usersfile,
outcomesfile,
numeric_attrs=[],
array_attrs=[],
outcome_attrs=None,
method_aggregation_outcome='VECTOR_VALUES',
itemsScope=[],
contexts_scope=[],
users_1_Scope=[],
users_2_Scope=[],
nb_items=float('inf'),
nb_individuals=float('inf'),
attributes_to_consider=None,
nb_items_entities=float('inf'),
nb_items_individuals=float('inf'),
hmt_to_itemset=False,
delimiter='\t'):
FULL_OUTCOME_CONSIDERED = False
ITEMS_METADATA_NEEDED = False
USE_CACHE = False
SIZE_ESTIMATION = False
SMALLER_DESCRIPTION_SPACE = True
VERBOSE = False
CLARIFY = True
to_consider_ids_in_contexts_scope = None
nb_outcome_considered = 0
if 'CACHE' in dir(process_outcome_dataset):
items, items_header, users, users_header, outcomes, outcomes_header, items_id, users_id, outcome_attrs, position_attr, outcomes_processed, considered_items, users1, users2, (
considered_items_ids), (considered_users_1_ids), (
considered_users_2_ids), (
considered_users_ids) = process_outcome_dataset.CACHE
else:
items, items_header = readCSVwithHeader(itemsfile,
numberHeader=numeric_attrs,
arrayHeader=array_attrs,
selectedHeader=None,
delimiter=delimiter)
users, users_header = readCSVwithHeader(usersfile,
numberHeader=numeric_attrs,
arrayHeader=array_attrs,
selectedHeader=None,
delimiter=delimiter)
outcomes, outcomes_header = readCSVwithHeader(
outcomesfile,
numberHeader=numeric_attrs,
arrayHeader=array_attrs,
selectedHeader=None,
delimiter=delimiter)
items_id = items_header[0]
users_id = users_header[0]
outcome_attrs = outcome_attrs if outcome_attrs is not None else [
outcomes_header[2]
]
position_attr = outcome_attrs[0]
outcomes_processed, vector_of_action = outcome_representation_in_reviews(
outcomes, position_attr, outcome_attrs, method_aggregation_outcome)
considered_items = filter_pipeline_obj(items, itemsScope)[0]
considered_items_in_contexts_scope = filter_pipeline_obj(
items, contexts_scope)[0]
users1 = filter_pipeline_obj(users, users_1_Scope)[0]
users2 = filter_pipeline_obj(users, users_2_Scope)[0]
get_items_ids = partial(map, itemgetter(items_id))
get_users_ids = partial(map, itemgetter(users_id))
considered_items_ids = set(get_items_ids(considered_items))
to_consider_ids_in_contexts_scope = set(
get_items_ids(considered_items_in_contexts_scope))
considered_users_1_ids = set(get_users_ids(users1))
considered_users_2_ids = set(get_users_ids(users2))
considered_users_ids = set(considered_users_1_ids) | set(
considered_users_2_ids)
if USE_CACHE:
process_outcome_dataset.CACHE = [
items, items_header, users, users_header, outcomes,
outcomes_header, items_id, users_id, outcome_attrs,
position_attr, outcomes_processed, considered_items[:],
users1[:], users2[:],
set(considered_items_ids),
set(considered_users_1_ids),
set(considered_users_2_ids),
set(considered_users_ids)
]
if nb_items < float('inf'):
nb_items = int(nb_items)
considered_items_ids = set(sorted(considered_items_ids)[:nb_items])
considered_items = [
x for x in considered_items if x[items_id] in considered_items_ids
]
if nb_individuals < float('inf'):
nb_individuals = int(nb_individuals)
considered_users_ids = set(
sorted(considered_users_ids)[:nb_individuals])
users1 = [x for x in users1 if x[users_id] in considered_users_ids]
users2 = [x for x in users2 if x[users_id] in considered_users_ids]
all_users_to_items_outcomes = {}
outcomes_considered = []
outcomes_considered_append = outcomes_considered.append
items_metadata = {
row[items_id]: row
for row in considered_items if row[items_id] in considered_items_ids
} if ITEMS_METADATA_NEEDED else {}
users_metadata = {
row[users_id]: row
for row in users if row[users_id] in considered_users_ids
}
for row in outcomes_processed:
v_id_rev = row[items_id]
u_id_rev = row[users_id]
if v_id_rev in considered_items_ids and u_id_rev in considered_users_ids:
pos_rev = row[position_attr]
if u_id_rev not in all_users_to_items_outcomes:
all_users_to_items_outcomes[u_id_rev] = {}
all_users_to_items_outcomes[u_id_rev][v_id_rev] = pos_rev
if FULL_OUTCOME_CONSIDERED:
outcomes_considered_append({
items_id: v_id_rev,
users_id: u_id_rev,
position_attr: pos_rev
})
nb_outcome_considered += 1
considered_users_1_sorted = sorted(users1, key=itemgetter(users_id))
considered_users_2_sorted = sorted(users2, key=itemgetter(users_id))
considered_items_sorted = sorted(considered_items,
key=itemgetter(items_id))
if SIZE_ESTIMATION:
from pympler.asizeof import asizeof
print asizeof(all_users_to_items_outcomes)
print asizeof(considered_items_sorted)
print asizeof(considered_users_1_sorted)
print asizeof(considered_users_2_sorted)
gc.collect()
if SMALLER_DESCRIPTION_SPACE:
NB_SELECTED_ITEMSET_ENTITIES = nb_items_entities #100
NB_SELECTED_ITEMSET_INDIVIDUALS = nb_items_individuals #100
from enumerator.enumerator_attribute_complex import init_attributes_complex, create_index_complex
from enumerator.enumerator_attribute_themes2 import tree_leafs, tree_remove
#####################################ENTITIES - DESCRIPTION SPACE LIMITING##########################################
concerned_attributes_entities_numeric = sorted(
set(attributes_to_consider) & set(numeric_attrs)
& set(items_header))
concerned_attributes_entities_hmt = sorted(
set(attributes_to_consider) & set(array_attrs) & set(items_header))
concerned_attributes_entities_categorical = sorted(
set(attributes_to_consider) & set(items_header) -
(set(concerned_attributes_entities_hmt)
| set(concerned_attributes_entities_numeric)))
attributes_plain = [{
'name': a,
'type': 'themes'
} for a in concerned_attributes_entities_hmt] + [{
'name': a,
'type': 'numeric'
} for a in concerned_attributes_entities_numeric] + [{
'name': a,
'type': 'simple'
} for a in concerned_attributes_entities_categorical]
attributes = [{
'name': a,
'type': 'themes'
} for a in concerned_attributes_entities_hmt] + [{
'name': a,
'type': 'numeric'
} for a in concerned_attributes_entities_numeric] + [{
'name': a,
'type': 'simple'
} for a in concerned_attributes_entities_categorical]
attributes = init_attributes_complex(considered_items_sorted,
attributes) #X
index_all = create_index_complex(considered_items_sorted,
attributes) #Y
nb_itemsets_all = 0
for attr in attributes:
if attr['type'] in {'numeric'}:
nb_itemsets_all += 2 * len(attr['domain'])
attr['nb_items'] = 2 * len(attr['domain'])
else:
nb_itemsets_all += len(attr['domain'])
attr['nb_items'] = len(attr['domain'])
if NB_SELECTED_ITEMSET_ENTITIES <= 1:
NB_SELECTED_ITEMSET_ENTITIES = int(nb_itemsets_all *
NB_SELECTED_ITEMSET_ENTITIES)
print "NB_SELECTED_ITEMSET_ENTITIES AFTER RATIO = ", NB_SELECTED_ITEMSET_ENTITIES
nb_itemsets_all_context = NB_SELECTED_ITEMSET_ENTITIES
nb_itemsets_to_remove = max(
0, nb_itemsets_all - NB_SELECTED_ITEMSET_ENTITIES)
if nb_itemsets_to_remove > 0:
if VERBOSE:
print 'Entities Search Space Modification : ', NB_SELECTED_ITEMSET_ENTITIES
factor = NB_SELECTED_ITEMSET_ENTITIES / float(nb_itemsets_all)
#print factor
for attr in attributes:
attr['nb_items_new'] = int(round(factor * attr['nb_items']))
attr['nb_items_to_remove'] = attr['nb_items'] - attr[
'nb_items_new']
attr_name = attr['name']
attr_type = attr['type']
attr_domain = attr['domain']
attr_labelmap = attr['labelmap']
if attr_type == 'themes':
tags_removed = set(
tree_remove(attr_domain, attr['nb_items_to_remove']))
tags_keeped = set(attr_domain) - tags_removed
for i, o in enumerate(index_all):
o[attr_name] = o[attr_name] & tags_keeped
considered_items_sorted[i][attr_name] = [
attr_labelmap[t] if t != '' else t
for t in sorted(o[attr_name])
]
elif attr_type == 'numeric':
values_to_remove = int(attr['nb_items_to_remove'] / 2)
values_to_keep = len(attr['domain']) - values_to_remove
if values_to_keep == 0:
values_to_keep = 1
new_attr_domain = [
attr_domain[int(
round((k / float(values_to_keep)) *
len(attr_domain)))]
for k in range(values_to_keep)
]
mapto = {
x:
new_attr_domain[bisect_right(new_attr_domain, x) - 1]
for x in attr_domain
}
for i, o in enumerate(index_all):
o[attr_name] = mapto[o[attr_name]]
considered_items_sorted[i][attr_name] = o[attr_name]
elif attr_type == 'simple':
if attr['nb_items_to_remove'] >= len(attr['domain']):
attr['nb_items_to_remove'] = -1
groupOfValues = 'From ' + attr['domain'][
0] + ' To ' + attr['domain'][
attr['nb_items_to_remove']]
new_attr_domain = [
groupOfValues
] + attr['domain'][attr['nb_items_to_remove'] + 1:]
mapto = {
x: x if x in new_attr_domain else groupOfValues
for x in attr_domain
}
for i, o in enumerate(index_all):
o[attr_name] = mapto[o[attr_name]]
considered_items_sorted[i][attr_name] = o[attr_name]
else:
if VERBOSE:
print 'no Entities Search Space Modification'
#####################################ENTITIES - DESCRIPTION SPACE LIMITING##########################################
#####################################INDIVIDUALS - DESCRIPTION SPACE LIMITING##########################################
concerned_attributes_individuals_numeric = sorted(
set(attributes_to_consider) & set(numeric_attrs)
& set(users_header))
concerned_attributes_individuals_hmt = sorted(
set(attributes_to_consider) & set(array_attrs) & set(users_header))
concerned_attributes_individuals_categorical = sorted(
set(attributes_to_consider) & set(users_header) -
(set(concerned_attributes_individuals_hmt)
| set(concerned_attributes_individuals_numeric)))
attributes_plain = [{
'name': a,
'type': 'themes'
} for a in concerned_attributes_individuals_hmt] + [{
'name': a,
'type': 'numeric'
} for a in concerned_attributes_individuals_numeric] + [{
'name':
a,
'type':
'simple'
} for a in concerned_attributes_individuals_categorical]
attributes = [{
'name': a,
'type': 'themes'
} for a in concerned_attributes_individuals_hmt] + [{
'name': a,
'type': 'numeric'
} for a in concerned_attributes_individuals_numeric] + [{
'name':
a,
'type':
'simple'
} for a in concerned_attributes_individuals_categorical]
considered_users_sorted = users_metadata.values()
attributes = init_attributes_complex(considered_users_sorted,
attributes) #X
index_all = create_index_complex(considered_users_sorted,
attributes) #Y
nb_itemsets_all = 0
for attr in attributes:
if attr['type'] in {'numeric'}:
nb_itemsets_all += 2 * len(attr['domain'])
attr['nb_items'] = 2 * len(attr['domain'])
else:
nb_itemsets_all += len(attr['domain'])
attr['nb_items'] = len(attr['domain'])
if NB_SELECTED_ITEMSET_INDIVIDUALS <= 1:
#print "nb_itemsets_all",nb_itemsets_all,NB_SELECTED_ITEMSET_INDIVIDUALS
NB_SELECTED_ITEMSET_INDIVIDUALS = int(
nb_itemsets_all * NB_SELECTED_ITEMSET_INDIVIDUALS)
print "NB_SELECTED_ITEMSET_INDIVIDUALS AFTER RATIO = ", NB_SELECTED_ITEMSET_INDIVIDUALS
nb_itemsets_all_individuals = nb_itemsets_all
nb_itemsets_to_remove = max(
0, nb_itemsets_all - NB_SELECTED_ITEMSET_INDIVIDUALS)
if nb_itemsets_to_remove > 0:
if VERBOSE:
print 'Individuals Search Space Modification : ', NB_SELECTED_ITEMSET_INDIVIDUALS
factor = NB_SELECTED_ITEMSET_INDIVIDUALS / float(nb_itemsets_all)
#print factor
for attr in attributes:
attr['nb_items_new'] = int(round(factor * attr['nb_items']))
attr['nb_items_to_remove'] = attr['nb_items'] - attr[
'nb_items_new']
attr_name = attr['name']
attr_type = attr['type']
attr_domain = attr['domain']
attr_labelmap = attr['labelmap']
if attr_type == 'themes':
tags_removed = set(
tree_remove(attr_domain, attr['nb_items_to_remove']))
tags_keeped = set(attr_domain) - tags_removed
for i, o in enumerate(index_all):
o[attr_name] = o[attr_name] & tags_keeped
considered_users_sorted[i][attr_name] = [
attr_labelmap[t] if t != '' else t
for t in sorted(o[attr_name])
]
elif attr_type == 'numeric':
values_to_remove = int(attr['nb_items_to_remove'] / 2)
values_to_keep = len(attr['domain']) - values_to_remove
if values_to_keep == 0:
values_to_keep = 1
new_attr_domain = [
attr_domain[int(
round((k / float(values_to_keep)) *
len(attr_domain)))]
for k in range(values_to_keep)
]
mapto = {
x:
new_attr_domain[bisect_right(new_attr_domain, x) - 1]
for x in attr_domain
}
for i, o in enumerate(index_all):
o[attr_name] = mapto[o[attr_name]]
considered_users_sorted[i][attr_name] = o[attr_name]
elif attr_type == 'simple':
# print len(attr['domain'])
# print attr['nb_items_to_remove']
# print attr['nb_items_new']
if attr['nb_items_to_remove'] >= len(attr['domain']):
attr['nb_items_to_remove'] = -1
groupOfValues = 'From ' + attr['domain'][
0] + ' To ' + attr['domain'][
attr['nb_items_to_remove']]
new_attr_domain = [
groupOfValues
] + attr['domain'][attr['nb_items_to_remove'] + 1:]
mapto = {
x: x if x in new_attr_domain else groupOfValues
for x in attr_domain
}
for i, o in enumerate(index_all):
o[attr_name] = mapto[o[attr_name]]
considered_users_sorted[i][attr_name] = o[attr_name]
else:
if VERBOSE:
print 'no Individuals Search Space Modification'
users1 = [
x for x in considered_users_sorted
if x[users_id] in considered_users_1_ids
]
users2 = [
x for x in considered_users_sorted
if x[users_id] in considered_users_2_ids
]
considered_users_1_sorted = sorted(users1, key=itemgetter(users_id))
considered_users_2_sorted = sorted(users2, key=itemgetter(users_id))
#####################################INDIVIDUALS - DESCRIPTION SPACE LIMITING##########################################
items_metadata = {
row[items_id]: row
for row in considered_items_sorted
if row[items_id] in considered_items_ids
} if ITEMS_METADATA_NEEDED else {}
users_metadata = {
row[users_id]: row
for row in considered_users_sorted
if row[users_id] in considered_users_ids
}
# if CLARIFY:
# concerned_attributes_entities_numeric=sorted(set(attributes_to_consider)&set(numeric_attrs)&set(items_header))
# concerned_attributes_entities_hmt=sorted(set(attributes_to_consider)&set(array_attrs)&set(items_header))
# concerned_attributes_entities_categorical=sorted(set(attributes_to_consider)&set(items_header)-(set(concerned_attributes_entities_hmt)|set(concerned_attributes_entities_numeric)))
# attributes=[{'name':a,'type':'themes'} for a in concerned_attributes_entities_hmt]+[{'name':a,'type':'numeric'} for a in concerned_attributes_entities_numeric]+[{'name':a,'type':'simple'} for a in concerned_attributes_entities_categorical]
# clarify_dataset(considered_items_sorted,attributes,items_id)
#print considered_items_sorted[0]
TOITEMSET = hmt_to_itemset
if TOITEMSET:
print ' '
print 'Transform HMT to Itemset ...'
print ' '
if TOITEMSET:
concerned_attributes_entities_numeric = sorted(
set(attributes_to_consider) & set(numeric_attrs)
& set(items_header))
concerned_attributes_entities_hmt = sorted(
set(attributes_to_consider) & set(array_attrs) & set(items_header))
concerned_attributes_entities_categorical = sorted(
set(attributes_to_consider) & set(items_header) -
(set(concerned_attributes_entities_hmt)
| set(concerned_attributes_entities_numeric)))
attributes_plain = [{
'name': a,
'type': 'themes'
} for a in concerned_attributes_entities_hmt] + [{
'name': a,
'type': 'numeric'
} for a in concerned_attributes_entities_numeric] + [{
'name': a,
'type': 'simple'
} for a in concerned_attributes_entities_categorical]
attributes = [{
'name': a,
'type': 'themes'
} for a in concerned_attributes_entities_hmt] + [{
'name': a,
'type': 'numeric'
} for a in concerned_attributes_entities_numeric] + [{
'name': a,
'type': 'simple'
} for a in concerned_attributes_entities_categorical]
attributes = init_attributes_complex(considered_items_sorted,
attributes) #X
index_all = create_index_complex(considered_items_sorted,
attributes) #Y
for attr in attributes:
#attr['nb_items_new']=int(round(factor*attr['nb_items']))
#attr['nb_items_to_remove']=attr['nb_items']-attr['nb_items_new']
attr_name = attr['name']
attr_type = attr['type']
attr_domain = attr['domain']
attr_labelmap = attr['labelmap']
index_tag_items = {
t: str(i).zfill(5)
for i, t in enumerate(attr_domain.keys())
}
index_tag_items = {
k: v + " " + attr_labelmap[k].partition(" ")[-1]
for k, v in index_tag_items.iteritems()
}
if attr_type == 'themes':
for i, o in enumerate(index_all):
# print ' '
# print considered_items_sorted[i][attr_name]
considered_items_sorted[i][attr_name] = [
index_tag_items[t] for t in sorted(o[attr_name])
if t != ''
]
# print considered_items_sorted[i][attr_name]
# print ' '
# raw_input('....')
REPLACING_IDS = False
if REPLACING_IDS:
ind_to_dict_items = {
i: x[items_id]
for i, x in enumerate(considered_items_sorted)
}
dict_to_ind_items = {v: k for k, v in ind_to_dict_items.iteritems()}
considered_items_sorted = [
dict([(items_id, dict_to_ind_items[x[items_id]])] +
[(k, v) for k, v in x.items() if k != items_id])
for x in considered_items_sorted
]
to_consider_ids_in_contexts_scope = {
dict_to_ind_items[x]
for x in to_consider_ids_in_contexts_scope
}
ind_to_dict_users = {i: x for i, x in enumerate(users_metadata)}
dict_to_ind_users = {v: k for k, v in ind_to_dict_users.iteritems()}
considered_users_1_sorted = [
dict([(users_id, dict_to_ind_users[x[users_id]])] +
[(k, v) for k, v in x.items() if k != users_id])
for x in considered_users_1_sorted
]
considered_users_2_sorted = [
dict([(users_id, dict_to_ind_users[x[users_id]])] +
[(k, v) for k, v in x.items() if k != users_id])
for x in considered_users_2_sorted
]
considered_users_sorted = [
dict([(users_id, dict_to_ind_users[x[users_id]])] +
[(k, v) for k, v in x.items() if k != users_id])
for x in considered_users_sorted
]
all_users_to_items_outcomes = {
dict_to_ind_users[u]:
{dict_to_ind_items[v]: o
for v, o in u_votes.iteritems()}
for u, u_votes in all_users_to_items_outcomes.iteritems()
}
items_metadata = {
row[items_id]: row
for row in considered_items_sorted
}
users_metadata = {
dict_to_ind_users[u]:
dict([(users_id, dict_to_ind_users[x[users_id]])] +
[(k, v) for k, v in x.items() if k != users_id])
for u, x in users_metadata.iteritems()
}
# print vector_of_action
# raw_input('**************')
else:
items_metadata = {
row[items_id]: row
for row in considered_items_sorted
}
process_outcome_dataset.STATS = {
'nb_items_entities': nb_itemsets_all_context,
'nb_items_individuals': nb_itemsets_all_individuals
}
return items_metadata, users_metadata, all_users_to_items_outcomes, outcomes_considered, items_id, users_id, considered_items_sorted, considered_users_1_sorted, considered_users_2_sorted, nb_outcome_considered, vector_of_action, to_consider_ids_in_contexts_scope