def transform_to_matrix_one_user(user_id):
print "loading data for user "+str(user_id)
categorized_data = DataExtractor.load_json_data(user_id)
data = DataExtractor.complete_data(categorized_data)
metadata = DataExtractor.complete_metadata(categorized_data)
#order the data by the alphabetic name of the features
print "ordering data "+str(user_id)
data = collections.OrderedDict(sorted(data.items()))
#get the first date and the last date
print "getting first date and last date "
end_date = date_min
start_date = datetime.now()
for feature, feature_data in data.iteritems():
feature_data = collections.OrderedDict(sorted(feature_data.items()))
begin_date = DataExtractor.start_date_of_realization(feature_data.keys()[0])
if begin_date < start_date:
start_date = begin_date
last_date = DataExtractor.start_date_of_realization(feature_data.keys()[len(feature_data.keys())-1])
if last_date > end_date:
end_date = last_date
data[feature] = feature_data
#construct the data matrix
#I- construct the matrices of all the features
print "constructing the matrixes "
rows = 0
transformers = {}
for feature, feature_date in data.iteritems():
if feature == "location":
transformers[feature] = MatrixLocationFeatureTransformer(feature, data[feature], metadata[feature], start_date, end_date, coocurring_precision)
elif feature == "bluetoothSeen" or feature == "bluetoothPaired":
transformers[feature] = MatrixBleutoothFeatureTransformer(feature, data[feature], metadata[feature], start_date, end_date, coocurring_precision)
else :
transformers[feature] = MatrixFeatureTransformer(feature, data[feature], metadata[feature], start_date, end_date, coocurring_precision)
if feature in features_importance_score_one:
transformers[feature].let_importance_scores_to_1 = True
transformers[feature].transform()
rows += transformers[feature].nbdimentions
#construct the time feature
transformers[MatrixTimeFeatureTransformer.feature_name] = MatrixTimeFeatureTransformer(start_date, end_date, coocurring_precision)
transformers[MatrixTimeFeatureTransformer.feature_name].transform()
rows += transformers[MatrixTimeFeatureTransformer.feature_name].nbdimentions
columns = transformers[MatrixTimeFeatureTransformer.feature_name].nbtimeslots
#II-concatenate all the matrices of each feature into one big matrix (do the same for the labels vector)
print "regrouping the matrixes "
data_matrix = np.zeros((columns, rows))
labels_vector = [""]* rows
dimentions_importance_score = np.zeros(rows)
transformers = collections.OrderedDict(sorted(transformers.items()))
begin_row_idex = 0
end_row_index = 0
for feature, feature_transformer in transformers.iteritems():
end_row_index = begin_row_idex + feature_transformer.nbdimentions
data_matrix[:, begin_row_idex:end_row_index] = feature_transformer.matrix_data
labels_vector[begin_row_idex:end_row_index] = feature_transformer.labels_vector
dimentions_importance_score[begin_row_idex:end_row_index]=feature_transformer.realization_importance_score
begin_row_idex = end_row_index
'''
The matrix contains a lot of feature vectors that contains 0 in all the features except the time features.
Those vectors corresponds to the times where any record has been done.
We want to eliminate those timestamps and their corresponding times
'''
time_vector = transformers.values()[0].time_vector
[data_matrix, time_vector] = eliminate_empty_records(data_matrix, time_vector)
data_matrix = np.transpose(data_matrix)
print "the labels are : "
print JsonUtils.dict_as_json_str(labels_vector)
print "first date of observation "+str(start_date)
print "first date of observation "+str(end_date)
print "dimension of the labels (features) vector : "+str(len(labels_vector))
print "dimension of the time vector : "+str(len(time_vector))
print "dimension of the resulted matrix (features, time) "+str(data_matrix.shape)
print "the number of non zeros values is : "+str(np.count_nonzero(data_matrix))+"/"+str(np.size(data_matrix))
print "the number of negative values in the matrix is : "+str(np.size(ma.masked_array(data_matrix, mask=(data_matrix>=0)).compressed()))
print "the data matrix printed : "
print Numpy.str(data_matrix)
#write the matrix data
MDataExtractor.save_matrix(user_id, data_matrix)
#write the labels vector, then the time vector and the importance scores
MDataExtractor.save_labels_vector(user_id, labels_vector)
MDataExtractor.save_time_vector(user_id, time_vector)
MDataExtractor.save_importance_scores(user_id, dimentions_importance_score)
def transform_to_rfv_one_user(user_id):
print "loading data for user "+str(user_id)
categorized_data = DataExtractor.load_json_data(user_id)
data = DataExtractor.complete_data(categorized_data)
metadata = DataExtractor.complete_metadata(categorized_data)
#order the data by the alphabetic name of the features
print "ordering data "+str(user_id)
data = collections.OrderedDict(sorted(data.items()))
#get the first date and the last date
print "getting first date and last date "
end_date = date_min
start_date = datetime.now()
for feature, feature_data in data.iteritems():
feature_data = collections.OrderedDict(sorted(feature_data.items()))
begin_date = DataExtractor.start_date_of_realization(feature_data.keys()[0])
if begin_date < start_date:
start_date = begin_date
last_date = DataExtractor.start_date_of_realization(feature_data.keys()[len(feature_data.keys())-1])
if last_date > end_date:
end_date = last_date
data[feature] = feature_data
#construct the values data
#I- construct the values for all the features
print "constructing the values data"
transformers = {}
features_name = []
records = []
values_name = {}
for feature, feature_date in data.iteritems():
if feature == "location":
transformers[feature] = ValuesFeatureTransformer(MatrixLocationFeatureTransformer, feature, data[feature], metadata[feature], start_date, end_date, coocurring_precision)
elif feature == "bluetoothSeen" or feature == "bluetoothPaired":
transformers[feature] = ValuesFeatureTransformer(MatrixBleutoothFeatureTransformer, feature, data[feature], metadata[feature], start_date, end_date, coocurring_precision)
else :
transformers[feature] = ValuesFeatureTransformer(MatrixFeatureTransformer, feature, data[feature], metadata[feature], start_date, end_date, coocurring_precision)
transformers[feature].transform()
features_name.append(feature)
values_name[features_name.index(feature)] = transformers[feature].values_labels
#construct the time feature
feature = "time"
timetrans = ValuesTimeFeatureTransformer(MatrixTimeFeatureTransformer, feature, start_date, end_date, coocurring_precision)
timetrans.transform()
transformers[ValuesTimeFeatureTransformer.day_label] = timetrans
transformers[ValuesTimeFeatureTransformer.hour_label] = timetrans
features_name.append(ValuesTimeFeatureTransformer.day_label)
values_name[features_name.index(ValuesTimeFeatureTransformer.day_label)] = timetrans.day_values_labels
features_name.append(ValuesTimeFeatureTransformer.hour_label)
values_name[features_name.index(ValuesTimeFeatureTransformer.hour_label)] = timetrans.time_values_labels
records_labels = timetrans.records_dates
records_nb = len(records_labels)
#make space for records
for r in range(records_nb):
records.append({})
#II-fill the records
for fid, fname in enumerate(features_name):
if fname == ValuesTimeFeatureTransformer.day_label:
for r in range(records_nb):
if transformers[fname].day_values_data[r]!= []: records[r][fid] = transformers[fname].day_values_data[r];
elif fname == ValuesTimeFeatureTransformer.hour_label:
for r in range(records_nb):
if transformers[fname].time_values_data[r]!= []: records[r][fid] = transformers[fname].time_values_data[r];
else:
for r in range(records_nb):
if transformers[fname].values_data[r]!= []: records[r][fid] = transformers[fname].values_data[r];
#remove the ones that only contain value for the time feature
for r in range(records_nb-1, -1, -1): #Decreasing loop over the records so that remove is possible
if len(records[r]) <= 2:
del records[r]
#for the remaining records, add non_present values for the non_persistant features that are not present in each record. non_persistant
for nf in nonpersistent_features: #add the non_present value as a value that can be taken by the non persistent features
if nf in features_name:
nfid = features_name.index(nf)
values_name[nfid].append(nonpresent_v)
rtv_data = {}
for idr, r in enumerate(records):
for nf in nonpersistent_features:
if nf in features_name:
nfid = features_name.index(nf)
if nfid not in r: r[nfid]=[values_name[nfid].index(nonpresent_v)];
rtv_data[idr]=r
print "first date of observation "+str(start_date)
print "first date of observation "+str(end_date)
print "features names "+str(features_name)
print "values names : "+str(values_name)
print "number of records "+str(len(rtv_data))
#write the data, the record dates, the feature names and the value names
RVFDataExtractor.save_rvf(user_id, rtv_data, features_name, values_name, records_labels)
