def memoize(*args, **kwargs):
"""
:param args:
:param kwargs:
:return:
"""
return cache.cached(*args, **kwargs)
def _get_account_name(self, account_uuid):
def get_account_name():
accounts = self.acs.listAccounts({'id': account_uuid, 'listall': 'true'}).get('account')
if accounts:
return accounts[0].get('name')
decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='account_' + account_uuid)
return decorator(get_account_name)()
def _get_user_name(self, user_uuid):
def get_user_name():
users = self.acs.listUsers({'id': user_uuid, 'listall': 'true'}).get('user')
if users:
return users[0].get('username')
decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='user_' + user_uuid)
return decorator(get_user_name)()
def cached_on_args(*args, **kwargs):
"""
:param args:
:param kwargs:
:return:
"""
kwargs.update({'key_prefix': specify_args_cache_key})
return cache.cached(*args, **kwargs)
class modelContainer():
def __init__(self, models_list, columns):
self.models_list = models_list
self.model_container = {}
self.columns = columns
cache.cached(timeout=3600)
def read_models(self):
for model_name in self.models_list:
self.model_container[model_name] = {
"pca_vectors": load('../models/vectors_pca_' + model_name),
"pca_tfidf": load('../models/tfidf_pca_' + model_name),
"clf_features":
load('../models/feature_selection_' + model_name),
"model": load_model('../models/' + model_name),
}
def process(self, modelName):
model = self.model_container[modelName]
return [
self.features["dummy_dates"] +
self.features["author_dummy"] +
self.features["division_dummy"] +
self.features["media_type_dummy"] +
model["pca_vectors"]["content"]\
.transform([self.features["vectors"]["content"]])[0].tolist() +
model["pca_vectors"]["highlight"]\
.transform([self.features["vectors"]["highlight"]])[0].tolist() +
model["pca_vectors"]["title"]\
.transform([self.features["vectors"]["title"]])[0].tolist() +
model["pca_vectors"]["media_desc"]\
.transform([self.features["vectors"]["media_desc"]])[0].tolist() +
model["pca_tfidf"]\
.transform([self.features["tf_idf"]])[0].tolist()
]
def predict_all(self, features):
if self.model_container == {}:
self.read_models()
self.features = features
pretidctions = {}
for model in self.models_list:
processedFeatures = self.process(model)
pca = self.model_container[model]["clf_features"]
selectedFeatures = pca.transform(processedFeatures)
tensorflowObj = self.model_container[model]["model"]
modelPrediction = tensorflowObj.predict(selectedFeatures)
if float(modelPrediction[0][0]) > 0.6:
pretidctions[model] = "reaction"
elif float(modelPrediction[0][0]) < 0.4:
pretidctions[model] = "no_reaction"
else:
"ambigious"
return pretidctions
def _get_account_name(self, account_uuid):
def get_account_name():
accounts = self.acs.listAccounts({
'id': account_uuid,
'listall': 'true'
}).get('account')
if accounts:
return accounts[0].get('name')
decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'],
key_prefix='account_' + account_uuid)
return decorator(get_account_name)()
def _get_user_name(self, user_uuid):
def get_user_name():
users = self.acs.listUsers({
'id': user_uuid,
'listall': 'true'
}).get('user')
if users:
return users[0].get('username')
decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'],
key_prefix='user_' + user_uuid)
return decorator(get_user_name)()
def decorator(view):
f = cache.cached(max_age, **ckwargs)(view)
@wraps(f)
def wrapper(*args, **wkwargs):
response = f(*args, **wkwargs)
response.cache_control.max_age = max_age
if max_age:
response.cache_control.public = True
extra = timedelta(seconds=max_age)
response.expires = response.last_modified + extra
else:
response.headers['Pragma'] = 'no-cache'
response.cache_control.must_revalidate = True
response.cache_control.no_cache = True
response.cache_control.no_store = True
response.expires = '-1'
return response.make_conditional(request)
return wrapper
def decorator(view):
f = cache.cached(max_age, **ckwargs)(view)
@wraps(f)
def wrapper(*args, **wkwargs):
response = f(*args, **wkwargs)
response.cache_control.max_age = max_age
if max_age:
response.cache_control.public = True
extra = timedelta(seconds=max_age)
else:
response.headers["Pragma"] = "no-cache"
response.cache_control.must_revalidate = True
response.cache_control.no_cache = True
response.cache_control.no_store = True
extra = timedelta(0)
response.expires = (response.last_modified or dt.utcnow()) + extra
return response.make_conditional(request)
return wrapper
def _get_disk_offerings(self):
decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='disk_offerings_' + self.region)
return decorator(lambda: self.acs.listDiskOfferings({}).get('diskoffering'))()
def _get_compute_offerings(self):
decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='compute_offerings_' + self.region)
return decorator(lambda: self.acs.listServiceOfferings({}).get('serviceoffering'))()
def _get_projects(self):
decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='projects_' + self.region)
params = {'simple': 'true', 'listall': 'true'}
return decorator(lambda: self.acs.listProjects(params).get('project'))()
class featureMaker():
def __init__(self, word2vec_path, columns):
self.word2vec_path = word2vec_path
self.columns = columns
self.months = {
'stycznia': 1,
'lutego': 2,
'marca': 3,
'kwietnia': 4,
'maja': 5,
'czerwca': 6,
'lipca': 7,
'sierpnia': 8,
'września': 9,
'października': 10,
'listopada': 11,
'grudnia': 12
}
cache.cached(timeout=3600)
def read_idf(self):
with open('../labeling/idf_dict') as f:
idf = {key: value for key, value in load(f).items() if value >= 100}
return idf
cache.cached(timeout=3600)
def read_word2vec(self):
with open(self.word2vec_path) as f:
lines = f.read().split('\n')
return {
x.split(' ')[0]: [float(x) for x in x.split(' ')[1:]] for x in lines
}
cache.cached(timeout=3600)
def read_dictionaries(self):
dictionaries = {}
for column in ['division', 'media', 'author']:
with open('../labeling/'+column+"_dict") as f:
dictionaries[column] = {x: i for i, x in enumerate(load(f))}
self.dicts = dictionaries
def calculate_tf(self):
bag_of_all_words = " ".join([value for key, value in self.data.items()
if key in self.columns])
tf = dict(Counter([word for word in bag_of_all_words.split(' ')]))
return tf
def calculate_tf_idf(self):
tf = self.calculate_tf()
idf = self.read_idf()
return [tf[word]/idf if word in tf.keys() else 0
for word, idf in idf.items()]
def calculate_vectors(self):
word2vec = self.read_word2vec()
word2vec_vocabulary = word2vec.keys()
vectors_container = {}
for column in self.columns:
vector = np.zeros(100, dtype = "float64")
n_words = 0
for word in self.data[column].split(' '):
if word in word2vec_vocabulary:
vector = np.add(vector, word2vec[word])
n_words += 1
if n_words:
vector = np.divide(vector, n_words)
vectors_container[column] = vector.tolist()
return vectors_container
def process_date(self):
date = self.data['date']
year = date.split('|')[0].split(' ')[2].strip()
month = str(self.months[date.split('|')[0].split(' ')[1].strip()])
day = date.split('|')[0].split(' ')[0].strip()
try:
hour = date.split('|')[1].strip()
except:
return None
return dt.strptime(" ".join([year, month, day, hour]), "%Y %m %d %H:%M")
def get_dummy_dates(self):
return_list = [0 for i in range(7+24)]
processed_date = self.process_date()
return_list[processed_date.weekday()] = 1
return_list[processed_date.hour+6] = 1
return return_list
def get_dummies_from(self, type):
return_list = [0 for x in range(len(self.dicts[type]))]
if self.data[type] in self.dicts[type].keys():
i = self.dicts[type][self.data[type]]
return_list[i] = 1
return return_list
def process(self, cleanedData):
self.data = cleanedData
tf_idf = self.calculate_tf_idf()
vectors = self.calculate_vectors()
dummy_dates = self.get_dummy_dates()
self.read_dictionaries()
author_dummy = self.get_dummies_from('author')
division_dummy = self.get_dummies_from('division')
media_type_dummy = self.get_dummies_from('media')
return {
"dummy_dates": dummy_dates,
"author_dummy": author_dummy,
"media_type_dummy": media_type_dummy,
"division_dummy": division_dummy,
"vectors": vectors,
"tf_idf": tf_idf,
}
def _get_projects(self):
decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'],
key_prefix='projects_' + self.region)
params = {'simple': 'true', 'listall': 'true'}
return decorator(
lambda: self.acs.listProjects(params).get('project'))()
