def memoize(*args, **kwargs): """ :param args: :param kwargs: :return: """ return cache.cached(*args, **kwargs)
def _get_account_name(self, account_uuid): def get_account_name(): accounts = self.acs.listAccounts({'id': account_uuid, 'listall': 'true'}).get('account') if accounts: return accounts[0].get('name') decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='account_' + account_uuid) return decorator(get_account_name)()
def _get_user_name(self, user_uuid): def get_user_name(): users = self.acs.listUsers({'id': user_uuid, 'listall': 'true'}).get('user') if users: return users[0].get('username') decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='user_' + user_uuid) return decorator(get_user_name)()
def cached_on_args(*args, **kwargs): """ :param args: :param kwargs: :return: """ kwargs.update({'key_prefix': specify_args_cache_key}) return cache.cached(*args, **kwargs)
class modelContainer(): def __init__(self, models_list, columns): self.models_list = models_list self.model_container = {} self.columns = columns cache.cached(timeout=3600) def read_models(self): for model_name in self.models_list: self.model_container[model_name] = { "pca_vectors": load('../models/vectors_pca_' + model_name), "pca_tfidf": load('../models/tfidf_pca_' + model_name), "clf_features": load('../models/feature_selection_' + model_name), "model": load_model('../models/' + model_name), } def process(self, modelName): model = self.model_container[modelName] return [ self.features["dummy_dates"] + self.features["author_dummy"] + self.features["division_dummy"] + self.features["media_type_dummy"] + model["pca_vectors"]["content"]\ .transform([self.features["vectors"]["content"]])[0].tolist() + model["pca_vectors"]["highlight"]\ .transform([self.features["vectors"]["highlight"]])[0].tolist() + model["pca_vectors"]["title"]\ .transform([self.features["vectors"]["title"]])[0].tolist() + model["pca_vectors"]["media_desc"]\ .transform([self.features["vectors"]["media_desc"]])[0].tolist() + model["pca_tfidf"]\ .transform([self.features["tf_idf"]])[0].tolist() ] def predict_all(self, features): if self.model_container == {}: self.read_models() self.features = features pretidctions = {} for model in self.models_list: processedFeatures = self.process(model) pca = self.model_container[model]["clf_features"] selectedFeatures = pca.transform(processedFeatures) tensorflowObj = self.model_container[model]["model"] modelPrediction = tensorflowObj.predict(selectedFeatures) if float(modelPrediction[0][0]) > 0.6: pretidctions[model] = "reaction" elif float(modelPrediction[0][0]) < 0.4: pretidctions[model] = "no_reaction" else: "ambigious" return pretidctions
def _get_account_name(self, account_uuid): def get_account_name(): accounts = self.acs.listAccounts({ 'id': account_uuid, 'listall': 'true' }).get('account') if accounts: return accounts[0].get('name') decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='account_' + account_uuid) return decorator(get_account_name)()
def _get_user_name(self, user_uuid): def get_user_name(): users = self.acs.listUsers({ 'id': user_uuid, 'listall': 'true' }).get('user') if users: return users[0].get('username') decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='user_' + user_uuid) return decorator(get_user_name)()
def decorator(view): f = cache.cached(max_age, **ckwargs)(view) @wraps(f) def wrapper(*args, **wkwargs): response = f(*args, **wkwargs) response.cache_control.max_age = max_age if max_age: response.cache_control.public = True extra = timedelta(seconds=max_age) response.expires = response.last_modified + extra else: response.headers['Pragma'] = 'no-cache' response.cache_control.must_revalidate = True response.cache_control.no_cache = True response.cache_control.no_store = True response.expires = '-1' return response.make_conditional(request) return wrapper
def decorator(view): f = cache.cached(max_age, **ckwargs)(view) @wraps(f) def wrapper(*args, **wkwargs): response = f(*args, **wkwargs) response.cache_control.max_age = max_age if max_age: response.cache_control.public = True extra = timedelta(seconds=max_age) else: response.headers["Pragma"] = "no-cache" response.cache_control.must_revalidate = True response.cache_control.no_cache = True response.cache_control.no_store = True extra = timedelta(0) response.expires = (response.last_modified or dt.utcnow()) + extra return response.make_conditional(request) return wrapper
def _get_disk_offerings(self): decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='disk_offerings_' + self.region) return decorator(lambda: self.acs.listDiskOfferings({}).get('diskoffering'))()
def _get_compute_offerings(self): decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='compute_offerings_' + self.region) return decorator(lambda: self.acs.listServiceOfferings({}).get('serviceoffering'))()
def _get_projects(self): decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='projects_' + self.region) params = {'simple': 'true', 'listall': 'true'} return decorator(lambda: self.acs.listProjects(params).get('project'))()
class featureMaker(): def __init__(self, word2vec_path, columns): self.word2vec_path = word2vec_path self.columns = columns self.months = { 'stycznia': 1, 'lutego': 2, 'marca': 3, 'kwietnia': 4, 'maja': 5, 'czerwca': 6, 'lipca': 7, 'sierpnia': 8, 'września': 9, 'października': 10, 'listopada': 11, 'grudnia': 12 } cache.cached(timeout=3600) def read_idf(self): with open('../labeling/idf_dict') as f: idf = {key: value for key, value in load(f).items() if value >= 100} return idf cache.cached(timeout=3600) def read_word2vec(self): with open(self.word2vec_path) as f: lines = f.read().split('\n') return { x.split(' ')[0]: [float(x) for x in x.split(' ')[1:]] for x in lines } cache.cached(timeout=3600) def read_dictionaries(self): dictionaries = {} for column in ['division', 'media', 'author']: with open('../labeling/'+column+"_dict") as f: dictionaries[column] = {x: i for i, x in enumerate(load(f))} self.dicts = dictionaries def calculate_tf(self): bag_of_all_words = " ".join([value for key, value in self.data.items() if key in self.columns]) tf = dict(Counter([word for word in bag_of_all_words.split(' ')])) return tf def calculate_tf_idf(self): tf = self.calculate_tf() idf = self.read_idf() return [tf[word]/idf if word in tf.keys() else 0 for word, idf in idf.items()] def calculate_vectors(self): word2vec = self.read_word2vec() word2vec_vocabulary = word2vec.keys() vectors_container = {} for column in self.columns: vector = np.zeros(100, dtype = "float64") n_words = 0 for word in self.data[column].split(' '): if word in word2vec_vocabulary: vector = np.add(vector, word2vec[word]) n_words += 1 if n_words: vector = np.divide(vector, n_words) vectors_container[column] = vector.tolist() return vectors_container def process_date(self): date = self.data['date'] year = date.split('|')[0].split(' ')[2].strip() month = str(self.months[date.split('|')[0].split(' ')[1].strip()]) day = date.split('|')[0].split(' ')[0].strip() try: hour = date.split('|')[1].strip() except: return None return dt.strptime(" ".join([year, month, day, hour]), "%Y %m %d %H:%M") def get_dummy_dates(self): return_list = [0 for i in range(7+24)] processed_date = self.process_date() return_list[processed_date.weekday()] = 1 return_list[processed_date.hour+6] = 1 return return_list def get_dummies_from(self, type): return_list = [0 for x in range(len(self.dicts[type]))] if self.data[type] in self.dicts[type].keys(): i = self.dicts[type][self.data[type]] return_list[i] = 1 return return_list def process(self, cleanedData): self.data = cleanedData tf_idf = self.calculate_tf_idf() vectors = self.calculate_vectors() dummy_dates = self.get_dummy_dates() self.read_dictionaries() author_dummy = self.get_dummies_from('author') division_dummy = self.get_dummies_from('division') media_type_dummy = self.get_dummies_from('media') return { "dummy_dates": dummy_dates, "author_dummy": author_dummy, "media_type_dummy": media_type_dummy, "division_dummy": division_dummy, "vectors": vectors, "tf_idf": tf_idf, }
def _get_projects(self): decorator = cache.cached(timeout=app.config['USAGE_CACHE_TIME'], key_prefix='projects_' + self.region) params = {'simple': 'true', 'listall': 'true'} return decorator( lambda: self.acs.listProjects(params).get('project'))()