def tag():
if 'user' in session:
user_id = session['user']['id']
tag_count, verify_count = get_tag_count(user_id)
q = request.args.get('q')
if (not session['is_admin']) and (q == 'verify' or q == '3-skips'):
flash('Invalid credentials', 'error')
return redirect(url_for('bp.login'))
price_range_text_list = config.PRICE_RANGE_TEXT_LIST
price_range_value_list = config.PRICE_RANGE_VALUE_LIST
return render_template('tag_product.html',
vendors=get_vendors(),
available_cats=get_categories(),
available_cats1=get_categories(),
price_range_text_list=price_range_text_list,
price_range_value_list=price_range_value_list,
username=session['user']['name'],
tag_count=tag_count,
verify_count=verify_count,
q=q,
autoescape=False)
else:
return redirect(url_for('bp.login'))
def tag():
if 'user' in session:
user_id = session['user']['id']
tag_count, verify_count = get_tag_count(user_id)
q = request.args.get('q')
if (not session['is_admin']) and (q == 'verify' or q == '3-skips'):
flash('Invalid credentials', 'error')
return redirect(url_for('bp.login'))
price_range_text_list = config.PRICE_RANGE_TEXT_LIST
price_range_value_list = config.PRICE_RANGE_VALUE_LIST
return render_template('tag_product.html',
vendors=get_vendors(),
available_cats=get_categories(),
available_cats1=get_categories(),
price_range_text_list=price_range_text_list,
price_range_value_list=price_range_value_list,
username=session['user']['name'],
tag_count=tag_count,
verify_count=verify_count,
q=q,
autoescape=False)
else:
return redirect(url_for('bp.login'))
def get_knn_keywords(k=100, top_n=3, text=sampletext, counts=None):
if not counts:
counts = classify(k, text)
category_lists = defaultdict(list)
for k, v in Counter({
label: counts[label]
for label in counts if label in utils.get_categories()
}).most_common(top_n):
lda_cat = category_models[k]
dictionary_cat = category_dicts[k]
bigram_cat = category_bigrams[k]
for w in lda_cat.get_document_topics(dictionary_cat.doc2bow(
bigram_cat[utils.tokenize(text)]),
per_word_topics=True)[2]:
category_lists[k].append(dictionary_cat[w[0]])
for w in lda_all.get_document_topics(dictionary_all.doc2bow(
bigram_all[utils.tokenize(text)]),
per_word_topics=True)[2]:
category_lists['all'].append(dictionary_all[w[0]])
return category_lists
def task_definition(task_id, task_instance=None):
task = utils.get_task_definition(task_id)
org_name = utils.get_org(task["organization_id"], "name")
org_slug = utils.get_org(task["organization_id"], "slug")
task_name = task["name"]
task_description = task["description"].replace("\n", "<br>")
task_tags = task["tags"]
mentors = task["assignments_profile_display_names"]
days = task["time_to_complete_in_days"]
task_categories = utils.get_categories(task_id)
attachments = None
if task_instance:
if task_instance not in utils.tasks_attachments_cache:
utils.tasks_attachments_cache[
task_instance] = utils.get_attachments(task_instance)
else:
utils.tasks_attachments_cache[task_instance] = [
"Too lazy to fix this."
]
return render_template(
"task.html",
org_name=org_name,
org_slug=org_slug,
task_name=task_name,
task_description=task_description,
tags=task_tags,
mentors=mentors,
days=days,
categories=task_categories,
attachments=utils.tasks_attachments_cache[task_instance])
def enrich_category(X):
items = X['item'].tolist()
cat_dict = utils.get_categories(items)
cat = [cat_dict[i] for i in items]
X.loc[:, 'category'] = pd.Series(cat).values
return X
def func(category1, category2):
start_time = time.time()
y_train1, y_train2 = get_categories(category1, category2)
compare = Compare(dataloader_workers=0, verbose=2,
distribution="bernoulli")
compare.fit(y_train1, y_train2, 10000)
elapsed_time = time.time() - start_time
return dict(
samples=pickle.dumps(compare.samples),
elapsed_time=elapsed_time,
)
def show_edit(request, videoid):
requesthead = 'http://spark.bokecc.com/api/video?'
q = {'videoid': videoid, 'userid': USERID, 'format': 'json'}
categories = get_categories()
my_json = get_json_result(requesthead, q)
video = my_json['video']
images = video ['image-alternate']
return render_to_response('edit.html', {'video': video,
'images': images,
'categories': categories
}
)
def func(category1, category2, averaging, nrefits):
start_time = time.time()
np.random.seed(10 * category1 + category2)
y_train1, y_train2 = get_categories(category1, category2)
np.random.seed()
htest = HTest(dataloader_workers=0,
verbose=1,
distribution="bernoulli",
averaging=averaging)
htest.fit(y_train1, y_train2, nrefits=nrefits)
elapsed_time = time.time() - start_time
return dict(
pvalue=htest.pvalue,
elapsed_time=elapsed_time,
)
def add_subcat():
if 'user' in session and session['is_admin']:
subcat_status = 'Not Added'
if request.form:
if len(request.form['subcat']) and request.form['category'] != '-1':
subcat_status = add_new_subcat( request.form['category'], request.form['subcat'] )
else:
subcat_status = 'Error'
user_id = session['user']['id']
tag_count, verify_count = get_tag_count(user_id)
return render_template("add_sub_cat.html",
username=session['user']['name'],
tag_count=tag_count,
verify_count=verify_count,
available_cats=get_categories(),
subcat_status=subcat_status)
else:
flash('Invalid credentials', 'error')
return redirect(url_for('bp.login'))
def enrich_category_count(X):
items = X['item'].tolist()
users = X['user'].tolist()
cat_dict = utils.get_categories(items)
nested = list(cat_dict.values())
cat = list(set([item for sublist in nested for item in sublist]))
print(cat)
print(len(cat))
user_cat_count = defaultdict(lambda: defaultdict(int))
for u, i in list(zip(users, items)):
i_cat = cat_dict[i]
for cat in i_cat:
cat_dict[u][cat] += 1
for u, i in list(zip(users, items)):
cat_vec = np.zeros(len(cat_dict))
i_cat = cat_dict[i]
def add_appliance():
form = NewApplianceForm()
form.category.choices = utils.get_categories()
if current_user.is_authenticated:
if form.validate_on_submit():
try:
appliance = Appliance(
name = form.name.data,
watts = form.watts.data,
category = form.category.data
)
db.session.add(appliance)
db.session.commit()
flash("New appliance added", "success")
except IntegrityError:
flash(f"{form.name.data} is already in the database", "danger")
return redirect("/addappliance")
return redirect("/")
return render_template("new-app.html", form=form)
def split(root_path, train_prop, test_prop):
val_prop = 1 - train_prop - test_prop
categories = utils.get_categories(root_path)
for cat in categories:
npy_path_list = utils.take_cat_npys(root_path, cat)
n = len(npy_path_list)
train_number = int(n * train_prop)
test_number = int(n * test_prop)
val_number = int(n * val_prop)
shuffle(npy_path_list)
for i in range(train_number):
save_to(npy_path_list[i], root_path, 'train', cat)
for i in range(train_number, train_number + test_number):
save_to(npy_path_list[i], root_path, 'test', cat)
for i in range(train_number + test_number, n):
save_to(npy_path_list[i], root_path, 'validation', cat)
def add_subcat():
if 'user' in session and session['is_admin']:
subcat_status = 'Not Added'
if request.form:
if len(request.form['subcat']
) and request.form['category'] != '-1':
subcat_status = add_new_subcat(request.form['category'],
request.form['subcat'])
else:
subcat_status = 'Error'
user_id = session['user']['id']
tag_count, verify_count = get_tag_count(user_id)
return render_template("add_sub_cat.html",
username=session['user']['name'],
tag_count=tag_count,
verify_count=verify_count,
available_cats=get_categories(),
subcat_status=subcat_status)
else:
flash('Invalid credentials', 'error')
return redirect(url_for('bp.login'))
import math
import csv
from collections import Counter
from collections import defaultdict
from gensim import corpora, models, similarities
from gensim.utils import simple_preprocess, ClippedCorpus
from gensim.parsing.preprocessing import STOPWORDS
import nltk.stem as stem
import utils
import re
import os
categories = utils.get_categories()
category_models = {
category:
models.LdaModel.load('./data/%s/category-titles-abstracts.lda' % category)
for category in categories
}
category_dicts = {
category: corpora.Dictionary.load(
'data/%s/category-titles-abstracts.dict' % category)
for category in categories
}
category_bigrams = {
category: models.Phrases.load('data/%s/bigram.bin' % category)
for category in categories
}
lda_all = models.LdaModel.load('./data/corpus-titles-abstracts.lda')
dictionary_all = corpora.Dictionary.load('data/corpus-titles-abstracts.dict')
def yolov2_coco(device=None, input_size=None, pretrained=True):
model = _get_model('cfg/yolov2.cfg', 'weights/yolov2.weights', device, input_size, pretrained)
cats = get_categories('classes/coco.names')
return model, cats
def yolov2_tiny_voc(device=None, input_size=None, pretrained=True):
model = _get_model('cfg/yolov2-tiny-voc.cfg', 'weights/yolov2-tiny-voc.weights', device, input_size, pretrained)
cats = get_categories('classes/voc.names')
return model, cats
# store the dictionary, for future reference
dictionary.save('./data/%s/%s.dict' % (category, category_filename))
# memory-friendly bag-of-words class
class BOW(object):
def __iter__(self):
for line, label in zip(open('./data/%s.csv' % corpus_filename),
open('./data/corpus-labels.csv')):
# assume there's one document per line, tokens separated by whitespace
if category in label:
yield dictionary.doc2bow(utils.tokenize(line))
else:
pass
# Now we can make a bag of words and do something with it by iterating over it
arxiv_bow = BOW()
corpora.MmCorpus.serialize('./data/%s/%s.mm' %
(category, category_filename),
arxiv_bow) # store to disk, for later use
if __name__ == '__main__':
#Set corpus name. This lets us select from "corpus-titles", "corpus-abstracts", and "corpus-titles-abstracts"
corpus_filename = 'corpus-titles-abstracts'
if args.category:
generate_bow(corpus_filename, args.category, True, 0.05, 10)
else:
for category in utils.get_categories():
generate_bow(corpus_filename, category, True, 0.05, 10)
def enum_categories_and_remove(dir, tolerance):
categories = utils.get_categories(dir)
for cat in categories:
remove_in_category(dir, cat, tolerance)
def show_search(request):
categories = get_categories()
return render_to_response('search.html', {'categories': categories})
def show_upload(request):
categories = get_categories()
return render_to_response('upload.html', {'categories': categories})
