def get(self, request):
error = 1
rg = request.GET
if rg:
if "create" in rg:
d = Dictionary(title=rg["title"], owner_id=self.request.user.id)
d.save()
error = 0
if "get_dictionary_content" in rg:
d = Dictionary(title=rg["titlr"], owner_id=self.request.user.id)
return HttpResponse(json.dumps({"error" : error}))
def reset_password(self, user):
''' Generate secure password, and email to user '''
random.seed(int(time()))
count = Dictionary.word_count()
index = random.randint(0, count)
word = Dictionary.at(index)
new_password = "******" % (
word,
random.randint(0, 9999),
)
user.password = new_password
dbsession.add(user)
dbsession.flush()
email_service = EmailService()
email_service.send_password_recovery(user, new_password)
def build_tweet_dictionary():
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.DEBUG)
from twitter_stream.models import Tweet
stoplist = stopwords.words('english')
texts = DbTexts(Tweet)
tokenized = Tokenizer(texts, stoplist=stoplist)
# build a dictionary
print "Building a dictionary"
dictionary = corpora.Dictionary(tokenized)
# Remove extremely rare words
dictionary.filter_extremes(no_below=2, no_above=0.5, keep_n=None)
dictionary.compactify()
# Save it in the database
print "Saving the tweet dict"
from models import Dictionary
return Dictionary.create_from_gensim_dictionary(dictionary,
"tweets dictionary")
def build_dictionary(self, dataset_id):
texts = DbTextIterator(self.queryset)
tokenized_texts = self.tokenizer(texts, *self.filters)
dataset = Dataset.objects.get(pk=dataset_id)
return Dictionary._create_from_texts(tokenized_texts=tokenized_texts,
name=self.name,
minimum_frequency=self.minimum_frequency,
dataset=dataset,
settings=self.get_dict_settings())
def build_dictionary(self, dataset_id):
texts = DbTextIterator(self.queryset)
tokenized_texts = self.tokenizer(texts, self.lemmatizer, *self.filters)
dataset = Dataset.objects.get(pk=dataset_id)
return Dictionary._create_from_texts(
tokenized_texts=tokenized_texts,
name=self.name,
minimum_frequency=self.minimum_frequency,
dataset=dataset,
settings=self.get_dict_settings())
def build_dictionary(self):
texts = DbTextIterator(self.queryset, textfield=self.textfield)
tokenized_texts = self.tokenizer(texts, stoplist=self.stoplist)
return Dictionary._create_from_texts(
tokenized_texts=tokenized_texts,
name=self.name,
minimum_frequency=self.minimum_frequency,
dataset=self.queryset.model.__name__,
settings=self.get_dict_settings())
def set_add():
if request.method == 'POST':
id = request.form['id']
key = request.form.get('key')
describe = request.form.get('describe')
types = request.form.get('types', 0, type=int)
content = request.form.get('content')
text = request.form.get('text')
imgfile = request.files.get('imgfile')
base_dir = os.path.dirname(__file__)
if id == '':
value = text
if types == 1:
value = content
if types == 2:
image_path = '/app/static/uploads/' + time.strftime("%Y%m%d")
image_path_all = os.path.dirname(base_dir) + image_path
if not os.path.exists(image_path_all):
os.mkdir(image_path_all)
imgfile.save(image_path_all + "/" + imgfile.filename)
value = image_path + "/" + imgfile.filename
times = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
set = Dictionary(key=key,
value=value,
types=types,
date=times,
describe=describe)
db.session.add(set)
db.session.commit()
else:
set = Dictionary.query.filter_by(id=id).first()
value = text
if types == 1:
value = content
if types == 2:
image_path = '/app/static/uploads/' + time.strftime("%Y%m%d")
image_path_all = os.path.dirname(base_dir) + image_path
if not os.path.exists(image_path_all):
os.mkdir(image_path_all)
imgfile.save(image_path_all + "/" + imgfile.filename)
value = '/static/uploads/' + time.strftime(
"%Y%m%d") + "/" + imgfile.filename
set.key = key
set.value = value
set.describe = describe
set.types = types
db.session.add(set)
db.session.commit()
return redirect(url_for('admin.set'))
id = request.args.get('id', 0)
set_info = Dictionary.query.get(id)
return render_template('admin/set_add.html', set_info=set_info)
def add_records_to_database(app, database_path):
'''
function to add data to the database
'''
# generate randomness
seed(1)
# set up database
setup_db(app, database_path)
# add data to Categories
categories = ['Verb', 'Substantiv', 'Adjektive', 'Adverb']
for category in categories:
new_cate = Categories(category)
new_cate.insert()
# add data to Dictionary
user_ids = [
'google-oauth2|104536530909866680796',
'auth0|5ff8909c8efe020068c0c1d5', 'google-oauth2|104536530909866680796'
]
example_words = ['Stockholm', 'nifiken', 'Svenska', 'lathunde', 'Studera']
example_cate = [2, 3, 2, 2, 1]
for user_id in user_ids:
for word, category in zip(example_words, example_cate):
A_random_value = 'random value' + str(randint(0, 100))
new_word = Dictionary(user_id,
word,
category,
meaninginenglish=A_random_value)
new_word.insert()
# for each user, add a random answer record
for user_id in user_ids:
answer = True
questions_user = Dictionary.query.\
filter(Dictionary.user_id == user_id).all()
for question in questions_user:
newanswer_record = AnswerRecords(question.id, user_id, answer)
newanswer_record.insert()
def index():
query_form = QueryForm()
# 调用WTF的函数实现验证
if query_form.validate_on_submit():
# 验证通过 获取数据
input_word = query_form.word.data
# 判断单词是否存在 查询数据库
session = Session()
word = session.query(Dictionary).filter_by(word=input_word).first()
if word:
# try:
words = session.query(Dictionary).filter_by(word=input_word)
# print(words)
session.close()
return render_template("index.html", form=query_form, words=words)
else:
try:
paraph = youdao.get_data(input_word)
new_word = Dictionary(word=paraph[0],
IPA=paraph[1],
paraphrase=paraph[2],
example_sentence=paraph[3] + paraph[4],
other=paraph[5])
session.add(new_word)
session.commit()
flash("添加成功")
words = session.query(Dictionary).filter_by(word=input_word)
session.close()
return render_template("index.html",
form=query_form,
words=words)
except Exception as e:
print(e)
flash("添加失败")
session.rollback()
words = session.query(Dictionary).order_by(
func.rand()).limit(5)
session.close()
return render_template("index.html",
form=query_form,
words=words)
session = Session()
words = session.query(Dictionary).order_by(func.rand()).limit(5)
session.close()
return render_template("index.html", form=query_form, words=words)
def build_tweet_dictionary():
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG)
from twitter_stream.models import Tweet
stoplist = stopwords.words('english')
texts = DbTexts(Tweet)
tokenized = Tokenizer(texts, stoplist=stoplist)
# build a dictionary
print "Building a dictionary"
dictionary = corpora.Dictionary(tokenized)
# Remove extremely rare words
dictionary.filter_extremes(no_below=2, no_above=0.5, keep_n=None)
dictionary.compactify()
# Save it in the database
print "Saving the tweet dict"
from models import Dictionary
return Dictionary.create_from_gensim_dictionary(dictionary, "tweets dictionary")
def _store(self,
id=None,
code=None,
description=None,
content=None,
common=None,
state=None,
meta=None,
profile_ids=None,
is_enabled=None,
hidden=None,
action=None,
kind=None,
parts=None,
session=None):
if action == 'update':
dictionary_model = self.dictionary.get(id, session=session)
if not dictionary_model:
raise ApiError(code="NOT_EXISTS",
message="Can't find dictionary with id=%r" % id)
else:
dictionary_model = Dictionary(
created=datetime.now(),
version=0,
)
if code is not None:
dictionary_model.code = code
if description is not None:
dictionary_model.description = description
if content is not None:
dictionary_model.content = content
if common is not None:
dictionary_model.common = common
if kind is not None:
dictionary_model.kind = kind
if state is not None:
dictionary_model.state = state
if meta is not None:
dictionary_model.meta = meta
if profile_ids is not None:
dictionary_model.profile_ids = profile_ids
if is_enabled is not None:
dictionary_model.is_enabled = is_enabled
if hidden is not None:
dictionary_model.hidden = hidden
if parts is not None:
dictionary_model.parts = parts
dictionary_model.version += 1
session.add(dictionary_model)
session.flush()
data = dictionary_model.to_dict()
return data
def build_script_dictionary(dataset_id):
dataset = Dataset.objects.get(pk=dataset_id)
dictionary = Dictionary._build_gensim_dictionary(dataset=dataset, scripts=dataset.scripts.all() )
dictionary._vectorize_corpus()
new_language.parent_client_id = parent_ids['client_id']
new_language.parent_object_id = parent_ids['object_id']
DBSession.flush()
dictionary_ids = dict()
for dictionary in old_DBSession.query(old_Dictionary).all():
if dictionary.parent_client_id and dictionary.parent_object_id:
parent_ids = language_ids[str(dictionary.parent_client_id) + '_' +
str(dictionary.parent_object_id)]
else:
parent_ids = {'client_id': None, 'object_id': None}
continue
new_dictionary = Dictionary(
client_id=dictionary.client_id,
parent_client_id=parent_ids['client_id'],
parent_object_id=parent_ids['object_id'],
translation_gist_client_id=translation_strings[
dictionary.translation_string]['ids']['client_id'],
translation_gist_object_id=translation_strings[
dictionary.translation_string]['ids']['object_id'],
marked_for_deletion=dictionary.marked_for_deletion)
if dictionary.additional_metadata:
new_dictionary.additional_metadata = json.loads(
dictionary.additional_metadata)
state_ids = translation_strings[dictionary.state]['ids']
new_dictionary.state_translation_gist_object_id = state_ids['object_id']
new_dictionary.state_translation_gist_client_id = state_ids['client_id']
DBSession.add(new_dictionary)
for group in old_DBSession.query(old_Group).join(old_BaseGroup).filter(
old_Group.subject_client_id == dictionary.client_id,
old_Group.subject_object_id == dictionary.object_id,
old_BaseGroup.dictionary_default == True).all():
parent_ids = language_ids[str(language.parent_client_id) + '_' + str(language.parent_object_id)]
new_language.parent_client_id = parent_ids['client_id']
new_language.parent_object_id = parent_ids['object_id']
DBSession.flush()
dictionary_ids = dict()
for dictionary in old_DBSession.query(old_Dictionary).all():
if dictionary.parent_client_id and dictionary.parent_object_id:
parent_ids = language_ids[str(dictionary.parent_client_id) + '_' + str(dictionary.parent_object_id)]
else:
parent_ids = {'client_id': None, 'object_id': None}
continue
new_dictionary = Dictionary(client_id=dictionary.client_id,
parent_client_id=parent_ids['client_id'],
parent_object_id=parent_ids['object_id'],
translation_gist_client_id=translation_strings[dictionary.translation_string]['ids'][
'client_id'],
translation_gist_object_id=translation_strings[dictionary.translation_string]['ids'][
'object_id'],
marked_for_deletion=dictionary.marked_for_deletion
)
if dictionary.additional_metadata:
new_dictionary.additional_metadata = json.loads(dictionary.additional_metadata)
state_ids = translation_strings[dictionary.state]['ids']
new_dictionary.state_translation_gist_object_id = state_ids['object_id']
new_dictionary.state_translation_gist_client_id = state_ids['client_id']
DBSession.add(new_dictionary)
for group in old_DBSession.query(old_Group).join(old_BaseGroup).filter(
old_Group.subject_client_id == dictionary.client_id,
old_Group.subject_object_id == dictionary.object_id,
old_BaseGroup.dictionary_default == True).all():
# print(group.id)
