def setKWtoTweet(n = 1000):
try:
db.create_tables([Tweets], True)# 第二引数がTrueの場合、存在している場合は、作成しない
with db.transaction():
tweets = Tweets.select().where(Tweets.screen_name != '_umiA' and ~Tweets.text.contains('RT')).order_by(Tweets.createdat.desc()).limit(n)
tweetslist = [utiltools.cleanText(tweet.text) for tweet in tweets if 'RT' not in tweet.text]
return tweetslist
except Exception as e:
langDB.rollback()
def setKWtoTweet(n=1000):
try:
db.create_tables([Tweets], True) # 第二引数がTrueの場合、存在している場合は、作成しない
with db.transaction():
tweets = Tweets.select().where(
Tweets.screen_name != '_umiA'
and ~Tweets.text.contains('RT')).order_by(
Tweets.createdat.desc()).limit(n)
tweetslist = [
utiltools.cleanText(tweet.text) for tweet in tweets
if 'RT' not in tweet.text
]
return tweetslist
except Exception as e:
langDB.rollback()
def incDocFreq(word):
try:
langDB.create_tables([managementInfos, tfidf], True)
with langDB.transaction():
try:
wdb, created = tfidf.get_or_create(word=word)
if created != True:
try:
wdb.df += 1
except:
wdb.df = 1
wdb.save()
except Exception as e:
print('')
langDB.commit()
except Exception as e:
langDB.rollback()
def incDocFreq(word):
try:
langDB.create_tables([managementInfos, tfidf], True)
with langDB.transaction():
try:
wdb, created = tfidf.get_or_create(word = word)
if created != True:
try:
wdb.df += 1
except:
wdb.df = 1
wdb.save()
except Exception as e:
print('')
langDB.commit()
except Exception as e:
langDB.rollback()
def upsertWord(ma, isLearn=False):
genkei = ma[7]
hinshi = ma[1]
hinshi2 = ma[2]
DF = 1
try:
langDB.create_tables([managementInfos, tfidf], True)
with langDB.transaction():
try:
wdb, created = tfidf.get_or_create(word=genkei,
hinshi=hinshi,
hinshi2=hinshi2)
if isLearn == False:
DF = 1
elif created == True:
wdb.word = genkei
wdb.yomi = ma[8]
wdb.hinshi = ma[1]
wdb.hinshi2 = ma[2]
wdb.info3 = ma[3]
wdb.termcnt = 1
wdb.df = 1
wdb.tf = 1
wdb.save()
DF = 1
else:
try:
wdb.df += 1
except:
wdb.df = 1
wdb.save()
DF = wdb.df
except Exception as e:
print('')
langDB.commit()
except Exception as e:
langDB.rollback()
# print(e)
return DF
def upsertWord(ma, isLearn = False):
genkei = ma[7]
hinshi = ma[1]
hinshi2 = ma[2]
DF = 1;
try:
langDB.create_tables([managementInfos, tfidf], True)
with langDB.transaction():
try:
wdb, created = tfidf.get_or_create(word = genkei, hinshi = hinshi, hinshi2 = hinshi2)
if isLearn == False:
DF = 1
elif created == True:
wdb.word = genkei
wdb.yomi = ma[8]
wdb.hinshi = ma[1]
wdb.hinshi2 = ma[2]
wdb.info3 = ma[3]
wdb.termcnt = 1
wdb.df = 1
wdb.tf = 1
wdb.save()
DF = 1
else:
try:
wdb.df += 1
except:
wdb.df = 1
wdb.save()
DF = wdb.df
except Exception as e:
print('')
langDB.commit()
except Exception as e:
langDB.rollback()
# print(e)
return DF
