def main(argvs, argc):
"""
このスクリプトでツイッターの情報を記録するデータベースを作成します
"""
if (argc != 2):
print "Usage #python %s dbname" % argvs[0]
return -1
dbname = argvs[1]
if os.path.exists(dbname):
print dbname, " already exists."
return -1
db = TwitterDb(dbname)
db.create_db()
print db.dbpath, " is created."
return 0
def main(argvs, argc):
"""
このスクリプトでツイッターの情報を記録するデータベースを作成します
"""
if(argc != 2):
print "Usage #python %s dbname" % argvs[0]
return -1
dbname = argvs[1]
if os.path.exists(dbname):
print dbname, " already exists."
return -1
db = TwitterDb(dbname)
db.create_db()
print db.dbpath, " is created."
return 0
def main():
search_term = ["トヨタ"]
if len(search_term) > 1:
term = " OR ".join(search_term)
since_id = 0
conf = ConfigParser.SafeConfigParser()
conf.read("twitter.ini")
print "==================================="
print "term :", term
print "==================================="
tw = TweeterSearcher(consumer_key=conf.get('Twitter',
'consumer_key'),
consumer_secret=conf.get('Twitter',
'consumer_secret'),
access_token_key=conf.get('Twitter',
'access_token_key'),
access_token_secret=conf.get('Twitter',
'access_token_secret'))
db = TwitterDb(host=conf.get("DB",
"host"),
user=conf.get("DB",
"user"),
pas=conf.get("DB",
"pass"),
database=conf.get("DB",
"database"))
"""
DBテーブルの生成。
存在しなければ作る。
"""
db.create_db()
cond = db.GetCondition(term)
tw_max_id = None
if cond is None:
tw_since_id = since_id
db.SetCondition(None, term, 0, 0, 0, since_id)
cond = db.GetCondition(term)
cond_id = cond['id']
else:
cond_id = cond['id']
if cond['found_since']:
tw_since_id = cond['max_id'] + 1
else:
tw_since_id = since_id
if cond['min_id'] > 0:
tw_max_id = cond['min_id']-1
print "tw_since_id :", tw_since_id
print "tw_max_id :", tw_max_id
ret = tw.StartSearch(term, tw_since_id, tw_max_id)
found = 0
for t in tw.GetTweets():
utc_dt = datetime.datetime.strptime(t['createdtime'], "%a %b %d %H:%M:%S +0000 %Y")
_zone = pytz.timezone("Asia/Tokyo")
jst_dt = _zone.fromutc(utc_dt)
db.AppendTweet(cond_id, t['id'], t['text'], t['user_id'], jst_dt.strftime("%Y-%m-%d %H:%M:%S"))
db.Commit()
if ret:
# 全レコードを読み込んだ
print "All tweets is imported."
found = 1
if len(tw.GetTweets()) > 0:
db.SetCondition(cond_id, term, found, tw.max_id, tw.min_id, since_id)
print "Import %d" % (len(tw.GetTweets()))
tm = tw.GetSearchReset()
print "REMAING:%d RESET-TIME: %d:%d" % (tw.GetSearchRemaining(),
tm.tm_hour,
tm.tm_min)
db.close()
return 0
def main(argvs, argc):
if argc != 4:
print "Usage: python search_to_db.py dbpath term since_id"
return -1
dbpath = argvs[1]
term = argvs[2]
since_id = int(argvs[3])
conf = ConfigParser.SafeConfigParser()
conf.read("twitter.ini")
print "==================================="
print "DB path:", dbpath
print "term :", term
print "since_id :", since_id
print "==================================="
# もし、UTF-8のターミナルなら以下の行はコメントアウトする
term = unicode(term, 'cp932').encode('utf-8')
tw = TweeterSearcher(consumer_key=conf.get('Twitter', 'consumer_key'),
consumer_secret=conf.get('Twitter',
'consumer_secret'),
access_token_key=conf.get('Twitter',
'access_token_key'),
access_token_secret=conf.get('Twitter',
'access_token_secret'))
db = TwitterDb(dbpath)
cond = db.GetCondition(term)
tw_max_id = None
tw_since_id = None
cond_id = None
if cond is None:
tw_since_id = since_id
db.SetCondition(None, term, 0, 0, 0, since_id)
cond = db.GetCondition(term)
cond_id = cond['id']
else:
cond_id = cond['id']
if cond['found_since']:
tw_since_id = cond['max_id'] + 1
else:
tw_since_id = since_id
if cond['min_id'] > 0:
tw_max_id = cond['min_id'] - 1
print "tw_since_id :", tw_since_id
print "tw_max_id :", tw_max_id
ret = tw.StartSearch(term, tw_since_id, tw_max_id)
found = 0
for t in tw.GetTweets():
utc = dateutil.parser.parse(t['createdtime']).utctimetuple()
tmstamp = time.mktime(utc)
db.AppendTweet(cond_id, t['id'], t['text'], t['user_id'], tmstamp)
if ret:
# 全レコードを読み込んだ
print "All tweets is imported."
found = 1
if len(tw.GetTweets()) > 0:
db.SetCondition(cond_id, term, found, tw.max_id, tw.min_id, since_id)
print "Import %d" % (len(tw.GetTweets()))
db.Commit()
tm = tw.GetSearchReset()
print "REMAING:%d RESET-TIME: %d:%d" % (tw.GetSearchRemaining(),
tm.tm_hour, tm.tm_min)
return 0
def setUp(self):
self._db = TwitterDb(self._dbpath)
self._db.create_db()
class TestTwitterDb(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._dbpath = (os.path.dirname(os.path.abspath(__file__)) +
"/test.sqlite")
if os.path.exists(cls._dbpath):
os.remove(cls._dbpath)
@classmethod
def tearDownClass(cls):
if os.path.exists(cls._dbpath):
os.remove(cls._dbpath)
def setUp(self):
self._db = TwitterDb(self._dbpath)
self._db.create_db()
def tearDown(self):
self._db.close()
self._db = None
os.remove(self._dbpath)
def test_createdb(self):
# createdb直後の状態をテストする
self.assertTrue(os.path.exists(self._dbpath), "Databaseファイルが存在しない")
self.assertIsNone(self._db.GetCondition("test"),
"作成直後のDBのsearch_conditionにデータが存在する")
def test_updateCondition(self):
#search_conditionの更新を確認する
self._db.SetCondition(None, "test", 0, 2, 3, 4)
cond = self._db.GetCondition("test")
self.assertEqual(cond['id'], 1)
self.assertEqual(cond['term'], 'test')
self.assertEqual(cond['found_since'], 0)
self.assertEqual(cond['max_id'], 2)
self.assertEqual(cond['min_id'], 3)
self.assertEqual(cond['since_id'], 4)
self._db.SetCondition(1, "test", 1, 3, 4, 5)
cond = self._db.GetCondition("test")
self.assertEqual(cond['id'], 1)
self.assertEqual(cond['term'], 'test')
self.assertEqual(cond['found_since'], 1)
self.assertEqual(cond['max_id'], 3)
self.assertEqual(cond['min_id'], 4)
self.assertEqual(cond['since_id'], 5)
def test_updateTweet(self):
# Tweetの追加の確認
self._db.AppendTweet(1, 1111, "TEST1", 2222, 3333)
self._db.AppendTweet(1, 1112, "TEST2", 2223, 3334)
self._db.AppendTweet(1, 1113, "TEST3", 2224, 3335)
self._db.AppendTweet(1, 1114, "TEST4", 2225, 3336)
self._db.AppendTweet(2, 9999, "TEST", 333, 777)
# 同じツイートが違う検索条件でひっかかる
self._db.AppendTweet(2, 1111, "TEST1", 2222, 3333)
self._db.Commit()
ret1 = self._db.GetTweets(1)
ret2 = self._db.GetTweets(2)
self.assertEqual(len(ret1), 4)
self.assertEqual(len(ret2), 2)
def setUp(self):
self._db = TwitterDb(self._dbpath)
self._db.create_db()
class TestTwitterDb(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls._dbpath = (os.path.dirname(os.path.abspath(__file__)) +
"/test.sqlite")
if os.path.exists(cls._dbpath):
os.remove(cls._dbpath)
@classmethod
def tearDownClass(cls):
if os.path.exists(cls._dbpath):
os.remove(cls._dbpath)
def setUp(self):
self._db = TwitterDb(self._dbpath)
self._db.create_db()
def tearDown(self):
self._db.close()
self._db = None
os.remove(self._dbpath)
def test_createdb(self):
# createdb直後の状態をテストする
self.assertTrue(os.path.exists(self._dbpath),
"Databaseファイルが存在しない")
self.assertIsNone(self._db.GetCondition("test"),
"作成直後のDBのsearch_conditionにデータが存在する")
def test_updateCondition(self):
#search_conditionの更新を確認する
self._db.SetCondition(None, "test", 0, 2, 3, 4)
cond = self._db.GetCondition("test")
self.assertEqual(cond['id'], 1)
self.assertEqual(cond['term'], 'test')
self.assertEqual(cond['found_since'], 0)
self.assertEqual(cond['max_id'], 2)
self.assertEqual(cond['min_id'], 3)
self.assertEqual(cond['since_id'], 4)
self._db.SetCondition(1, "test", 1, 3, 4, 5)
cond = self._db.GetCondition("test")
self.assertEqual(cond['id'], 1)
self.assertEqual(cond['term'], 'test')
self.assertEqual(cond['found_since'], 1)
self.assertEqual(cond['max_id'], 3)
self.assertEqual(cond['min_id'], 4)
self.assertEqual(cond['since_id'], 5)
def test_updateTweet(self):
# Tweetの追加の確認
self._db.AppendTweet(1, 1111, "TEST1", 2222, 3333)
self._db.AppendTweet(1, 1112, "TEST2", 2223, 3334)
self._db.AppendTweet(1, 1113, "TEST3", 2224, 3335)
self._db.AppendTweet(1, 1114, "TEST4", 2225, 3336)
self._db.AppendTweet(2, 9999, "TEST", 333, 777)
# 同じツイートが違う検索条件でひっかかる
self._db.AppendTweet(2, 1111, "TEST1", 2222, 3333)
self._db.Commit()
ret1 = self._db.GetTweets(1)
ret2 = self._db.GetTweets(2)
self.assertEqual(len(ret1), 4)
self.assertEqual(len(ret2), 2)