def test_03_user_scan(self):
rd = red.Red(TEST_RED_CONFIG)
# scanner
cursor = 0
pattern = rd.key_word()
# push random words
ws = ('hello', 'hi', 'word', '한글', '구글', '아마존')
for w in ws:
wk = rd.key_word(w)
rd.entity_mset(wk, {'word': w, 'timestamp': int(time.time())})
fits = list()
while cursor is not None:
(cursor, keys) = rd.scan(cursor, pattern, 2)
if cursor == 0: break
if len(keys) <= 0: continue
for rk in keys:
self.assertTrue(rd.entity_exists(rk))
self.assertTrue(rd.entity_hexists(rk, 'word'))
w = rd.entity_get(rk, 'word')
#print(w)
fits.append(w)
rd.connection().delete(rk)
self.assertEqual(len(ws), len(fits))
self.assertEqual(len(fits), len(set(fits)))
def __init__(self, name, red_config_path) :
self._name = name
self.rd = red.Red(red_config_path)
self.rk = self.rd.key_service(self._name)
self.cursor = self.get_cursor()
self.loops = 0
self.inserts = list()
self.deletes = list()
self.updates = list()
self.log('Service %s Started with %d'%(self._name, self.cursor))
def test_02_sets(self):
rd = red.Red(TEST_RED_CONFIG)
cnx = rd.connection()
test_rk = 'ict://test'
rv = int(time.time())
cnx.hset(test_rk, 'timestamp', rv)
cnx.expire(test_rk, 1)
self.assertTrue(cnx.exists(test_rk))
self.assertTrue(cnx.hexists(test_rk, 'timestamp'))
self.assertEqual(int(cnx.hget(test_rk, 'timestamp')), rv)
time.sleep(1)
self.assertFalse(cnx.exists(test_rk))
def __init__(self, config_path, red_config, channel, scan_pattern):
super().__init__(config_path, red_config)
self.rd = red.Red(red_config)
self.channel = channel
#channel_id = self.rd.entity_has_id(self.rd.key_channel(self.channel))
if self.channel is not None:
self.channel_id = self.CHANNEL_IDMAP[self.channel]
self.cached_users = dict()
self.cached_media = dict()
self.cached_interactions = list()
self.cursor = 0
self.scan_pattern = scan_pattern
#self.stacked = list()
self.access_token = None
self.session = requests.Session()
self.rk = None
#! -*- coding: utf -*-
import red
import requests
# insert api keys
rd = red.Red()
API_HOST = {
'facebook': 'https://graph.facebook.com/v2.11',
'instagram': 'https://api.instagram.com/v1',
'youtube': 'https://www.googleapis.com/youtube/v3',
'twitter': 'https://api.twitter.com/1.1'
}
ACCESS_KEYS = {
'facebook': [],
'instagram': [
'1245321415.1d02407.9813f1466f544fdf90844bc70eeeecb3',
'1296009759.1d02407.fa4e0770b50b4a61ab770e86590eadee',
'2265298105.1d02407.a358aeb0bb5d43e2b6834d969fad2bbd',
'242390511.1d02407.bbf2f14f6b9248149b7edf0dcd8f3f1c',
'2461051507.1d02407.98744e3033c844cca282b3a72656399a',
'2969404.1d02407.1628b3ba7faf4e889783e5333d162d35',
'3133448196.1d02407.0fb0e8226f6447ab8b9b7a6faec7558f',
'3134019004.1d02407.b360214bcee5427198369690391ac514',
'3176426362.1d02407.cba864b180864c2aa0617bbc03afd591',
'361296566.1d02407.ce11b3ca4df5432d8b4c8f585d1e18ba',
'3613877784.1d02407.3893150016fd4c74ad0828de23b9270f',
'3614340281.1d02407.8e61cac78d784aa586c7ac81c5b4a658',
'3614670677.1d02407.2106b368e9b34035b76f5ae76661741b',
class testPull(unittest.TestCase):
rd = red.Red(TEST_RED_CONFIG)
db = sql.Sql(TEST_SQL_CONFIG)
def setUp(self):
self.rd.connection().flushall()
def test_pull_91_words(self):
daemon = pull_words.Service(TEST_RED_CONFIG, TEST_SQL_CONFIG)
daemon.serve(0.1, 5, 600)
self.assertTrue(self.rd.entity_exists(daemon.rk))
self.assertTrue(self.rd.entity_hexists(daemon.rk, 'cursor'))
#self.assertTrue(int(self.rd.entity_get(daemon.rk, 'cursor'))>0)
(cursor, word_rks) = self.rd.scan(0, self.rd.key_channel_word(), 2000)
self.assertTrue(cursor is not None)
self.assertTrue(0 < len(word_rks))
for wk in word_rks:
self.assertTrue(self.rd.entity_has_id(wk))
word_data = self.rd.entity_gets(wk)
self.assertTrue(word_data is not None)
#print(word_data)
the_word_key = self.rd.key_word(word_data['word'])
self.assertEqual(int(self.rd.entity_get_id(the_word_key)),
int(word_data['word_id']))
def test_pull_92_users(self):
daemon = pull_users.Service(TEST_RED_CONFIG, TEST_SQL_CONFIG)
daemon.serve(0.1, 5, 600)
self.assertTrue(self.rd.entity_exists(daemon.rk))
self.assertTrue(self.rd.entity_hexists(daemon.rk, 'cursor'))
#self.assertTrue(int(self.rd.entity_get(daemon.rk, 'cursor'))>0)
(cursor, user_rks) = self.rd.scan(0, self.rd.key_channel_user(), 2000)
self.assertTrue(0 < len(user_rks))
for uk in user_rks:
self.assertTrue(self.rd.entity_has_id(uk))
user_data = self.rd.entity_gets(uk)
self.assertTrue(user_data is not None)
#print(user_data)
@unittest.skip('media not fulfilled')
def test_pull_93_media(self):
daemon = pull_media.Service(TEST_RED_CONFIG, TEST_SQL_CONFIG)
daemon.serve(0.1, 5, 600)
self.assertTrue(self.rd.entity_exists(daemon.rk))
self.assertTrue(self.rd.entity_hexists(daemon.rk, 'cursor'))
#self.assertTrue(int(self.rd.entity_get(daemon.rk, 'cursor'))>0)
(cursor, media_rks) = self.rd.scan(0, self.rd.key_channel_media(),
2000)
self.assertTrue(0 < len(media_rks))
for uk in media_rks:
self.assertTrue(self.rd.entity_has_id(uk))
media_data = self.rd.entity_gets(uk)
self.assertTrue(media_data is not None)
print(media_data)
def tearDown(self):
pass
def copy_tables(ldb, lcs, rcs, tables):
for tb in tables:
rcs.execute('select * from %s order by id desc limit 3000' % (tb))
rss = [rs for rs in rcs]
rcs.execute('desc %s' % (tb))
_atts = ','.join(['%s' for rs in rcs])
lcs.executemany('insert ignore %s values (%s)' % (tb, _atts), rss)
ldb.commit()
if FORCE_INIT:
rdb = connect_mysql(REMOTE_SQL_CONFIG)
rcs = rdb.cursor()
ldb = connect_mysql(TEST_SQL_CONFIG)
lcs = ldb.cursor()
rd = red.Red(TEST_RED_CONFIG)
rd.connection().flushall()
# drop every tables ldb has;
drop_all_tables(ldb, lcs)
# copy table structures from rdb;
create_stmts = import_table_creates(rcs)
create_all_tables(ldb, lcs, create_stmts)
# copy required tables
copy_tables(ldb, lcs, rcs,
('channels', 'users', 'currents', 'keywords',
'channel_keywords', 'channel_users', 'channel_media'))
lcs.execute(
'select id,title,searchable,user_trackable,media_trackable from channels'
)
def test_01_connection(self):
rd = red.Red(TEST_RED_CONFIG)
self.assertTrue(rd is not None)
self.assertTrue(rd.connection() is not None)
import sys
import time
from datetime import datetime
if __name__ == '__main__':
red_config = sys.argv[1] if 1 < len(sys.argv) and 1 < len(
sys.argv[1]) else './test/redis.json'
scan_pattern = sys.argv[2] if 2 < len(sys.argv) and 1 < len(
sys.argv[2]) else 'ict://'
scan_mnt = int(
sys.argv[3]) if 3 < len(sys.argv) and 1 < len(sys.argv[3]) else 100
print('%s> LOAD CONFIG %s' % (str(datetime.now()), red_config))
rd = red.Red(red_config)
cursor = 0
while cursor is not None:
print('%s> SCAN %d MATCH %s COUNT %d' %
(str(datetime.now()), cursor, scan_pattern, scan_mnt))
rs = rd.scan(cursor, scan_pattern, scan_mnt)
print(rs[0])
print(rs[1])
if rs[0] <= 0:
cursor = None
for rk in rs[1]:
if rd.entity_has_assigned(rk):
#! -*- coding: utf8 -*-
import sys
import red
import requests
# insert api keys
rd = red.Red(sys.argv[1]
if 1 < len(sys.argv) and 0 < len(sys.argv[1]) else 'redis.json')
API_HOST = {
'facebook': 'https://graph.facebook.com/v2.11',
'instagram': 'https://api.instagram.com/v1',
'youtube': 'https://www.googleapis.com/youtube/v3',
'twitter': 'https://api.twitter.com/1.1'
}
ACCESS_KEYS = {
'facebook': [],
'instagram': [
'1245321415.1d02407.9813f1466f544fdf90844bc70eeeecb3',
'1296009759.1d02407.fa4e0770b50b4a61ab770e86590eadee',
'2265298105.1d02407.a358aeb0bb5d43e2b6834d969fad2bbd',
'242390511.1d02407.bbf2f14f6b9248149b7edf0dcd8f3f1c',
'2461051507.1d02407.98744e3033c844cca282b3a72656399a',
'2969404.1d02407.1628b3ba7faf4e889783e5333d162d35',
'3133448196.1d02407.0fb0e8226f6447ab8b9b7a6faec7558f',
'3134019004.1d02407.b360214bcee5427198369690391ac514',
'3176426362.1d02407.cba864b180864c2aa0617bbc03afd591',
'361296566.1d02407.ce11b3ca4df5432d8b4c8f585d1e18ba',
def __init__(self, config_path, red_config, createIfNotExists=False):
self._config_path = config_path
self.rd = red.Red(red_config)
self.load_configure(createIfNotExists)
self.cursor = 0
class testTransact(unittest.TestCase):
rd = red.Red(TEST_RED_CONFIG)
db = sql.Sql(TEST_SQL_CONFIG)
rword = str(uuid.uuid4())
ruid = str(uuid.uuid4())
rmid = str(uuid.uuid4())
def setUp(self):
self.rd.connection().flushall()
def test_transact_91_word_insert(self):
rk = self.rd.key_channel_word('facebook', self.rword)
self.rd.entity_mset(
rk, {
'word': self.rword,
'channel': 'facebook',
'channel_id': 2,
'amounts': 100,
'uploads': 10,
'users': 5,
})
self.rd.entity_set_update(rk)
print((rk, self.rd.entity_gets(rk)))
daemon = transaction_words.Service(TEST_RED_CONFIG, TEST_SQL_CONFIG)
daemon.serve(0.1, 5, 60)
rss = self.db.select('select id,word from keywords where word like %s',
(self.rword, ))
self.assertTrue(0 < len(rss))
(word_id, word) = rss[0]
#print(('>>WORD', word_id, word))
rss = self.db.select(
'select * from channel_keywords where word_id=%d limit 1' %
(word_id))
self.assertTrue(0 < len(rss))
print(rss)
@unittest.skip('unhandle')
def test_transact_92_word_update(self):
rk = self.rd.key_channel_word('facebook', self.rword)
#ts = self.rd.entity_timestamp(rk)
rss = self.db.select('select id,word from keywords where word like %s',
(self.rword, ))
self.assertTrue(0 < len(rss))
(word_id, the_word) = rss[0]
self.rd.entity_set(rk, 'amounts', 1000)
self.rd.entity_set_update(rk)
self.assertTrue(self.rd.entity_is_update(rk))
daemon = transaction_words.Service(TEST_RED_CONFIG, TEST_SQL_CONFIG)
daemon.serve(0.1, 5, 600)
# now assigned;
rss = self.db.select(
'select id,amounts from channel_keywords where word_id=%s',
(word_id, ))
(channel_word_id, amounts) = rss[0]
self.assertEqual(amounts, int(self.rd.entity_get(rk, 'amounts')))
def test_transact_93_user_insert(self):
rk = self.rd.key_channel_user('instagram', self.ruid)
self.rd.entity_mset(rk, {
'uid': self.ruid,
'channel': 'instagram',
'channel_id': 3,
})
self.rd.entity_set_update(rk)
daemon = transaction_users.Service(TEST_RED_CONFIG, TEST_SQL_CONFIG)
daemon.serve(0.1, 5, 600)
self.assertTrue(self.rd.entity_has_id(rk))
self.assertFalse(self.rd.entity_is_update(rk))
print(self.rd.entity_gets(rk))
@unittest.skip('unhandle')
def test_transact_94_user_update(self):
rk = self.rd.key_channel_user('instagram', self.ruid)
#ts = self.rd.entity_timestamp(rk)
self.rd.entity_set(rk, 'subscribers', 1000)
self.rd.entity_set_update(rk)
daemon = transaction_users.Service(TEST_RED_CONFIG, TEST_SQL_CONFIG)
daemon.serve(0.1, 5, 600)
# now assigned;
self.assertFalse(self.rd.entity_is_update(rk))
def test_transact_95_media_insert(self):
rk = self.rd.key_channel_media('instagram', self.ruid, self.rmid)
self.rd.entity_mset(
rk, {
'uid': self.ruid,
'mid': self.rmid,
'channel': 'instagram',
'channel_id': 3,
})
self.rd.entity_set_update(rk)
daemon = transaction_media.Service(TEST_RED_CONFIG, TEST_SQL_CONFIG)
daemon.serve(0.1, 5, 600)
self.assertTrue(self.rd.entity_has_id(rk))
self.assertFalse(self.rd.entity_is_update(rk))
print(self.rd.entity_gets(rk))
@unittest.skip('unhandle')
def test_transact_96_media_update(self):
rk = self.rd.key_channel_media('instagram', self.ruid, self.rmid)
#ts = self.rd.entity_timestamp(rk)
self.rd.entity_set(rk, 'interactions', 1000)
self.rd.entity_set_update(rk)
daemon = transaction_media.Service(TEST_RED_CONFIG, TEST_SQL_CONFIG)
daemon.serve(0.1, 5, 600)
# now assigned;
self.assertFalse(self.rd.entity_is_update(rk))
def tearDown(self):
pass