def setExpire(keyList, redisType, extime):
if redisType not in ["string", "list", "set", "hash", "sortedset"
] or len(keyList) == 0:
print "key type error! type: %s" % (redisType)
return
data = {}
data[EX_KEY] = {}
for key in keyList:
stringKey = "%s_%s" % (redisType, key)
data[EX_KEY][stringKey] = int(time.time()) + extime
print "expire key: %s" % (stringKey)
print "get expire key size: %d" % (len(data))
#print data
#put data to ssdb
s = SSDB(SSDB_HOST, SSDB_PORT)
for key in data:
s.multi_zset(key, **data[key])
print "zadd key: %s" % (key)
print "set expire key finish"
def processSet(arg):
keyList = arg[0]
redisHost = arg[1]
redisPort = arg[2]
ssdbHost = arg[3]
ssdbPort = arg[4]
if len(keyList) == 0:
return
try:
#get data from redis
data = {}
r = redis.Redis(host=redisHost, port=redisPort)
pipeRedis = r.pipeline()
#get keys
for item in keyList:
pipeRedis.smembers(item)
retList = pipeRedis.execute()
#print retList
for i in range(len(retList)):
if len(retList[i]) > 0:
data[keyList[i]] = {}
for item in retList[i]:
data[keyList[i]][item] = 0
#put keys
print "get redis data size: %d" % (len(data))
#print data
#put data to ssdb
s = SSDB(SSDB_HOST, SSDB_PORT)
for key in data:
s.zclear(key)
s.multi_zset(key, **data[key])
print "zadd key: %s" % (key)
#set expire
setExpire(keyList, "sortedset", EX_TIME)
except Exception, e:
print e.message
import arrow
from ssdb import SSDB
import json
ssdb = SSDB(host='104.131.54.255', port=8888)
onlyfiles = [f for f in listdir(".") if isfile(join(".", f))]
onlyfiles = [i for i in onlyfiles if "results-" in i]
for file_name in onlyfiles:
timestamp = arrow.utcnow().timestamp
res = json.loads(open("results.json").read())[:]
values = dict([[i["username"],i["followers"]] for i in res])
#print values[values.keys()[0]], values.keys()[0]
ssdb.multi_zset("instagram-followers", **values)
ssdb.multi_zset("test-instagram-followers", **values)
values = dict([[i["username"], timestamp] for i in res])
ssdb.multi_zset("ig-last-crawled", **values)
ssdb.multi_zset("ig-last-updated", **values)
values = dict([[i[_id], i["username"]] for i in res])
ssdb.multi_hset("ig-username-id", **values)
for c, user in enumerate(json.loads(open("results.json").read())[:]):
print c
un = user["username"]
cols = ["profile_pic_url","full_name","followers","following","username"]
vals = [user[c] for c in cols]
info = dict(zip(cols, vals))
#Information
onlyfiles = [f for f in listdir(".") if isfile(join(".", f))]
onlyfiles = [i for i in onlyfiles if "html" in i]
results = []
for i in onlyfiles:
print i
try:
f = open(i)
html = f.read()
results.append(Instagram()._profile(html))
f.close()
except Exception as e:
print e
f = open("results.json", "w")
f.write(json.dumps(results))
f.close()
# upload ssdb
res = json.loads(open("results.json").read())[:]
values = dict([[i["username"],i["followers"]] for i in res])
ssdb.multi_zset("instagram-followers", **values)
ssdb.multi_zset("swarm-instagram-followers", **values)
try:
bashCommand = "rm *.html"
output = subprocess.check_output(['bash','-c', bashCommand])
except Exception as e:
print e
