def setExpire(keyList, redisType, extime): if redisType not in ["string", "list", "set", "hash", "sortedset" ] or len(keyList) == 0: print "key type error! type: %s" % (redisType) return data = {} data[EX_KEY] = {} for key in keyList: stringKey = "%s_%s" % (redisType, key) data[EX_KEY][stringKey] = int(time.time()) + extime print "expire key: %s" % (stringKey) print "get expire key size: %d" % (len(data)) #print data #put data to ssdb s = SSDB(SSDB_HOST, SSDB_PORT) for key in data: s.multi_zset(key, **data[key]) print "zadd key: %s" % (key) print "set expire key finish"
def processSet(arg): keyList = arg[0] redisHost = arg[1] redisPort = arg[2] ssdbHost = arg[3] ssdbPort = arg[4] if len(keyList) == 0: return try: #get data from redis data = {} r = redis.Redis(host=redisHost, port=redisPort) pipeRedis = r.pipeline() #get keys for item in keyList: pipeRedis.smembers(item) retList = pipeRedis.execute() #print retList for i in range(len(retList)): if len(retList[i]) > 0: data[keyList[i]] = {} for item in retList[i]: data[keyList[i]][item] = 0 #put keys print "get redis data size: %d" % (len(data)) #print data #put data to ssdb s = SSDB(SSDB_HOST, SSDB_PORT) for key in data: s.zclear(key) s.multi_zset(key, **data[key]) print "zadd key: %s" % (key) #set expire setExpire(keyList, "sortedset", EX_TIME) except Exception, e: print e.message
import arrow from ssdb import SSDB import json ssdb = SSDB(host='104.131.54.255', port=8888) onlyfiles = [f for f in listdir(".") if isfile(join(".", f))] onlyfiles = [i for i in onlyfiles if "results-" in i] for file_name in onlyfiles: timestamp = arrow.utcnow().timestamp res = json.loads(open("results.json").read())[:] values = dict([[i["username"],i["followers"]] for i in res]) #print values[values.keys()[0]], values.keys()[0] ssdb.multi_zset("instagram-followers", **values) ssdb.multi_zset("test-instagram-followers", **values) values = dict([[i["username"], timestamp] for i in res]) ssdb.multi_zset("ig-last-crawled", **values) ssdb.multi_zset("ig-last-updated", **values) values = dict([[i[_id], i["username"]] for i in res]) ssdb.multi_hset("ig-username-id", **values) for c, user in enumerate(json.loads(open("results.json").read())[:]): print c un = user["username"] cols = ["profile_pic_url","full_name","followers","following","username"] vals = [user[c] for c in cols] info = dict(zip(cols, vals)) #Information
onlyfiles = [f for f in listdir(".") if isfile(join(".", f))] onlyfiles = [i for i in onlyfiles if "html" in i] results = [] for i in onlyfiles: print i try: f = open(i) html = f.read() results.append(Instagram()._profile(html)) f.close() except Exception as e: print e f = open("results.json", "w") f.write(json.dumps(results)) f.close() # upload ssdb res = json.loads(open("results.json").read())[:] values = dict([[i["username"],i["followers"]] for i in res]) ssdb.multi_zset("instagram-followers", **values) ssdb.multi_zset("swarm-instagram-followers", **values) try: bashCommand = "rm *.html" output = subprocess.check_output(['bash','-c', bashCommand]) except Exception as e: print e