def resubmit_jobs():
if config.get('ckan.harvest.mq.type') != 'redis':
return
redis = get_connection()
log.debug('_create_or_update_package')
harvest_object_pending = redis.keys('harvest_object_id:*')
for key in harvest_object_pending:
date_of_key = datetime.datetime.strptime(redis.get(key),
"%Y-%m-%d %H:%M:%S.%f")
if (datetime.datetime.now() - date_of_key).seconds > 180: # 3 minuites for fetch and import max
redis.rpush('harvest_object_id',
json.dumps({'harvest_object_id': key.split(':')[-1]})
)
redis.delete(key)
harvest_jobs_pending = redis.keys('harvest_job_id:*')
for key in harvest_jobs_pending:
date_of_key = datetime.datetime.strptime(redis.get(key),
"%Y-%m-%d %H:%M:%S.%f")
if (datetime.datetime.now() - date_of_key).seconds > 7200: # 3 hours for a gather
redis.rpush('harvest_job_id',
json.dumps({'harvest_job_id': key.split(':')[-1]})
)
redis.delete(key)
def prepare_sms_object():
no_of_sms = len(redis.keys('sms_*'))
sms_keys = redis.keys('sms_*')
sms_text = []
sms_time = []
sms_from = []
sms_to = []
sms_key = []
sms_replied = []
for key in sms_keys:
sms_replied.append(redis.hget(key,'Replied'))
sms_text.append(redis.hget(key,'Text'))
sms_from.append(redis.hget(key,'From'))
sms_to.append(redis.hget(key,'Plivo DID'))
sms_time.append(redis.hget(key,'Time Received'))
sms_key.append(key)
sms_object = {'sms_from':sms_from, 'sms_to':sms_to, 'sms_text':sms_text, 'sms_time':sms_time, 'sms_key':sms_key, \
'sms_replied':sms_replied}
response = make_response(render_template("sms_dashboard.html", no_of_sms = no_of_sms-1,\
sms_object = sms_object))
response.headers['Content-type'] = 'text/html'
return response
def resubmit_jobs():
if config.get('ckan.harvest.mq.type') != 'redis':
return
redis = get_connection()
log.debug('_create_or_update_package')
harvest_object_pending = redis.keys('harvest_object_id:*')
for key in harvest_object_pending:
date_of_key = datetime.datetime.strptime(redis.get(key),
"%Y-%m-%d %H:%M:%S.%f")
if (datetime.datetime.now() - date_of_key
).seconds > 180: # 3 minuites for fetch and import max
redis.rpush('harvest_object_id',
json.dumps({'harvest_object_id': key.split(':')[-1]}))
redis.delete(key)
harvest_jobs_pending = redis.keys('harvest_job_id:*')
for key in harvest_jobs_pending:
date_of_key = datetime.datetime.strptime(redis.get(key),
"%Y-%m-%d %H:%M:%S.%f")
if (datetime.datetime.now() -
date_of_key).seconds > 7200: # 3 hours for a gather
redis.rpush('harvest_job_id',
json.dumps({'harvest_job_id': key.split(':')[-1]}))
redis.delete(key)
def resubmit_jobs():
'''
Examines the fetch and gather queues for items that are suspiciously old.
These are removed from the queues and placed back on them afresh, to ensure
the fetch & gather consumers are triggered to process it.
'''
if config.get('ckan.harvest.mq.type') != 'redis':
return
redis = get_connection()
# fetch queue
harvest_object_pending = redis.keys(get_fetch_routing_key() + ':*')
for key in harvest_object_pending:
date_of_key = datetime.datetime.strptime(redis.get(key),
"%Y-%m-%d %H:%M:%S.%f")
# 3 minutes for fetch and import max
if (datetime.datetime.now() - date_of_key).seconds > 180:
redis.rpush(get_fetch_routing_key(),
json.dumps({'harvest_object_id': key.split(':')[-1]})
)
redis.delete(key)
# gather queue
harvest_jobs_pending = redis.keys(get_gather_routing_key() + ':*')
for key in harvest_jobs_pending:
date_of_key = datetime.datetime.strptime(redis.get(key),
"%Y-%m-%d %H:%M:%S.%f")
# 3 hours for a gather
if (datetime.datetime.now() - date_of_key).seconds > 7200:
redis.rpush(get_gather_routing_key(),
json.dumps({'harvest_job_id': key.split(':')[-1]})
)
redis.delete(key)
def resubmit_jobs():
'''
Examines the fetch and gather queues for items that are suspiciously old.
These are removed from the queues and placed back on them afresh, to ensure
the fetch & gather consumers are triggered to process it.
'''
if config.get('ckan.harvest.mq.type') != 'redis':
return
redis = get_connection()
# fetch queue
harvest_object_pending = redis.keys(get_fetch_routing_key() + ':*')
for key in harvest_object_pending:
date_of_key = datetime.datetime.strptime(redis.get(key),
"%Y-%m-%d %H:%M:%S.%f")
# 3 minutes for fetch and import max
if (datetime.datetime.now() - date_of_key).seconds > 180:
redis.rpush(get_fetch_routing_key(),
json.dumps({'harvest_object_id': key.split(':')[-1]})
)
redis.delete(key)
# gather queue
harvest_jobs_pending = redis.keys(get_gather_routing_key() + ':*')
for key in harvest_jobs_pending:
date_of_key = datetime.datetime.strptime(redis.get(key),
"%Y-%m-%d %H:%M:%S.%f")
# 3 hours for a gather
if (datetime.datetime.now() - date_of_key).seconds > 7200:
redis.rpush(get_gather_routing_key(),
json.dumps({'harvest_job_id': key.split(':')[-1]})
)
redis.delete(key)
def add_custom_commute_score_to_all(walk_weight=1,
bike_weight=1,
transit_weight=1,
drive_weight=1):
listings = redis.keys('house-search:listings/*')
if walk_weight < 0:
walk_weight = 1
log.info("Reset negative walk weight to 1")
if bike_weight < 0:
bike_weight = 1
log.info("Reset negative bike weight to 1")
if transit_weight < 0:
transit_weight = 1
log.info("Reset negative transit weight to 1")
if drive_weight < 0:
drive_weight = 1
log.info("Reset negative drive weight to 1")
weighted_sum = walk_weight + bike_weight + transit_weight + drive_weight
if weighted_sum <= 0:
# Someone entered weird values
log.info("Resetting weights due to weighted_sum being <=0")
weighted_sum = 4
walk_weight = bike_weight = transit_weight = drive_weight = 1
for i in listings:
k = i.decode()
if "latitude" in k or "longitude" in k or "/downtown" in k or "/poi/" in k or "score" in k:
pass
# these keys do not represent a listing
else:
try:
walk_score = bike_score = transit_score = drive_score = 0
pois = redis.keys(k + '/poi/*')
if len(pois) > 0:
for place in pois:
str_place = place.decode()
data = ast.literal_eval(redis.get(str_place).decode())
walk_score = walk_score + calculate_walk_score(
data['walk_time'])
bike_score = bike_score + calculate_bike_score(
data['bike_time'])
drive_score = drive_score + calculate_drive_score(
data['drive_time'])
transit_score = transit_score + calculate_transit_score(
data['transit_time'],
data['transit_route']['routes'][0])
score = (walk_score * walk_weight + bike_score *
bike_weight + transit_score * transit_weight +
drive_score * drive_weight) / (weighted_sum *
len(pois))
redis.set(k + "/custom_commute_score", score)
except Exception as e:
log.exception(traceback.format_exc())
def resubmit_jobs():
'''
Examines the fetch and gather queues for items that are suspiciously old.
These are removed from the queues and placed back on them afresh, to ensure
the fetch & gather consumers are triggered to process it.
'''
if config.get('ckan.harvest.mq.type') != 'redis':
return
redis = get_connection()
# fetch queue
harvest_object_pending = redis.keys(get_fetch_routing_key() + ':*')
for key in harvest_object_pending:
redis_value = redis.get(key)
if redis_value is None:
log.info(
'Fetch Queue: Redis cannot get value for key {}'.format(key))
continue
date_of_key = datetime.datetime.strptime(redis_value,
"%Y-%m-%d %H:%M:%S.%f")
log.debug('[Fetch queue]: Check key {} with value {}'.format(
key, date_of_key))
# 3 minutes for fetch and import max
if (datetime.datetime.now() - date_of_key).seconds > 180:
log.debug(
'[Fetch queue]: Re-new harvest object with KEY {} in redis'.
format(key))
redis.rpush(get_fetch_routing_key(),
json.dumps({'harvest_object_id': key.split(':')[-1]}))
redis.delete(key)
# gather queue
harvest_jobs_pending = redis.keys(get_gather_routing_key() + ':*')
for key in harvest_jobs_pending:
redis_value = redis.get(key)
if redis_value is None:
log.info(
'Gather Queue: Redis cannot get value for key {}'.format(key))
continue
date_of_key = datetime.datetime.strptime(redis_value,
"%Y-%m-%d %H:%M:%S.%f")
log.debug('[Gather queue]: Check key {} with value {}'.format(
key, date_of_key))
# 3 hours for a gather
if (datetime.datetime.now() - date_of_key).seconds > 7200:
log.debug(
'[Gather queue]: Re-new harvest job with KEY {} in redis'.
format(key))
redis.rpush(get_gather_routing_key(),
json.dumps({'harvest_job_id': key.split(':')[-1]}))
redis.delete(key)
def add_custom_commute_score_to_one(location,
walk_weight=1,
bike_weight=1,
transit_weight=1,
drive_weight=1):
listings = redis.keys('house-search:listings/*')
if walk_weight < 0:
walk_weight = 1
log.info("Reset negative walk weight to 1")
if bike_weight < 0:
bike_weight = 1
log.info("Reset negative bike weight to 1")
if transit_weight < 0:
transit_weight = 1
log.info("Reset negative transit weight to 1")
if drive_weight < 0:
drive_weight = 1
log.info("Reset negative drive weight to 1")
weighted_sum = walk_weight + bike_weight + transit_weight + drive_weight
if weighted_sum <= 0:
# Someone entered weird values
log.info("Resetting weights due to weighted_sum being <=0")
weighted_sum = 4
walk_weight = bike_weight = transit_weight = drive_weight = 1
pois = redis.keys(location.listing_key + '/poi/*')
if len(pois) > 0:
try:
walk_score = bike_score = transit_score = drive_score = 0
for place in pois:
str_place = place.decode()
data = ast.literal_eval(redis.get(str_place).decode())
walk_score = walk_score + calculate_walk_score(
data['walk_time'])
bike_score = bike_score + calculate_bike_score(
data['bike_time'])
drive_score = drive_score + calculate_drive_score(
data['drive_time'])
transit_score = transit_score + calculate_transit_score(
data['transit_time'], data['transit_route']['routes'][0])
score = (walk_score * walk_weight + bike_score * bike_weight +
transit_score * transit_weight +
drive_score * drive_weight) / (weighted_sum * len(pois))
redis.set(location.listing_key + "/custom_commute_score", score)
except Exception as e:
log.exception(traceback.format_exc())
def GET(self):
header = render.header()
nav = render.nav('master')
playlist_keys = redis.keys('playlist:*')
playlists = []
for i in playlist_keys:
playlists.append(i.split(':')[1])
playlists = sorted(playlists)
cue_keys = redis.keys("cue:*")
cues = []
for i in cue_keys:
cues.append(i.split(':')[1])
cues = sorted(cues)
return render.master(header, nav, playlists, cues)
def inject_availability_message():
if len(redis.keys('AVAILABILITY_MESSAGE')) == 1:
return {
"availability_message":
redis.get('AVAILABILITY_MESSAGE').decode('utf-8')
}
return {}
def do_create_total(redis):
count = 0
for key in redis.keys(FORMAT_LOGIN_DATE_TABLE4FISH%('*')):
count+=convert_util.to_int(redis.scard(key))
print 'do_create_total total[%s]'%(count)
redis.set("fish:login:per:day:total",count)
def getYearData(year):
'''
'''
if len(year) != 4:
return {"error": "Invalid parameter"}
redis = brconfig.getRedis()
#redis=getRedis()
pipe = redis.pipeline(transaction=True)
keyList = redis.keys("ids:date:%s????" % year)
monthList = []
for key in keyList:
monthList.append(key[9:15]) #like:201206
monthList = list(set(monthList)) #remove duplicate
yearData = {}
for month in monthList:
monthData = getMonthData(month)
errorCount = 0
liveCount = 0
for item in monthData:
errorCount += monthData[item]["error"]
liveCount += monthData[item]["live"]
yearData[month[4:6]] = {
"error": errorCount,
"live": liveCount,
"link": "summary?month=%s" % month
}
return yearData
def POST(self):
playlists = []
playlist_keys = redis.keys('playlist:*')
for i in playlist_keys:
playlists.append(i.split(':')[1])
print playlists
return json.dumps(playlists)
def export_data_to_csv():
try:
records = []
for key in redis.keys():
if not key in [
'last_block', 'last_block_time_seconds',
'last_usage_total_sent'
]:
accounts = list(set([key[:-12] for key in redis.hkeys(key)]))
for account in accounts:
cpu_usage_us = redis.hget(key, f'{account}-cpu-archive')
net_usage_words = redis.hget(key, f'{account}-net-archive')
record = {
'date': key,
'account': account,
'cpu_usage_us': cpu_usage_us,
'net_usage_words': net_usage_words
}
records.append(record)
with open('/data/accounts-usage.csv', 'w', encoding='utf8',
newline='') as output_file:
fc = csv.DictWriter(output_file, fieldnames=records[0].keys())
fc.writeheader()
fc.writerows(records)
logger.info('Exported DB to CSV!')
except Exception as e:
logger.info('Could not export data!')
logger.info(traceback.format_exc())
def list_users():
users = {}
for key in redis.keys():
if key.startswith(redis_prefix):
name = key[len(redis_prefix):]
users[name] = load(name)
return users
def getYearData(year):
'''
'''
if len(year)!=4:
return {"error":"Invalid parameter"}
redis=brconfig.getRedis()
#redis=getRedis()
pipe=redis.pipeline(transaction=True)
keyList=redis.keys("ids:date:%s????"%year)
monthList=[]
for key in keyList:
monthList.append(key[9:15])#like:201206
monthList=list(set(monthList))#remove duplicate
yearData={}
for month in monthList:
monthData=getMonthData(month)
errorCount=0
liveCount=0
for item in monthData:
errorCount+=monthData[item]["error"]
liveCount+=monthData[item]["live"]
yearData[month[4:6]]={"error":errorCount,"live":liveCount,"link":"summary?month=%s"%month}
return yearData
def print_all(redis, name):
"""
Function to print all the entries in our database
:param redis: Redis connection
:param name: task name
"""
if name is None:
# printing names of all the task lists in db
tasks = redis.keys()
if tasks:
stri = "Here are the names of all the task lists:\n"
for task in tasks:
stri += task + '\n'
return stri
else:
logging.warning('Oops..smth went wrong. List is empty')
return 'Nothing to print.'
else:
# printing all the tasks in the task list
stri = ''
tasks = List.pull_from_redis(redis, name, False)
if tasks and tasks is not None:
for key, value in tasks.iteritems():
stri += 'Task ' + key + ': ' + '\n'
for k, v in value.iteritems():
stri += k + ': ' + v + '\n'
stri += '--------------\n'
return stri
else:
logging.warning('Oops..smth went wrong. List is empty')
return 'Nothing to print.'
def GET(self):
header = render.header()
nav = render.nav('clips')
cue_keys = redis.keys("cue:*")
cues = []
for i in cue_keys:
cues.append(i.split(':')[1])
return render.clips(header, nav, cues)
def injectGlobals(bot, fromSession):
# Get the list of keys stored in this session
q = "{}.{}.*.*".format(GLOBALS_KEY, fromSession)
keys = redis.keys(q)
for k in keys:
keyname = k.split('.')[-2]
value = retrieveVariable(k)
bot.__dict__[keyname] = value
def cull_playlists(item):
playlist_keys = redis.keys('playlist:*')
for i in playlist_keys:
playlist_name = i.split(':')[1]
playlist = json.loads(redis.get(i))
cull_playlist(playlist, item)
redis.set('playlist:' + playlist_name, json.dumps(playlist))
redis.save()
def disks_usage(self):
disks = {}
disks_for_host = redis.keys('%s:disk_usage:*' % self.host)
for disk in disks_for_host:
disk_name = disk.split(':')[-2]
usage = map(clean_timeseries_array, redis.lrange(disk, -200, -1))
disks[disk_name] = usage
return disks
def disks_usage(self):
disks = {}
disks_for_host = redis.keys('%s:disk_usage:*' % self.host)
for disk in disks_for_host:
disk_name = disk.split(':')[-2]
usage = map(clean_timeseries_array, redis.lrange(disk, -200, -1))
disks[disk_name] = usage
return disks
def do_create_recharge_total(redis):
""" 统计所有的充值总数 """
count = 0
for key in redis.keys(FISH_SYSTEM_DATE_RECHARGE_TOTAL%('*')):
print key
count+=convert_util.to_int(redis.hget(key,'recharge_coin_total'))
print 'do_create_recharge_total total[%s]'%(count)
redis.set(FISH_SYSTEM_RECHARGE_TOTAL,count)
def GET(self):
header = render.header()
nav = render.nav('scheduling')
playlist_keys = redis.keys("playlist:*")
playlists = []
for i in playlist_keys:
playlists.append(i.split(':')[1])
playlists = sorted(playlists)
return render.scheduling(header, nav, playlists)
def render_list(key):
strategies = {
'::': render
, ':#': render
, ':-': lambda x: '\n'.join([ render(each) for each in redis.keys(x + ':*') ])
, '': str
}
return '\n'.join ([ definite_strategy (strategies, key[0:2], key, '')
for key in redis.lrange ( key, 0, -1 ) ])
def top():
keys = sorted(redis.keys())
res = ''
for key in keys:
s = '<p>%s %s</p>' % (key.decode('utf-8'),
redis.get(key).decode('utf-8'))
res += s
print(res) #debug
return res
def strings_redis():
import redis
#charset="utf-8", decode_responses=True => avoid b' in redis python
redis = redis.Redis(host='127.0.0.1',
port=6379,
db=0,
charset="utf-8",
decode_responses=True)
print("-------------")
print("STRINGS")
print("-------------")
#info()
print(redis.info())
print("-------------")
#monitor()
print(redis.monitor())
print("-------------")
#set()
redis.set("name", "javier")
redis.set("name", "jaime")
print("key: ", redis.get("name"))
print("-------------")
print("all keys: ", redis.keys())
print("keys with a 'name...': ", redis.keys("name*"))
print("keys with a 'e': ", redis.keys("*e*"))
print("-------------")
#setnx(name, value)
redis.set("name", "javier")
#mset(name, value)
redis.mset({"name": "peter", "name": "david"})
print("name: ", redis.mget("name"))
print("-------------")
#getrange(name, start, end) - substrings of the value
print("range : ", redis.getrange("name", 0, 3))
#delete all keys
for key in redis.scan_iter("prefix:*"):
redis.delete(key)
def index():
tweets = [redis.get(k) for k in redis.keys('tweet:*')]
return """
<form action="/tweet" method="post">
<textarea name="message"></textarea><br />
<input type="submit" name="submit" value="Enviar">
</form>
<br />
%s
""" % '<br /><br />'.join(tweets)
def POST(self):
print "GETTING CLIPS"
cues = {}
cue_keys = redis.keys('cue:*')
print cue_keys
for i in cue_keys:
cueName = i.split(':')[1]
cues[cueName] = redis.get(i)
print cues
return json.dumps(cues)
def repr_list(key):
strategies = {
'::': repr
, ':#': repr
, ':!': repr
, ':-': lambda x: [ repr(each) for each in redis.keys(x + ':*') ]
, '': str
}
return [ definite_strategy (strategies, key[0:2], key, '')
for key in redis.lrange ( key, 0, -1 ) ]
def prune_data():
try:
for key in redis.keys():
if not key in ['last_block', 'last_block_time_seconds', 'last_usage_total_sent']:
if datetime.strptime(key, '%Y-%m-%d') < datetime.utcnow() - timedelta(days=8):
redis.delete(key)
logger.info(f'Deleted old data from DB: {key}')
except Exception as e:
logger.info('Could not prune data!')
logger.info(traceback.format_exc())
def edit_playlists(old_name, new_name):
print "fixing playlists"
playlist_keys = redis.keys('playlist:*')
for i in playlist_keys:
playlist_name = i.split(':')[1]
playlist = json.loads(redis.get(i))
#print playlist
edit_playlist(playlist, old_name, new_name)
redis.set('playlist:' + playlist_name, json.dumps(playlist))
redis.save()
def dump_all(redis=r):
keys = redis.keys('*')
pairs = {}
for key in keys:
type = redis.type(key)
val = redis.get(key)
try:
pairs[key] = eval(val)
except:
pass
return pairs
def route_post_initialize():
redis = get_redis()
for key in redis.keys('isu4:*'):
redis.delete(key)
shutil.rmtree(get_dir('log'))
response = make_response('OK')
response.headers['Content-Type'] = 'text/plain'
return response
def dump_all(redis=r):
keys = redis.keys('*')
pairs = {}
for key in keys:
type = redis.type(key)
val = redis.get(key)
try:
pairs[key] = eval(val)
except Exception as e:
print pairs, key, val, e
return pairs
def route_post_initialize():
redis = get_redis()
for key in redis.keys('isu4:*'):
redis.delete(key)
shutil.rmtree(get_dir('log'))
response = make_response('OK')
response.headers['Content-Type'] = 'text/plain'
return response
def _get_like_list(ad_id):
likes = redis.keys('user:like:*:%s' % ad_id)
like_list = []
for like in likes:
l_user = redis.get(like)
l_date = redis.get('date:%s' % ":".join(like.split(':')[1:]))
like_list.append({'user': l_user, 'date': l_date})
return like_list
def pull_redis_queue(self, host="localhost", port=6379, **kwargs): #拉取对应redis下的队列
if not redis_enable:
raise RedisImportException
redis = redis.Redis(host = host, port = port, **kwargs)
for key in redis.keys():
if name[:11] == "redis_queue":
self.queue_dict[key] = RedisQ(key, **kwargs)
if self.queue_name_counter.has_key(queue_type):
self.queue_name_counter["redis_queue"] += 1
else:
self.queue_name_counter["redis_queue"] = 0
def process_queues(cls):
with utils.get_redis_for_cache() as redis:
LOG.info("smart strict workflow loop start")
for queue_name in redis.keys("strict-merge-queues~*"):
queue = cls.from_queue_name(redis, queue_name)
try:
queue.process()
except exceptions.MergifyNotInstalled:
queue.delete()
except Exception:
queue.log.error("Fail to process merge queue", exc_info=True)
LOG.info("smart strict workflow loop end")
def calc_per_recharge():
if not redis.exists(FISH_SYSTEM_RECHARGE_TOTAL):
do_create_recharge_total(redis)
else:
already_create_day = len(redis.keys(FISH_SYSTEM_DATE_RECHARGE_TOTAL%('*')))
already_recharge_total = convert_util.to_int(redis.get(FISH_SYSTEM_RECHARGE_TOTAL))
result = already_recharge_total/already_create_day
print 'already_create_day[%s] already_recharge[%s] result[%s]'%(already_create_day,already_recharge_total,result)
redis.set("fish:per:recharge:rate",result)
#删除当天充值人数统计
redis.delete(FISH_RECHARGE_USER_DAY_IDS)
def _get_scheduled_items(self, params):
date = params.date
date_key = 'scheduledItem:' + date + ':*'
scheduled_keys = redis.keys(date_key)
scheduled_items = {}
for i in scheduled_keys:
print "s item" + i
split_key = i.split(':')
hour = split_key[2]
minute = split_key[3]
playlist = redis.get(i)
scheduled_items[hour + ':' + minute] = playlist
return json.dumps(scheduled_items)
def calc_per_login():
if not redis.exists("fish:login:per:day:total"):
do_create_total(redis)
else:
today = date.today()
do_add_yesterday_data(redis,today-timeDelt)
already_create_day = len(redis.keys(FORMAT_LOGIN_DATE_TABLE4FISH%('*')))
already_login_total = convert_util.to_int(redis.get("fish:login:per:day:total"))
result = already_login_total/already_create_day
print 'already_create_day[%s] already_login_total[%s] result[%s]'%(already_create_day,already_login_total,result)
redis.set("fish:per:login:rate",result)
def get_recipes(search):
# returns all recipes with the search string in the name or description
results = []
for r in [redis.get(k) for k in redis.keys('*recipe*')]:
r = loads(r)
score = compute_match(search, r)
if score > 0:
results.append((r, score))
sorted_r = sorted(results, key=operator.itemgetter(1))
sorted_r.reverse()
print sorted_r
i = min(len(sorted_r), 10)
return sorted_r[:i]
def process_queues(cls):
# NOTE(sileht): Don't use the celery retry mechanism here, the
# periodic tasks already retries. This ensure a repo can't block
# another one.
redis = utils.get_redis_for_cache()
LOG.info("smart strict workflow loop start")
for queue_name in redis.keys("strict-merge-queues~*"):
queue = cls.from_queue_name(redis, queue_name)
queue.log.info("handling queue")
try:
queue.process()
except Exception:
queue.log.error("Fail to process merge queue", exc_info=True)
LOG.info("smart strict workflow loop end")
def pull_redis_queue(self,
host="localhost",
port=6379,
**kwargs): #拉取对应redis下的队列
if not redis_enable:
raise RedisImportException
redis = redis.Redis(host=host, port=port, **kwargs)
for key in redis.keys():
if name[:11] == "redis_queue":
self.queue_dict[key] = RedisQ(key, **kwargs)
if self.queue_name_counter.has_key(queue_type):
self.queue_name_counter["redis_queue"] += 1
else:
self.queue_name_counter["redis_queue"] = 0
def GET(self):
header = render.header()
nav = render.nav('playlists')
playlists = []
playlist_keys = redis.keys('playlist:*')
cues = []
cue_keys = redis.keys('cue:*')
for i in cue_keys:
cues.append(i.split(':')[1])
for i in playlist_keys:
playlist_name = i.split(':')[1]
playlist = json.loads(redis.get(i))
#new_item = {'name':playlist_name, 'val':playlist}
playlists.append({'name':playlist_name, 'val': playlist})
#playlists[playlist_name] = playlist
#playlists = sorted(playlists)
playlists = sorted(playlists, key=lambda k: k['name'])
playlist_keys = []
for i in playlists:
playlist_keys.append(i['name'])
print "returning plists"
print playlists
return render.playlists(header, nav, playlists, cues, playlist_keys)
def merge():
topic_count = defaultdict(int)
f = "word2count.txt"
keys = redis.keys("*")
for pos, key in enumerate(keys):
l = redis.hgetall(key)
print "1",pos, key
for k,v in l.iteritems():
topic_count[int(k)]+=int(v)
#word_topic_freq = defaultdict(list)
with open("word_tf.txt", "w") as word_freq:
for pos, word in enumerate(keys):
tf = []
l = redis.hgetall(word)
for topic, freq in l.iteritems():
topic = int(topic)
count = topic_count[topic]
if count < 10000:
continue
freq = int(freq)*500000/count
if freq > 0:
tf.append((topic, freq))
fcount = sum(i[1] for i in tf)
tf = dict(tf)
id = NAME2ID.get(name_tidy(word), 0)
if id:
t = tf.get(id,0)
diff = fcount - t
tf[id] = fcount
fcount += diff
if not fcount:
continue
t = []
for topic, f in tf.iteritems():
rank = int(f*10000/fcount)
if rank:
t.append((topic, rank))
if t:
word_freq.write(
dumps([word, t])+"\n"
)
def retrieve_mail(rcpt):
output = ""
try:
for key in redis.keys():
if key != 'mail:id':
if redis.hget(key, 'To:') == rcpt:
mail = redis.hgetall(key)
output += 'Date: ' + mail['Date:'] + '\n'
output += 'From: ' + mail['From:'] + '\n'
output += 'Message: \n' + mail['Message:'] + '\n'
output += "-------------------------------------\n"
except:
print 'Unexpected error:', sys.exc_info()[0]
exit(1)
return output
def stations():
try:
keys = [
key.decode().split(':').pop()
for key in redis.keys('services:*')
]
keys.sort()
key = 'services:{}'.format(keys.pop())
return redis.get(key)
except:
subprocess.Popen(['scrapy', 'crawl', 'bikesampa'])
return json.dumps([])
def getInvitationData(redis):
returnData = []
for invitation in redis.keys():
data = json.loads(redis.get(invitation))
data['user'] = invitation.decode()
data['url'] = getRegistrationUrl(invitation.decode(), data['token'])
del data['token']
returnData.append(data)
if len(returnData
) == 1: # add empty row so json2table will convert consistent
emptyDict = data.copy()
for key in emptyDict.keys():
emptyDict[key] = ''
returnData.append(emptyDict)
return returnData
def merge():
topic_count = defaultdict(int)
f = "word2count.txt"
keys = redis.keys("*")
for pos, key in enumerate(keys):
l = redis.hgetall(key)
print "1", pos, key
for k, v in l.iteritems():
topic_count[int(k)] += int(v)
#word_topic_freq = defaultdict(list)
with open("word_tf.txt", "w") as word_freq:
for pos, word in enumerate(keys):
tf = []
l = redis.hgetall(word)
for topic, freq in l.iteritems():
topic = int(topic)
count = topic_count[topic]
if count < 10000:
continue
freq = int(freq) * 500000 / count
if freq > 0:
tf.append((topic, freq))
fcount = sum(i[1] for i in tf)
tf = dict(tf)
id = NAME2ID.get(name_tidy(word), 0)
if id:
t = tf.get(id, 0)
diff = fcount - t
tf[id] = fcount
fcount += diff
if not fcount:
continue
t = []
for topic, f in tf.iteritems():
rank = int(f * 10000 / fcount)
if rank:
t.append((topic, rank))
if t:
word_freq.write(dumps([word, t]) + "\n")
def _is_reached_limit_like(user):
hour_ago = datetime.now() - timedelta(hours=1)
like_counter = 0
limit = 5
likes = redis.keys('user:like:*:%s' % 'ad:*')
like_list = []
for like in likes:
l_user = redis.get(like)
l_date = redis.get('date:%s' % ":".join(like.split(':')[1:]))
if user == l_user and hour_ago < datetime.strptime(
l_date, '%Y-%m-%d %H:%M'):
like_counter += 1
return like_counter >= limit
def setStuInfo2Class():
stu = Student.instance()
redis = stu.getDB()
sks = redis.keys('student:20*')
for sk in sks :
stuid = sk[8:]
try:
si = stu.getStuInfo(stuid)
except Exception,e:
continue
if si==None:
print "student None:",stuid
continue
cid = si['class']
ct = si['teacher']
try:
ci = stu.getClassInfo(cid)
except Exception,e:
print e
continue
# Contructing IO Devices
# a remote has to be attached to a controller
# Multiple controllers can interface to udp server but not to same controller
#
cf.add_udp_io_sever(name="main_remote", ip = "192.168.1.82", redis_key="MODBUS_STATISTICS:127.0.0.1",remote_type= "UDP", port=5005 )
cf.add_rtu_interface(name = "rtu_2",protocol="modify_modbus",baud_rate=38400 )
cf.add_remote( name="satellite_1",modbus_address=100,irrigation_station_number=44, card_dict={"open":"Remote 1 Open Wire","short":"Remote 1 Shorted Selenoid","connectivity":"Remote 1 Connectivity"})
cf.add_remote( name="satellite_2",modbus_address=125 ,irrigation_station_number=22,card_dict={"open":"Remote 2 Open Wire","short":"Remote 2 Shorted Selenoid","connectivity":"Remote 2 Connectivity"})
cf.add_remote( name="satellite_3",modbus_address=170,irrigation_station_number=22,card_dict={"open":"Remote 3 Open Wire","short":"Remote 3 Shorted Selenoid","connectivity":"Remote 3 Connectivity"})
cf.end_rtu_interface()
cf.end_udp_io_server()
cf.end_controller()
cf.end_site()
cf.end_system()
keys = redis.keys("*")
for i in keys:
print "+++++++++++++:"
print i
temp = i.split( common.sep)
print len(temp)
print redis.hgetall(i)
print "----------------"
print "lenght",len(keys)
print "testing query functions"
print qc.match_labels( "CONTROLLER" ) # match single item
temp = qc.match_labels( "REMOTE" ) # match single item
print len(temp),temp
from logicalblocks import getblocks
from logicalblocks import getdistance
from clusters import makeclusters
redis = redis.Redis(host='localhost', port=6379, db=0)
# DEBUG levels
# extractor
# distance
# cluster
# generic
DEBUG = 'clusters'
#DEBUG = 'distance'
filter = "*2015/02/13/7058423*"
keys = redis.keys(filter)
id = 0
for key in keys:
id = id + 1
type = redis.type(key);
value = ''
if type == 'string':
val = redis.get(key);
html = val
result = chardet.detect(html)
charset = result['encoding']
if charset == 'utf-8':
data = html
else:
def find(redis, key_namespace, predicate, action): keys = redis.keys(key_namespace + "*") for key in keys: value = redis.get(key) if predicate(key, value): action(key, value)
def delete_all(redis, prefix):
for key in redis.keys(REDIS_PREFIX + "*"):
redis.delete(key)
def del_all(redis=r):
keys = redis.keys('*')
for k in keys:
print 'Deleting:', k, 'result is', redis.delete(k)
for i in keys:
self.redis.delete(i)
if __name__ == "__main__":
# test driver
redis = redis.StrictRedis( host = "127.0.0.1", port=6379, db = 11 )
common = Redis_Graph_Common( redis)
redis_key, new_namespace =common.construct_node( [], "","head","head" )
print redis_key,new_namespace
print redis.hgetall(redis_key)
redis_key, new_namespace =common.construct_node( new_namespace,"relation 1","level_one","h1" )
print redis_key,new_namespace
print redis.hgetall(redis_key)
redis_key, new_namespace =common.construct_node( new_namespace,"relation 2","level_two","h2" )
print redis_key,new_namespace
print redis.hgetall(redis_key)
print "simple match"
print common.match( "relation 2","level_two","h2")
print "starting match"
print common.match( "*","level_two","h2",[["","head","head"]])
print "all the keys"
print redis.keys("*")
print "none of the keys"
common.delete_all()
print redis.keys("*")
