def logs_new_guild(guild_name, guild_server, guild_region):
try:
r = redis.from_url(os.environ.get('REDISTOGO_URL'))
except:
import config
r = redis.from_url(config.REDISTOGO_URL)
guild = {}
guild["guild_name"] = guild_name
guild["guild_server"] = guild_server
guild["guild_region"] = guild_region
guild["logs"] = []
start_time = 1438387200000
response = requests.get("https://www.warcraftlogs.com:443/v1/reports/guild/"+guild_name+"/"+guild_server+"/"+guild_region+"?start="+str(start_time)+"&api_key=9457bbf774422ab14b5625efb2b35e36")
response = json.loads(response.text)
# print response
for log in response:
if log["zone"] == 8:
new_log = {}
new_log["log_id"] = log["id"]
new_log["title"] = log["title"]
new_log["start"] = log["start"]/1000
new_log["date"] = datetime.date.fromtimestamp(log["start"]/1000)
new_log["owner"] = log["owner"]
guild["logs"].append(new_log)
guild["last_checked"] = int(time.time())
guild["last_checked_dt"] = datetime.datetime.fromtimestamp(guild["last_checked"])
guild_id_string = guild_name+"_"+guild_server+"_"+guild_region
guild = analyze_guild_logs(guild, r)
r.hmset(guild_id_string, guild)
return guild
def image_upload():
"""
The image upload route
"""
urls = request.json['urls']
# get credentials from the redis server
rd = redis.from_url(current_app.config['REDIS_URL'])
try:
credentials = rd.get("credentials")
except:
response_object = {'status': "error, no access token is found, without it you can no longer upload image files."}
return jsonify(response_object)
# create new Task
tsk = Task()
tsk.initialize(urls,credentials)
# create and add Job
with Connection(redis.from_url(current_app.config['REDIS_URL'])):
q = Queue()
task = q.enqueue('project.server.main.works.long_work', tsk)
if task:
response_object = {'jobId': task.get_id()}
else:
response_object = {
'status': "error, could start job, this may mean that there are no workers running, or the redis server is down"}
return jsonify(response_object), 202
def last_log_from_guild(guild_name, guild_server, guild_region, r):
try:
r = redis.from_url(os.environ.get('REDISTOGO_URL'))
except:
import config
r = redis.from_url(config.REDISTOGO_URL)
guild = {}
guild["guild_name"] = guild_name
guild["guild_server"] = guild_server
guild["guild_region"] = guild_region
guild["logs"] = []
start_time = 1435734000000
end_time = 1437465661000
try:
response = requests.get("https://www.warcraftlogs.com:443/v1/reports/guild/"+guild_name+"/"+guild_server+"/"+guild_region+"?start="+str(start_time)+"&api_key=9457bbf774422ab14b5625efb2b35e36")
response = json.loads(response.text)
log_id = response[0]["id"]
try:
report = analyze(log_id)
except:
good_log = False
for log_dict in response[::-1]:
try:
log_id = log_dict["id"]
report = analyze(log_id)
good_log = True
except:
pass
if good_log == True:
break
except:
report = False
return report
def __get_connection(self) -> redis.Redis:
"""
Get a Redis connection
:return: Redis connection instance
:rtype: redis.Redis
"""
if self.__redis_use_socket:
r = redis.from_url(
'unix://{:s}?db={:d}'.format(
self.__redis_host,
self.__redis_db
)
)
else:
r = redis.from_url(
'redis://{:s}:{:d}/{:d}'.format(
self.__redis_host,
self.__redis_port,
self.__redis_db
)
)
if BlackRed.Settings.REDIS_AUTH is not None:
r.execute_command('AUTH {:s}'.format(BlackRed.Settings.REDIS_AUTH))
return r
def from_settings(settings):
host = settings.get('REDIS_HOST',REDIS_HOST)
port = settings.get('REDIS_PORT',REDIS_PORT)
redis_url = settings.get('REDIS_URL',REDIS_URL)
if redis_url:
redis.from_url(redis_url)
else:
redis.Redis(host = host,port = int(port))
def setup_rq_connection():
# 'RQ_DASHBOARD_REDIS_URL' environmental variable takes priority;
# otherwise, we look at the Flask app's config for the redis information.
if os.environ.get('RQ_DASHBOARD_REDIS_URL', None):
redis_conn = from_url(os.environ.get('RQ_DASHBOARD_REDIS_URL'))
elif current_app.config.get('REDIS_URL'):
redis_conn = from_url(current_app.config.get('REDIS_URL'))
else:
redis_conn = Redis(host=current_app.config.get('REDIS_HOST', 'localhost'),
port=current_app.config.get('REDIS_PORT', 6379),
password=current_app.config.get('REDIS_PASSWORD', None),
db=current_app.config.get('REDIS_DB', 0))
push_connection(redis_conn)
def get_redis_connection(config):
"""
Returns a redis connection from a connection config
"""
if "URL" in config:
return redis.from_url(config["URL"], db=config["DB"])
if "USE_REDIS_CACHE" in config.keys():
from django.core.cache import get_cache
cache = get_cache(config["USE_REDIS_CACHE"])
if hasattr(cache, "client"):
# We're using django-redis. The cache's `client` attribute
# is a pluggable backend that return its Redis connection as
# its `client`
try:
# To get Redis connection on django-redis >= 3.4.0
# we need to use cache.client.get_client() instead of
# cache.client.client used in older versions
try:
return cache.client.get_client()
except AttributeError:
return cache.client.client
except NotImplementedError:
pass
else:
# We're using django-redis-cache
return cache._client
return redis.Redis(host=config["HOST"], port=config["PORT"], db=config["DB"], password=config.get("PASSWORD", None))
def __get_redis(app_ctx):
"""Constructs Redis Client object.
:return: Redis Client object
"""
return redis.from_url(app_ctx.config['REDIS_URL'])
def main():
global LAST_UPDATE_ID
telegram_token = os.environ.get("TELEGRAM_TOKEN")
logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
logger = logging.getLogger("Maslahat.uz")
logger.setLevel(logging.DEBUG)
# logger.debug("Initalizing bot ...")
try:
bot = telegram.Bot(telegram_token)
# logger.debug("Connected to Telegram API")
except telegram.error.TelegramError:
pass
# logger.warning("Cannot connect to Telegram server!")
redis_url = os.environ.get("REDIS_URL")
redis_conn = redis.from_url(redis_url)
# logger.debug("Connected to Redis")
# logger.debug("Receiving updates ...")
try:
LAST_UPDATE_ID = bot.getUpdates()[-1].update_id
# logger.debug("Updates received")
except IndexError:
# logger.warning("No update received")
LAST_UPDATE_ID = None
# logger.debug("Starting heartbeat ...")
heart_beat(logger, stat)
# logger.debug("Waiting for updates ...")
while True:
bot_worker(redis_conn, bot, logger)
check_facebook(redis_conn, bot, logger)
check_announcements(redis_conn, bot, logger)
def __init__(self, cb):
# these instance variables are just for convenience
self.user = cb.user
self.config = cb.config
self.ws = cb
self.thread_pool_executor = futures.ThreadPoolExecutor(max_workers=20)
self.triggers = []
self.join_time = {}
self.current_users = cb.currentusers
self.battling = False # self.config.get('Chatbot', 'battle')
try:
redis_uname = self.config.get('External', 'redis_uname')
redis_pass = self.config.get('External', 'redis_pass')
redis_server = self.config.get('External', 'redis_server')
redis_url = os.getenv('REDISTOGO_URL', 'redis://%s:%s@%s' %
(redis_uname, redis_pass, redis_server))
self.redis = redis.from_url(redis_url)
# self.redis = redis.from_url('redis://127.0.0.1:6379')
except Exception as e:
print e
print "Redis connection failed (ignore if you're not using redis)"
self.initialise_triggers(self.config)
self.initialise_queue()
def get_redis_connection(config):
"""
Returns a redis connection from a connection config
"""
if 'URL' in config:
return redis.from_url(config['URL'], db=config['DB'])
if 'USE_REDIS_CACHE' in config.keys():
from django.core.cache import get_cache
cache = get_cache(config['USE_REDIS_CACHE'])
if hasattr(cache, 'client'):
# We're using django-redis. The cache's `client` attribute
# is a pluggable backend that return its Redis connection as
# its `client`
try:
return cache.client.client
except NotImplementedError:
pass
else:
# We're using django-redis-cache
return cache._client
return redis.Redis(host=config['HOST'],
port=config['PORT'], db=config['DB'],
password=config.get('PASSWORD', None))
def from_settings(settings):
"""
:param: settings object
:return: Channel object
"""
connection_type = settings.get('RABBITMQ_CONNECTION_TYPE', RABBITMQ_CONNECTION_TYPE)
connection_parameters = settings.get('RABBITMQ_CONNECTION_PARAMETERS', RABBITMQ_CONNECTION_PARAMETERS)
connection = {
'blocking': pika.BlockingConnection,
'libev': pika.LibevConnection,
'select': pika.SelectConnection,
'tornado': pika.TornadoConnection,
'twisted': pika.TwistedConnection
}[connection_type](pika.ConnectionParameters(**connection_parameters))
channel = connection.channel()
channel.basic_qos(prefetch_count=1)
url = settings.get('REDIS_URL', REDIS_URL)
host = settings.get('REDIS_HOST', REDIS_HOST)
port = settings.get('REDIS_PORT', REDIS_PORT)
# REDIS_URL takes precedence over host/port specification.
if url:
redis_server = redis.from_url(url)
else:
redis_server = redis.Redis(host=host, port=port)
return channel, redis_server
def __init__(self, hot_key='hot', url="redis://127.0.0.1", db=None, **kwargs):
self._conn = None
self._conn = redis.from_url(url, db=db, **kwargs)
try:
self._conn.ping()
except redis.ConnectionError, err:
raise ConnectionError(str(err))
def handle(self, *args, **options):
redis_conn = redis.from_url(os.getenv("REDISTOGO_URL", "redis://localhost:6379"))
states = redis_conn.get("states")
if states is None:
raise CommandError('"states" are not in redis cache!')
states = loads(states)
for state_data in states:
self.stdout.write("Processing state: %s\n" % state_data["NAME"])
get_statistics_for_area(None, state_data["state"])
counties = redis_conn.get("counties")
if counties is None:
raise CommandError('"counties" are not in redis cache!')
counties = loads(counties)
for county_data in counties:
self.stdout.write("Processing county: %s\n" % county_data["NAME"])
get_statistics_for_area(None, "%s,%s" % (county_data["state"], county_data["county"]))
for state_data in states:
state_msas = redis_conn.get("msa,%s" % state_data["state"])
if state_msas is None:
raise CommandError('"MSAs" for %s are not in redis cache!' % state_data["state"])
state_msas = loads(state_msas)
for msa_data in state_msas:
self.stdout.write("Processing msa: %s\n" % msa_data["NAME"])
get_statistics_for_area(
None,
"%s,%s,"
% (msa_data["state"], msa_data["metropolitan statistical area/micropolitan statistical area"]),
)
def __init__(self, url, key, db=None, includes=None, excludes=None, **kwargs):
super(RedisHandler, self).__init__()
self.__key = key
self.__includes = includes
self.__excludes = excludes or ("args", "exc_info", "msg", "stack_info")
self.__redis = redis.from_url(url, db, **kwargs)
self.__redis.delete(self.__key) # remove key if exists
def settings_update(request, setting_to_set, new_value=None):
if request.method == "POST":
new_value = request.POST["new_value"]
# only except post? they have to be logged in anyway though...
try: # this could get a get_or_create but that limits us and would just make us write code if we wanted slightly different functionality
setting = AccountSetting.objects.get(user=request.user, setting_name=setting_to_set)
if setting.setting_value == new_value:
return HttpResponse(json.dumps({"msg": "no change"}), content_type='application/json', status=200)
except Exception as e:
print("Hoping it just didn't exit yes, just in case :: {}".format(e))
setting = AccountSetting()
setting.user = request.user
setting.setting_name = setting_to_set
print(setting_to_set, new_value)
if setting.setting_value == new_value:
pass
else:
setting.setting_value = new_value
setting.save()
# now that it's saved in the DB lets save it in the cache! someday...
try:
r = redis.from_url(os.getenv('REDISTOGO_URL', 'redis://localhost:6379'))
r.hset("user.settings.{}.hash".format(request.user.id), setting_to_set, new_value)
except Exception as e:
print(e)
return HttpResponse(json.dumps({"msg": "I'm not a useful return..."}), content_type='application/json', status=200)
def main():
rclient = redis.from_url(redis_url)
cache = rcache.Rcache(cache_url, server_id)
log_file = rclient.get("log_file")
log_pos = rclient.get("log_pos")
log_pos = int(log_pos) if log_pos else None
only_events = _trans_events(events)
only_events.append(RotateEvent)
stream = BinLogStreamReader(
connection_settings=mysql_settings,
server_id=server_id,
blocking=blocking,
only_events=only_events,
only_tables=tables,
only_schemas=schemas,
resume_stream=True, # for resuming
freeze_schema=False, # do not support alter table event for faster
log_file=log_file,
log_pos=log_pos)
row_count = 0
for binlogevent in stream:
if int(time.time()) - binlogevent.timestamp > binlog_max_latency:
logger.warn("latency[{}] too large".format(
int(time.time()) - binlogevent.timestamp))
logger.debug("catch {}".format(binlogevent.__class__.__name__))
if isinstance(binlogevent, RotateEvent): #listen log_file changed event
rclient.set("log_file", binlogevent.next_binlog)
rclient.set("log_pos", binlogevent.position)
logger.info("log_file:{}, log_position:{}".format(
binlogevent.next_binlog, binlogevent.position))
else:
row_count += 1
table = "%s.%s" % (binlogevent.schema, binlogevent.table)
vals_lst = _get_row_values(binlogevent)
if not binlogevent.primary_key:
tables_without_primary_key.get(table, None)
try:
cache.save(table, binlogevent.primary_key, vals_lst)
logger.debug("save {} {} rows to cache".format(
table, len(vals_lst)))
except rcache.SaveIgnore as err:
logger.warning(str(err))
except rcache.FullError as err:
logger.info("cache OOM occured: {}.trigger dump command".format(
str(err)))
dump_code = _trigger_dumping()
cache.save(table, binlogevent.primary_key, vals_lst)
if cache_max_rows and cache.size > cache_max_rows:
logger.info("cache size:{} >= {}, trigger dumping".format(
cache.size, cache_max_rows))
_trigger_dumping()
rclient.set("log_pos", binlogevent.packet.log_pos)
if row_count % 1000 == 0:
logger.info("save {} changed rows".format(row_count))
stream.close()
def __init__(self, author, text, bot, add_from_twitch=True, useOnlyText=False):
self.redis = redis.from_url(config.redisURL)
twitch_from_prefix = "(From Twitch)"
self.author = author
options_exist = "options" in bot
if options_exist:
options = bot['options']
display_exist = options_exist and "displayFromMessages" in options
display_from_messages_disabled = display_exist and options['displayFromMessages'] is False
if display_from_messages_disabled:
add_from_twitch = False
message = ""
if add_from_twitch:
message = twitch_from_prefix + " "
if author and useOnlyText is not True:
message = message + author + ": "
self.message = message + text
self.bot_id = bot['_id']
def assignTemplateForView(request):
"""
accept post, else use get function
"""
if request.method == "POST":
try:
print(request.POST["viewName"])
temp = UserTemplates.objects.get(user=request.user, viewName=request.POST["viewName"])
except:
temp = UserTemplates()
form = UserTemplatesForm(request.POST, instance=temp)
# print(dir(form))
try:
form.full_clean()
itm = form.save()
redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379') # this is for the heroku install!
r = redis.from_url(redis_url)
try:
print("{0} - {1} - {2}".format("user.settings.{}.hash".format(itm.user.id), itm.viewName, itm.pathToTemplate))
x = r.hset("user.settings.{}.hash".format(itm.user.id), itm.viewName, itm.pathToTemplate)
print(x)
except Exception as e:
print("post set...")
print(e)
print("...post e")
except Exception as e:
print("Clean or save failed...")
print(e)
print(form.errors)
else:
form = UserTemplatesForm(initial={"user": request.user})
return render_to_response("user/user_template_form.html", {'form': form}, RequestContext(request))
def __init__(self, *args, **config):
super(RedisObserver, self).__init__(*args, **config)
self.redis = redis.from_url(config.get("redis_url",
"redis://localhost:6379/0"),
float(config.get("timeout", 5)))
self.restart_on_timeout = config.get("restart_on_timeout", None)
def setup_redis(args):
if args.url is not None:
redis_conn = redis.from_url(args.url, db=args.db)
else:
redis_conn = redis.Redis(host=args.host, port=args.port, db=args.db,
password=args.password)
use_connection(redis_conn)
def __init__(self):
self.redis_instance = redis.from_url(REDIS_URL)
self.recarea_list = []
self.small_recarea_dict = {
key.split('_')[0]: json.loads(self.redis_instance.get(key))
for key in self.redis_instance.keys() if key.find('_small') != -1
}
def test_get_website_from_redis(self):
#clear redis
redis = get_redis()
redis.flushall()
#store known website in redis
website = Website.objects.all()[0]
logging.debug('website from DB: %s' % website)
website.prepare()
website._special_attribute = 'special'
cache_website(website, get_website_redis_key(website.token), redis)
# retrieve via normal method
website_from_method = get_website(website.token)
self.assertEqual(website, website_from_method)
self.assertTrue(hasattr(website_from_method, '_special_attribute') and website_from_method._special_attribute == 'special')
#clear redis and store website in backup...
redis.flushall()
redis = switch_redis()
website._special_attribute = 'special backup'
cache_website(website, get_website_redis_key(website.token), redis)
# be very clear that it's in the backup
main_redis = redis.from_url(REDIS_URL) # Redis(**REDIS_DBS['default'])
main_redis.flushdb()
# retrieve via normal method
website_from_method = get_website(website.token)
self.assertEqual(website, website_from_method)
self.assertTrue(hasattr(website_from_method, '_special_attribute') and website_from_method._special_attribute == 'special backup')
def run(self):
REDIS_URL = os.environ.get('OPENREDIS_URL', 'redis://localhost:6379')
client = redis.from_url(REDIS_URL)
pubsub = client.pubsub()
pubsub.subscribe(['table'])
for message in pubsub.listen():
if message['type'] == 'subscribe':
continue
data = json.loads(message['data'])
self.output_device(data)
print('player: {} next: {} data: {}'
.format(self.name, self.next_player, data))
if data['action'] == 'quit':
if data['player'] == self.name:
break
if data['player'] == self.next_player:
self.next_player = data['next']
continue
if data['action'] == 'won':
if data['player'] == self.name:
break
if data['player'] == self.next_player:
self.next_player = data['next']
continue
if data['action'] == 'join':
if data['before'] == self.next_player:
self.next_player = data['player']
continue
self.current_middle = data['middle']
if data['next'] == self.name:
self.play(data)
def notification():
logging.basicConfig()
redis_instance = redis.from_url(settings.REDIS_URL)
garoa_response = requests.get('http://status.garoa.net.br/status')
if garoa_response.status_code == 200:
garoa_response_json = garoa_response.json()
garoa_opened = garoa_response_json.get('open', False)
garoa_opened_last_check = bool(redis_instance.get('GAROA_OPEN'))
if garoa_opened:
garoa_status = 'aberto!'
redis_instance.set('GAROA_OPEN', True)
else:
garoa_status = 'fechado.'
redis_instance.delete('GAROA_OPEN')
if garoa_opened != garoa_opened_last_check:
notification_request = {
'title': 'Garoa {status}'.format(status=garoa_status),
'type': 'note',
}
for client in PushbulletClient.objects.all():
response = requests.post('https://api.pushbullet.com/v2/pushes', auth=(client.access_token, ''), data=notification_request)
if response.status_code == 200:
logging.warn(u'Usuário {id} notificado.'.format(id=client.pk))
else:
logging.warn(u'Erro na notificação do Usuário {id}.'.format(id=client.pk))
else:
logging.warn(u'O status do Garoa não mudou desde a última verificação.')
else:
logging.warn(u'Erro na consulta à API do status do Garoa.')
def run(self, debug=None):
"""
启动
:param debug: 是否debug
:return:
"""
assert len(self.config['FORWARDER_INPUT_ADDRESS_LIST']) == len(self.config['FORWARDER_OUTPUT_ADDRESS_LIST'])
if self.config['REDIS_URL']:
import redis
rds = redis.from_url(self.config['REDIS_URL'])
self.share_store = ShareStore(rds,
self.config['REDIS_KEY_SHARE_PREFIX'] + self.config['REDIS_USER_KEY_PREFIX'],
self.config['REDIS_KEY_SHARE_PREFIX'] + self.config['REDIS_NODES_KEY'],
self.config['REDIS_USER_MAX_AGE']
)
if debug is not None:
self.debug = debug
workers = len(self.config['FORWARDER_INPUT_ADDRESS_LIST'])
def run_wrapper():
logger.info('Running server, debug: %s, workers: %s',
self.debug, workers)
setproctitle.setproctitle(self._make_proc_name('forwarder:master'))
# 只能在主线程里面设置signals
self._handle_parent_proc_signals()
self.proc_mgr.spawn_workers(workers, self._worker_run)
run_wrapper()
def particle(hydrodataset, part, model):
from paegan.logger import logger
from paegan.logger.redis_handler import RedisHandler
rhandler = RedisHandler(model.redis_log_channel, model.redis_url)
rhandler.setLevel(logging.PROGRESS)
logger.addHandler(rhandler)
try:
redis_connection = redis.from_url(model.redis_url)
forcer = BaseForcer(hydrodataset,
particle=part,
common_variables=model.common_variables,
times=model.times,
start_time=model.start,
models=model._models,
release_location_centroid=model.reference_location.point,
usebathy=model._use_bathymetry,
useshore=model._use_shoreline,
usesurface=model._use_seasurface,
reverse_distance=model.reverse_distance,
bathy_path=model.bathy_path,
shoreline_path=model.shoreline_path,
shoreline_feature=model.shoreline_feature,
time_method=model.time_method,
redis_url=model.redis_url,
redis_results_channel=model.redis_results_channel,
shoreline_index_buffer=model.shoreline_index_buffer
)
forcer.run()
except Exception:
redis_connection.publish(model.redis_results_channel, json.dumps({"status" : "FAILED", "uid" : part.uid }))
else:
redis_connection.publish(model.redis_results_channel, json.dumps({"status" : "COMPLETED", "uid" : part.uid }))
def send_directions_page(recipient, page_size):
redis_client = redis.from_url(os.getenv('REDIS_URL', 'redis://localhost:6379'))
key = STEPS_KEY_TMPL.format(phone_number=recipient)
steps = redis_client.lrange(key, 0, page_size - 1)
length = redis_client.llen(key)
redis_client.ltrim(key, page_size, length - 1)
head, tail = steps[:-1], steps[-1]
for step in head:
decoded = json.loads(step)
send_message(
recipient,
TWILIO_SHORTCODE,
body=decoded['text'],
media_urls=[decoded['image']],
)
decoded = json.loads(tail)
if redis_client.llen(key) > 0:
body = '{} (Reply "next" for next page)'.format(decoded['text'])
redis_client.expire(key, REDIS_EXPIRATION)
else:
body = decoded['text']
send_message(
recipient,
TWILIO_SHORTCODE,
body=body,
media_urls=[decoded['image']],
)
def test_add_command_two_lang(self):
get_dao().add_commands(gen_test_commands(10, 'zh_TW'))
get_dao().add_commands(gen_test_commands(20, 'en'))
r = redis.from_url(REDIS_URL)
assert 10 == len(r.keys('COMMAND::zh_TW::*'))
assert 20 == len(r.keys('COMMAND::en::*'))
assert 30 == len(r.keys('COMMAND::*'))
def update_progress(self, finish=None):
""" Function was rewritten from original """
if self.request.id:
if finish:
self.update_state(None, PROGRESS, {
'progress_percent': 100,
'time_remaining': 0,
})
else:
try:
r = redis.from_url(settings.BROKER_URL)
celery_task_key = 'celery_%s' % self.__class__.name
t_start, t_estimated = r.hmget(celery_task_key,
['%s_time_start' % self.request.id,
'%s_time_estimated' % self.request.id])
t_start, t_estimated = int(t_start), int(t_estimated)
cur_time = int(time())
total_time = t_estimated - t_start
part_time = cur_time - t_start
if total_time:
progress_percent = 100 * part_time / total_time
time_remaining = t_estimated - cur_time
else:
progress_percent = 100
time_remaining = 0
self.update_state(None, PROGRESS, {
'progress_percent': progress_percent,
'time_remaining': time_remaining,
})
except Exception as e:
logger.debug(e)
logger.debug('Redis doesn\'t work 4')
def __init__(self):
self.rediscon = redis.from_url('redis://localhost:6379')
self.underlying_data_nodes = {}
from flask import Flask, redirect
from flask import Flask, flash, redirect, render_template, request, session, abort, url_for, make_response
import os, random
from redis import StrictRedis, from_url
# http://cristian.regolo.cc/2015/07/07/introducing-the-geo-api-in-redis.html
app = Flask(__name__)
app.secret_key = 'super secret key'
## connect to redis at startup ; connection string in case of Heroku
connection_string = 'redis://*****:*****@app.route("/drivers/<int:id>/location", methods=["PUT"])
def put_driver(id):
s_lat = request.form.get('latitude')
s_lon = request.form.get('longitude')
print s_lat, s_lon, id, type(id)
lat = float(s_lat)
lon = float(s_lon)
if (lat > 90 or lat < -90 or lon > 90 or lon < -90):
return make_response(
'{"error" : "Invalid lat/lon must be between -90 to 90"}', 422)
if id > 50000 or id < 1:
return make_response('{}', 404)
def initialize(context):
log.info('Initializing Algorithm')
# Adjustable variables
context.avoid_trades = []
context.trade_restrictions = [
'ANDV', 'DDAIF', 'DHR', 'HL', 'FTV', 'LPX', 'DDAIF', 'NNHE', 'NVST',
'PNM', 'SKY', 'XSAU'
]
rebalance_hours = 0.0
rebalance_minutes = 8.0
context.long_exposure = 0.85
context.short_exposure = -0.85
context.num_longs = 15
context.num_shorts = 15
context.spyleverage = 1.25
context.std_cutoff = 0.15
# Fixed variables
rebalance_start_time = rebalance_hours * 60 + rebalance_minutes
context.combined_restrictions = context.avoid_trades + context.trade_restrictions
context.rebalance_complete = False
context.trade_queue = {}
context.rolling_portfolios = []
context.clear_queue_run = 0
context.long_weight = context.long_exposure / context.num_longs
context.short_weight = context.short_exposure / context.num_shorts
context.SPY = symbol('VOO')
r = redis.from_url(os.environ.get("REDIS_URL"))
try:
loaded_state = pickle.loads(r.get('pylivetrader_redis_state'))
loaded_ndays = loaded_state['ndays']
log.debug('Loaded ndays = {}'.format(loaded_ndays))
except:
loaded_ndays = 0
log.debug('No state has been loaded: ndays = {}'.format(loaded_ndays))
context.ndays = loaded_ndays
context.idays = 3
attach_pipeline(make_pipeline(), 'pipeline')
# Scheduling functions
schedule_function(rebalance, date_rules.every_day(),
time_rules.market_open(minutes=rebalance_start_time))
clear_queue_frequency = 1
clear_queue_duration = 56
clear_queue_start = int(rebalance_start_time) + 4
for minutez in range(clear_queue_start,
clear_queue_start + clear_queue_duration,
clear_queue_frequency):
schedule_function(clear_queue, date_rules.every_day(),
time_rules.market_open(minutes=minutez))
check_order_frequency = 10
check_order_duration = 50
check_order_start = int(rebalance_start_time) + 10
for minutez in range(check_order_start,
check_order_start + check_order_duration,
check_order_frequency):
schedule_function(check_order_status, date_rules.every_day(),
time_rules.market_open(minutes=minutez))
eod_operations_start_time = int(rebalance_start_time) + 70
schedule_function(
eod_operations, date_rules.every_day(),
time_rules.market_open(minutes=eod_operations_start_time))
def get_redis():
global REDIS_CONNECTION
if REDIS_CONNECTION is None:
REDIS_CONNECTION = redis.from_url(get_redis_url())
return REDIS_CONNECTION
import redis
from rq import Worker, Connection
from config import REDIS_QUEUE, REDIS_HOST
if __name__ == '__main__':
redis_connection = redis.from_url(REDIS_HOST)
with Connection(redis_connection):
worker = Worker(REDIS_QUEUE)
worker.work()
'naturaldate': humanize.naturaldate,
'naturaltime': humanize.naturaltime,
'naturalsize': humanize.naturalsize,
'datetime': format_datetime,
'isodatetime': format_isodatetime,
'format_currency': format_currency,
'uuid': format_uuid,
})
app.static_folder = 'static'
if app.config['PROXY_FIX']:
from werkzeug.contrib.fixers import ProxyFix
app.wsgi_app = ProxyFix(app.wsgi_app,
num_proxies=app.config['PROXY_FIX_NUM_PROXIES'])
redis_conn = redis.from_url(app.config['REDIS_URL'])
cache = RedisCache(host=redis_conn)
csrf = CSRFProtect(app)
db = SQLAlchemy(app)
migrate = Migrate(app, db)
from wuvt.auth import AuthManager
auth_manager = AuthManager()
auth_manager.db = db
auth_manager.init_app(app)
if len(app.config['SENTRY_DSN']) > 0:
sentry_sdk.init(app.config['SENTRY_DSN'],
integrations=[
FlaskIntegration(),
def deploy_sandbox_shared_setup(log, verbose=True, app=None, exp_config=None):
"""Set up Git, push to Heroku, and launch the app."""
if verbose:
out = None
else:
out = open(os.devnull, "w")
config = get_config()
if not config.ready:
config.load()
heroku.sanity_check(config)
(heroku_app_id, tmp) = setup_experiment(
log, debug=False, app=app, exp_config=exp_config
)
# Register the experiment using all configured registration services.
if config.get("mode") == "live":
log("Registering the experiment on configured services...")
registration.register(heroku_app_id, snapshot=None)
# Log in to Heroku if we aren't already.
log("Making sure that you are logged in to Heroku.")
heroku.log_in()
config.set("heroku_auth_token", heroku.auth_token())
log("", chevrons=False)
# Change to temporary directory.
cwd = os.getcwd()
os.chdir(tmp)
# Commit Heroku-specific files to tmp folder's git repo.
git = GitClient(output=out)
git.init()
git.add("--all")
git.commit('"Experiment {}"'.format(heroku_app_id))
# Initialize the app on Heroku.
log("Initializing app on Heroku...")
team = config.get("heroku_team", None)
heroku_app = HerokuApp(dallinger_uid=heroku_app_id, output=out, team=team)
heroku_app.bootstrap()
heroku_app.buildpack("https://github.com/stomita/heroku-buildpack-phantomjs")
# Set up add-ons and AWS environment variables.
database_size = config.get("database_size")
redis_size = config.get("redis_size")
addons = [
"heroku-postgresql:{}".format(quote(database_size)),
"heroku-redis:{}".format(quote(redis_size)),
"papertrail",
]
if config.get("sentry"):
addons.append("sentry")
for name in addons:
heroku_app.addon(name)
heroku_config = {
"aws_access_key_id": config["aws_access_key_id"],
"aws_secret_access_key": config["aws_secret_access_key"],
"aws_region": config["aws_region"],
"auto_recruit": config["auto_recruit"],
"smtp_username": config["smtp_username"],
"smtp_password": config["smtp_password"],
"whimsical": config["whimsical"],
"DASHBOARD_PASSWORD": fake.password(length=20, special_chars=False),
"DASHBOARD_USER": config.get("dashboard_user", "admin"),
"FLASK_SECRET_KEY": codecs.encode(os.urandom(16), "hex"),
}
# Set up the preferred class as an environment variable, if one is set
# This is needed before the config is parsed, but we also store it in the
# config to make things easier for recording into bundles.
preferred_class = config.get("EXPERIMENT_CLASS_NAME", None)
if preferred_class:
heroku_config["EXPERIMENT_CLASS_NAME"] = preferred_class
heroku_app.set_multiple(**heroku_config)
# Wait for Redis database to be ready.
log("Waiting for Redis...")
ready = False
while not ready:
try:
r = redis.from_url(heroku_app.redis_url)
r.set("foo", "bar")
ready = True
except (ValueError, redis.exceptions.ConnectionError):
time.sleep(2)
log("Saving the URL of the postgres database...")
config.extend({"database_url": heroku_app.db_url})
config.write()
git.add("config.txt")
time.sleep(0.25)
git.commit("Save URL for database")
time.sleep(0.25)
log("Generating dashboard links...")
heroku_addons = heroku_app.addon_parameters()
heroku_addons = json.dumps(heroku_addons)
if six.PY2:
heroku_addons = heroku_addons.decode("utf-8")
config.extend({"infrastructure_debug_details": heroku_addons})
config.write()
git.add("config.txt")
time.sleep(0.25)
git.commit("Save URLs for heroku addon management")
time.sleep(0.25)
# Launch the Heroku app.
log("Pushing code to Heroku...")
git.push(remote="heroku", branch="HEAD:master")
log("Scaling up the dynos...")
default_size = config.get("dyno_type")
for process in ["web", "worker"]:
size = config.get("dyno_type_" + process, default_size)
qty = config.get("num_dynos_" + process)
heroku_app.scale_up_dyno(process, qty, size)
if config.get("clock_on"):
heroku_app.scale_up_dyno("clock", 1, size)
time.sleep(8)
# Launch the experiment.
log("Launching the experiment on the remote server and starting recruitment...")
launch_url = "{}/launch".format(heroku_app.url)
log("Calling {}".format(launch_url), chevrons=False)
launch_data = _handle_launch_data(launch_url, error=log)
result = {
"app_name": heroku_app.name,
"app_home": heroku_app.url,
"dashboard_url": "{}/dashboard/".format(heroku_app.url),
"recruitment_msg": launch_data.get("recruitment_msg", None),
}
log("Experiment details:")
log("App home: {}".format(result["app_home"]), chevrons=False)
log("Dashboard URL: {}".format(result["dashboard_url"]), chevrons=False)
log(
"Dashboard user: {}".format(heroku_config.get("DASHBOARD_USER")), chevrons=False
)
log(
"Dashboard password: {}".format(heroku_config.get("DASHBOARD_PASSWORD")),
chevrons=False,
)
log("Recruiter info:")
log(result["recruitment_msg"], chevrons=False)
# Return to the branch whence we came.
os.chdir(cwd)
log(
"Completed Heroku deployment of experiment ID {} using app ID {}.".format(
config.get("id"), heroku_app_id
)
)
return result
import os, sys
from environs import Env
sys.path.append(os.path.dirname(os.path.realpath(__file__)))
import redis
from rq import Worker, Queue, Connection
env = Env()
env.read_env()
listen = ['default']
redis_url = env.str("REDISTOGO_URL", default="redis://localhost:6379")
conn = redis.from_url(redis_url)
if __name__ == '__main__':
with Connection(conn):
worker = Worker(list(map(Queue, listen)))
worker.work(with_scheduler=True)
# Braintree settings
BRAINTREE_MERCHANT_ID = 'wpb3qct435mfmfqp'
BRAINTREE_PUBLIC_KEY = 's2g7pgqyz7x7z656' # Merchant ID
# Public Key
BRAINTREE_PRIVATE_KEY = 'e662bef77c6008f8491262b9b9d30024' # Private key
from braintree import Configuration, Environment
Configuration.configure(Environment.Sandbox, BRAINTREE_MERCHANT_ID,
BRAINTREE_PUBLIC_KEY, BRAINTREE_PRIVATE_KEY)
# REDIS Settings
REDIS_HOST = 'localhost'
REDIS_PORT = 6379
REDIS_DB = 1
import redis
redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')
redis = redis.from_url(redis_url)
CHANNEL_LAYERS = {
"default": {
"BACKEND": "asgi_redis.RedisChannelLayer",
"CONFIG": {
"hosts": [os.environ.get('REDIS_URL', 'redis://localhost:6379')],
},
"ROUTING": "chat.routing.channel_routing",
},
}
# coding=utf-8
import telebot
from emoji import emojize
import redis
from redis import StrictRedis
r = redis.from_url(
'redis://*****:*****@ec2-54-247-139-72.eu-west-1.compute.amazonaws.com:23429'
)
TOKEN = '204715944:AAHhnfXiWQDcpKAPiCRuv0GVOENNoDQylvw'
bot = telebot.TeleBot(TOKEN)
heart = emojize(':heart:', use_aliases=True)
right = emojize(':right_arrow:', use_aliases=True)
left = emojize(':left_arrow:', use_aliases=True)
ledger = emojize(':ledger:', use_aliases=True)
phone = emojize(':speech_balloon:', use_aliases=True)
faqq = emojize(':page_facing_up:', use_aliases=True)
info = emojize(':information:', use_aliases=True)
house = emojize(':house:', use_aliases=True)
r.set(int(0), "Лера\nhttps://telegra.ph/file/fb134b7947d17e4522981.png"
"\n\nStatus: ")
r.set(int(1), "Софи\nhttps://telegra.ph/file/2848b75621de5868a9814.png"
"\n\nStatus: ")
r.set(
int(2), "Виктория\nhttps://telegra.ph/file/59880dca747f0556b3948.png"
"\n\nStatus: ")
def __init__(self):
self.client = redis.from_url(settings.REDIS_URL)
def __init__(self, secret, redis_uri="redis://localhost:6379"):
if secret is None:
raise ValueError("Secret required")
print "secret", secret, redis_uri
self.redis = redis.from_url(redis_uri)
self.secret = secret
import json
import os
import threading
import urllib
from dropbox import Dropbox, DropboxOAuth2Flow
from dropbox.files import DeletedMetadata, FolderMetadata, WriteMode
from flask import abort, Flask, redirect, render_template
from flask import Response, request, session, url_for
import redis
from data_cleaning_tools import clean_data
redis_url = os.environ['REDISTOGO_URL']
print("hello world! this is the redis url: {}".format(redis_url))
redis_client = redis.from_url(redis_url)
# App key and secret from the App console (dropbox.com/developers/apps)
APP_KEY = os.environ['APP_KEY']
APP_SECRET = os.environ['APP_SECRET']
app = Flask(__name__)
app.debug = True
# A random secret used by Flask to encrypt session data cookies
app.secret_key = os.environ['FLASK_SECRET_KEY']
def get_url(route):
'''Generate a proper URL, forcing HTTPS if not running locally'''
print("GET_URL...")
def __init__(self, setting, crawler):
RetryMiddleware.__init__(self, setting)
self.rconn = redis.from_url(setting['REDIS_URL'],
db=1,
decode_response=True)
init_cookie(self.rconn, crawler.spider.name)
def log_post(id):
r = redis.from_url(os.environ.get("REDIS_URL"))
r.set(id, 'true')
app.debug = True
app.config['DEBUG'] = True
app.config["DEBUG_TB_INTERCEPT_REDIRECTS"] = False
app.config["SQLALCHEMY_RECORD_QUERIES"] = True
app.config["SECRET_KEY"] = os.getenv("SECRET_KEY")
toolbar = DebugToolbarExtension(app)
# gzip responses
Compress(app)
app.config["COMPRESS_DEBUG"] = compress_json
# for running rq jobs
ti_queues = []
redis_rq_conn = redis.from_url(os.getenv("REDIS_URL",
"redis://127.0.0.1:6379"),
db=0)
for i in range(0, 2): # number of queues to spin up
ti_queues.append(Queue("ti-queue-{}".format(i), connection=redis_rq_conn))
# aws s3 connection
s3_conn = boto.connect_s3(os.getenv("AWS_ACCESS_KEY_ID"),
os.getenv("AWS_SECRET_ACCESS_KEY"))
requests_cache_bucket = s3_conn.get_bucket('tng-requests-cache')
# imports got here for tables that need auto-created.
# import publication
# import version
#
# db.create_all()
try:
with urllib.request.urlopen("https://raw.githubusercontent.com/corbindavenport/tootbot/update-check/current-version.txt") as url:
s = url.read()
new_version = s.decode("utf-8").rstrip()
current_version = 2.4 # Current version of script
if (current_version < float(new_version)):
print('[WARN] A new version of Tootbot (' + str(new_version) + ') is available! (you have ' + str(current_version) + ')')
print('[WARN] Get the latest update from here: https://github.com/corbindavenport/tootbot/releases')
else:
print('[ OK ] You have the latest version of Tootbot (' + str(current_version) + ')')
url.close()
except BaseException as e:
print('[EROR] Error while checking for updates:', str(e))
# Connect to Redis database
try:
r = redis.from_url(os.environ.get("REDIS_URL"))
except BaseException as e:
print('[EROR] Error while connecting to Redis:', str(e))
print('[EROR] Tootbot cannot continue, now shutting down')
exit()
# General settings
DELAY_BETWEEN_TWEETS = int(os.environ.get('DELAY_BETWEEN_POSTS', None))
POST_LIMIT = int(os.environ.get('POST_LIMIT', None))
SUBREDDIT_TO_MONITOR = os.environ.get('SUBREDDIT_TO_MONITOR', None)
NSFW_POSTS_ALLOWED = bool(distutils.util.strtobool(
os.environ.get('NSFW_POSTS_ALLOWED', None)))
SPOILERS_ALLOWED = bool(distutils.util.strtobool(
os.environ.get('SPOILERS_ALLOWED', None)))
SELF_POSTS_ALLOWED = bool(distutils.util.strtobool(
os.environ.get('SELF_POSTS_ALLOWED', None)))
if os.environ.get('HASHTAGS', None) == 'false':
def duplicate_check(id):
r = redis.from_url(os.environ.get("REDIS_URL"))
if r.get(id):
return True
else:
return False
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'posthog.settings')
app = Celery('posthog')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
# Connect to our Redis instance to store the heartbeat
redis_instance = redis.from_url(settings.REDIS_URL, db=0)
@app.on_after_configure.connect
def setup_periodic_tasks(sender, **kwargs):
# Heartbeat every 10sec to make sure the worker is alive
sender.add_periodic_task(10.0,
redis_heartbeat.s(),
name='10 sec heartbeat')
sender.add_periodic_task(
crontab(day_of_week='mon,fri'), # check twice a week
update_event_partitions.s(),
)
sender.add_periodic_task(15 * 60, calculate_cohort.s(), name='debug')
import arrow
import logging
import os
import requests
import redis
import sys
logger = logging.getLogger('osm')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
logger.addHandler(ch)
redis_url = os.environ.get("REDIS_URL")
r = redis.from_url(redis_url) if redis_url else dict()
new_user_json_url = 'https://s3.amazonaws.com/data.openstreetmap.us/users/newest.json'
slack_url = os.environ.get('SLACK_WEBHOOK_URL')
def send_to_slack(message):
logger.info("Telling Slack: %s", message)
return requests.post(slack_url, json={'text': message})
def interesting_change(feature):
props = feature.get('properties')
contains = props.get('inside')
if not contains:
return False
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.session = requests.Session()
self.redis = redis.from_url(settings.REDIS_URL)
self.to_save = []
2) Some classes instantiated (including DG auth, Google Cloud Storage)
create complex local context and require asyncio event
scheduling that cannot be pickled and therefore cannot be queued.
Via these wrappers, the context is created only in the worker process.
"""
import json
import os
import redis
from grabbers import base, dg, landsat, planet_grabber
# Heroku provides the env variable REDIS_URL for Heroku redis;
# the default redis://redis_db:6379 points to the local docker redis
redis_url = os.getenv('REDIS_URL', 'redis://redis_db:6379')
connection = redis.from_url(redis_url, decode_responses=True)
PROVIDER_CLASSES = {
'digital_globe': dg.DGImageGrabber,
'landsat': landsat.LandsatThumbnails,
'planet': planet_grabber.PlanetGrabber
}
def pull(db_key, provider, bbox, **specs):
"""Pull an image."""
grabber = PROVIDER_CLASSES[provider](**specs)
looped = base.loop(grabber.pull)
records = looped(bbox)
reformatted = _format_exceptions(*records)
connection.set(db_key, json.dumps(reformatted))
#!/usr/bin/env python
import os
import json
import redis
from flask import Flask
from flask import request
from linkextractor import extract_links
app = Flask(__name__)
redis_conn = redis.from_url(os.getenv("REDIS_URL", "redis://*****:*****@app.route("/")
def index():
return "Usage: http://<hostname>[:<prt>]/api/<url>"
@app.route("/api/<path:url>")
def api(url):
qs = request.query_string.decode("utf-8")
if qs != "":
url += "?" + qs
jsonlinks = redis_conn.get(url)
if not jsonlinks:
links = extract_links(url)
jsonlinks = json.dumps(links, indent=2)
redis_conn.set(url, jsonlinks)
response = app.response_class(status=200,
from rq import Connection, Queue
import redis
import requests
from flask import Flask, request, url_for, abort, \
render_template, send_from_directory, Response, redirect
from store import store_number
app = Flask(__name__, template_folder='templates')
app.config.update(dict(
DEBUG=True,
ENVIRONMENT="development",
))
conn = redis.from_url('redis://*****:*****@app.route('/trait')
def trait():
return render_template('trait.html')
@app.route('/insert', methods=["GET", "POST"])
def insert():
data = dict(request.get_json())['data']
if "value" not in data:
return json.dumps({
'success': False,
'err': 'value is missing'
def end_rate_limit(page):
r = redis.from_url(os.environ.get("REDIS_URL"))
r.set(redis_key(page, 'started'), pickle.dumps(None))
r.set(redis_key(page, 'finished'), pickle.dumps(datetime.utcnow()))
import os
import logging
import coloredlogs
from configparser import ConfigParser
from flask import Flask, escape, request, jsonify
import redis as redis_
from cloudapp import cloudapp
logger = logging.getLogger(__name__)
coloredlogs.install(level='INFO')
config = ConfigParser()
config.read('config.cfg')
app = Flask(__name__)
redis = redis_.from_url(config.get('database', 'redis'))
cloud = cloudapp(config.get('cloudapp', 'acc'),
config.get('cloudapp', 'passwd'))
def get_auth(auth_key):
result = list(redis.scan_iter('*'))
collect = []
collect_auth = []
for data in result:
collect.append(data.decode())
# data.decode()
for d in collect:
collect_auth.append(redis.get(d).decode())
logger.info(collect_auth)
if auth_key in collect_auth:
import os
from flask import Flask, render_template, request, redirect, jsonify
import shortener
import redis
from scout_apm.flask import ScoutApm
app = Flask(__name__)
ScoutApm(app)
db = redis.from_url(os.environ['REDISCLOUD_URL'])
# db=redis.Redis(host='localhost', port=6379, password='')
@app.route('/')
def main():
return render_template('index.html')
@app.route('/checkcustom/<url>')
def checkcustom(url):
response = db.hexists("custom", url)
print(type(response))
if (response):
return "1"
else:
return "0"
@app.route('/shorten/', methods=['POST', 'GET'])
def shorten():
def __init__(self, bot):
self.bot = bot
self.r = redis.from_url(os.environ['REDIS_URL'])
from flask import Flask, render_template, request, jsonify, url_for, redirect
from flask_caching import Cache
import search
import os
import redis
app = Flask(__name__)
# cache = Cache(app, config={"CACHE_TYPE": "simple"})
cache = redis.from_url(os.environ["REDISCLOUD_URL"])
@app.route("/", methods=["POST", "GET"])
def index():
# check if Lobby Name form is submitted via POST
if request.method == "POST":
lobby_name = request.form["Lobby Name"]
return redirect(url_for("show_lobby_name", lobby_name=lobby_name))
# otherwise return regular index.html
return render_template("index.html")
# opens a new lobby page with given lobby name
@app.route("/lobby/<lobby_name>")
def show_lobby_name(lobby_name):
return render_template("lobby.html", name=lobby_name)
@app.route("/test")
def show_test():
return render_template("test.html")
def run_worker():
redis_url = app.config['REDIS_URL']
redis_connection = redis.from_url(redis_url)
with Connection(redis_connection):
worker = Worker(app.config['QUEUES'])
worker.work()