def run(app_name, command, process_name_start_to_restart=None):
cloud = heroku.from_key(os.getenv("HEROKU_API_KEY"))
app = cloud.apps[app_name]
if command == "memory":
# need o have done this for it to work: heroku labs:enable log-runtime-metrics
for process in app.processes:
process_name = process.process
logger.info(process_name)
for line in app.logs(num=100000, ps=process_name).split("\n"):
try:
if u"Error R14 (Memory quota exceeded)" in line or u"Error R15 (Memory quota vastly exceeded)" in line:
logger.info(line)
except Exception:
pass
if command == "restart":
print(
u"restarting {app_name}, processes that start with {process_name}".
format(app_name=app_name,
process_name=process_name_start_to_restart))
for process in app.processes:
process_name = process.process
process_name_start = process_name.split(".")[0]
if process_name_start == process_name_start_to_restart or not process_name_start_to_restart:
process.restart()
print(u"upon request in heroku_api, restarted {process_name}".
format(process_name=process_name))
def home(request):
cloud = heroku.from_key(settings.MAKAHIKI_HEROKU_KEY)
apps = cloud.apps
app_dict = {}
for app in apps:
if app.name not in ("kukuicup-uh", "kukuicup-ewc", "kukuicup-demo",
"makahiki-farm"):
app_info = app.__dict__
today = timezone.now() - datetime.timedelta(minutes=3)
complete_status = "complete"
if app_info["slug_size"] is None or app_info["slug_size"] == 0:
complete_status = "pending"
elif app_info["created_at"] >= today:
# if newly created, ping the app url to see if it is up
r = requests.get('https://%s.herokuapp.com' % app.name)
print "%s response code=%d" % (app.name, r.status_code)
if 200 != r.status_code:
complete_status = "pending"
app_info["completed_status"] = complete_status
app_dict[app.name] = app_info
keys = sorted(app_dict.keys())
app_list = []
for key in keys:
app_list.append((key, app_dict[key]))
return render_to_response('index.html', {'apps': app_list},
context_instance=RequestContext(request))
def _scale(self, num_change):
if self.key:
cloud = heroku.from_key(self.key)
app = cloud.apps[self.name]
workers = app.processes['worker']
len_workers = len([worker for worker in workers])
workers.scale(len_workers+num_change)
def setUp(self):
self.cloud = heroku.from_key(settings.HEROKU_KEY)
if 'rocket-1-1' in [x.name for x in self.cloud.apps]:
self.cloud.apps['rocket-1-1'].destroy()
self.user = OurUser.objects.get(id=1)
self.name = create_instance(self.user)
def load_remote_heroku_config(required=None, remote_config=None):
remote_config = None
new_config = {}
if not remote_config and "HEROKU_REMOTE_CONFIG" in os.environ:
remote_config = os.environ["HEROKU_REMOTE_CONFIG"]
if remote_config:
if not required:
raise Exception("Must pass required options if using HEROKU_REMOTE_CONFIG")
parts = remote_config.split(":")
if len(parts) < 2:
raise Exception("Error, bad format for HEROKU_REMOTE_CONFIG. Should be key:app-name")
key = parts[0]
app = parts[1]
import heroku
h = heroku.from_key(key)
h_app = h.apps[app]
for key in required:
if key in h_app.config.data:
if "DEBUG_SETTINGS" in os.environ:
print "Importing %s from %s" % (key, app)
val = h_app.config.data[key]
new_config[key] = cast_env_vars(val)
os.environ[key] = val
return new_config
def home(request):
cloud = heroku.from_key(settings.MAKAHIKI_HEROKU_KEY)
apps = cloud.apps
app_dict = {}
for app in apps:
if app.name not in ("kukuicup-uh", "kukuicup-ewc", "kukuicup-demo", "makahiki-farm"):
app_info = app.__dict__
today = timezone.now() - datetime.timedelta(minutes=3)
complete_status = "complete"
if app_info["slug_size"] is None or app_info["slug_size"] == 0:
complete_status = "pending"
elif app_info["created_at"] >= today:
# if newly created, ping the app url to see if it is up
r = requests.get('https://%s.herokuapp.com' % app.name)
print "%s response code=%d" % (app.name, r.status_code)
if 200 != r.status_code:
complete_status = "pending"
app_info["completed_status"] = complete_status
app_dict[app.name] = app_info
keys = sorted(app_dict.keys())
app_list = []
for key in keys:
app_list.append((key, app_dict[key]))
return render_to_response('index.html',
{'apps': app_list},
context_instance=RequestContext(request))
def monitor():
cloud = heroku.from_key(os.getenv("HEROKU_API_KEY"))
app_name = os.getenv("MONITOR_APP_NAME")
app = cloud.apps[app_name]
dynos = app.processes["web"]
logs = app.logs(num=1000, ps="web").splitlines()
logs.reverse()
memory_error_logs = []
for log in logs:
if "Error R14" in log:
memory_error_logs.append(log)
if os.getenv("DYNO_MONITOR_DEBUG"):
keen.add_event("dyno_monitor_debug", {
"app": app_name,
})
i = 1
for dyno in dynos:
try:
dyno_response = next(x for x in memory_error_logs if "web.%s" % i in x)
if dyno_response:
dynos[i-1].restart()
keen.add_event("dyno_restart", {
"app": app_name,
"dyno": "web.%s" % i,
})
except StopIteration:
pass
i += 1
def logs(request, appname):
cloud = heroku.from_key(settings.MAKAHIKI_HEROKU_KEY)
app = cloud.apps[appname]
logs = app.logs(num=10).split("\n")
return render_to_response('logs.html',
{'appname': appname,
'logs': logs},
context_instance=RequestContext(request))
def run(app, command):
cloud = heroku.from_key(os.getenv("HEROKU_API_KEY"))
app = cloud.apps[app]
if command == "restart":
print(u"restarting {app}".format(app=app))
for process in app.processes:
process.restart()
print(u"upon request in heroku_api, restarted {process}".format(
process=process))
def get_heroku_release_version(self):
try:
import heroku
key = settings.HEROKU_API_KEY
cloud = heroku.from_key(key)
app = cloud.apps[settings.HEROKU_APP_LABEL]
latest_release = app.releases[-1]
return latest_release.name
except:
return None
def get_heroku_release_version(self):
try:
import heroku
key = settings.HEROKU_API_KEY
cloud = heroku.from_key(key)
app = cloud.apps[settings.HEROKU_APP_LABEL]
latest_release = app.releases[-1]
return latest_release.name
except:
return None
def logs(request, appname):
cloud = heroku.from_key(settings.MAKAHIKI_HEROKU_KEY)
app = cloud.apps[appname]
logs = app.logs(num=10).split("\n")
return render_to_response('logs.html', {
'appname': appname,
'logs': logs
},
context_instance=RequestContext(request))
def fetch_article(user, category, feed):
"""
to parse articles of feeds
"""
if settings.DEBUG == False:
cloud = heroku.from_key(settings.HEROKU_APIKEY)
app = cloud.apps['theinfoholic']
try:
app.processes['worker']
except KeyError, e:
cloud._http_resource(method='POST', resource=(
'apps', 'theinfoholic', 'ps', 'scale'),
data={'type': 'worker', 'qty': 1})
def start_heroku_process(command):
try:
import heroku
except ImportError:
logger.error('heroku module is required to start a Heroku process')
else:
h = heroku.from_key(current_app.config['HEROKU_API_KEY'])
try:
app = h.apps[current_app.config['HEROKU_APP_NAME']]
app.processes.add(command)
logger.debug("Heroku process '%s' is started", command)
except Exception:
logger.exception('Failed to start the Heroku process')
def __init__(self, config_file = "config.ini"):
"""Initialise the daemon with the given config file.
Argument:
- config_file: (optional, default = "config.ini") The configuration file to use.
"""
self._conf = HAConfigParser.loadConf(config_file)
self._pd_wrapper = PingdomAPIWrapper(self._conf.getPingdomAPIKey(),
self._conf.getPingdomLogin(),
self._conf.getPingdomPassword())
hrk_cloud = heroku.from_key(self._conf.getHerokuAPIKey())
self._heroku_app = hrk_cloud.apps[self._conf.getHerokuAppName()]
def run(app_name, process_name_start_to_restart, command):
cloud = heroku.from_key(os.getenv("HEROKU_API_KEY"))
app = cloud.apps[app_name]
if command=="restart":
print(u"restarting {app_name}, processes that start with {process_name}".format(
app_name=app_name, process_name=process_name_start_to_restart))
for process in app.processes:
process_name = process.process
process_name_start = process_name.split(".")[0]
if process_name_start==process_name_start_to_restart:
process.restart()
print(u"upon request in heroku_api, restarted {process_name}".format(
process_name=process_name))
def start_heroku_process(command):
try:
import heroku
except ImportError:
logger.error('heroku module is required to start a Heroku process')
else:
h = heroku.from_key(current_app.config['HEROKU_API_KEY'])
try:
app = h.apps[current_app.config['HEROKU_APP_NAME']]
app.processes.add(command)
logger.debug("Heroku process '%s' is started", command)
except Exception:
logger.exception('Failed to start the Heroku process')
def apply(self):
try:
self.cloud = heroku.from_key(self.params.key.as_string())
except errors.NoMatching:
try:
username = self.params.username.as_string()
password = self.params.password.as_string()
except errors.NoMatching:
raise errors.TypeError(
"Must specify key or username and password",
anchor=self.params.anchor)
self.cloud = heroku.from_pass(username, password)
if self.root.readonly:
return
changed = False
app_id = self.params.application_id.as_string()
if not app_id in self.cloud.apps:
self.action("Creating new app named '%s'" % app_id)
if not self.root.simulate:
self.app = self.cloud.apps.add(app_id)
else:
self.app = None
changed = True
else:
self.app = self.cloud.apps[app_id]
self.action("Entering maintenance mode")
if not self.root.simulate and self.app:
self.app.maintenance(on=True)
self.apply_configuration()
self.apply_scaling()
self.apply_addons()
self.action("Leaving maintenance mode")
if not self.root.simulate and self.app:
self.app.maintenance(on=False)
self.apply_domains()
# May generate e-mail so do it last - we don't want someone looking
# half way through set up
self.apply_collaborators()
self.root.changed(changed)
def start_worker():
"""
wake up worker
"""
if settings.DEBUG == False:
import heroku
cloud = heroku.from_key(settings.HEROKU_APIKEY)
app = cloud.apps['theinfoholic']
#worker_start_time = time.time()
try:
app.processes['worker']
except KeyError, e:
cloud._http_resource(method='POST', resource=(
'apps', 'theinfoholic', 'ps', 'scale'),
data={'type': 'worker', 'qty': 1})
def apply(self):
try:
self.cloud = heroku.from_key(self.params.key.as_string())
except errors.NoMatching:
try:
username = self.params.username.as_string()
password = self.params.password.as_string()
except errors.NoMatching:
raise errors.TypeError(
"Must specify key or username and password", anchor=self.params.anchor)
self.cloud = heroku.from_pass(username, password)
if self.root.readonly:
return
changed = False
app_id = self.params.application_id.as_string()
if not app_id in self.cloud.apps:
self.action("Creating new app named '%s'" % app_id)
if not self.root.simulate:
self.app = self.cloud.apps.add(app_id)
else:
self.app = None
changed = True
else:
self.app = self.cloud.apps[app_id]
self.action("Entering maintenance mode")
if not self.root.simulate and self.app:
self.app.maintenance(on=True)
self.apply_configuration()
self.apply_scaling()
self.apply_addons()
self.action("Leaving maintenance mode")
if not self.root.simulate and self.app:
self.app.maintenance(on=False)
self.apply_domains()
# May generate e-mail so do it last - we don't want someone looking
# half way through set up
self.apply_collaborators()
self.root.changed(changed)
def handle(self, *args, **options):
use_maintenance = options['maintenance']
bump_assets = options['bump_assets']
collect_static = options['collect_static']
use_all = options['use_all']
flush_cache = options['flush_cache']
print "All right, deploying your shit now"
os.chdir(MANAGE_DIR)
if use_maintenance or use_all:
os.system("heroku maintenance:on")
# Deploy to heroku
print "\nDeploying to heroku..."
os.system("git push heroku master")
if collect_static or use_all:
print "\nCollecting all staticfiles..."
os.system("heroku run python manage.py collectstatic --noinput")
if bump_assets or use_all:
print "\nUpdating asset version"
cloud = heroku.from_key(HEROKU_API_KEY)
app = cloud.apps[HEROKU_APP]
last_release = app.releases[-1]
app.config['APP_REVISION'] = last_release.name
if flush_cache or use_all:
print "\nFlushing memcache..."
cmd = (
'''heroku run "python -c \\"'''
'''import os;'''
'''os.environ['DJANGO_SETTINGS_MODULE'] = 'davecx.settings';'''
'''from django.core.cache import cache;'''
'''cache.clear();'''
'''\\" " '''
) # Closing quotes for 'heroku run' and 'python -c'
os.system(cmd)
if use_maintenance or use_all:
os.system("heroku maintenance:off")
print "\nFinished!!!"
def handle(self, *args, **options):
use_maintenance = options['maintenance']
bump_assets = options['bump_assets']
collect_static = options['collect_static']
use_all = options['use_all']
flush_cache = options['flush_cache']
print "All right, deploying your shit now"
os.chdir(MANAGE_DIR)
if use_maintenance or use_all:
os.system("heroku maintenance:on")
# Deploy to heroku
print "\nDeploying to heroku..."
os.system("git push heroku master")
if collect_static or use_all:
print "\nCollecting all staticfiles..."
os.system("heroku run python manage.py collectstatic --noinput")
if bump_assets or use_all:
print "\nUpdating asset version"
cloud = heroku.from_key(HEROKU_API_KEY)
app = cloud.apps[HEROKU_APP]
last_release = app.releases[-1]
app.config['APP_REVISION'] = last_release.name
if flush_cache or use_all:
print "\nFlushing memcache..."
cmd = ('''heroku run "python -c \\"'''
'''import os;'''
'''os.environ['DJANGO_SETTINGS_MODULE'] = 'davecx.settings';'''
'''from django.core.cache import cache;'''
'''cache.clear();'''
'''\\" " ''') # Closing quotes for 'heroku run' and 'python -c'
os.system(cmd)
if use_maintenance or use_all:
os.system("heroku maintenance:off")
print "\nFinished!!!"
def update_api_keys_from_config():
engine = _get_database()
Session = sessionmaker(bind=engine)
session = Session()
apps = session.query(App).order_by(App.appname).all()
HEROKU_API_KEY = os.environ.get('HEROKU_API_KEY', False)
heroku_conn = heroku.from_key(HEROKU_API_KEY)
heroku_apps = heroku_conn.apps()
for app in apps:
print "checking {0} for api_key".format(app.appname)
if app.appname in heroku_apps:
config = heroku_apps[app.appname].config()
new_api_key = config['HEROKU_API_KEY']
if new_api_key:
print "[{0}] setting api_key to {1}".format(app.appname, new_api_key)
app.api_key = new_api_key
session.commit()
def update_api_keys_from_config():
engine = _get_database()
Session = sessionmaker(bind=engine)
session = Session()
apps = session.query(App).order_by(App.appname).all()
HEROKU_API_KEY = os.environ.get('HEROKU_API_KEY', False)
heroku_conn = heroku.from_key(HEROKU_API_KEY)
heroku_apps = heroku_conn.apps()
for app in apps:
print "checking {0} for api_key".format(app.appname)
if app.appname in heroku_apps:
config = heroku_apps[app.appname].config()
new_api_key = config['HEROKU_API_KEY']
if new_api_key:
print "[{0}] setting api_key to {1}".format(app.appname, new_api_key)
app.api_key = new_api_key
session.commit()
def work(request):
appname = request.POST['appname']
cloud = heroku.from_key(settings.MAKAHIKI_HEROKU_KEY)
apps = cloud.apps
print "create app %s" % appname
try:
apps.add(appname)
except:
pass
config_json = {
"BUILDPACK_URL":
"https://github.com/yongwen/makahiki-buildpack.git",
"MAKAHIKI_USE_MEMCACHED":
"True",
"MAKAHIKI_USE_HEROKU":
"True",
"MAKAHIKI_USE_S3":
"True",
"MAKAHIKI_ADMIN_INFO":
settings.MAKAHIKI_ADMIN_INFO,
"MAKAHIKI_AWS_ACCESS_KEY_ID":
settings.MAKAHIKI_AWS_ACCESS_KEY_ID,
"MAKAHIKI_AWS_SECRET_ACCESS_KEY":
settings.MAKAHIKI_AWS_SECRET_ACCESS_KEY,
"MAKAHIKI_AWS_STORAGE_BUCKET_NAME":
settings.MAKAHIKI_AWS_STORAGE_BUCKET_NAME
}
# copy from https://github.com/heroku/heroku.py/blob/master/heroku/models.py
try:
cloud._http_resource(method='PUT',
resource=('apps', appname, 'config_vars'),
data=json.dumps(config_json))
except:
pass
try:
cloud._http_resource(method='POST',
resource=('apps', appname, 'addons', 'memcache'))
except:
pass
sshkey_cmd = "rm -rf .ssh; mkdir .ssh; ssh-keygen -q -N '' -f .ssh/id_rsa"
os.system(sshkey_cmd)
knownhosts_cmd = 'echo "|1|v2fAE9r+64rPyeKTVWZamQa95N8=|8cihuAGn19m0ljoDHJITbpNx618= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAu8erSx6jh+8ztsfHwkNeFr/SZaSOcvoa8AyMpaerGIPZDB2TKNgNkMSYTLYGDK2ivsqXopo2W7dpQRBIVF80q9mNXy5tbt1WE04gbOBB26Wn2hF4bk3Tu+BNMFbvMjPbkVlC2hcFuQJdH4T2i/dtauyTpJbD/6ExHR9XYVhdhdMs0JsjP/Q5FNoWh2ff9YbZVpDQSTPvusUp4liLjPfa/i0t+2LpNCeWy8Y+V9gUlDWiyYwrfMVI0UwNCZZKHs1Unpc11/4HLitQRtvuk0Ot5qwwBxbmtvCDKZvj1aFBid71/mYdGRPYZMIxq1zgP1acePC1zfTG/lvuQ7d0Pe0kaw==" > .ssh/known_hosts'
os.system(knownhosts_cmd)
cloud = heroku.from_key(settings.MAKAHIKI_HEROKU_KEY)
keys = cloud.keys
for key in keys:
if not re.search("== .*@(\w+)\.(\w+)", key.contents):
key.delete()
file = open(".ssh/id_rsa.pub")
for line in file:
cloud.keys.add(line)
print "setup ssh key"
app_source = "git://github.com/yongwen/makahiki-min.git"
clone_cmd = "rm -rf git-tmp; git clone %s git-tmp; " % app_source
push_cmd = 'cd git-tmp; git push [email protected]:%s.git master; sleep 5; curl -s -o /dev/null http://%s.herokuapp.com/init/; echo %s' % (
appname, appname, COMPLETED_TEXT)
cmd = 'echo "%s" > /tmp/push.sh' % (clone_cmd + push_cmd)
os.system(cmd)
print "git clone and push"
os.system("/bin/bash /tmp/push.sh &")
return HttpResponseRedirect(reverse("home", args=()))
def work(request):
appname = request.POST['appname']
cloud = heroku.from_key(settings.MAKAHIKI_HEROKU_KEY)
apps = cloud.apps
print "create app %s" % appname
try:
apps.add(appname)
except:
pass
config_json = {
"BUILDPACK_URL" : "https://github.com/yongwen/makahiki-buildpack.git",
"MAKAHIKI_USE_MEMCACHED" : "True",
"MAKAHIKI_USE_HEROKU" : "True",
"MAKAHIKI_USE_S3" : "True",
"MAKAHIKI_ADMIN_INFO" : settings.MAKAHIKI_ADMIN_INFO,
"MAKAHIKI_AWS_ACCESS_KEY_ID" : settings.MAKAHIKI_AWS_ACCESS_KEY_ID,
"MAKAHIKI_AWS_SECRET_ACCESS_KEY" : settings.MAKAHIKI_AWS_SECRET_ACCESS_KEY,
"MAKAHIKI_AWS_STORAGE_BUCKET_NAME" : settings.MAKAHIKI_AWS_STORAGE_BUCKET_NAME
}
# copy from https://github.com/heroku/heroku.py/blob/master/heroku/models.py
try:
cloud._http_resource(
method='PUT',
resource=('apps', appname, 'config_vars'),
data=json.dumps(config_json))
except:
pass
try:
cloud._http_resource(
method='POST',
resource=('apps', appname, 'addons', 'memcache'))
except:
pass
sshkey_cmd = "rm -rf .ssh; mkdir .ssh; ssh-keygen -q -N '' -f .ssh/id_rsa"
os.system(sshkey_cmd)
knownhosts_cmd = 'echo "|1|v2fAE9r+64rPyeKTVWZamQa95N8=|8cihuAGn19m0ljoDHJITbpNx618= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAu8erSx6jh+8ztsfHwkNeFr/SZaSOcvoa8AyMpaerGIPZDB2TKNgNkMSYTLYGDK2ivsqXopo2W7dpQRBIVF80q9mNXy5tbt1WE04gbOBB26Wn2hF4bk3Tu+BNMFbvMjPbkVlC2hcFuQJdH4T2i/dtauyTpJbD/6ExHR9XYVhdhdMs0JsjP/Q5FNoWh2ff9YbZVpDQSTPvusUp4liLjPfa/i0t+2LpNCeWy8Y+V9gUlDWiyYwrfMVI0UwNCZZKHs1Unpc11/4HLitQRtvuk0Ot5qwwBxbmtvCDKZvj1aFBid71/mYdGRPYZMIxq1zgP1acePC1zfTG/lvuQ7d0Pe0kaw==" > .ssh/known_hosts'
os.system(knownhosts_cmd)
cloud = heroku.from_key(settings.MAKAHIKI_HEROKU_KEY)
keys = cloud.keys
for key in keys:
if not re.search("== .*@(\w+)\.(\w+)", key.contents):
key.delete()
file = open(".ssh/id_rsa.pub")
for line in file:
cloud.keys.add(line)
print "setup ssh key"
app_source = "git://github.com/yongwen/makahiki-min.git"
clone_cmd = "rm -rf git-tmp; git clone %s git-tmp; " % app_source
push_cmd = 'cd git-tmp; git push [email protected]:%s.git master; sleep 5; curl -s -o /dev/null http://%s.herokuapp.com/init/; echo %s' % (appname, appname, COMPLETED_TEXT)
cmd = 'echo "%s" > /tmp/push.sh' % (clone_cmd + push_cmd)
os.system(cmd)
print "git clone and push"
os.system("/bin/bash /tmp/push.sh &")
return HttpResponseRedirect(reverse("home", args=()))
from celery import Celery
from utils import configure_app
app = Flask(__name__)
configure_app(app, 'config.DevelopmentConfig')
# configure_error_handlers(app)
# configure_middleware_handlers(app)
# toolbar = DebugToolbarExtension(app)
env = Environment(loader=FileSystemLoader('./templates/'))
celery = Celery()
celery.add_defaults(app.config)
cloud = heroku.from_key(app.config.get('HEROKU_API_KEY'))
def send_reminder(venue, user, last_checkin, checkin):
scale_celery(1)
if user.get('send_on') == 'instantly':
if user.get('email'):
print 'Delaying email'
email_reminder.delay(venue, user, last_checkin, checkin)
if user.get('phone'):
print 'Delaying sms'
sms_reminder.delay(venue, user, last_checkin, checkin)
import sys, subprocess, os
import heroku
heroku_token = sys.argv[1]
subprocess.Popen(['heroku', 'config:add',
'HEROKU_TOKEN=' + heroku_token]).wait()
subprocess.Popen(['ssh-keygen', '-t', 'rsa', '-f', 'tmpkey', '-N', '']).wait()
cloud = heroku.from_key(heroku_token)
cloud.keys.add(file('tmpkey.pub').read())
subprocess.Popen(
['heroku', 'config:add', 'SSH_PRIVKEY=' + file('tmpkey').read()]).wait()
os.unlink('tmpkey')
os.unlink('tmpkey.pub')
def __init__(self, stack, *args, **kwargs):
super(HerokuAdapter, self).__init__(*args, **kwargs)
self.stack = stack
self.heroku = heroku.from_key(settings.HEROKU_API_KEY)
def delete(request):
myapp = request.POST['appname']
cloud = heroku.from_key(settings.MAKAHIKI_HEROKU_KEY)
app = cloud.apps[myapp]
app.destroy()
return HttpResponseRedirect(reverse("home", args=()))
def delete(request):
myapp = request.POST['appname']
cloud = heroku.from_key(settings.MAKAHIKI_HEROKU_KEY)
app = cloud.apps[myapp]
app.destroy()
return HttpResponseRedirect(reverse("home", args=()))
try:
while (True):
print "\n\n====================[Beginning Run]=======================\n".ljust(
max_str_length)
sqlsession = Session()
apps = sqlsession.query(App).order_by("app_appname").all()
ratelimits = {}
for app in apps:
heroku_conn = None
rl = None
key_type = 'General Key'
if app.api_key is None:
print "[{0}] processing app {0}".format(
app.appname, HEROKU_API_KEY)
heroku_conn = heroku.from_key(HEROKU_API_KEY)
rl = heroku_conn.ratelimit_remaining()
print "rate_limit_remaining = {0} for Generic API_KEY".format(
rl)
else:
print "[{0}] processing app {1}".format(
app.appname, app.api_key)
heroku_conn = heroku.from_key(app.api_key)
rl = heroku_conn.ratelimit_remaining()
print "rate_limit_remaining = {0} for app configured key {1}".format(
rl, app.api_key)
key_type = app.api_key
if app.appname in ratelimits:
prev_limit = ratelimits[app.appname]['prev_limit']
prev_time = ratelimits[app.appname]['prev_time']
def main(dyno_type, quantity):
cloud = heroku.from_key(os.environ.get('HEROKU_API_KEY'))
app = cloud.apps[config.app_name]
scale(app, dyno_type, quantity)
engine = create_engine(DATABASE_URL)
Session = scoped_session(sessionmaker(bind=engine))
while(True):
print "\n\n====================[Beginning Run]=======================\n".ljust(max_str_length)
sqlsession = Session()
apps = sqlsession.query(App).order_by("app_appname").all()
ratelimits = {}
for app in apps:
heroku_conn = None
rl = None
key_type = 'General Key'
if app.api_key is None:
print "[{0}] processing app {0}".format(app.appname, HEROKU_API_KEY)
heroku_conn = heroku.from_key(HEROKU_API_KEY)
rl = heroku_conn.ratelimit_remaining()
print "rate_limit_remaining = {0} for Generic API_KEY".format(rl)
else:
print "[{0}] processing app {1}".format(app.appname, app.api_key)
heroku_conn = heroku.from_key(app.api_key)
rl = heroku_conn.ratelimit_remaining()
print "rate_limit_remaining = {0} for app configured key {1}".format(rl, app.api_key)
key_type = app.api_key
if app.appname in ratelimits:
prev_limit = ratelimits[app.appname]['prev_limit']
prev_time = ratelimits[app.appname]['prev_time']
current_time = time.time()
if (prev_limit - rl) > RATE_LIMIT_INTERVAL:
time_diff = current_time - prev_time
help='minimum number of timings needed from a dyno to include it in response time aggregation')
parser.add_argument('--kill-threshold', type=int, default=DEFAULT_KILL_THRESHOLD,
help='number of stddevs above the mean that determine a slow dyno')
parser.add_argument('--min-threshold', type=float, default=DEFAULT_MIN_THRESHOLD,
help='minimum response time measurement a dyno must exceed before it can be considered slow')
parser.add_argument('--dry-run', '-n', action='store_true',
help='perform analysis but do not interfere with running dynos')
parser.add_argument('--verbose', '-v', action='store_true',
help='verbose output (log level DEBUG)')
args = parser.parse_args()
log_level = logging.INFO
if getattr(args, 'verbose'):
log_level = logging.DEBUG
h = heroku.from_key(os.getenv('HEROKU_API_KEY'))
heroku_app_name = os.getenv('HEROKU_APP_NAME')
app = h.apps[heroku_app_name]
logging.basicConfig(level=log_level, stream=sys.stderr,
format='%(asctime)s %(module)s:{} %(levelname)s %(message)s'.format(heroku_app_name))
# ignore noisy output from requests
if log_level > logging.DEBUG:
logging.getLogger("requests.packages.urllib3").setLevel(logging.WARN)
logging.info('starting slow dyno detection')
try:
exit_code = main(app, vars(args))
except:
logging.error('slow dyno detection failed', exc_info=1)
exit_code = 1
#import socket
#import httplib
#import logging
#httplib.HTTPConnection.debuglevel = 1
#logging.basicConfig() # you need to initialize logging, otherwise you will not see anything from requests
#logging.getLogger().setLevel(logging.INFO)
#requests_log = logging.getLogger("requests.packages.urllib3")
#requests_log.setLevel(logging.INFO)
#requests_log.propagate = True
HEROKU_API_KEY = os.environ.get('HEROKU_API_KEY', False)
HEROKU_APPNAME = os.environ.get('HEROKU_APPNAME', False)
TEST_EMAIL = os.environ.get('TEST_EMAIL', False)
heroku_conn = heroku.from_key(HEROKU_API_KEY)
#app = heroku_conn.create_app(name='testy2124app', stack_id_or_name='cedar', region_id_or_name='us')
#print app.addons()
#print heroku_conn.addons('testy123app')
#for addon in app.addons():
#addon.delete()
#del config['TEST1']
#del config['TEST2']
#del config['TEST3']
#del config['Z01']
#del config['Z02']
#print config
#config['TEST1'] = u'MM1'
#config['TEST2'] = u'MM2'
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'
}
}
CACHES = get_cache()
FEEDBACK_APP_SETTINGS = {
'respond_to_feedback_request_url_template':
'{scheme}://{host}/feedback/#/todo/{id}'
}
if os.environ.get("HEROKU_APP_NAME", False):
import heroku
api = heroku.from_key(os.environ['HEROKU_API_KEY'])
app = api.apps[os.environ.get("HEROKU_APP_NAME")]
release = app.releases[-1]
os.environ['HEROKU_RELEASE_NAME'] = release.name
RAVEN_CONFIG = {
'dsn':
'https://*****:*****@sentry.io/105604',
'release': os.environ.get('HEROKU_RELEASE_NAME', None),
'tags': {},
}
LOGGING = {
'version': 1,
'disable_existing_loggers': True,
'root': {
def update():
"""Enqueues an update job so that a worker process update the database.
This feature is enabled when ``USE_QUEUE`` is ``True``. To process enqueued
jobs, run a worker process (``cpucoolerchart runworker``).
There is a special feature that starts a worker process to reduce the
server cost. Only Heroku is supported for now. Set ``START_WORKER_NODE``
to ``"heroku"`` and ``HEROKU_API_KEY`` and ``HEROKU_APP_NAME`` to your
Heroku API key and app name respectively.
To prevent DDoS attacks, it cannot be requested more frequently than once
per 5 minutes when ``app.debug`` is ``False``.
**Example request**:
.. sourcecode:: http
POST /update HTTP/1.1
Host: example.com
Accept: application/json
**Example response**:
.. sourcecode:: http
HTTP/1.1 202 OK
Content-Type: application/json
{
"msg": "process started"
}
It returns a JSON object containing a message in the *msg* property.
Possible messages are:
+------------------------+------------------------------------------------+
| message | description |
+========================+================================================+
| process started | An update job is enqueued and a worker process |
| | has started |
+------------------------+------------------------------------------------+
| too many requests | There was a request within the last 5 minutes |
+------------------------+------------------------------------------------+
| already up to date | data is already up to date |
+------------------------+------------------------------------------------+
| invalid worker type | worker type is other than ``"heroku"`` |
+------------------------+------------------------------------------------+
| Heroku API key is not | ``HEROKU_API_KEY`` is not set in config |
| set | |
+------------------------+------------------------------------------------+
| Heroku app name is not | ``HEROKU_APP_NAME`` is not set in config |
| set | |
+------------------------+------------------------------------------------+
| heroku is not | :mod:`heroku` Python module is not installed |
| installed. Add heroku | on your dyno. |
| to your | |
| requirements.txt | |
+------------------------+------------------------------------------------+
| failed to enqueue a | an error occurred during enqueuing a job |
| job | |
+------------------------+------------------------------------------------+
| failed to start a | an error occurred during starting a worker |
| worker | |
+------------------------+------------------------------------------------+
:status 202: an update job is enqueued and a worker process has started
:status 404: the app is not configured to update data via HTTP
:status 429: there was a request within the last 5 minutes
:status 500: an error occurred during enqueuing a job or starting a worker
:status 503: worker settings are not valid
"""
if not current_app.config.get('USE_QUEUE'):
return jsonify(
msg='the app is not configured to update data via HTTP'), 404
if not current_app.debug:
if cache.get('prevent_further_requests'):
return jsonify(msg='too many requests'), 429
cache.set('prevent_further_requests', True, timeout=300)
if not is_update_needed():
return jsonify(msg='already up to date'), 202
try:
update_queue.enqueue_call(update_data, result_ttl=0)
except Exception:
current_app.logger.exception('An error occurred during enqueuing')
return jsonify(msg='failed to enqueue a job'), 500
worker_type = current_app.config.get('START_WORKER_NODE')
if not worker_type:
return jsonify(msg='process started'), 202
if worker_type != 'heroku':
return jsonify(msg='invalid worker type'), 503
if not current_app.config.get('HEROKU_API_KEY'):
return jsonify(msg='Heroku API key is not set'), 503
elif not current_app.config.get('HEROKU_APP_NAME'):
return jsonify(msg='Heroku app name is not set'), 503
elif heroku is None:
return jsonify(msg='heroku is not installed. '
'Add heroku to your requirements.txt'), 503
try:
client = heroku.from_key(current_app.config['HEROKU_API_KEY'])
herokuapp = client.apps[current_app.config['HEROKU_APP_NAME']]
herokuapp.processes.add(current_app.config['HEROKU_WORKER_NAME'])
return jsonify(msg='process started'), 202
except Exception:
current_app.logger.exception("Couldn't start heroku process")
return jsonify(msg='failed to start a worker'), 500
def add_app(appname, app_api_url=False, min_dynos=0, max_dynos=5, count_boundary=0, api_key=False):
"""heroku run fab add_app:martinsharehoodadmin,"https://*****:*****@martinsharehoodadmin.herokuapp.com/api/celery_proc_scalar",min_dynos=1,count_boundary=0,max_dynos=5"""
engine = _get_database()
Session = sessionmaker(bind=engine)
session = Session()
app = session.query(App).filter_by(appname=appname).first()
if app is not None:
print "App '%s' already exists, updating it" % appname
else:
print "Configuring new app for '%s'" % (appname)
app = App(appname=appname)
session.add(app)
#app.control_app = control_app
full_url = "https://%s.herokuapp.com/heroku_proc_scalar/proc_count" % appname
if app_api_url:
print "Got app_api url %s" % app_api_url
url = urlparse(app_api_url)
hostname = url.hostname
scheme = url.scheme
if scheme is None:
scheme = 'https'
port = url.port
if port is not None:
port = ":%s" % port
else:
port = ''
path = url.path
username = url.username
password = url.password
if password:
app.password = password
if username:
app.username = username
full_url = "%s://%s%s%s" % (scheme, hostname, port, path)
app.app_api_url = full_url
app.min_dynos = min_dynos
app.max_dynos = max_dynos
app.count_boundary = count_boundary
if api_key:
app.api_key = api_key
else:
HEROKU_API_KEY = os.environ.get('HEROKU_API_KEY', False)
heroku_conn = heroku.from_key(HEROKU_API_KEY)
heroku_app = heroku_conn.app(appname)
print "Setting HEROKU_API_KEY directly from {0}".format(appname)
config = heroku_app.config()
new_api_key = config['HEROKU_API_KEY']
app.api_key = new_api_key
session.commit()
print "Updated %s to :-" % appname
print "appname = %s" % app.appname
print "app_api_url = %s" % app.app_api_url
print "username = %s" % app.username
print "password = %s" % app.password
print "min_dynos = %s" % app.min_dynos
print "max_dynos = %s" % app.max_dynos
print "count_boundary = %s" % app.count_boundary
print "api_key = %s" % app.api_key
app.config['HEROKU_API_KEY'] = os.environ.get('HEROKU_API_KEY')
app.config['DEBUG_TB_ENABLED'] = True
app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False
# Bootstrap Heroku environment variables.
heroku_env = Heroku(app)
# Intialize databse configuration.
db.init_app(app)
sentry = Sentry(app)
auth = GoogleAuth(app)
toolbar = DebugToolbarExtension(app)
celery = Celery(app)
heroku = heroku.from_key(app.config['HEROKU_API_KEY'] )
@celery.task
def build_task(request):
app.logger.info('Starting build for {0}.'.format(request))
# Replace # w/ @
_split_url = request.application_url.split('@')
clone_url = _split_url[0]
clone_ref = _split_url[1] if len(_split_url) > 1 else None
h_app = heroku.apps.create(stack=cedar)
h_app.config['BUILDPACK_URL'] = request.buildpack_url
app.logger.info('Heroku app {0} created for {1}.'.format(h_app, request))
import os
import sys
import heroku
"""Scale heroku web processes using the heroku python API."""
# you may want to add better argument processing, use argparse, etc.
dynos = int(sys.argv[1])
cloud = heroku.from_key(os.environ.get('HEROKU_API_KEY'))
app = cloud.apps['nhlplayoffscheer']
try:
# you may want to add a maximum dyno check here to prevent costly mistakes ;)
webproc = app.processes['worker']
webproc.scale(dynos)
except KeyError:
# note: scaling to 0 dynos or attempting to scale up if 0 web dynos exist
# both throw this error. Make sure you have at least one dyno.
print >> sys.stderr, "Could not scale web processes - are there 0 web dynos running?"
def heroku_app(self):
if not hasattr(self, "_heroku_app"):
cloud = heroku.from_key(self.settings.HEROKU_API_KEY)
self._heroku_app = cloud.apps[self.settings.HEROKU_APP_NAME]
return self._heroku_app
def scale_worker(num):
cloud = heroku.from_key(os.environ['HEROKU_API_KEY'])
app = cloud.apps['kindy']
app.processes['worker'].scale(num)
def restart_dyno(app_name, dyno_name):
cloud = heroku.from_key(os.getenv("HEROKU_API_KEY"))
app = cloud.apps[app_name]
process = app.processes[dyno_name]
process.restart()
print u"restarted {} on {}!".format(dyno_name, app_name)
from settings import *
import heroku
cloud = heroku.from_key('e7605a0ee08cf20e3867f1f1a0ac0486e32d1cb9')
app = cloud.apps['nhlplayoffscheer']
print app.config
DEBUG = False
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql_psycopg2', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME': 'd5s0sml2ufd7fj', # Or path to database file if using sqlite3.
# The following settings are not used with sqlite3:
'USER': '******',
'PASSWORD': '******',
'HOST': 'ec2-54-225-69-193.compute-1.amazonaws.com', # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
'PORT': '5432', # Set to empty string for default.
}
}
BROKER_URL = app.config.data['REDISTOGO_URL']
CELERY_RESULT_BACKEND = app.config.data['REDISTOGO_URL']
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
def __init__(self, stack, *args, **kwargs):
super(HerokuAdapter, self).__init__(*args, **kwargs)
self.stack = stack
self.heroku = heroku.from_key(settings.HEROKU_API_KEY)
import heroku
import requests
import os
import sched
import time
def poke(domain):
print 'Poking {domain}'.format(domain=domain)
requests.get(domain)
def poke_all(apps):
for app in apps:
poke('http://{app}.herokuapp.com'.format(app=app.name))
def run(cloud, sch):
poke_all(cloud.apps)
sch.enter(3600, 1, run, (cloud, sch))
if __name__ == '__main__':
cloud = heroku.from_key(os.environ['HEROKU_API_KEY'])
scheduler = sched.scheduler(time.time, time.sleep)
run(cloud, scheduler)
scheduler.run()
import os
import time
import logging
import heroku
from celery import task
from celery.result import AsyncResult
from models import Instance
from django.conf import settings
# connect to heroku
cloud = heroku.from_key(settings.HEROKU_KEY)
def ex(call, ignore_error=False):
logging.info('Exec: %s' % str(call))
num = os.system(call)
if num and not ignore_error:
raise Exception('Error code: ' + str(num))
@task.task()
def setup_enviroment(session_key, tutorial_id):
# try to find env by session id,
# if not - setup new
logger = setup_enviroment.get_logger()
logger.info('Setup enviroment for %s session_key: %s' %
(tutorial_id, session_key))
inst = get_instance(session_key)
init_tutorial(inst, tutorial_id)
def add_app(appname, app_api_url=False, min_dynos=0, max_dynos=5, count_boundary=0, api_key=False):
"""heroku run fab add_app:martinsharehoodadmin,"https://*****:*****@martinsharehoodadmin.herokuapp.com/api/celery_proc_scalar",min_dynos=1,count_boundary=0,max_dynos=5"""
engine = _get_database()
Session = sessionmaker(bind=engine)
session = Session()
app = session.query(App).filter_by(appname=appname).first()
if app is not None:
print "App '%s' already exists, updating it" % appname
else:
print "Configuring new app for '%s'" % (appname)
app = App(appname=appname)
session.add(app)
#app.control_app = control_app
full_url = "https://%s.herokuapp.com/heroku_proc_scalar/proc_count" % appname
if app_api_url:
print "Got app_api url %s" % app_api_url
url = urlparse(app_api_url)
hostname = url.hostname
scheme = url.scheme
if scheme is None:
scheme = 'https'
port = url.port
if port is not None:
port = ":%s" % port
else:
port = ''
path = url.path
username = url.username
password = url.password
if password:
app.password = password
if username:
app.username = username
full_url = "%s://%s%s%s" % (scheme, hostname, port, path)
app.app_api_url = full_url
app.min_dynos = min_dynos
app.max_dynos = max_dynos
app.count_boundary = count_boundary
if api_key:
app.api_key = api_key
else:
HEROKU_API_KEY = os.environ.get('HEROKU_API_KEY', False)
heroku_conn = heroku.from_key(HEROKU_API_KEY)
heroku_app = heroku_conn.app(appname)
print "Setting HEROKU_API_KEY directly from {0}".format(appname)
config = heroku_app.config()
new_api_key = config['HEROKU_API_KEY']
app.api_key = new_api_key
session.commit()
print "Updated %s to :-" % appname
print "appname = %s" % app.appname
print "app_api_url = %s" % app.app_api_url
print "username = %s" % app.username
print "password = %s" % app.password
print "min_dynos = %s" % app.min_dynos
print "max_dynos = %s" % app.max_dynos
print "count_boundary = %s" % app.count_boundary
print "api_key = %s" % app.api_key