def handle(self, *args, **options):
self.root_dir = settings.FILEPATH
prefix = os.getcwd()
current_snapshot = gitinfo.get_project_snapshot(self.root_dir, submodules=False)
current_sha = current_snapshot['commits'][0]['sha']
print "Current commit SHA: %s" % current_sha
if 'clear' in args:
print "clearing resource cache"
rcache.delete_pattern(RESOURCE_PREFIX % '*')
existing_resource_str = rcache.get(RESOURCE_PREFIX % current_sha, None)
if existing_resource_str:
print "getting resource dict from cache"
self.output_resources(existing_resource_str)
return
self.resources = {}
for finder in finders.get_finders():
for path, storage in finder.list(['.*', '*~', '* *']):
if not storage.location.startswith(prefix):
continue
url = os.path.join(storage.prefix, path) if storage.prefix else path
parts = (storage.location + '/' + path).split('/')
self.generate_output(url, parts)
resource_str = simplejson.dumps(self.resources, indent=2)
rcache.set(RESOURCE_PREFIX % current_sha, resource_str, 86400)
self.output_resources(resource_str)
def handle(self, *args, **options):
prefix = os.getcwd()
current_snapshot = gitinfo.get_project_snapshot(self.root_dir, submodules=False)
current_sha = current_snapshot['commits'][0]['sha']
print "Current commit SHA: %s" % current_sha
if 'clear' in args:
print "clearing resource cache"
rcache.delete_pattern(RESOURCE_PREFIX % '*')
existing_resource_str = rcache.get(RESOURCE_PREFIX % current_sha, None)
if existing_resource_str:
print "getting resource dict from cache"
self.output_resources(existing_resource_str)
return
resources = {}
for finder in finders.get_finders():
for path, storage in finder.list(['.*', '*~', '* *']):
if not storage.location.startswith(prefix):
continue
if not getattr(storage, 'prefix', None):
url = path
else:
url = os.path.join(storage.prefix, path)
filename = os.path.join(storage.location, path)
resources[url] = self.get_hash(filename)
resource_str = json.dumps(resources, indent=2)
rcache.set(RESOURCE_PREFIX % current_sha, resource_str, 86400)
self.output_resources(resource_str)
def handle(self, num_pool, username, **options):
email = options['mail']
root_dir = settings.FILEPATH
git_snapshot = gitinfo.get_project_snapshot(
root_dir,
submodules=False,
log_count=1,
)
head = git_snapshot['commits'][0]
if options['check']:
exit(0 if get_preindex_complete(head) else 1)
if get_preindex_complete(head) and email:
mail_admins('Already preindexed', "Skipping this step")
return
else:
clear_preindex_complete()
commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % (
git_snapshot['current_branch'], head['sha'])
pre_message = list()
pre_message.append("Heads up, %s has started preindexing" % username)
pre_message.append(commit_info)
if email:
mail_admins(" HQAdmin preindex_everything started",
'\n'.join(pre_message))
def couch_preindex():
call_command('sync_prepare_couchdb_multi', str(num_pool), username,
**{'no_mail': True})
print("Couch preindex done")
def pillow_preindex():
call_command('ptop_preindex')
print("ptop_preindex_done")
jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)]
gevent.joinall(jobs)
try:
for job in jobs:
job.get()
except Exception:
subject = " HQAdmin preindex_everything failed"
message = get_traceback_string()
else:
subject = " HQAdmin preindex_everything may or may not be complete"
message = ("We heard a rumor that preindex is complete,\n"
"but it's on you to check that all tasks are complete.")
set_preindex_complete(head)
if email:
mail_admins(subject, message)
else:
print('{}\n\n{}'.format(subject, message))
def handle(self, *args, **options):
self.root_dir = settings.FILEPATH
prefix = os.getcwd()
current_snapshot = gitinfo.get_project_snapshot(self.root_dir,
submodules=False)
current_sha = current_snapshot['commits'][0]['sha']
print "Current commit SHA: %s" % current_sha
if 'clear' in args:
print "clearing resource cache"
rcache.delete_pattern(RESOURCE_PREFIX % '*')
existing_resource_str = rcache.get(RESOURCE_PREFIX % current_sha, None)
if existing_resource_str:
print "getting resource dict from cache"
self.output_resources(existing_resource_str)
return
pool = Pool(10)
self.resources = {}
for finder in finders.get_finders():
for path, storage in finder.list(['.*', '*~', '* *']):
if not storage.location.startswith(prefix):
continue
url = os.path.join(storage.prefix,
path) if storage.prefix else path
parts = (storage.location + '/' + path).split('/')
pool.spawn(self.generate_output, url, parts)
pool.join()
resource_str = simplejson.dumps(self.resources, indent=2)
rcache.set(RESOURCE_PREFIX % current_sha, resource_str, 86400)
self.output_resources(resource_str)
def handle(self, *args, **options):
prefix = os.getcwd()
current_snapshot = gitinfo.get_project_snapshot(self.root_dir, submodules=False)
current_sha = current_snapshot['commits'][0]['sha']
print "Current commit SHA: %s" % current_sha
if 'clear' in args:
print "clearing resource cache"
rcache.delete_pattern(RESOURCE_PREFIX % '*')
existing_resource_str = rcache.get(RESOURCE_PREFIX % current_sha, None)
if existing_resource_str:
print "getting resource dict from cache"
self.output_resources(existing_resource_str)
return
resources = {}
for finder in finders.get_finders():
for path, storage in finder.list(['.*', '*~', '* *']):
if not storage.location.startswith(prefix):
continue
if not getattr(storage, 'prefix', None):
url = path
else:
url = os.path.join(storage.prefix, path)
filename = os.path.join(storage.location, path)
resources[url] = self.get_hash(filename)
resource_str = json.dumps(resources, indent=2)
rcache.set(RESOURCE_PREFIX % current_sha, resource_str, 86400)
self.output_resources(resource_str)
def handle(self, *args, **options):
root_dir = settings.FILEPATH
git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True)
git_snapshot['diff_url'] = options.get('url', None)
deploy = HqDeploy(
date=datetime.utcnow(),
user=options['user'],
environment=options['environment'],
code_snapshot=git_snapshot,
)
deploy.save()
# reset PillowTop errors in the hope that a fix has been deployed
rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow())
if rows_updated:
print "\n---------------- Pillow Errors Reset ----------------\n" \
"{} pillow errors queued for retry\n".format(rows_updated)
if options['mail_admins']:
snapshot_table = render_to_string('hqadmin/partials/project_snapshot.html', dictionary={'snapshot': git_snapshot})
message = "Deployed by %s, cheers!" % options['user']
snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % (message, snapshot_table)
call_command('mail_admins', snapshot_body, **{'subject': 'Deploy successful', 'html': True})
def handle(self, *args, **options):
if len(args) == 0:
num_pool = POOL_SIZE
else:
num_pool = int(args[0])
if len(args) > 1:
username = args[1]
else:
username = '******'
email = options['mail']
root_dir = settings.FILEPATH
git_snapshot = gitinfo.get_project_snapshot(
root_dir,
submodules=False,
log_count=1,
)
head = git_snapshot['commits'][0]
commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % (
git_snapshot['current_branch'], head['sha'])
pre_message = list()
pre_message.append("Heads up, %s has started preindexing" % username)
pre_message.append(commit_info)
if email:
mail_admins(
" HQAdmin preindex_everything started", '\n'.join(pre_message)
)
def couch_preindex():
call_command('sync_prepare_couchdb_multi', num_pool, username,
**{'no_mail': True})
print "Couch preindex done"
def pillow_preindex():
call_command('ptop_preindex')
print "ptop_preindex_done"
jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)]
gevent.joinall(jobs)
message = '\n'.join([
"We heard a rumor that preindex is complete, "
"but it's on you to check that all tasks are complete."
])
if email:
mail_admins(
" HQAdmin preindex_everything may or may not be complete",
message
)
else:
print message
def handle(self, *args, **options):
if len(args) == 0:
num_pool = POOL_SIZE
else:
num_pool = int(args[0])
if len(args) > 1:
username = args[1]
else:
username = "******"
email = options["mail"]
root_dir = settings.FILEPATH
git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=False, log_count=1)
head = git_snapshot["commits"][0]
commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % (git_snapshot["current_branch"], head["sha"])
pre_message = list()
pre_message.append("Heads up, %s has started preindexing" % username)
pre_message.append(commit_info)
if email:
mail_admins(" HQAdmin preindex_everything started", "\n".join(pre_message))
def couch_preindex():
call_command("sync_prepare_couchdb_multi", num_pool, username, **{"no_mail": True})
print "Couch preindex done"
def pillow_preindex():
call_command("ptop_preindex")
print "ptop_preindex_done"
jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)]
gevent.joinall(jobs)
try:
for job in jobs:
job.get()
except Exception:
subject = " HQAdmin preindex_everything failed"
f = StringIO()
traceback.print_exc(file=f)
message = f.getvalue()
else:
subject = " HQAdmin preindex_everything may or may not be complete"
message = (
"We heard a rumor that preindex is complete,\n" "but it's on you to check that all tasks are complete."
)
if email:
mail_admins(subject, message)
else:
print "{}\n\n{}".format(subject, message)
def handle(self, *args, **options):
root_dir = settings.FILEPATH
git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True)
compare_url = git_snapshot['diff_url'] = options.get('url', None)
deploy = HqDeploy(
date=datetime.utcnow(),
user=options['user'],
environment=options['environment'],
code_snapshot=git_snapshot,
)
deploy.save()
# reset PillowTop errors in the hope that a fix has been deployed
rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow())
if rows_updated:
print "\n---------------- Pillow Errors Reset ----------------\n" \
"{} pillow errors queued for retry\n".format(rows_updated)
deploy_notification_text = (
"CommCareHQ has been successfully deployed to *{}* by *{}*. "
"Find the diff {{diff_link}}".format(
options['environment'],
options['user'],
)
)
if hasattr(settings, 'MIA_THE_DEPLOY_BOT_API'):
link = diff_link(STYLE_SLACK, compare_url)
requests.post(settings.MIA_THE_DEPLOY_BOT_API, data=json.dumps({
"username": "******",
"text": deploy_notification_text.format(diff_link=link),
}))
if settings.DATADOG_API_KEY:
tags = ['environment:{}'.format(options['environment'])]
link = diff_link(STYLE_MARKDOWN, compare_url)
datadog_api.Event.create(
title="Deploy Success",
text=deploy_notification_text.format(diff_link=link),
tags=tags,
alert_type="success"
)
print "\n=============================================================\n" \
"Congratulations! Deploy Complete.\n\n" \
"Don't forget to keep an eye on the deploy dashboard to " \
"make sure everything is running smoothly.\n\n" \
"https://p.datadoghq.com/sb/5c4af2ac8-1f739e93ef" \
"\n=============================================================\n"
if options['mail_admins']:
message_body = get_deploy_email_message_body(
environment=options['environment'], user=options['user'],
compare_url=compare_url)
call_command('mail_admins', message_body, **{'subject': 'Deploy successful', 'html': True})
def handle(self, *args, **options):
start = datetime.utcnow()
if len(args) == 0:
num_pool = POOL_SIZE
else:
num_pool = int(args[0])
if len(args) > 1:
username = args[1]
else:
username = "******"
email = options["mail"]
root_dir = settings.FILEPATH
git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=False, log_count=1)
head = git_snapshot["commits"][0]
commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % (git_snapshot["current_branch"], head["sha"])
pre_message = list()
pre_message.append("Heads up, %s has started preindexing" % username)
pre_message.append(commit_info)
if email:
mail_admins(" HQAdmin preindex_everything started", "\n".join(pre_message))
def couch_preindex():
call_command("sync_prepare_couchdb_multi", num_pool, username, **{"no_mail": True})
print "Couch preindex done"
def pillow_preindex():
call_command("ptop_preindex")
print "ptop_preindex_done"
jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)]
gevent.joinall(jobs)
message = list()
message.append("Preindex results:")
message.append("\tInitiated by: %s" % username)
delta = datetime.utcnow() - start
message.append("Total time: %d seconds" % delta.seconds)
message.append("\nYou may now deploy")
message.append(commit_info)
if email:
mail_admins(" HQAdmin preindex_everything complete", "\n".join(message))
else:
print message
def handle(self, *args, **options):
root_dir = settings.FILEPATH
git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True)
git_snapshot['diff_url'] = options.get('url', None)
deploy = HqDeploy(
date=datetime.utcnow(),
user=options['user'],
environment=options['environment'],
code_snapshot=git_snapshot,
)
deploy.save()
# reset PillowTop errors in the hope that a fix has been deployed
rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow())
if rows_updated:
print "\n---------------- Pillow Errors Reset ----------------\n" \
"{} pillow errors queued for retry\n".format(rows_updated)
deploy_notification_text = (
"CommCareHQ has been successfully deployed to *{}* by *{}*. "
"Find the diff {{diff_link}}".format(
options['environment'],
options['user'],
)
)
if hasattr(settings, 'MIA_THE_DEPLOY_BOT_API'):
link = diff_link(STYLE_SLACK, git_snapshot['diff_url'])
requests.post(settings.MIA_THE_DEPLOY_BOT_API, data=json.dumps({
"channel": "#dev",
"username": "******",
"text": deploy_notification_text.format(diff_link=link),
"icon_emoji": ":see_no_evil:"
}))
if settings.DATADOG_API_KEY:
tags = ['environment:{}'.format(options['environment'])]
link = diff_link(STYLE_MARKDOWN, git_snapshot['diff_url'])
datadog_api.Event.create(
title="Deploy Success",
text=deploy_notification_text.format(diff_link=link),
tags=tags
)
if options['mail_admins']:
snapshot_table = render_to_string('hqadmin/partials/project_snapshot.html', dictionary={'snapshot': git_snapshot})
message = "Deployed by %s, cheers!" % options['user']
snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % (message, snapshot_table)
call_command('mail_admins', snapshot_body, **{'subject': 'Deploy successful', 'html': True})
def handle(self, *args, **options):
current_snapshot = gitinfo.get_project_snapshot(self.root_dir, submodules=False)
self.current_sha = current_snapshot['commits'][0]['sha']
print "Current commit SHA: %s" % self.current_sha
if 'save' in args:
self.save_manifest()
else:
existing_resource_str = rcache.get(COMPRESS_PREFIX % self.current_sha, None)
if existing_resource_str:
self.output_manifest(existing_resource_str,
is_soft_update='soft' in args)
else:
raise ResourceCompressError(
"Could not find manifest.json in redis! Deploying under this "
"condition will cause the server to 500."
)
def handle(self, *args, **options):
root_dir = settings.FILEPATH
git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True)
deploy = HqDeploy(
date=datetime.utcnow(),
user=options['user'],
environment=options['environment'],
code_snapshot=git_snapshot,
)
deploy.save()
if options['mail_admins']:
snapshot_table = render_to_string('hqadmin/partials/project_snapshot.html', dictionary={'snapshot': git_snapshot})
message = "Deployed by %s, cheers!" % options['user']
snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % (message, snapshot_table)
call_command('mail_admins', snapshot_body, **{'subject': 'Deploy successful', 'html': True})
def handle(self, *args, **options):
root_dir = settings.FILEPATH
git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True)
deploy = HqDeploy(
date=datetime.utcnow(),
user=options['user'],
environment=options['environment'],
code_snapshot=git_snapshot,
)
deploy.save()
if options['mail_admins']:
snapshot_table = render_to_string(
'hqadmin/partials/project_snapshot.html',
dictionary={'snapshot': git_snapshot})
message = "Deployed by %s, cheers!" % options['user']
snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % (
message, snapshot_table)
call_command('mail_admins', snapshot_body, **{
'subject': 'Deploy successful',
'html': True
})
def handle(self, *args, **options):
if len(args) == 0:
num_pool = POOL_SIZE
else:
num_pool = int(args[0])
if len(args) > 1:
username = args[1]
else:
username = '******'
email = options['mail']
root_dir = settings.FILEPATH
git_snapshot = gitinfo.get_project_snapshot(
root_dir,
submodules=False,
log_count=1,
)
head = git_snapshot['commits'][0]
commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % (
git_snapshot['current_branch'], head['sha'])
pre_message = list()
pre_message.append("Heads up, %s has started preindexing" % username)
pre_message.append(commit_info)
if email:
mail_admins(
" HQAdmin preindex_everything started", '\n'.join(pre_message)
)
def couch_preindex():
call_command('sync_prepare_couchdb_multi', num_pool, username,
**{'no_mail': True})
print "Couch preindex done"
def pillow_preindex():
call_command('ptop_preindex')
print "ptop_preindex_done"
jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)]
gevent.joinall(jobs)
try:
for job in jobs:
job.get()
except Exception:
subject = " HQAdmin preindex_everything failed"
f = StringIO()
traceback.print_exc(file=f)
message = f.getvalue()
else:
subject = " HQAdmin preindex_everything may or may not be complete"
message = (
"We heard a rumor that preindex is complete,\n"
"but it's on you to check that all tasks are complete."
)
if email:
mail_admins(subject, message)
else:
print '{}\n\n{}'.format(subject, message)
def current_sha(self):
current_snapshot = gitinfo.get_project_snapshot(self.root_dir, submodules=False)
sha = current_snapshot['commits'][0]['sha']
print("Current commit SHA: %s" % sha)
return sha
def current_sha(self):
current_snapshot = gitinfo.get_project_snapshot(self.root_dir,
submodules=False)
sha = current_snapshot['commits'][0]['sha']
print("Current commit SHA: %s" % sha)
return sha
def handle(self, *args, **options):
if len(args) == 0:
num_pool = POOL_SIZE
else:
num_pool = int(args[0])
if len(args) > 1:
username = args[1]
else:
username = '******'
email = options['mail']
root_dir = settings.FILEPATH
git_snapshot = gitinfo.get_project_snapshot(
root_dir,
submodules=False,
log_count=1,
)
head = git_snapshot['commits'][0]
if options['check']:
exit(0 if get_preindex_complete(head) else 1)
if get_preindex_complete(head):
mail_admins('Already preindexed', "Skipping this step")
return
else:
clear_preindex_complete()
commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % (
git_snapshot['current_branch'], head['sha'])
pre_message = list()
pre_message.append("Heads up, %s has started preindexing" % username)
pre_message.append(commit_info)
if email:
mail_admins(
" HQAdmin preindex_everything started", '\n'.join(pre_message)
)
def couch_preindex():
call_command('sync_prepare_couchdb_multi', num_pool, username,
**{'no_mail': True})
print "Couch preindex done"
def pillow_preindex():
call_command('ptop_preindex')
print "ptop_preindex_done"
jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)]
gevent.joinall(jobs)
try:
for job in jobs:
job.get()
except Exception:
subject = " HQAdmin preindex_everything failed"
f = StringIO()
traceback.print_exc(file=f)
message = f.getvalue()
else:
subject = " HQAdmin preindex_everything may or may not be complete"
message = (
"We heard a rumor that preindex is complete,\n"
"but it's on you to check that all tasks are complete."
)
set_preindex_complete(head)
if email:
mail_admins(subject, message)
else:
print '{}\n\n{}'.format(subject, message)
