def handle(self, *args, **options): self.root_dir = settings.FILEPATH prefix = os.getcwd() current_snapshot = gitinfo.get_project_snapshot(self.root_dir, submodules=False) current_sha = current_snapshot['commits'][0]['sha'] print "Current commit SHA: %s" % current_sha if 'clear' in args: print "clearing resource cache" rcache.delete_pattern(RESOURCE_PREFIX % '*') existing_resource_str = rcache.get(RESOURCE_PREFIX % current_sha, None) if existing_resource_str: print "getting resource dict from cache" self.output_resources(existing_resource_str) return self.resources = {} for finder in finders.get_finders(): for path, storage in finder.list(['.*', '*~', '* *']): if not storage.location.startswith(prefix): continue url = os.path.join(storage.prefix, path) if storage.prefix else path parts = (storage.location + '/' + path).split('/') self.generate_output(url, parts) resource_str = simplejson.dumps(self.resources, indent=2) rcache.set(RESOURCE_PREFIX % current_sha, resource_str, 86400) self.output_resources(resource_str)
def handle(self, *args, **options): prefix = os.getcwd() current_snapshot = gitinfo.get_project_snapshot(self.root_dir, submodules=False) current_sha = current_snapshot['commits'][0]['sha'] print "Current commit SHA: %s" % current_sha if 'clear' in args: print "clearing resource cache" rcache.delete_pattern(RESOURCE_PREFIX % '*') existing_resource_str = rcache.get(RESOURCE_PREFIX % current_sha, None) if existing_resource_str: print "getting resource dict from cache" self.output_resources(existing_resource_str) return resources = {} for finder in finders.get_finders(): for path, storage in finder.list(['.*', '*~', '* *']): if not storage.location.startswith(prefix): continue if not getattr(storage, 'prefix', None): url = path else: url = os.path.join(storage.prefix, path) filename = os.path.join(storage.location, path) resources[url] = self.get_hash(filename) resource_str = json.dumps(resources, indent=2) rcache.set(RESOURCE_PREFIX % current_sha, resource_str, 86400) self.output_resources(resource_str)
def handle(self, num_pool, username, **options): email = options['mail'] root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot( root_dir, submodules=False, log_count=1, ) head = git_snapshot['commits'][0] if options['check']: exit(0 if get_preindex_complete(head) else 1) if get_preindex_complete(head) and email: mail_admins('Already preindexed', "Skipping this step") return else: clear_preindex_complete() commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % ( git_snapshot['current_branch'], head['sha']) pre_message = list() pre_message.append("Heads up, %s has started preindexing" % username) pre_message.append(commit_info) if email: mail_admins(" HQAdmin preindex_everything started", '\n'.join(pre_message)) def couch_preindex(): call_command('sync_prepare_couchdb_multi', str(num_pool), username, **{'no_mail': True}) print("Couch preindex done") def pillow_preindex(): call_command('ptop_preindex') print("ptop_preindex_done") jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)] gevent.joinall(jobs) try: for job in jobs: job.get() except Exception: subject = " HQAdmin preindex_everything failed" message = get_traceback_string() else: subject = " HQAdmin preindex_everything may or may not be complete" message = ("We heard a rumor that preindex is complete,\n" "but it's on you to check that all tasks are complete.") set_preindex_complete(head) if email: mail_admins(subject, message) else: print('{}\n\n{}'.format(subject, message))
def handle(self, *args, **options): self.root_dir = settings.FILEPATH prefix = os.getcwd() current_snapshot = gitinfo.get_project_snapshot(self.root_dir, submodules=False) current_sha = current_snapshot['commits'][0]['sha'] print "Current commit SHA: %s" % current_sha if 'clear' in args: print "clearing resource cache" rcache.delete_pattern(RESOURCE_PREFIX % '*') existing_resource_str = rcache.get(RESOURCE_PREFIX % current_sha, None) if existing_resource_str: print "getting resource dict from cache" self.output_resources(existing_resource_str) return pool = Pool(10) self.resources = {} for finder in finders.get_finders(): for path, storage in finder.list(['.*', '*~', '* *']): if not storage.location.startswith(prefix): continue url = os.path.join(storage.prefix, path) if storage.prefix else path parts = (storage.location + '/' + path).split('/') pool.spawn(self.generate_output, url, parts) pool.join() resource_str = simplejson.dumps(self.resources, indent=2) rcache.set(RESOURCE_PREFIX % current_sha, resource_str, 86400) self.output_resources(resource_str)
def handle(self, *args, **options): root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True) git_snapshot['diff_url'] = options.get('url', None) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], code_snapshot=git_snapshot, ) deploy.save() # reset PillowTop errors in the hope that a fix has been deployed rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow()) if rows_updated: print "\n---------------- Pillow Errors Reset ----------------\n" \ "{} pillow errors queued for retry\n".format(rows_updated) if options['mail_admins']: snapshot_table = render_to_string('hqadmin/partials/project_snapshot.html', dictionary={'snapshot': git_snapshot}) message = "Deployed by %s, cheers!" % options['user'] snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % (message, snapshot_table) call_command('mail_admins', snapshot_body, **{'subject': 'Deploy successful', 'html': True})
def handle(self, *args, **options): if len(args) == 0: num_pool = POOL_SIZE else: num_pool = int(args[0]) if len(args) > 1: username = args[1] else: username = '******' email = options['mail'] root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot( root_dir, submodules=False, log_count=1, ) head = git_snapshot['commits'][0] commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % ( git_snapshot['current_branch'], head['sha']) pre_message = list() pre_message.append("Heads up, %s has started preindexing" % username) pre_message.append(commit_info) if email: mail_admins( " HQAdmin preindex_everything started", '\n'.join(pre_message) ) def couch_preindex(): call_command('sync_prepare_couchdb_multi', num_pool, username, **{'no_mail': True}) print "Couch preindex done" def pillow_preindex(): call_command('ptop_preindex') print "ptop_preindex_done" jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)] gevent.joinall(jobs) message = '\n'.join([ "We heard a rumor that preindex is complete, " "but it's on you to check that all tasks are complete." ]) if email: mail_admins( " HQAdmin preindex_everything may or may not be complete", message ) else: print message
def handle(self, *args, **options): if len(args) == 0: num_pool = POOL_SIZE else: num_pool = int(args[0]) if len(args) > 1: username = args[1] else: username = "******" email = options["mail"] root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=False, log_count=1) head = git_snapshot["commits"][0] commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % (git_snapshot["current_branch"], head["sha"]) pre_message = list() pre_message.append("Heads up, %s has started preindexing" % username) pre_message.append(commit_info) if email: mail_admins(" HQAdmin preindex_everything started", "\n".join(pre_message)) def couch_preindex(): call_command("sync_prepare_couchdb_multi", num_pool, username, **{"no_mail": True}) print "Couch preindex done" def pillow_preindex(): call_command("ptop_preindex") print "ptop_preindex_done" jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)] gevent.joinall(jobs) try: for job in jobs: job.get() except Exception: subject = " HQAdmin preindex_everything failed" f = StringIO() traceback.print_exc(file=f) message = f.getvalue() else: subject = " HQAdmin preindex_everything may or may not be complete" message = ( "We heard a rumor that preindex is complete,\n" "but it's on you to check that all tasks are complete." ) if email: mail_admins(subject, message) else: print "{}\n\n{}".format(subject, message)
def handle(self, *args, **options): root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True) compare_url = git_snapshot['diff_url'] = options.get('url', None) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], code_snapshot=git_snapshot, ) deploy.save() # reset PillowTop errors in the hope that a fix has been deployed rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow()) if rows_updated: print "\n---------------- Pillow Errors Reset ----------------\n" \ "{} pillow errors queued for retry\n".format(rows_updated) deploy_notification_text = ( "CommCareHQ has been successfully deployed to *{}* by *{}*. " "Find the diff {{diff_link}}".format( options['environment'], options['user'], ) ) if hasattr(settings, 'MIA_THE_DEPLOY_BOT_API'): link = diff_link(STYLE_SLACK, compare_url) requests.post(settings.MIA_THE_DEPLOY_BOT_API, data=json.dumps({ "username": "******", "text": deploy_notification_text.format(diff_link=link), })) if settings.DATADOG_API_KEY: tags = ['environment:{}'.format(options['environment'])] link = diff_link(STYLE_MARKDOWN, compare_url) datadog_api.Event.create( title="Deploy Success", text=deploy_notification_text.format(diff_link=link), tags=tags, alert_type="success" ) print "\n=============================================================\n" \ "Congratulations! Deploy Complete.\n\n" \ "Don't forget to keep an eye on the deploy dashboard to " \ "make sure everything is running smoothly.\n\n" \ "https://p.datadoghq.com/sb/5c4af2ac8-1f739e93ef" \ "\n=============================================================\n" if options['mail_admins']: message_body = get_deploy_email_message_body( environment=options['environment'], user=options['user'], compare_url=compare_url) call_command('mail_admins', message_body, **{'subject': 'Deploy successful', 'html': True})
def handle(self, *args, **options): start = datetime.utcnow() if len(args) == 0: num_pool = POOL_SIZE else: num_pool = int(args[0]) if len(args) > 1: username = args[1] else: username = "******" email = options["mail"] root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=False, log_count=1) head = git_snapshot["commits"][0] commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % (git_snapshot["current_branch"], head["sha"]) pre_message = list() pre_message.append("Heads up, %s has started preindexing" % username) pre_message.append(commit_info) if email: mail_admins(" HQAdmin preindex_everything started", "\n".join(pre_message)) def couch_preindex(): call_command("sync_prepare_couchdb_multi", num_pool, username, **{"no_mail": True}) print "Couch preindex done" def pillow_preindex(): call_command("ptop_preindex") print "ptop_preindex_done" jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)] gevent.joinall(jobs) message = list() message.append("Preindex results:") message.append("\tInitiated by: %s" % username) delta = datetime.utcnow() - start message.append("Total time: %d seconds" % delta.seconds) message.append("\nYou may now deploy") message.append(commit_info) if email: mail_admins(" HQAdmin preindex_everything complete", "\n".join(message)) else: print message
def handle(self, *args, **options): root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True) git_snapshot['diff_url'] = options.get('url', None) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], code_snapshot=git_snapshot, ) deploy.save() # reset PillowTop errors in the hope that a fix has been deployed rows_updated = PillowError.bulk_reset_attempts(datetime.utcnow()) if rows_updated: print "\n---------------- Pillow Errors Reset ----------------\n" \ "{} pillow errors queued for retry\n".format(rows_updated) deploy_notification_text = ( "CommCareHQ has been successfully deployed to *{}* by *{}*. " "Find the diff {{diff_link}}".format( options['environment'], options['user'], ) ) if hasattr(settings, 'MIA_THE_DEPLOY_BOT_API'): link = diff_link(STYLE_SLACK, git_snapshot['diff_url']) requests.post(settings.MIA_THE_DEPLOY_BOT_API, data=json.dumps({ "channel": "#dev", "username": "******", "text": deploy_notification_text.format(diff_link=link), "icon_emoji": ":see_no_evil:" })) if settings.DATADOG_API_KEY: tags = ['environment:{}'.format(options['environment'])] link = diff_link(STYLE_MARKDOWN, git_snapshot['diff_url']) datadog_api.Event.create( title="Deploy Success", text=deploy_notification_text.format(diff_link=link), tags=tags ) if options['mail_admins']: snapshot_table = render_to_string('hqadmin/partials/project_snapshot.html', dictionary={'snapshot': git_snapshot}) message = "Deployed by %s, cheers!" % options['user'] snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % (message, snapshot_table) call_command('mail_admins', snapshot_body, **{'subject': 'Deploy successful', 'html': True})
def handle(self, *args, **options): current_snapshot = gitinfo.get_project_snapshot(self.root_dir, submodules=False) self.current_sha = current_snapshot['commits'][0]['sha'] print "Current commit SHA: %s" % self.current_sha if 'save' in args: self.save_manifest() else: existing_resource_str = rcache.get(COMPRESS_PREFIX % self.current_sha, None) if existing_resource_str: self.output_manifest(existing_resource_str, is_soft_update='soft' in args) else: raise ResourceCompressError( "Could not find manifest.json in redis! Deploying under this " "condition will cause the server to 500." )
def handle(self, *args, **options): root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], code_snapshot=git_snapshot, ) deploy.save() if options['mail_admins']: snapshot_table = render_to_string('hqadmin/partials/project_snapshot.html', dictionary={'snapshot': git_snapshot}) message = "Deployed by %s, cheers!" % options['user'] snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % (message, snapshot_table) call_command('mail_admins', snapshot_body, **{'subject': 'Deploy successful', 'html': True})
def handle(self, *args, **options): root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot(root_dir, submodules=True) deploy = HqDeploy( date=datetime.utcnow(), user=options['user'], environment=options['environment'], code_snapshot=git_snapshot, ) deploy.save() if options['mail_admins']: snapshot_table = render_to_string( 'hqadmin/partials/project_snapshot.html', dictionary={'snapshot': git_snapshot}) message = "Deployed by %s, cheers!" % options['user'] snapshot_body = "<html><head><title>Deploy Snapshot</title></head><body><h2>%s</h2>%s</body></html>" % ( message, snapshot_table) call_command('mail_admins', snapshot_body, **{ 'subject': 'Deploy successful', 'html': True })
def handle(self, *args, **options): if len(args) == 0: num_pool = POOL_SIZE else: num_pool = int(args[0]) if len(args) > 1: username = args[1] else: username = '******' email = options['mail'] root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot( root_dir, submodules=False, log_count=1, ) head = git_snapshot['commits'][0] commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % ( git_snapshot['current_branch'], head['sha']) pre_message = list() pre_message.append("Heads up, %s has started preindexing" % username) pre_message.append(commit_info) if email: mail_admins( " HQAdmin preindex_everything started", '\n'.join(pre_message) ) def couch_preindex(): call_command('sync_prepare_couchdb_multi', num_pool, username, **{'no_mail': True}) print "Couch preindex done" def pillow_preindex(): call_command('ptop_preindex') print "ptop_preindex_done" jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)] gevent.joinall(jobs) try: for job in jobs: job.get() except Exception: subject = " HQAdmin preindex_everything failed" f = StringIO() traceback.print_exc(file=f) message = f.getvalue() else: subject = " HQAdmin preindex_everything may or may not be complete" message = ( "We heard a rumor that preindex is complete,\n" "but it's on you to check that all tasks are complete." ) if email: mail_admins(subject, message) else: print '{}\n\n{}'.format(subject, message)
def current_sha(self): current_snapshot = gitinfo.get_project_snapshot(self.root_dir, submodules=False) sha = current_snapshot['commits'][0]['sha'] print("Current commit SHA: %s" % sha) return sha
def handle(self, *args, **options): if len(args) == 0: num_pool = POOL_SIZE else: num_pool = int(args[0]) if len(args) > 1: username = args[1] else: username = '******' email = options['mail'] root_dir = settings.FILEPATH git_snapshot = gitinfo.get_project_snapshot( root_dir, submodules=False, log_count=1, ) head = git_snapshot['commits'][0] if options['check']: exit(0 if get_preindex_complete(head) else 1) if get_preindex_complete(head): mail_admins('Already preindexed', "Skipping this step") return else: clear_preindex_complete() commit_info = "\nCommit Info:\nOn Branch %s, SHA: %s" % ( git_snapshot['current_branch'], head['sha']) pre_message = list() pre_message.append("Heads up, %s has started preindexing" % username) pre_message.append(commit_info) if email: mail_admins( " HQAdmin preindex_everything started", '\n'.join(pre_message) ) def couch_preindex(): call_command('sync_prepare_couchdb_multi', num_pool, username, **{'no_mail': True}) print "Couch preindex done" def pillow_preindex(): call_command('ptop_preindex') print "ptop_preindex_done" jobs = [gevent.spawn(couch_preindex), gevent.spawn(pillow_preindex)] gevent.joinall(jobs) try: for job in jobs: job.get() except Exception: subject = " HQAdmin preindex_everything failed" f = StringIO() traceback.print_exc(file=f) message = f.getvalue() else: subject = " HQAdmin preindex_everything may or may not be complete" message = ( "We heard a rumor that preindex is complete,\n" "but it's on you to check that all tasks are complete." ) set_preindex_complete(head) if email: mail_admins(subject, message) else: print '{}\n\n{}'.format(subject, message)