def dumpJson(json_object, dump_file): try: with open(dump_file, 'w') as out: json.dump(json_object, out, indent=1) except Exception as err: traceback.print_exc() raise StackAdvisorException("Error writing to file {0} : {1}".format(dump_file, str(err)))
def put_structured_out(self, sout): Script.structuredOut.update(sout) try: with open(self.stroutfile, 'w') as fp: json.dump(Script.structuredOut, fp) except IOError, err: Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile})
def update_definitions(self, heartbeat): """ Updates the persisted alert definitions JSON. :param heartbeat: :return: """ if 'alertDefinitionCommands' not in heartbeat: logger.warning("There are no alert definition commands in the heartbeat; unable to update definitions") return # prune out things we don't want to store alert_definitions = [] for command in heartbeat['alertDefinitionCommands']: command_copy = command.copy() # no need to store these since we always use the in-memory cached values if 'configurations' in command_copy: del command_copy['configurations'] alert_definitions.append(command_copy) # write out the new definitions with open(os.path.join(self.cachedir, self.FILENAME), 'w') as f: json.dump(alert_definitions, f, indent=2) # reschedule only the jobs that have changed self.reschedule()
def delete_storm_local_data(self, env): """ Deletes Storm data from local directories. This will create a marker file with JSON data representing the upgrade stack and request/stage ID. This will prevent multiple Storm components on the same host from removing the local directories more than once. :return: """ import params Logger.info('Clearing Storm data from local directories...') storm_local_directory = params.local_dir if storm_local_directory is None: raise Fail("The storm local directory specified by storm-site/storm.local.dir must be specified") request_id = default("/requestId", None) stage_id = default("/stageId", None) stack_version = params.version stack_name = params.stack_name json_map = {} json_map["requestId"] = request_id json_map["stageId"] = stage_id json_map["stackVersion"] = stack_version json_map["stackName"] = stack_name temp_directory = params.tmp_dir upgrade_file = os.path.join(temp_directory, "storm-upgrade-{0}.json".format(stack_version)) if os.path.exists(upgrade_file): try: with open(upgrade_file) as file_pointer: existing_json_map = json.load(file_pointer) if cmp(json_map, existing_json_map) == 0: Logger.info("The storm upgrade has already removed the local directories for {0}-{1} for request {2} and stage {3}".format( stack_name, stack_version, request_id, stage_id)) # nothing else to do here for this as it appears to have already been # removed by another component being upgraded return except: Logger.error("The upgrade file {0} appears to be corrupt; removing...".format(upgrade_file)) File(upgrade_file, action="delete") else: # delete the upgrade file since it does not match File(upgrade_file, action="delete") # delete from local directory Directory(storm_local_directory, action="delete", recursive=True) # recreate storm local directory Directory(storm_local_directory, mode=0755, owner = params.storm_user, group = params.user_group, recursive = True) # the file doesn't exist, so create it with open(upgrade_file, 'w') as file_pointer: json.dump(json_map, file_pointer, indent=2)
def new_cached_exec(key, file_path, kinit_path, temp_dir, exec_user, keytab_file, principal, hostname): """ Entry point of an actual execution - triggered when timeout on the cache expired or on fresh execution """ now = datetime.now() temp_kinit_cache_fd, temp_kinit_cache_filename = mkstemp(dir=temp_dir) command = "%s -c %s -kt %s %s" % \ (kinit_path, temp_kinit_cache_filename, keytab_file, principal.replace("_HOST", hostname)) os.close(temp_kinit_cache_fd) try: # Ensure the proper user owns this file File(temp_kinit_cache_filename, owner=exec_user, mode=0600) # Execute the kinit Execute(command, user=exec_user) with open(file_path, 'w+') as cache_file: result = {key: {"last_successful_execution": str(now)}} json.dump(result, cache_file) finally: File(temp_kinit_cache_filename, action='delete')
def persist_cache(self): # ensure that our cache directory exists if not os.path.exists(self.cluster_cache_dir): os.makedirs(self.cluster_cache_dir) with self.__file_lock: with open(self.__current_cache_json_file, 'w') as f: json.dump(self, f, indent=2) if self.hash is not None: with open(self.__current_cache_hash_file, 'w') as fp: fp.write(self.hash)
def persist_cache(self, cache_hash): # ensure that our cache directory exists if not os.path.exists(self.cluster_cache_dir): os.makedirs(self.cluster_cache_dir) with self.__file_lock: with open(self.__current_cache_json_file, 'w') as f: json.dump(self, f, indent=2) if self.hash is not None: with open(self.__current_cache_hash_file, 'w') as fp: fp.write(cache_hash) # if all of above are successful finally set the hash self.hash = cache_hash
def new_cached_exec(key, file_path, kinit_path, exec_user, keytab_file, principal, hostname): """ Entry point of an actual execution - triggered when timeout on the cache expired or on fresh execution """ now = datetime.now() _, temp_kinit_cache_file = mkstemp() command = "su -s /bin/bash - %s -c '%s -c %s -kt %s %s'" % \ (exec_user, kinit_path, temp_kinit_cache_file, keytab_file, principal.replace("_HOST", hostname)) try: Execute(command) with open(file_path, 'w+') as cache_file: result = {key: {"last_successful_execution": str(now)}} json.dump(result, cache_file) finally: os.remove(temp_kinit_cache_file)
def new_cached_exec(key, file_path, kinit_path, temp_dir, exec_user, keytab_file, principal, hostname): """ Entry point of an actual execution - triggered when timeout on the cache expired or on fresh execution """ now = datetime.now() temp_kinit_cache_fd, temp_kinit_cache_filename = mkstemp(dir=temp_dir) command = "%s -c %s -kt %s %s" % \ (kinit_path, temp_kinit_cache_filename, keytab_file, principal.replace("_HOST", hostname)) os.close(temp_kinit_cache_fd) try: Execute(command, user=exec_user) with open(file_path, 'w+') as cache_file: result = {key: {"last_successful_execution": str(now)}} json.dump(result, cache_file) finally: File(temp_kinit_cache_filename, action='delete')
def delete_storm_local_data(self, env): """ Deletes Storm data from local directories. This will create a marker file with JSON data representing the upgrade stack and request/stage ID. This will prevent multiple Storm components on the same host from removing the local directories more than once. :return: """ import params Logger.info('Clearing Storm data from local directories...') storm_local_directory = params.local_dir if storm_local_directory is None: raise Fail( "The storm local directory specified by storm-site/storm.local.dir must be specified" ) request_id = default("/requestId", None) stage_id = default("/stageId", None) stack_version = params.version stack_name = params.stack_name json_map = {} json_map["requestId"] = request_id json_map["stageId"] = stage_id json_map["stackVersion"] = stack_version json_map["stackName"] = stack_name temp_directory = params.tmp_dir upgrade_file = os.path.join( temp_directory, "storm-upgrade-{0}.json".format(stack_version)) if os.path.exists(upgrade_file): try: with open(upgrade_file) as file_pointer: existing_json_map = json.load(file_pointer) if cmp(json_map, existing_json_map) == 0: Logger.info( "The storm upgrade has already removed the local directories for {0}-{1} for request {2} and stage {3}" .format(stack_name, stack_version, request_id, stage_id)) # nothing else to do here for this as it appears to have already been # removed by another component being upgraded return except: Logger.error( "The upgrade file {0} appears to be corrupt; removing...". format(upgrade_file)) File(upgrade_file, action="delete") else: # delete the upgrade file since it does not match File(upgrade_file, action="delete") # delete from local directory Directory(storm_local_directory, action="delete", recursive=True) # recreate storm local directory Directory(storm_local_directory, mode=0755, owner=params.storm_user, group=params.user_group, recursive=True) # the file doesn't exist, so create it with open(upgrade_file, 'w') as file_pointer: json.dump(json_map, file_pointer, indent=2)
def write_config(config, cfg_type, tag): file_name = cfg_type + "_" + tag if os.path.isfile(file_name): os.remove(file_name) json.dump(config, open(file_name, 'w'))
"request {2} and direction {3}. Nothing else to do.".format(stack_name, stack_version, request_id, upgrade_direction)) # Nothing else to do here for this as it appears to have already been # removed by another component being upgraded return else: Logger.info("The marker file differs from the new value. Will proceed to delete Storm local dir, " "and generate new file. Current marker file: {0}".format(str(existing_json_map))) except Exception, e: Logger.error("The marker file {0} appears to be corrupt; removing it. Error: {1}".format(marker_file, str(e))) File(marker_file, action="delete") else: Logger.info('The marker file {0} does not exist; will attempt to delete local Storm directory if it exists.'.format(marker_file)) # Delete from local directory if os.path.isdir(storm_local_directory): Logger.info("Deleting storm local directory, {0}".format(storm_local_directory)) Directory(storm_local_directory, action="delete", recursive=True) # Recreate storm local directory Logger.info("Recreating storm local directory, {0}".format(storm_local_directory)) Directory(storm_local_directory, mode=0755, owner=params.storm_user, group=params.user_group, recursive=True) # The file doesn't exist, so create it Logger.info("Saving marker file to {0} with contents: {1}".format(marker_file, str(json_map))) with open(marker_file, 'w') as file_pointer: json.dump(json_map, file_pointer, indent=2) if __name__ == "__main__": StormUpgrade().execute()
""" logger.info("Updating cached configurations for cluster {0}".format(cluster_name)) self.__cache_lock.acquire() try: self.__configurations[cluster_name] = configuration except Exception, exception : logger.exception("Unable to update configurations for cluster {0}".format(cluster_name)) finally: self.__cache_lock.release() self.__file_lock.acquire() try: with open(self.__config_json_file, 'w') as f: json.dump(self.__configurations, f, indent=2) except Exception, exception : logger.exception("Unable to update configurations for cluster {0}".format(cluster_name)) finally: self.__file_lock.release() def get_configuration_value(self, cluster_name, key): """ Gets a value from the cluster configuration map for the given cluster and key. The key is expected to be of the form 'foo-bar/baz' or 'foo-bar/bar-baz/foobarbaz' where every / denotes a new mapping :param key: a lookup key, like 'foo-bar/baz' :return: the value, or None if not found """ self.__cache_lock.acquire()
def write_file(self, filename, tags): runDir = self.findRunDir() conf_file = open(os.path.join(runDir, filename), 'w') json.dump(tags, conf_file) conf_file.close()
Logger.info( 'The marker file {0} does not exist; will attempt to delete local Storm directory if it exists.' .format(marker_file)) # Delete from local directory if os.path.isdir(storm_local_directory): Logger.info("Deleting storm local directory, {0}".format( storm_local_directory)) Directory(storm_local_directory, action="delete", create_parents=True) # Recreate storm local directory Logger.info("Recreating storm local directory, {0}".format( storm_local_directory)) Directory(storm_local_directory, mode=0755, owner=params.storm_user, group=params.user_group, create_parents=True) # The file doesn't exist, so create it Logger.info("Saving marker file to {0} with contents: {1}".format( marker_file, str(json_map))) with open(marker_file, 'w') as file_pointer: json.dump(json_map, file_pointer, indent=2) if __name__ == "__main__": StormUpgrade().execute()
def delete_storm_local_data(self, env): """ Deletes Storm data from local directories. This will create a marker file with JSON data representing the upgrade stack and request/stage ID. This will prevent multiple Storm components on the same host from removing the local directories more than once. :return: """ import params Logger.info('Clearing Storm data from local directories...') storm_local_directory = params.local_dir if storm_local_directory is None: raise Fail( "The storm local directory specified by storm-site/storm.local.dir must be specified" ) request_id = default("/requestId", None) stack_name = params.stack_name stack_version = params.version upgrade_direction = params.upgrade_direction json_map = {} json_map["requestId"] = request_id json_map["stackName"] = stack_name json_map["stackVersion"] = stack_version json_map["direction"] = upgrade_direction temp_directory = params.tmp_dir marker_file = os.path.join( temp_directory, "storm-upgrade-{0}.json".format(stack_version)) Logger.info("Marker file for upgrade/downgrade of Storm, {0}".format( marker_file)) if os.path.exists(marker_file): Logger.info("The marker file exists.") try: with open(marker_file) as file_pointer: existing_json_map = json.load(file_pointer) if cmp(json_map, existing_json_map) == 0: Logger.info( "The storm upgrade has already removed the local directories for {0}-{1} for " "request {2} and direction {3}. Nothing else to do.". format(stack_name, stack_version, request_id, upgrade_direction)) # Nothing else to do here for this as it appears to have already been # removed by another component being upgraded return else: Logger.info( "The marker file differs from the new value. Will proceed to delete Storm local dir, " "and generate new file. Current marker file: {0}". format(str(existing_json_map))) except Exception as e: Logger.error( "The marker file {0} appears to be corrupt; removing it. Error: {1}" .format(marker_file, str(e))) File(marker_file, action="delete") else: Logger.info( 'The marker file {0} does not exist; will attempt to delete local Storm directory if it exists.' .format(marker_file)) # Delete from local directory if os.path.isdir(storm_local_directory): Logger.info("Deleting storm local directory, {0}".format( storm_local_directory)) Directory(storm_local_directory, action="delete", create_parents=True) # Recreate storm local directory Logger.info("Recreating storm local directory, {0}".format( storm_local_directory)) Directory(storm_local_directory, mode=0755, owner=params.storm_user, group=params.user_group, create_parents=True) # The file doesn't exist, so create it Logger.info("Saving marker file to {0} with contents: {1}".format( marker_file, str(json_map))) with open(marker_file, 'w') as file_pointer: json.dump(json_map, file_pointer, indent=2)
def write_mapping(hostmapping): if os.path.isfile(Options.MR_MAPPING_FILE): os.remove(Options.MR_MAPPING_FILE) json.dump(hostmapping, open(Options.MR_MAPPING_FILE, 'w'))