def save_config(connection, path, config_name): """ Saves config file with new content. Returns empty string for valid flask response and error message otherwise. """ new_content = request.values.get('content', '') root_passwd = request.values.get('root_passwd', '') if new_content: try: f = connection.get_file(os.path.join(path, config_name), 'w') except Exception as e: logger.exception(e) return 'Error while opening file %s.<br>%s' % (config_name, e) else: f.write(new_content) f.close() logger.info('%s was successfully saved.' % config_name) elif not new_content: msg = 'Attempting for writing of empty content to %s.' % config_name logger.error(msg) return msg else: return 'Invalid root password.' return ''
def get_mfsmaster_info(connection, host, port, configs_path): """ returns some of mfsmaster options in a dictionary or gives default values, asigned in this module """ errors = [] mfsmaster_cfg_path = os.path.join(configs_path, CONFIGS[0]) meta_path = meta_path = moose_options.get('meta_path', '') version = '' try: mfsmaster_cfg = connection.get_file(mfsmaster_cfg_path, 'r') except mfs_exceptions.OpenRemoteFileFailed: error = 'Cannot find \"%s\" in %s.' % (CONFIGS[0], configs_path) logger.error(mfs_exceptions.MetafilesException(error)) else: lines = mfsmaster_cfg.readlines() if not meta_path: meta_path = get_value_by_key('DATA_PATH', lines) mfsmaster_cfg.close() if not meta_path: meta_path = DEFAULT_META_PATH version = get_master_version(host, port) if not isinstance(version, tuple): errors.append(version) return { 'meta_path': meta_path, 'port': port, 'version': version, 'host': host, 'errors': errors }
def resolv_check(config_obj): # Addresses validation ip_pattern = re.compile("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") name_pattern = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE) address_for_valid = {'general': 'host', 'moose_options': 'master_host'} if config_obj.get('general', 'host') is not None: for i in address_for_valid: if not ip_pattern.match(config_obj.get(i, address_for_valid[i])): if not name_pattern.match(config_obj.get(i, address_for_valid[i])): msg = '%s address is not valid in %s section.' % (address_for_valid[i], i) logger.error(msg) raise mfs_exceptions.AddressValidError(msg) else: try: socket.gethostbyname(config_obj.get(i, address_for_valid[i])) except Exception: msg = "%s doesn't resolve" % str(config_obj.get(i, address_for_valid[i])) logger.error(msg) raise mfs_exceptions.HostResolveError(msg) # Check whether master address is valid master_host = config_obj.get('moose_options', 'master_host') try: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((master_host, 22)) except Exception: msg = '%s host is innaccessible' % str(master_host) logger(msg) raise mfs_exceptions.MooseConnectionFailed(msg) finally: s.close()
def get_mfsmaster_info(connection, host, port, configs_path): """ returns some of mfsmaster options in a dictionary or gives default values, asigned in this module """ errors = [] mfsmaster_cfg_path = os.path.join(configs_path, CONFIGS[0]) meta_path = meta_path = moose_options.get('meta_path', '') version = '' try: mfsmaster_cfg = connection.get_file(mfsmaster_cfg_path, 'r') except mfs_exceptions.OpenRemoteFileFailed: error = 'Cannot find \"%s\" in %s.' % (CONFIGS[0], configs_path) logger.error(mfs_exceptions.MetafilesException(error)) else: lines = mfsmaster_cfg.readlines() if not meta_path: meta_path = get_value_by_key('DATA_PATH', lines) mfsmaster_cfg.close() if not meta_path: meta_path = DEFAULT_META_PATH version = get_master_version(host, port) if not isinstance(version, tuple): errors.append(version) return { 'meta_path' : meta_path, 'port' : port, 'version' : version, 'host' : host, 'errors' : errors }
def get_master_version(host, port): try: mfs = moose_lib.MooseFS(masterhost=host, masterport=int(port)) version = mfs.masterversion except Exception as e: error = 'Failed to connect to %s:%s.<br/>%s' % (host, port, e) logger.error(error) return mfs_exceptions.MooseConnectionFailed(error) else: return version
def _sftp_connect(self): """ Helper method. It initialize sftp connection with remote host. >>> obj._sftp_connect() """ try: sftp = self.ssh.open_sftp() except Exception as e: msg = "Failed to open sftp session: %s" % str(e) logger.error(e) raise mfs_exceptions.MooseConnectionFailed(msg) return sftp
def mfs_mounts(self): clients = [] try: s = self.bind_to_master() self.mysend(s, struct.pack(">LLB", CLTOMA_SESSION_LIST, 1, 1)) header = self.myrecv(s, 8) cmd, length = struct.unpack(">LL", header) if cmd == MATOCL_SESSION_LIST and self.masterversion >= (1, 5, 14): data = self.myrecv(s, length) statscnt = struct.unpack(">H", data[0:2])[0] pos = 2 while pos < length: sessionid, ip1, ip2, ip3, ip4, v1, v2, v3, ileng = struct.unpack( ">LBBBBHBBL", data[pos:pos + 16]) ipnum = "%d.%d.%d.%d" % (ip1, ip2, ip3, ip4) ver = "%d.%d.%d" % (v1, v2, v3) pos += 16 info = data[pos:pos + ileng] pos += ileng pleng = struct.unpack(">L", data[pos:pos + 4])[0] pos += 4 path = data[pos:pos + pleng] pos += pleng try: host = (socket.gethostbyaddr(ipnum))[0] except Exception: host = "(unresolved)" clients.append({ 'host': host, 'ip': ipnum, 'mount_point': info, 'mfsmount_root': path }) return clients else: msg = 'Error during clients list obtainig. Check master version, it could be obsolite' logger.error(msg) raise mfs_exceptions.MooseError(msg) except Exception as e: msg = 'Error during connect to master: %s' % str(e) logger.error(msg) raise mfs_exceptions.MooseConnectionFailed(msg) finally: s.close() return clients
def _sftp_connect(self): """ Helper method. It initialize sftp connection with remote host. >>> obj._sftp_connect() """ try: sftp = self.ssh.open_sftp() except Exception, e: msg = "Failed to open sftp session: %s" % str(e) logger.error(e) raise errors.MooseConnectionFailed(msg)
def mfs_mounts(self): clients = [] try: s = self.bind_to_master() self.mysend(s, struct.pack(">LLB", CLTOMA_SESSION_LIST, 1, 1)) header = self.myrecv(s, 8) cmd, length = struct.unpack(">LL", header) if cmd == MATOCL_SESSION_LIST and self.masterversion >= (1, 5, 14): data = self.myrecv(s, length) statscnt = struct.unpack(">H", data[0:2])[0] pos = 2 while pos < length: sessionid, ip1, ip2, ip3, ip4, v1, v2, v3, ileng = struct.unpack(">LBBBBHBBL", data[pos:pos+16]) ipnum = "%d.%d.%d.%d" % (ip1, ip2, ip3, ip4) ver = "%d.%d.%d" % (v1, v2, v3) pos += 16 info = data[pos:pos+ileng] pos += ileng pleng = struct.unpack(">L", data[pos:pos+4])[0] pos += 4 path = data[pos:pos+pleng] pos += pleng try: host = (socket.gethostbyaddr(ipnum))[0] except Exception: host = "(unresolved)" clients.append( { 'host': host, 'ip': ipnum, 'mount_point': info, 'mfsmount_root': path } ) return clients else: msg = 'Error during clients list obtainig. Check master version, it could be obsolite' logger.error(msg) raise mfs_exceptions.MooseError(msg) except Exception as e: msg = 'Error during connect to master: %s' % str(e) logger.error(msg) raise mfs_exceptions.MooseConnectionFailed(msg) finally: s.close() return clients
def get_file(self, path, mode): """ Return file object related with remote file. Method takes path to the file and mode args. >>> obj.get_file('/etc/mfs/mfshdd.cfg', 'w') >>> obj.get_file('/etc/mfs/mfshdd.cfg', 'rb') """ try: file_o = self.remote.file(path, mode) except IOError, e: msg = "Failed to open remote file: %s" % str(e) logger.error(e) raise errors.MooseConnectionFailed(msg)
def get_file(self, path, mode): """ Return file object related with remote file. Method takes path to the file and mode args. >>> obj.get_file('/etc/mfs/mfshdd.cfg', 'w') >>> obj.get_file('/etc/mfs/mfshdd.cfg', 'rb') """ try: file_o = self.remote.file(path, mode) except IOError as e: msg = "Failed to open remote file: %s" % str(e) logger.error(e) raise mfs_exceptions.OpenRemoteFileFailed(msg) return file_o
def _connect(self, ssh_client): """ Helper for connect method. According to __init__ arguments (auth type, login, pwd) return appropriate ssh connection. >>> obj._connect(ssh_client) """ try: ssh_client.connect(self.host, username=self.user, key_filename=self.private_key_file) except Exception as e: msg = "\n".join(['ssh connection failed with the following error:\n%s' % str(e), 'CONNECTION DETAILS:', 'host: %s' % self.host, 'username: %s' % self.user, 'private key file: %s' % self.private_key_file]) logger.error(e) raise mfs_exceptions.MooseConnectionFailed(msg) return ssh_client
def _connect(self, ssh_client): """ Helper for connect method. According to __init__ arguments (auth type, login, pwd) return appropriate ssh connection. >>> obj._connect(ssh_client) """ try: if self.remote_auth_type == 'pwd': ssh_client.connect(self.host, username=self.user, password=self.password) elif self.remote_auth_type == 'key': if not self.remote_auth_passwd: ssh_client.connect(self.host, username=self.user, key_filename=self.private_key_file) elif self.remote_auth_type == 'key' and self.remote_auth_passwd: ssh_client.connect(self.host, username=self.user, key_filename=self.private_key_file, password=self.password) except Exception, e: msg = 'Error during ssh client connect: %s' % str(e) logger.error(e) raise errors.MooseConnectionFailed(msg)
def _connect(self, ssh_client): """ Helper for connect method. According to __init__ arguments (auth type, login, pwd) return appropriate ssh connection. >>> obj._connect(ssh_client) """ try: ssh_client.connect(self.host, username=self.user, key_filename=self.private_key_file) except Exception as e: msg = "\n".join([ 'ssh connection failed with the following error:\n%s' % str(e), 'CONNECTION DETAILS:', 'host: %s' % self.host, 'username: %s' % self.user, 'private key file: %s' % self.private_key_file ]) logger.error(e) raise mfs_exceptions.MooseConnectionFailed(msg) return ssh_client
def get_meta_info(meta_path, connection, configs_path): """ Gets DATA_PATH option from configs_path/mfsmaster.cfg file and tries to get information about metafiles in DATA_PATH. Adds errors if mfsmaster.cfg is missing or DATA_PATH is not readable or not exists. """ errors = [] metafiles = [] if connection.path_exists(meta_path): metafiles = connection.get_files_info(meta_path) else: msg = 'Metafiles path % s does not exist.' % meta_path logger.error(mfs_exceptions.MetafilesException(msg)) errors.append(''.join([msg, '<br/>Check \"meta_path\" option in \"%s\" or \"DATA_PATH\" option in %s.' % \ (DEFAULT_MFSTOOL_CONFIG_PATH, CONFIGS[0])])) return { 'metafiles' : metafiles, 'errors' : errors, }
def get_meta_info(meta_path, connection, configs_path): """ Gets DATA_PATH option from configs_path/mfsmaster.cfg file and tries to get information about metafiles in DATA_PATH. Adds errors if mfsmaster.cfg is missing or DATA_PATH is not readable or not exists. """ errors = [] metafiles = [] if connection.path_exists(meta_path): metafiles = connection.get_files_info(meta_path) else: msg = 'Metafiles path % s does not exist.' % meta_path logger.error(mfs_exceptions.MetafilesException(msg)) errors.append(''.join([msg, '<br/>Check \"meta_path\" option in \"%s\" or \"DATA_PATH\" option in %s.' % \ (DEFAULT_MFSTOOL_CONFIG_PATH, CONFIGS[0])])) return { 'metafiles': metafiles, 'errors': errors, }
def network_check(config_obj): # Check whether app port is accessible port = config_obj.get('general', 'port') address = config_obj.get('general', 'host') s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if port is None and address is None: port = DEFAULT_APP_PORT address = DEFAULT_APP_ADDRESS result = s.connect_ex((DEFAULT_APP_ADDRESS, int(port))) elif port is None and address is not None: port = DEFAULT_APP_PORT result = s.connect_ex((address, DEFAULT_APP_PORT)) else: address = DEFAULT_APP_ADDRESS result = s.connect_ex((DEFAULT_APP_ADDRESS, int(port))) if result == 0: s.close() msg = '%s port is already used' % str(port) logger.error(msg) raise mfs_exceptions.PortUsageError(msg)
def mfs_servers(self): # Return list of chunk servers servers = [] try: s = self.bind_to_master() self.mysend(s, struct.pack(">LL", CLTOMA_CSERV_LIST, 0)) header = self.myrecv(s, 8) cmd, length = struct.unpack(">LL", header) if cmd == MATOCL_CSERV_LIST and self.masterversion >= (1, 5, 13) and (length % 54) == 0: data = self.myrecv(s, length) n = length/54 for i in xrange(n): d = data[i*54:(i+1)*54] disconnected,v1,v2,v3,ip1,ip2,ip3,ip4,port,used,total,chunks,tdused,tdtotal,tdchunks, \ errcnt = struct.unpack(">BBBBBBBBHQQLQQLL",d) strip = "%u.%u.%u.%u" % (ip1,ip2,ip3,ip4) try: host = (socket.gethostbyaddr(strip))[0] except Exception as e: host = "(unresolved)" msg = 'Error during server hostname getting %s' % str(e) logger.error(msg) servers.append({ 'host': host, 'ip': strip, 'port': port }) return servers else: msg = 'Error during servers list obtainig. Check master version, it could be obsolite' logger.error(msg) raise mfs_exceptions.MooseError(msg) except Exception as e: msg = 'Error during connect to master: %s' % str(e) logger.error(msg) raise mfs_exceptions.MooseConnectionFailed(msg) finally: s.close()
def directives_check(config_obj): # Check whether options = {'moose_options': 'master_host', 'ssh_options': 'key'} for i in options: if not config_obj.has_option(i, options.get(i)): msg = '%s value is required for %s directive.' % (options.get(i), i) logger.error(msg) raise mfs_exceptions.ValueError(msg) # Check whether ssh key exists if not config_obj.get('ssh_options', 'key'): msg = 'SSH key file is not specified.' logger.error(msg) raise mfs_exceptions.KeyFileMissing(msg) elif not os.path.isfile(config_obj.get('ssh_options', 'key')): msg = 'Specified SSH key file is absent' logger.error(msg) raise mfs_exceptions.InvalidKeyFile(msg)
def mfs_backup_servers(self): # Return list of metaloggers servers metaloggers = [] try: s = self.bind_to_master() self.mysend(s, struct.pack(">LL", 522, 0)) header = self.myrecv(s, 8) cmd, length = struct.unpack(">LL", header) if cmd == MATOCL_MLOG_LIST and (length % 8) == 0: data = self.myrecv(s, length) n = length / 8 for i in xrange(n): d = data[i * 8:(i + 1) * 8] v1, v2, v3, ip1, ip2, ip3, ip4 = struct.unpack( ">HBBBBBB", d) ip = '.'.join([str(ip1), str(ip2), str(ip3), str(ip4)]) ver = '.'.join([str(v1), str(v2), str(v3)]) try: host = (socket.gethostbyaddr("%u.%u.%u.%u" % (ip1, ip2, ip3, ip4)))[0] except Exception as e: host = "(unresolved)" msg = 'Error during server hostname getting %s' % str( e) logger.error(msg) raise mfs_exceptions.MooseError(msg) metaloggers.append({'host': host, 'ip': ip}) else: msg = 'Error during metaloggers list obtainig. Check master version, it could be obsolite' logger.error(msg) raise mfs_exceptions.MooseError(msg) except Exception as e: msg = 'Error during connect to master: %s' % str(e) logger.error(msg) raise mfs_exceptions.MooseConnectionFailed(msg) finally: s.close() return metaloggers
def mfs_servers(self): # Return list of chunk servers servers = [] try: s = self.bind_to_master() self.mysend(s, struct.pack(">LL", CLTOMA_CSERV_LIST, 0)) header = self.myrecv(s, 8) cmd, length = struct.unpack(">LL", header) if cmd == MATOCL_CSERV_LIST and self.masterversion >= ( 1, 5, 13) and (length % 54) == 0: data = self.myrecv(s, length) n = length / 54 for i in xrange(n): d = data[i * 54:(i + 1) * 54] disconnected,v1,v2,v3,ip1,ip2,ip3,ip4,port,used,total,chunks,tdused,tdtotal,tdchunks, \ errcnt = struct.unpack(">BBBBBBBBHQQLQQLL",d) strip = "%u.%u.%u.%u" % (ip1, ip2, ip3, ip4) try: host = (socket.gethostbyaddr(strip))[0] except Exception as e: host = "(unresolved)" msg = 'Error during server hostname getting %s' % str( e) logger.error(msg) servers.append({'host': host, 'ip': strip, 'port': port}) return servers else: msg = 'Error during servers list obtainig. Check master version, it could be obsolite' logger.error(msg) raise mfs_exceptions.MooseError(msg) except Exception as e: msg = 'Error during connect to master: %s' % str(e) logger.error(msg) raise mfs_exceptions.MooseConnectionFailed(msg) finally: s.close()
def get_configs_info(connection, files_list=CONFIGS.values()): """ Tries to get configs_path from moosefs_tool.ini. Checks files from files_list in configs_path on remote host. Adds config's errors if they are. """ errors = [] configs_path = '' configs = [] try: configs_path = moose_options.get('configs_path', DEFAULT_CONFIGS_PATH) except Exception as e: msg = 'Cannot get \"configs_path\".<br/>%s' % str(e) logger.error(mfs_exceptions.ConfigsException(msg)) errors.append(msg) else: for file in files_list: full_path = os.path.join(configs_path, file) if connection.path_exists(full_path): f = connection.get_file(full_path, 'r') configs.append(file) f.close() else: msg = '\"%s\" is missing in %s.' % (file, configs_path) logger.error(mfs_exceptions.ConfigsException(msg)) errors.append(msg) if not configs: msg = 'You have no configs in %s.<br/>' % configs_path logger.error(mfs_exceptions.ConfigsException(msg)) errors.append(''.join([msg, 'Please check \"configs_path\" option in %s' % \ DEFAULT_MFSTOOL_CONFIG_PATH])) return { 'configs_path': configs_path, 'configs': configs, 'errors': errors, }
def mfs_backup_servers(self): # Return list of metaloggers servers metaloggers = [] try: s = self.bind_to_master() self.mysend(s, struct.pack(">LL", 522, 0)) header = self.myrecv(s, 8) cmd, length = struct.unpack(">LL", header) if cmd == MATOCL_MLOG_LIST and (length % 8) == 0: data = self.myrecv(s, length) n = length/8 for i in xrange(n): d = data[i*8:(i+1)*8] v1, v2, v3, ip1, ip2, ip3, ip4 = struct.unpack(">HBBBBBB", d) ip = '.'.join([str(ip1), str(ip2), str(ip3), str(ip4)]) ver = '.'.join([str(v1), str(v2), str(v3)]) try: host = (socket.gethostbyaddr("%u.%u.%u.%u" % (ip1, ip2, ip3, ip4)))[0] except Exception as e: host = "(unresolved)" msg = 'Error during server hostname getting %s' % str(e) logger.error(msg) raise mfs_exceptions.MooseError(msg) metaloggers.append({ 'host': host, 'ip': ip }) else: msg = 'Error during metaloggers list obtainig. Check master version, it could be obsolite' logger.error(msg) raise mfs_exceptions.MooseError(msg) except Exception as e: msg = 'Error during connect to master: %s' % str(e) logger.error(msg) raise mfs_exceptions.MooseConnectionFailed(msg) finally: s.close() return metaloggers
def get_meta_info(connection, errors, config_path): """ Gets DATA_PATH option from config_path/mfsmaster.cfg file and tries to get information about metafiles in DATA_PATH. Adds errors if mfsmaster.cfg is missing or DATA_PATH is not readable or not exists. """ errors['metafiles'] = [] meta_path = '' metafiles = [] mfsmaster_cfg_path = os.path.join(config_path, CONFIGS[0]) try: mfsmaster_cfg = connection.get_file(mfsmaster_cfg_path, 'r') data_line = ''.join([l for l in mfsmaster_cfg.readlines() \ if 'DATA_PATH' in l]) meta_path = re.split(' ?= ?', data_line)[1].strip() if connection.path_exists(meta_path): metafiles = connection.get_files_info(meta_path) else: msg = 'Metafiles path % s does not exist.' % meta_path logger.error(mfs_exceptions.MetafilesException(msg)) errors['metafiles'].append(''.join([msg, '<br/>Change DATA_PATH option in mfsmaster.cfg.'])) except mfs_exceptions.OpenRemoteFileFailed: msg = 'Cannot find \"%s\" in %s.' % (CONFIGS[0], config_path) logger.error(mfs_exceptions.MetafilesException(msg)) errors['metafiles'].append(msg) except IndexError: msg = 'Cannot read DATA_PATH option in %s.' % CONFIGS[0] logger.error(mfs_exceptions.MetafilesException(msg)) errors['metafiles'].append(msg) else: mfsmaster_cfg.close() return meta_path, metafiles, errors
def get_configs_info(connection, files_list=CONFIGS.values()): """ Tries to get configs_path from moosefs_tool.ini. Checks files from files_list in configs_path on remote host. Adds config's errors if they are. """ errors = [] configs_path = '' configs = [] try: configs_path = moose_options.get('configs_path', DEFAULT_CONFIGS_PATH) except Exception as e: msg = 'Cannot get \"configs_path\".<br/>%s' % str(e) logger.error(mfs_exceptions.ConfigsException(msg)) errors.append(msg) else: for file in files_list: full_path = os.path.join(configs_path, file) if connection.path_exists(full_path): f = connection.get_file(full_path, 'r') configs.append(file) f.close() else: msg = '\"%s\" is missing in %s.' % (file, configs_path) logger.error(mfs_exceptions.ConfigsException(msg)) errors.append(msg) if not configs: msg = 'You have no configs in %s.<br/>' % configs_path logger.error(mfs_exceptions.ConfigsException(msg)) errors.append(''.join([msg, 'Please check \"configs_path\" option in %s' % \ DEFAULT_MFSTOOL_CONFIG_PATH])) return { 'configs_path' : configs_path, 'configs' : configs, 'errors' : errors, }
def get_config_info(connection, errors, files_list=CONFIGS.values()): """ Tries to get config_path from moosefs_tool.ini. Checks files from files_list in config_path on remote host. Adds config's errors if they are. """ errors['configs'] = [] config_path = '' configs = [] try: config_path = roots['config_path'] except KeyError: msg = 'Cannot get \"config_path\" option from moosefs_tool.ini' logger.error(mfs_exceptions.ConfigsException(msg)) errors['configs'].append(''.join([msg, '<br/>Please set \"config_path\" option', 'and restart application.'])) else: for file in files_list: full_path = os.path.join(config_path, file) if connection.path_exists(full_path): f = connection.get_file(full_path, 'r') configs.append(file) f.close() else: msg = '\"%s\" is missing in %s.' % (file, config_path) logger.error(mfs_exceptions.ConfigsException(msg)) errors['configs'].append(msg) if not configs: msg = 'You have no configs in %s.<br/>' % config_path logger.error(mfs_exceptions.ConfigsException(msg)) errors['configs'].append(''.join([msg, 'Please check \"config_path\" option in moosefs_tool.ini'])) return config_path, configs, errors
def sections_check(config_obj): for i in ('moose_options', 'ssh_options'): if not config_obj.has_section(i): msg = '%s section is missing. This section is required' % str(i) logger.error(msg) raise mfs_exceptions.SectionMissing(msg)