def create_connection(module, blade):
"""Create connection between arrays"""
changed = True
if not module.check_mode:
remote_array = module.params['target_url']
try:
remote_system = PurityFb(module.params['target_url'])
remote_system.login(module.params['target_api'])
remote_array = remote_system.arrays.list_arrays().items[0].name
remote_conn_cnt = remote_system.array_connections.list_array_connections(
).pagination_info.total_item_count
# TODO: SD - Update with new max when fan-in/fan-out is enabled for FB
if remote_conn_cnt == 1:
module.fail_json(
msg=
"Remote array {0} already connected to another array. Fan-In not supported"
.format(remote_array))
connection_key = remote_system.array_connections.create_array_connections_connection_keys(
).items[0].connection_key
remote_array = remote_system.arrays.list_arrays().items[0].name
connection_info = ArrayConnectionPost(
management_address=module.params['target_url'],
encrypted=module.params['encrypted'],
connection_key=connection_key)
blade.array_connections.create_array_connections(
array_connection=connection_info)
except Exception:
module.fail_json(msg="Failed to connect to remote array {0}.".
format(remote_array))
module.exit_json(changed=changed)
def __init__(self, endpoint, api_token):
# self.fb = PurityFb(endpoint, conn_timeo=ctimeo, read_timeo=rtimeo,
# retries=retries)
self.flashblade = PurityFb(host=endpoint)
self.flashblade.disable_verify_ssl()
self.flashblade._api_client.user_agent = 'Purity_FB_Prometheus_exporter/1.0'
self.flashblade.request_timeout = urllib3.Timeout(connect=2.0,
read=60.0)
self.flashblade.login(api_token)
self.filesystems = []
self.buckets = []
self.array_performance = {}
self.array_performance['nfs'] = None
self.array_performance['http'] = None
self.array_performance['s3'] = None
self.array_performance['smb'] = None
self.array_specific_perf = {}
self.array_specific_perf['nfs'] = None
self.array_specific_perf['http'] = None
self.array_specific_perf['s3'] = None
self.array_space = None
self.nfs_filesystems_performance = []
self.buckets_performance = []
self.buckets_replica_links = []
self.filesystems_replica_links = []
self.users_usage = []
self.groups_usage = []
self.clients_performance = []
def __init__(self, target, api_token, request='all'):
# self.fb = PurityFb(endpoint, conn_timeo=ctimeo, read_timeo=rtimeo, retries=retries)
self.fb = PurityFb(host=target)
self.fb.disable_verify_ssl()
self.fb._api_client.user_agent = 'Purity_FB_Prometheus_exporter/1.0'
self.fb.request_timeout = urllib3.Timeout(connect=2.0, read=60.0)
self.fb.login(api_token)
self.request = request
self.filesystems = self.fb.file_systems.list_file_systems()
self.buckets = self.fb.buckets.list_buckets()
def _check_connected(module, blade):
connected_blades = blade.array_connections.list_array_connections()
for target in range(0, len(connected_blades.items)):
if connected_blades.items[target].management_address is None:
try:
remote_system = PurityFb(module.params['target_url'])
remote_system.login(module.params['target_api'])
remote_array = remote_system.arrays.list_arrays().items[0].name
if connected_blades.items[target].remote.name == remote_array:
return connected_blades.items[target]
except Exception:
module.fail_json(msg="Failed to connect to remote array {0}.".format(module.params['target_url']))
if connected_blades.items[target].management_address == module.params['target_url'] and \
connected_blades.items[target].status in ["connected", "connecting", "partially_connected"]:
return connected_blades.items[target]
return None
def get_alerts(self):
"""Gets active alerts from FlashBlade."""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
fb = PurityFb(self.endpoint)
fb.disable_verify_ssl()
fb.login(self.apitoken)
fbinfo = fb.alerts.list_alerts(filter='(state=\'closing\' or state=\'open\') and (severity=\'warning\' or severity=\'critical\')')
fb.logout()
return(fbinfo)
def get_status(self):
"""Gets hardware component status from FlashBlade."""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
fb = PurityFb(self.endpoint)
fb.disable_verify_ssl()
fb.login(self.apitoken)
fbinfo = fb.hardware.list_hardware(names=[self.component])
fb.logout()
return(fbinfo)
def get_status(self):
"""Gets hardware component status from FlashBlade."""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
fbinfo={}
try:
fb = PurityFb(self.endpoint)
fb.disable_verify_ssl()
fb.login(self.apitoken)
fbinfo = fb.hardware.list_hardware(names=[self.component])
fb.logout()
except Exception as e:
self.logger.error('FB REST call returned "%s" ', e)
return(fbinfo)
def get_alerts(self):
"""Gets active alerts from FlashBlade."""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
fbinfo = {}
try:
fb = PurityFb(self.endpoint)
fb.disable_verify_ssl()
fb.login(self.apitoken)
fbinfo = fb.alerts.list_alerts(filter="state='open'")
fb.logout()
except Exception as e:
self.logger.error('FB REST call returned "%s" ', e)
return (fbinfo)
def connect_fb(endpoint, apitoken):
"""Establish connection with FlashBlade."""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
fb = PurityFb(endpoint, conn_timeo=2.0, read_timeo=5.0, retries=1)
fb.disable_verify_ssl()
fb.login(apitoken)
return fb
def get_perf(self):
"""Gets performance counters from FlashBlade."""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
fb = PurityFb(self.endpoint)
fb.disable_verify_ssl()
fb.login(self.apitoken)
if (self.proto is None):
fbinfo = fb.arrays.list_arrays_performance()
else:
fbinfo = fb.arrays.list_arrays_performance(protocol=self.proto)
fb.logout()
return (fbinfo)
def connect_fb(endpoint, apitoken, ctimeo=2.0, rtimeo=5.0, retries=3):
"""Establish connection with FlashBlade."""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
fb = PurityFb(endpoint, conn_timeo=ctimeo, read_timeo=rtimeo, retries=retries)
fb.disable_verify_ssl()
fb._api_client.user_agent = 'Purity_FB_Zabbix_plugin/1.0'
fb.login(apitoken)
return fb
def create_session(flashBlade, token):
fb = PurityFb(flashBlade)
# Disable SSL verification
fb.disable_verify_ssl()
fb.login(token) # login to the array with your API_TOKEN
return (fb)
def get_occupancy(self):
"""Gets occupancy values from FlasgBlade ."""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
fb = PurityFb(self.endpoint)
fb.disable_verify_ssl()
fb.login(self.apitoken)
if (self.volname):
fbinfo = fb.file_systems.list_file_systems(names=[self.volname])
elif (self.type == 'filesystem'):
fbinfo = fb.arrays.list_arrays_space(type='file-system')
elif (self.type == 'objectstore'):
fbinfo = fb.arrays.list_arrays_space(type='object-store')
else:
fbinfo = fb.arrays.list_arrays_space()
fb.logout()
return (fbinfo)
def pull_fb_stats(self,
host: str,
object_type: str,
objects: str,
num_days: int = 30,
interval: int = 86400000) -> BucketPerformanceResponse:
# Default num_days is 30, interval time is 1 day in milliseconds
end = int(time.time() * 1000) # Convert seconds to milliseconds
start = int(end - (num_days * interval))
fb = PurityFb(self.flashblades[host]['ip'])
if object_type == 'nfs':
try:
fb.login(self.flashblades[host]['auth_token'])
fb_stats = fb.file_systems.list_file_systems_performance(
start_time=start,
end_time=end,
resolution=interval,
protocol='nfs',
names=objects)
except rest.ApiException as e:
print("Exception: %s\n" % e)
elif object_type == 's3':
try:
fb.login(self.flashblades[host]['auth_token'])
fb_stats = fb.buckets.list_buckets_performance(
start_time=start,
end_time=end,
resolution=interval,
names=objects)
except rest.ApiException as e:
print("Exception: %s\n" % e)
else:
raise Exception(
"Wrong object type specified. Object type should be 'nfs' or 's3'."
)
return fb_stats
def pull_fb_objects(self, host: str, object_type: str) -> list:
fb = PurityFb(self.flashblades[host]['ip'])
if object_type == 'nfs':
try:
fb.login(self.flashblades[host]['auth_token'])
fb_file_systems = fb.file_systems.list_file_systems()
except rest.ApiException as e:
print("Exception: %s\n" % e)
elif object_type == 's3':
try:
fb.login(self.flashblades[host]['auth_token'])
fb_file_systems = fb.buckets.list_buckets()
except rest.ApiException as e:
print("Exception: %s\n" % e)
else:
raise Exception(
"Wrong object type specified. Object type should be 'nfs' or 's3'."
)
names = []
for item in fb_file_systems.items:
names.append(item.name)
return names
def main():
# Setup variables
global DEBUG_FLAG
exit_code = 0
# Check for command line parameters
options = parsecl()
API_TOKEN = options.API_TOKEN
flashBlade = options.flashBlade
fs = options.fs
suffix = options.suffix
DEBUG_FLAG = options.DEBUG_FLAG
VERBOSE_FLAG = options.VERBOSE_FLAG
if DEBUG_FLAG:
print('API Token:', API_TOKEN)
print('FlashBlade:', flashBlade)
print('File System:', fs)
print('Suffix:', suffix)
print('Debug Flag:', DEBUG_FLAG)
print('Verbose Flag:', VERBOSE_FLAG)
if flashBlade == None:
sys.exit('Exiting: You must provide FlashBlade details')
if API_TOKEN == None:
sys.exit('Exiting: You must provide FlashBlade API Token details')
if fs == None:
sys.exit('Exiting: You must provide FlashBlade file system')
print(BANNER)
print(HEADER + ' - ' + flashBlade)
print(strftime('%d/%m/%Y %H:%M:%S %Z', gmtime()))
print(BANNER)
# create PurityFb object for a certain array
fb = PurityFb(flashBlade)
# this is required for versions before Purity//FB 2.1.3 because they only supports self-signed
# certificates. in later versions, this may be unnecessary if you have imported a certificate.
fb.disable_verify_ssl()
try:
res= fb.login(API_TOKEN)
except rest.ApiException as e:
print("Exception when logging in to the array: %s\n" % e)
if res:
try:
if suffix:
# create a snapshot with suffix for flashblade file system
res = fb.file_system_snapshots.create_file_system_snapshots(sources=[fs],
suffix=SnapshotSuffix(suffix))
else:
# create a snapshot for the file system
res = fb.file_system_snapshots.create_file_system_snapshots(sources=[fs])
if VERBOSE_FLAG:
print(res)
print('Snapshot created for', fs, 'suffix', res.items[0].suffix)
except rest.ApiException as e:
print("Exception when creating file system snapshots: %s\n" % e)
fb.logout()
print(BANNER)
print(strftime('%d/%m/%Y %H:%M:%S %Z', gmtime()))
print(BANNER)
sys.exit(exit_code)
class FlashBlade():
"""
Base class for FlashBlade Prometheus array info
"""
def __init__(self, endpoint, api_token):
# self.fb = PurityFb(endpoint, conn_timeo=ctimeo, read_timeo=rtimeo,
# retries=retries)
self.flashblade = PurityFb(host=endpoint)
self.flashblade.disable_verify_ssl()
self.flashblade._api_client.user_agent = 'Purity_FB_Prometheus_exporter/1.0'
self.flashblade.request_timeout = urllib3.Timeout(connect=2.0,
read=60.0)
self.flashblade.login(api_token)
self.filesystems = []
self.buckets = []
self.array_performance = {}
self.array_performance['nfs'] = None
self.array_performance['http'] = None
self.array_performance['s3'] = None
self.array_performance['smb'] = None
self.array_specific_perf = {}
self.array_specific_perf['nfs'] = None
self.array_specific_perf['http'] = None
self.array_specific_perf['s3'] = None
self.array_space = None
self.nfs_filesystems_performance = []
self.buckets_performance = []
self.buckets_replica_links = []
self.filesystems_replica_links = []
self.users_usage = []
self.groups_usage = []
self.clients_performance = []
def __del__(self):
if self.flashblade is not None:
self.flashblade.logout()
def get_array_info(self):
return self.flashblade.arrays.list_arrays().items[0]
def get_open_alerts(self):
return self.flashblade.alerts.list_alerts(filter="state='open'").items
def get_hardware_status(self):
return self.flashblade.hardware.list_hardware().items
def get_array_performance(self, proto):
if self.array_performance[proto] is None:
try:
self.array_performance[
proto] = self.flashblade.arrays.list_arrays_performance(
protocol=proto).items[0]
except Exception:
pass
return self.array_performance[proto]
def get_array_specific_performance(self, proto):
if proto == 'http':
if self.array_specific_perf['http'] is None:
try:
self.array_specific_perf[
'http'] = self.flashblade.arrays.list_arrays_http_specific_performance(
).items[0]
except Exception:
pass
return self.array_specific_perf['http']
if proto == 'nfs':
if self.array_specific_perf['nfs'] is None:
try:
self.array_specific_perf[
'nfs'] = self.flashblade.arrays.list_arrays_nfs_specific_performance(
).items[0]
except Exception:
pass
return self.array_specific_perf['nfs']
if proto == 's3':
if self.array_specific_perf['s3'] is None:
try:
self.array_specific_perf[
's3'] = self.flashblade.arrays.list_arrays_s3_specific_performance(
).items[0]
except Exception:
pass
return self.array_specific_perf['s3']
def get_filesystems(self):
if not self.filesystems:
try:
self.filesystems = self.flashblade.file_systems.list_file_systems(
).items
except Exception:
pass
return self.filesystems
def get_array_space(self):
if self.array_space is None:
try:
self.array_space = self.flashblade.arrays.list_arrays_space(
).items[0]
except Exception:
pass
return self.array_space
def get_buckets(self):
if not self.buckets:
try:
self.buckets = self.flashblade.buckets.list_buckets().items
except Exception:
pass
return self.buckets
def get_nfs_filesystems_performance(self):
if not self.nfs_filesystems_performance:
for f in self.get_filesystems():
try:
self.nfs_filesystems_performance.append(
self.flashblade.file_systems.
list_file_systems_performance(protocol='nfs',
names=[f.name]).items[0])
except Exception:
pass
return self.nfs_filesystems_performance
def get_buckets_performance(self):
if not self.buckets_performance:
for b in self.get_buckets():
try:
self.buckets_performance.append(
self.flashblade.buckets.
list_buckets_s3_specific_performance(
names=[b.name]).items[0])
except Exception:
pass
return self.buckets_performance
def get_bucket_replica_links(self):
if not self.buckets_replica_links:
try:
self.buckets_replica_links = self.flashblade.bucket_replica_links.list_bucket_replica_links(
).items
except Exception:
pass
return self.buckets_replica_links
def get_filesystem_replica_links(self):
if not self.filesystems_replica_links:
try:
self.filesystems_replica_links = self.flashblade.file_system_replica_links.list_file_system_replica_links(
).items
except Exception:
pass
return self.filesystems_replica_links
def get_users_usage(self):
if not self.users_usage:
for f in self.get_filesystems():
try:
uu = self.flashblade.usage_users.list_user_usage(
file_system_names=[f.name]).items
if len(uu) == 0:
continue
self.users_usage = self.users_usage + uu
except Exception:
pass
return self.users_usage
def get_groups_usage(self):
if not self.groups_usage:
for f in self.get_filesystems():
try:
gu = self.flashblade.usage_groups.list_group_usage(
file_system_names=[f.name]).items
if len(gu) == 0:
continue
self.groups_usage = self.groups_usage + gu
except Exception:
pass
return self.groups_usage
def get_clients_performance(self):
if not self.clients_performance:
try:
self.clients_performance = self.flashblade.arrays.list_clients_performance(
).items
except Exception:
pass
return self.clients_performance
# before use.
# Disable warnings related to unsigned SSL certificates on the FlashBlade.
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# Create PurityFb object for a certain array using environment variables.
FB_MGMT = os.environ.get('FB_MGMT_VIP')
TOKEN = os.environ.get('FB_MGMT_TOKEN')
# Constants.
FS_NAME="ior-benchmark"
FS_SIZE=10 * 1024 * 1024 * 1024 * 1024 # 10TB
# Create management object.
fb = PurityFb(FB_MGMT)
fb.disable_verify_ssl()
try:
fb.login(TOKEN)
except rest.ApiException as e:
print("Exception: %s\n" % e)
try:
# First, if the filesystem already exists, delete it.
res = fb.file_systems.list_file_systems(names=[FS_NAME])
if len(res.items) == 1:
print("Found existing filesystem {}, deleting.".format(FS_NAME))
fb.file_systems.update_file_systems(name=FS_NAME,
attributes=FileSystem(nfs=NfsRule(v3_enabled=False)))
class FlashbladeCollector():
"""
Instantiates the collector's methods and properties to retrieve status,
space occupancy and performance metrics from Puretorage FlasBlade.
Provides also a 'collect' method to allow Prometheus client registry
to work properly.
:param target: IP address or domain name of the target array's management
interface.
:type target: str
:param api_token: API token of the user with which to log in.
:type api_token: str
"""
def __init__(self, target, api_token, request='all'):
# self.fb = PurityFb(endpoint, conn_timeo=ctimeo, read_timeo=rtimeo, retries=retries)
self.fb = PurityFb(host=target)
self.fb.disable_verify_ssl()
self.fb._api_client.user_agent = 'Purity_FB_Prometheus_exporter/1.0'
self.fb.request_timeout = urllib3.Timeout(connect=2.0, read=60.0)
self.fb.login(api_token)
self.request = request
self.filesystems = self.fb.file_systems.list_file_systems()
self.buckets = self.fb.buckets.list_buckets()
def __del__(self):
if self.fb is not None:
self.fb.logout()
def array_info(self):
"""Assemble a simple information metric defining the scraped system."""
data = self.fb.arrays.list_arrays().items[0]
yield InfoMetricFamily('purefb',
'FlashBlade system information',
value={
'array_name': data.name,
'system_id': data.id,
'os': data.os,
'version': data.version
})
def array_hw(self):
"""
Create a metric of gauge type for components status,
with the hardware component name as label.
Metrics values can be iterated over.
"""
fb_hw = self.fb.hardware.list_hardware().items
status = GaugeMetricFamily('purefb_hw_status',
'Hardware components status',
labels=['hw_id'])
for h in fb_hw:
state = h.status
name = h.name
labels_v = [name]
if state == 'unused' or state == 'not_installed':
continue
elif state == 'healthy':
status.add_metric(labels_v, 1)
else:
status.add_metric(labels_v, 0)
yield status
def array_events(self):
"""
Create a metric of gauge type for the number of open alerts:
critical, warning and info, with the severity as label.
Metrics values can be iterated over.
"""
fb_events = self.fb.alerts.list_alerts(filter="state='open'").items
labels = ['severity']
events = GaugeMetricFamily('purefb_open_events_total',
'FlashBlade number of open events',
labels=labels)
# Inrement each counter for each type of event
c_crit, c_warn, c_info = 0, 0, 0
for msg in fb_events:
if msg.severity == 'critical':
c_crit += 1
if msg.severity == 'warning':
c_warn += 1
if msg.severity == 'info':
c_info += 1
events.add_metric(['critical'], c_crit)
events.add_metric(['warning'], c_warn)
events.add_metric(['info'], c_info)
yield events
def array_space(self):
"""
Create metrics of gauge type for array space indicators.
Metrics values can be iterated over.
"""
fb_space = self.fb.arrays.list_arrays_space().items[0]
data_reduction = GaugeMetricFamily('purefb_array_space_data_reduction',
'FlashBlade overall data reduction',
labels=[])
space = GaugeMetricFamily('purefb_array_space_bytes',
'FlashBlade total space capacity',
labels=['dimension'])
data_reduction.add_metric([], fb_space.space.data_reduction)
space.add_metric(['capacity'], fb_space.capacity)
space.add_metric(['total_physical'], fb_space.space.total_physical)
space.add_metric(['snapshots'], fb_space.space.snapshots)
yield data_reduction
yield space
def buckets_space(self):
"""
Create metrics of gauge type for buckets space indicators, with the
account name and the bucket name as labels.
Metrics values can be iterated over.
"""
datareduction = GaugeMetricFamily('purefb_buckets_data_reduction',
'FlashBlade buckets data reduction',
labels=['account', 'name'])
objcount = GaugeMetricFamily('purefb_buckets_object_count',
'FlashBlade buckets objects counter',
labels=['account', 'name'])
space = GaugeMetricFamily('purefb_buckets_space_bytes',
'FlashBlade buckets space',
labels=['account', 'name', 'dimension'])
for b in self.buckets.items:
if b.space.data_reduction is None:
b.space.data_reduction = 0
datareduction.add_metric([b.account.name, b.name],
b.space.data_reduction)
objcount.add_metric([b.account.name, b.name], b.object_count)
space.add_metric([b.account.name, b.name, 'snapshots'],
b.space.snapshots)
space.add_metric([b.account.name, b.name, 'total_physical'],
b.space.total_physical)
space.add_metric([b.account.name, b.name, 'virtual'],
b.space.virtual)
space.add_metric([b.account.name, b.name, 'unique'],
b.space.unique)
yield datareduction
yield objcount
yield space
def filesystems_space(self):
"""
Create metrics of gauge type for filesystems space indicators,
with filesystem name as label.
Metrics values can be iterated over.
"""
datareduction = GaugeMetricFamily(
'purefb_filesystems_data_reduction',
'FlashBlade filesystems data reduction',
labels=['name'])
space = GaugeMetricFamily('purefb_filesystems_space_bytes',
'FlashBlade filesystems space',
labels=['name', 'dimension'])
for f in self.filesystems.items:
if f.space.data_reduction is None:
f.space.data_reduction = 0
datareduction.add_metric([f.name], f.space.data_reduction)
space.add_metric([f.name, 'provisioned'], f.provisioned)
space.add_metric([f.name, 'snapshots'], f.space.snapshots)
space.add_metric([f.name, 'total_physical'],
f.space.total_physical)
space.add_metric([f.name, 'virtual'], f.space.virtual)
space.add_metric([f.name, 'unique'], f.space.unique)
yield datareduction
yield space
def array_perf(self):
"""
Create array performance metrics of gauge type.
Metrics values can be iterated over.
"""
protocols = ['http', 'nfs', 's3', 'smb']
bpops = GaugeMetricFamily(
'purefb_array_performance_opns_bytes',
'FlashBlade array average bytes per operations',
labels=['protocol', 'dimension'])
latency = GaugeMetricFamily('purefb_array_performance_latency_usec',
'FlashBlade array latency',
labels=['protocol', 'dimension'])
iops = GaugeMetricFamily('purefb_array_performance_iops',
'FlashBlade array IOPS',
labels=['protocol', 'dimension'])
throughput = GaugeMetricFamily(
'purefb_array_performance_throughput_bytes',
'FlashBlade array throughput',
labels=['protocol', 'dimension'])
for proto in protocols:
fb_perf = self.fb.arrays.list_arrays_performance(
protocol=proto).items[0]
bpops.add_metric([proto, 'per_op'], fb_perf.bytes_per_op)
bpops.add_metric([proto, 'read'], fb_perf.bytes_per_read)
bpops.add_metric([proto, 'write'], fb_perf.bytes_per_write)
latency.add_metric([proto, 'read'], fb_perf.usec_per_read_op)
latency.add_metric([proto, 'write'], fb_perf.usec_per_write_op)
latency.add_metric([proto, 'other'], fb_perf.usec_per_other_op)
iops.add_metric([proto, 'read'], fb_perf.reads_per_sec)
iops.add_metric([proto, 'write'], fb_perf.writes_per_sec)
iops.add_metric([proto, 'other'], fb_perf.others_per_sec)
#iops.add_metric([proto, 'in'], fb_perf.input_per_sec)
#iops.add_metric([proto, 'out'], fb_perf.output_per_sec)
throughput.add_metric([proto, 'read'], fb_perf.read_bytes_per_sec)
throughput.add_metric([proto, 'write'],
fb_perf.write_bytes_per_sec)
yield bpops
yield latency
yield iops
yield throughput
def filesystems_perf(self):
"""
Create metrics of gauge type for filesystems performance indicators,
with filesystem name as label.
Metrics values can be iterated over.
"""
bpops = GaugeMetricFamily(
'purefb_filesystem_performance_opns_bytes',
'FlashBlade filesystem average bytes per operations',
labels=['protocol', 'name', 'dimension'])
latency = GaugeMetricFamily(
'purefb_filesystem_performance_latency_usec',
'FlashBlade filesystem latency',
labels=['protocol', 'name', 'dimension'])
iops = GaugeMetricFamily('purefb_filesystem_performance_iops',
'FlashBlade filesystem IOPS',
labels=['protocol', 'name', 'dimension'])
throughput = GaugeMetricFamily(
'purefb_filesystem_performance_throughput_bytes',
'FlashBlade filesystem throughput',
labels=['protocol', 'name', 'dimension'])
for f in self.filesystems.items:
if not f.nfs.enabled:
continue
fb_fs_perf = None
try:
fb_fs_perf = self.fb.file_systems.list_file_systems_performance(
protocol='nfs', names=[f.name]).items[0]
except Exception as e:
continue
bpops.add_metric(['nfs', f.name, 'per_op'],
fb_fs_perf.bytes_per_op)
bpops.add_metric(['nfs', f.name, 'read'],
fb_fs_perf.bytes_per_read)
bpops.add_metric(['nfs', f.name, 'write'],
fb_fs_perf.bytes_per_write)
latency.add_metric(['nfs', f.name, 'read'],
fb_fs_perf.usec_per_read_op)
latency.add_metric(['nfs', f.name, 'write'],
fb_fs_perf.usec_per_write_op)
latency.add_metric(['nfs', f.name, 'other'],
fb_fs_perf.usec_per_other_op)
iops.add_metric(['nfs', f.name, 'read'], fb_fs_perf.reads_per_sec)
iops.add_metric(['nfs', f.name, 'write'],
fb_fs_perf.writes_per_sec)
iops.add_metric(['nfs', f.name, 'other'],
fb_fs_perf.others_per_sec)
throughput.add_metric(['nfs', f.name, 'read'],
fb_fs_perf.read_bytes_per_sec)
throughput.add_metric(['nfs', f.name, 'write'],
fb_fs_perf.write_bytes_per_sec)
yield bpops
yield latency
yield iops
yield throughput
def buckets_perf(self):
"""
Create metrics of gauge type for buckets performace indicators, with the
account name and the bucket name as labels.
Metrics values can be iterated over.
"""
latency = GaugeMetricFamily('purefb_bucket_performance_latency_usec',
'FlashBlade bucket latency',
labels=['name', 'dimension'])
throughput = GaugeMetricFamily(
'purefb_bucket_performance_throughput_bytes',
'FlashBlade bucket throughput',
labels=['name', 'dimension'])
for b in self.buckets.items:
try:
bperf = self.fb.buckets.list_buckets_s3_specific_performance(
names=[b.name]).items[0]
except Exception as e:
continue
#bperf = self.fb.buckets.list_buckets_performance(names=[b.name])
latency.add_metric([b.name, 'read_buckets'],
bperf.usec_per_read_bucket_op)
latency.add_metric([b.name, 'read_objects'],
bperf.usec_per_read_object_op)
latency.add_metric([b.name, 'write_buckets'],
bperf.usec_per_write_bucket_op)
latency.add_metric([b.name, 'write_objects'],
bperf.usec_per_write_object_op)
latency.add_metric([b.name, 'other'], bperf.usec_per_other_op)
throughput.add_metric([b.name, 'read_buckets'],
bperf.read_buckets_per_sec)
throughput.add_metric([b.name, 'read_objects'],
bperf.read_objects_per_sec)
throughput.add_metric([b.name, 'write_buckets'],
bperf.write_buckets_per_sec)
throughput.add_metric([b.name, 'write_objects'],
bperf.write_objects_per_sec)
throughput.add_metric([b.name, 'other'], bperf.others_per_sec)
yield latency
yield throughput
def clientperf(self):
"""
Create metrics of gauge type for client performance metrics.
Metrics values can be iterated over.
"""
fb_clientperf = self.fb.arrays.list_clients_performance()
bpops = GaugeMetricFamily(
'purefb_client_performance_opns_bytes',
'FlashBlade client average bytes per operations',
labels=['name', 'port', 'dimension'])
latency = GaugeMetricFamily('purefb_client_performance_latency_usec',
'FlashBlade latency',
labels=['name', 'port', 'dimension'])
iops = GaugeMetricFamily('purefb_client_performance_iops',
'FlashBlade IOPS',
labels=['name', 'port', 'dimension'])
throughput = GaugeMetricFamily(
'purefb_client_performance_throughput_bytes',
'FlashBlade client_throughput',
labels=['name', 'port', 'dimension'])
for cperf in fb_clientperf.items:
client, port = cperf.name.split(':')
bpops.add_metric([client, port, 'per_op'], cperf.bytes_per_op)
bpops.add_metric([client, port, 'read'], cperf.bytes_per_read)
bpops.add_metric([client, port, 'write'], cperf.bytes_per_write)
iops.add_metric([client, port, 'read'], cperf.reads_per_sec)
iops.add_metric([client, port, 'write'], cperf.writes_per_sec)
iops.add_metric([client, port, 'other'], cperf.others_per_sec)
latency.add_metric([client, port, 'read'], cperf.usec_per_read_op)
latency.add_metric([client, port, 'write'],
cperf.usec_per_write_op)
latency.add_metric([client, port, 'other'],
cperf.usec_per_other_op)
throughput.add_metric([client, port, 'read'],
cperf.read_bytes_per_sec)
throughput.add_metric([client, port, 'write'],
cperf.write_bytes_per_sec)
yield bpops
yield latency
yield iops
yield throughput
def collect(self):
"""Global collector method for all the collected array metrics."""
if (self.request == 'all' or self.request == 'array'):
yield from self.array_info()
yield from self.array_hw()
yield from self.array_events()
yield from self.array_perf()
yield from self.array_space()
yield from self.filesystems_space()
yield from self.buckets_space()
yield from self.filesystems_perf()
yield from self.buckets_perf()
if (self.request == 'all' or self.request == 'clients'):
yield from self.clientperf()
class FlashbladeCollector:
""" Instantiates the collector's methods and properties to retrieve metrics
from Puretorage FlasBlade.
Provides also a 'collect' method to allow Prometheus client registry
to work
:param target: IP address or domain name of the target array's management
interface.
:type target: str
:param api_token: API token of the user with which to log in.
:type api_token: str
"""
def __init__(self, target, api_token):
# self.fb = PurityFb(endpoint, conn_timeo=ctimeo, read_timeo=rtimeo, retries=retries)
self.fb = PurityFb(host=target, api_token=api_token)
self.fb.disable_verify_ssl()
def array_hw(self):
""" Create a metric of gauge type for components status,
with the hardware component name as label.
Metrics values can be iterated over.
"""
fb_hw = self.fb.hardware.list_hardware().items
labels = ['hw_id']
status = GaugeMetricFamily('pure_fb_hw_status',
'Hardware components status',
labels=labels)
for h in fb_hw:
state = h.status
name = h.name
labels_v = [name]
if state == 'unused' or state == 'not_installed':
status.add_metric(labels_v, -1)
elif state == 'healty':
status.add_metric(labels_v, 1)
else:
status.add_metric(labels_v, 0)
yield status
def array_events(self):
""" Create a metric of gauge type for the number of open alerts:
critical, warning and info, with the severity as label.
Metrics values can be iterated over.
"""
fb_events = self.fb.alerts.list_alerts(filter="state='open'").items
labels = ['severity']
events = GaugeMetricFamily('pure_fb_open_events_total',
'FlashBlade number of open events',
labels=labels)
ccounter = 0
wcounter = 0
icounter = 0
for msg in fb_events:
severity = msg.severity
if severity == 'critical':
ccounter += 1
if severity == 'warning':
wcounter += 1
if severity == 'info':
icounter += 1
events.add_metric(['critical'], ccounter)
events.add_metric(['warning'], wcounter)
events.add_metric(['info'], icounter)
yield events
def array_space(self):
""" Create metrics of gauge type for array space indicators.
Metrics values can be iterated over.
"""
fb_space = self.fb.arrays.list_arrays_space().items[0]
capacity_tot = GaugeMetricFamily('pure_fb_space_tot_capacity_bytes',
'FlashBlade total space capacity',
labels=[])
data_reduction = GaugeMetricFamily('pure_fb_space_data_reduction',
'FlashBlade overall data reduction',
labels=[])
tot_physical = GaugeMetricFamily('pure_fb_space_tot_physical_bytes',
'FlashBlade overall occupied space',
labels=[])
tot_snapshots = GaugeMetricFamily(
'pure_fb_space_tot_snapshot_bytes',
'FlashBlade occupied space for snapshots',
labels=[])
capacity_tot.add_metric([], fb_space.capacity)
data_reduction.add_metric([], fb_space.space.data_reduction)
tot_physical.add_metric([], fb_space.space.total_physical)
tot_snapshots.add_metric([], fb_space.space.snapshots)
yield capacity_tot
yield data_reduction
yield tot_physical
yield tot_snapshots
def buckets_space(self):
""" Create metrics of gauge type for buckets space indicators, with the
account name and the bucket name as labels.
Metrics values can be iterated over.
"""
fb_buckets = self.fb.buckets.list_buckets()
labels = ['account', 'name']
data_reduct = GaugeMetricFamily('pure_fb_buckets_data_reduction',
'FlashBlade buckets data reduction',
labels=labels)
obj_cnt = GaugeMetricFamily('pure_fb_buckets_object_count',
'FlashBlade buckets objects counter',
labels=labels)
space_snap = GaugeMetricFamily(
'pure_fb_buckets_snapshots_bytes',
'FlashBlade buckets occupied snapshots space',
labels=labels)
tot_phy = GaugeMetricFamily('pure_fb_buckets_total_bytes',
'FlashBlade buckets total physical space',
labels=labels)
virt_space = GaugeMetricFamily('pure_fb_buckets_virtual_bytes',
'FlashBlade buckets virtual space',
labels=labels)
uniq_space = GaugeMetricFamily('pure_fb_buckets_unique_bytes',
'FlashBlade buckets unique space',
labels=labels)
for b in fb_buckets.items:
if b.space.data_reduction is None:
b.space.data_reduction = 0
data_reduct.add_metric([b.account.name, b.name],
b.space.data_reduction)
obj_cnt.add_metric([b.account.name, b.name], b.object_count)
space_snap.add_metric([b.account.name, b.name], b.space.snapshots)
tot_phy.add_metric([b.account.name, b.name],
b.space.total_physical)
virt_space.add_metric([b.account.name, b.name], b.space.virtual)
uniq_space.add_metric([b.account.name, b.name], b.space.unique)
yield data_reduct
yield obj_cnt
yield space_snap
yield tot_phy
yield virt_space
yield uniq_space
def filesystems_space(self):
""" Create metrics of gauge type for filesystems space indicators,
with filesystem name as label.
Metrics values can be iterated over.
"""
fb_filesystems = self.fb.file_systems.list_file_systems()
labels = ['name']
data_reduct = GaugeMetricFamily(
'pure_fb_filesystems_data_reduction',
'FlashBlade filesystems data reduction',
labels=labels)
space_snap = GaugeMetricFamily(
'pure_fb_filesystems_snapshots_bytes',
'FlashBlade filesystems occupied snapshots space',
labels=labels)
tot_phy = GaugeMetricFamily(
'pure_fb_filesystems_total_bytes',
'FlashBlade filesystems total physical space',
labels=labels)
virt_space = GaugeMetricFamily('pure_fb_filesystems_virtual_bytes',
'FlashBlade filesystems virtual space',
labels=labels)
uniq_space = GaugeMetricFamily('pure_fb_filesystems_unique_bytes',
'FlashBlade filesystems unique space',
labels=labels)
for f in fb_filesystems.items:
if f.space.data_reduction is None:
f.space.data_reduction = 0
data_reduct.add_metric([f.name], f.space.data_reduction)
space_snap.add_metric([f.name], f.space.snapshots)
tot_phy.add_metric([f.name], f.space.total_physical)
virt_space.add_metric([f.name], f.space.virtual)
uniq_space.add_metric([f.name], f.space.unique)
yield data_reduct
yield space_snap
yield tot_phy
yield virt_space
yield uniq_space
def collect(self):
"""Global collector method for all the collected metrics."""
yield from self.array_hw()
yield from self.array_events()
yield from self.array_space()
yield from self.buckets_space()
yield from self.filesystems_space()
def get_blade(module):
"""Return System Object or Fail"""
user_agent = "%(base)s %(class)s/%(version)s (%(platform)s)" % {
"base": USER_AGENT_BASE,
"class": __name__,
"version": VERSION,
"platform": platform.platform(),
}
blade_name = module.params["fb_url"]
api = module.params["api_token"]
if HAS_PURITY_FB:
if blade_name and api:
blade = PurityFb(blade_name)
blade.disable_verify_ssl()
try:
blade.login(api)
versions = blade.api_version.list_versions().versions
if API_AGENT_VERSION in versions:
blade._api_client.user_agent = user_agent
except Exception:
module.fail_json(
msg="Pure Storage FlashBlade authentication failed. Check your credentials"
)
elif environ.get("PUREFB_URL") and environ.get("PUREFB_API"):
blade = PurityFb(environ.get("PUREFB_URL"))
blade.disable_verify_ssl()
try:
blade.login(environ.get("PUREFB_API"))
versions = blade.api_version.list_versions().versions
if API_AGENT_VERSION in versions:
blade._api_client.user_agent = user_agent
except Exception:
module.fail_json(
msg="Pure Storage FlashBlade authentication failed. Check your credentials"
)
else:
module.fail_json(
msg="You must set PUREFB_URL and PUREFB_API environment variables "
"or the fb_url and api_token module arguments"
)
else:
module.fail_json(msg="purity_fb SDK not installed.")
return blade
def __init__(self, target, api_token):
# self.fb = PurityFb(endpoint, conn_timeo=ctimeo, read_timeo=rtimeo, retries=retries)
self.fb = PurityFb(host=target, api_token=api_token)
self.fb.disable_verify_ssl()
def format_bytes(size):
# 2**10 = 1024
power = 2**10
n = 0
power_labels = {0: '', 1: ' KB', 2: ' MB', 3: ' GB', 4: ' TB'}
while size > power:
size /= power
n += 1
size = round(size, 0)
size_str = str(size) + power_labels[n]
return size_str
fb = PurityFb(FB_IP)
try:
fs = str(sys.argv[1])
# Set API token
fb.login(API_TOKEN)
# Get default user quota information and store in "res"
res = fb.quotas_users.list_user_quotas(file_system_names=[fs])
# Get user quota information for this filesystem
res = fb.usage_users.list_user_usage(file_system_names=[fs])
fs_def_user_quota = str(
format_bytes(res.items[0].file_system_default_quota))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--account",
default='datateam',
help="Service account name.")
parser.add_argument("--user", default='spark', help="Account user name.")
parser.add_argument("--outfile",
default='credentials',
help="Output file for key credentials.")
args = parser.parse_args()
# Create PurityFb object for a certain array using environment variables.
FB_MGMT = os.environ.get('FB_MGMT_VIP')
TOKEN = os.environ.get('FB_MGMT_TOKEN')
# Fail fast if necessary env variables not available.
if not FB_MGMT or not TOKEN:
print(
"Requires environment variables for logging into FlashBlade REST: please set FB_MGMT_VIP and FB_MGMT_TOKEN"
)
exit(1)
# Step 1: login to the FlashBlade management API
fb = PurityFb(FB_MGMT)
fb.disable_verify_ssl()
try:
fb.login(TOKEN)
except rest.ApiException as e:
print("Exception: %s\n" % e)
exit()
# Step 2: Create service account
try:
res = fb.object_store_accounts.create_object_store_accounts(
names=[args.account])
print("Creating service account {}".format(args.account))
except:
print("Service account {} already exists".format(args.account))
# Step 3: Create user account
accountuser = args.account + '/' + args.user
try:
# post the object store user object myobjuser on the array
print("Creating user account {}".format(accountuser))
res = fb.object_store_users.create_object_store_users(
names=[accountuser])
except:
print("User %s creation failed.".format(accountuser))
# Step 4: Create access keys
res = fb.object_store_access_keys.list_object_store_access_keys(
filter="user.name=\'{}\'".format(accountuser))
if len(res.items) == 2:
print("User {} cannot create more access keys.".format(accountuser))
exit(1)
# generate access key and secret key for object store user
# note: you need to handle the secret key since you can't retrieve it from the array after create
accesskey = ""
secretkey = ""
try:
res = fb.object_store_access_keys.create_object_store_access_keys(
object_store_access_key={'user': {
'name': accountuser
}})
accesskey = res.items[0].name
secretkey = res.items[0].secret_access_key
except rest.ApiException as e:
print("Exception when creating object store access key: %s\n" % e)
exit(1)
# Step 5: Create bucket
bucketname = args.user + "-working"
print("Creating bucket %s\n" % bucketname)
try:
attr = Bucket()
# Each bucket must be associated with a service account.
attr.account = Reference(name=args.account)
res = fb.buckets.create_buckets(names=[bucketname], account=attr)
except rest.ApiException as e:
print("Exception when creating bucket: %s\n" % e)
# Output
with open(args.outfile, "w") as outf:
outf.write("AWS_ACCESS_KEY_ID={}\n".format(accesskey))
outf.write("AWS_SECRET_ACCESS_KEY_ID={}\n".format(secretkey))
print(
"Access newly created bucket in Spark at s3a://{}/".format(bucketname))
fb.logout()
def get_blade(module):
"""Return System Object or Fail"""
# Note: user_agent not included in FlashBlade API 1.1
# user_agent = '%(base)s %(class)s/%(version)s (%(platform)s)' % {
# 'base': USER_AGENT_BASE,
# 'class': __name__,
# 'version': VERSION,
# 'platform': platform.platform()
# }
blade_name = module.params['fb_url']
api = module.params['api_token']
if blade_name and api:
blade = PurityFb(blade_name)
blade.disable_verify_ssl()
try:
blade.login(api)
except rest.ApiException as e:
module.fail_json(msg="Pure Storage FlashBlade authentication failed. Check your credentials")
elif environ.get('PUREFB_URL') and environ.get('PUREFB_API'):
blade = PurityFb(environ.get('PUREFB_URL'))
blade.disable_verify_ssl()
try:
blade.login(environ.get('PUREFB_API'))
except rest.ApiException as e:
module.fail_json(msg="Pure Storage FlashBlade authentication failed. Check your credentials")
else:
module.fail_json(msg="You must set PUREFB_URL and PUREFB_API environment variables or the fb_url and api_token module arguments")
return blade
def _get_blade():
'''
Get Pure Storage FlasBlade configuration
1) From the minion config
pure_tags:
fb:
san_ip: management vip or hostname for the FlashBlade
api_token: A valid api token for the FlashBlade being managed
2) From environment (PUREFB_IP and PUREFB_API)
3) From the pillar (PUREFB_IP and PUREFB_API)
'''
try:
blade_name = __opts__['pure_tags']['fb'].get('san_ip')
api_token = __opts__['pure_tags']['fb'].get('api_token')
if blade_name and api:
blade = PurityFb(blade_name)
blade.disable_verify_ssl()
except (KeyError, NameError, TypeError):
try:
blade_name = os.environ.get('PUREFB_IP')
api_token = os.environ.get('PUREFB_API')
if blade_name:
blade = PurityFb(blade_name)
blade.disable_verify_ssl()
except (ValueError, KeyError, NameError):
try:
api_token = __pillar__['PUREFB_API']
blade = PurityFb(__pillar__['PUREFB_IP'])
blade.disable_verify_ssl()
except (KeyError, NameError):
raise CommandExecutionError(
'No Pure Storage FlashBlade credentials found.')
try:
blade.login(api_token)
except Exception:
raise CommandExecutionError(
'Pure Storage FlashBlade authentication failed.')
return blade
def _get_blade():
"""
Get Pure Storage FlasBlade configuration
1) From the minion config
pure_tags:
fb:
san_ip: management vip or hostname for the FlashBlade
api_token: A valid api token for the FlashBlade being managed
2) From environment (PUREFB_IP and PUREFB_API)
3) From the pillar (PUREFB_IP and PUREFB_API)
"""
try:
blade_name = __opts__["pure_tags"]["fb"].get("san_ip")
api_token = __opts__["pure_tags"]["fb"].get("api_token")
if blade_name and api:
blade = PurityFb(blade_name)
blade.disable_verify_ssl()
except (KeyError, NameError, TypeError):
try:
blade_name = os.environ.get("PUREFB_IP")
api_token = os.environ.get("PUREFB_API")
if blade_name:
blade = PurityFb(blade_name)
blade.disable_verify_ssl()
except (ValueError, KeyError, NameError):
try:
api_token = __pillar__["PUREFB_API"]
blade = PurityFb(__pillar__["PUREFB_IP"])
blade.disable_verify_ssl()
except (KeyError, NameError):
raise CommandExecutionError(
"No Pure Storage FlashBlade credentials found.")
try:
blade.login(api_token)
except Exception: # pylint: disable=broad-except
raise CommandExecutionError(
"Pure Storage FlashBlade authentication failed.")
return blade