def _process_group_metadata(response, group, elapsed_time=None, end_time=None):
logger.debug('Cluster updating: group {0} meta key read time: {1}.{2}'.format(
group.group_id, elapsed_time.tsec, elapsed_time.tnsec))
meta = response.data
group.parse_meta(meta)
couple = group.meta.get('couple')
if couple is None:
logger.error('Read symmetric groups from group {} (no couple data): {}'.format(
group.group_id, group.meta))
return
logger.info('Read symmetric groups from group {}: {}'.format(group.group_id, couple))
for gid in couple:
if gid != group.group_id:
logger.info('Scheduling update for group {}'.format(gid))
_queue.add(gid)
couple_str = ':'.join((str(gid) for gid in sorted(couple)))
logger.debug('{0} in storage.couples: {1}'.format(
couple_str, couple_str in storage.couples))
if couple_str not in storage.couples and couple_str not in storage.cache_couples:
ns_id = group.meta.get('namespace')
if ns_id is None:
logger.error('Inconsistent meta read from group {}, '
'missing namespace: {}'.format(group, group.meta))
return
if group.type == storage.Group.TYPE_DATA:
logger.info('Creating couple {0}'.format(couple_str))
for gid in couple:
if gid not in storage.groups:
logger.info('Group {} is not found adding fake group for '
'couple {}'.format(gid, couple))
storage.groups.add(gid)
c = storage.couples.add(storage.groups[gid] for gid in couple)
logger.info('Created couple {0} {1}'.format(c, repr(c)))
elif group.type == storage.Group.TYPE_CACHE:
logger.info('Creating cache couple {0}'.format(couple_str))
c = storage.cache_couples.add(storage.groups[gid] for gid in couple)
logger.info('Created cache couple {0} {1}'.format(c, repr(c)))
else:
raise ValueError('Unknown group type for group {}: {}'.format(
group, group.type))
for gid in couple:
infrastructure.update_group_history(storage.groups[gid])
if ns_id not in storage.namespaces:
logger.info('Creating storage namespace {}'.format(ns_id))
ns = storage.namespaces.add(ns_id)
else:
ns = storage.namespaces[ns_id]
ns.add_couple(c)
return
def _create_groupset_if_needed(groups, group_type, ns_id):
for gid in groups:
if gid not in storage.groups:
logger.info(
'Group {group} is not found, adding fake group '
'for groupset {groups}'.format(
group=gid,
groups=groups,
))
storage.groups.add(gid)
groupset_str = ':'.join((str(gid) for gid in sorted(groups)))
if groupset_str not in storage.groupsets:
# TODO: somehow check that couple type matches group.type
# for all groups in couple (not very easy when metakey read
# fails)
logger.info(
'Creating groupset {groups}, group type "{group_type}"'.
format(
groups=groupset_str,
group_type=group_type,
))
c = storage.groupsets.add(
groups=(storage.groups[gid] for gid in groups),
group_type=group_type,
)
for gid in groups:
infrastructure.update_group_history(storage.groups[gid])
if ns_id not in storage.namespaces:
logger.info('Creating storage namespace {}'.format(ns_id))
ns = storage.namespaces.add(ns_id)
else:
ns = storage.namespaces[ns_id]
ns.add_couple(c)
return storage.groupsets[groupset_str]
def _create_groupset_if_needed(groups, group_type, ns_id):
for gid in groups:
if gid not in storage.groups:
logger.info(
'Group {group} is not found, adding fake group '
'for groupset {groups}'.format(
group=gid,
groups=groups,
)
)
storage.groups.add(gid)
groupset_str = ':'.join((str(gid) for gid in sorted(groups)))
if groupset_str not in storage.groupsets:
# TODO: somehow check that couple type matches group.type
# for all groups in couple (not very easy when metakey read
# fails)
logger.info('Creating groupset {groups}, group type "{group_type}"'.format(
groups=groupset_str,
group_type=group_type,
))
c = storage.groupsets.add(
groups=(storage.groups[gid] for gid in groups),
group_type=group_type,
)
for gid in groups:
infrastructure.update_group_history(storage.groups[gid])
if ns_id not in storage.namespaces:
logger.info('Creating storage namespace {}'.format(ns_id))
ns = storage.namespaces.add(ns_id)
else:
ns = storage.namespaces[ns_id]
ns.add_couple(c)
return storage.groupsets[groupset_str]
def _process_backend_statistics(node, b_stat, backend_stats, collect_ts,
processed_fss, processed_node_backends):
backend_id = b_stat['backend_id']
nb_config = (b_stat['config']
if 'config' in b_stat else b_stat['backend']['config'])
gid = nb_config['group']
if gid == 0:
# skip zero group ids
return
b_stat['stats'] = backend_stats.get(backend_id, {})
update_group_history = False
node_backend_addr = '{0}/{1}'.format(node, backend_id)
if node_backend_addr not in storage.node_backends:
node_backend = storage.node_backends.add(node, backend_id)
update_group_history = True
else:
node_backend = storage.node_backends[node_backend_addr]
if b_stat['status']['state'] != 1:
logger.info('Node backend {0} is not enabled: state {1}'.format(
str(node_backend), b_stat['status']['state']))
node_backend.disable()
return
node_backend.enable()
if gid not in storage.groups:
logger.debug('Adding group {0}'.format(gid))
group = storage.groups.add(gid)
else:
group = storage.groups[gid]
fsid = b_stat['backend']['vfs']['fsid']
fsid_key = '{host}:{fsid}'.format(host=node.host, fsid=fsid)
if fsid_key not in storage.fs:
logger.debug('Adding fs {0}'.format(fsid_key))
fs = storage.fs.add(node.host, fsid)
else:
fs = storage.fs[fsid_key]
if node_backend not in fs.node_backends:
fs.add_node_backend(node_backend)
if fs not in processed_fss:
fs.update_statistics(b_stat['backend'], collect_ts)
processed_fss.add(fs)
logger.info(
'Updating statistics for node backend {}'.format(node_backend))
prev_base_path = node_backend.base_path
try:
node_backend.update_statistics(b_stat, collect_ts)
except KeyError as e:
logger.warn('Bad stat for node backend {0} ({1}): {2}'.format(
node_backend, e, b_stat))
pass
if node_backend.base_path != prev_base_path:
update_group_history = True
if b_stat['status']['read_only'] or node_backend.stat_commit_errors > 0:
node_backend.make_read_only()
else:
node_backend.make_writable()
if node_backend.group is not group:
logger.debug('Adding node backend {0} to group {1}{2}'.format(
node_backend, group.group_id,
' (moved from group {0})'.format(node_backend.group.group_id)
if node_backend.group else ''))
group.add_node_backend(node_backend)
update_group_history = True
# these backends' commands stat are used later to update accumulated
# node commands stat
processed_node_backends.append(node_backend)
if update_group_history:
logger.debug(
'Group {} history may be outdated, adding to update queue'.
format(group))
infrastructure.update_group_history(group)
def _process_group_metadata(response,
group,
elapsed_time=None,
end_time=None):
logger.debug(
'Cluster updating: group {0} meta key read time: {1}.{2}'.
format(group.group_id, elapsed_time.tsec, elapsed_time.tnsec))
meta = response.data
group.parse_meta(meta)
couple = group.meta.get('couple')
if couple is None:
logger.error(
'Read symmetric groups from group {} (no couple data): {}'.
format(group.group_id, meta))
return
logger.info('Read symmetric groups from group {}: {}'.format(
group.group_id, couple))
for gid in couple:
if gid != group.group_id:
logger.info('Scheduling update for group {}'.format(gid))
_queue.add(gid)
couple_str = ':'.join((str(gid) for gid in sorted(couple)))
logger.debug('{0} in storage.couples: {1}'.format(
couple_str, couple_str in storage.couples))
if couple_str not in storage.couples and couple_str not in storage.cache_couples:
ns_id = group.meta.get('namespace')
if ns_id is None:
logger.error('Inconsistent meta read from group {}, '
'missing namespace: {}'.format(
group, group.meta))
return
if group.type == storage.Group.TYPE_DATA:
logger.info('Creating couple {0}'.format(couple_str))
for gid in couple:
if gid not in storage.groups:
logger.info(
'Group {} is not found adding fake group for '
'couple {}'.format(gid, couple))
storage.groups.add(gid)
c = storage.couples.add(
[storage.groups[gid] for gid in couple])
logger.info('Created couple {0} {1}'.format(c, repr(c)))
elif group.type == storage.Group.TYPE_CACHE:
logger.info('Creating cache couple {0}'.format(couple_str))
c = storage.cache_couples.add(
[storage.groups[gid] for gid in couple])
logger.info('Created cache couple {0} {1}'.format(
c, repr(c)))
else:
raise ValueError(
'Unknown group type for group {}: {}'.format(
group, group.type))
for gid in couple:
infrastructure.update_group_history(storage.groups[gid])
if ns_id not in storage.namespaces:
logger.info('Creating storage namespace {}'.format(ns_id))
ns = storage.namespaces.add(ns_id)
else:
ns = storage.namespaces[ns_id]
ns.add_couple(c)
return
def update_statistics(node, stat, elapsed_time=None):
logger.debug(
'Cluster updating: node {0} statistics time: {1:03f}'.format(
node, elapsed_time))
collect_ts = mh.elliptics_time_to_ts(stat['timestamp'])
try:
try:
node.update_statistics(stat, collect_ts)
except KeyError as e:
logger.warn('Bad procfs stat for node {0} ({1}): {2}'.format(
node, e, stat))
pass
fss = set()
good_node_backends = []
backend_stats = NodeInfoUpdater._parsed_stats(stat['stats'])
for b_stat in stat['backends'].itervalues():
backend_id = b_stat['backend_id']
b_stat['stats'] = backend_stats.get(backend_id, {})
update_group_history = False
node_backend_addr = '{0}/{1}'.format(node, backend_id)
if node_backend_addr not in storage.node_backends:
node_backend = storage.node_backends.add(node, backend_id)
update_group_history = True
else:
node_backend = storage.node_backends[node_backend_addr]
nb_config = (b_stat['config'] if 'config' in b_stat else
b_stat['backend']['config'])
gid = nb_config['group']
if gid == 0:
# skip zero group ids
continue
if b_stat['status']['state'] != 1:
logger.info(
'Node backend {0} is not enabled: state {1}'.format(
str(node_backend), b_stat['status']['state']))
node_backend.disable()
continue
if gid not in storage.groups:
logger.debug('Adding group {0}'.format(gid))
group = storage.groups.add(gid)
else:
group = storage.groups[gid]
if 'vfs' not in b_stat['backend']:
logger.error(
'Failed to parse statistics for node backend {0}, '
'vfs key not found: {1}'.format(node_backend, b_stat))
continue
fsid = b_stat['backend']['vfs']['fsid']
fsid_key = '{host}:{fsid}'.format(host=node.host, fsid=fsid)
if fsid_key not in storage.fs:
logger.debug('Adding fs {0}'.format(fsid_key))
fs = storage.fs.add(node.host, fsid)
else:
fs = storage.fs[fsid_key]
if node_backend not in fs.node_backends:
fs.add_node_backend(node_backend)
fs.update_statistics(b_stat['backend'], collect_ts)
fss.add(fs)
good_node_backends.append(node_backend)
node_backend.enable()
logger.info('Updating statistics for node backend %s' %
(str(node_backend)))
if 'backend' not in b_stat:
logger.warn('No backend in b_stat: {0}'.format(b_stat))
elif 'dstat' not in b_stat['backend']:
logger.warn('No dstat in backend: {0}'.format(
b_stat['backend']))
prev_base_path = node_backend.base_path
try:
node_backend.update_statistics(b_stat, collect_ts)
except KeyError as e:
logger.warn(
'Bad stat for node backend {0} ({1}): {2}'.format(
node_backend, e, b_stat))
pass
if node_backend.base_path != prev_base_path:
update_group_history = True
if b_stat['status'][
'read_only'] or node_backend.stat_commit_errors > 0:
node_backend.make_read_only()
else:
node_backend.make_writable()
if node_backend.group is not group:
logger.debug(
'Adding node backend {0} to group {1}{2}'.format(
node_backend, group.group_id,
' (moved from group {0})'.format(
node_backend.group.group_id)
if node_backend.group else ''))
update_group_history = True
group.add_node_backend(node_backend)
if update_group_history:
infrastructure.update_group_history(group)
for fs in fss:
fs.update_commands_stats()
node.update_commands_stats(good_node_backends)
except Exception as e:
logger.exception(
'Unable to process statistics for node {}'.format(node))
def _process_backend_statistics(node,
b_stat,
backend_stats,
collect_ts,
processed_fss,
processed_node_backends):
backend_id = b_stat['backend_id']
nb_config = (b_stat['config']
if 'config' in b_stat else
b_stat['backend']['config'])
gid = nb_config['group']
if gid == 0:
# skip zero group ids
return
b_stat['stats'] = backend_stats.get(backend_id, {})
update_group_history = False
node_backend_addr = '{0}/{1}'.format(node, backend_id)
if node_backend_addr not in storage.node_backends:
node_backend = storage.node_backends.add(node, backend_id)
update_group_history = True
else:
node_backend = storage.node_backends[node_backend_addr]
if b_stat['status']['state'] != 1:
logger.info('Node backend {0} is not enabled: state {1}'.format(
str(node_backend), b_stat['status']['state']))
node_backend.disable()
return
node_backend.enable()
if gid not in storage.groups:
logger.debug('Adding group {0}'.format(gid))
group = storage.groups.add(gid)
else:
group = storage.groups[gid]
fsid = b_stat['backend']['vfs']['fsid']
fsid_key = '{host}:{fsid}'.format(host=node.host, fsid=fsid)
if fsid_key not in storage.fs:
logger.debug('Adding fs {0}'.format(fsid_key))
fs = storage.fs.add(node.host, fsid)
else:
fs = storage.fs[fsid_key]
if node_backend not in fs.node_backends:
fs.add_node_backend(node_backend)
if fs not in processed_fss:
fs.update_statistics(b_stat['backend'], collect_ts)
processed_fss.add(fs)
logger.info('Updating statistics for node backend {}'.format(node_backend))
prev_base_path = node_backend.base_path
try:
node_backend.update_statistics(b_stat, collect_ts)
except KeyError as e:
logger.warn('Bad stat for node backend {0} ({1}): {2}'.format(
node_backend, e, b_stat))
pass
if node_backend.base_path != prev_base_path:
update_group_history = True
if b_stat['status']['read_only'] or node_backend.stat_commit_errors > 0:
node_backend.make_read_only()
else:
node_backend.make_writable()
if node_backend.group is not group:
logger.debug('Adding node backend {0} to group {1}{2}'.format(
node_backend, group.group_id,
' (moved from group {0})'.format(node_backend.group.group_id)
if node_backend.group else ''))
group.add_node_backend(node_backend)
update_group_history = True
# these backends' commands stat are used later to update accumulated
# node commands stat
processed_node_backends.append(node_backend)
if update_group_history:
logger.debug('Group {} history may be outdated, adding to update queue'.format(group))
infrastructure.update_group_history(group)
def update_statistics(node, stat, elapsed_time=None):
logger.debug(
'Cluster updating: node {0} statistics time: {1:03f}'.format(
node, elapsed_time
)
)
collect_ts = mh.elliptics_time_to_ts(stat['timestamp'])
try:
try:
node.update_statistics(stat, collect_ts)
except KeyError as e:
logger.warn('Bad procfs stat for node {0} ({1}): {2}'.format(node, e, stat))
pass
fss = set()
good_node_backends = []
backend_stats = NodeInfoUpdater._parsed_stats(stat['stats'])
for b_stat in stat['backends'].itervalues():
backend_id = b_stat['backend_id']
b_stat['stats'] = backend_stats.get(backend_id, {})
update_group_history = False
node_backend_addr = '{0}/{1}'.format(node, backend_id)
if node_backend_addr not in storage.node_backends:
node_backend = storage.node_backends.add(node, backend_id)
update_group_history = True
else:
node_backend = storage.node_backends[node_backend_addr]
nb_config = (b_stat['config']
if 'config' in b_stat else
b_stat['backend']['config'])
gid = nb_config['group']
if gid == 0:
# skip zero group ids
continue
if b_stat['status']['state'] != 1:
logger.info('Node backend {0} is not enabled: state {1}'.format(
str(node_backend), b_stat['status']['state']))
node_backend.disable()
continue
if gid not in storage.groups:
logger.debug('Adding group {0}'.format(gid))
group = storage.groups.add(gid)
else:
group = storage.groups[gid]
if 'vfs' not in b_stat['backend']:
logger.error(
'Failed to parse statistics for node backend {0}, '
'vfs key not found: {1}'.format(node_backend, b_stat))
continue
fsid = b_stat['backend']['vfs']['fsid']
fsid_key = '{host}:{fsid}'.format(host=node.host, fsid=fsid)
if fsid_key not in storage.fs:
logger.debug('Adding fs {0}'.format(fsid_key))
fs = storage.fs.add(node.host, fsid)
else:
fs = storage.fs[fsid_key]
if node_backend not in fs.node_backends:
fs.add_node_backend(node_backend)
fs.update_statistics(b_stat['backend'], collect_ts)
fss.add(fs)
good_node_backends.append(node_backend)
node_backend.enable()
logger.info('Updating statistics for node backend %s' % (str(node_backend)))
if 'backend' not in b_stat:
logger.warn('No backend in b_stat: {0}'.format(b_stat))
elif 'dstat' not in b_stat['backend']:
logger.warn('No dstat in backend: {0}'.format(b_stat['backend']))
prev_base_path = node_backend.base_path
try:
node_backend.update_statistics(b_stat, collect_ts)
except KeyError as e:
logger.warn('Bad stat for node backend {0} ({1}): {2}'.format(
node_backend, e, b_stat))
pass
if node_backend.base_path != prev_base_path:
update_group_history = True
if b_stat['status']['read_only'] or node_backend.stat_commit_errors > 0:
node_backend.make_read_only()
else:
node_backend.make_writable()
if node_backend.group is not group:
logger.debug('Adding node backend {0} to group {1}{2}'.format(
node_backend, group.group_id,
' (moved from group {0})'.format(node_backend.group.group_id)
if node_backend.group else ''))
update_group_history = True
group.add_node_backend(node_backend)
if update_group_history:
infrastructure.update_group_history(group)
for fs in fss:
fs.update_commands_stats()
node.update_commands_stats(good_node_backends)
except Exception as e:
logger.exception('Unable to process statistics for node {}'.format(node))
