def main_query(args):
vlevel = set_logging(args.verbose, args.log_file)
results = query(args.mirror_url,
args.max_items,
args.filters,
verbosity=vlevel)
try:
if args.output_format == FORMAT_JSON:
print(util.json_dumps(results).decode())
else:
output = []
for item in results:
try:
output.append(args.output_format % item)
except KeyError as e:
sys.stderr.write("output format failed (%s) for: %s\n" %
(e, item))
sys.exit(1)
for line in sorted(output):
print(line)
except IOError as e:
if e.errno == errno.EPIPE:
sys.exit(0x80 | signal.SIGPIPE)
raise
def validate_config(config, sourcefile=None):
"""Validate storage config object."""
if not sourcefile:
sourcefile = ''
try:
import jsonschema
jsonschema.validate(config, STORAGE_CONFIG_SCHEMA)
except ImportError:
LOG.error('Cannot validate storage config, missing jsonschema')
raise
except jsonschema.exceptions.ValidationError as e:
if isinstance(e.instance, int):
msg = 'Unexpected value (%s) for property "%s"' % (e.path[0],
e.instance)
raise ValueError(msg)
if 'type' not in e.instance:
msg = "%s in %s" % (e.message, e.instance)
raise ValueError(msg)
instance_type = e.instance['type']
stype = get_storage_types().get(instance_type)
if stype:
try:
jsonschema.validate(e.instance, stype.schema)
except jsonschema.exceptions.ValidationError as f:
msg = "%s in %s\n%s" % (f.message, sourcefile,
util.json_dumps(e.instance))
raise (ValueError(msg))
else:
msg = "Unknown storage type: %s in %s" % (instance_type,
e.instance)
raise ValueError(msg)
def get_partition_sfdisk_info(devpath, sfdisk_info=None):
if not sfdisk_info:
sfdisk_info = sfdisk_info(devpath)
entry = [
part for part in sfdisk_info['partitions'] if part['node'] == devpath
]
if len(entry) != 1:
raise RuntimeError(
'Device %s not present in sfdisk dump:\n%s' % devpath,
util.json_dumps(sfdisk_info))
return entry.pop()
def insert_products(self, path, target, content):
# The super classes' insert_products will
# we override this because default mirror inserts content
# where as we want to insert the rendered 'target' tree
# the difference is that 'content' is the original (with gpg sign)
# so our content will no longer have that signature.
dpath = self.products_data_path(target['content_id'])
self.store.insert_content(dpath, util.json_dumps(target))
if not path:
return
# this will end up writing the content exactly as it
# was in the source, leaving the signed data in-tact
self.store.insert_content(path, content)
# for our vmtest content id, we want to write
# a json file in streams/v1/<distro>.json that can be queried
# even though it will not appear in index
vmtest_json = VMTEST_CONTENT_ID_PATH_MAP.get(target['content_id'])
if vmtest_json:
self.store.insert_content(vmtest_json, util.json_dumps(target))
def test_basic_zfsroot_update_storage_config(self):
zfsroot_volname = "/ROOT/zfsroot"
pool_id = self.zfsroot_id + '_zfsroot_pool'
newents = [
{
'type': 'zpool',
'id': pool_id,
'pool': 'rpool',
'vdevs': ['disk1p1'],
'mountpoint': '/'
},
{
'type': 'zfs',
'id': self.zfsroot_id + '_zfsroot_container',
'pool': pool_id,
'volume': '/ROOT',
'properties': {
'canmount': 'off',
'mountpoint': 'none'
}
},
{
'type': 'zfs',
'id': self.zfsroot_id + '_zfsroot_fs',
'pool': pool_id,
'volume': zfsroot_volname,
'properties': {
'canmount': 'noauto',
'mountpoint': '/'
}
},
]
expected = OrderedDict([(i['id'], i)
for i in self.base + newents + self.extra])
scfg = block_meta.extract_storage_ordered_dict({
'storage': {
'version': 1,
'config': self.base + self.zfsroots + self.extra
}
})
found = block_meta.zfsroot_update_storage_config(scfg)
print(util.json_dumps([(k, v) for k, v in found.items()]))
self.assertEqual(expected, found)
def get_blockdev_sector_size(devpath):
"""
Get the logical and physical sector size of device at devpath
Returns a tuple of integer values (logical, physical).
"""
info = _lsblock([devpath])
LOG.debug('get_blockdev_sector_size: info:\n%s' % util.json_dumps(info))
# (LP: 1598310) The call to _lsblock() may return multiple results.
# If it does, then search for a result with the correct device path.
# If no such device is found among the results, then fall back to previous
# behavior, which was taking the first of the results
assert len(info) > 0
for (k, v) in info.items():
if v.get('device_path') == devpath:
parent = k
break
else:
parent = list(info.keys())[0]
return (int(info[parent]['LOG-SEC']), int(info[parent]['PHY-SEC']))
def zero_device(devpath, force=False):
""" Wipe mdadm member device at data offset.
For mdadm devices with metadata version 1.1 or newer location
of the data offset is provided. This value is used to determine
the location to start wiping data to clear data.
If metadata version is older then fallback to wiping 1MB at
start and end of device; metadata was at end of device.
"""
assert_valid_devpath(devpath)
metadata = mdadm_examine(devpath, export=False)
if not metadata and not force:
LOG.debug('%s not mdadm member, force=False so skiping zeroing',
devpath)
return
LOG.debug('mdadm.examine metadata:\n%s', util.json_dumps(metadata))
version = metadata.get('version')
offsets = []
# wipe at start, end of device for metadata older than 1.1
if version and version in ["1.1", "1.2"]:
LOG.debug('mdadm %s has metadata version=%s, extracting offsets',
devpath, version)
for field in ['super_offset', 'data_offset']:
offset, unit = metadata[field].split()
if unit == "sectors":
offsets.append(int(offset) * 512)
else:
LOG.warning('Unexpected offset unit: %s', unit)
if not offsets:
offsets = [0, -(1024 * 1024)]
LOG.info('mdadm: wiping md member %s @ offsets %s', devpath, offsets)
zero_file_at_offsets(devpath,
offsets,
buflen=1024,
count=1024,
strict=True)
LOG.info('mdadm: successfully wiped %s' % devpath)
def block_info_main(args):
"""get information about block devices, similar to lsblk"""
if not args.devices:
raise ValueError('devices to scan must be specified')
if not all(block.is_block_device(d) for d in args.devices):
raise ValueError('invalid device(s)')
def add_size_to_holders_tree(tree):
"""add size information to generated holders trees"""
size_file = os.path.join(tree['device'], 'size')
# size file is always represented in 512 byte sectors even if
# underlying disk uses a larger logical_block_size
size = ((512 * int(util.load_file(size_file)))
if os.path.exists(size_file) else None)
tree['size'] = util.bytes2human(size) if args.human else str(size)
for holder in tree['holders']:
add_size_to_holders_tree(holder)
return tree
def format_name(tree):
"""format information for human readable display"""
res = {
'name': ' - '.join((tree['name'], tree['dev_type'], tree['size'])),
'holders': []
}
for holder in tree['holders']:
res['holders'].append(format_name(holder))
return res
trees = [
add_size_to_holders_tree(t) for t in
[block.clear_holders.gen_holders_tree(d) for d in args.devices]
]
print(
util.json_dumps(trees) if args.json else '\n'.join(
block.clear_holders.format_holders_tree(t)
for t in [format_name(tree) for tree in trees]))
return 0
def get_blockdev_sector_size(devpath):
"""
Get the logical and physical sector size of device at devpath
Returns a tuple of integer values (logical, physical).
"""
info = {}
try:
info = _lsblock([devpath])
except util.ProcessExecutionError as e:
# raise on all errors except device missing error
if str(e.exit_code) != "32":
raise
if info:
LOG.debug('get_blockdev_sector_size: info:\n%s', util.json_dumps(info))
# (LP: 1598310) The call to _lsblock() may return multiple results.
# If it does, then search for a result with the correct device path.
# If no such device is found among the results, then fall back to
# previous behavior, which was taking the first of the results
assert len(info) > 0
for (k, v) in info.items():
if v.get('device_path') == devpath:
parent = k
break
else:
parent = list(info.keys())[0]
logical = info[parent]['LOG-SEC']
physical = info[parent]['PHY-SEC']
else:
sys_path = sys_block_path(devpath)
logical = util.load_file(
os.path.join(sys_path, 'queue/logical_block_size'))
physical = util.load_file(
os.path.join(sys_path, 'queue/hw_sector_size'))
LOG.debug('get_blockdev_sector_size: (log=%s, phys=%s)', logical, physical)
return (int(logical), int(physical))
