def data_size(self):
sz = 0
if self.resource.resource_file and os.path.exists(self.resource.resource_file.path):
sz += os.stat(self.resource.resource_file.path).st_size
for line in sh.du("-cs", self.cache_path):
if line.startswith("."):
sz += int(line.split(" ")[-1])
return sz
def test_piping():
from sh import sort, du, glob, wc, ls
# sort this directory by biggest file
print sort(du(glob('*'), '-sb'), '-rn')
# print the number of folders and files in /etc
print wc(ls('/etc', '-l'), '-l')
def _get_dir_used_space(directory):
used_space = 0
try:
used_space = int(sh.du(directory, s=True).split()[0])
except:
# TODO: Improve exception handling and error msg
log.error('Could not obtain the disk usage')
return used_space
def main() -> None:
count_new_dirs = 0
start_time = time.time()
src_dirs = set(sh.ssh(host, f'ls {src_dir}').split())
dst_dirs = set(sh.ls(f'{dst_dir}').split())
new_dirs = src_dirs - dst_dirs
if new_dirs:
count_new_dirs = len(new_dirs)
log('backup_img', f'На сервере обнаружено новых каталогов: {count_new_dirs}. Воссоздание дерева каталогов')
for new_dir in new_dirs:
sh.mkdir('-p', f'{dst_dir}/{new_dir}')
del src_dirs, dst_dirs, new_dirs
src_files = set(sh.ssh(host, f'cd {src_dir}; find -type f').split())
sh.cd(dst_dir)
dst_files = set(sh.find('-type', 'f').split())
new_files = src_files - dst_files
size_before_copying = int(sh.du('-s').split()[0])
if new_files:
count_new_files = len(new_files)
log('backup_img', f'На сервере обнаружено новых файлов: {count_new_files}. Начинаю копирование...')
for cnt, new_file in enumerate(new_files):
if (cnt + 1) % 10 == 0:
log('backup_img', f'Скопировано файлов: {cnt}...')
current_dir, current_file = new_file.split('/')[1:]
sh.scp('-p', '-r', f'{host}:{src_dir}/{current_dir}/{current_file}', f'{dst_dir}/{current_dir}/{current_file}')
size_after_copying = int(sh.du('-s').split()[0])
size_delta = round((size_after_copying - size_before_copying) / 1024, 1)
log('backup_img', f'Размер скопированных файлов {size_delta} Мб')
time_delta = int(time.time() - start_time)
time_delta = f'{str(time_delta // 3600).zfill(2)}:{str(time_delta // 60 % 60).zfill(2)}:{str(time_delta % 60).zfill(2)}'
log('backup_img', f'Время копирования: {time_delta}')
message = f'{get_time_stamp()} New images copying completed.\n' \
f'Copied file size: {size_delta}Mb\n' \
f'New files: {count_new_files}\n' \
f'New dirs: {count_new_dirs}\n' \
f'Time spent copying: {time_delta}'
send_message_telegram(message)
del src_files, dst_files, new_files
return None
def main(argv):
if not os.path.exists(IMAGE_DIR):
logger.error('Cannot find IMAGE_DIR: ' + IMAGE_DIR)
exit(1)
# Create ROOTFS_DIR if not present
sh.mkdir(ROOTFS_DIR, "-p")
if len(argv) == 0 or argv[0] == '-h' or argv[0] == '--help':
print_help()
exit(0)
single_arg_cmds = ['ls', 'rm', 'mkdir', 'file', 'vim', 'nano', 'cat']
auto_color_cmds = ['ls']
# Remove one element from argument list
command = argv.pop(0)
if command == 'list':
image_list = find_image_list()
print('{:<40}{:<20}{:<20}'.format('IMAGE NAME', 'TYPE', 'SIZE'))
for image in image_list:
image_size_text = sh.du('-L',
'-h',
image['name'],
_ok_code=range(255))
image['sizeB'] = image_size_text.split()[0]
print('{:<40}{:<20}{:<20}'.format(image['name'], image['type'],
image['sizeB']))
elif command == 'push':
do_push(argv)
elif command == 'pull':
do_pull(argv)
elif command in single_arg_cmds:
extra_argv = []
if command in auto_color_cmds:
extra_argv += ['--color=auto']
do_single_arg_cmd(argv, command, extra_argv)
elif command == 'query':
subcommand = argv.pop(0)
do_query(argv, subcommand)
elif command == 'mount':
do_mount(argv)
elif command == 'umount':
do_umount(argv)
elif command == 'userMount':
do_mount(argv, user=True)
elif command == 'userUmount':
do_umount(argv, user=True)
else:
logger.error('Command not supported: ' + str(command))
exit(1)
def do_query(argv, subcommand):
# Match any of command, i.e. *of, e.g. typeof.
if re.match(r'^\w+of$', subcommand):
if os.path.isfile(argv[0]):
path = argv[0]
else:
path = os.path.join(IMAGE_DIR, argv[0])
if subcommand == 'list':
image_list = find_image_list()
for image in image_list:
print(image['name'])
elif subcommand == 'listWithType':
image_list = find_image_list()
for image in image_list:
print('{} {}'.format(
image['name'],
image['type'],
))
elif subcommand == 'typeof':
print(imageParser.parse_image_type(path))
elif subcommand == 'sizeof':
image_size_text = sh.du('-L', '-h', path, _ok_code=range(255))
print(image_size_text.split()[0])
elif subcommand == 'pathof':
print(imageParser.locate_image_path(path))
elif subcommand == 'partitionTableof':
path = imageParser.locate_image_path(path)
img = imageParser.parse_partition_table(path)
for idx, part in enumerate(img['partitions']):
# index, start, end, size, offset, sizelimit, mountable
print('{} {} {} {} {} {} {}'.format(idx + 1, part['start'],
part['end'], part['size'],
part['offset'],
part['sizelimit'],
part['mountable']))
else:
logger.error('Query command not supported: ' + str(subcommand))
exit(1)
def get_space_used(path, host=None):
if not os.path.isdir(path):
return 0
_user = os.path.basename(path)
# Get /home usage from ZFS
if os.path.dirname(path) == "/home":
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username='******')
stdin, stdout, stderr = ssh.exec_command(
"zfs get -H -p -o value userused@%s tank/home" % _user)
out = stdout.read().splitlines()
logger.debug1("HOME zfs output for %s: %s", _user, out)
if type(out) is list:
_value = out[0]
else:
_value = out
if _value == '-':
_value = 0
return int(_value)
# Get /fdata usage from quota reports in /tmp
if "/fdata" in path:
if _user in FDATA_USERS:
return 0
if _user in BEEGFS_USED:
FDATA_USERS.append(_user)
return int(BEEGFS_USED[_user])
# If the above methods failed for some reason, use du
logger.debug1("Executing: du -s -x %s", path)
_du_out = du("-s", "-x", path)
_du_split = _du_out.split()
_du_used = _du_split[0]
_du_path = _du_split[1]
if _du_path != path:
logger.error("du %s invalid", path)
return False
return int(_du_used) * 1024
def get_space_used(path, host=None):
if not os.path.isdir(path):
return 0
_user = os.path.basename(path)
# Get /home usage from ZFS
if os.path.dirname(path) == "/home":
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username='******')
stdin, stdout, stderr = ssh.exec_command("zfs get -H -p -o value userused@%s tank/home" % _user)
out = stdout.read().splitlines()
logger.debug1("HOME zfs output for %s: %s", _user, out)
if type(out) is list:
_value = out[0]
else:
_value = out
if _value == '-':
_value = 0
return int(_value)
# Get /fdata usage from quota reports in /tmp
if "/fdata" in path:
if _user in FDATA_USERS:
return 0
if _user in FHGFS_USED:
FDATA_USERS.append(_user)
return int(FHGFS_USED[_user])
# If the above methods failed for some reason, use du
logger.debug1("Executing: du -s -x %s", path)
_du_out = du("-s", "-x", path)
_du_split = _du_out.split()
_du_used = _du_split[0]
_du_path = _du_split[1]
if _du_path != path:
logger.error("du %s invalid", path)
return False
return int(_du_used) * 1024
from sh import git, ssh, head, tail, wc, sort, grep, du
# Get list of files
grep('*')
# Get sizes of each
du('-hM', grep('*'))
# Sort, numerically
sort(du('-hM', grep('*')), '-n')
# And get the largest
tail(sort(du('-hM', grep('*')), '-n'), '-n' 5)
def get_expected_size(path):
'''Ground truth by gnu utils'''
return int(du('-sb', path).split()[:1][0])
def data_size(self):
sz = self.resource.resource_file.size if self.resource.resource_file else 0
for line in sh.du('-cs', self.cache_path):
if line.startswith('.'):
sz += int(line.split(' ')[-1])
return sz
def data_size(self):
sz = self.resource.resource_file.size if self.resource.resource_file else 0
for line in sh.du('-cs', self.cache_path):
if line.startswith('.'):
sz += int(line.split(' ')[-1])
return sz
def getFolderSize(path):
return int(sh.du("-s", path).split("\t")[0])
pass
i += 1
time.sleep(1)
vm = imp.load_source(sys.argv[1], '/root/vms/%s' % sys.argv[1])
this = wait_for(sys.argv[1], 90 if not hasattr(vm, 'systemd_timeout') else getattr(vm, 'systemd_timeout'))
if not this:
sys.exit(1)
if len(sys.argv) > 2:
sys.exit(0)
# if it's the first boot, it could take a while
# first boot defined by 'local' FS size > the newly formatted size of ~(483328+sizeof(vm-support))
def local_size():
return int(sh.zfs('list',o='refer',H=True,p='xenfs/vms/%s/local' % sys.argv[1]))
vm_size = int(sh.du(s='/root/vm-support',block_size=1).split('\t')[0])
zero_size = 490000
zero_plus_vm = int(zero_size+vm_size*1.1) # 1.1 for FS overhead
while True:
try:
this.ping()
time.sleep(5)
except dbus.exceptions.DBusException, e:
if local_size() < zero_plus_vm:
this = wait_for(sys.argv[1])
continue
print 'VM exited'
sys.exit(0)
sys.exit(1)
def get_dir_size(dname):
"""Get size of directory in kilobytes using unix ``du -s``"""
size = sh.du('-s', dname).split()[0]
return int(size)
def get_expected_size(path):
'''Ground truth by gnu utils'''
return int(du('-sb', path).split()[:1][0])
def get_du(p):
if os.path.exists(p):
return int(str(sh.du('-k', p)).split()[0]) * 1024
else:
return 0L