def compress(cls, src, dst):
logger.debug("Compressing '%s' to '%s'...", src,
dst if isinstance(dst, str) else dst.name)
with ZipFile(dst, 'w', allowZip64=True) as zipf:
for root, dirs, files in os.walk(src):
zipf.write(root, os.path.relpath(root, src))
for fn in files:
filename = pthjoin(root, fn)
if os.path.isfile(filename):
arcname = pthjoin(os.path.relpath(root, src), fn)
zipf.write(filename, arcname)
def argparser_setup(cls, parser, env):
parser.add_argument(
'--no-zip',
dest='nozip',
action='store_true',
help=u"Не запаковывать резервную копию в ZIP-архив, резервная" +
u" копия будет создана в виде директории")
settings = env.core.settings
if 'backup.path' in settings:
default_target = pthjoin(
settings['backup.path'],
datetime.today().strftime(settings['backup.filename']) +
'.ngwbackup')
else:
default_target = None
parser.add_argument(
'target',
type=str,
metavar='path',
default=default_target,
nargs='?' if default_target is not None else None,
help=u"Имя файла или директории (см. --no-zip), в который будет" +
u" сохранена резервная копия")
def execute(cls, args, env):
target = args.target
if target is None:
if env.core.options['backup.path']:
autoname = datetime.today().strftime(env.core.options['backup.filename'])
target = pthjoin(env.core.options['backup.path'], autoname)
else:
target = NamedTemporaryFile(delete=False).name
os.unlink(target)
logger.warn("Backup path not set. Writing backup to temporary file %s!", target)
to_stdout = target == '-'
tmp_root = env.core.options.get('backup.tmpdir', None if to_stdout
else os.path.split(target)[0])
if not to_stdout and os.path.exists(target):
raise RuntimeError("Target already exists!")
if args.nozip:
@contextmanager
def tgt_context():
tmpdir = mkdtemp(dir=tmp_root)
try:
yield tmpdir
logger.debug("Renaming [%s] to [%s]...", tmpdir, target)
os.rename(tmpdir, target)
except Exception:
rmtree(tmpdir)
raise
elif to_stdout:
@contextmanager
def tgt_context():
with TemporaryDirectory(dir=tmp_root) as tmp_dir:
yield tmp_dir
cls.compress(tmp_dir, sys.stdout.buffer)
else:
@contextmanager
def tgt_context():
with TemporaryDirectory(dir=tmp_root) as tmp_dir:
yield tmp_dir
tmp_arch = mkstemp(dir=tmp_root)[1]
os.unlink(tmp_arch)
try:
cls.compress(tmp_dir, tmp_arch)
logger.debug("Renaming [%s] to [%s]...", tmp_arch, target)
os.rename(tmp_arch, target)
except Exception:
os.unlink(tmp_arch)
raise
with tgt_context() as tgt:
backup(env, tgt)
if not to_stdout:
print(target)
def argparser_setup(cls, parser, env):
parser.add_argument(
'--upload',
dest='upload',
action='store_true',
help="upload backup to remote storage using S3 protocol")
parser.add_argument('--upload-object',
dest='upload_object',
metavar='name')
parser.add_argument('--upload-bucket',
dest='upload_bucket',
metavar='name',
default=env.core._backup_upload_bucket)
parser.add_argument('--upload-server',
dest='upload_server',
metavar='host:port',
default=env.core._backup_upload_server)
parser.add_argument('--upload-access-key',
dest='upload_access_key',
metavar='key',
default=env.core._backup_upload_access_key)
parser.add_argument('--upload-secret-key',
dest='upload_secret_key',
metavar='secret',
default=env.core._backup_upload_secret_key)
parser.add_argument(
'--no-zip',
dest='nozip',
action='store_true',
help='use directory instead of zip-file as backup format')
settings = env.core.settings
if 'backup.path' in settings:
default_target = pthjoin(
settings['backup.path'],
datetime.today().strftime(settings['backup.filename']) +
'.ngwbackup') # NOQA: W503
else:
default_target = None
parser.add_argument('target',
type=str,
metavar='path',
default=default_target,
nargs='?',
help='backup destination path')
def argparser_setup(cls, parser, env):
parser.add_argument(
'--no-zip', dest='nozip', action='store_true',
help=u"Не запаковывать резервную копию в ZIP-архив, резервная"
+ u" копия будет создана в виде директории")
settings = env.core.settings
if 'backup.path' in settings:
default_target = pthjoin(
settings['backup.path'],
datetime.today().strftime(settings['backup.filename']) +
'.ngwbackup')
else:
default_target = None
parser.add_argument(
'target', type=str, metavar='path', default=default_target,
nargs='?' if default_target is not None else None,
help=u"Имя файла или директории (см. --no-zip), в который будет"
+ u" сохранена резервная копия")
def execute(cls, args, env):
target = args.target
autoname = datetime.today().strftime(env.core._backup_filename)
if target is None:
if env.core._backup_path:
target = pthjoin(env.core._backup_path, autoname)
else:
target = NamedTemporaryFile(delete=False).name
os.unlink(target)
upload_object = args.upload_object
if upload_object is None:
upload_object = autoname
if args.nozip and args.upload:
raise RuntimeError("Incompatible options: no-zip and upload")
if args.upload and upload_object is None:
raise RuntimeError("Upload object name required")
env.core.logger.info("Running backup to %s", target)
backup(env, target, nozip=args.nozip)
if args.upload:
env.core.logger.info(
"Uploading backup to server=%s, bucket=%s, object=%s",
args.upload_server, args.upload_bucket, upload_object)
client = Minio(args.upload_server,
secure=False,
access_key=args.upload_access_key,
secret_key=args.upload_secret_key)
if args.upload_secret_key:
if not client.bucket_exists(args.upload_bucket):
env.core.logger.info("Bucket not exist so creating new")
client.make_bucket(args.upload_bucket)
client.fput_object(args.upload_bucket, upload_object, target)
from cffi import FFI
from os.path import dirname
from os.path import join as pthjoin
from os.path import relpath
ffi = FFI()
ffi.cdef("int LZ4_compress (const char*, char*, int);")
ffi.cdef("int LZ4_decompress_safe (const char*, char*, int, int);")
ffi.cdef("int LZ4_compressHC (const char*, char*, int);")
ffi.cdef("int LZ4_compressBound (int);")
path = dirname(__file__)
lz4_files = ['lz4.c', 'lz4hc.c', 'xxhash.c']
sources = [relpath(pthjoin(path, p)) for p in lz4_files]
_lz4 = ffi.verify(sources=sources)
def _store_le32(c, x):
c[0] = chr(x & 0xff)
c[1] = chr((x >> 8) & 0xff)
c[2] = chr((x >> 16) & 0xff)
c[3] = chr((x >> 24) & 0xff)
def _load_le32(c):
return ord(c[0]) | \
(ord(c[1]) << 8) | \
(ord(c[2]) << 16) | \
(ord(c[3]) << 24)
cmph_io_adapter_t *cmph_io_function_adapter(char*(*readfn)(),
void(*rewindfn)(),
void(*destroyfn)(),
cmph_uint32(*keylenfn)(),
cmph_uint32 nkeys);
void cmph_io_function_adapter_destroy(cmph_io_adapter_t * adapter);
""")
__VERSION__ = '0.3.0'
ffi.C = ffi.dlopen(None)
path = dirname(__file__)
sources = [relpath(src) for src in
glob(pthjoin(path, '*.c'))]
@ffi.callback("void(int, char*)")
def _cmph_py_logger(level, message):
log_fn = {
0: logger.error,
1: logger.warn,
2: logger.info,
}.get(level, logger.debug)
log_fn(ffi.string(message).strip())
_cmph = ffi.verify('''
#include <cmph.h>
#include <pylogging.h>
#include <pyadapter.h>
cmph_io_adapter_t *cmph_io_function_adapter(char*(*readfn)(),
void(*rewindfn)(),
void(*destroyfn)(),
cmph_uint32(*keylenfn)(),
cmph_uint32 nkeys);
void cmph_io_function_adapter_destroy(cmph_io_adapter_t * adapter);
""")
__VERSION__ = '0.3.0'
ffi.C = ffi.dlopen(None)
path = dirname(__file__)
sources = [relpath(src) for src in
glob(pthjoin(path, '*.c'))]
@ffi.callback("void(int, char*)")
def _cmph_py_logger(level, message):
log_fn = {
0: logger.error,
1: logger.warn,
2: logger.info,
}.get(level, logger.debug)
log_fn(ffi.string(message).strip())
_cmph = ffi.verify('''
#include <cmph.h>
#include <pylogging.h>
#include <pyadapter.h>
from cffi import FFI
from os.path import dirname
from os.path import join as pthjoin
from os.path import relpath
ffi = FFI()
ffi.cdef("int LZ4_compress (const char*, char*, int);")
ffi.cdef("int LZ4_decompress_safe (const char*, char*, int, int);")
ffi.cdef("int LZ4_compressHC (const char*, char*, int);")
ffi.cdef("int LZ4_compressBound (int);")
path = dirname(__file__)
lz4_files = ['lz4.c', 'lz4hc.c', 'xxhash.c']
sources = [relpath(pthjoin(path, p)) for p in lz4_files]
_lz4 = ffi.verify(sources=sources)
def _store_le32(c, x):
c[0] = chr(x & 0xff)
c[1] = chr((x >> 8) & 0xff)
c[2] = chr((x >> 16) & 0xff)
c[3] = chr((x >> 24) & 0xff)
def _load_le32(c):
return ord(c[0]) | \
(ord(c[1]) << 8) | \
(ord(c[2]) << 16) | \
(ord(c[3]) << 24)
