def compress(cls, src, dst): logger.debug("Compressing '%s' to '%s'...", src, dst if isinstance(dst, str) else dst.name) with ZipFile(dst, 'w', allowZip64=True) as zipf: for root, dirs, files in os.walk(src): zipf.write(root, os.path.relpath(root, src)) for fn in files: filename = pthjoin(root, fn) if os.path.isfile(filename): arcname = pthjoin(os.path.relpath(root, src), fn) zipf.write(filename, arcname)
def argparser_setup(cls, parser, env): parser.add_argument( '--no-zip', dest='nozip', action='store_true', help=u"Не запаковывать резервную копию в ZIP-архив, резервная" + u" копия будет создана в виде директории") settings = env.core.settings if 'backup.path' in settings: default_target = pthjoin( settings['backup.path'], datetime.today().strftime(settings['backup.filename']) + '.ngwbackup') else: default_target = None parser.add_argument( 'target', type=str, metavar='path', default=default_target, nargs='?' if default_target is not None else None, help=u"Имя файла или директории (см. --no-zip), в который будет" + u" сохранена резервная копия")
def execute(cls, args, env): target = args.target if target is None: if env.core.options['backup.path']: autoname = datetime.today().strftime(env.core.options['backup.filename']) target = pthjoin(env.core.options['backup.path'], autoname) else: target = NamedTemporaryFile(delete=False).name os.unlink(target) logger.warn("Backup path not set. Writing backup to temporary file %s!", target) to_stdout = target == '-' tmp_root = env.core.options.get('backup.tmpdir', None if to_stdout else os.path.split(target)[0]) if not to_stdout and os.path.exists(target): raise RuntimeError("Target already exists!") if args.nozip: @contextmanager def tgt_context(): tmpdir = mkdtemp(dir=tmp_root) try: yield tmpdir logger.debug("Renaming [%s] to [%s]...", tmpdir, target) os.rename(tmpdir, target) except Exception: rmtree(tmpdir) raise elif to_stdout: @contextmanager def tgt_context(): with TemporaryDirectory(dir=tmp_root) as tmp_dir: yield tmp_dir cls.compress(tmp_dir, sys.stdout.buffer) else: @contextmanager def tgt_context(): with TemporaryDirectory(dir=tmp_root) as tmp_dir: yield tmp_dir tmp_arch = mkstemp(dir=tmp_root)[1] os.unlink(tmp_arch) try: cls.compress(tmp_dir, tmp_arch) logger.debug("Renaming [%s] to [%s]...", tmp_arch, target) os.rename(tmp_arch, target) except Exception: os.unlink(tmp_arch) raise with tgt_context() as tgt: backup(env, tgt) if not to_stdout: print(target)
def argparser_setup(cls, parser, env): parser.add_argument( '--upload', dest='upload', action='store_true', help="upload backup to remote storage using S3 protocol") parser.add_argument('--upload-object', dest='upload_object', metavar='name') parser.add_argument('--upload-bucket', dest='upload_bucket', metavar='name', default=env.core._backup_upload_bucket) parser.add_argument('--upload-server', dest='upload_server', metavar='host:port', default=env.core._backup_upload_server) parser.add_argument('--upload-access-key', dest='upload_access_key', metavar='key', default=env.core._backup_upload_access_key) parser.add_argument('--upload-secret-key', dest='upload_secret_key', metavar='secret', default=env.core._backup_upload_secret_key) parser.add_argument( '--no-zip', dest='nozip', action='store_true', help='use directory instead of zip-file as backup format') settings = env.core.settings if 'backup.path' in settings: default_target = pthjoin( settings['backup.path'], datetime.today().strftime(settings['backup.filename']) + '.ngwbackup') # NOQA: W503 else: default_target = None parser.add_argument('target', type=str, metavar='path', default=default_target, nargs='?', help='backup destination path')
def execute(cls, args, env): target = args.target autoname = datetime.today().strftime(env.core._backup_filename) if target is None: if env.core._backup_path: target = pthjoin(env.core._backup_path, autoname) else: target = NamedTemporaryFile(delete=False).name os.unlink(target) upload_object = args.upload_object if upload_object is None: upload_object = autoname if args.nozip and args.upload: raise RuntimeError("Incompatible options: no-zip and upload") if args.upload and upload_object is None: raise RuntimeError("Upload object name required") env.core.logger.info("Running backup to %s", target) backup(env, target, nozip=args.nozip) if args.upload: env.core.logger.info( "Uploading backup to server=%s, bucket=%s, object=%s", args.upload_server, args.upload_bucket, upload_object) client = Minio(args.upload_server, secure=False, access_key=args.upload_access_key, secret_key=args.upload_secret_key) if args.upload_secret_key: if not client.bucket_exists(args.upload_bucket): env.core.logger.info("Bucket not exist so creating new") client.make_bucket(args.upload_bucket) client.fput_object(args.upload_bucket, upload_object, target)
from cffi import FFI from os.path import dirname from os.path import join as pthjoin from os.path import relpath ffi = FFI() ffi.cdef("int LZ4_compress (const char*, char*, int);") ffi.cdef("int LZ4_decompress_safe (const char*, char*, int, int);") ffi.cdef("int LZ4_compressHC (const char*, char*, int);") ffi.cdef("int LZ4_compressBound (int);") path = dirname(__file__) lz4_files = ['lz4.c', 'lz4hc.c', 'xxhash.c'] sources = [relpath(pthjoin(path, p)) for p in lz4_files] _lz4 = ffi.verify(sources=sources) def _store_le32(c, x): c[0] = chr(x & 0xff) c[1] = chr((x >> 8) & 0xff) c[2] = chr((x >> 16) & 0xff) c[3] = chr((x >> 24) & 0xff) def _load_le32(c): return ord(c[0]) | \ (ord(c[1]) << 8) | \ (ord(c[2]) << 16) | \ (ord(c[3]) << 24)
cmph_io_adapter_t *cmph_io_function_adapter(char*(*readfn)(), void(*rewindfn)(), void(*destroyfn)(), cmph_uint32(*keylenfn)(), cmph_uint32 nkeys); void cmph_io_function_adapter_destroy(cmph_io_adapter_t * adapter); """) __VERSION__ = '0.3.0' ffi.C = ffi.dlopen(None) path = dirname(__file__) sources = [relpath(src) for src in glob(pthjoin(path, '*.c'))] @ffi.callback("void(int, char*)") def _cmph_py_logger(level, message): log_fn = { 0: logger.error, 1: logger.warn, 2: logger.info, }.get(level, logger.debug) log_fn(ffi.string(message).strip()) _cmph = ffi.verify(''' #include <cmph.h> #include <pylogging.h> #include <pyadapter.h>