예제 #1
0
    def snapshot(self, path):
        """Takes a snapshot of an individual file."""

        # try grabbing the temp filesystem system path
        temp_dir = None
        temp_dir = self.tmp.getsyspath('/')

        # Create a temp file system to be snapshotted
        temp_snapshot_fs = TempFS(temp_dir=temp_dir)
        src_path = temp_snapshot_fs.getsyspath('/')

        with self.fs.open(path, 'rb') as source_file:
            with temp_snapshot_fs.open('datafile', 'wb') as temp_file:
                shutil.copyfileobj(source_file, temp_file)

        # snapshot destination directory
        dest_dir = self.snapshot_snap_path(path)

        command = ['rdiff-backup',
                   '--parsable-output',
                   '--no-eas',
                   '--no-file-statistics',
                   '--no-acls',
                   '--tempdir', self.tmp.getsyspath('/'),
                   src_path, dest_dir]

        # speed up the tests
        if self.__testing:
            command.insert(5, '--current-time')
            command.insert(6, str(self.__testing['time']))
            self.__testing['time'] += 1

        process = Popen(command, stdout=PIPE, stderr=PIPE)
        stderr = process.communicate()[1]

        ignore = [lambda x: x.startswith("Warning: could not determine case")]

        if len(stderr) is not 0:
            for rule in ignore:
                if not rule(stderr):
                    raise SnapshotError(stderr)

        # close the temp snapshot filesystem
        temp_snapshot_fs.close()
예제 #2
0
파일: moyapi.py 프로젝트: chrmorais/moya
    def run_install(self):
        args = self.args
        console = self.console
        installed = []
        install_package = args.package

        install_select = package_select = self.call('package.select', package=install_package)
        install_notes = package_select['notes']

        if package_select['version'] is None:
            raise CommandError("no install candidate for '{}', run 'moya-pm list' to see available packages".format(install_package))

        package_name = package_select['name']
        install_version = versioning.Version(package_select['version'])

        filename = package_select['md5']
        download_url = package_select['download']
        package_filename = download_url.rsplit('/', 1)[-1]

        libs = []
        output_fs = fsopendir(args.output)
        force = args.force

        installed_libs = {}

        archive = None
        if not args.download:
            try:
                application = WSGIApplication(self.location, args.settings, disable_autoreload=True)
                archive = application.archive
                if archive is None:
                    console.text('unable to load project, use the --force switch to force installation')
                    return -1
            except Exception as e:
                if not args.force:
                    console.exception(e)
                    console.text('unable to load project, use the --force switch to force installation')
                    return -1
            else:
                libs = [(lib.long_name, lib.version, lib.install_location)
                        for lib in archive.libs.values() if lib.long_name == package_name]

                installed_libs = archive.libs.copy()

                if not force:
                    for name, version, location in libs:
                        if name == package_name:
                            if version > install_version:
                                if not args.force:
                                    raise CommandError("a newer version ({}) is already installed, use --force to force installation".format(version))
                            elif install_version == version:
                                if not args.force:
                                    raise CommandError("version {} is already installed, use --force to force installation".format(version))
                            else:
                                if not args.upgrade:
                                    raise CommandError("an older version ({}) is installed, use --upgrade to force upgrade".format(version))
                            force = True

        username = self.settings.get('upload', 'username', None)
        password = self.settings.get('upload', 'password', None)
        if username and password:
            auth = (username, password)
        else:
            auth = None

        install_app = args.app or package_name.split('.')[-1]
        packages = dependencies.gather_dependencies(self.rpc,
                                                    install_app,
                                                    args.mount,
                                                    install_package,
                                                    console,
                                                    no_deps=args.no_deps)

        if not args.no_add:
            for package_name, (app_name, mount, package_select) in packages.items():
                if package_select['version'] is None:
                    raise CommandError("no install candidate for required package '{}', run 'moya-pm list {}' to see available packages".format(package_name, package_name))

        download_temp_fs = TempFS()
        for package_name, (app_name, mount, package_select) in packages.items():

            package_name = package_select['name']
            install_version = versioning.Version(package_select['version'])

            filename = "{}-{}.{}".format(package_name, install_version, package_select['md5'])
            download_url = package_select['download']
            package_filename = download_url.rsplit('/', 1)[-1]

            with download_temp_fs.open(filename, 'wb') as package_file:
                checksum = downloader.download(download_url,
                                               package_file,
                                               console=console,
                                               auth=auth,
                                               verify_ssl=False,
                                               msg="requesting {name}=={version}".format(**package_select))
                if checksum != package_select['md5']:
                    raise CommandError("md5 checksum of download doesn't match server! download={}, server={}".format(checksum, package_select['md5']))

            if args.download:
                with fsopendir(args.download) as dest_fs:
                    fs.utils.copyfile(download_temp_fs, filename, dest_fs, package_filename)

        if args.download:
            return 0

        changed_server_xml = False
        for package_name, (app_name, mount, package_select) in packages.items():

            package_name = package_select['name']
            install_version = versioning.Version(package_select['version'])

            filename = "{}-{}.{}".format(package_name, install_version, package_select['md5'])
            download_url = package_select['download']
            package_filename = download_url.rsplit('/', 1)[-1]

            install_location = relativefrom(self.location, pathjoin(self.location, args.output, package_select['name']))
            package_select['location'] = install_location

            with download_temp_fs.open(filename, 'rb') as package_file:
                with ZipFS(package_file, 'r') as package_fs:
                    with output_fs.makeopendir(package_select['name']) as lib_fs:
                        #if not lib_fs.isdirempty('/') and not force:
                        #    raise CommandError("install directory is not empty, use --force to erase and overwrite")
                        fs.utils.remove_all(lib_fs, '/')
                        fs.utils.copydir(package_fs, lib_fs)
                        installed.append((package_select, mount))

            if not args.no_add and archive:
                server_xml = archive.cfg.get('project', 'startup')
                changed_server_xml =\
                    installer.install(project_path=self.location,
                                      server_xml_location=archive.cfg.get('project', 'location'),
                                      server_xml=server_xml,
                                      server_name=application.server_ref,
                                      lib_path=install_location,
                                      lib_name=package_name,
                                      app_name=app_name,
                                      mount=mount)

        table = []
        for _package, mount in installed:
            table.append([Cell("{name}=={version}".format(**_package), fg="magenta", bold=True),
                          Cell(_package['location'], fg="blue", bold=True),
                          Cell(mount or '', fg="cyan", bold=True)])

        if table:
            console.table(table, ['package', 'location', 'mount'])

        if install_notes:
            console.table([[install_notes]], ['{} v{} release notes'.format(install_select['name'], install_select['version'])])

        if changed_server_xml:
            console.text("moya-pm modified '{}' -- please check changes".format(server_xml), fg="green", bold="yes")
예제 #3
0
from fs import open_fs
from fs.tempfs import TempFS
# from fs_s3fs import S3FS


# Generate temp files to upload:
tempfs = TempFS()
for num in range(0, 100):
    with tempfs.open('file' + str(num), 'w') as fi:
        fi.write("I AM A FILE\n" * num)


# Connect to S3
# s3fs = S3FS(
#     'default',
#     aws_access_key_id="minio",
#     aws_secret_access_key="minio123",
#     endpoint_url="http://*****:*****@default/?endpoint_url=http://localhost:9009')

# Print some info for us to use.
count = len(s3fs.listdir('/'))
print("File Count in S3: {}".format(count))
count = len(tempfs.listdir('/'))
print("File Count in tempfs: {}".format(count))

# Upload the files to S3
files = tempfs.listdir('/')
for fi in files: