def snapshot(self, path):
"""Takes a snapshot of an individual file."""
# try grabbing the temp filesystem system path
temp_dir = None
temp_dir = self.tmp.getsyspath('/')
# Create a temp file system to be snapshotted
temp_snapshot_fs = TempFS(temp_dir=temp_dir)
src_path = temp_snapshot_fs.getsyspath('/')
with self.fs.open(path, 'rb') as source_file:
with temp_snapshot_fs.open('datafile', 'wb') as temp_file:
shutil.copyfileobj(source_file, temp_file)
# snapshot destination directory
dest_dir = self.snapshot_snap_path(path)
command = ['rdiff-backup',
'--parsable-output',
'--no-eas',
'--no-file-statistics',
'--no-acls',
'--tempdir', self.tmp.getsyspath('/'),
src_path, dest_dir]
# speed up the tests
if self.__testing:
command.insert(5, '--current-time')
command.insert(6, str(self.__testing['time']))
self.__testing['time'] += 1
process = Popen(command, stdout=PIPE, stderr=PIPE)
stderr = process.communicate()[1]
ignore = [lambda x: x.startswith("Warning: could not determine case")]
if len(stderr) is not 0:
for rule in ignore:
if not rule(stderr):
raise SnapshotError(stderr)
# close the temp snapshot filesystem
temp_snapshot_fs.close()
def run_install(self):
args = self.args
console = self.console
installed = []
install_package = args.package
install_select = package_select = self.call('package.select', package=install_package)
install_notes = package_select['notes']
if package_select['version'] is None:
raise CommandError("no install candidate for '{}', run 'moya-pm list' to see available packages".format(install_package))
package_name = package_select['name']
install_version = versioning.Version(package_select['version'])
filename = package_select['md5']
download_url = package_select['download']
package_filename = download_url.rsplit('/', 1)[-1]
libs = []
output_fs = fsopendir(args.output)
force = args.force
installed_libs = {}
archive = None
if not args.download:
try:
application = WSGIApplication(self.location, args.settings, disable_autoreload=True)
archive = application.archive
if archive is None:
console.text('unable to load project, use the --force switch to force installation')
return -1
except Exception as e:
if not args.force:
console.exception(e)
console.text('unable to load project, use the --force switch to force installation')
return -1
else:
libs = [(lib.long_name, lib.version, lib.install_location)
for lib in archive.libs.values() if lib.long_name == package_name]
installed_libs = archive.libs.copy()
if not force:
for name, version, location in libs:
if name == package_name:
if version > install_version:
if not args.force:
raise CommandError("a newer version ({}) is already installed, use --force to force installation".format(version))
elif install_version == version:
if not args.force:
raise CommandError("version {} is already installed, use --force to force installation".format(version))
else:
if not args.upgrade:
raise CommandError("an older version ({}) is installed, use --upgrade to force upgrade".format(version))
force = True
username = self.settings.get('upload', 'username', None)
password = self.settings.get('upload', 'password', None)
if username and password:
auth = (username, password)
else:
auth = None
install_app = args.app or package_name.split('.')[-1]
packages = dependencies.gather_dependencies(self.rpc,
install_app,
args.mount,
install_package,
console,
no_deps=args.no_deps)
if not args.no_add:
for package_name, (app_name, mount, package_select) in packages.items():
if package_select['version'] is None:
raise CommandError("no install candidate for required package '{}', run 'moya-pm list {}' to see available packages".format(package_name, package_name))
download_temp_fs = TempFS()
for package_name, (app_name, mount, package_select) in packages.items():
package_name = package_select['name']
install_version = versioning.Version(package_select['version'])
filename = "{}-{}.{}".format(package_name, install_version, package_select['md5'])
download_url = package_select['download']
package_filename = download_url.rsplit('/', 1)[-1]
with download_temp_fs.open(filename, 'wb') as package_file:
checksum = downloader.download(download_url,
package_file,
console=console,
auth=auth,
verify_ssl=False,
msg="requesting {name}=={version}".format(**package_select))
if checksum != package_select['md5']:
raise CommandError("md5 checksum of download doesn't match server! download={}, server={}".format(checksum, package_select['md5']))
if args.download:
with fsopendir(args.download) as dest_fs:
fs.utils.copyfile(download_temp_fs, filename, dest_fs, package_filename)
if args.download:
return 0
changed_server_xml = False
for package_name, (app_name, mount, package_select) in packages.items():
package_name = package_select['name']
install_version = versioning.Version(package_select['version'])
filename = "{}-{}.{}".format(package_name, install_version, package_select['md5'])
download_url = package_select['download']
package_filename = download_url.rsplit('/', 1)[-1]
install_location = relativefrom(self.location, pathjoin(self.location, args.output, package_select['name']))
package_select['location'] = install_location
with download_temp_fs.open(filename, 'rb') as package_file:
with ZipFS(package_file, 'r') as package_fs:
with output_fs.makeopendir(package_select['name']) as lib_fs:
#if not lib_fs.isdirempty('/') and not force:
# raise CommandError("install directory is not empty, use --force to erase and overwrite")
fs.utils.remove_all(lib_fs, '/')
fs.utils.copydir(package_fs, lib_fs)
installed.append((package_select, mount))
if not args.no_add and archive:
server_xml = archive.cfg.get('project', 'startup')
changed_server_xml =\
installer.install(project_path=self.location,
server_xml_location=archive.cfg.get('project', 'location'),
server_xml=server_xml,
server_name=application.server_ref,
lib_path=install_location,
lib_name=package_name,
app_name=app_name,
mount=mount)
table = []
for _package, mount in installed:
table.append([Cell("{name}=={version}".format(**_package), fg="magenta", bold=True),
Cell(_package['location'], fg="blue", bold=True),
Cell(mount or '', fg="cyan", bold=True)])
if table:
console.table(table, ['package', 'location', 'mount'])
if install_notes:
console.table([[install_notes]], ['{} v{} release notes'.format(install_select['name'], install_select['version'])])
if changed_server_xml:
console.text("moya-pm modified '{}' -- please check changes".format(server_xml), fg="green", bold="yes")
from fs import open_fs
from fs.tempfs import TempFS
# from fs_s3fs import S3FS
# Generate temp files to upload:
tempfs = TempFS()
for num in range(0, 100):
with tempfs.open('file' + str(num), 'w') as fi:
fi.write("I AM A FILE\n" * num)
# Connect to S3
# s3fs = S3FS(
# 'default',
# aws_access_key_id="minio",
# aws_secret_access_key="minio123",
# endpoint_url="http://*****:*****@default/?endpoint_url=http://localhost:9009')
# Print some info for us to use.
count = len(s3fs.listdir('/'))
print("File Count in S3: {}".format(count))
count = len(tempfs.listdir('/'))
print("File Count in tempfs: {}".format(count))
# Upload the files to S3
files = tempfs.listdir('/')
for fi in files:
