def _backup(self, clear_volume=False):
JBoxVol.log_info("Backing up " + self.sessname + " at " + str(JBoxVol.BACKUP_LOC))
bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
for f in os.listdir(self.disk_path):
if f.startswith('.') and (f in ['.juliabox']):
continue
full_path = os.path.join(self.disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f))
bkup_tar.close()
os.chmod(bkup_file, 0666)
if clear_volume:
ensure_delete(self.disk_path)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is not None and JBoxVol.BACKUP_BUCKET is not None:
if plugin.push(JBoxVol.BACKUP_BUCKET, bkup_file,
metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
JBoxVol.log_info("Moved backup to S3 " + self.sessname)
def _backup(self, clear_volume=False):
JBoxVol.log_info("Backing up " + self.sessname + " at " +
str(JBoxVol.BACKUP_LOC))
bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
for f in os.listdir(self.disk_path):
if f.startswith('.') and (f in ['.juliabox']):
continue
full_path = os.path.join(self.disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f))
bkup_tar.close()
os.chmod(bkup_file, 0666)
if clear_volume:
ensure_delete(self.disk_path)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is not None and JBoxVol.BACKUP_BUCKET is not None:
if plugin.push(
JBoxVol.BACKUP_BUCKET,
bkup_file,
metadata={'backup_time':
bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
JBoxVol.log_info("Moved backup to S3 " + self.sessname)
def refresh_disk(self, mark_refreshed=True):
self.log_debug("blanking out disk at %s", self.disk_path)
ensure_delete(self.disk_path)
self.log_debug("restoring common data on disk at %s", self.disk_path)
self.restore_user_home(True)
self.setup_instance_config()
if mark_refreshed:
self.mark_refreshed()
self.log_debug("refreshed disk at %s", self.disk_path)
def refresh_disk(self, mark_refreshed=True):
self.log_debug("blanking out disk at %s", self.disk_path)
ensure_delete(self.disk_path)
self.log_debug("restoring common data on disk at %s", self.disk_path)
self.restore_user_home(True)
self.setup_instance_config()
if mark_refreshed:
self.mark_refreshed()
self.log_debug("refreshed disk at %s", self.disk_path)
def _del_unused_package_extracts(usedpkgs=None):
if usedpkgs is None:
usedpkgs = JBoxDefaultPackagesVol.BUNDLES_IN_USE
# usedpkgs is the list of volumes mounted by all containers
currdirname = os.path.basename(JBoxDefaultPackagesVol.CURRENT_BUNDLE)
for pkgdir in os.listdir(JBoxDefaultPackagesVol.FS_LOC):
dirname = os.path.basename(pkgdir)
if dirname not in usedpkgs and dirname != currdirname:
# no container uses it, delete
JBoxDefaultPackagesVol.log_info("Deleting unused packages folder %s", dirname)
ensure_delete(os.path.join(JBoxDefaultPackagesVol.FS_LOC, dirname), include_itself=True)
def _del_unused_package_extracts(usedpkgs=None):
if usedpkgs is None:
usedpkgs = JBoxDefaultPackagesVol.BUNDLES_IN_USE
# usedpkgs is the list of volumes mounted by all containers
currdirname = os.path.basename(JBoxDefaultPackagesVol.CURRENT_BUNDLE)
for pkgdir in os.listdir(JBoxDefaultPackagesVol.FS_LOC):
dirname = os.path.basename(pkgdir)
if dirname not in usedpkgs and dirname != currdirname:
# no container uses it, delete
JBoxDefaultPackagesVol.log_info("Deleting unused packages folder %s", dirname)
ensure_delete(os.path.join(JBoxDefaultPackagesVol.FS_LOC, dirname), include_itself=True)
def _unpack_config(self):
if os.path.exists(self.disk_path):
JBoxDefaultConfigVol.log_debug("Config folder exists %s. Deleting...", self.disk_path)
ensure_delete(self.disk_path, include_itself=True)
JBoxDefaultConfigVol.log_debug("Config folder deleted %s", self.disk_path)
JBoxDefaultConfigVol.log_debug("Will unpack config to %s", self.disk_path)
os.mkdir(self.disk_path)
JBoxDefaultConfigVol.log_debug("Created config folder %s", self.disk_path)
self.restore_user_home(True)
JBoxDefaultConfigVol.log_debug("Restored config files to %s", self.disk_path)
self.setup_instance_config()
JBoxDefaultConfigVol.log_debug("Setup instance config at %s", self.disk_path)
def restore_user_home(self, new_disk):
with tarfile.open(JBoxVol.USER_HOME_IMG, 'r:gz') as user_home:
if new_disk:
user_home.extractall(self.disk_path)
else:
# extract .juliabox, .ipython/README, .ipython/kernels,
# .ipython/profile_julia, .ipython/profile_default
for path in JBoxVol.USER_HOME_ESSENTIALS:
full_path = os.path.join(self.disk_path, path)
if os.path.exists(full_path):
ensure_delete(full_path, include_itself=True)
for info in user_home.getmembers():
if not JBoxVol._is_path_user_home_essential(info.name):
continue
user_home.extract(info, self.disk_path)
def restore_user_home(self, new_disk):
with tarfile.open(JBoxVol.USER_HOME_IMG, 'r:gz') as user_home:
if new_disk:
user_home.extractall(self.disk_path)
else:
# extract .juliabox, .ipython/README, .ipython/kernels,
# .ipython/profile_julia, .ipython/profile_default
for path in JBoxVol.USER_HOME_ESSENTIALS:
full_path = os.path.join(self.disk_path, path)
if os.path.exists(full_path):
ensure_delete(full_path, include_itself=True)
for info in user_home.getmembers():
if not JBoxVol._is_path_user_home_essential(info.name):
continue
user_home.extract(info, self.disk_path)
def _unpack_julia_packages():
pkg_name = os.path.basename(JBoxVol.PKG_IMG).split('.')[0]
pkgdir = os.path.join(JBoxDefaultPackagesVol.FS_LOC, pkg_name)
if os.path.exists(pkgdir):
JBoxDefaultPackagesVol.log_debug("Packages folder exists %s. Deleting...", pkgdir)
ensure_delete(pkgdir, include_itself=True)
JBoxDefaultPackagesVol.log_debug("Packages folder deleted %s", pkgdir)
JBoxDefaultPackagesVol.log_debug("Will unpack packages to %s", pkgdir)
os.mkdir(pkgdir)
JBoxDefaultPackagesVol.log_debug("Created packages folder")
# unpack the latest image from PKG_IMG
with tarfile.open(JBoxVol.PKG_IMG, 'r:gz') as pkgs:
pkgs.extractall(pkgdir)
JBoxDefaultPackagesVol.CURRENT_BUNDLE = pkg_name
JBoxDefaultPackagesVol.log_info("Current packages folder set to %s", pkgdir)
def _unpack_config(self):
if os.path.exists(self.disk_path):
JBoxDefaultConfigVol.log_debug(
"Config folder exists %s. Deleting...", self.disk_path)
ensure_delete(self.disk_path, include_itself=True)
JBoxDefaultConfigVol.log_debug("Config folder deleted %s",
self.disk_path)
JBoxDefaultConfigVol.log_debug("Will unpack config to %s",
self.disk_path)
os.mkdir(self.disk_path)
JBoxDefaultConfigVol.log_debug("Created config folder %s",
self.disk_path)
self.restore_user_home(True)
JBoxDefaultConfigVol.log_debug("Restored config files to %s",
self.disk_path)
self.setup_instance_config()
JBoxDefaultConfigVol.log_debug("Setup instance config at %s",
self.disk_path)
def _unpack_julia_packages():
pkg_name = os.path.basename(JBoxVol.PKG_IMG).split('.')[0]
pkgdir = os.path.join(JBoxDefaultPackagesVol.FS_LOC, pkg_name)
if os.path.exists(pkgdir):
JBoxDefaultPackagesVol.log_debug(
"Packages folder exists %s. Deleting...", pkgdir)
ensure_delete(pkgdir, include_itself=True)
JBoxDefaultPackagesVol.log_debug("Packages folder deleted %s",
pkgdir)
JBoxDefaultPackagesVol.log_debug("Will unpack packages to %s", pkgdir)
os.mkdir(pkgdir)
JBoxDefaultPackagesVol.log_debug("Created packages folder")
# unpack the latest image from PKG_IMG
with tarfile.open(JBoxVol.PKG_IMG, 'r:gz') as pkgs:
pkgs.extractall(pkgdir)
JBoxDefaultPackagesVol.CURRENT_BUNDLE = pkg_name
JBoxDefaultPackagesVol.log_info("Current packages folder set to %s",
pkgdir)
def _unpack_julia_packages():
pkg_name = os.path.basename(JBoxVol.PKG_IMG).split('.')[0]
pkgdir = os.path.join(JBoxDefaultPackagesVol.FS_LOC, pkg_name)
if os.path.exists(pkgdir):
JBoxDefaultPackagesVol.log_debug("Packages folder exists %s. Deleting...", pkgdir)
ensure_delete(pkgdir, include_itself=True)
JBoxDefaultPackagesVol.log_debug("Packages folder deleted %s", pkgdir)
JBoxDefaultPackagesVol.log_debug("Will unpack packages to %s", pkgdir)
os.mkdir(pkgdir)
JBoxDefaultPackagesVol.log_debug("Created packages folder")
# unpack the latest image from PKG_IMG
result = sh.tar("-xzf", JBoxVol.PKG_IMG, "-C", pkgdir)
if result.exit_code != 0:
JBoxDefaultPackagesVol.log_error("Error extracting tar file %r", result.exit_code)
raise Exception("Error extracting packages")
# with tarfile.open(JBoxVol.PKG_IMG, 'r:gz') as pkgs:
# pkgs.extractall(pkgdir)
JBoxDefaultPackagesVol.CURRENT_BUNDLE = pkg_name
JBoxDefaultPackagesVol.log_info("Current packages folder set to %s", pkgdir)
def _unpack_julia_packages():
pkg_name = os.path.basename(JBoxVol.PKG_IMG).split('.')[0]
pkgdir = os.path.join(JBoxDefaultPackagesVol.FS_LOC, pkg_name)
if os.path.exists(pkgdir):
JBoxDefaultPackagesVol.log_debug("Packages folder exists %s. Deleting...", pkgdir)
ensure_delete(pkgdir, include_itself=True)
JBoxDefaultPackagesVol.log_debug("Packages folder deleted %s", pkgdir)
JBoxDefaultPackagesVol.log_debug("Will unpack packages to %s", pkgdir)
os.mkdir(pkgdir)
JBoxDefaultPackagesVol.log_debug("Created packages folder")
# unpack the latest image from PKG_IMG
result = sh.tar("-xzf", JBoxVol.PKG_IMG, "-C", pkgdir)
if result.exit_code != 0:
JBoxDefaultPackagesVol.log_error("Error extracting tar file %r", result.exit_code)
raise Exception("Error extracting packages")
# with tarfile.open(JBoxVol.PKG_IMG, 'r:gz') as pkgs:
# pkgs.extractall(pkgdir)
JBoxDefaultPackagesVol.CURRENT_BUNDLE = pkg_name
JBoxDefaultPackagesVol.log_info("Current packages folder set to %s", pkgdir)
def refresh_disk(self):
self.log_debug("blanking out disk at %s", self.disk_path)
ensure_delete(self.disk_path)
self.log_debug("refreshed disk at %s", self.disk_path)
def refresh_disk(self):
if JBoxVol.BACKUP_LOC is not None:
self.log_debug("blanking out disk at %s", self.disk_path)
ensure_delete(self.disk_path)
self.log_debug("refreshed disk at %s", self.disk_path)
def release(self, backup=False):
ensure_delete(self.disk_path, include_itself=True)
def refresh_disk(self):
self.log_debug("blanking out disk at %s", self.disk_path)
ensure_delete(self.disk_path)
self.log_debug("refreshed disk at %s", self.disk_path)
def release(self, backup=False):
ensure_delete(self.disk_path, include_itself=True)
def refresh_disk(self):
if JBoxVol.BACKUP_LOC is not None:
self.log_debug("blanking out disk at %s", self.disk_path)
ensure_delete(self.disk_path)
self.log_debug("refreshed disk at %s", self.disk_path)
