def _backup(self, clear_volume=False):
JBoxVol.log_info("Backing up " + self.sessname + " at " + str(JBoxVol.BACKUP_LOC))
bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
for f in os.listdir(self.disk_path):
if f.startswith('.') and (f in ['.julia', '.ipython']):
continue
full_path = os.path.join(self.disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f))
bkup_tar.close()
os.chmod(bkup_file, 0666)
if clear_volume:
ensure_delete(self.disk_path)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
if JBoxVol.BACKUP_BUCKET is not None:
if CloudHost.push_file_to_s3(JBoxVol.BACKUP_BUCKET, bkup_file,
metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
JBoxVol.log_info("Moved backup to S3 " + self.sessname)
def _backup(self, clear_volume=False):
JBoxVol.log_info("Backing up " + self.sessname + " at " +
str(JBoxVol.BACKUP_LOC))
bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
for f in os.listdir(self.disk_path):
if f.startswith('.') and (f in ['.julia', '.juliabox']):
continue
full_path = os.path.join(self.disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f))
bkup_tar.close()
os.chmod(bkup_file, 0666)
if clear_volume:
ensure_delete(self.disk_path)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
if JBoxVol.BACKUP_BUCKET is not None:
if CloudHost.push_file_to_s3(
JBoxVol.BACKUP_BUCKET,
bkup_file,
metadata={'backup_time':
bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
JBoxVol.log_info("Moved backup to S3 " + self.sessname)
def create_disk():
disk_id = JBoxContainer.get_unused_disk_id()
if disk_id < 0:
raise Exception("No free disk available")
disk_path = os.path.join(JBoxContainer.FS_LOC, str(disk_id))
ensure_delete(disk_path)
JBoxContainer.restore_user_home(disk_path)
JBoxContainer.setup_instance_config(disk_path)
return disk_id, disk_path
def refresh_disk(self, mark_refreshed=True):
self.log_debug("blanking out disk at %s", self.disk_path)
ensure_delete(self.disk_path)
self.log_debug("restoring common data on disk at %s", self.disk_path)
self.restore_user_home()
self.setup_instance_config()
if mark_refreshed:
self.mark_refreshed()
self.log_debug("refreshed disk at %s", self.disk_path)
def refresh_disk(self, mark_refreshed=True):
self.log_debug("blanking out disk at %s", self.disk_path)
ensure_delete(self.disk_path)
self.log_debug("restoring common data on disk at %s", self.disk_path)
self.restore_user_home(True)
self.setup_instance_config()
if mark_refreshed:
self.mark_refreshed()
self.log_debug("refreshed disk at %s", self.disk_path)
def backup(self):
JBoxContainer.log_info("Backing up " + self.debug_str() + " at " + str(JBoxContainer.BACKUP_LOC))
cname = self.get_name()
if cname is None:
return
bkup_file = os.path.join(JBoxContainer.BACKUP_LOC, cname[1:] + ".tar.gz")
if not self.is_running():
k = JBoxContainer.pull_from_s3(bkup_file, True)
bkup_file_mtime = None
if k is not None:
bkup_file_mtime = JBoxContainer.parse_iso_time(k.get_metadata('backup_time'))
elif os.path.exists(bkup_file):
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxContainer.LOCAL_TZ_OFFSET)
if bkup_file_mtime is not None:
tstart = self.time_started()
tstop = self.time_finished()
tcomp = tstart if ((tstop is None) or (tstart > tstop)) else tstop
if tcomp <= bkup_file_mtime:
JBoxContainer.log_info("Already backed up " + self.debug_str())
return
disk_ids_used = self.get_disk_ids_used()
if len(disk_ids_used) > 0:
if len(disk_ids_used) > 1:
JBoxContainer.log_info("Can not backup more than one disks per user yet. Backing up the first disk.")
elif len(disk_ids_used) == 0:
JBoxContainer.log_info("No disks to backup")
return
disk_id_used = disk_ids_used[0]
disk_path = os.path.join(JBoxContainer.FS_LOC, str(disk_id_used))
bkup_tar = tarfile.open(bkup_file, 'w:gz')
for f in os.listdir(disk_path):
if f.startswith('.') and (f in ['.julia', '.ipython']):
continue
full_path = os.path.join(disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f))
bkup_tar.close()
os.chmod(bkup_file, 0666)
ensure_delete(disk_path)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + datetime.timedelta(
seconds=JBoxContainer.LOCAL_TZ_OFFSET)
if JBoxContainer.BACKUP_BUCKET is not None:
if CloudHelper.push_file_to_s3(JBoxContainer.BACKUP_BUCKET, bkup_file,
metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
JBoxContainer.log_info("Moved backup to S3 " + self.debug_str())
def restore_user_home(self, new_disk):
with tarfile.open(JBoxVol.USER_HOME_IMG, 'r:gz') as user_home:
if new_disk:
user_home.extractall(self.disk_path)
else:
# extract .juliabox, .ipython/README, .ipython/kernels, .ipython/profile_julia, .ipython/profile_default
for path in JBoxVol.USER_HOME_ESSENTIALS:
full_path = os.path.join(self.disk_path, path)
if os.path.exists(full_path):
ensure_delete(full_path, include_itself=True)
for info in user_home.getmembers():
if not JBoxVol._is_path_user_home_essential(info.name):
continue
user_home.extract(info, self.disk_path)
def restore_user_home(self, new_disk):
with tarfile.open(JBoxVol.USER_HOME_IMG, 'r:gz') as user_home:
if new_disk:
user_home.extractall(self.disk_path)
else:
# extract .juliabox, .ipython/README, .ipython/kernels,
# .ipython/profile_julia, .ipython/profile_default, .ipython/profile_jboxjulia
for path in JBoxVol.USER_HOME_ESSENTIALS:
full_path = os.path.join(self.disk_path, path)
if os.path.exists(full_path):
ensure_delete(full_path, include_itself=True)
for info in user_home.getmembers():
if not JBoxVol._is_path_user_home_essential(info.name):
continue
user_home.extract(info, self.disk_path)
