def pull_from_bucketstore(local_file, metadata_only=False):
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is None or JBoxVol.BACKUP_BUCKET is None:
return None
return plugin.pull(JBoxVol.BACKUP_BUCKET,
local_file,
metadata_only=metadata_only)
def update_user_home_image(fetch=True):
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is None:
VolMgr.log_info("No plugin provided for bucketstore. Can not update packages and user home images")
return
home_img_dir, curr_home_img = os.path.split(JBoxVol.USER_HOME_IMG)
pkg_img_dir, curr_pkg_img = os.path.split(JBoxVol.PKG_IMG)
bucket, new_pkg_img, new_home_img = JBoxDynConfig.get_user_home_image(Compute.get_install_id())
new_home_img_path = os.path.join(home_img_dir, new_home_img)
new_pkg_img_path = os.path.join(pkg_img_dir, new_pkg_img)
updated = False
for img_path in (new_home_img_path, new_pkg_img_path):
if not os.path.exists(img_path):
if fetch:
VolMgr.log_debug("fetching new image to %s", img_path)
k = plugin.pull(bucket, img_path)
if k is not None:
VolMgr.log_debug("fetched new image")
if os.path.exists(new_home_img_path):
VolMgr.log_debug("set new home image to %s", new_home_img_path)
JBoxVol.USER_HOME_IMG = new_home_img_path
updated = True
if os.path.exists(new_pkg_img_path):
VolMgr.log_debug("set new pkg image to %s", new_pkg_img_path)
JBoxVol.PKG_IMG = new_pkg_img_path
updated = True
return updated
def _backup(self, clear_volume=False):
JBoxVol.log_info("Backing up " + self.sessname + " at " +
str(JBoxVol.BACKUP_LOC))
bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
for f in os.listdir(self.disk_path):
if f.startswith('.') and (f in ['.juliabox']):
continue
full_path = os.path.join(self.disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f))
bkup_tar.close()
os.chmod(bkup_file, 0666)
if clear_volume:
ensure_delete(self.disk_path)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is not None and JBoxVol.BACKUP_BUCKET is not None:
if plugin.push(
JBoxVol.BACKUP_BUCKET,
bkup_file,
metadata={'backup_time':
bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
JBoxVol.log_info("Moved backup to S3 " + self.sessname)
def _backup(self, clear_volume=False):
JBoxVol.log_info("Backing up " + self.sessname + " at " + str(JBoxVol.BACKUP_LOC))
bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz")
bkup_tar = tarfile.open(bkup_file, 'w:gz')
for f in os.listdir(self.disk_path):
if f.startswith('.') and (f in ['.juliabox']):
continue
full_path = os.path.join(self.disk_path, f)
bkup_tar.add(full_path, os.path.join('juser', f))
bkup_tar.close()
os.chmod(bkup_file, 0666)
if clear_volume:
ensure_delete(self.disk_path)
# Upload to S3 if so configured. Delete from local if successful.
bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is not None and JBoxVol.BACKUP_BUCKET is not None:
if plugin.push(JBoxVol.BACKUP_BUCKET, bkup_file,
metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
os.remove(bkup_file)
JBoxVol.log_info("Moved backup to S3 " + self.sessname)
def update_user_home_image(fetch=True):
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is None:
VolMgr.log_info("No plugin provided for bucketstore. Can not update packages and user home images")
return
home_img_dir, curr_home_img = os.path.split(JBoxVol.USER_HOME_IMG)
pkg_img_dir, curr_pkg_img = os.path.split(JBoxVol.PKG_IMG)
bucket, new_pkg_img, new_home_img = JBoxDynConfig.get_user_home_image(Compute.get_install_id())
new_home_img_path = os.path.join(home_img_dir, new_home_img)
new_pkg_img_path = os.path.join(pkg_img_dir, new_pkg_img)
updated = False
for img_path in (new_home_img_path, new_pkg_img_path):
if not os.path.exists(img_path):
if fetch:
VolMgr.log_debug("fetching new image to %s", img_path)
k = plugin.pull(bucket, img_path)
if k is not None:
VolMgr.log_debug("fetched new image")
if os.path.exists(new_home_img_path):
VolMgr.log_debug("set new home image to %s", new_home_img_path)
JBoxVol.USER_HOME_IMG = new_home_img_path
updated = True
if os.path.exists(new_pkg_img_path):
VolMgr.log_debug("set new pkg image to %s", new_pkg_img_path)
JBoxVol.PKG_IMG = new_pkg_img_path
updated = True
return updated
def restore(self):
sessname = unique_sessname(self.user_email)
old_sessname = esc_sessname(self.user_email)
src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz")
pull_from_bucketstore = JBoxVol.pull_from_bucketstore
mig_hndl = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_MIGRATE)
if mig_hndl and mig_hndl.should_migrate(self.user_email):
pull_from_bucketstore = mig_hndl.pull_from_bucketstore
k = pull_from_bucketstore(src) # download from S3 if exists
if not os.path.exists(src):
if old_sessname is not None:
src = os.path.join(JBoxVol.BACKUP_LOC, old_sessname + ".tar.gz")
k = pull_from_bucketstore(src) # download from S3 if exists
if not os.path.exists(src):
return
JBoxVol.log_info("Filtering out restore info from backup " + src + " to " + self.disk_path)
src_tar = tarfile.open(src, 'r:gz')
try:
perms = {}
for info in src_tar.getmembers():
if not info.name.startswith('juser/'):
continue
extract_name = info.name[6:]
if (info.type == tarfile.LNKTYPE or info.type == tarfile.SYMTYPE) and \
info.linkname.startswith('juser/'):
info.linkname = info.linkname[6:]
if info.name.startswith('juser/.'):
if JBoxVol._is_path_user_home_essential(extract_name):
continue
info.name = extract_name
if len(info.name) == 0:
continue
src_tar.extract(info, self.disk_path)
extracted_path = os.path.join(self.disk_path, extract_name)
if os.path.isdir(extracted_path) and not os.access(extracted_path, os.W_OK):
st = os.stat(extracted_path)
perms[extracted_path] = st
os.chmod(extracted_path, st.st_mode | stat.S_IWRITE)
if len(perms) > 0:
JBoxVol.log_debug("resetting permissions on %d folders", len(perms))
for extracted_path, perm in perms.iteritems():
os.chmod(extracted_path, perm)
JBoxVol.log_info("Restored backup at " + self.disk_path)
except IOError as ioe:
if ioe.errno == errno.ENOSPC:
# continue login on ENOSPC to allow user to delete files
JBoxVol.log_exception("No space left to restore backup for %s", sessname)
else:
raise
finally:
src_tar.close()
# delete local copy of backup if we have it on bucketstore
if k is not None:
os.remove(src)
def _make_alias_hostname(instance_id=None):
dns_name = CompEC2.get_instance_id() if instance_id is None else instance_id
if CompEC2.AUTOSCALE_GROUP is not None:
dns_name += ('-' + CompEC2.AUTOSCALE_GROUP)
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_DNS)
dns_name += ('.' + plugin.domain())
return dns_name
def calc_stat(user_email):
VolMgr.STATS['num_users'] += 1
sessname = unique_sessname(user_email)
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is not None:
k = plugin.pull(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True)
if k is not None:
VolMgr.STATS['loopback']['sizes'].append(k.size)
def configure():
if not EmailVerifyHandler.CONFIGURED:
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SENDMAIL)
if plugin is None:
EmailVerifyHandler.log_error("No plugin found for sending mails. Cannot send verification mail.")
EmailVerifyHandler.EMAIL_PLUGIN = plugin
EmailVerifyHandler.EMAIL_SENDER = JBoxCfg.get('user_activation')['sender']
EmailVerifyHandler.CONFIGURED = True
def calc_stat(user_email):
VolMgr.STATS["num_users"] += 1
sessname = unique_sessname(user_email)
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is not None:
k = plugin.pull(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True)
if k is not None:
VolMgr.STATS["loopback"]["sizes"].append(k.size)
def test():
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SENDMAIL)
plugin.send_email('*****@*****.**', '*****@*****.**', "test SES",
"""hello world
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt
ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in
reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit
anim id est laborum.""")
def configure():
if not EmailVerifyHandler.CONFIGURED:
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SENDMAIL)
if plugin is None:
EmailVerifyHandler.log_error(
"No plugin found for sending mails. Cannot send verification mail."
)
EmailVerifyHandler.EMAIL_PLUGIN = plugin
EmailVerifyHandler.EMAIL_SENDER = JBoxCfg.get(
'user_activation')['sender']
EmailVerifyHandler.CONFIGURED = True
def get_monitoring_plugin():
if CompGCE.MONITORING_PLUGIN == None:
CompGCE.MONITORING_PLUGIN = JBPluginCloud.jbox_get_plugin(
JBPluginCloud.JBP_MONITORING_GOOGLE)
return CompGCE.MONITORING_PLUGIN
def restore(self):
sessname = unique_sessname(self.user_email)
old_sessname = esc_sessname(self.user_email)
src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz")
pull_from_bucketstore = JBoxVol.pull_from_bucketstore
mig_hndl = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_MIGRATE)
if mig_hndl and mig_hndl.should_migrate(self.user_email):
pull_from_bucketstore = mig_hndl.pull_from_bucketstore
k = pull_from_bucketstore(src) # download from S3 if exists
if not os.path.exists(src):
if old_sessname is not None:
src = os.path.join(JBoxVol.BACKUP_LOC,
old_sessname + ".tar.gz")
k = pull_from_bucketstore(src) # download from S3 if exists
if not os.path.exists(src):
return
JBoxVol.log_info("Filtering out restore info from backup " + src +
" to " + self.disk_path)
src_tar = tarfile.open(src, 'r:gz')
try:
perms = {}
for info in src_tar.getmembers():
if not info.name.startswith('juser/'):
continue
extract_name = info.name[6:]
if (info.type == tarfile.LNKTYPE or info.type == tarfile.SYMTYPE) and \
info.linkname.startswith('juser/'):
info.linkname = info.linkname[6:]
if info.name.startswith('juser/.'):
if JBoxVol._is_path_user_home_essential(extract_name):
continue
info.name = extract_name
if len(info.name) == 0:
continue
src_tar.extract(info, self.disk_path)
extracted_path = os.path.join(self.disk_path, extract_name)
if os.path.isdir(extracted_path) and not os.access(
extracted_path, os.W_OK):
st = os.stat(extracted_path)
perms[extracted_path] = st
os.chmod(extracted_path, st.st_mode | stat.S_IWRITE)
if len(perms) > 0:
JBoxVol.log_debug("resetting permissions on %d folders",
len(perms))
for extracted_path, perm in perms.iteritems():
os.chmod(extracted_path, perm)
JBoxVol.log_info("Restored backup at " + self.disk_path)
except IOError as ioe:
if ioe.errno == errno.ENOSPC:
# continue login on ENOSPC to allow user to delete files
JBoxVol.log_exception("No space left to restore backup for %s",
sessname)
else:
raise
finally:
src_tar.close()
# delete local copy of backup if we have it on bucketstore
if k is not None:
os.remove(src)
def init():
LoggerMixin.configure()
db.configure()
VolMgr.configure()
JBoxDisk.PLUGIN = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if __name__ == "__main__":
conf_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../engine/conf'))
conf_file = os.path.join(conf_dir, 'tornado.conf')
user_conf_file = os.path.join('/jboxengine/conf', 'jbox.user')
JBoxCfg.read(conf_file, user_conf_file)
JBoxCfg.dckr = docker.Client()
LoggerMixin.configure()
db.configure()
SessContainer.configure()
VolMgr.configure()
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is None:
VolMgr.log_error("No plugin found for bucketstore")
exit(1)
ts = JBoxVol._get_user_home_timestamp()
tsstr = ts.strftime("%Y%m%d_%H%M")
VolMgr.log_debug("user_home_timestamp: %s", tsstr)
imgf, pkgf = copy_for_upload(tsstr)
copy_for_boot()
bucket = 'juliabox-user-home-templates'
VolMgr.log_debug("pushing new image files to bucketstore at: %s", bucket)
plugin.push(bucket, imgf)
def init():
LoggerMixin.configure()
db.configure()
VolMgr.configure()
JBoxDisk.PLUGIN = JBPluginCloud.jbox_get_plugin(
JBPluginCloud.JBP_BUCKETSTORE)
def _get_scaler_plugin():
if CompGCE.SCALER_PLUGIN is None:
CompGCE.SCALER_PLUGIN = JBPluginCloud.jbox_get_plugin(
JBPluginCloud.JBP_SCALER)
return CompGCE.SCALER_PLUGIN
def get_monitoring_plugin():
if CompGCE.MONITORING_PLUGIN == None:
CompGCE.MONITORING_PLUGIN = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_MONITORING_GOOGLE)
return CompGCE.MONITORING_PLUGIN
def pull_from_bucketstore(local_file, metadata_only=False):
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if plugin is None or JBoxVol.BACKUP_BUCKET is None:
return None
return plugin.pull(JBoxVol.BACKUP_BUCKET, local_file, metadata_only=metadata_only)
def get_alias_hostname():
plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_DNS)
if plugin is None:
return CompEC2.get_instance_public_hostname()
return CompEC2._make_alias_hostname()
def _get_scaler_plugin():
if CompGCE.SCALER_PLUGIN is None:
CompGCE.SCALER_PLUGIN = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SCALER)
return CompGCE.SCALER_PLUGIN