def pull_from_bucketstore(local_file, metadata_only=False): plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE) if plugin is None or JBoxVol.BACKUP_BUCKET is None: return None return plugin.pull(JBoxVol.BACKUP_BUCKET, local_file, metadata_only=metadata_only)
def update_user_home_image(fetch=True): plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE) if plugin is None: VolMgr.log_info("No plugin provided for bucketstore. Can not update packages and user home images") return home_img_dir, curr_home_img = os.path.split(JBoxVol.USER_HOME_IMG) pkg_img_dir, curr_pkg_img = os.path.split(JBoxVol.PKG_IMG) bucket, new_pkg_img, new_home_img = JBoxDynConfig.get_user_home_image(Compute.get_install_id()) new_home_img_path = os.path.join(home_img_dir, new_home_img) new_pkg_img_path = os.path.join(pkg_img_dir, new_pkg_img) updated = False for img_path in (new_home_img_path, new_pkg_img_path): if not os.path.exists(img_path): if fetch: VolMgr.log_debug("fetching new image to %s", img_path) k = plugin.pull(bucket, img_path) if k is not None: VolMgr.log_debug("fetched new image") if os.path.exists(new_home_img_path): VolMgr.log_debug("set new home image to %s", new_home_img_path) JBoxVol.USER_HOME_IMG = new_home_img_path updated = True if os.path.exists(new_pkg_img_path): VolMgr.log_debug("set new pkg image to %s", new_pkg_img_path) JBoxVol.PKG_IMG = new_pkg_img_path updated = True return updated
def _backup(self, clear_volume=False): JBoxVol.log_info("Backing up " + self.sessname + " at " + str(JBoxVol.BACKUP_LOC)) bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz") bkup_tar = tarfile.open(bkup_file, 'w:gz') for f in os.listdir(self.disk_path): if f.startswith('.') and (f in ['.juliabox']): continue full_path = os.path.join(self.disk_path, f) bkup_tar.add(full_path, os.path.join('juser', f)) bkup_tar.close() os.chmod(bkup_file, 0666) if clear_volume: ensure_delete(self.disk_path) # Upload to S3 if so configured. Delete from local if successful. bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \ datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET) plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE) if plugin is not None and JBoxVol.BACKUP_BUCKET is not None: if plugin.push( JBoxVol.BACKUP_BUCKET, bkup_file, metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None: os.remove(bkup_file) JBoxVol.log_info("Moved backup to S3 " + self.sessname)
def _backup(self, clear_volume=False): JBoxVol.log_info("Backing up " + self.sessname + " at " + str(JBoxVol.BACKUP_LOC)) bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz") bkup_tar = tarfile.open(bkup_file, 'w:gz') for f in os.listdir(self.disk_path): if f.startswith('.') and (f in ['.juliabox']): continue full_path = os.path.join(self.disk_path, f) bkup_tar.add(full_path, os.path.join('juser', f)) bkup_tar.close() os.chmod(bkup_file, 0666) if clear_volume: ensure_delete(self.disk_path) # Upload to S3 if so configured. Delete from local if successful. bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \ datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET) plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE) if plugin is not None and JBoxVol.BACKUP_BUCKET is not None: if plugin.push(JBoxVol.BACKUP_BUCKET, bkup_file, metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None: os.remove(bkup_file) JBoxVol.log_info("Moved backup to S3 " + self.sessname)
def restore(self): sessname = unique_sessname(self.user_email) old_sessname = esc_sessname(self.user_email) src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz") pull_from_bucketstore = JBoxVol.pull_from_bucketstore mig_hndl = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_MIGRATE) if mig_hndl and mig_hndl.should_migrate(self.user_email): pull_from_bucketstore = mig_hndl.pull_from_bucketstore k = pull_from_bucketstore(src) # download from S3 if exists if not os.path.exists(src): if old_sessname is not None: src = os.path.join(JBoxVol.BACKUP_LOC, old_sessname + ".tar.gz") k = pull_from_bucketstore(src) # download from S3 if exists if not os.path.exists(src): return JBoxVol.log_info("Filtering out restore info from backup " + src + " to " + self.disk_path) src_tar = tarfile.open(src, 'r:gz') try: perms = {} for info in src_tar.getmembers(): if not info.name.startswith('juser/'): continue extract_name = info.name[6:] if (info.type == tarfile.LNKTYPE or info.type == tarfile.SYMTYPE) and \ info.linkname.startswith('juser/'): info.linkname = info.linkname[6:] if info.name.startswith('juser/.'): if JBoxVol._is_path_user_home_essential(extract_name): continue info.name = extract_name if len(info.name) == 0: continue src_tar.extract(info, self.disk_path) extracted_path = os.path.join(self.disk_path, extract_name) if os.path.isdir(extracted_path) and not os.access(extracted_path, os.W_OK): st = os.stat(extracted_path) perms[extracted_path] = st os.chmod(extracted_path, st.st_mode | stat.S_IWRITE) if len(perms) > 0: JBoxVol.log_debug("resetting permissions on %d folders", len(perms)) for extracted_path, perm in perms.iteritems(): os.chmod(extracted_path, perm) JBoxVol.log_info("Restored backup at " + self.disk_path) except IOError as ioe: if ioe.errno == errno.ENOSPC: # continue login on ENOSPC to allow user to delete files JBoxVol.log_exception("No space left to restore backup for %s", sessname) else: raise finally: src_tar.close() # delete local copy of backup if we have it on bucketstore if k is not None: os.remove(src)
def _make_alias_hostname(instance_id=None): dns_name = CompEC2.get_instance_id() if instance_id is None else instance_id if CompEC2.AUTOSCALE_GROUP is not None: dns_name += ('-' + CompEC2.AUTOSCALE_GROUP) plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_DNS) dns_name += ('.' + plugin.domain()) return dns_name
def calc_stat(user_email): VolMgr.STATS['num_users'] += 1 sessname = unique_sessname(user_email) plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE) if plugin is not None: k = plugin.pull(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True) if k is not None: VolMgr.STATS['loopback']['sizes'].append(k.size)
def configure(): if not EmailVerifyHandler.CONFIGURED: plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SENDMAIL) if plugin is None: EmailVerifyHandler.log_error("No plugin found for sending mails. Cannot send verification mail.") EmailVerifyHandler.EMAIL_PLUGIN = plugin EmailVerifyHandler.EMAIL_SENDER = JBoxCfg.get('user_activation')['sender'] EmailVerifyHandler.CONFIGURED = True
def calc_stat(user_email): VolMgr.STATS["num_users"] += 1 sessname = unique_sessname(user_email) plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE) if plugin is not None: k = plugin.pull(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True) if k is not None: VolMgr.STATS["loopback"]["sizes"].append(k.size)
def test(): plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SENDMAIL) plugin.send_email('*****@*****.**', '*****@*****.**', "test SES", """hello world Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.""")
def configure(): if not EmailVerifyHandler.CONFIGURED: plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SENDMAIL) if plugin is None: EmailVerifyHandler.log_error( "No plugin found for sending mails. Cannot send verification mail." ) EmailVerifyHandler.EMAIL_PLUGIN = plugin EmailVerifyHandler.EMAIL_SENDER = JBoxCfg.get( 'user_activation')['sender'] EmailVerifyHandler.CONFIGURED = True
def get_monitoring_plugin(): if CompGCE.MONITORING_PLUGIN == None: CompGCE.MONITORING_PLUGIN = JBPluginCloud.jbox_get_plugin( JBPluginCloud.JBP_MONITORING_GOOGLE) return CompGCE.MONITORING_PLUGIN
def restore(self): sessname = unique_sessname(self.user_email) old_sessname = esc_sessname(self.user_email) src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz") pull_from_bucketstore = JBoxVol.pull_from_bucketstore mig_hndl = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_MIGRATE) if mig_hndl and mig_hndl.should_migrate(self.user_email): pull_from_bucketstore = mig_hndl.pull_from_bucketstore k = pull_from_bucketstore(src) # download from S3 if exists if not os.path.exists(src): if old_sessname is not None: src = os.path.join(JBoxVol.BACKUP_LOC, old_sessname + ".tar.gz") k = pull_from_bucketstore(src) # download from S3 if exists if not os.path.exists(src): return JBoxVol.log_info("Filtering out restore info from backup " + src + " to " + self.disk_path) src_tar = tarfile.open(src, 'r:gz') try: perms = {} for info in src_tar.getmembers(): if not info.name.startswith('juser/'): continue extract_name = info.name[6:] if (info.type == tarfile.LNKTYPE or info.type == tarfile.SYMTYPE) and \ info.linkname.startswith('juser/'): info.linkname = info.linkname[6:] if info.name.startswith('juser/.'): if JBoxVol._is_path_user_home_essential(extract_name): continue info.name = extract_name if len(info.name) == 0: continue src_tar.extract(info, self.disk_path) extracted_path = os.path.join(self.disk_path, extract_name) if os.path.isdir(extracted_path) and not os.access( extracted_path, os.W_OK): st = os.stat(extracted_path) perms[extracted_path] = st os.chmod(extracted_path, st.st_mode | stat.S_IWRITE) if len(perms) > 0: JBoxVol.log_debug("resetting permissions on %d folders", len(perms)) for extracted_path, perm in perms.iteritems(): os.chmod(extracted_path, perm) JBoxVol.log_info("Restored backup at " + self.disk_path) except IOError as ioe: if ioe.errno == errno.ENOSPC: # continue login on ENOSPC to allow user to delete files JBoxVol.log_exception("No space left to restore backup for %s", sessname) else: raise finally: src_tar.close() # delete local copy of backup if we have it on bucketstore if k is not None: os.remove(src)
def init(): LoggerMixin.configure() db.configure() VolMgr.configure() JBoxDisk.PLUGIN = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
if __name__ == "__main__": conf_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../engine/conf')) conf_file = os.path.join(conf_dir, 'tornado.conf') user_conf_file = os.path.join('/jboxengine/conf', 'jbox.user') JBoxCfg.read(conf_file, user_conf_file) JBoxCfg.dckr = docker.Client() LoggerMixin.configure() db.configure() SessContainer.configure() VolMgr.configure() plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE) if plugin is None: VolMgr.log_error("No plugin found for bucketstore") exit(1) ts = JBoxVol._get_user_home_timestamp() tsstr = ts.strftime("%Y%m%d_%H%M") VolMgr.log_debug("user_home_timestamp: %s", tsstr) imgf, pkgf = copy_for_upload(tsstr) copy_for_boot() bucket = 'juliabox-user-home-templates' VolMgr.log_debug("pushing new image files to bucketstore at: %s", bucket) plugin.push(bucket, imgf)
def init(): LoggerMixin.configure() db.configure() VolMgr.configure() JBoxDisk.PLUGIN = JBPluginCloud.jbox_get_plugin( JBPluginCloud.JBP_BUCKETSTORE)
def _get_scaler_plugin(): if CompGCE.SCALER_PLUGIN is None: CompGCE.SCALER_PLUGIN = JBPluginCloud.jbox_get_plugin( JBPluginCloud.JBP_SCALER) return CompGCE.SCALER_PLUGIN
def get_monitoring_plugin(): if CompGCE.MONITORING_PLUGIN == None: CompGCE.MONITORING_PLUGIN = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_MONITORING_GOOGLE) return CompGCE.MONITORING_PLUGIN
def get_alias_hostname(): plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_DNS) if plugin is None: return CompEC2.get_instance_public_hostname() return CompEC2._make_alias_hostname()
def _get_scaler_plugin(): if CompGCE.SCALER_PLUGIN is None: CompGCE.SCALER_PLUGIN = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SCALER) return CompGCE.SCALER_PLUGIN