Beispiel #1
0
 def pull_from_bucketstore(local_file, metadata_only=False):
     plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
     if plugin is None or JBoxVol.BACKUP_BUCKET is None:
         return None
     return plugin.pull(JBoxVol.BACKUP_BUCKET,
                        local_file,
                        metadata_only=metadata_only)
Beispiel #2
0
    def update_user_home_image(fetch=True):
        plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
        if plugin is None:
            VolMgr.log_info("No plugin provided for bucketstore. Can not update packages and user home images")
            return

        home_img_dir, curr_home_img = os.path.split(JBoxVol.USER_HOME_IMG)
        pkg_img_dir, curr_pkg_img = os.path.split(JBoxVol.PKG_IMG)

        bucket, new_pkg_img, new_home_img = JBoxDynConfig.get_user_home_image(Compute.get_install_id())

        new_home_img_path = os.path.join(home_img_dir, new_home_img)
        new_pkg_img_path = os.path.join(pkg_img_dir, new_pkg_img)
        updated = False
        for img_path in (new_home_img_path, new_pkg_img_path):
            if not os.path.exists(img_path):
                if fetch:
                    VolMgr.log_debug("fetching new image to %s", img_path)
                    k = plugin.pull(bucket, img_path)
                    if k is not None:
                        VolMgr.log_debug("fetched new image")

        if os.path.exists(new_home_img_path):
            VolMgr.log_debug("set new home image to %s", new_home_img_path)
            JBoxVol.USER_HOME_IMG = new_home_img_path
            updated = True

        if os.path.exists(new_pkg_img_path):
            VolMgr.log_debug("set new pkg image to %s", new_pkg_img_path)
            JBoxVol.PKG_IMG = new_pkg_img_path
            updated = True

        return updated
Beispiel #3
0
    def _backup(self, clear_volume=False):
        JBoxVol.log_info("Backing up " + self.sessname + " at " +
                         str(JBoxVol.BACKUP_LOC))

        bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz")
        bkup_tar = tarfile.open(bkup_file, 'w:gz')

        for f in os.listdir(self.disk_path):
            if f.startswith('.') and (f in ['.juliabox']):
                continue
            full_path = os.path.join(self.disk_path, f)
            bkup_tar.add(full_path, os.path.join('juser', f))
        bkup_tar.close()
        os.chmod(bkup_file, 0666)

        if clear_volume:
            ensure_delete(self.disk_path)

        # Upload to S3 if so configured. Delete from local if successful.
        bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
            datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
        plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
        if plugin is not None and JBoxVol.BACKUP_BUCKET is not None:
            if plugin.push(
                    JBoxVol.BACKUP_BUCKET,
                    bkup_file,
                    metadata={'backup_time':
                              bkup_file_mtime.isoformat()}) is not None:
                os.remove(bkup_file)
                JBoxVol.log_info("Moved backup to S3 " + self.sessname)
Beispiel #4
0
    def _backup(self, clear_volume=False):
        JBoxVol.log_info("Backing up " + self.sessname + " at " + str(JBoxVol.BACKUP_LOC))

        bkup_file = os.path.join(JBoxVol.BACKUP_LOC, self.sessname + ".tar.gz")
        bkup_tar = tarfile.open(bkup_file, 'w:gz')

        for f in os.listdir(self.disk_path):
            if f.startswith('.') and (f in ['.juliabox']):
                continue
            full_path = os.path.join(self.disk_path, f)
            bkup_tar.add(full_path, os.path.join('juser', f))
        bkup_tar.close()
        os.chmod(bkup_file, 0666)

        if clear_volume:
            ensure_delete(self.disk_path)

        # Upload to S3 if so configured. Delete from local if successful.
        bkup_file_mtime = datetime.datetime.fromtimestamp(os.path.getmtime(bkup_file), pytz.utc) + \
            datetime.timedelta(seconds=JBoxVol.LOCAL_TZ_OFFSET)
        plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
        if plugin is not None and JBoxVol.BACKUP_BUCKET is not None:
            if plugin.push(JBoxVol.BACKUP_BUCKET, bkup_file,
                           metadata={'backup_time': bkup_file_mtime.isoformat()}) is not None:
                os.remove(bkup_file)
                JBoxVol.log_info("Moved backup to S3 " + self.sessname)
Beispiel #5
0
    def update_user_home_image(fetch=True):
        plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
        if plugin is None:
            VolMgr.log_info("No plugin provided for bucketstore. Can not update packages and user home images")
            return

        home_img_dir, curr_home_img = os.path.split(JBoxVol.USER_HOME_IMG)
        pkg_img_dir, curr_pkg_img = os.path.split(JBoxVol.PKG_IMG)

        bucket, new_pkg_img, new_home_img = JBoxDynConfig.get_user_home_image(Compute.get_install_id())

        new_home_img_path = os.path.join(home_img_dir, new_home_img)
        new_pkg_img_path = os.path.join(pkg_img_dir, new_pkg_img)
        updated = False
        for img_path in (new_home_img_path, new_pkg_img_path):
            if not os.path.exists(img_path):
                if fetch:
                    VolMgr.log_debug("fetching new image to %s", img_path)
                    k = plugin.pull(bucket, img_path)
                    if k is not None:
                        VolMgr.log_debug("fetched new image")

        if os.path.exists(new_home_img_path):
            VolMgr.log_debug("set new home image to %s", new_home_img_path)
            JBoxVol.USER_HOME_IMG = new_home_img_path
            updated = True

        if os.path.exists(new_pkg_img_path):
            VolMgr.log_debug("set new pkg image to %s", new_pkg_img_path)
            JBoxVol.PKG_IMG = new_pkg_img_path
            updated = True

        return updated
Beispiel #6
0
    def restore(self):
        sessname = unique_sessname(self.user_email)
        old_sessname = esc_sessname(self.user_email)
        src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz")

        pull_from_bucketstore = JBoxVol.pull_from_bucketstore
        mig_hndl = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_MIGRATE)
        if mig_hndl and mig_hndl.should_migrate(self.user_email):
            pull_from_bucketstore = mig_hndl.pull_from_bucketstore

        k = pull_from_bucketstore(src)  # download from S3 if exists
        if not os.path.exists(src):
            if old_sessname is not None:
                src = os.path.join(JBoxVol.BACKUP_LOC, old_sessname + ".tar.gz")
                k = pull_from_bucketstore(src)  # download from S3 if exists

        if not os.path.exists(src):
            return

        JBoxVol.log_info("Filtering out restore info from backup " + src + " to " + self.disk_path)

        src_tar = tarfile.open(src, 'r:gz')
        try:
            perms = {}
            for info in src_tar.getmembers():
                if not info.name.startswith('juser/'):
                    continue
                extract_name = info.name[6:]
                if (info.type == tarfile.LNKTYPE or info.type == tarfile.SYMTYPE) and \
                        info.linkname.startswith('juser/'):
                    info.linkname = info.linkname[6:]
                if info.name.startswith('juser/.'):
                    if JBoxVol._is_path_user_home_essential(extract_name):
                        continue
                info.name = extract_name
                if len(info.name) == 0:
                    continue
                src_tar.extract(info, self.disk_path)
                extracted_path = os.path.join(self.disk_path, extract_name)
                if os.path.isdir(extracted_path) and not os.access(extracted_path, os.W_OK):
                    st = os.stat(extracted_path)
                    perms[extracted_path] = st
                    os.chmod(extracted_path, st.st_mode | stat.S_IWRITE)
            if len(perms) > 0:
                JBoxVol.log_debug("resetting permissions on %d folders", len(perms))
                for extracted_path, perm in perms.iteritems():
                    os.chmod(extracted_path, perm)
            JBoxVol.log_info("Restored backup at " + self.disk_path)
        except IOError as ioe:
            if ioe.errno == errno.ENOSPC:
                # continue login on ENOSPC to allow user to delete files
                JBoxVol.log_exception("No space left to restore backup for %s", sessname)
            else:
                raise
        finally:
            src_tar.close()
        # delete local copy of backup if we have it on bucketstore
        if k is not None:
            os.remove(src)
Beispiel #7
0
    def _make_alias_hostname(instance_id=None):
        dns_name = CompEC2.get_instance_id() if instance_id is None else instance_id
        if CompEC2.AUTOSCALE_GROUP is not None:
            dns_name += ('-' + CompEC2.AUTOSCALE_GROUP)
        plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_DNS)
        dns_name += ('.' + plugin.domain())

        return dns_name
Beispiel #8
0
    def calc_stat(user_email):
        VolMgr.STATS['num_users'] += 1
        sessname = unique_sessname(user_email)

        plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
        if plugin is not None:
            k = plugin.pull(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True)
            if k is not None:
                VolMgr.STATS['loopback']['sizes'].append(k.size)
Beispiel #9
0
    def configure():
        if not EmailVerifyHandler.CONFIGURED:
            plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SENDMAIL)
            if plugin is None:
                EmailVerifyHandler.log_error("No plugin found for sending mails. Cannot send verification mail.")
            EmailVerifyHandler.EMAIL_PLUGIN = plugin
            EmailVerifyHandler.EMAIL_SENDER = JBoxCfg.get('user_activation')['sender']

            EmailVerifyHandler.CONFIGURED = True
Beispiel #10
0
    def calc_stat(user_email):
        VolMgr.STATS["num_users"] += 1
        sessname = unique_sessname(user_email)

        plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
        if plugin is not None:
            k = plugin.pull(JBoxVol.BACKUP_BUCKET, sessname + ".tar.gz", metadata_only=True)
            if k is not None:
                VolMgr.STATS["loopback"]["sizes"].append(k.size)
Beispiel #11
0
 def test():
     plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SENDMAIL)
     plugin.send_email('*****@*****.**', '*****@*****.**', "test SES",
                       """hello world
                       Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt
                       ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
                       ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in
                       reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
                       Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit
                       anim id est laborum.""")
Beispiel #12
0
    def configure():
        if not EmailVerifyHandler.CONFIGURED:
            plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SENDMAIL)
            if plugin is None:
                EmailVerifyHandler.log_error(
                    "No plugin found for sending mails. Cannot send verification mail."
                )
            EmailVerifyHandler.EMAIL_PLUGIN = plugin
            EmailVerifyHandler.EMAIL_SENDER = JBoxCfg.get(
                'user_activation')['sender']

            EmailVerifyHandler.CONFIGURED = True
Beispiel #13
0
 def get_monitoring_plugin():
     if CompGCE.MONITORING_PLUGIN == None:
         CompGCE.MONITORING_PLUGIN = JBPluginCloud.jbox_get_plugin(
             JBPluginCloud.JBP_MONITORING_GOOGLE)
     return CompGCE.MONITORING_PLUGIN
Beispiel #14
0
    def restore(self):
        sessname = unique_sessname(self.user_email)
        old_sessname = esc_sessname(self.user_email)
        src = os.path.join(JBoxVol.BACKUP_LOC, sessname + ".tar.gz")

        pull_from_bucketstore = JBoxVol.pull_from_bucketstore
        mig_hndl = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_MIGRATE)
        if mig_hndl and mig_hndl.should_migrate(self.user_email):
            pull_from_bucketstore = mig_hndl.pull_from_bucketstore

        k = pull_from_bucketstore(src)  # download from S3 if exists
        if not os.path.exists(src):
            if old_sessname is not None:
                src = os.path.join(JBoxVol.BACKUP_LOC,
                                   old_sessname + ".tar.gz")
                k = pull_from_bucketstore(src)  # download from S3 if exists

        if not os.path.exists(src):
            return

        JBoxVol.log_info("Filtering out restore info from backup " + src +
                         " to " + self.disk_path)

        src_tar = tarfile.open(src, 'r:gz')
        try:
            perms = {}
            for info in src_tar.getmembers():
                if not info.name.startswith('juser/'):
                    continue
                extract_name = info.name[6:]
                if (info.type == tarfile.LNKTYPE or info.type == tarfile.SYMTYPE) and \
                        info.linkname.startswith('juser/'):
                    info.linkname = info.linkname[6:]
                if info.name.startswith('juser/.'):
                    if JBoxVol._is_path_user_home_essential(extract_name):
                        continue
                info.name = extract_name
                if len(info.name) == 0:
                    continue
                src_tar.extract(info, self.disk_path)
                extracted_path = os.path.join(self.disk_path, extract_name)
                if os.path.isdir(extracted_path) and not os.access(
                        extracted_path, os.W_OK):
                    st = os.stat(extracted_path)
                    perms[extracted_path] = st
                    os.chmod(extracted_path, st.st_mode | stat.S_IWRITE)
            if len(perms) > 0:
                JBoxVol.log_debug("resetting permissions on %d folders",
                                  len(perms))
                for extracted_path, perm in perms.iteritems():
                    os.chmod(extracted_path, perm)
            JBoxVol.log_info("Restored backup at " + self.disk_path)
        except IOError as ioe:
            if ioe.errno == errno.ENOSPC:
                # continue login on ENOSPC to allow user to delete files
                JBoxVol.log_exception("No space left to restore backup for %s",
                                      sessname)
            else:
                raise
        finally:
            src_tar.close()
        # delete local copy of backup if we have it on bucketstore
        if k is not None:
            os.remove(src)
Beispiel #15
0
 def init():
     LoggerMixin.configure()
     db.configure()
     VolMgr.configure()
     JBoxDisk.PLUGIN = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)

if __name__ == "__main__":
    conf_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../engine/conf'))
    conf_file = os.path.join(conf_dir, 'tornado.conf')
    user_conf_file = os.path.join('/jboxengine/conf', 'jbox.user')

    JBoxCfg.read(conf_file, user_conf_file)
    JBoxCfg.dckr = docker.Client()

    LoggerMixin.configure()
    db.configure()
    SessContainer.configure()
    VolMgr.configure()

    plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
    if plugin is None:
        VolMgr.log_error("No plugin found for bucketstore")
        exit(1)

    ts = JBoxVol._get_user_home_timestamp()
    tsstr = ts.strftime("%Y%m%d_%H%M")
    VolMgr.log_debug("user_home_timestamp: %s", tsstr)

    imgf, pkgf = copy_for_upload(tsstr)
    copy_for_boot()

    bucket = 'juliabox-user-home-templates'

    VolMgr.log_debug("pushing new image files to bucketstore at: %s", bucket)
    plugin.push(bucket, imgf)
Beispiel #17
0
 def init():
     LoggerMixin.configure()
     db.configure()
     VolMgr.configure()
     JBoxDisk.PLUGIN = JBPluginCloud.jbox_get_plugin(
         JBPluginCloud.JBP_BUCKETSTORE)
Beispiel #18
0
 def _get_scaler_plugin():
     if CompGCE.SCALER_PLUGIN is None:
         CompGCE.SCALER_PLUGIN = JBPluginCloud.jbox_get_plugin(
             JBPluginCloud.JBP_SCALER)
     return CompGCE.SCALER_PLUGIN
Beispiel #19
0
 def get_monitoring_plugin():
     if CompGCE.MONITORING_PLUGIN == None:
         CompGCE.MONITORING_PLUGIN = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_MONITORING_GOOGLE)
     return CompGCE.MONITORING_PLUGIN
Beispiel #20
0
 def pull_from_bucketstore(local_file, metadata_only=False):
     plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_BUCKETSTORE)
     if plugin is None or JBoxVol.BACKUP_BUCKET is None:
         return None
     return plugin.pull(JBoxVol.BACKUP_BUCKET, local_file, metadata_only=metadata_only)
Beispiel #21
0
 def get_alias_hostname():
     plugin = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_DNS)
     if plugin is None:
         return CompEC2.get_instance_public_hostname()
     return CompEC2._make_alias_hostname()
Beispiel #22
0
 def _get_scaler_plugin():
     if CompGCE.SCALER_PLUGIN is None:
         CompGCE.SCALER_PLUGIN = JBPluginCloud.jbox_get_plugin(JBPluginCloud.JBP_SCALER)
     return CompGCE.SCALER_PLUGIN