Пример #1
0
def cmd(mycmd, env=None, debug=False, fail_func=None):
    """Run the external |mycmd|.

    If |mycmd| is a string, then it's assumed to be a bash snippet and will
    be run through bash.  Otherwise, it's a standalone command line and will
    be run directly.
    """
    log.debug('cmd: %r', mycmd)
    sys.stdout.flush()

    if env is None:
        env = {}
    if 'BASH_ENV' not in env:
        env = env.copy()
        env['BASH_ENV'] = '/etc/spork/is/not/valid/profile.env'

    args = []
    if isinstance(mycmd, str):
        args.append(BASH_BINARY)
        if debug:
            args.append('-x')
        args.extend(['-c', mycmd])
    else:
        args.extend(mycmd)

    log.debug('args: %r', args)
    proc = Popen(args, env=env)
    ret = proc.wait()
    if ret:
        if fail_func:
            log.error('cmd(%r) exited %s; running fail_func().', args, ret)
            fail_func()
        raise CatalystError('cmd(%r) exited %s' % (args, ret),
                            print_traceback=False)
Пример #2
0
def cmd(mycmd, env=None, debug=False, fail_func=None):
	"""Run the external |mycmd|.

	If |mycmd| is a string, then it's assumed to be a bash snippet and will
	be run through bash.  Otherwise, it's a standalone command line and will
	be run directly.
	"""
	log.debug('cmd: %r', mycmd)
	sys.stdout.flush()

	if env is None:
		env = {}
	if 'BASH_ENV' not in env:
		env = env.copy()
		env['BASH_ENV'] = '/etc/spork/is/not/valid/profile.env'

	args = []
	if isinstance(mycmd, str):
		args.append(BASH_BINARY)
		if debug:
			args.append('-x')
		args.extend(['-c', mycmd])
	else:
		args.extend(mycmd)

	log.debug('args: %r', args)
	proc = Popen(args, env=env)
	ret = proc.wait()
	if ret:
		if fail_func:
			log.error('cmd(%r) exited %s; running fail_func().', args, ret)
			fail_func()
		raise CatalystError('cmd(%r) exited %s' % (args, ret),
			print_traceback=False)
Пример #3
0
def clear_dir(target,
              mode=0o755,
              chg_flags=False,
              remove=False,
              clear_nondir=True):
    '''Universal directory clearing function

	@target: string, path to be cleared or removed
	@mode: integer, desired mode to set the directory to
	@chg_flags: boolean used for FreeBSD hosts
	@remove: boolean, passed through to clear_dir()
	@return boolean
	'''
    log.debug('start: %s', target)
    if not target:
        log.debug('no target... returning')
        return False

    mystat = None
    if os.path.isdir(target) and not os.path.islink(target):
        log.notice('Emptying directory: %s', target)
        # stat the dir, delete the dir, recreate the dir and set
        # the proper perms and ownership
        try:
            log.debug('os.stat()')
            mystat = os.stat(target)
            # There's no easy way to change flags recursively in python
            if chg_flags and os.uname()[0] == "FreeBSD":
                cmd(['chflags', '-R', 'noschg', target])
            log.debug('shutil.rmtree()')
            shutil.rmtree(target)
        except Exception:
            log.error('clear_dir failed', exc_info=True)
            return False
    elif os.path.exists(target):
        if clear_nondir:
            log.debug("Clearing (unlinking) non-directory: %s", target)
            os.unlink(target)
        else:
            log.info('clear_dir failed: %s: is not a directory', target)
            return False
    else:
        log.debug("Condidtions not met to clear: %s", target)
        log.debug("                       isdir: %s", os.path.isdir(target))
        log.debug("                      islink: %s", os.path.islink(target))
        log.debug("                      exists: %s", os.path.exists(target))

    if not remove:
        log.debug('ensure_dirs()')
        ensure_dirs(target, mode=mode)
        if mystat:
            os.chown(target, mystat[ST_UID], mystat[ST_GID])
            os.chmod(target, mystat[ST_MODE])

    log.debug('DONE, returning True')
    return True
Пример #4
0
	def run(self):
		if "purgeonly" in self.settings["options"]:
			self.purge()
			return True

		if "purge" in self.settings["options"]:
			self.purge()

		success = True
		self.setup()
		log.notice('Creating %s tree snapshot %s from %s ...',
			self.settings["repo_name"], self.settings['version_stamp'],
			self.settings['portdir'])

		mytmp=self.settings["tmp_path"]
		ensure_dirs(mytmp)

		cmd(['rsync', '-a', '--no-o', '--no-g', '--delete',
			'--exclude=/packages/',
			'--exclude=/distfiles/',
			'--exclude=/local/',
			'--exclude=CVS/',
			'--exclude=.svn',
			'--exclude=.git/',
			'--filter=H_**/files/digest-*',
			self.settings['portdir'] + '/',
			mytmp + '/' + self.settings['repo_name'] + '/'],
			env=self.env)

		log.notice('Compressing %s snapshot tarball ...', self.settings["repo_name"])
		compressor = CompressMap(self.settings["compress_definitions"],
			env=self.env, default_mode=self.settings['compression_mode'],
			comp_prog=self.settings["comp_prog"])
		infodict = compressor.create_infodict(
			source=self.settings["repo_name"],
			destination=self.settings["snapshot_path"],
			basedir=mytmp,
			filename=self.settings["snapshot_path"],
			mode=self.settings["compression_mode"],
			auto_extension=True
			)
		if not compressor.compress(infodict):
			success = False
			log.error('Snapshot compression failure')
		else:
			filename = '.'.join([self.settings["snapshot_path"],
				compressor.extension(self.settings["compression_mode"])])
			log.notice('Snapshot successfully written to %s', filename)
			self.gen_contents_file(filename)
			self.gen_digest_file(filename)
		if "keepwork" not in self.settings["options"]:
			self.cleanup()
		if success:
			log.info('snapshot: complete!')
		return success
Пример #5
0
	def run(self):
		if "purgeonly" in self.settings["options"]:
			self.purge()
			return True

		if "purge" in self.settings["options"]:
			self.purge()

		success = True
		self.setup()
		log.notice('Creating %s tree snapshot %s from %s ...',
			self.settings["repo_name"], self.settings['version_stamp'],
			self.settings['portdir'])

		mytmp=self.settings["tmp_path"]
		ensure_dirs(mytmp)

		cmd(['rsync', '-a', '--no-o', '--no-g', '--delete',
			'--exclude=/packages/',
			'--exclude=/distfiles/',
			'--exclude=/local/',
			'--exclude=CVS/',
			'--exclude=.svn',
			'--filter=H_**/files/digest-*',
			self.settings['portdir'] + '/',
			mytmp + '/' + self.settings['repo_name'] + '/'],
			env=self.env)

		log.notice('Compressing %s snapshot tarball ...', self.settings["repo_name"])
		compressor = CompressMap(self.settings["compress_definitions"],
			env=self.env, default_mode=self.settings['compression_mode'],
			comp_prog=self.settings["comp_prog"])
		infodict = compressor.create_infodict(
			source=self.settings["repo_name"],
			destination=self.settings["snapshot_path"],
			basedir=mytmp,
			filename=self.settings["snapshot_path"],
			mode=self.settings["compression_mode"],
			auto_extension=True
			)
		if not compressor.compress(infodict):
			success = False
			log.error('Snapshot compression failure')
		else:
			filename = '.'.join([self.settings["snapshot_path"],
				compressor.extension(self.settings["compression_mode"])])
			log.notice('Snapshot successfully written to %s', filename)
			self.gen_contents_file(filename)
			self.gen_digest_file(filename)
		if "keepwork" not in self.settings["options"]:
			self.cleanup()
		if success:
			log.info('snapshot: complete!')
		return success
Пример #6
0
def clear_dir(target, mode=0o755, chg_flags=False, remove=False,
		clear_nondir=True):
	'''Universal directory clearing function

	@target: string, path to be cleared or removed
	@mode: integer, desired mode to set the directory to
	@chg_flags: boolean used for FreeBSD hosts
	@remove: boolean, passed through to clear_dir()
	@return boolean
	'''
	log.debug('start: %s', target)
	if not target:
		log.debug('no target... returning')
		return False

	mystat = None
	if os.path.isdir(target) and not os.path.islink(target):
		log.notice('Emptying directory: %s', target)
		# stat the dir, delete the dir, recreate the dir and set
		# the proper perms and ownership
		try:
			log.debug('os.stat()')
			mystat = os.stat(target)
			# There's no easy way to change flags recursively in python
			if chg_flags and os.uname()[0] == "FreeBSD":
				cmd(['chflags', '-R', 'noschg', target])
			log.debug('shutil.rmtree()')
			shutil.rmtree(target)
		except Exception:
			log.error('clear_dir failed', exc_info=True)
			return False
	elif os.path.exists(target):
		if clear_nondir:
			log.debug("Clearing (unlinking) non-directory: %s", target)
			os.unlink(target)
		else:
			log.info('clear_dir failed: %s: is not a directory', target)
			return False
	else:
		log.debug("Condidtions not met to clear: %s", target)
		log.debug("                       isdir: %s", os.path.isdir(target))
		log.debug("                      islink: %s", os.path.islink(target))
		log.debug("                      exists: %s", os.path.exists(target))

	if not remove:
		log.debug('ensure_dirs()')
		ensure_dirs(target, mode=mode)
		if mystat:
			os.chown(target, mystat[ST_UID], mystat[ST_GID])
			os.chmod(target, mystat[ST_MODE])

	log.debug('DONE, returning True')
	return True
Пример #7
0
 def base_dirs(self):
     if os.uname()[0] == "FreeBSD":
         # baselayout no longer creates the .keep files in proc and dev for FreeBSD as it
         # would create them too late...we need them earlier before bind mounting filesystems
         # since proc and dev are not writeable, so...create them here
         ensure_dirs(self.settings["stage_path"] + "/proc")
         ensure_dirs(self.settings["stage_path"] + "/dev")
         for f in ('/proc', '/dev'):
             f = self.settings['stage_path'] + f + '/.keep'
             if not os.path.isfile(f):
                 try:
                     fileutils.touch(f)
                 except IOError:
                     log.error('Failed to create %s', f)
Пример #8
0
	def base_dirs(self):
		if os.uname()[0] == "FreeBSD":
			# baselayout no longer creates the .keep files in proc and dev for FreeBSD as it
			# would create them too late...we need them earlier before bind mounting filesystems
			# since proc and dev are not writeable, so...create them here
			ensure_dirs(self.settings["stage_path"]+"/proc")
			ensure_dirs(self.settings["stage_path"]+"/dev")
			for f in ('/proc', '/dev'):
				f = self.settings['stage_path'] + f + '/.keep'
				if not os.path.isfile(f):
					try:
						fileutils.touch(f)
					except IOError:
						log.error('Failed to create %s', f)
Пример #9
0
	def disable(self, point):
		'''Sets the resume point 'OFF'

		@param point: string.  name of the resume point to disable
		@return boolean
		'''
		if point not in self._points:
			return True
		try:
			os.unlink(self._points[point])
			self._points.pop(point)
		except Exception as e:
			log.error('AutoResumeError: %s', e)
			return False
		return True
Пример #10
0
    def disable(self, point):
        '''Sets the resume point 'OFF'

		@param point: string.  name of the resume point to disable
		@return boolean
		'''
        if point not in self._points:
            return True
        try:
            os.unlink(self._points[point])
            self._points.pop(point)
        except Exception as e:
            log.error('AutoResumeError: %s', e)
            return False
        return True
Пример #11
0
	def run(self):
		if "purgeonly" in self.settings["options"]:
			self.purge()
			return True

		if "purge" in self.settings["options"]:
			self.purge()

		success = True
		self.setup()
		log.notice('Creating Portage tree snapshot %s from %s ...',
			self.settings['version_stamp'], self.settings['portdir'])

		mytmp=self.settings["tmp_path"]
		ensure_dirs(mytmp)

		target_snapshot = self.settings["portdir"] + "/ " + mytmp + "/%s/" % self.settings["repo_name"]
		cmd("rsync -a --no-o --no-g --delete --exclude /packages/ --exclude /distfiles/ " +
			"--exclude /local/ --exclude CVS/ --exclude .svn --filter=H_**/files/digest-* " +
			target_snapshot,
			"Snapshot failure", env=self.env)

		log.notice('Compressing Portage snapshot tarball ...')
		compressor = CompressMap(self.settings["compress_definitions"],
			env=self.env, default_mode=self.settings['compression_mode'])
		infodict = compressor.create_infodict(
			source=self.settings["repo_name"],
			destination=self.settings["snapshot_path"],
			basedir=mytmp,
			filename=self.settings["snapshot_path"],
			mode=self.settings["compression_mode"],
			auto_extension=True
			)
		if not compressor.compress(infodict):
			success = False
			log.error('Snapshot compression failure')
		else:
			filename = '.'.join([self.settings["snapshot_path"],
				compressor.extension(self.settings["compression_mode"])])
			log.notice('Snapshot successfully written to %s', filename)
			self.gen_contents_file(filename)
			self.gen_digest_file(filename)

		self.cleanup()
		if success:
			log.info('snapshot: complete!')
		return success
Пример #12
0
	def get(self, point, no_lf=True):
		'''Gets any data stored inside a resume point

		@param point: string.  name of the resume point to enable
		@return data: string of information stored, or None
		'''
		if point in self._points:
			try:
				with open(self._points[point], 'r') as myf:
					data = myf.read()
				if data and no_lf:
					data = data.replace('\n', ' ')
			except OSError as e:
				log.error('AutoResumeError: %s', e)
				return None
			return data
		return None
Пример #13
0
    def get(self, point, no_lf=True):
        '''Gets any data stored inside a resume point

		@param point: string.  name of the resume point to enable
		@return data: string of information stored, or None
		'''
        if point in self._points:
            try:
                with open(self._points[point], 'r') as myf:
                    data = myf.read()
                if data and no_lf:
                    data = data.replace('\n', ' ')
            except OSError as e:
                log.error('AutoResumeError: %s', e)
                return None
            return data
        return None
Пример #14
0
def clear_dir(target, mode=0o755, remove=False):
    '''Universal directory clearing function

    @target: string, path to be cleared or removed
    @mode: integer, desired mode to set the directory to
    @remove: boolean, passed through to clear_dir()
    @return boolean
    '''
    log.debug('start: %s', target)
    if not target:
        log.debug('no target... returning')
        return False

    mystat = None
    if os.path.isdir(target) and not os.path.islink(target):
        log.notice('Emptying directory: %s', target)
        # stat the dir, delete the dir, recreate the dir and set
        # the proper perms and ownership
        try:
            log.debug('os.stat()')
            mystat = os.stat(target)
            log.debug('shutil.rmtree()')
            shutil.rmtree(target)
        except Exception:
            log.error('clear_dir failed', exc_info=True)
            return False
    elif os.path.exists(target):
        log.debug("Clearing (unlinking) non-directory: %s", target)
        os.unlink(target)
    else:
        log.debug("Conditions not met to clear: %s", target)
        log.debug("                      isdir: %s", os.path.isdir(target))
        log.debug("                     islink: %s", os.path.islink(target))
        log.debug("                     exists: %s", os.path.exists(target))

    if not remove:
        log.debug('ensure_dirs()')
        ensure_dirs(target, mode=mode)
        if mystat:
            os.chown(target, mystat[ST_UID], mystat[ST_GID])
            os.chmod(target, mystat[ST_MODE])

    log.debug('DONE, returning True')
    return True
Пример #15
0
    def run(self):
        if self.settings['snapshot_treeish'] == 'stable':
            treeish = self.update_ebuild_repo()
        else:
            treeish = self.settings['snapshot_treeish']

        self.set_snapshot(treeish)

        git_cmd = [
            self.git, '-C', self.gitdir, 'archive', '--format=tar', treeish
        ]
        tar2sqfs_cmd = [
            command('tar2sqfs'),
            str(self.snapshot), '-q', '-f', '-j1', '-c', 'gzip'
        ]

        log.notice('Creating %s tree snapshot %s from %s',
                   self.settings['repo_name'], treeish, self.gitdir)
        log.notice('>>> ' + ' '.join([*git_cmd, '|']))
        log.notice('    ' + ' '.join(tar2sqfs_cmd))

        lockfile = self.snapshot.with_suffix('.lock')
        with write_lock(lockfile):
            git = subprocess.Popen(git_cmd,
                                   stdout=subprocess.PIPE,
                                   stderr=sys.stderr,
                                   close_fds=False)
            tar2sqfs = subprocess.Popen(tar2sqfs_cmd,
                                        stdin=git.stdout,
                                        stdout=sys.stdout,
                                        stderr=sys.stderr,
                                        close_fds=False)
            git.stdout.close()
            git.wait()
            tar2sqfs.wait()

        if tar2sqfs.returncode == 0:
            log.notice('Wrote snapshot to %s', self.snapshot)
        else:
            log.error('Failed to create snapshot')
        return tar2sqfs.returncode == 0
Пример #16
0
	def enable(self, point, data=None):
		'''Sets the resume point 'ON'

		@param point: string.  name of the resume point to enable
		@param data: string of information to store, or None
		@return boolean
		'''
		if point in self._points and not data:
			return True
		fname = pjoin(self.basedir, point)
		if data:
			with open(fname,"w") as myf:
				myf.write(data)
		else:
			try:
				fileutils.touch(fname)
				self._points[point] = fname
			except Exception as e:
				log.error('AutoResumeError: %s', e)
				return False
		return True
Пример #17
0
def move_path(src, dest):
    '''Move a source target to a new destination

    :param src: source path to move
    :param dest: destination path to move it to
    :returns: boolean
    '''
    log.debug('Start move_path(%s, %s)', src, dest)
    if os.path.isdir(src) and not os.path.islink(src):
        if os.path.exists(dest):
            log.warning('Removing existing target destination: %s', dest)
            if not clear_dir(dest, remove=True):
                return False
        log.debug('Moving source...')
        try:
            shutil.move(src, dest)
        except Exception:
            log.error('move_path failed', exc_info=True)
            return False
        return True
    return False
Пример #18
0
    def enable(self, point, data=None):
        '''Sets the resume point 'ON'

		@param point: string.  name of the resume point to enable
		@param data: string of information to store, or None
		@return boolean
		'''
        if point in self._points and not data:
            return True
        fname = pjoin(self.basedir, point)
        if data:
            with open(fname, "w") as myf:
                myf.write(data)
        else:
            try:
                fileutils.touch(fname)
                self._points[point] = fname
            except Exception as e:
                log.error('AutoResumeError: %s', e)
                return False
        return True
Пример #19
0
def move_path(src, dest):
	'''Move a source target to a new destination

	:param src: source path to move
	:param dest: destination path to move it to
	:returns: boolean
	'''
	log.debug('Start move_path(%s, %s)', src, dest)
	if os.path.isdir(src) and not os.path.islink(src):
		if os.path.exists(dest):
			log.warning('Removing existing target destination: %s', dest)
			if not clear_dir(dest, remove=True):
				return False
		log.debug('Moving source...')
		try:
			shutil.move(src, dest)
		except Exception:
			log.error('move_path failed', exc_info=True)
			return False
		return True
	return False
Пример #20
0
def cmd(mycmd, myexc="", env=None, debug=False, fail_func=None):
	if env is None:
		env = {}
	log.debug('cmd: %r', mycmd)
	sys.stdout.flush()
	args=[BASH_BINARY]
	if "BASH_ENV" not in env:
		env = env.copy()
		env["BASH_ENV"] = "/etc/spork/is/not/valid/profile.env"
	if debug:
		args.append("-x")
	args.append("-c")
	args.append(mycmd)

	log.debug('args: %r', args)
	proc = Popen(args, env=env)
	if proc.wait() != 0:
		if fail_func:
			log.error('CMD(), NON-Zero command return.  Running fail_func().')
			fail_func()
		raise CatalystError("cmd() NON-zero return value from: %s" % myexc,
			print_traceback=False)
Пример #21
0
 def clean_stage1(self):
     '''seedcache is enabled, so salvage the /tmp/stage1root,
     remove the seed chroot'''
     log.notice('Salvaging the stage1root from the chroot path ...')
     # move the self.settings["stage_path"] outside of the self.settings["chroot_path"]
     tmp_path = normpath(self.settings["storedir"] + "/tmp/" + "stage1root")
     if move_path(self.settings["stage_path"], tmp_path):
         self.remove_chroot()
         # move it to self.settings["chroot_path"]
         if not move_path(tmp_path, self.settings["chroot_path"]):
             log.error(
                 'clean_stage1 failed, see previous log messages for details'
             )
             return False
         log.notice(
             'Successfully moved and cleaned the stage1root for the seedcache'
         )
         return True
     log.error(
         'clean_stage1 failed to move the stage1root to a temporary loation'
     )
     return False
Пример #22
0
def clear_dir(target, mode=0o755, chg_flags=False, remove=False):
    '''Universal directory clearing function

	@target: string, path to be cleared or removed
	@mode: integer, desired mode to set the directory to
	@chg_flags: boolean used for FreeBSD hoosts
	@remove: boolean, passed through to clear_dir()
	@return boolean
	'''
    log.debug('start: %s', target)
    if not target:
        log.debug('no target... returning')
        return False
    if os.path.isdir(target):
        log.info('Emptying directory: %s', target)
        # stat the dir, delete the dir, recreate the dir and set
        # the proper perms and ownership
        try:
            log.debug('os.stat()')
            mystat = os.stat(target)
            # There's no easy way to change flags recursively in python
            if chg_flags and os.uname()[0] == "FreeBSD":
                os.system("chflags -R noschg " + target)
            log.debug('shutil.rmtree()')
            shutil.rmtree(target)
            if not remove:
                log.debug('ensure_dirs()')
                ensure_dirs(target, mode=mode)
                os.chown(target, mystat[ST_UID], mystat[ST_GID])
                os.chmod(target, mystat[ST_MODE])
        except Exception:
            log.error('clear_dir failed', exc_info=True)
            return False
    else:
        log.info('clear_dir failed: %s: is not a directory', target)
    log.debug('DONE, returning True')
    return True
Пример #23
0
def clear_dir(target, mode=0o755, chg_flags=False, remove=False):
	'''Universal directory clearing function

	@target: string, path to be cleared or removed
	@mode: integer, desired mode to set the directory to
	@chg_flags: boolean used for FreeBSD hoosts
	@remove: boolean, passed through to clear_dir()
	@return boolean
	'''
	log.debug('start: %s', target)
	if not target:
		log.debug('no target... returning')
		return False
	if os.path.isdir(target):
		log.info('Emptying directory: %s', target)
		# stat the dir, delete the dir, recreate the dir and set
		# the proper perms and ownership
		try:
			log.debug('os.stat()')
			mystat=os.stat(target)
			# There's no easy way to change flags recursively in python
			if chg_flags and os.uname()[0] == "FreeBSD":
				os.system("chflags -R noschg " + target)
			log.debug('shutil.rmtree()')
			shutil.rmtree(target)
			if not remove:
				log.debug('ensure_dirs()')
				ensure_dirs(target, mode=mode)
				os.chown(target, mystat[ST_UID], mystat[ST_GID])
				os.chmod(target, mystat[ST_MODE])
		except Exception:
			log.error('clear_dir failed', exc_info=True)
			return False
	else:
		log.info('clear_dir failed: %s: is not a directory', target)
	log.debug('DONE, returning True')
	return True
Пример #24
0
 def __init__(self, message, print_traceback=False):
     if message:
         log.error('CatalystError: %s', message, exc_info=print_traceback)
Пример #25
0
	def __init__(self, message, print_traceback=False):
		if message:
			log.error('CatalystError: %s', message, exc_info=print_traceback)