def setUp(self): self.venv = tempfile.mkdtemp() self.env = {} current_venv = None for k, v in os.environ.iteritems(): if k == 'VIRTUAL_ENV': current_venv = v continue elif ('VIRTUALENV' in k or 'VIRTUAL_ENV' in k or k == 'PATH'): continue self.env[k] = v path = os.environ['PATH'] if current_venv and current_venv in path: m = re.search('(.*)%s[^:]*:?(.*)' % current_venv, path) if m: path = ':'.join([e for e in m.groups() if e]) self.env['PATH'] = path shell('virtualenv {venv}'.format(venv=self.venv), env=self.env, exit_on_fail=False) self.steel = 'python ' + os.path.abspath( os.path.splitext(steelscript.commands.steel.__file__)[0]) + '.py'
def _git_repo(self): # check if git enabled for this project try: shell(cmd='git status', cwd=settings.PROJECT_ROOT, allow_fail=True) return True except ShellFailed: return False
def main(self): cwd = os.getcwd() if not os.path.exists('manage.py'): console('This command must be run inside the project directory.') return shell('python manage.py reload --trace', msg='Reloading app framework reports', cwd=cwd)
def main(self): cwd = os.getcwd() if not os.path.exists('manage.py'): console('This command must be run inside the project directory to initialize.') return shell('python manage.py initialize --trace', msg='Initializing project using default settings', cwd=cwd)
def shell(cls, cmd): if cmd.startswith('steel' or cmd.startswith(cls.steel)): opts=' --loglevel debug --logfile -' else: opts='' return shell('{cmd}{opts}'.format(cmd=cmd, opts=opts), exit_on_fail=False, save_output=True)
def run_table(*args, **kwargs): # combine base arguments argstr = ' '.join(args) # split out criteria criteria, options = process_criteria(kwargs) critargs = ' '.join('--criteria=%s:%s' % (k, v) for k, v in criteria.iteritems()) argstr = '%s %s' % (argstr, critargs) # format remaining keyword args kws = [] for k, v in kwargs.iteritems(): if v.lower() in ('true', ''): # handle empty or True attrs as command-line flags kws.append('--%s' % k) else: kws.append('--%s=%s' % (k, v)) kws = ' '.join(kws) cmd = 'python manage.py %s %s' % (argstr, kws) logger.debug('running command: %s' % cmd) try: results = shell(cmd, cwd=os.getcwd(), save_output=True, allow_fail=False, exit_on_fail=False, log_output=False) except ShellFailed as e: logger.error('Error processing table. Error code: %s, ' 'stdout results: %s' % (e.returncode, e.output))
def main(self): cwd = os.getcwd() if not os.path.exists('manage.py'): console('This command must be run inside the project directory.') return if self.options.interactive: yn = prompt_yn('This will delete and re-initialize the database from ' 'scratch. There is no undo.\nAre you sure?', False) if not yn: console('Aborting.') sys.exit() shell('python manage.py reset_appfwk --force --trace', msg='Resetting project database', cwd=cwd)
def get(self, request, namespace, report_slug): report = get_object_or_404(Report, namespace=namespace, slug=report_slug) from ansi2html import Ansi2HTMLConverter conv = Ansi2HTMLConverter(inline=True, dark_bg=False) ansi = shell('git diff --color %s' % report.filepath, save_output=True) html = conv.convert(ansi, full=False) if (not html and shell('git ls-files %s' % report.filepath, save_output=True)): html = 'No changes.' else: html = 'File not committed to git repository.' return render_to_response('editdiff.html', {'report': report, 'diffhtml': html}, context_instance=RequestContext(request))
def shell(self, cmd): cmd = cmd.replace('vsteel', self.steel) if cmd.startswith('steel' or cmd.startswith(self.steel)): opts = ' --loglevel debug --logfile -' else: opts = '' logger.info("shell command: {cmd}{opts}".format(cmd=cmd, opts=opts)) return shell('. {venv}/bin/activate; {cmd}{opts}' .format(venv=self.venv, cmd=cmd, opts=opts), env=self.env, exit_on_fail=False, save_output=True)
def create_debug_zipfile(no_summary=False): """ Collects logfiles and system info into a zipfile for download/email `no_summary` indicates whether to include system information from the helper script `steel about` as part of the zipped package. Default is to include the file. """ # setup correct timezone based on admin settings admin = AppfwkUser.objects.filter(is_superuser=True)[0] tz = pytz.timezone(admin.timezone) current_tz = os.environ['TZ'] try: # save TZ to environment for zip to use os.environ['TZ'] = str(tz) # if zlib is available, then let's compress the files # otherwise we will just append them like a tarball try: import zlib compression = zipfile.ZIP_DEFLATED except ImportError: compression = zipfile.ZIP_STORED # setup the name, correct timezone, and open the zipfile now = datetime_to_seconds(datetime.now(tz)) archive_name = os.path.join(settings.PROJECT_ROOT, 'debug-%d.zip' % now) myzip = zipfile.ZipFile(archive_name, 'w', compression=compression) try: # find all of the usual logfiles logging.debug('zipping log files ...') for fname in find_logs(): debug_fileinfo(fname) myzip.write(fname) if not no_summary: logging.debug('running about script') response = shell('steel about -v', save_output=True) logging.debug('zipping about script') myzip.writestr('system_summary.txt', response) finally: myzip.close() finally: # return env to its prior state os.environ['TZ'] = current_tz return archive_name
def initialize_git(self, dirpath): """If git installed, initialize project folder as new repo. """ try: check_git() except ShellFailed: return False # we have git, lets make a repo shell('git init', msg='Initializing project as git repo', cwd=dirpath) shell('git add .', msg=None, cwd=dirpath) shell('git commit -a -m "Initial commit."', msg='Creating initial git commit', cwd=dirpath) shell('git tag -a 0.0.1 -m 0.0.1', msg='Tagging as release 0.0.1', cwd=dirpath) return True
def initialize_git(self, dirpath): """Initialize project folder as new repo, if git installed.""" try: check_git() except ShellFailed: return # we have git, lets make a repo shell('git init', msg='Initializing project as git repo', cwd=dirpath) fname = os.path.join(dirpath, '.gitignore') with open(fname, 'w') as f: f.write(GITIGNORE) shell('git add .', msg=None, cwd=dirpath) shell('git commit -a -m "Initial commit."', msg='Creating initial git commit', cwd=dirpath)
def initialize_project(self, dirpath): shell('python manage.py initialize -v 3 --trace', msg='Initializing project with default settings', cwd=dirpath)
def get_offline_js(self, dirpath): console("Downloading offline JavaScript files...") offline_js_dir = os.path.join(dirpath, 'offline') self.mkdir(offline_js_dir) failedurls = set() offline_files = settings.OFFLINE_JS_FILES + settings.OFFLINE_CSS_FILES for url, dirname in offline_files: filename = url.rsplit('/', 1)[1] console("Downloading {0}... ".format(url), newline=False) connectionfailed = False try: r = requests.get(url, stream=True, allow_redirects=True) except requests.exceptions.Timeout: console("failed: request timed out.".format(filename)) connectionfailed = True except requests.exceptions.ConnectionError as e: console("failed with connection error: {0}".format(e)) connectionfailed = True if connectionfailed: failedurls.add(url) elif r.status_code != requests.codes.ok: console("failed with HTTP status code {0}.".format( filename, r.status_code)) failedurls.add(url) else: if dirname is not None: f = tempfile.NamedTemporaryFile(delete=False) downloadpath = f.name else: downloadpath = os.path.join(offline_js_dir, filename) f = open(downloadpath, 'w') for chunk in r: f.write(chunk) f.close() console("success.") # If dirname is not None, that means the file is a zip or tar # archive and should be extracted to that subdirectory. if dirname is not None: finaldir = os.path.join(offline_js_dir, dirname) console("Extracting to " + finaldir + "... ", newline=False) os.mkdir(finaldir) try: # when original url gets redirected to the cloud, # the zip file would be moved to the middle # hence search for string of '.zip' followed by # Non-alphanumeric letters if r.url.endswith('zip') or \ re.search('.zip[^a-zA-Z\d]', r.url): # Unzip into temporary dir, then move the contents # of the outermost dir where we want. (With tar we # can just use --strip-components 1.) unzipdir = tempfile.mkdtemp() shell("unzip {0} -d {1}".format( downloadpath, unzipdir)) shell("mv -v {0}/*/* {1}".format( unzipdir, finaldir)) shell("rm -rf {0}".format(unzipdir)) else: # Not a zip, assume tarball. self.mkdir(finaldir) shell(("tar xvf {0} --strip-components 1 " "--directory {1}").format( downloadpath, finaldir)) except Exception as e: # This will probably be a ShellFailed exception, but # we need to clean up the file no matter what. raise e finally: os.remove(downloadpath) console("success.") if failedurls: console("Warning: the following offline JavaScript files failed " "to download. To complete this installation, you must " "manually download these files to " + offline_js_dir + ".") for url, dirname in settings.OFFLINE_JS_FILES: if url in failedurls: console(" {0}".format(url)) if dirname is not None: console( " (this file is an archive -- extract to %s)" % os.path.join(offline_js_dir, dirname)) else: console("Done.")
def collectreports(self, dirpath): shell('python manage.py collectreports -v 3 --trace', msg='Collecting default reports', cwd=dirpath)
def get_offline_js(self, dirpath): console("Downloading offline JavaScript files...") offline_js_dir = os.path.join(dirpath, 'offline') self.mkdir(offline_js_dir) failedurls = set() offline_files = settings.OFFLINE_JS_FILES + settings.OFFLINE_CSS_FILES for url, dirname in offline_files: filename = url.rsplit('/', 1)[1] console("Downloading {0}... ".format(url), newline=False) connectionfailed = False try: r = requests.get(url, stream=True, allow_redirects=True) except requests.exceptions.Timeout: console("failed: request timed out.".format(filename)) connectionfailed = True except requests.exceptions.ConnectionError as e: console("failed with connection error: {0}".format(e)) connectionfailed = True if connectionfailed: failedurls.add(url) elif r.status_code != requests.codes.ok: console("failed with HTTP status code {0}.".format(filename, r.status_code)) failedurls.add(url) else: if dirname is not None: f = tempfile.NamedTemporaryFile(delete=False) downloadpath = f.name else: downloadpath = os.path.join(offline_js_dir, filename) f = open(downloadpath, 'w') for chunk in r: f.write(chunk) f.close() console("success.") # If dirname is not None, that means the file is a zip or tar # archive and should be extracted to that subdirectory. if dirname is not None: finaldir = os.path.join(offline_js_dir, dirname) console("Extracting to " + finaldir + "... ", newline=False) os.mkdir(finaldir) try: # when original url gets redirected to the cloud, # the zip file would be moved to the middle # hence search for string of '.zip' followed by # Non-alphanumeric letters if r.url.endswith('zip') or \ re.search('.zip[^a-zA-Z\d]', r.url): # Unzip into temporary dir, then move the contents # of the outermost dir where we want. (With tar we # can just use --strip-components 1.) unzipdir = tempfile.mkdtemp() shell("unzip {0} -d {1}".format(downloadpath, unzipdir)) shell("mv -v {0}/*/* {1}".format(unzipdir, finaldir)) shell("rm -rf {0}".format(unzipdir)) else: # Not a zip, assume tarball. self.mkdir(finaldir) shell(("tar xvf {0} --strip-components 1 " "--directory {1}").format(downloadpath, finaldir)) except Exception as e: # This will probably be a ShellFailed exception, but # we need to clean up the file no matter what. raise e finally: os.remove(downloadpath) console("success.") if failedurls: console("Warning: the following offline JavaScript files failed " "to download. To complete this installation, you must " "manually download these files to " + offline_js_dir + ".") for url, dirname in settings.OFFLINE_JS_FILES: if url in failedurls: console(" {0}".format(url)) if dirname is not None: console(" (this file is an archive -- extract to %s)" % os.path.join(offline_js_dir, dirname)) else: console("Done.")
def main(self): options = self.options interactive = not options.non_interactive if options.wave: if options.name and options.name != 'wave': self.parser.error("Name may not be specified for the sample " "WaveGenerator plugin") sys.exit(1) options.name = 'wave' options.title = 'Sample WaveGenerator' options.description = 'Generate sine and cosine waves' options.author = 'Riverbed' options.author_email = '*****@*****.**' else: # Ask questions if options.name: if not re.match('^[a-z0-9_]+$', options.name): self.parser.error('Invalid name: please use only ' 'lowercase letters, numbers, and ' 'underscores.\n') else: done = False while not done: options.name = prompt( 'Give a simple name for your plugin (a-z, 0-9, _)') if not re.match('^[a-z0-9_]+$', options.name): self.parser.error('Invalid name: please use only ' 'lowercase letters, numbers, and ' 'underscores.\n') else: done = True if not options.title and interactive: options.title = prompt('Give your plugin a title', default='') if not options.description and interactive: options.description = prompt('Briefly describe your plugin', default='') if not options.author and interactive: options.author = prompt("Author's name", default='') if not options.author_email and interactive: options.author_email = prompt("Author's email", default='') options.Name = options.name[0:1].upper() + options.name[1:] if not options.title: options.title = options.name which = 'wave' if options.wave else '__plugin__' basedir = os.path.join( os.path.dirname(steelscript.appfwk.commands.__file__), 'data', 'steelscript-' + which) targetbasedir = os.path.join(os.path.abspath(options.dir), 'steelscript-' + options.name) for (dir, subdirs, files) in os.walk(basedir): targetdir = dir.replace(basedir, targetbasedir) targetdir = targetdir.replace(which, options.name) if dir == basedir: if os.path.exists(targetdir): self.parser.error(('Target directory already exists: ' '{0}').format(targetdir)) os.mkdir(targetdir.format(which=options.name)) for f in files: if ( f.endswith('~') or f.endswith('.pyc') or f.endswith('#')): continue srcfile = os.path.join(dir, f) dstfile = os.path.join(targetdir, (f.replace('.py.in', '.py') .replace(which, options.name))) process_file(srcfile, dstfile, vars(options)) print('Writing: {dst}'.format(dst=dstfile)) shell('(cd {dir}; python setup.py develop )'.format(dir=targetbasedir)) write_relver = True if not self.options.nogit: if self.initialize_git(targetbasedir): write_relver = False if write_relver: relver = open(os.path.join(targetbasedir, 'RELEASE-VERSION'), 'w') relver.write('0.0.1') relver.close()
def main(self): options = self.options interactive = not options.non_interactive if options.wave: if options.name and options.name != 'wave': self.parser.error("Name may not be specified for the sample " "WaveGenerator plugin") sys.exit(1) options.name = 'wave' options.title = 'Sample WaveGenerator' options.description = 'Generate sine and cosine waves' options.author = 'Riverbed' options.author_email = '*****@*****.**' else: # Ask questions if options.name: if not re.match('^[a-z0-9_]+$', options.name): self.parser.error('Invalid name: please use only ' 'lowercase letters, numbers, and ' 'underscores.\n') else: done = False while not done: options.name = prompt( 'Give a simple name for your plugin (a-z, 0-9, _)') if not re.match('^[a-z0-9_]+$', options.name): self.parser.error('Invalid name: please use only ' 'lowercase letters, numbers, and ' 'underscores.\n') else: done = True if not options.title and interactive: options.title = prompt('Give your plugin a title', default='') if not options.description and interactive: options.description = prompt('Briefly describe your plugin', default='') if not options.author and interactive: options.author = prompt("Author's name", default='') if not options.author_email and interactive: options.author_email = prompt("Author's email", default='') options.Name = options.name[0:1].upper() + options.name[1:] if not options.title: options.title = options.name which = 'wave' if options.wave else '__plugin__' basedir = os.path.join( os.path.dirname(steelscript.appfwk.commands.__file__), 'data', 'steelscript-' + which) targetbasedir = os.path.join(os.path.abspath(options.dir), 'steelscript-' + options.name) for (dir, subdirs, files) in os.walk(basedir): targetdir = dir.replace(basedir, targetbasedir) targetdir = targetdir.replace(which, options.name) if dir == basedir: if os.path.exists(targetdir): self.parser.error(('Target directory already exists: ' '{0}').format(targetdir)) os.mkdir(targetdir.format(which=options.name)) for f in files: if (f.endswith('~') or f.endswith('.pyc') or f.endswith('#')): continue srcfile = os.path.join(dir, f) dstfile = os.path.join( targetdir, (f.replace('.py.in', '.py').replace(which, options.name))) process_file(srcfile, dstfile, vars(options)) print('Writing: {dst}'.format(dst=dstfile)) shell('(cd {dir}; python setup.py develop )'.format(dir=targetbasedir)) write_relver = True if not self.options.nogit: if self.initialize_git(targetbasedir): write_relver = False if write_relver: relver = open(os.path.join(targetbasedir, 'RELEASE-VERSION'), 'w') relver.write('0.0.1') relver.close()