def setUp(self):
self.venv = tempfile.mkdtemp()
self.env = {}
current_venv = None
for k, v in os.environ.iteritems():
if k == 'VIRTUAL_ENV':
current_venv = v
continue
elif ('VIRTUALENV' in k or
'VIRTUAL_ENV' in k or
k == 'PATH'):
continue
self.env[k] = v
path = os.environ['PATH']
if current_venv and current_venv in path:
m = re.search('(.*)%s[^:]*:?(.*)' % current_venv, path)
if m:
path = ':'.join([e for e in m.groups() if e])
self.env['PATH'] = path
shell('virtualenv {venv}'.format(venv=self.venv),
env=self.env, exit_on_fail=False)
self.steel = 'python ' + os.path.abspath(
os.path.splitext(steelscript.commands.steel.__file__)[0]) + '.py'
def _git_repo(self):
# check if git enabled for this project
try:
shell(cmd='git status', cwd=settings.PROJECT_ROOT, allow_fail=True)
return True
except ShellFailed:
return False
def _git_repo(self):
# check if git enabled for this project
try:
shell(cmd='git status', cwd=settings.PROJECT_ROOT, allow_fail=True)
return True
except ShellFailed:
return False
def setUp(self):
self.venv = tempfile.mkdtemp()
self.env = {}
current_venv = None
for k, v in os.environ.iteritems():
if k == 'VIRTUAL_ENV':
current_venv = v
continue
elif ('VIRTUALENV' in k or 'VIRTUAL_ENV' in k or k == 'PATH'):
continue
self.env[k] = v
path = os.environ['PATH']
if current_venv and current_venv in path:
m = re.search('(.*)%s[^:]*:?(.*)' % current_venv, path)
if m:
path = ':'.join([e for e in m.groups() if e])
self.env['PATH'] = path
shell('virtualenv {venv}'.format(venv=self.venv),
env=self.env,
exit_on_fail=False)
self.steel = 'python ' + os.path.abspath(
os.path.splitext(steelscript.commands.steel.__file__)[0]) + '.py'
def main(self):
cwd = os.getcwd()
if not os.path.exists('manage.py'):
console('This command must be run inside the project directory.')
return
shell('python manage.py reload --trace',
msg='Reloading app framework reports',
cwd=cwd)
def main(self):
cwd = os.getcwd()
if not os.path.exists('manage.py'):
console('This command must be run inside the project directory.')
return
shell('python manage.py reload --trace',
msg='Reloading app framework reports',
cwd=cwd)
def main(self):
cwd = os.getcwd()
if not os.path.exists('manage.py'):
console('This command must be run inside the project directory to initialize.')
return
shell('python manage.py initialize --trace',
msg='Initializing project using default settings',
cwd=cwd)
def shell(cls, cmd):
if cmd.startswith('steel' or cmd.startswith(cls.steel)):
opts=' --loglevel debug --logfile -'
else:
opts=''
return shell('{cmd}{opts}'.format(cmd=cmd, opts=opts),
exit_on_fail=False, save_output=True)
def shell(cls, cmd):
if cmd.startswith('steel' or cmd.startswith(cls.steel)):
opts=' --loglevel debug --logfile -'
else:
opts=''
return shell('{cmd}{opts}'.format(cmd=cmd, opts=opts),
exit_on_fail=False, save_output=True)
def run_table(*args, **kwargs):
# combine base arguments
argstr = ' '.join(args)
# split out criteria
criteria, options = process_criteria(kwargs)
critargs = ' '.join('--criteria=%s:%s' % (k, v)
for k, v in criteria.iteritems())
argstr = '%s %s' % (argstr, critargs)
# format remaining keyword args
kws = []
for k, v in kwargs.iteritems():
if v.lower() in ('true', ''):
# handle empty or True attrs as command-line flags
kws.append('--%s' % k)
else:
kws.append('--%s=%s' % (k, v))
kws = ' '.join(kws)
cmd = 'python manage.py %s %s' % (argstr, kws)
logger.debug('running command: %s' % cmd)
try:
results = shell(cmd, cwd=os.getcwd(), save_output=True,
allow_fail=False, exit_on_fail=False, log_output=False)
except ShellFailed as e:
logger.error('Error processing table. Error code: %s, '
'stdout results: %s' % (e.returncode, e.output))
def run_table(*args, **kwargs):
# combine base arguments
argstr = ' '.join(args)
# split out criteria
criteria, options = process_criteria(kwargs)
critargs = ' '.join('--criteria=%s:%s' % (k, v)
for k, v in criteria.iteritems())
argstr = '%s %s' % (argstr, critargs)
# format remaining keyword args
kws = []
for k, v in kwargs.iteritems():
if v.lower() in ('true', ''):
# handle empty or True attrs as command-line flags
kws.append('--%s' % k)
else:
kws.append('--%s=%s' % (k, v))
kws = ' '.join(kws)
cmd = 'python manage.py %s %s' % (argstr, kws)
logger.debug('running command: %s' % cmd)
try:
results = shell(cmd, cwd=os.getcwd(), save_output=True,
allow_fail=False, exit_on_fail=False, log_output=False)
except ShellFailed as e:
logger.error('Error processing table. Error code: %s, '
'stdout results: %s' % (e.returncode, e.output))
def main(self):
cwd = os.getcwd()
if not os.path.exists('manage.py'):
console('This command must be run inside the project directory.')
return
if self.options.interactive:
yn = prompt_yn('This will delete and re-initialize the database from '
'scratch. There is no undo.\nAre you sure?', False)
if not yn:
console('Aborting.')
sys.exit()
shell('python manage.py reset_appfwk --force --trace',
msg='Resetting project database',
cwd=cwd)
def get(self, request, namespace, report_slug):
report = get_object_or_404(Report, namespace=namespace,
slug=report_slug)
from ansi2html import Ansi2HTMLConverter
conv = Ansi2HTMLConverter(inline=True, dark_bg=False)
ansi = shell('git diff --color %s' % report.filepath, save_output=True)
html = conv.convert(ansi, full=False)
if (not html and shell('git ls-files %s' % report.filepath,
save_output=True)):
html = 'No changes.'
else:
html = 'File not committed to git repository.'
return render_to_response('editdiff.html',
{'report': report,
'diffhtml': html},
context_instance=RequestContext(request))
def get(self, request, namespace, report_slug):
report = get_object_or_404(Report, namespace=namespace,
slug=report_slug)
from ansi2html import Ansi2HTMLConverter
conv = Ansi2HTMLConverter(inline=True, dark_bg=False)
ansi = shell('git diff --color %s' % report.filepath, save_output=True)
html = conv.convert(ansi, full=False)
if (not html and shell('git ls-files %s' % report.filepath,
save_output=True)):
html = 'No changes.'
else:
html = 'File not committed to git repository.'
return render_to_response('editdiff.html',
{'report': report,
'diffhtml': html},
context_instance=RequestContext(request))
def shell(self, cmd):
cmd = cmd.replace('vsteel', self.steel)
if cmd.startswith('steel' or cmd.startswith(self.steel)):
opts = ' --loglevel debug --logfile -'
else:
opts = ''
logger.info("shell command: {cmd}{opts}".format(cmd=cmd, opts=opts))
return shell('. {venv}/bin/activate; {cmd}{opts}'
.format(venv=self.venv, cmd=cmd, opts=opts),
env=self.env, exit_on_fail=False, save_output=True)
def shell(self, cmd):
cmd = cmd.replace('vsteel', self.steel)
if cmd.startswith('steel' or cmd.startswith(self.steel)):
opts = ' --loglevel debug --logfile -'
else:
opts = ''
logger.info("shell command: {cmd}{opts}".format(cmd=cmd, opts=opts))
return shell('. {venv}/bin/activate; {cmd}{opts}'
.format(venv=self.venv, cmd=cmd, opts=opts),
env=self.env, exit_on_fail=False, save_output=True)
def create_debug_zipfile(no_summary=False):
""" Collects logfiles and system info into a zipfile for download/email
`no_summary` indicates whether to include system information from
the helper script `steel about` as part of the
zipped package. Default is to include the file.
"""
# setup correct timezone based on admin settings
admin = AppfwkUser.objects.filter(is_superuser=True)[0]
tz = pytz.timezone(admin.timezone)
current_tz = os.environ['TZ']
try:
# save TZ to environment for zip to use
os.environ['TZ'] = str(tz)
# if zlib is available, then let's compress the files
# otherwise we will just append them like a tarball
try:
import zlib
compression = zipfile.ZIP_DEFLATED
except ImportError:
compression = zipfile.ZIP_STORED
# setup the name, correct timezone, and open the zipfile
now = datetime_to_seconds(datetime.now(tz))
archive_name = os.path.join(settings.PROJECT_ROOT,
'debug-%d.zip' % now)
myzip = zipfile.ZipFile(archive_name, 'w', compression=compression)
try:
# find all of the usual logfiles
logging.debug('zipping log files ...')
for fname in find_logs():
debug_fileinfo(fname)
myzip.write(fname)
if not no_summary:
logging.debug('running about script')
response = shell('steel about -v', save_output=True)
logging.debug('zipping about script')
myzip.writestr('system_summary.txt', response)
finally:
myzip.close()
finally:
# return env to its prior state
os.environ['TZ'] = current_tz
return archive_name
def create_debug_zipfile(no_summary=False):
""" Collects logfiles and system info into a zipfile for download/email
`no_summary` indicates whether to include system information from
the helper script `steel about` as part of the
zipped package. Default is to include the file.
"""
# setup correct timezone based on admin settings
admin = AppfwkUser.objects.filter(is_superuser=True)[0]
tz = pytz.timezone(admin.timezone)
current_tz = os.environ['TZ']
try:
# save TZ to environment for zip to use
os.environ['TZ'] = str(tz)
# if zlib is available, then let's compress the files
# otherwise we will just append them like a tarball
try:
import zlib
compression = zipfile.ZIP_DEFLATED
except ImportError:
compression = zipfile.ZIP_STORED
# setup the name, correct timezone, and open the zipfile
now = datetime_to_seconds(datetime.now(tz))
archive_name = os.path.join(settings.PROJECT_ROOT, 'debug-%d.zip' % now)
myzip = zipfile.ZipFile(archive_name, 'w', compression=compression)
try:
# find all of the usual logfiles
logging.debug('zipping log files ...')
for fname in find_logs():
debug_fileinfo(fname)
myzip.write(fname)
if not no_summary:
logging.debug('running about script')
response = shell('steel about -v', save_output=True)
logging.debug('zipping about script')
myzip.writestr('system_summary.txt', response)
finally:
myzip.close()
finally:
# return env to its prior state
os.environ['TZ'] = current_tz
return archive_name
def initialize_git(self, dirpath):
"""If git installed, initialize project folder as new repo.
"""
try:
check_git()
except ShellFailed:
return False
# we have git, lets make a repo
shell('git init', msg='Initializing project as git repo', cwd=dirpath)
shell('git add .', msg=None, cwd=dirpath)
shell('git commit -a -m "Initial commit."',
msg='Creating initial git commit',
cwd=dirpath)
shell('git tag -a 0.0.1 -m 0.0.1',
msg='Tagging as release 0.0.1',
cwd=dirpath)
return True
def initialize_git(self, dirpath):
"""If git installed, initialize project folder as new repo.
"""
try:
check_git()
except ShellFailed:
return False
# we have git, lets make a repo
shell('git init', msg='Initializing project as git repo',
cwd=dirpath)
shell('git add .',
msg=None,
cwd=dirpath)
shell('git commit -a -m "Initial commit."',
msg='Creating initial git commit',
cwd=dirpath)
shell('git tag -a 0.0.1 -m 0.0.1',
msg='Tagging as release 0.0.1',
cwd=dirpath)
return True
def initialize_git(self, dirpath):
"""Initialize project folder as new repo, if git installed."""
try:
check_git()
except ShellFailed:
return
# we have git, lets make a repo
shell('git init', msg='Initializing project as git repo', cwd=dirpath)
fname = os.path.join(dirpath, '.gitignore')
with open(fname, 'w') as f:
f.write(GITIGNORE)
shell('git add .', msg=None, cwd=dirpath)
shell('git commit -a -m "Initial commit."',
msg='Creating initial git commit',
cwd=dirpath)
def initialize_git(self, dirpath):
"""Initialize project folder as new repo, if git installed."""
try:
check_git()
except ShellFailed:
return
# we have git, lets make a repo
shell('git init', msg='Initializing project as git repo',
cwd=dirpath)
fname = os.path.join(dirpath, '.gitignore')
with open(fname, 'w') as f:
f.write(GITIGNORE)
shell('git add .',
msg=None,
cwd=dirpath)
shell('git commit -a -m "Initial commit."',
msg='Creating initial git commit',
cwd=dirpath)
def initialize_project(self, dirpath):
shell('python manage.py initialize -v 3 --trace',
msg='Initializing project with default settings',
cwd=dirpath)
def get_offline_js(self, dirpath):
console("Downloading offline JavaScript files...")
offline_js_dir = os.path.join(dirpath, 'offline')
self.mkdir(offline_js_dir)
failedurls = set()
offline_files = settings.OFFLINE_JS_FILES + settings.OFFLINE_CSS_FILES
for url, dirname in offline_files:
filename = url.rsplit('/', 1)[1]
console("Downloading {0}... ".format(url), newline=False)
connectionfailed = False
try:
r = requests.get(url, stream=True, allow_redirects=True)
except requests.exceptions.Timeout:
console("failed: request timed out.".format(filename))
connectionfailed = True
except requests.exceptions.ConnectionError as e:
console("failed with connection error: {0}".format(e))
connectionfailed = True
if connectionfailed:
failedurls.add(url)
elif r.status_code != requests.codes.ok:
console("failed with HTTP status code {0}.".format(
filename, r.status_code))
failedurls.add(url)
else:
if dirname is not None:
f = tempfile.NamedTemporaryFile(delete=False)
downloadpath = f.name
else:
downloadpath = os.path.join(offline_js_dir, filename)
f = open(downloadpath, 'w')
for chunk in r:
f.write(chunk)
f.close()
console("success.")
# If dirname is not None, that means the file is a zip or tar
# archive and should be extracted to that subdirectory.
if dirname is not None:
finaldir = os.path.join(offline_js_dir, dirname)
console("Extracting to " + finaldir + "... ",
newline=False)
os.mkdir(finaldir)
try:
# when original url gets redirected to the cloud,
# the zip file would be moved to the middle
# hence search for string of '.zip' followed by
# Non-alphanumeric letters
if r.url.endswith('zip') or \
re.search('.zip[^a-zA-Z\d]', r.url):
# Unzip into temporary dir, then move the contents
# of the outermost dir where we want. (With tar we
# can just use --strip-components 1.)
unzipdir = tempfile.mkdtemp()
shell("unzip {0} -d {1}".format(
downloadpath, unzipdir))
shell("mv -v {0}/*/* {1}".format(
unzipdir, finaldir))
shell("rm -rf {0}".format(unzipdir))
else: # Not a zip, assume tarball.
self.mkdir(finaldir)
shell(("tar xvf {0} --strip-components 1 "
"--directory {1}").format(
downloadpath, finaldir))
except Exception as e:
# This will probably be a ShellFailed exception, but
# we need to clean up the file no matter what.
raise e
finally:
os.remove(downloadpath)
console("success.")
if failedurls:
console("Warning: the following offline JavaScript files failed "
"to download. To complete this installation, you must "
"manually download these files to " + offline_js_dir + ".")
for url, dirname in settings.OFFLINE_JS_FILES:
if url in failedurls:
console(" {0}".format(url))
if dirname is not None:
console(
" (this file is an archive -- extract to %s)" %
os.path.join(offline_js_dir, dirname))
else:
console("Done.")
def collectreports(self, dirpath):
shell('python manage.py collectreports -v 3 --trace',
msg='Collecting default reports',
cwd=dirpath)
def initialize_project(self, dirpath):
shell('python manage.py initialize -v 3 --trace',
msg='Initializing project with default settings',
cwd=dirpath)
def get_offline_js(self, dirpath):
console("Downloading offline JavaScript files...")
offline_js_dir = os.path.join(dirpath, 'offline')
self.mkdir(offline_js_dir)
failedurls = set()
offline_files = settings.OFFLINE_JS_FILES + settings.OFFLINE_CSS_FILES
for url, dirname in offline_files:
filename = url.rsplit('/', 1)[1]
console("Downloading {0}... ".format(url), newline=False)
connectionfailed = False
try:
r = requests.get(url, stream=True, allow_redirects=True)
except requests.exceptions.Timeout:
console("failed: request timed out.".format(filename))
connectionfailed = True
except requests.exceptions.ConnectionError as e:
console("failed with connection error: {0}".format(e))
connectionfailed = True
if connectionfailed:
failedurls.add(url)
elif r.status_code != requests.codes.ok:
console("failed with HTTP status code {0}.".format(filename,
r.status_code))
failedurls.add(url)
else:
if dirname is not None:
f = tempfile.NamedTemporaryFile(delete=False)
downloadpath = f.name
else:
downloadpath = os.path.join(offline_js_dir, filename)
f = open(downloadpath, 'w')
for chunk in r:
f.write(chunk)
f.close()
console("success.")
# If dirname is not None, that means the file is a zip or tar
# archive and should be extracted to that subdirectory.
if dirname is not None:
finaldir = os.path.join(offline_js_dir, dirname)
console("Extracting to " + finaldir + "... ",
newline=False)
os.mkdir(finaldir)
try:
# when original url gets redirected to the cloud,
# the zip file would be moved to the middle
# hence search for string of '.zip' followed by
# Non-alphanumeric letters
if r.url.endswith('zip') or \
re.search('.zip[^a-zA-Z\d]', r.url):
# Unzip into temporary dir, then move the contents
# of the outermost dir where we want. (With tar we
# can just use --strip-components 1.)
unzipdir = tempfile.mkdtemp()
shell("unzip {0} -d {1}".format(downloadpath,
unzipdir))
shell("mv -v {0}/*/* {1}".format(unzipdir,
finaldir))
shell("rm -rf {0}".format(unzipdir))
else: # Not a zip, assume tarball.
self.mkdir(finaldir)
shell(("tar xvf {0} --strip-components 1 "
"--directory {1}").format(downloadpath,
finaldir))
except Exception as e:
# This will probably be a ShellFailed exception, but
# we need to clean up the file no matter what.
raise e
finally:
os.remove(downloadpath)
console("success.")
if failedurls:
console("Warning: the following offline JavaScript files failed "
"to download. To complete this installation, you must "
"manually download these files to " + offline_js_dir + ".")
for url, dirname in settings.OFFLINE_JS_FILES:
if url in failedurls:
console(" {0}".format(url))
if dirname is not None:
console(" (this file is an archive -- extract to %s)"
% os.path.join(offline_js_dir, dirname))
else:
console("Done.")
def collectreports(self, dirpath):
shell('python manage.py collectreports -v 3 --trace',
msg='Collecting default reports',
cwd=dirpath)
def main(self):
options = self.options
interactive = not options.non_interactive
if options.wave:
if options.name and options.name != 'wave':
self.parser.error("Name may not be specified for the sample "
"WaveGenerator plugin")
sys.exit(1)
options.name = 'wave'
options.title = 'Sample WaveGenerator'
options.description = 'Generate sine and cosine waves'
options.author = 'Riverbed'
options.author_email = '*****@*****.**'
else:
# Ask questions
if options.name:
if not re.match('^[a-z0-9_]+$', options.name):
self.parser.error('Invalid name: please use only '
'lowercase letters, numbers, and '
'underscores.\n')
else:
done = False
while not done:
options.name = prompt(
'Give a simple name for your plugin (a-z, 0-9, _)')
if not re.match('^[a-z0-9_]+$', options.name):
self.parser.error('Invalid name: please use only '
'lowercase letters, numbers, and '
'underscores.\n')
else:
done = True
if not options.title and interactive:
options.title = prompt('Give your plugin a title', default='')
if not options.description and interactive:
options.description = prompt('Briefly describe your plugin',
default='')
if not options.author and interactive:
options.author = prompt("Author's name", default='')
if not options.author_email and interactive:
options.author_email = prompt("Author's email", default='')
options.Name = options.name[0:1].upper() + options.name[1:]
if not options.title:
options.title = options.name
which = 'wave' if options.wave else '__plugin__'
basedir = os.path.join(
os.path.dirname(steelscript.appfwk.commands.__file__),
'data', 'steelscript-' + which)
targetbasedir = os.path.join(os.path.abspath(options.dir),
'steelscript-' + options.name)
for (dir, subdirs, files) in os.walk(basedir):
targetdir = dir.replace(basedir, targetbasedir)
targetdir = targetdir.replace(which, options.name)
if dir == basedir:
if os.path.exists(targetdir):
self.parser.error(('Target directory already exists: '
'{0}').format(targetdir))
os.mkdir(targetdir.format(which=options.name))
for f in files:
if ( f.endswith('~') or
f.endswith('.pyc') or
f.endswith('#')):
continue
srcfile = os.path.join(dir, f)
dstfile = os.path.join(targetdir,
(f.replace('.py.in', '.py')
.replace(which, options.name)))
process_file(srcfile, dstfile, vars(options))
print('Writing: {dst}'.format(dst=dstfile))
shell('(cd {dir}; python setup.py develop )'.format(dir=targetbasedir))
write_relver = True
if not self.options.nogit:
if self.initialize_git(targetbasedir):
write_relver = False
if write_relver:
relver = open(os.path.join(targetbasedir, 'RELEASE-VERSION'), 'w')
relver.write('0.0.1')
relver.close()
def main(self):
options = self.options
interactive = not options.non_interactive
if options.wave:
if options.name and options.name != 'wave':
self.parser.error("Name may not be specified for the sample "
"WaveGenerator plugin")
sys.exit(1)
options.name = 'wave'
options.title = 'Sample WaveGenerator'
options.description = 'Generate sine and cosine waves'
options.author = 'Riverbed'
options.author_email = '*****@*****.**'
else:
# Ask questions
if options.name:
if not re.match('^[a-z0-9_]+$', options.name):
self.parser.error('Invalid name: please use only '
'lowercase letters, numbers, and '
'underscores.\n')
else:
done = False
while not done:
options.name = prompt(
'Give a simple name for your plugin (a-z, 0-9, _)')
if not re.match('^[a-z0-9_]+$', options.name):
self.parser.error('Invalid name: please use only '
'lowercase letters, numbers, and '
'underscores.\n')
else:
done = True
if not options.title and interactive:
options.title = prompt('Give your plugin a title', default='')
if not options.description and interactive:
options.description = prompt('Briefly describe your plugin',
default='')
if not options.author and interactive:
options.author = prompt("Author's name", default='')
if not options.author_email and interactive:
options.author_email = prompt("Author's email", default='')
options.Name = options.name[0:1].upper() + options.name[1:]
if not options.title:
options.title = options.name
which = 'wave' if options.wave else '__plugin__'
basedir = os.path.join(
os.path.dirname(steelscript.appfwk.commands.__file__), 'data',
'steelscript-' + which)
targetbasedir = os.path.join(os.path.abspath(options.dir),
'steelscript-' + options.name)
for (dir, subdirs, files) in os.walk(basedir):
targetdir = dir.replace(basedir, targetbasedir)
targetdir = targetdir.replace(which, options.name)
if dir == basedir:
if os.path.exists(targetdir):
self.parser.error(('Target directory already exists: '
'{0}').format(targetdir))
os.mkdir(targetdir.format(which=options.name))
for f in files:
if (f.endswith('~') or f.endswith('.pyc') or f.endswith('#')):
continue
srcfile = os.path.join(dir, f)
dstfile = os.path.join(
targetdir,
(f.replace('.py.in', '.py').replace(which, options.name)))
process_file(srcfile, dstfile, vars(options))
print('Writing: {dst}'.format(dst=dstfile))
shell('(cd {dir}; python setup.py develop )'.format(dir=targetbasedir))
write_relver = True
if not self.options.nogit:
if self.initialize_git(targetbasedir):
write_relver = False
if write_relver:
relver = open(os.path.join(targetbasedir, 'RELEASE-VERSION'), 'w')
relver.write('0.0.1')
relver.close()
