def job(self, run: "Run", payload: dict, device: Device) -> dict:
now = controller.strip_all(controller.get_time())
source = Path.cwd() / "projects" / "migrations" / f"backup_{now}.tgz"
controller.migrate_export(
Path.cwd(), {"import_export_types": import_classes, "name": f"backup_{now}"}
)
with open_tar(source, "w:gz") as tar:
tar.add(
Path.cwd() / "projects" / "migrations" / f"backup_{now}", arcname="/"
)
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(AutoAddPolicy())
ssh_client.connect(
device.ip_address,
username=device.username,
password=device.password,
look_for_keys=False,
)
destination = f"{run.sub(run.destination_path, locals())}/backup_{now}.tgz"
run.transfer_file(ssh_client, [(source, destination)])
ssh_client.close()
if run.delete_folder:
rmtree(Path.cwd() / "projects" / "migrations" / f"backup_{now}")
if run.delete_archive:
remove(source)
return {
"success": True,
"result": f"backup stored in {destination} ({device.ip_address})",
}
def job(self, payload: dict, device: Device) -> dict:
path_backup = Path.cwd() / "logs" / "job_logs"
now = strip_all(str(datetime.now()))
path_dir = path_backup / f"logs_{now}"
source = path_backup / f"logs_{now}.tgz"
makedirs(path_dir)
for job in fetch_all("Job"):
with open(path_dir / f"{job.name}.json", "w") as log_file:
dump(job.logs, log_file)
with open_tar(source, "w:gz") as tar:
tar.add(path_dir, arcname="/")
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(AutoAddPolicy())
ssh_client.connect(
device.ip_address,
username=device.username,
password=device.password,
look_for_keys=False,
)
destination = f"{self.destination_path}/logs_{now}.tgz"
self.transfer_file(ssh_client, [(source, destination)])
ssh_client.close()
if self.delete_folder:
rmtree(path_dir)
if self.delete_archive:
remove(source)
return {
"success": True,
"result": f"logs stored in {destination} ({device.ip_address})",
}
def restore(options):
if options.files and options.backup_file is None:
raise Exception('no backup file specified')
if options.files:
if exists('campaign-data') or \
exists('simics-workspace/gold-checkpoints'):
if input('existing data will be deleted before restore '
'operation, continue? [Y/n]: ') in ('n', 'N', 'no', 'No',
'NO'):
return
if exists('campaign-data'):
rmtree('campaign-data')
if exists('simics-workspace/gold-checkpoints'):
rmtree('simics-workspace/gold-checkpoints')
print('restoring files...', end='')
stdout.flush()
with open_tar(options.backup_file, 'r:gz') as backup:
backup.extractall()
print('done')
for item in listdir('campaign-data'):
if '.sql' in item:
print('restoring database from {}...'.format(item))
restore_database(options, join('campaign-data', item))
print('database restored')
remove(join('campaign-data', item))
break
else:
print('could not find .sql file to restore')
def job(self, payload: dict, device: Device) -> dict:
now = strip_all(str(datetime.now()))
source = Path.cwd() / "migrations" / f"backup_{now}.tgz"
migrate_export(
Path.cwd(),
{
"import_export_types": list(import_properties),
"name": f"backup_{now}"
},
)
with open_tar(source, "w:gz") as tar:
tar.add(Path.cwd() / "migrations" / f"backup_{now}", arcname="/")
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(AutoAddPolicy())
ssh_client.connect(
device.ip_address,
username=device.username,
password=device.password,
look_for_keys=False,
)
destination = f"{self.destination_path}/backup_{now}.tgz"
self.transfer_file(ssh_client, [(source, destination)])
ssh_client.close()
if self.delete_folder:
rmtree(Path.cwd() / "migrations" / f"backup_{now}")
if self.delete_archive:
remove(source)
return {
"success": True,
"result": f"backup stored in {destination} ({device.ip_address})",
}
def job(self, device, _):
path_backup = Path.cwd() / 'logs' / 'job_logs'
now = strip_all(str(datetime.now()))
path_dir = path_backup / f'logs_{now}'
source = path_backup / f'logs_{now}.tgz'
makedirs(path_dir)
for job in fetch_all('Job'):
with open(path_dir / f'{job.name}.json', 'w') as log_file:
dump(job.logs, log_file)
with open_tar(source, 'w:gz') as tar:
tar.add(path_dir, arcname='/')
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(AutoAddPolicy())
ssh_client.connect(device.ip_address,
username=device.username,
password=device.password,
look_for_keys=False)
destination = f'{self.destination_path}/logs_{now}.tgz'
self.transfer_file(ssh_client, source, destination)
ssh_client.close()
if self.delete_folder:
rmtree(path_dir)
if self.delete_archive:
remove(source)
return {
'success': True,
'result': f'logs stored in {destination} ({device.ip_address})'
}
def retrieve_render(handle, folder):
"""Performs a retrieve request on the server with the given handle
The data is decompressed and written in a new subfolder inside 'folder'
Returns th parsed response"""
data = \
{
"pass": password,
"command": command,
"action": "retrieve",
"data": handle
}
#get the tar.gz
res = None
try:
res = post(server_address, data=data, **extra_params)
except Exception:
pass
payload = try_parse_res(res)
if payload["code"] >= 0:
#decompresse the gz
decompressed_file = decompress_gzip(bytearray.fromhex(payload["data"]))
file_like = BytesIO(decompressed_file)
#extract the tar
tar = open_tar(fileobj=file_like, mode="r")
tar.extractall(folder)
return payload
def job(self, device, _):
now = strip_all(str(datetime.now()))
source = Path.cwd() / 'migrations' / f'backup_{now}.tgz'
migrate_export(Path.cwd(), {
'import_export_types': list(import_properties),
'name': f'backup_{now}'
})
with open_tar(source, 'w:gz') as tar:
tar.add(
Path.cwd() / 'migrations' / f'backup_{now}',
arcname='/'
)
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(AutoAddPolicy())
ssh_client.connect(
device.ip_address,
username=device.username,
password=device.password,
look_for_keys=False
)
destination = f'{self.destination_path}/backup_{now}.tgz'
self.transfer_file(ssh_client, source, destination)
ssh_client.close()
if self.delete_folder:
rmtree(Path.cwd() / 'migrations' / f'backup_{now}')
if self.delete_archive:
remove(source)
return {
'success': True,
'result': f'backup stored in {destination} ({device.ip_address})'
}
def import_jobs(self, **kwargs: Any) -> None:
jobs = kwargs["jobs_to_import"]
path = self.path / "projects" / "exported_jobs"
for file in scandir(path / "services"):
if file.name == ".gitkeep" or file.name not in jobs:
continue
with open(file.path, "r") as instance_file:
instance = yaml.load(instance_file)
model = instance.pop("type")
factory(model, **self.objectify(model, instance))
Session.commit()
for workflow in listdir(path / "workflows"):
if workflow == ".gitkeep" or workflow not in jobs:
continue
workflow_name = workflow.split(".")[0]
with open_tar(path / "workflows" / workflow) as tar_file:
tar_file.extractall(path=path / "workflows")
for instance_type in ("jobs", "workflow", "edges"):
path_job = path / "workflows" / workflow_name / instance_type
for file in scandir(path_job):
with open(path_job / file.name, "r") as instance_file:
instance = yaml.load(instance_file)
model = instance.pop("type")
factory(model, **self.objectify(model, instance))
Session.commit()
rmtree(path / "workflows" / workflow_name)
def _extract(self):
with open_tar(self.path) as tar:
for name in tar.getnames():
path = self.base_path / name
if path.exists():
continue
logger.info(f'extracting {path}')
tar.extract(name, path=self.base_path)
def backup(options):
def traverse_directory(directory, archive=None, progress=None):
num_items = 0
for item in listdir(directory):
if isdir(join(directory, item)):
num_items += traverse_directory(join(directory, item), archive,
progress)
else:
num_items += 1
if archive is not None:
try:
archive.add(join(directory, item))
except FileNotFoundError:
pass
if progress is not None:
progress[0] += 1
progress[1].update(progress[0])
return num_items
# def backup(options):
if not exists('backups'):
mkdir('backups')
if not exists('campaign-data'):
mkdir('campaign-data')
sql_backup = 'campaign-data/{}.sql'.format(options.db_name)
print('dumping database...')
backup_database(options, sql_backup)
print('database dumped')
if options.files:
backup_name = 'backups/{}_{}'.format(
'-'.join([
'{:02}'.format(unit) for unit in datetime.now().timetuple()[:3]
]), '-'.join([
'{:02}'.format(unit)
for unit in datetime.now().timetuple()[3:6]
]))
num_items = 0
directories = ['campaign-data']
if exists('simics-workspace/gold-checkpoints'):
directories.append('simics-workspace/gold-checkpoints')
print('discovering files to archive')
for directory in directories:
num_items += traverse_directory(directory)
print('archiving files...')
with open_tar('{}.tar.gz'.format(backup_name), 'w:gz') \
as backup, ProgressBar(max_value=num_items, widgets=[
Percentage(), ' (',
SimpleProgress(format='%(value)d/%(max_value)d'), ') ', Bar(),
' ', Timer()]) as progress_bar:
progress = [0, progress_bar]
for directory in directories:
traverse_directory(directory, backup, progress)
remove(sql_backup)
print('backup complete')
def import_service(self, archive):
service_name = archive.split(".")[0]
path = self.path / "files" / "services"
with open_tar(path / archive) as tar_file:
tar_file.extractall(path=path)
status = self.migration_import(
folder="services",
name=service_name,
import_export_types=["service", "workflow_edge"],
)
rmtree(path / service_name)
return status
def tarball_data(tar_name):
"""Extract a tarball for test usage
This fixture extracts a tarball, and returns the path to the extracted
files.
:see: `tar_name`
"""
data_dir = path.join(path.dirname(__file__), 'data', 'git')
with open_tar(path.join(data_dir, tar_name + '.tar'), 'r:') as tar:
with TemporaryDirectory() as temp_dir:
tar.extractall(temp_dir)
yield str(path.join(temp_dir, tar_name))
def import_service(self, archive):
service_name = archive.split(".")[0]
path = self.path / "files" / "services"
with open_tar(path / archive) as tar_file:
tar_file.extractall(path=path)
status = self.migration_import(
folder="services",
name=service_name,
import_export_types=["service", "workflow_edge"],
skip_pool_update=True,
skip_model_update=True,
update_pools=True,
)
rmtree(path / service_name, ignore_errors=True)
return status
def tarball_data(tar_name):
"""Extract a tarball for test usage
This fixture extracts a tarball, and returns the path to the extracted
files.
:see: `tar_name`
"""
data_dir = path.join(path.dirname(path.abspath(__file__)), 'data', 'git')
with closing(open_tar(path.join(data_dir, tar_name + '.tar'))) as tar:
try:
temp_dir = mkdtemp()
tar.extractall(temp_dir)
yield str(path.join(temp_dir, tar_name))
finally:
rmtree(temp_dir)
def export_service(self, service_id):
service = db.fetch("service", id=service_id)
path = Path(self.path / "files" / "services" / service.filename)
path.mkdir(parents=True, exist_ok=True)
services = service.deep_services if service.type == "workflow" else [service]
services = [service.to_dict(export=True) for service in services]
for service_dict in services:
for relation in ("devices", "pools", "events"):
service_dict.pop(relation)
with open(path / "service.yaml", "w") as file:
yaml.dump(services, file)
if service.type == "workflow":
with open(path / "workflow_edge.yaml", "w") as file:
yaml.dump(
[edge.to_dict(export=True) for edge in service.deep_edges], file
)
with open_tar(f"{path}.tgz", "w:gz") as tar:
tar.add(path, arcname=service.filename)
rmtree(path, ignore_errors=True)
def index_dataset_basic(path_to_dataset, db_session, max_movies_rated):
"""
Load a small sub-set of the Netflix data into the database.
"""
# Loading the movies:
movies_path = os.path.join(path_to_dataset, "movie_titles.txt")
for movie in get_movies(movies_path):
db_session.add(movie)
# Loading the ratings:
training_set = os.path.join(path_to_dataset, "training_set")
if not os.path.isdir(training_set):
# The ratings haven't been extracted. Let's do it!
try:
tar_file = open_tar(training_set + ".tar")
tar_file.extractall(path_to_dataset)
except TarError, exc:
raise IndexingError("Could not open training file: %s" % exc)
finally:
def index_dataset_basic(path_to_dataset, db_session, max_movies_rated):
"""
Load a small sub-set of the Netflix data into the database.
"""
# Loading the movies:
movies_path = os.path.join(path_to_dataset, "movie_titles.txt")
for movie in get_movies(movies_path):
db_session.add(movie)
# Loading the ratings:
training_set = os.path.join(path_to_dataset, "training_set")
if not os.path.isdir(training_set):
# The ratings haven't been extracted. Let's do it!
try:
tar_file = open_tar(training_set + ".tar")
tar_file.extractall(path_to_dataset)
except TarError, exc:
raise IndexingError("Could not open training file: %s" % exc)
finally:
def tarfile(self, format, filename, content_type):
from .root.histogram import Histogram
from .combination import Combination
imgformat = "eps"
tarred_contents = StringIO()
with closing(open_tar(mode="w" + format, fileobj=tarred_contents)) as tar:
for key, context in self.resource_to_render.indexed_contexts:
if not context_renderable_as(context, imgformat):
continue
name = "/".join(map(str, key))
content = context.rendered(imgformat).content.body
info = TarInfo(name=name + "." + imgformat)
info.size = len(content)
tar.addfile(tarinfo=info, fileobj=StringIO(content))
return Response(tarred_contents.getvalue(), content_type=content_type,
content_disposition=("Content-Disposition: attachment; filename={0};"
.format(filename)))
def unpack_tar_file(directory):
"""
extract files in results.tar which is in directory into this directory
then remove the results.tar file from directory
:param directory: (string)
:return:
"""
message(mode='INFO', text=' Unpack')
tar_file = adapt_path('{}results.tar'.format(directory))
try:
chdir(directory)
tar = open_tar(tar_file)
except FileNotFoundError:
message(mode='ERROR', text='No tar file')
except Exception as err:
message(mode='ERROR', text="{}".format(err))
else:
tar.extractall()
tar.close()
remove(tar_file)
def export_job(self, job_id: str) -> None:
job = fetch("Job", id=job_id)
if job.type == "Workflow":
path = self.path / "projects" / "exported_jobs" / "workflows" / job.filename
path.mkdir(parents=True, exist_ok=True)
for instance_type in ("jobs", "workflow", "edges"):
Path(path / instance_type).mkdir(parents=True, exist_ok=True)
for sub_job in job.jobs:
with open(path / "jobs" / f"{sub_job.filename}.yaml",
"w") as file:
sub_job_as_dict = sub_job.to_dict(export=True)
for relation in ("devices", "pools", "events"):
sub_job_as_dict.pop(relation)
if sub_job.type == "Workflow":
sub_job_as_dict["type"] = "Workflow"
yaml.dump(sub_job_as_dict, file)
for edge in job.edges:
name = self.strip_all(
f"{edge.workflow}{edge.source}{edge.destination}")
with open(path / "edges" / f"{name}.yaml", "w") as file:
edge = {
**edge.to_dict(export=True), "type": "WorkflowEdge"
}
yaml.dump(edge, file)
with open(path / "workflow" / f"{job.filename}.yaml", "w") as file:
job_as_dict = job.to_dict(export=True)
for relation in ("devices", "pools", "events"):
job_as_dict.pop(relation)
yaml.dump({**job_as_dict, "type": "Workflow"}, file)
with open_tar(f"{path}.tgz", "w:gz") as tar:
tar.add(path, arcname=job.filename)
rmtree(path)
else:
path = self.path / "projects" / "exported_jobs" / "services"
with open(path / f"{job.filename}.yaml", "w") as file:
job_as_dict = job.to_dict(export=True)
for relation in ("devices", "pools", "events"):
job_as_dict.pop(relation)
yaml.dump(job_as_dict, file)
def pack_tar_file(directory):
"""
pack files with configurationShared.outputFileEndings into results.tar in directory
before that, remove old tar file if it exists
:param directory: (string)
:return:
"""
message(mode='INFO', text=' Pack')
tar_file = adapt_path('{}results.tar'.format(directory))
try:
if path.isfile(tar_file): # remove old tar file if it exists
remove(tar_file)
chdir(directory)
tar = open_tar(tar_file, 'w')
except Exception as err:
message(mode='ERROR', text="{}".format(err))
else:
for extension in outputFileEndings:
for file in listdir(directory):
if file.endswith('.{}'.format(extension)):
tar.add(file)
tar.close()
def export_service(self, service_id):
service = db.fetch("service", id=service_id)
path = Path(self.path / "files" / "services" / service.filename)
path.mkdir(parents=True, exist_ok=True)
services = service.deep_services if service.type == "workflow" else [
service
]
exclude = ("target_devices", "target_pools", "pools", "events")
services = [
service.to_dict(export=True,
private_properties=True,
exclude=exclude) for service in services
]
with open(path / "service.yaml", "w") as file:
yaml.dump(services, file)
if service.type == "workflow":
edges = [edge.to_dict(export=True) for edge in service.deep_edges]
with open(path / "workflow_edge.yaml", "w") as file:
yaml.dump(edges, file)
with open_tar(f"{path}.tgz", "w:gz") as tar:
tar.add(path, arcname=service.filename)
rmtree(path, ignore_errors=True)
return path
def tarfile(self, format, filename, content_type):
from .root.histogram import Histogram
from .combination import Combination
imgformat = "eps"
tarred_contents = StringIO()
with closing(open_tar(mode="w" + format,
fileobj=tarred_contents)) as tar:
for key, context in self.resource_to_render.indexed_contexts:
if not context_renderable_as(context, imgformat):
continue
name = "/".join(map(str, key))
content = context.rendered(imgformat).content.body
info = TarInfo(name=name + "." + imgformat)
info.size = len(content)
tar.addfile(tarinfo=info, fileobj=StringIO(content))
return Response(
tarred_contents.getvalue(),
content_type=content_type,
content_disposition=(
"Content-Disposition: attachment; filename={0};".format(
filename)))
def __init__(self, embedding_file, vocab):
self._vocab = vocab
with open_tar(embedding_file, 'r') as fin:
with closing(fin.extractfile('parameters')) as f:
params = json.loads(f.read())
self.N = params['N']
self.K = params['K']
self.mu = numpy.empty([self.N, self.K], dtype=numpy.float)
self.context = numpy.empty([self.N, self.K], dtype=numpy.float)
self.sigma = numpy.empty([2 * self.N, self.K], dtype=numpy.float)
with closing(fin.extractfile('word_mu')) as f:
for i, line in enumerate(f):
vec = line.strip().split()[1:]
self.mu[i,:] = [float(ele) for ele in vec]
with closing(fin.extractfile('mu_context')) as f:
for i, line in enumerate(f):
vec = line.strip().split()
self.context[i,:] = [float(ele) for ele in vec]
with closing(fin.extractfile('sigma')) as f:
for i, line in enumerate(f):
vec = line.strip().split()
self.sigma[i,:] = [float(ele) for ele in vec]
#!/usr/bin/env python3
from compileall import compile_dir
from os import chdir, remove, walk
from os.path import abspath, dirname, join
from tarfile import open as open_tar
top_dir = dirname(dirname(abspath(__file__)))
chdir(top_dir)
compile_dir('src', force=True, quiet=1, legacy=True)
with open_tar('drseus.tar.gz', 'w:gz') as package:
package.add('scripts/install_dependencies.sh', 'install_dependencies.sh')
package.add('scripts/setup_environment.sh', 'setup_environment.sh')
package.add('drseus.py')
package.add('README.md')
for root, dirs, files in walk(join(top_dir, 'src')):
if 'migrations' not in root and '__pycache__' not in root:
for file_ in files:
if not file_.endswith('.py'):
package.add(join(root.replace(top_dir + '/', ''), file_))
if file_.endswith('.pyc'):
remove(join(root, file_))
def _generate_state(self, sm):
with self.handle.follow(sm).make_path() as r:
with open_tar(str(r), "r") as tp:
yield tp
def get_test_filenames(test_path, force_download):
"""Return the test file names
If the test files does'nt exists, we download it
"""
uris = {'http://download.wikimedia.org/qualitywiki/latest':
[('qualitywiki-latest-stub-articles.xml', '.gz'), #~ 3.1 KB
('qualitywiki-latest-stub-meta-current.xml', '.gz'), #~ 11.0 KB
('qualitywiki-latest-stub-meta-history.xml', '.gz')], #~ 28.9 KB
'http://download.wikimedia.org/tawiki/latest':
[('tawiki-latest-stub-articles.xml', '.gz'), #~ 1.2 MB
('tawiki-latest-stub-meta-history.xml', '.gz')], #~ 7.3 MB
'http://www.w3.org/XML/Test/': [('xmlts20080205', '.tar.gz')]
}
compressed_dir_path = join(test_path, 'compressed_files')
if force_download is True:
if lfs.exists(compressed_dir_path):
print 'Remove compressed directory ', compressed_dir_path
lfs.remove(compressed_dir_path)
for names in uris.itervalues():
for (name, ext) in names:
path = join(test_path, name)
if lfs.exists(path):
print 'Remove %s file' % path
lfs.remove(path)
# test directory
if lfs.exists(test_path) is False:
lfs.make_folder(test_path)
# compressed directory
if lfs.exists(compressed_dir_path) is False:
lfs.make_folder(compressed_dir_path)
else:
lfs.open(compressed_dir_path)
test_dir_filenames = lfs.get_names(test_path)
for base_uri, names in uris.iteritems():
for (name, ext) in names:
if test_dir_filenames.count(name):
continue
compressed_dest = join(compressed_dir_path, '%s%s' % (name, ext))
# check if tarball already exists
if lfs.exists(compressed_dest) is False:
src = join(base_uri, '%s%s' % (name, ext))
print 'GET %s file' % src
dest = join(test_path, name)
if vfs.exists(src) is False:
print "%s uri does not exists" % src
continue
src_file = vfs.open(src)
# save Gzip file
compressed_dest_file = lfs.make_file(compressed_dest)
compressed_dest_file.write(src_file.read())
compressed_dest_file.close()
src_file.close()
print 'Extract file %s' % compressed_dest
# Uncompressed File Path
if name == 'xmlts20080205':
# uncompress only xmlconf.xml file
tar = open_tar(compressed_dest)
xmlconf_file = tar.extractfile('xmlconf/xmlconf.xml')
ucf_path = join(test_path, name)
ucf_file = lfs.make_file(ucf_path)
ucf_file.write(xmlconf_file.read())
ucf_file.close()
else:
# untar Gzip file
compressed_dest_file = lfs.open(compressed_dest)
gzip_file = GzipFile(compressed_dest)
ucf_path = join(test_path, name)
ucf_file = lfs.make_file(ucf_path)
ucf_file.write(gzip_file.read())
compressed_dest_file.close()
gzip_file.close()
ucf_file.close()
tests = []
# update test dir name
test_dir_filenames = lfs.get_names(test_path)
for filename in test_dir_filenames:
real_path = join(test_path, filename)
if lfs.is_file(real_path):
bytes = lfs.get_size(real_path)
tests.append((real_path, filename, bytes,
get_string_size(bytes)))
tests.sort(key=lambda x: x[2])
return tests
def results_page(request, campaign_id=None):
error_title = None
error_message = None
result_filter = None
if campaign_id is not None:
campaign = models.campaign.objects.get(id=campaign_id)
else:
campaign = None
if request.method == 'GET' and 'view_output' in request.GET and \
'view_all' not in request.GET and 'select_box' in request.GET:
result_ids = map(int, dict(request.GET)['select_box'])
results = models.result.objects.filter(
id__in=result_ids).order_by('-id')
else:
if campaign_id is not None:
campaign_items_ = campaign_items
output_file = 'campaign-data/{}/gold_{}'.format(
campaign_id, campaign.output_file)
if exists(output_file) and guess_type(output_file)[0] is not None:
output_file = True
else:
output_file = False
results = campaign.result_set.all()
else:
campaign_items_ = None
output_file = True
results = models.result.objects.all()
result_filter = filters.result(request.GET, queryset=results)
if not result_filter.qs.count() and results.count():
error_title = 'Filter Error'
error_message = 'Filter did not return any results and was ignored.'
result_filter = filters.result(None, queryset=results)
else:
results = result_filter.qs.order_by('-id')
if request.method == 'GET' and 'view_output' in request.GET:
if 'view_dut_output' in request.GET:
if 'view_download' in request.GET:
temp_file = TemporaryFile()
start = perf_counter()
with open_tar(fileobj=temp_file, mode='w:gz') as archive:
for result in results:
with BytesIO(result.dut_output.encode('utf-8')) as \
byte_file:
info = TarInfo('{}_dut_output.txt'.format(
result.id))
info.size = len(result.dut_output)
archive.addfile(info, byte_file)
print('archive created', round(perf_counter() - start, 2),
'seconds')
response = FileResponse(
temp_file, content_type='application/x-compressed')
response['Content-Disposition'] = \
'attachment; filename=dut_outputs.tar.gz'
response['Content-Length'] = temp_file.tell()
temp_file.seek(0)
return response
else:
return render(
request, 'output.html', {
'campaign': campaign,
'campaign_items': campaign_items if campaign else None,
'navigation_items': navigation_items,
'results': results,
'type': 'dut_output'
})
elif 'view_aux_output' in request.GET:
if 'view_download' in request.GET:
temp_file = TemporaryFile()
start = perf_counter()
with open_tar(fileobj=temp_file, mode='w:gz') as archive:
for result in results:
with BytesIO(result.aux_output.encode('utf-8')) as \
byte_file:
info = TarInfo('{}_aux_output.txt'.format(
result.id))
info.size = len(result.aux_output)
archive.addfile(info, byte_file)
print('archive created', round(perf_counter() - start, 2),
'seconds')
response = FileResponse(
temp_file, content_type='application/x-compressed')
response['Content-Disposition'] = \
'attachment; filename=aux_outputs.tar.gz'
response['Content-Length'] = temp_file.tell()
temp_file.seek(0)
return response
else:
return render(
request, 'output.html', {
'campaign': campaign,
'campaign_items': campaign_items if campaign else None,
'navigation_items': navigation_items,
'results': results,
'type': 'aux_output'
})
elif 'view_debugger_output' in request.GET:
if 'view_download' in request.GET:
temp_file = TemporaryFile()
start = perf_counter()
with open_tar(fileobj=temp_file, mode='w:gz') as archive:
for result in results:
with BytesIO(
result.debugger_output.encode('utf-8')) as \
byte_file:
info = TarInfo('{}_debugger_output.txt'.format(
result.id))
info.size = len(result.debugger_output)
archive.addfile(info, byte_file)
print('archive created', round(perf_counter() - start, 2),
'seconds')
response = FileResponse(
temp_file, content_type='application/x-compressed')
response['Content-Disposition'] = \
'attachment; filename=debugger_outputs.tar.gz'
response['Content-Length'] = temp_file.tell()
temp_file.seek(0)
return response
else:
return render(
request, 'output.html', {
'campaign': campaign,
'campaign_items': campaign_items if campaign else None,
'navigation_items': navigation_items,
'results': results,
'type': 'debugger_output'
})
elif 'view_output_file' in request.GET:
result_ids = []
for result in results:
if exists('campaign-data/{}/results/{}/{}'.format(
result.campaign_id, result.id,
result.campaign.output_file)):
result_ids.append(result.id)
results = models.result.objects.filter(
id__in=result_ids).order_by('-id')
if 'view_download' in request.GET:
temp_file = TemporaryFile()
start = perf_counter()
with open_tar(fileobj=temp_file, mode='w:gz') as archive:
for result in results:
archive.add(
'campaign-data/{}/results/{}/{}'.format(
result.campaign_id, result.id,
result.campaign.output_file),
'{}_{}'.format(result.id,
result.campaign.output_file))
print('archive created', round(perf_counter() - start, 2),
'seconds')
response = FileResponse(
temp_file, content_type='application/x-compressed')
response['Content-Disposition'] = \
'attachment; filename=output_files.tar.gz'
response['Content-Length'] = temp_file.tell()
temp_file.seek(0)
return response
else:
return render(
request, 'output.html', {
'campaign': campaign,
'campaign_items': campaign_items if campaign else None,
'navigation_items': navigation_items,
'results': results,
'type': 'output_file'
})
elif 'view_log_file' in request.GET:
if 'view_download' in request.GET:
temp_file = TemporaryFile()
start = perf_counter()
with open_tar(fileobj=temp_file, mode='w:gz') as archive:
for result in results:
for log_file in result.campaign.log_files:
archive.add(
'campaign-data/{}/results/{}/{}'.format(
result.campaign_id, result.id, log_file),
'{}_{}'.format(result.id, log_file))
print('archive created', round(perf_counter() - start, 2),
'seconds')
response = FileResponse(
temp_file, content_type='application/x-compressed')
response['Content-Disposition'] = \
'attachment; filename=log_files.tar.gz'
response['Content-Length'] = temp_file.tell()
temp_file.seek(0)
return response
else:
return render(
request, 'output.html', {
'campaign': campaign,
'campaign_items': campaign_items if campaign else None,
'navigation_items': navigation_items,
'results': results,
'type': 'log_file'
})
elif request.method == 'POST':
if 'new_outcome_category' in request.POST:
results.values('outcome_category').update(
outcome_category=request.POST['new_outcome_category'])
elif 'new_outcome' in request.POST:
results.values('outcome').update(
outcome=request.POST['new_outcome'])
elif 'delete' in request.POST and 'results[]' in request.POST:
result_ids = [
int(result_id) for result_id in dict(request.POST)['results[]']
]
results_to_delete = models.result.objects.filter(id__in=result_ids)
for result in results_to_delete:
if exists('campaign-data/{}/results/{}'.format(
result.campaign_id, result.id)):
rmtree('campaign-data/{}/results/{}'.format(
result.campaign_id, result.id))
results_to_delete.delete()
elif 'delete_all' in request.POST:
for result in results:
if exists('campaign-data/{}/results/{}'.format(
result.campaign_id, result.id)):
rmtree('campaign-data/{}/results/{}'.format(
result.campaign_id, result.id))
results.delete()
if campaign_id:
return redirect('/campaign/{}/results'.format(campaign_id))
else:
return redirect('/results')
result_table = tables.results(results)
RequestConfig(request, paginate={
'per_page': table_length
}).configure(result_table)
return render(
request, 'results.html', {
'campaign': campaign,
'campaign_items': campaign_items_,
'error_message': error_message,
'error_title': error_title,
'filter': result_filter,
'filter_tabs': True,
'navigation_items': navigation_items,
'output_file': output_file,
'result_count': '{:,}'.format(results.count()),
'result_table': result_table
})
def result_page(request, result_id):
result = models.result.objects.get(id=result_id)
if request.method == 'GET':
if 'get_dut_output' in request.GET:
response = HttpResponse(result.dut_output,
content_type='text/plain')
response['Content-Disposition'] = \
'attachment; filename="{}_dut_output.txt"'.format(
result_id)
return response
elif 'get_debugger_output' in request.GET:
response = HttpResponse(result.debugger_output,
content_type='text/plain')
response['Content-Disposition'] = \
'attachment; filename="{}_debugger_output.txt"'.format(
result_id)
return response
elif 'get_aux_output' in request.GET:
response = HttpResponse(result.aux_output,
content_type='text/plain')
response['Content-Disposition'] = \
'attachment; filename="{}_aux_output.txt"'.format(
result_id)
return response
elif 'get_output_file' in request.GET:
response = get_file(result.campaign.output_file, result_id)
response['Content-Disposition'] = \
'attachment; filename={}_{}'.format(
result_id, result.campaign.output_file)
return response
elif 'get_log_file' in request.GET:
temp_file = TemporaryFile()
with open_tar(fileobj=temp_file, mode='w:gz') as archive:
for log_file in result.campaign.log_files:
archive.add(
'campaign-data/{}/results/{}/{}'.format(
result.campaign_id, result.id, log_file),
'{}_{}'.format(result.id, log_file))
response = FileResponse(temp_file,
content_type='application/x-compressed')
response['Content-Disposition'] = \
'attachment; filename={}_log_files.tar.gz'.format(result.id)
response['Content-Length'] = temp_file.tell()
temp_file.seek(0)
return response
campaign_items_ = [(item[0],
'/campaign/{}/{}'.format(result.campaign_id,
item[1]), item[2], item[3])
for item in campaign_items]
if result.campaign.output_file:
output_file = 'campaign-data/{}/results/{}/{}'.format(
result.campaign_id, result_id, result.campaign.output_file)
output_file = \
exists(output_file) and guess_type(output_file)[0] is not None
else:
output_file = False
result_table = tables.result(models.result.objects.filter(id=result_id))
events = result.event_set.all()
event_table = tables.event(events)
if request.method == 'POST' and 'launch' in request.POST:
Popen([
argv[0], '--campaign_id',
str(result.campaign_id), 'regenerate', result_id
])
if request.method == 'POST' and 'save' in request.POST:
result.outcome = request.POST['outcome']
result.outcome_category = request.POST['outcome_category']
result.save()
elif request.method == 'POST' and 'delete' in request.POST:
if exists('campaign-data/{}/results/{}'.format(result.campaign_id,
result.id)):
rmtree('campaign-data/{}/results/{}'.format(
result.campaign_id, result.id))
result.delete()
return HttpResponse('Result deleted')
injections = result.injection_set.all()
if result.campaign.simics:
if injections.count():
injection_table = tables.injection(injections)
else:
injection_table = None
register_diffs = result.simics_register_diff_set.all()
register_filter = filters.simics_register_diff(request.GET,
queryset=register_diffs)
register_diff_count = register_filter.qs.count()
register_table = tables.simics_register_diff(register_filter.qs)
RequestConfig(request, paginate={
'per_page': table_length
}).configure(register_table)
memory_diffs = result.simics_memory_diff_set.all()
memory_diff_count = memory_diffs.count()
memory_table = tables.simics_memory_diff(memory_diffs)
RequestConfig(request, paginate={
'per_page': table_length
}).configure(memory_table)
else:
register_filter = None
memory_diff_count = 0
memory_table = None
register_diff_count = 0
register_table = None
if injections.count():
injection_table = tables.injection(injections)
else:
injection_table = None
RequestConfig(request, paginate=False).configure(result_table)
RequestConfig(request, paginate=False).configure(event_table)
if injection_table:
RequestConfig(request, paginate=False).configure(injection_table)
return render(
request, 'result.html', {
'campaign_items': campaign_items_,
'event_count': '{:,}'.format(events.count()),
'event_table': event_table,
'filter': register_filter,
'injection_table': injection_table,
'memory_diff_count': '{:,}'.format(memory_diff_count),
'memory_table': memory_table,
'navigation_items': navigation_items,
'output_file': output_file,
'register_diff_count': '{:,}'.format(register_diff_count),
'register_table': register_table,
'result': result,
'result_table': result_table
})
