def update_choices():
print('generating filter choices...')
events = models.event.objects.all()
choice_count = len(event_choices) + len(injection_choices) + \
len(result_choices)
progress_bar = ProgressBar(
max_value=choice_count,
widgets=[
Percentage(), ' (',
SimpleProgress(format='%(value)d/%(max_value)d'), ') ',
Bar(), ' ',
Timer()
])
count = 0
for attribute in event_choices:
count += 1
progress_bar.update(count)
event_choices[attribute] = choices(events, attribute)
injections = models.injection.objects.all()
for attribute in injection_choices:
count += 1
progress_bar.update(count)
injection_choices[attribute] = choices(injections, attribute)
results = models.result.objects.all()
for attribute in result_choices:
count += 1
progress_bar.update(count)
result_choices[attribute] = choices(results, attribute)
print()
def update_hashes(options):
campaign = get_campaign(options)
progress_bar = ProgressBar(
max_value=campaign.result_set.count(),
widgets=[
Percentage(), ' (',
SimpleProgress(format='%(value)d/%(max_value)d'), ') ',
Bar(), ' ',
Timer()
])
count = 0
with open(
'campaign-data/{}/gold/{}'.format(campaign.id,
campaign.output_file),
'rb') as solution_file:
solution_contents = solution_file.read()
gold_hash = sha256(solution_contents).hexdigest()
for result in campaign.result_set.all():
if result.data_diff == 1.0:
result.data_hash = gold_hash
result.save()
else:
result_location = 'campaign-data/{}/results/{}/{}'.format(
campaign.id, result.id, campaign.output_file)
if exists(result_location):
with open(result_location, 'rb') as result_file:
result_contents = result_file.read()
result.data_hash = sha256(result_contents).hexdigest()
result.save()
count += 1
progress_bar.update(count)
def backup(options):
def traverse_directory(directory, archive=None, progress=None):
num_items = 0
for item in listdir(directory):
if isdir(join(directory, item)):
num_items += traverse_directory(join(directory, item), archive,
progress)
else:
num_items += 1
if archive is not None:
try:
archive.add(join(directory, item))
except FileNotFoundError:
pass
if progress is not None:
progress[0] += 1
progress[1].update(progress[0])
return num_items
# def backup(options):
if not exists('backups'):
mkdir('backups')
if not exists('campaign-data'):
mkdir('campaign-data')
sql_backup = 'campaign-data/{}.sql'.format(options.db_name)
print('dumping database...')
backup_database(options, sql_backup)
print('database dumped')
if options.files:
backup_name = 'backups/{}_{}'.format(
'-'.join([
'{:02}'.format(unit) for unit in datetime.now().timetuple()[:3]
]), '-'.join([
'{:02}'.format(unit)
for unit in datetime.now().timetuple()[3:6]
]))
num_items = 0
directories = ['campaign-data']
if exists('simics-workspace/gold-checkpoints'):
directories.append('simics-workspace/gold-checkpoints')
print('discovering files to archive')
for directory in directories:
num_items += traverse_directory(directory)
print('archiving files...')
with open_tar('{}.tar.gz'.format(backup_name), 'w:gz') \
as backup, ProgressBar(max_value=num_items, widgets=[
Percentage(), ' (',
SimpleProgress(format='%(value)d/%(max_value)d'), ') ', Bar(),
' ', Timer()]) as progress_bar:
progress = [0, progress_bar]
for directory in directories:
traverse_directory(directory, backup, progress)
remove(sql_backup)
print('backup complete')
def compute_availabilitymatrix(cutout,
shapes,
excluder,
nprocesses=None,
disable_progressbar=False):
"""
Compute the eligible share within cutout cells in the overlap with shapes.
For parallel calculation (nprocesses not None) the excluder must not be
initialized and all raster references must be strings. Otherwise processes
are colliding when reading from one common rasterio.DatasetReader.
Parameters
----------
cutout : atlite.Cutout
Cutout which the availability matrix is aligned to.
shapes : geopandas.Series/geopandas.DataFrame
Geometries for which the availabilities are calculated.
excluder : atlite.gis.ExclusionContainer
Container of all meta data or objects which to exclude, i.e.
rasters and geometries.
nprocesses : int, optional
Number of processes to use for calculating the matrix. The paralle-
lization can heavily boost the calculation speed. The default is None.
disable_progressbar: bool, optional
Disable the progressbar if nprocesses is not None. Then the `map`
function instead of the `imap` function is used for the multiprocessing
pool. This speeds up the calculation.
Returns
-------
availabilities : xr.DataArray
DataArray of shape (|shapes|, |y|, |x|) containing all the eligible
share of cutout cell (x,y) in the overlap with shape i.
Notes
-----
The rasterio (or GDAL) average downsampling returns different results
dependent on how the target raster (the cutout raster) is spanned.
Either it is spanned from the top left going downwards,
e.g. Affine(0.25, 0, 0, 0, -0.25, 50), or starting in the
lower left corner and going up, e.g. Affine(0.25, 0, 0, 0, 0.25, 50).
Here we stick to the top down version which is why we use
`cutout.transform_r` and flipping the y-axis in the end.
"""
availability = []
shapes = shapes.geometry if isinstance(shapes,
gpd.GeoDataFrame) else shapes
shapes = shapes.to_crs(excluder.crs)
progress = SimpleProgress(format='(%s)' % SimpleProgress.DEFAULT_FORMAT)
widgets = [
Percentage(), ' ', progress, ' ',
Bar(), ' ',
Timer(), ' ',
ETA()
]
progressbar = ProgressBar(prefix='Compute availabily matrix: ',
widgets=widgets,
max_value=len(shapes))
args = (excluder, cutout.transform_r, cutout.crs, cutout.shape)
if nprocesses is None:
for i in progressbar(shapes.index):
_ = shape_availability_reprojected(shapes.loc[[i]], *args)[0]
availability.append(_)
else:
assert excluder.all_closed, ('For parallelization all raster files '
'in excluder must be closed')
kwargs = {
'initializer': _init_process,
'initargs': (shapes, *args),
'maxtasksperchild': 20,
'processes': nprocesses
}
with mp.Pool(**kwargs) as pool:
if disable_progressbar:
availability = list(pool.map(_process_func, shapes.index))
else:
imap = pool.imap(_process_func, shapes.index)
availability = list(progressbar(imap))
availability = np.stack(availability)[:, ::-1] # flip axis, see Notes
coords = [(shapes.index), ('y', cutout.data.y), ('x', cutout.data.x)]
return xr.DataArray(availability, coords=coords)
def injections_page(request, campaign_id=None):
if campaign_id is not None:
campaign = models.campaign.objects.get(id=campaign_id)
campaign_items_ = campaign_items
injections = models.injection.objects.filter(
result__campaign_id=campaign_id)
else:
campaign = None
campaign_items_ = None
injections = models.injection.objects.all()
injection_filter = filters.injection(request.GET, queryset=injections)
error_title = None
error_message = None
if not injection_filter.qs.count() and injections.count():
error_title = 'Filter Error'
error_message = 'Filter did not return any injections and was ignored.'
injection_filter = filters.injection(None, queryset=injections)
injections = injection_filter.qs
print('filtering for failed registers...')
failed_registers = []
all_regs = injections.values_list('register', flat=True).distinct()
progress_bar = ProgressBar(
max_value=all_regs.count(),
widgets=[
Percentage(), ' (',
SimpleProgress(format='%(value)d/%(max_value)d'), ') ',
Bar(), ' ',
Timer()
])
for count, register in enumerate(all_regs, start=1):
progress_bar.update(count)
reg_injections = injections.filter(register=register)
if reg_injections.filter(success=True).count():
continue
failed_bits = reg_injections.values_list('bit', flat=True).distinct()
if len(failed_bits) != max(failed_bits) - 1:
continue
failed_registers.append(register)
print()
injections = injections.filter(register__in=failed_registers)
if injections.count() > 0:
chart_data, chart_list = injections_charts(injections)
chart_list = sorted(chart_list, key=lambda x: x['order'])
else:
chart_data = None
chart_list = None
injection_table = tables.injections(injections)
RequestConfig(request, paginate={
'per_page': table_length
}).configure(injection_table)
return render(
request, 'injections.html', {
'campaign': campaign,
'campaign_items': campaign_items_,
'chart_data': chart_data,
'chart_list': chart_list,
'error_message': error_message,
'error_title': error_title,
'filter': injection_filter,
'injection_count': '{:,}'.format(injections.count()),
'injection_table': injection_table,
'navigation_items': navigation_items
})