def expand_volume(peers: List[Peer], vol_info: Optional[Volume]) -> Result:
# Expands the volume by X servers+bricks
# Adds bricks and then runs a rebalance
"""
:param peers:
:param vol_info:
:return:
"""
volume_name = config("volume_name")
# Are there new peers
log("Checking for new peers to expand the volume named {}".format(
volume_name))
# Build the brick list
brick_list = get_brick_list(peers, vol_info)
if brick_list.is_ok():
log("Expanding volume with brick list: {}".format(brick_list.value),
INFO)
return volume_add_brick(volume_name, brick_list.value, True)
else:
if brick_list.value is Status.WaitForMorePeers:
log("Waiting for more peers", INFO)
return Ok(0)
elif brick_list.value is Status.InvalidConfig:
return Err(brick_list.value)
else:
# Some other error
return Err("Unknown error in expand volume: {}".format(
brick_list.value))
def server_changed() -> Result:
"""
:return:
"""
volume_name = config("volume_name")
if is_leader():
log("I am the leader: {}".format(relation_id()))
log("Loading config", INFO)
status_set(workload_state="maintenance",
message="Checking for new peers to probe")
peers = peer_list()
if peers.is_err():
return Err(peers.value)
log("peer list: {}".format(peers))
probe_in_units(peers.value, related_units())
# Update our peer list
peers = peer_list()
if peers.is_err():
return Err(peers.value)
# Everyone is in. Lets see if a volume exists
vol_info = volume_info(volume_name)
existing_volume = False
if vol_info.is_err():
return Err("Volume info command failed: {}".format(vol_info.value))
if len(vol_info.value) > 0:
log("Expanding volume {}".format(volume_name), INFO)
status_set(workload_state="maintenance",
message="Expanding volume {}".format(volume_name))
expand_vol = expand_volume(peers.value, vol_info.value)
if expand_vol.is_ok():
log("Expand volume succeeded.", INFO)
status_set(workload_state="active",
message="Expand volume succeeded.")
# Poke the other peers to update their status
relation_set("expanded", "True")
# Ensure the cluster is mounted
# mount_cluster(volume_name)
# setup_ctdb()
# setup_samba(volume_name)
return Ok(())
else:
log(
"Expand volume failed with output: {}".format(
expand_vol.value), ERROR)
status_set(workload_state="blocked",
message="Expand volume failed. Please check juju "
"debug-log.")
return Err(expand_vol.value)
if not existing_volume:
log("Creating volume {}".format(volume_name), INFO)
status_set(workload_state="maintenance",
message="Creating volume {}".format(volume_name))
create_gluster_volume(volume_name, peers.value)
# mount_cluster(volume_name)
# setup_ctdb()
# setup_samba(volume_name)
return Ok(())
def convert_sample( # type: ignore # reason: incompatible return type, but actually compatible
self,
sample: "GstSample", # type: ignore # noqa F821
) -> Result[PIL_Image, Union[RuntimeError, ValueError]]:
caps = sample.get_caps()
structure = caps.get_structure(0)
logger.debug(f"structure: {structure}")
format_ = structure.get_value("format")
logger.debug(f"format: {format_}")
format__ = AppsinkColorFormat._from_caps_format(format_)
if format__.is_err():
return Err(format__.unwrap_err())
raw_mode = format__.unwrap()._to_PIL_raw_mode()
shape = (structure.get_value("width"), structure.get_value("height"))
logger.debug(f"shape: {shape}")
# Note that `gst_buffer_extract_dup()` cause a memory leak.
# c.f. https://github.com/beetbox/audioread/pull/84
buf = sample.get_buffer()
success, info = buf.map(self._Gst.MapFlags.READ)
if success:
data = info.data
ret = PIL.Image.frombytes("RGB", shape, data, "raw", raw_mode)
buf.unmap(info)
return Ok(ret)
else:
return Err(RuntimeError("`gst_buffer_map()` failed"))
def shrink_volume(peer: Peer, vol_info: Optional[Volume]):
"""
:param peer:
:param vol_info:
:return:
"""
volume_name = config("volume_name")
log("Shrinking volume named {}".format(volume_name), INFO)
peers = [peer]
# Build the brick list
brick_list = get_brick_list(peers, vol_info)
if brick_list.is_ok():
log("Shrinking volume with brick list: {}".format(brick_list), INFO)
return volume_remove_brick(volume_name, brick_list.value, True)
else:
if brick_list.value == Status.WaitForMorePeers:
log("Waiting for more peers", INFO)
return Ok(0)
elif brick_list.value == Status.InvalidConfig:
return Err(brick_list.value)
else:
# Some other error
return Err("Unknown error in shrink volume: {}".format(
brick_list.value))
def top_parser(amount: dict, sorting_type: dict, preset: dict,
user_id: str) -> Result[dict, str]:
answer = {}
# amount parsing
amount_str = amount['selected_option']['value']
amount = try_parse_int(amount_str)
if amount is None:
return Err(f"Erroneous number: {amount_str}")
if amount <= 0:
return Err(
f"Number of messages should be positive, provided value: {amount}")
answer['top_count'] = amount
# sorting_type parsing
sorting_type = sorting_type['selected_option']['value']
if sorting_type not in {"reply_count", "thread_length", "reactions_rate"}:
return Err(f"Unknown sorting type: {sorting_type}")
answer['sorting_type'] = sorting_type
answer['user_id'] = user_id
# preset parsing
preset = preset['selected_option']['value']
if preset == "all":
pass
elif preset.startswith("<#") and preset.endswith(">"):
answer['channel_id'] = preset[2:-1]
else:
answer['preset_name'] = preset
return Ok(answer)
def resolve_to_ip(address: str) -> Result:
"""
Resolves an dns address to an ip address. Relies on dig
:param address: String. Hostname to resolve to an ip address
:return: result
"""
if address == "localhost":
local_ip = get_local_ip()
if local_ip.is_err():
return Err(local_ip.value)
try:
ip_addr = ip_address(local_ip.value)
return Ok(ip_addr)
except ValueError:
return Err("failed to parse ip address: {}".format(local_ip.value))
arg_list = ["+short", address.strip()]
output = run_command("dig", arg_list, False, False)
if output.is_err():
return Err("dig cmd failed with error:{}".format(output.value))
# Remove the trailing . and newline
trimmed = output.value.strip().rstrip(".")
try:
ip_addr = ip_address(trimmed)
return Ok(ip_addr)
except ValueError:
return Err("failed to parse ip address: {}".format(trimmed))
def read_config(root: str) -> Result[errstr, Config]:
path = path_join(root, "config.json")
try:
with open(path) as f:
j = json.loads(f.read())
c = Config()
props = [
"posts dir",
"include before",
"include after",
"static src",
"static dest",
]
for prop in props:
try:
v = j[prop]
setattr(c, prop.replace(" ", "_"), v)
except KeyError as e:
continue
return Ok(c)
except TypeError as e:
return Err(f"Failed to read config: {e}")
except FileNotFoundError as e:
return Err(f"Couldn't find {path}. Does it exist?")
def build(self) -> Result["_BuiltPipeline", PipelineBuildError]: # noqa F821 (Hey linter, see below.)
try:
elements = [f() for f in self._thunks]
logger.debug(f"_caps_string: {self._caps_string}")
caps = self._Gst.caps_from_string(self._caps_string)
elements[-1].set_property("caps", caps)
pipeline = self._Gst.Pipeline()
for x in elements:
pipeline.add(x)
for (x, y) in zip(elements, elements[1:]):
# c.f. http://gstreamer-devel.966125.n4.nabble.com/Problem-linking-rtspsrc-to-any-other-element-td3051725.html
if x.get_static_pad("src"):
logger.info(f"get static pad of src of {x}")
if not x.link(y):
return Err(PipelineBuildError(f"failed to link {x} {y}"))
else:
def f(x: "Gst.Element", y: "Gst.Element") -> None: # type: ignore # noqa F821
logger.info(f"linking {x} and {y}")
x.link(y)
x.connect("pad-added", lambda _a, _b, x=x, y=y: f(x, y))
return Ok(_BuiltPipeline(pipeline=pipeline, sink=elements[-1]))
except PipelineBuildError as err:
return Err(err)
except Exception as err:
try:
raise PipelineBuildError(err) from err
except PipelineBuildError as err:
return Err(err)
def _ready(self, file, watched_dir, archive_dir, error_dir):
"""Validate that the files in the directory are ready for processing
This is kept to my normal functional style because the only reason I use a class here is because
of the library
Positional Arguments:
file -- full path to a file
watched_dir -- the directory this event handler is watching
archive_dir -- the directory to move processed files into
error_dir -- the directory to move files to when we cannot handle them
Returns:
A result.Ok containing a ReadyDir if we are ready, or a result.Err if not
Error:
Should only return result.Err. No expected raises
"""
changed_dir = os.path.dirname(file)
if changed_dir != watched_dir:
return Err('File {} is not in watched directory {}. Ignoring'.format(file, watched_dir))
dir_files = os.listdir(changed_dir)
if len(dir_files) < 3:
return Err('We expect at least 3 files to operate. Waiting')
# TODO validate that filetypes are of images
# TODO ignore non-image files
rd = wtde.ReadyDir(
files=dir_files,
directory=changed_dir,
archive_dir=archive_dir,
error_dir=error_dir,
play_dt=None,
remove_srcdir=False)
return Ok(rd)
def get_processed_trace(
clr: Clr, test_result: TestResult, need_mechanisms_and_reasons: bool, need_join_info: bool
) -> Result[str, ProcessedTrace]:
test_status = option_to_result(
test_result.load_test_status(), lambda: "Need a test status file"
)
if test_result.trace_path is None:
if need_join_info:
return Err("Can't get join info without a trace.")
else:
return Ok(
ProcessedTrace(
clr=clr,
test_result=test_result,
test_status=test_status,
process_info=None,
process_names=cast(ThreadToProcessToName, None),
process_query=None,
join_info=Err("Did not request join info"),
mechanisms_and_reasons=None,
gcs_result=Err("Did not collect a trace"),
)
)
else:
return Ok(
_get_processed_trace_from_process(
clr,
test_status,
test_result,
need_join_info=need_join_info,
need_mechanisms_and_reasons=need_mechanisms_and_reasons,
)
)
def __call__(self,
request: LoadAtlasRequest) -> Result[LoadAtlasData, str]:
if isinstance(request, LoadBrainglobeAtlas):
atlas_data = self._remote_atlas_reader.read(name=request.name)
if atlas_data is None:
return Err("Atlas not loaded.")
atlas = Atlas(
volume=atlas_data.registration_volume,
resolution_um=atlas_data.resolution_um,
annotation_volume=atlas_data.annotation_volume,
)
elif isinstance(request, LoadAtlasFromFile):
atlas_data2 = self._local_atlas_reader.read(
filename=request.filename)
if atlas_data2 is None:
return Err("Atlas not loaded.")
atlas = Atlas(
volume=atlas_data2.registration_volume,
resolution_um=request.resolution_um,
annotation_volume=None,
)
self._repo.set_atlas(atlas=atlas)
return Ok(
LoadAtlasData(
volume=atlas.volume,
transform=atlas.shared_space_transform,
resolution=atlas.resolution_um,
annotation_volume=atlas.annotation_volume,
))
def _get_processed_trace_from_process(
clr: Clr,
test_status: Failable[TestRunStatus],
test_result: TestResult,
need_join_info: bool,
need_mechanisms_and_reasons: bool,
) -> ProcessedTrace:
if test_result.process is None:
ts = unwrap(
map_err(
test_status,
lambda _: "Didn't specify --process and there's no test status to specify PID.\n"
" (hint: maybe specify the test output '.yaml' file instead of the trace file)",
)
)
if ts.process_id is not None:
process_predicate = process_predicate_from_id(ts.process_id)
else:
process_predicate = process_predicate_from_parts(ts.get_process_data_tuple())
else:
assert (
test_status.is_err()
), "'--process' is unnecessary as the test result specifies the PID"
process_predicate = process_predicate_from_parts(test_result.process)
process_names, proc = get_process_names_and_process_info(
clr,
non_null(test_result.trace_path),
str(test_result),
process_predicate,
# TODO: make this optional; though the metric FirstEventToFirstGCSeconds needs this too.
collect_event_names=True,
)
assert len(proc.gcs) > 0, (
f"Process '{proc.process.Name}' in Trace File '{proc.trace_path.name}' "
"has no GC's to analyze."
)
# TODO: just do this lazily (getting join info)
join_info = (
get_join_info_for_all_gcs(clr, proc) if need_join_info else Err("Did not request join info")
)
res = ProcessedTrace(
clr=clr,
test_result=test_result,
test_status=test_status,
process_info=proc,
process_names=process_names,
process_query=test_result.process,
join_info=join_info,
# TODO: just do this lazily
mechanisms_and_reasons=get_mechanisms_and_reasons_for_process_info(proc)
if need_mechanisms_and_reasons
else None,
gcs_result=Err("temporary err, will be overwritten"),
)
return _init_processed_trace(res, proc)
def MemoryPressure(self) -> FailableFloat:
ghh = self.trace_gc.GlobalHeapHistory
if ghh is None:
return Err("No GlobalHeapHistory")
elif ghh.HasMemoryPressure:
return Ok(ghh.MemoryPressure)
else:
return Err("GlobalHeapHistory#HasMemoryPressure was false")
def test_eq():
assert Ok(1) == Ok(1)
assert Err(1) == Err(1)
assert Ok(1) != Err(1)
assert Ok(1) != Ok(2)
assert not (Ok(1) != Ok(1))
assert Ok(1) != "abc"
assert Ok("0") != Ok(0)
def is_allowed(self, parsed_query):
if not parsed_query:
return Err("Could not parse query")
for rule in self.rules.itervalues():
check = rule.check(parsed_query)
if not check.is_ok():
return Err(check.value)
return Ok(True)
def FirstToLastGCSeconds(self) -> FailableFloat:
if self.process_info is None:
return Err("Need a trace")
gcs = self.process_info.all_gcs_including_incomplete
if len(gcs) < 2:
return Err("Need at least 2 gcs")
else:
return Ok(
msec_to_seconds(gcs[-1].StartRelativeMSec -
gcs[0].StartRelativeMSec))
def preprocess(self, inputs: Any) -> Result[Tensor, ValueError]:
'''
Preprocesses inputs using the preprocessor.
'''
if not self.preprocessor_exists():
try:
return Ok(self.preprocessor(inputs))
except Exception as error: return Err(ValueError(error))
else:
return Err(ValueError('tried to call <CNN>.preprocessor on a None value'))
def _get_mark_times(clr: Clr, gc: AbstractTraceGC, hp_i: int) -> Failable[AbstractMarkInfo]:
m = gc.PerHeapMarkTimes
if m is None:
return Err("No PerHeapMarkTimes")
else:
res = clr.PythonnetUtil.TryGetValue(m, hp_i)
if res is None:
return Err(f"PerHeapMarkTimes contains no heap {hp_i}")
else:
return Ok(res)
def load_md_file(state: State, metadata: FileMetadata,
path: str) -> Result[errstr, str]:
try:
with open(path) as f:
return Ok(f.read())
except IOError as e:
Err(str(e))
except FileNotFoundError as e:
Err(str(e))
return Err(f"Failed to compile file {path}")
def FirstEventToFirstGCSeconds(self) -> FailableFloat:
if self.process_info is None:
return Err("Need a trace")
ts = self.process_info.events_time_span
if ts is None:
return Err("Did not specify to collect events")
else:
return Ok(
msec_to_seconds(
self.process_info.all_gcs_including_incomplete[0].
StartRelativeMSec - ts.StartMSec))
def build_rover(rover_position_as_string: str, rover_id: int) -> Result[Rover, str]:
if not re.match(rover_position_pattern, rover_position_as_string):
return Err(f"Positional data for rover {rover_id + 1} is not in the correct format: {rover_position_as_string}")
split_position = rover_position_as_string.split()
try:
position_x = int(split_position[0])
position_y = int(split_position[1])
direction = split_position[2].upper()
except TypeError as e:
return Err(str(e))
return Ok(Rover(rover_id=rover_id, position_x=position_x, position_y=position_y, facing=direction))
def check_for_new_devices() -> Result:
"""
:return:
"""
log("Checking for new devices", INFO)
log("Checking for ephemeral unmount")
ephemeral_unmount()
# if config.changed("brick_devices"))
brick_devices = []
# Get user configured storage devices
manual_brick_devices = get_manual_bricks()
if manual_brick_devices.is_err():
return Err(manual_brick_devices.value)
brick_devices.extend(manual_brick_devices.value)
# Get the juju storage block devices
juju_config_brick_devices = get_juju_bricks()
if juju_config_brick_devices.is_err():
return Err(juju_config_brick_devices.value)
brick_devices.extend(juju_config_brick_devices.value)
log("storage devices: {}".format(brick_devices))
format_handles = []
brick_paths = []
# Format all drives in parallel
for device in brick_devices:
if not device.initialized:
log("Calling initialize_storage for {}".format(device.dev_path))
# Spawn all format commands in the background
format_handles.append(initialize_storage(device))
else:
# The device is already initialized, lets add it to our
# usable paths list
log("{} is already initialized".format(device.dev_path))
brick_paths.append(device.mount_path)
# Wait for all children to finish formatting their drives
for handle in format_handles:
output_result = handle.format_child.wait_with_output()
if output_result.is_ok():
# success
# 1. Run any post setup commands if needed
finish_initialization(handle.device.dev_path)
brick_paths.append(handle.device.mount_path)
else:
# Failed
log(
"Device {} formatting failed with error: {}. Skipping".format(
handle.device.dev_path, output_result.value), ERROR)
log("Usable brick paths: {}".format(brick_paths))
return Ok(())
def _get_per_heap_histories(gc: AbstractTraceGC) -> Sequence[Result[str, AbstractGCPerHeapHistory]]:
if gc.HeapCount == 1:
return [Err("Workstation GC has no AbstractGCPerHeapHistories")]
else:
n = len(gc.PerHeapHistories)
if n != gc.HeapCount:
print(
f"WARN: GC {gc.Number} has {gc.HeapCount} heaps, but {n} PerHeapHistories. It's a "
+ f" It's a {get_gc_kind_for_abstract_trace_gc(gc).name}."
)
return repeat(Err("GC has wrong number of PerHeapHistories"), gc.HeapCount)
else:
return [Ok(h) for h in gc.PerHeapHistories]
def _get_processed_trace_from_process(
clr: Clr,
test_status: Failable[TestRunStatus],
test_result: TestResult,
trace_path: Path,
process: ProcessQuery,
need_join_info: bool,
need_mechanisms_and_reasons: bool,
) -> ProcessedTrace:
if process is None:
ts = unwrap(
map_err(
test_status,
lambda _:
"Didn't specify --process and there's no test status to specify PID\n"
" (hint: maybe specify the test output '.yaml' file instead of the trace file)",
))
if ts.process_id is None:
raise Exception(
"Test status file exists but does not specify process_id")
process_predicate = process_predicate_from_id(ts.process_id)
else:
assert (test_status.is_err(
)), "'--process' is unnecessary as the test result specifies the PID"
process_predicate = process_predicate_from_parts(process)
process_names, proc = get_process_names_and_process_info(
clr,
trace_path,
str(test_result),
process_predicate,
# TODO: make this optional; though the metric FirstEventToFirstGCSeconds needs this too.
collect_event_names=True,
)
# TODO: just do this lazily (getting join info)
join_info = (get_join_info_for_all_gcs(clr, proc)
if need_join_info else Err("Did not request join info"))
res = ProcessedTrace(
clr=clr,
test_result=test_result,
test_status=test_status,
process_info=proc,
process_names=process_names,
process_query=process,
join_info=join_info,
# TODO: just do this lazily
mechanisms_and_reasons=get_mechanisms_and_reasons_for_process_info(
proc) if need_mechanisms_and_reasons else None,
gcs_result=Err("temporary err, will be overwritten"),
)
return _init_processed_trace(res, proc)
def _get_server_gc_heap_histories(
gc: AbstractTraceGC) -> Sequence[Result[str, AbstractServerGcHistory]]:
if gc.HeapCount == 1:
return [Err("Workstation GC has no ServerGcHeapHistories")]
else:
n = len(gc.ServerGcHeapHistories)
if n != gc.HeapCount:
print(
f"WARN: GC {gc.Number} has {gc.HeapCount} heaps, but {n} ServerGcHeapHistories"
)
return repeat(Err("GC has wrong number of ServerGcHeapHistories"),
gc.HeapCount)
else:
return [Ok(h) for h in gc.ServerGcHeapHistories]
def process_rover_routes(rover_input) -> Result[List[str], str]:
# TODO: Expand this to take other formats, namely from a text file and dictionary input
input_result = read_input(rover_input)
if input_result.is_err():
return Err(input_result.unwrap_err())
mars_rover_data = input_result.unwrap()
final_positions: List[str] = []
for rover in mars_rover_data.rovers:
try:
rover.read_instructions(mars_rover_data.map, mars_rover_data.instructions[rover.id])
except ValueError as e:
return Err(str(e))
final_positions.append(str(rover))
return Ok(final_positions)
def _get_processed_gc(
proc: ProcessedTrace, gc_index: int, join_info: Result[str, AbstractJoinInfoForGC]
) -> ProcessedGC:
gc = non_null(proc.process_info).gcs[gc_index]
heap_join_infos: Iterable[Result[str, AbstractJoinInfoForHeap]] = match(
join_info, lambda j: [Ok(h) for h in j.Heaps], lambda e: repeat(Err(e), gc.HeapCount)
)
res = ProcessedGC(proc=proc, index=gc_index, trace_gc=gc, join_info=join_info, heaps=())
res.heaps = [
ProcessedHeap(
gc=res,
index=hp_i,
per_heap_history=phh,
server_gc_history=sgh,
_mark_times=map_ok(mark_times, lambda m: m.MarkTimes),
_mark_promoted=map_ok(mark_times, lambda m: m.MarkPromoted),
join_info=heap_join_info,
)
for hp_i, (phh, sgh, heap_join_info) in enumerate(
zip_check_3(
_get_per_heap_histories(gc), _get_server_gc_heap_histories(gc), heap_join_infos
)
)
for mark_times in (_get_mark_times(proc.clr, gc, hp_i),)
]
return res
def create_gluster_volume(volume_name: str, peers: List[Peer]) -> Result:
"""
Create a new gluster volume with a name and a list of peers
:param volume_name: str. Name of the volume to create
:param peers: List[Peer]. List of the peers to use in this volume
:return:
"""
create_vol = create_volume(peers, None)
if create_vol.is_ok():
if create_vol.value == Status.Created:
log("Create volume succeeded.", INFO)
status_set(workload_state="maintenance",
message="Create volume succeeded")
start_gluster_volume(volume_name)
# Poke the other peers to update their status
relation_set("started", "True")
return Ok(())
elif create_vol.value == Status.WaitForMorePeers:
log("Waiting for all peers to enter the Peer in Cluster status")
status_set(workload_state="maintenance",
message="Waiting for all peers to enter "
"the \"Peer in Cluster status\"")
return Ok(())
else:
# Status is failed
# What should I return here
return Ok(())
else:
log("Create volume failed with output: {}".format(create_vol.value),
ERROR)
status_set(workload_state="blocked",
message="Create volume failed. Please check "
"juju debug-log.")
return Err(create_vol.value)
def SurvivalPct(self) -> FailableFloat:
pct = self._trace_gc.SurvivalPercent(self._gen_value)
if isnan(pct):
return Err(f"{Gens(self._gen_value).name} not collected?")
else:
assert 0 <= pct <= 100
return Ok(pct)
def map(self, hook: StateMapHook[T]) -> Result[U, ValueError]:
'''
Gets a value, applies the hook and returns the result.
Does not mutate anything in place.
'''
return Err(ValueError(NotImplementedError))
