import math
import threading
import sys
from random import choice
from enum import Enum
# DFS Meta Data
CHUNK_SIZE = 2 * 1024 * 1024
BUFSIZE = 4 * 1024 * 1024
NUM_DATA_SERVER = 4
NUM_REPLICATION = 4
# Data Node Addr
DATA_NODE_ADDR = [
'121.37.138.6', '124.70.177.49', '124.70.128.29', '124.70.153.253'
]
DATA_NODE_PORT = 20000
CHUNK_PATTERN = 'file-%s-%s'
TIME_OUT = 3
SYSTEM_Mode = 'DISTRIBUTED' # LOCAL/DISTRIBUTED 本地/分布式
# Name Node Meta Data
NAME_NODE_META_PATH = './dfs/namenode/meta.pkl'
# Local Data Node
DATA_NODE_DIR = './dfs/datanode%s'
# Operations
operation_names = ('put', 'read', 'fetch', 'quit', 'ls', 'delete', 'll')
COMMAND = Enum('COMMAND', operation_names)
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License."""
__revision__ = " $Id: LingData.py 2 2017-07-25 14:32:00Z damir $ "
__docformat__ = 'reStructuredText'
__author__ = 'Damir Cavar <*****@*****.**>, Atreyee M.'
__version__ = '0.1'
from enum import Enum
#import PSTree
labels = Enum(
'labels',
'id word lemma POS SPOS NER foreign isReferent hasAntecedent RefText RefS RefFrom RefTo RefHead'
)
wtypes = Enum('wtypes', 'verb noun pronoun copula unknown')
def main():
pass
class Document():
"""This is the main class that holds documents, which consist of a set of sentences.
"""
def __init__(self):
self.sentences = [] #: list of sentences
self.sIDs = {
} #: dictionary with sentence IDs as keys and the sentence number in the list as value
opts.wrap_mode = None
opts.prefix = prefix
opts.cmd_line_options = {}
return opts
def get_fake_env(sdir='', bdir=None, prefix='', opts=None):
if opts is None:
opts = get_fake_options(prefix)
env = Environment(sdir, bdir, opts)
env.coredata.compiler_options.host['c']['args'] = FakeCompilerOptions()
env.machines.host.cpu_family = 'x86_64' # Used on macOS inside find_library
return env
Backend = Enum('Backend', 'ninja vs xcode')
if 'MESON_EXE' in os.environ:
meson_exe = mesonlib.split_args(os.environ['MESON_EXE'])
else:
meson_exe = None
if mesonlib.is_windows() or mesonlib.is_cygwin():
exe_suffix = '.exe'
else:
exe_suffix = ''
def get_meson_script():
'''
Guess the meson that corresponds to the `mesonbuild` that has been imported
import trp
from typing import List, Optional
from tabulate import tabulate
from enum import Enum
from io import StringIO
import csv
import logging
logger = logging.getLogger(__name__)
Textract_Pretty_Print = Enum('Textract_Pretty_Print',
["WORDS", "LINES", "FORMS", "TABLES"],
start=0)
Pretty_Print_Table_Format = Enum('Pretty_Print_Table_Format', [
"csv", "plain", "simple", "github", "grid", "fancy_grid", "pipe", "orgtbl",
"jira", "presto", "pretty", "psql", "rst", "mediawiki", "moinmoin",
"youtrack", "html", "unsafehtml", "latex", "latex_raw", "latex_booktabs",
"latex_longtable", "textile", "tsv"
])
def get_string(
textract_json: dict,
output_type: Optional[List[Textract_Pretty_Print]] = None,
table_format: Pretty_Print_Table_Format = Pretty_Print_Table_Format.github
):
result_value = ""
for t in output_type:
if t == Textract_Pretty_Print.WORDS:
result_value += get_words_string(textract_json=textract_json)
if t == Textract_Pretty_Print.LINES:
#枚举类型
from enum import Enum
Month = Enum('Month', ('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug',
'Sep', 'oct', 'Nov', 'Dec'))
for name, member in Month.__members__.items():
print(name, '=>', member, ',', member.value)
class HelpParser:
'''A parser for the Bitcoin Core RPC command line help'''
Section = Enum(
'Section', 'command description result literal_result past_result '
'arguments literal_argument examples')
def __init__(self):
self.json_level = 0
def parse_help_result(self, result_line):
match = re.match(r'([^\s]*)\s+\((.*)\)\s+(.*)', result_line)
if match:
name = match.group(1)
if name[0] == '"' and name[-1] == '"':
name = name[1:-1]
return {'format': 'table', 'name': name, "type": match.group(2),
"description": match.group(3)}
else:
return None
def parse_help_argument(self, result_line):
match = re.match(r'\d\.\s+([^\s]*)\s+\((.*?)\)\s*(.*)', result_line)
if match:
name = match.group(1)
if name[0] == '"' and name[-1] == '"':
name = name[1:-1]
return {"name": name, "type": match.group(2),
"description": match.group(3)}
else:
return None
def next_section(self, line, help_data):
"""Check line for section identifiers and move state to next section if
found. Returns if a new section was found and the state was changed.
"""
previous_section = self.section
result_match = re.match(r'^Result:?([^:]*):?$', line)
if result_match and (':' in line or line == 'Result' or line.startswith('Result (')):
self.section = self.Section.result
help_data['results'].append(
{'title_extension': result_match.group(1)})
elif line == "Arguments:":
self.section = self.Section.arguments
elif re.match(r'Examples?:?', line):
self.section = self.Section.examples
return previous_section != self.section
def check_opening_json(self, line):
"""Checks line for brackets opening a JSON object and sets the
json_level accordingly. Detects one-line JSON which is closed in the
same line.
"""
if line.endswith('},') or line.endswith('],'):
line = line[:-1]
match = re.match(r"^ *([\[{])", line)
if match:
pairs = [
['{', '}'],
['[', ']'],
]
for pair in pairs:
if match.group(1) == pair[0] and (line[-len(pair[1]):] != pair[1]):
self.json_level += 1
return
def parse_help_overview(self, help_text):
command_list = CommandList()
group = ""
for line in help_text.splitlines():
pattern = re.compile("== (.*) ==")
match = pattern.match(line)
if match:
group = match.group(1)
else:
if len(line) > 0:
command_list.add(group, line.rstrip())
return command_list
def parse_help_command(self, help_text):
self.section = self.Section.command
help_data = {
"command": "",
"description": "",
"results": [],
"arguments": [],
"examples": [],
}
for line in help_text.splitlines():
# print("LINE (" + self.section.name + ") " + line)
if self.section == self.Section.command:
help_data['command'] = line.rstrip()
self.section = self.Section.description
elif self.section == self.Section.description:
if not self.next_section(line, help_data):
if line:
if help_data["description"] and help_data["description"][-2] in ['.', ':']:
help_data["description"] += '\n'
help_data["description"] += line.rstrip() + '\n'
elif self.section == self.Section.arguments:
if not self.next_section(line, help_data):
if line:
argument = self.parse_help_argument(line)
if argument:
help_data["arguments"].append(argument)
else:
if help_data['arguments']:
last_argument = help_data["arguments"][-1]
if last_argument:
self.check_opening_json(line)
if self.json_level > 0:
last_argument['literal_description'] = line + '\n'
self.section = self.Section.literal_argument
else:
if line.startswith(' '):
if last_argument['description']:
last_argument['description'] += '\n '
last_argument['description'] += line.lstrip()
elif self.section == self.Section.literal_argument:
last_argument = help_data['arguments'][-1]
last_argument['literal_description'] += line + '\n'
self.check_opening_json(line)
if re.match(r"^ *[\]}]", line):
self.json_level -= 1
if self.json_level == 0:
self.section = self.Section.arguments
elif self.section == self.Section.result:
if not self.next_section(line, help_data):
if line.startswith("{") or line.startswith("["):
self.section = self.Section.literal_result
result_data = help_data['results'][-1]
result_data.update(
{'format': 'literal', 'text': ' ' + line + '\n'})
else:
if line and not line.startswith(" "):
result_data_line = self.parse_help_result(line)
result_data = help_data['results'][-1]
if result_data_line:
result_data.update(result_data_line)
else:
result_data.update(
{'format': 'literal', 'text': ' ' + line + '\n'})
self.section = self.Section.past_result
elif self.section == self.Section.literal_result:
last_result = help_data['results'][-1]
last_result['text'] += ' ' + line.rstrip() + '\n'
if line == "}" or line == ']':
self.section = self.Section.past_result
elif self.section == self.Section.past_result:
self.next_section(line, help_data)
elif self.section == self.Section.examples:
if line:
help_data["examples"].append(line)
return help_data
'arm64',
'armhf',
'host',
]
ModelFormatStrs = [
"file",
"code",
]
PlatformTypeStrs = [
"tensorflow",
"caffe",
"onnx",
]
PlatformType = Enum('PlatformType', [(ele, ele) for ele in PlatformTypeStrs],
type=str)
RuntimeTypeStrs = ["cpu", "gpu", "dsp", "hta", "apu", "cpu+gpu"]
InOutDataTypeStrs = [
"int32",
"float32",
]
InOutDataType = Enum('InputDataType',
[(ele, ele) for ele in InOutDataTypeStrs],
type=str)
FPDataTypeStrs = [
"fp16_fp32",
"fp32_fp32",
from enum import Enum
from collections import namedtuple
Type = Enum("Type", ("CURE", "HEALTHY", "SICK", "DYING", "DEAD"))
Agent = namedtuple("Agent", ("name", "category"))
def meetup(agent_listing: tuple) -> list:
"""Model the outcome of the meetings of pairs of agents.
The pairs of agents are ((a[0], a[1]), (a[2], a[3]), ...). If there's an uneven
number of agents, the last agent will remain the same.
Notes
-----
The rules governing the meetings were described in the question. The outgoing
listing may change its internal ordering relative to the incoming one.
Parameters
----------
agent_listing : tuple of Agent
A listing (tuple in this case) in which each element is of the Agent
type, containing a 'name' field and a 'category' field, with 'category' being
of the type Type.
Returns
-------
updated_listing : list
A list of Agents with their 'category' field changed according to the result
of the meeting.
"""
class RemapLabels(Transform, CliPlugin):
DefaultAction = Enum('DefaultAction', ['keep', 'delete'])
@staticmethod
def _split_arg(s):
parts = s.split(':')
if len(parts) != 2:
import argparse
raise argparse.ArgumentTypeError()
return (parts[0], parts[1])
@classmethod
def build_cmdline_parser(cls, **kwargs):
parser = super().build_cmdline_parser(**kwargs)
parser.add_argument('-l', '--label', action='append',
type=cls._split_arg, dest='mapping',
help="Label in the form of: '<src>:<dst>' (repeatable)")
parser.add_argument('--default',
choices=[a.name for a in cls.DefaultAction],
default=cls.DefaultAction.keep.name,
help="Action for unspecified labels")
return parser
def __init__(self, extractor, mapping, default=None):
super().__init__(extractor)
assert isinstance(default, (str, self.DefaultAction))
if isinstance(default, str):
default = self.DefaultAction[default]
assert isinstance(mapping, (dict, list))
if isinstance(mapping, list):
mapping = dict(mapping)
self._categories = {}
src_label_cat = self._extractor.categories().get(AnnotationType.label)
if src_label_cat is not None:
self._make_label_id_map(src_label_cat, mapping, default)
src_mask_cat = self._extractor.categories().get(AnnotationType.mask)
if src_mask_cat is not None:
assert src_label_cat is not None
dst_mask_cat = MaskCategories(attributes=src_mask_cat.attributes)
dst_mask_cat.colormap = {
id: src_mask_cat.colormap[id]
for id, _ in enumerate(src_label_cat.items)
if self._map_id(id) or id == 0
}
self._categories[AnnotationType.mask] = dst_mask_cat
src_points_cat = self._extractor.categories().get(AnnotationType.points)
if src_points_cat is not None:
assert src_label_cat is not None
dst_points_cat = PointsCategories(attributes=src_points_cat.attributes)
dst_points_cat.items = {
id: src_points_cat.items[id]
for id, item in enumerate(src_label_cat.items)
if self._map_id(id) or id == 0
}
self._categories[AnnotationType.points] = dst_points_cat
def _make_label_id_map(self, src_label_cat, label_mapping, default_action):
dst_label_cat = LabelCategories(attributes=src_label_cat.attributes)
id_mapping = {}
for src_index, src_label in enumerate(src_label_cat.items):
dst_label = label_mapping.get(src_label.name)
if not dst_label and default_action == self.DefaultAction.keep:
dst_label = src_label.name # keep unspecified as is
if not dst_label:
continue
dst_index = dst_label_cat.find(dst_label)[0]
if dst_index is None:
dst_index = dst_label_cat.add(dst_label,
src_label.parent, src_label.attributes)
id_mapping[src_index] = dst_index
if log.getLogger().isEnabledFor(log.DEBUG):
log.debug("Label mapping:")
for src_id, src_label in enumerate(src_label_cat.items):
if id_mapping.get(src_id):
log.debug("#%s '%s' -> #%s '%s'",
src_id, src_label.name, id_mapping[src_id],
dst_label_cat.items[id_mapping[src_id]].name
)
else:
log.debug("#%s '%s' -> <deleted>", src_id, src_label.name)
self._map_id = lambda src_id: id_mapping.get(src_id, None)
self._categories[AnnotationType.label] = dst_label_cat
def categories(self):
return self._categories
def transform_item(self, item):
# TODO: provide non-inplace version
annotations = []
for ann in item.annotations:
if ann.type in { AnnotationType.label, AnnotationType.mask,
AnnotationType.points, AnnotationType.polygon,
AnnotationType.polyline, AnnotationType.bbox
} and ann.label is not None:
conv_label = self._map_id(ann.label)
if conv_label is not None:
ann._label = conv_label
annotations.append(ann)
else:
annotations.append(ann)
item._annotations = annotations
return item
name='followup_priorities',
validate_strings=True)
sqla_enum_types = [
allowed_bandpasses,
thumbnail_types,
instrument_types,
followup_priorities,
]
api_classnames = sa.Enum(
*[
k for k, v in facility_apis.__dict__.items()
if inspect.isclass(v) and issubclass(v, facility_apis.FollowUpAPI)
and v is not facility_apis.FollowUpAPI
],
name='followup_apis',
validate_strings=True,
)
listener_classnames = sa.Enum(
*LISTENER_CLASSNAMES,
name='followup_listeners',
validate_strings=True,
)
py_allowed_magsystems = Enum('magsystems', ALLOWED_MAGSYSTEMS)
py_allowed_bandpasses = Enum('bandpasses', ALLOWED_BANDPASSES)
py_thumbnail_types = Enum('thumbnail_types', THUMBNAIL_TYPES)
py_followup_priorities = Enum('priority', FOLLOWUP_PRIORITIES)
import sys
from enum import Enum
from error import print_error_msg, ERRNUM
from loader import DictionaryLoader
# ex: python -m "name"
# ex: python -m "name" "ox"
ARGS = Enum("PY", "WORD", "TYPE", "MAX", AVAILABLE=2)
def parse_arguements(args):
if len(args) < ARGS.AVAILABLE:
print_error_msg(ERRNUM.E_ARGS)
word = args[ARGS.WORD]
try:
module = args[ARGS.TYPE]
except IndexError:
module = 'ox' # default module
return (word, module)
if __name__ == "__main__":
word, module = parse_arguements(sys.argv)
dict_ins = DictionaryLoader.get_module(module)
dict_ins.find_word(word).show_word()
Any issues can be reported to https://github.com/CalabreseLab
---
"""
import numpy as np
from enum import Enum
from collections import defaultdict
from itertools import product
from pandas import DataFrame
from seekr.my_tqdm import my_tqdm
from seekr.fasta_reader import Reader
Log2 = Enum('Log2', ('pre', 'post', 'none'))
class BasicCounter:
"""Generates overlapping kmer counts for a fasta file
Parameters
----------
infasta: str (default=None)
Full path to fasta file to be counted
outfile: str (default=None)
Full path to the counts file to be saved
k: int (default=6)
Size of kmer to be counted
binary: bool (default=True)
Saves as numpy array if True, else saves as csv
class VW_EGolf(Device, HTTP):
pass
"""
the main class for the Golf charge device /!\ actually unable to work
"""
BASE = "http://{h}/r?rapi=%24"
Cmd = Enum("Cmd", "sleep reset enable disable")
__CmdToUrl = {
Cmd.sleep: "FS",
Cmd.reset: "FR",
Cmd.enable: "FE",
Cmd.disable: "FD"
}
def __init__(self, host):
"""
Construtor of the class VW_EGolf:
:param host: the host of the charger
:type host: string
"""
Device.__init__(self)
BASE = BASE.format(h=host)
self.__charging_rate = self.get_charging_rate()
def command(self, cmd):
"""
command to the charger:
:param cmd: the command to pass to the charger
:type cmd: Cmd enum
:return: if it has been done or not
:rtype: bool
"""
pass
url = BASE + __CmdToUrl[cmd]
result = self._get(url)
def get_charging_rate(self):
"""charging rate getter from the charger"""
pass
url = BASE + "GC"
result = self._get(url)
#sort infos to get the value and return it
@property
def charging_rate(self):
return __charging_rate
@charging_rate.setter
def set_charging_rate(self, charging_rate):
"""
change the charging rate in ampere:
:param charging_rate: new charging rate
:type charging_rate: int
:return: if it has been done or not
:rtype: bool
"""
pass
url = BASE + "SC+" + str(amp)
result = self._get(url)
from enum import Enum
ACTIONS = Enum('API functions', 'MATCHES PLAYERS TEAMS')
BASE_URL = 'https://api.dc01.gamelockerapp.com/shards/global'
SUCCESS_CODES = [200]
# coding: utf-8
from enum import Enum
import time
PizzaProgress = Enum('PizzaProgress', 'queued preparation baking ready')
PizzaDough = Enum('PizzaDough', 'thin thick')
PizzaSauce = Enum('PizzaSauce', 'tomato creme_fraiche')
PizzaTopping = Enum('PizzaTopping', 'mozzarella double_mozzarella bacon ham mushrooms red_onion oregano')
STEP_DELAY = 3 # 考虑是示例，单位为秒
# 这里没有一个AbstractBuilder
class Pizza:
def __init__(self, name):
self.name = name
self.dough = None
self.sauce = None
self.topping = []
def __str__(self):
return self.name
def prepare_dough(self, dough):
self.dough = dough
print('preparing the {} dough of your {}...'.format(self.dough.name, self))
time.sleep(STEP_DELAY)
print('done with the {} dough'.format(self.dough.name))
# ConcreateBuilder
Contours as a list of numpy.ndarray.
"""
output = []
for contour in input_contours:
x, y, w, h = cv2.boundingRect(contour)
if (w < min_width or w > max_width):
continue
if (h < min_height or h > max_height):
continue
area = cv2.contourArea(contour)
if (area < min_area):
continue
if (cv2.arcLength(contour, True) < min_perimeter):
continue
hull = cv2.convexHull(contour)
solid = 100 * area / cv2.contourArea(hull)
if (solid < solidity[0] or solid > solidity[1]):
continue
if (len(contour) < min_vertex_count
or len(contour) > max_vertex_count):
continue
ratio = (float)(w) / h
if (ratio < min_ratio or ratio > max_ratio):
continue
output.append(contour)
return output
BlurType = Enum('BlurType',
'Box_Blur Gaussian_Blur Median_Filter Bilateral_Filter')
"""Returns a function that prepends the supplied prefix and converts
underscores to the dashes required by Cloud constants..
"""
def inner(name: str) -> str:
return "{}-{}".format(prefix, name.replace('_', '-'))
return inner
US_REGIONS = {"west1", "west2", "central1", "east1", "east4"}
EURO_REGIONS = {"west1", "west4", "north1"}
ASIA_REGIONS = {"southeast1", "east1", "northeast1"}
# Actual enum types.
US = Enum('US', u.dict_by(US_REGIONS, _vfn("us")))
Europe = Enum('Europe', u.dict_by(EURO_REGIONS, _vfn("europe")))
Asia = Enum('Asia', u.dict_by(ASIA_REGIONS, _vfn("asia")))
Region = Union[US, Europe, Asia]
Zone = str
def valid_regions(zone: Optional[str] = None) -> List[Region]:
"""Returns valid region strings for Cloud, for the globe or for a particular
region if specified.
"""
if zone is None:
return valid_regions("americas") \
+ valid_regions("europe") \
+ valid_regions("asia")
def parse_header_type(cfg):
header_type = Enum('HeaderType', ' '.join(cfg.HeaderType), module=__name__)
return header_type
'Const',
'Gather',
'StridedSlice',
'Slice',
'ReverseV2',
'Stack',
'Pack',
'Unstack',
'Unpack',
'Cast',
'ArgMax',
'Split',
'FakeQuantWithMinMaxVars',
]
TFOpType = Enum('TFOpType', [(op, op) for op in TFSupportedOps], type=str)
TFSupportedOps = [six.b(op) for op in TFSupportedOps]
TFTransformGraphOptions = {
base_converter.DeviceType.CPU.value: [
'strip_unused_nodes', 'remove_nodes(op=Identity, op=CheckNumerics)',
'fold_constants(ignore_errors=true)', 'fold_batch_norms',
'fold_old_batch_norms', 'remove_control_dependencies',
'strip_unused_nodes', 'sort_by_execution_order'
],
base_converter.DeviceType.GPU.value: [
'strip_unused_nodes', 'remove_nodes(op=Identity, op=CheckNumerics)',
'fold_constants(ignore_errors=true)', 'flatten_atrous_conv',
'fold_batch_norms', 'fold_old_batch_norms',
'remove_control_dependencies', 'strip_unused_nodes',
from enum import Enum
import re
from typing import Dict, List, Optional, TextIO
from linesep import read_paragraphs
from pydantic import BaseModel
ParserState = Enum("ParserState", "BADGES POST_LINKS POST_CONTENTS INTRO SECTIONS")
HEADER_LINK_RGX = r"`(?P<label>[^`<>]+) <(?P<url>[^>]+)>`_"
IMAGE_START = ".. image:: "
class Image(BaseModel):
href: str
target: Optional[str]
alt: Optional[str]
@classmethod
def parse(cls, s: str) -> "Image":
if not s.startswith(IMAGE_START):
raise ValueError(f"Not an RST image: {s!r}")
lines = s.splitlines(keepends=True)
href = lines[0][len(IMAGE_START) :].strip()
options: Dict[str, Optional[str]] = {
"target": None,
"alt": None,
}
opt_name: Optional[str] = None
opt_value: Optional[str] = None
for ln in lines[1:]:
def run_mpi_sim(args,
numbermodelruns,
inputfile,
usernamespace,
optparams=None):
"""Run mixed mode MPI/OpenMP simulation - MPI task farm for models with each model parallelised with OpenMP
Args:
args (dict): Namespace with command line arguments
numbermodelruns (int): Total number of model runs.
inputfile (str): Name of the input file to open.
usernamespace (dict): Namespace that can be accessed by user in any Python code blocks in input file.
optparams (dict): Optional argument. For Taguchi optimisation it provides the parameters to optimise and their values.
"""
from mpi4py import MPI
# Define MPI message tags
tags = Enum('tags', {'READY': 0, 'DONE': 1, 'EXIT': 2, 'START': 3})
# Initializations and preliminaries
comm = MPI.COMM_WORLD # get MPI communicator object
size = comm.Get_size() # total number of processes
rank = comm.Get_rank() # rank of this process
status = MPI.Status() # get MPI status object
name = MPI.Get_processor_name() # get name of processor/host
tsimstart = perf_counter()
# Master process
if rank == 0:
modelrun = 1
numworkers = size - 1
closedworkers = 0
print('Master: PID {} on {} using {} workers.'.format(
os.getpid(), name, numworkers))
while closedworkers < numworkers:
data = comm.recv(source=MPI.ANY_SOURCE,
tag=MPI.ANY_TAG,
status=status)
source = status.Get_source()
tag = status.Get_tag()
if tag == tags.READY.value: # Worker is ready, so send it a task
if modelrun < numbermodelruns + 1:
comm.send(modelrun, dest=source, tag=tags.START.value)
print('Master: sending model {} to worker {}.'.format(
modelrun, source))
modelrun += 1
else:
comm.send(None, dest=source, tag=tags.EXIT.value)
elif tag == tags.DONE.value:
print('Worker {}: completed.'.format(source))
elif tag == tags.EXIT.value:
print('Worker {}: exited.'.format(source))
closedworkers += 1
# Worker process
else:
print('Worker {}: PID {} on {}.'.format(rank, os.getpid(), name))
while True:
comm.send(None, dest=0, tag=tags.READY.value)
modelrun = comm.recv(
source=0, tag=MPI.ANY_TAG, status=status
) # Receive a model number to run from the master
tag = status.Get_tag()
# Run a model
if tag == tags.START.value:
if optparams: # If Taguchi optimistaion, add specific value for each parameter to optimise for each experiment to user accessible namespace
tmp = {}
tmp.update((key, value[modelrun - 1])
for key, value in optparams.items())
modelusernamespace = usernamespace.copy()
modelusernamespace.update({'optparams': tmp})
else:
modelusernamespace = usernamespace
run_model(args, modelrun, numbermodelruns, inputfile,
modelusernamespace)
comm.send(None, dest=0, tag=tags.DONE.value)
elif tag == tags.EXIT.value:
break
comm.send(None, dest=0, tag=tags.EXIT.value)
tsimend = perf_counter()
simcompletestr = '\n=== Simulation completed in [HH:MM:SS]: {}'.format(
datetime.timedelta(seconds=int(tsimend - tsimstart)))
print('{} {}\n'.format(
simcompletestr,
'=' * (get_terminal_width() - 1 - len(simcompletestr))))
class GeneticAlgorithm(Generic[C]):
SelectionType = Enum("SelectionType", "ROULETTE TOURNAMENT")
def __init__(
self,
initial_population: List[C],
threshold: float,
max_generations: int = 100,
mutation_chance: float = 0.01,
crossover_chance: float = 0.7,
selection_type: SelectionType = SelectionType.TOURNAMENT) -> None:
self._population: List[C] = initial_population
self._threshold: float = threshold
self._max_generations: int = max_generations
self._mutation_chance: float = mutation_chance
self._crossover_chance: float = crossover_chance
self._selection_type: GeneticAlgorithm.SelectionType = selection_type
self._fitness_key: Callable = type(self._population[0]).fitness
# Use the probability distribution wheel to pick 2 parents
# Note: will not work with negative fitness results
def _pick_roulette(self, wheel: List[float]) -> Tuple[C, C]:
return tuple(choices(self._population, weights=wheel, k=2))
# Choose num_participants at random and take the best 2
def _pick_tournament(self, num_participants: int) -> Tuple[C, C]:
participants: List[C] = choices(self._population, k=num_participants)
return tuple(nlargest(2, participants, key=self._fitness_key))
# Replace the population with a new generation of individuals
def _reproduce_and_replace(self) -> None:
new_population: List[C] = []
# keep going until we've filled the new generation
while len(new_population) < len(self._population):
# pick the 2 parents
if self._selection_type == GeneticAlgorithm.SelectionType.ROULETTE:
parents: Tuple[C, C] = self._pick_roulette(
[x.fitness() for x in self._population])
else:
parents = self._pick_tournament(len(self._population) // 2)
# potentially crossover the 2 parents
if random() < self._crossover_chance:
new_population.extend(parents[0].crossover(parents[1]))
else:
new_population.extend(parents)
# if we had an odd number, we'll have 1 extra, so we remove it
if len(new_population) > len(self._population):
new_population.pop()
self._population = new_population # replace reference
# With _mutation_chance probability mutate each individual
def _mutate(self) -> None:
for individual in self._population:
if random() < self._mutation_chance:
individual.mutate()
# Run the genetic algorithm for max_generations iterations
# and return the best individual found
def run(self) -> C:
best: C = max(self._population, key=self._fitness_key)
for generation in range(self._max_generations):
if best.fitness(
) >= self._threshold: # early exit if we beat threshold
return best
print(
f"Generation {generation} Best {best.fitness()} Avg {mean(map(self._fitness_key, self._population))}"
)
self._reproduce_and_replace()
self._mutate()
highest: C = max(self._population, key=self._fitness_key)
if highest.fitness() > best.fitness():
best = highest # found a new best
return best # best we found in max_generations
BG_DIR = os.path.join(GFX_DIR, "bg")
BGD_DIR = os.path.join(GFX_DIR, "bgd")
CUTIN_DIR = os.path.join(GFX_DIR, "cutin")
EVENT_DIR = os.path.join(GFX_DIR, "events")
FLASH_DIR = os.path.join(GFX_DIR, "flash")
FONT_FOLDER = os.path.join(GFX_DIR, "font")
MENU_DIR = os.path.join(GFX_DIR, "menu")
MOVIEFRAME_DIR = os.path.join(GFX_DIR, "movieframes") # Full-sized background frames
MOVIE_DIR = os.path.join(GFX_DIR, "movies") # Icons for the movie gallery
NAMETAG_DIR = os.path.join(GFX_DIR, "nametags")
PRESENT_DIR = os.path.join(GFX_DIR, "presents")
SPRITE_DIR = os.path.join(GFX_DIR, "sprites")
TEXTBOX_DIR = os.path.join(GFX_DIR, "textbox")
TRIAL_DIR = os.path.join(GFX_DIR, "trial")
TEXT_ALIGN = Enum("left", "right", "center", "offcenter")
#TEXT_V_ALIGN = Enum("normal", "nonstop")
IMG_FILTERS = Enum("unfiltered", "sepia", "inverted")
#SCENE_MODES = Enum("normal", "trial", "rules", "ammo", "ammoname", "present", "presentname", "debate", "mtb", "climax", "anagram", "menu", "map", "report", "report2", "skill", "skill2", "music", "eventname", "moviename", "theatre", "help", "other")
TEXT_FORMAT = {
common.SCENE_MODES.normal: {"x": 18, "y": 202, "w": 444, "h": 24, "a": TEXT_ALIGN.left, "clt": 0, "killblanks": True},
common.SCENE_MODES.trial: {"x": 18, "y": 202, "w": 444, "h": 24, "a": TEXT_ALIGN.left, "clt": 0, "killblanks": True},
common.SCENE_MODES.rules: {"x": 32, "y": 159, "w": 416, "h": 18, "a": TEXT_ALIGN.center, "clt": 0, "killblanks": True},
common.SCENE_MODES.ammo: {"x": 247, "y": 72, "w": 185, "h": 14, "a": TEXT_ALIGN.left, "clt": 7, "killblanks": False},
common.SCENE_MODES.ammoname: {"x": 32, "y": 199, "w": 200, "h": 14, "a": TEXT_ALIGN.center, "clt": 7, "killblanks": True},
common.SCENE_MODES.ammosummary: {"x": 41, "y": 192, "w": 200, "h": 12, "a": TEXT_ALIGN.left, "clt": 7, "killblanks": False},
common.SCENE_MODES.present: {"x": 247, "y": 72, "w": 185, "h": 14, "a": TEXT_ALIGN.left, "clt": 7, "killblanks": False},
common.SCENE_MODES.presentname: {"x": 32, "y": 199, "w": 200, "h": 14, "a": TEXT_ALIGN.center, "clt": 7, "killblanks": True},
common.SCENE_MODES.debate: {"x": 18, "y": 160, "w": 444, "h": 24, "a": TEXT_ALIGN.center, "clt": 8, "killblanks": True},
common.SCENE_MODES.mtb: {"x": 18, "y": 160, "w": 444, "h": 24, "a": TEXT_ALIGN.center, "clt": 12, "killblanks": True},
class Uint128Struct(c.Structure):
_fields_ = [("hi", c.c_uint64), ("lo", c.c_uint64)]
ObjT = Enum(
'ObjT',
[
# These are the only conf object types we care about.
('PROCESS', 0x7200000000000001),
('SERVICE', 0x7300000000000001),
('SDEV', 0x6400000000000001),
('DRIVE', 0x6b00000000000001),
('PROFILE', 0x7000000000000001),
('OBJV', 0x6a00000000000001),
('NODE', 0x6e00000000000001),
('SITE', 0x5300000000000001),
('RACK', 0x6100000000000001),
('ENCLOSURE', 0x6500000000000001),
('CONTROLLER', 0x6300000000000001),
('ROOT', 0x7400000000000001),
('POOL', 0x6f00000000000001),
('PVER', 0x7600000000000001)
])
ObjT.__doc__ = 'Motr conf object types and their m0_fid.f_container values'
class HaNoteStruct(c.Structure):
# Constants for no_state field values as they are described in ha/note.h
def pad(self, word_amount: int, char_amount: int) -> None:
if word_amount >= 0:
self.X_words = np.pad(self.X_words, [(0, 0), (0, word_amount)],
"constant")
else:
self.X_words = self.X_words[:, :word_amount]
if char_amount >= 0:
self.X_chars = np.pad(self.X_chars, [(0, 0), (0, char_amount)],
"constant")
else:
self.X_chars = self.X_chars[:, :char_amount]
ClusterHandling = Enum("ClusterHandling", "CONCAT MEAN CENTER")
class GermanDataset(Dataset):
def __init__(
self,
files: List[str],
gmm_files: List[str],
num_clusterlabels: int,
num_gmm_clusters: int,
window_before: int,
window_after: int,
word_vocab: Optional[Vocab] = None,
char_vocab: Optional[Vocab] = None,
bag_of_words: bool = False,
cluster_handling: ClusterHandling = ClusterHandling.CONCAT
class QueryHistory(models.Model):
"""
Holds metadata about all queries that have been executed.
"""
STATE = Enum('submitted', 'running', 'available', 'failed', 'expired')
SERVER_TYPE = ((BEESWAX, 'Beeswax'), (HIVE_SERVER2, 'Hive Server 2'))
owner = models.ForeignKey(User, db_index=True)
query = models.TextField()
last_state = models.IntegerField(db_index=True)
has_results = models.BooleanField(default=False) # If true, this query will eventually return tabular results.
submission_date = models.DateTimeField(auto_now_add=True)
# In case of multi statements in a query, these are the id of the currently running statement
server_id = models.CharField(max_length=1024, null=True) # Aka secret, only query in the "submitted" state is allowed to have no server_id
server_guid = models.CharField(max_length=1024, null=True, default=None)
statement_number = models.SmallIntegerField(default=0) # The index of the currently running statement
operation_type = models.SmallIntegerField(null=True)
modified_row_count = models.FloatField(null=True)
log_context = models.CharField(max_length=1024, null=True)
server_host = models.CharField(max_length=128, help_text=_('Host of the query server.'), default='')
server_port = models.SmallIntegerField(help_text=_('Port of the query server.'), default=0)
server_name = models.CharField(max_length=128, help_text=_('Name of the query server.'), default='')
server_type = models.CharField(max_length=128, help_text=_('Type of the query server.'), default=BEESWAX, choices=SERVER_TYPE)
query_type = models.SmallIntegerField(help_text=_('Type of the query.'), default=HQL, choices=((HQL, 'HQL'), (IMPALA, 'IMPALA')))
design = models.ForeignKey('SavedQuery', to_field='id', null=True) # Some queries (like read/create table) don't have a design
notify = models.BooleanField(default=False) # Notify on completion
class Meta:
ordering = ['-submission_date']
@staticmethod
def build(*args, **kwargs):
if kwargs['server_type'] == HIVE_SERVER2:
return HiveServerQueryHistory(*args, **kwargs)
else:
return BeeswaxQueryHistory(*args, **kwargs)
def get_full_object(self):
if self.server_type == HiveServerQueryHistory.node_type:
return HiveServerQueryHistory.objects.get(id=self.id)
else:
return BeeswaxQueryHistory.objects.get(id=self.id)
@staticmethod
def get(id):
if QueryHistory.objects.filter(id=id, server_type=BEESWAX).exists():
return BeeswaxQueryHistory.objects.get(id=id)
else:
return HiveServerQueryHistory.objects.get(id=id)
def get_type_name(self):
if self.query_type == 1:
return 'impala'
else:
return 'beeswax'
def get_query_server_config(self):
from beeswax.server.dbms import get_query_server_config
query_server = get_query_server_config(self.get_type_name())
query_server.update({
'server_name': self.server_name,
'server_host': self.server_host,
'server_port': self.server_port,
'server_type': self.server_type,
})
return query_server
def get_current_statement(self):
if self.design is not None:
design = self.design.get_design()
return design.get_query_statement(self.statement_number)
else:
return self.query
def is_finished(self):
is_statement_finished = not self.is_running()
if self.design is not None:
design = self.design.get_design()
return is_statement_finished and self.statement_number + 1 == design.statement_count # Last statement
else:
return is_statement_finished
def is_running(self):
return self.last_state in (QueryHistory.STATE.running.index, QueryHistory.STATE.submitted.index)
def is_success(self):
return self.last_state in (QueryHistory.STATE.available.index,)
def is_failure(self):
return self.last_state in (QueryHistory.STATE.expired.index, QueryHistory.STATE.failed.index)
def set_to_running(self):
self.last_state = QueryHistory.STATE.running.index
def set_to_failed(self):
self.last_state = QueryHistory.STATE.failed.index
def set_to_available(self):
self.last_state = QueryHistory.STATE.available.index
except Exception as e:
log.warning("Failed to convert attribute '%s'='%s': %s" % \
(name, value, e))
return default
def _write_xml_bbox(bbox, parent_elem):
x, y, w, h = bbox
bbox_elem = ET.SubElement(parent_elem, 'bndbox')
ET.SubElement(bbox_elem, 'xmin').text = str(x)
ET.SubElement(bbox_elem, 'ymin').text = str(y)
ET.SubElement(bbox_elem, 'xmax').text = str(x + w)
ET.SubElement(bbox_elem, 'ymax').text = str(y + h)
return bbox_elem
LabelmapType = Enum('LabelmapType', ['voc', 'source'])
class VocConverter(Converter):
DEFAULT_IMAGE_EXT = VocPath.IMAGE_EXT
BUILTIN_ATTRS = {'difficult', 'pose', 'truncated', 'occluded' }
@staticmethod
def _split_tasks_string(s):
return [VocTask[i.strip()] for i in s.split(',')]
@staticmethod
def _get_labelmap(s):
if osp.isfile(s):
return s
try:
return LabelmapType[s].name
class TestState(util.SubscribableStateMixin):
"""This class handles tracking the state of a running Test.
This class encapsulates all the interesting transient bits of a running Test,
as opposed to the openhtf.TestDescriptor class, which encapsulates static
data associated with a Test (that is, it remains the same across invocations
of Test.Execute()).
Init Args:
test_desc: openhtf.TestDescriptor instance describing the test to run,
used to initialize some values here, but it is not modified.
Attributes:
test_record: TestRecord instance for the currently running test.
logger: Logger that logs to test_record's log_records attribute.
running_phase_state: PhaseState object for the currently running phase,
if any, otherwise None.
user_defined_state: Dictionary for users to persist state across phase
invokations. It's passed to the user via test_api.
test_api: An openhtf.TestApi instance for passing to test phases,
providing test authors access to necessary state information, while
protecting internal-only structures from being accidentally modified.
Note that if there is no running phase, test_api is also None.
execution_uid: A UUID that is specific to this execution.
"""
Status = Enum('Status', ['WAITING_FOR_TEST_START', 'RUNNING', 'COMPLETED'])
def __init__(self, test_desc, execution_uid):
super(TestState, self).__init__()
self._status = self.Status.WAITING_FOR_TEST_START
self.test_record = test_record.TestRecord(
dut_id=None,
station_id=conf.station_id,
code_info=test_desc.code_info,
# Copy metadata so we don't modify test_desc.
metadata=copy.deepcopy(test_desc.metadata))
self.logger = logs.initialize_record_logger(execution_uid,
self.test_record,
self.notify_update)
self.plug_manager = plugs.PlugManager(test_desc.plug_types,
self.logger)
self.running_phase_state = None
self.user_defined_state = {}
self.execution_uid = execution_uid
@property
def test_api(self):
"""Create a TestApi for access to this TestState.
The returned TestApi should be passed as the first argument to test
phases. Note that the return value is none if there is no
self.running_phase_state set. As such, this attribute should only
be accessed within a RunningPhaseContext().
"""
running_phase_state = self.running_phase_state
return (running_phase_state and openhtf.TestApi(
self.logger, self.user_defined_state, self.test_record,
measurements.Collection(running_phase_state.measurements),
running_phase_state.attachments, running_phase_state.attach,
running_phase_state.attach_from_file, self.notify_update))
@contextlib.contextmanager
def running_phase_context(self, phase_desc):
"""Create a context within which a single phase is running.
Yields a PhaseState object for tracking transient state during the
execution of the phase, including the output PhaseRecord. That PhaseState
provides the TestApi to be passed into the test phase.
Within this context, the Station API will report the given phase as the
currently running phase.
"""
assert not self.running_phase_state, 'Phase already running!'
self.running_phase_state = PhaseState.from_descriptor(
phase_desc, self.notify_update)
try:
with self.running_phase_state.record_timing_context:
self.notify_update() # New phase started.
yield self.running_phase_state
finally:
# Clear notification callbacks so we can serialize measurements.
for meas in self.running_phase_state.measurements.values():
meas.set_notification_callback(None)
self.test_record.phases.append(
self.running_phase_state.phase_record)
self.running_phase_state = None
self.notify_update() # Phase finished.
def _asdict(self):
"""Return a dict representation of the test's state."""
return {
'status': self._status,
'test_record': self.test_record,
'plugs': self.plug_manager._asdict(),
'running_phase_state': self.running_phase_state,
}
@property
def is_finalized(self):
return self._status == self.Status.COMPLETED
@property
def last_run_phase_name(self):
"""Get the name of the currently running phase, or None.
Note that this name is not guaranteed to still be accurate by the time this
method returns, so this should only be used for log messages/user display
and not for programmatic purposes.
"""
return self.running_phase_state and self.running_phase_state.name
def set_status_from_phase_outcome(self, phase_outcome):
"""Set our internal state based on the given phase outcome.
Args:
phase_outcome: An instance of phase_executor.PhaseOutcome
Returns: True if the test has finished prematurely (failed).
"""
assert not self.is_finalized, 'Test already completed!'
# Handle a few cases where the test is ending prematurely.
if phase_outcome.raised_exception:
self.logger.debug('Finishing test execution early due to phase '
'exception, outcome ERROR.')
result = phase_outcome.phase_result
if isinstance(result, phase_executor.ExceptionInfo):
code = result.exc_type.__name__
description = str(result.exc_val).decode('utf8', 'replace')
else:
# threads.ThreadTerminationError gets str'd directly.
code = str(type(phase_outcome.phase_result).__name__)
description = str(phase_outcome.phase_result).decode(
'utf8', 'replace')
self.test_record.add_outcome_details(code, description)
self.finalize(test_record.Outcome.ERROR)
elif phase_outcome.is_timeout:
self.logger.debug('Finishing test execution early due to phase '
'timeout, outcome TIMEOUT.')
self.finalize(test_record.Outcome.TIMEOUT)
elif phase_outcome.phase_result == openhtf.PhaseResult.STOP:
self.logger.debug('Finishing test execution early due to '
'PhaseResult.STOP, outcome FAIL.')
# TODO(madsci): Decouple flow control from pass/fail.
self.finalize(test_record.Outcome.ABORTED)
if self.is_finalized != phase_outcome.is_terminal:
raise openhtf.InvalidTestStateError(
'Unexpected finalized state (%s) after PhaseOutcome %s.',
self.is_finalized, phase_outcome)
return self.is_finalized
def mark_test_started(self):
"""Set the TestRecord's start_time_millis field."""
# Blow up instead of blowing away a previously set start_time_millis.
assert self.test_record.start_time_millis is 0
self.test_record.start_time_millis = util.time_millis()
self.notify_update()
def set_status_running(self):
"""Mark the test as actually running, can't be done once finalized."""
assert self._status == self.Status.WAITING_FOR_TEST_START
self._status = self.Status.RUNNING
self.notify_update()
def finalize(self, test_outcome=None):
"""Mark the state as finished.
This method is called with no arguments on normal test completion, or
with an argument if the test stopped under some other condition, where
the test_outcome argument specifies what the Test's outcome was.
When a Test completes normally, the outcome will be either PASS or FAIL,
depending on measurements' PASS/FAIL status. Any UNSET measurements will
cause the Test to FAIL unless conf.allow_unset_measurements is set True.
Args:
test_outcome: If specified, use this as the Test outcome.
"""
assert not self.is_finalized, 'Test already completed!'
# Sanity check to make sure we have a DUT ID by the end of the test.
if not self.test_record.dut_id:
raise BlankDutIdError(
'Blank or missing DUT ID, HTF requires a non-blank ID.')
if test_outcome:
# Override measurement-based PASS/FAIL with a specific test outcome.
self.test_record.outcome = test_outcome
else:
allowed_outcomes = {measurements.Outcome.PASS}
if conf.allow_unset_measurements:
allowed_outcomes.add(measurements.Outcome.UNSET)
if any(meas.outcome not in allowed_outcomes
for phase in self.test_record.phases
for meas in phase.measurements.itervalues()):
self.test_record.outcome = test_record.Outcome.FAIL
else:
self.test_record.outcome = test_record.Outcome.PASS
# A message has already been logged if we were called with test_outcome
# set, but if we're finishing normally, log it here.
self.logger.debug(
'Finishing test execution normally with outcome %s.',
self.test_record.outcome.name)
# The test is done at this point, no further updates to test_record.
self.logger.handlers = []
self.test_record.end_time_millis = util.time_millis()
self._status = self.Status.COMPLETED
self.notify_update()
def __str__(self):
return '<%s: %s@%s Running Phase: %s>' % (
type(self).__name__,
self.test_record.dut_id,
self.test_record.station_id,
self.last_run_phase_name,
)
return items
@staticmethod
def _lazy_extract_mask(mask, c):
return lambda: mask == c
class CamvidImporter(Importer):
@classmethod
def find_sources(cls, path):
return cls._find_sources_recursive(path, '.txt', 'camvid',
file_filter=lambda p: osp.basename(p) != CamvidPath.LABELMAP_FILE)
LabelmapType = Enum('LabelmapType', ['camvid', 'source'])
class CamvidConverter(Converter):
DEFAULT_IMAGE_EXT = CamvidPath.IMAGE_EXT
@classmethod
def build_cmdline_parser(cls, **kwargs):
parser = super().build_cmdline_parser(**kwargs)
parser.add_argument('--apply-colormap', type=str_to_bool, default=True,
help="Use colormap for class masks (default: %(default)s)")
parser.add_argument('--label-map', type=cls._get_labelmap, default=None,
help="Labelmap file path or one of %s" % \
', '.join(t.name for t in LabelmapType))
def __init__(self, extractor, save_dir,
from enum import Enum
import mutablerecords
from openhtf.util import logs
_LOG = logging.getLogger(__name__)
class InvalidMeasurementDimensions(Exception):
"""Raised when a measurement is taken with the wrong number of dimensions."""
OutcomeDetails = collections.namedtuple('OutcomeDetails', 'code description')
Outcome = Enum('Outcome', ['PASS', 'FAIL', 'ERROR', 'TIMEOUT', 'ABORTED']) # pylint: disable=invalid-name
# LogRecord is in openhtf.util.logs.LogRecord.
class Attachment(collections.namedtuple('Attachment', 'data mimetype')):
"""Encapsulate attachment data and guessed MIME type."""
@property
def sha1(self):
return hashlib.sha1(self.data).hexdigest()
class TestRecord( # pylint: disable=no-init
mutablerecords.Record(
'TestRecord', ['dut_id', 'station_id'], {
'start_time_millis': int,
'end_time_millis': None,
