def write_dot_graph(
member_meta: Dict[str, Dict[str, str]],
local_deps: DepMap,
areas: Dict[str, List[str]],
out: IO,
) -> None:
def disp(val: str, out: IO = out, **kwargs: Any) -> None:
print(val, file=out, **kwargs)
disp("digraph packages {")
for area, members in areas.items():
disp(f" subgraph cluster_{area} " "{")
disp(f' label = "/{area}";')
disp(" color = blue;")
for member in members:
description = member_meta[member]["description"]
disp(f' "{member}" [tooltip="{description}"', end="")
if member_meta[member]["has_bin"]:
disp(",shape=Mdiamond,color=red", end="")
disp("];")
disp(" }")
for package, deps in local_deps.items():
for dep in deps:
disp(
f' "{package}" -> "{dep}" [edgetooltip="{package} -> {dep}",URL="none"',
end="",
)
if dep in ("timely", "differential-dataflow"):
disp("color=green,style=dashed", end="")
disp("];")
disp("}")
out.flush()
def input_ex(s: str,
file_out: IO = sys.stdout,
file_in: IO = sys.stdin) -> str:
print(s, file=file_out, end="")
file_out.flush()
return sys.stdin.readline().rstrip("\n")
def write_to_csv(file: IO, frames: List[List]):
"""
Write a list of frame to a csv file.
:file (IO) An opened file descriptor
:frames (List[List]) A list of list of values
(Each element contains the equivalent of a frame dictionary, as a list)
Doesn't close the file when done, be careful
"""
for frame in frames:
for it in range(0, 2):
if isinstance(frame[it], int):
frame[it] = dt.utcfromtimestamp(
frame[it]).strftime('%Y-%m-%d %H:%M:%S')
else:
try:
frame[it] = dt.strptime(frame[it], '%Y-%m-%d %H:%M:%S')
except ValueError:
frame[it] = dt.strptime(frame[it].split('.')[0],
'%Y-%m-%d %H:%M:%S')
writer = csv.writer(file, delimiter='\t')
writer.writerows(frames)
file.flush()
file.seek(0)
def _write_equivalences_file(self, fp: IO, equivalences: List[Equivalence]) -> None:
# I don't even know why this is a thing.
# cf. https://stackoverflow.com/questions/51272814/python-yaml-dumping-pointer-references
yaml = YAML()
yaml.representer.ignore_aliases = lambda *data: True
yaml.dump({"equivalences": [e.to_json() for e in equivalences]}, fp)
fp.flush()
def write_result_to_file(self, output_file: IO, data: Dict[str, object]):
columns: List[str] = data["algorithm"].get_columns()
output_data = [data.get(column) for column in columns]
output: str = f'{" ".join(map(str, output_data))}'
output = output.replace("[", "").replace("]", "").replace("\n", "").replace(".cnf", "")
output = re.sub('\s+', ' ', output).strip()
output_file.write(f'{output}\n')
output_file.flush()
def write_result_to_file(self, output_file: IO, data: Dict[str, object]):
columns: List[str] = data["algorithm"].get_columns()
if data.get("things") is not None:
data["things"] = "".join(map(str, data["things"]))
output_data = [data.get(column) for column in columns]
output: str = f'{" ".join(map(str, output_data))}'
output_file.write(f'{output}\n')
output_file.flush()
def _write_multiple_objects(
self, f: IO, object_refs: List[ObjectRef], owner_addresses: List[str], url: str
) -> List[str]:
"""Fuse all given objects into a given file handle.
Args:
f(IO): File handle to fusion all given object refs.
object_refs(list): Object references to fusion to a single file.
owner_addresses(list): Owner addresses for the provided objects.
url(str): url where the object ref is stored
in the external storage.
Return:
List of urls_with_offset of fused objects.
The order of returned keys are equivalent to the one
with given object_refs.
"""
keys = []
offset = 0
ray_object_pairs = self._get_objects_from_store(object_refs)
for ref, (buf, metadata), owner_address in zip(
object_refs, ray_object_pairs, owner_addresses
):
address_len = len(owner_address)
metadata_len = len(metadata)
if buf is None and len(metadata) == 0:
error = f"Object {ref.hex()} does not exist."
raise ValueError(error)
buf_len = 0 if buf is None else len(buf)
payload = (
address_len.to_bytes(8, byteorder="little")
+ metadata_len.to_bytes(8, byteorder="little")
+ buf_len.to_bytes(8, byteorder="little")
+ owner_address
+ metadata
+ (memoryview(buf) if buf_len else b"")
)
# 24 bytes to store owner address, metadata, and buffer lengths.
payload_len = len(payload)
assert (
self.HEADER_LENGTH + address_len + metadata_len + buf_len == payload_len
)
written_bytes = f.write(payload)
assert written_bytes == payload_len
url_with_offset = create_url_with_offset(
url=url, offset=offset, size=written_bytes
)
keys.append(url_with_offset.encode())
offset += written_bytes
# Necessary because pyarrow.io.NativeFile does not flush() on close().
f.flush()
return keys
def flush_file_handle(file_handle: IO):
"""
Attempts to flush the handle. First tries to call .flush and then
tries to apply `os.fsync` to the file descriptor to truly flush the
buffer to the file.
:param file_handle: The file handle to attempt to flush
:type file_handle: IO
"""
if hasattr(file_handle, 'flush'):
file_handle.flush()
if hasattr(file_handle, 'fileno'):
os.fsync(file_handle.fileno())
def _print_simple(num_completed: int, num_simulations: int, td: timedelta,
end: str, fd: IO) -> None:
if fd.closed:
return
print(
timedelta(td.days, td.seconds),
num_completed,
'of',
num_simulations,
'simulations',
f'({num_completed / num_simulations:.0%})',
end=end,
file=fd,
)
fd.flush()
def _set_output(plumed_file: str, colvar_output_file: str,
running_file: typing.IO) -> None:
"""Copy template plumed input and set printed output file.
Parameters
----------
plumed_file
Template file that contains the CVs to be calculated.
colvar_output_file
File for the CVs to be printed to
running_file
Open file in append mode to copy plumed and print statement to
"""
with open(plumed_file, "r") as source:
shutil.copyfileobj(source, running_file)
running_file.write(f"PRINT ARG=* FILE={colvar_output_file}")
running_file.flush()
def fdump(self, f: IO, size=0x8000, console=sys.stdout) -> None:
self._send(b'd') # send dump command
cnt = 0
while cnt < size:
buf = self._receive(ack=True)[:size - cnt]
f.write(buf)
f.flush()
cnt += len(buf)
if console:
print('\r%d%%' % ((cnt / 0x8000) * 100), end='', file=console)
if console:
print('\nComplete.', file=console)
if size < 0x8000:
self.reset()
# consume the remaining packet
self._receive(ack=False)
def start(input: IO, output: IO):
env = obj.Environment()
while True:
print(PROMPT, end="")
output.flush()
scanned: str = input.readline()
if not scanned:
return
lex = lexer.Lexer(scanned)
psr = parser.Parser(lex)
program = psr.parse()
if len(psr.errors) > 0:
print_parser_errors(output, psr.errors)
continue
evaluated = evaluator.eval(program, env)
if evaluated:
print(str(evaluated), file=output)
def _print_progress(
sim_index: Optional[int],
now: Union[int, float],
t_stop: Optional[Union[int, float]],
timescale: TimeValue,
end: str,
fd: IO,
) -> None:
parts = []
if sim_index:
parts.append(f'Sim {sim_index}')
magnitude, units = timescale
if magnitude == 1:
parts.append(f'{now:6.0f} {units}')
else:
parts.append(f'{magnitude}x{now:6.0f} {units}')
if t_stop:
parts.append(f'({100 * now / t_stop:.0f}%)')
else:
parts.append('(N/A%)')
print(*parts, end=end, file=fd)
fd.flush()
def silence(fd: IO):
"""Silence any output from fd."""
from os import close, dup, dup2, fdopen, pipe
# Do not silence when debugging.
if not DEBUG:
# Backup the file
old_fd = fd
# Flush the file so it can be silenced properly.
fd.flush()
# Create a duplicate of fd
new_fd = dup(fd.fileno())
# Create a pipe to write to.
read, write = pipe()
# Set the write to the fd filenumber
dup2(write, fd.fileno())
# Close the pipe.
close(write)
close(read)
# Set fd to the new fd
fd = fdopen(new_fd, 'w')
try:
# Run the commands in the 'with' statement.
yield
finally:
if not DEBUG:
# Return the fd back to its original state.
dup2(fd.fileno(), old_fd.fileno())
fd = old_fd
def _print(self, stream: IO, message: str, **kwargs: Any) -> None:
if None in (stream, message):
return
stream_tty = stream.isatty()
print_tty = kwargs.pop('tty', True) if self._tty else kwargs.pop(
'tty', False)
print_notty = kwargs.pop('notty', True) if self._notty else kwargs.pop(
'notty', False)
if (stream_tty and print_tty) or (not stream_tty and print_notty):
prefix = None
if kwargs.pop('prefix', self._prefix):
if callable(self._prefix):
prefix = self._prefix()
elif self._prefix:
prefix = str(self._prefix)
message = f'{prefix} {message}' if prefix else message
if not stream.isatty() or not kwargs.pop('colors_enabled',
self._colors_enabled):
message = self.strip_style(message)
else:
style_args = {
k: v
for (k, v) in kwargs.items() if k in _style_keys
}
if len(style_args) > 0:
message = self.style(message, **style_args)
stream.write(message)
endl = kwargs.pop('endl', self._endl)
stream.write(endl)
stream.flush()
def _write_decrypted_blob(handle: typing.IO, data: bytes):
handle.write(data)
handle.flush()
def evaluate(recognizer: TimedChessRecognizer,
output_file: typing.IO,
dataset_folder: Path,
save_fens: bool = False):
"""Perform the performance evaluation, saving the results to a CSV output file.
Args:
recognizer (TimedChessRecognizer): the instance of the chess recognition pipeline
output_file (typing.IO): the output file object
dataset_folder (Path): the folder of the dataset to evaluate
save_fens (bool, optional): whether to save the FEN outputs for every sample. Defaults to False.
"""
time_keys = [
"corner_detection", "occupancy_classification", "piece_classification",
"prepare_results"
]
output_file.write(",".join([
"file", "error", "num_incorrect_squares", "num_incorrect_corners",
"occupancy_classification_mistakes", "piece_classification_mistakes",
"actual_num_pieces", "predicted_num_pieces", *(
["fen_actual", "fen_predicted", "fen_predicted_is_valid"]
if save_fens else []), "time_corner_detection",
"time_occupancy_classification", "time_piece_classification",
"time_prepare_results"
]) + "\n")
for i, img_file in enumerate(dataset_folder.glob("*.png")):
json_file = img_file.parent / (img_file.stem + ".json")
with json_file.open("r") as f:
label = json.load(f)
img = cv2.imread(str(img_file))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
groundtruth_board = chess.Board(label["fen"])
groundtruth_corners = sort_corner_points(np.array(label["corners"]))
error = None
try:
predicted_board, predicted_corners, times = recognizer.predict(
img, label["white_turn"])
except RecognitionException as e:
error = e
predicted_board = chess.Board()
predicted_board.clear_board()
predicted_corners = np.zeros((4, 2))
times = {k: -1 for k in time_keys}
mistakes = _get_num_mistakes(groundtruth_board, predicted_board)
incorrect_corners = np.sum(
np.linalg.norm(groundtruth_corners - predicted_corners, axis=-1) >
(10 / 1200 * img.shape[1]))
occupancy_mistakes = _get_num_occupancy_mistakes(
groundtruth_board, predicted_board)
piece_mistakes = _get_num_piece_mistakes(groundtruth_board,
predicted_board)
output_file.write(",".join(
map(str, [
img_file.name, error, mistakes, incorrect_corners,
occupancy_mistakes, piece_mistakes,
len(groundtruth_board.piece_map()),
len(predicted_board.piece_map()), *([
groundtruth_board.board_fen(),
predicted_board.board_fen(),
predicted_board.status() == Status.VALID
] if save_fens else []), *(times[k] for k in time_keys)
])) + "\n")
if (i + 1) % 5 == 0:
output_file.flush()
logging.info(f"Processed {i+1} files from {dataset_folder}")
def _finish(cls, file: IO, *, at_exit: bool = False) -> None:
print(cls.cursor_show, end='', file=file)
file.flush()
def _w(fd: IO, *items: Any, flush: bool = True):
"""Writes all items to specified file descriptor and flushes the fd if flush is set
to True."""
fd.write("".join([item.__str__() for item in items]))
if flush:
fd.flush()
def overwrite_helper(
out_path: typing.Union[str, pathlib.Path],
contents: str,
*,
do_write: bool,
show_diff: bool,
show_diff_side_by_side: bool = False,
answer_yes: bool = False,
out_file: typing.IO = sys.stderr,
) -> None:
out_path_obj = pathlib.Path(out_path)
with tempfile.NamedTemporaryFile(mode="w+t") as sheet_file:
lines = []
# Write sheet to temporary file.
sheet_file.write(contents)
sheet_file.flush()
sheet_file.seek(0)
new_lines = sheet_file.read().splitlines(keepends=False)
# Compare sheet with output if exists and --show-diff given.
if show_diff:
if out_path != "-" and out_path_obj.exists():
with out_path_obj.open("rt") as inputf:
old_lines = inputf.read().splitlines(keepends=False)
else:
old_lines = []
if not show_diff_side_by_side:
lines = list(
difflib.unified_diff(old_lines,
new_lines,
fromfile=str(out_path),
tofile=str(out_path)))
for line in lines:
if line.startswith(("+++", "---")):
print(colored(line, color="white", attrs=("bold", )),
end="",
file=out_file)
elif line.startswith("@@"):
print(colored(line, color="cyan", attrs=("bold", )),
end="",
file=out_file)
elif line.startswith("+"):
print(colored(line, color="green", attrs=("bold", )),
file=out_file)
elif line.startswith("-"):
print(colored(line, color="red", attrs=("bold", )),
file=out_file)
else:
print(line, file=out_file)
else:
cd = icdiff.ConsoleDiff(cols=get_terminal_columns(),
line_numbers=True)
lines = list(
cd.make_table(
old_lines,
new_lines,
fromdesc=str(out_path),
todesc=str(out_path),
context=True,
numlines=3,
))
for line in lines:
line = "%s\n" % line
if hasattr(out_file, "buffer"):
out_file.buffer.write(
line.encode("utf-8")) # type: ignore
else:
out_file.write(line)
out_file.flush()
if not lines:
logger.info("File %s not changed, no diff...", out_path)
# Actually copy the file contents.
if (not show_diff or lines) and do_write:
logger.info("About to write file contents to %s", out_path)
sheet_file.seek(0)
if out_path == "-":
shutil.copyfileobj(sheet_file, sys.stdout)
else:
if show_diff:
logger.info("See above for the diff that will be applied.")
if answer_yes or input("Is this OK? [yN] ").lower().startswith(
"y"):
with out_path_obj.open("wt") as output_file:
shutil.copyfileobj(sheet_file, output_file)
def send_midi(midi_out: IO, b: bytes):
midi_out.write(b)
midi_out.flush()
def writefp(self, fp: typing.IO) -> None:
for line in self.dumplines():
fp.write(line)
fp.flush()
def generate_next_layer(name,
input_feature,
input_label,
n_train,
n_test,
statfile: typing.IO,
patch_radius=2,
nystrom_dim=200,
pooling_size=2,
pooling_stride=2,
gamma=2,
regularization_param=100,
learning_rate=0.2,
crop_ratio=1,
n_iter=5000,
chunk_size=5000,
max_channel=16):
X_raw = input_feature
label = input_label
n = n_train + n_test
# detecting image parameters
pixel_per_image = X_raw.shape[2]
pixel_per_side = int(math.sqrt(pixel_per_image))
patch_per_side = int(pixel_per_side - 2 * patch_radius)
patch_per_image = patch_per_side * patch_per_side
patch_size = patch_radius * 2 + 1
pixel_per_patch = patch_size * patch_size
pooling_per_side = int(patch_per_side / pooling_stride)
pooling_per_image = pooling_per_side * pooling_per_side
tprint("Raw size = " + str(X_raw.shape))
n_channel = min(max_channel, X_raw.shape[1])
selected_channel_list = range(0, n_channel)
selected_group_size = [n_channel]
feature_dim = len(selected_group_size) * nystrom_dim
# construct patches
tprint("Construct patches...")
print(
"patch : n = {}, patch per image = {}, selected_channel_list = {}, pixel_per_patch = {}"
.format(n, patch_per_image, len(selected_channel_list),
pixel_per_patch))
patch = np.zeros(
(n, patch_per_image, len(selected_channel_list), pixel_per_patch),
dtype=np.float32)
for y in range(0, patch_per_side):
for x in range(0, patch_per_side):
for i in selected_channel_list:
indices = get_pixel_vector(x + patch_radius, y + patch_radius,
patch_radius, pixel_per_side)
patch_id = x + y * patch_per_side
patch[:, patch_id, i] = X_raw[:, selected_channel_list[i],
indices]
tprint("Patch size = " + str(patch.shape))
# local contrast normalization and ZCA whitening
tprint('local contrast normalization and ZCA whitening...')
patch = patch.reshape((n * patch_per_image, n_channel * pixel_per_patch))
patch -= np.mean(patch, axis=1).reshape((patch.shape[0], 1))
patch /= LA.norm(patch, axis=1).reshape((patch.shape[0], 1)) + 0.1
patch = zca_whitening(patch)
patch = patch.reshape((n, patch_per_image, n_channel, pixel_per_patch))
# create features
tprint("Create features...")
transformer = [0]
base = 0
X_reduced = np.zeros((n, pooling_per_image, feature_dim), dtype=np.float32)
print(
"X_reduced : n = {}, pooling_per_image = {}, feature_dim = {}".format(
n, pooling_per_image, feature_dim))
while base < n:
tprint(" sample id " + str(base) + "-" +
str(min(n, base + chunk_size)))
X_reduced[base:min(n, base +
chunk_size)], transformer = transform_and_pooling(
patch=patch[base:min(n, base + chunk_size)],
transformer=transformer,
selected_group_size=selected_group_size,
gamma=gamma,
nystrom_dim=nystrom_dim,
patch_per_side=patch_per_side,
pooling_size=pooling_size,
pooling_stride=pooling_stride)
base = min(n, base + chunk_size)
gc.collect()
# normalization
X_reduced = X_reduced.reshape((n * pooling_per_image, feature_dim))
X_reduced -= np.mean(X_reduced, axis=0)
X_reduced /= LA.norm(X_reduced) / math.sqrt(n * pooling_per_image)
X_reduced = X_reduced.reshape((n, pooling_per_image * feature_dim))
# Learning_filters
tprint("Training...")
binary_label = label_binarize(label, classes=range(0, 10))
filter, stats = low_rank_matrix_regression(X_train=X_reduced[0:n_train],
Y_train=binary_label[0:n_train],
X_test=X_reduced[n_train:],
Y_test=binary_label[n_train:],
d1=pooling_per_image,
d2=feature_dim,
n_iter=n_iter,
reg=regularization_param,
learning_rate=learning_rate,
ratio=crop_ratio)
filter_dim = filter.shape[0]
tprint("Apply filters...")
output = np.dot(X_reduced.reshape((n * pooling_per_image, feature_dim)),
filter.T)
output = np.reshape(output, (n, pooling_per_image, filter_dim))
output = np.transpose(output, (0, 2, 1))
tprint("feature dimension = " + str(output[0].size))
# print stats
for key in stats:
statfile.write('{}_{}\t{}\n'.format(name, key, stats[key]))
statfile.flush()
return output
def zip_extract(zip_file: zipfile.ZipFile, file_name: str,
target_file_obj: IO):
with zip_file.open(file_name) as fp:
shutil.copyfileobj(fp, target_file_obj)
target_file_obj.flush()
def gunzip(gzip_file_name: str, target_file_obj: IO):
with gzip.open(gzip_file_name, "rb") as gzip_file:
shutil.copyfileobj(gzip_file, target_file_obj)
target_file_obj.flush()
def loss_log_callback(step: int, loss: float, theta: np.ndarray,
grad: np.ndarray, opt_state: Any, file_handle: IO):
file_handle.write(f'{step},{loss}\n')
file_handle.flush()
