def match_using_block_desc(pblks_labeled):
start = time.time()
#Ensure that physical block (1) has not been labeled already and (2) intersects streets with names
temp_pblk_st_dict = {
k: v
for k, v in pblk_st_dict.items()
if (int(k) not in pblks_labeled) and (len(v) > 0)
}
#Use multiprocessing to compare street lists for matching blocks
pool = Pool(4)
pool_results = pool.map(check_block, temp_pblk_st_dict.items())
pool.close()
#Convert list of dictionaries into one dictionary, as long as there is only one microdata block guess
temp_match_dict = {
k: v[0]
for d in pool_results for k, v in d.items() if len(v) == 1
}
temp_pblks_labeled, temp_labels = zip(*temp_match_dict.items())
num_matches = len(temp_match_dict)
per_micro_blocks = round(100 * float(num_matches) / num_micro_blocks, 1)
end = time.time()
run_time = round(float(end - start) / 60, 1)
cprint("Matches based on block description: " + str(num_matches) + " (" +
str(per_micro_blocks) + r"% of microdata blocks)",
file=AnsiToWin32(sys.stdout))
cprint("Matching took " + str(run_time) + " minutes to complete\n",
'cyan',
file=AnsiToWin32(sys.stdout))
return temp_match_dict, list(temp_pblks_labeled), list(
temp_labels), num_matches
def wrap(file_obj, use_color=True):
"""
Wrap file_obj in another stream which handles ANSI color codes using colorama
NOTE:
imports colorama here to avoid dependency from setup.py importing VUnit before colorama is installed
"""
from colorama import AnsiToWin32 # pylint: disable=import-outside-toplevel
if use_color:
return AnsiToWin32(file_obj).stream
return AnsiToWin32(file_obj, strip=True, convert=False).stream
def __init__(
self,
stream=None,
colorizer=None,
highlighter=None,
attributes_map=None,
):
"""
Initializes a colorizing stream handler.
:param stream: The stream to use for output.
:param colorizer: The colorizer to use for colorizing the output. If
not specified, a :class:`chromalog.colorizer.Colorizer` is
instantiated.
:param highlighter: The colorizer to use for highlighting the output
when color is not supported.
:param attributes_map: A map of LogRecord attributes/color tags.
"""
if not stream:
stream = sys.stderr
self.has_color_support = stream_has_color_support(stream)
self.color_disabled = False
self.attributes_map = attributes_map or self.default_attributes_map
if self.has_color_support:
stream = AnsiToWin32(stream).stream
super(ColorizingStreamHandler, self).__init__(stream)
self.colorizer = colorizer or Colorizer()
self.highlighter = highlighter
self.setFormatter(ColorizingFormatter())
def wrap_stream(stream, convert, strip, autoreset, wrap):
if wrap:
wrapper = AnsiToWin32(stream, convert=convert,
strip=strip, autoreset=autoreset)
if wrapper.should_wrap():
stream = wrapper.stream
return stream
def _init_log(self):
if not self.is_log_initiated:
self.log = AnsiToWin32(
open(_get_log_file_name(_get_logs_dir_name()),
"a",
encoding="utf-8")).stream
self.is_log_initiated = True
def __init__(self, f=None, colorize=True, level=None, log_format=None):
'''
log_format supports:
time -> Time in local timezone. (like what `date` outputs)
clock -> Timestamp delta from first call to import logger.
level -> Log level
message -> What you want to log
'''
self._initTime = time.time()
if level is not None:
self._level = level
else:
self._level = Level.DEBUG
if f is not None:
self.fd = f
else:
self.fd = sys.stderr
if log_format is None:
self._format = '[{clock}][{level}] {message}'
else:
self._format = log_format
self.colorize = colorize and self.fd.isatty()
if sys.platform.startswith(
'win') and self.colorize: # Windows wrapper.
self.fd = AnsiToWin32(self.fd).stream
self._log(Level.INFO, 'Logger initialized at %s' % time.asctime())
def __init__(self,
log_folder=None,
log_level='info',
output_level='success',
out_stream=sys.stdout,
output_to_stderr=False):
"""Initialize the logger."""
self.colored_out = AnsiToWin32(out_stream).stream
t = self.get_type(log_level)
if t != 'unknown':
self.log_level = LEVEL[t]
else:
if isinstance(log_level, int):
self.log_level = log_level # discouraged
elif str(log_level).lower() == 'none':
self.log_level = 5
else:
self.log_level = 1
t = self.get_type(output_level)
if t != 'unknown':
self.output_level = LEVEL[t]
else:
if isinstance(output_level, int):
self.output_level = output_level # discouraged
elif str(output_level).lower() == 'none':
self.output_level = 5
else:
self.output_level = 2
self.output_to_stderr = output_to_stderr
self.log_folder = None
if log_folder != None:
self.log_folder = log_folder
if (not os.path.isdir(log_folder)):
os.makedirs(log_folder)
self.make_logs()
def __init__(self, config, *args):
# no color on Windows yet, this keeps the output from looking insane with all the ANSI
if os.name == 'nt':
self.stream = AnsiToWin32(sys.stdout, convert=True).stream
init()
#init(convert=True)
commands.Command.__init__(self, config, *args)
config.add_option('SuperVerbose',
short_option='s',
help='Display per page validation results.',
action='store_true',
default=False)
config.add_option('ExtraTotals',
short_option='x',
help='List of all misses per-module.',
action='store_true',
default=False)
config.add_option('DumpFolder',
short_option='D',
help='Dump the failed blocks to a specified folder',
default=None)
config.add_option(
'FailFile',
short_option='F',
help=
'Output file containing detailed information about unverifiable memory',
default='FailedValidation.txt')
if os.name is not 'nt':
os.system('setterm -cursor off')
else:
os.system('color 0f')
def setOutputStream(self, stream):
if platform.startswith('win'):
on_stream = AnsiToWin32(stream).stream
else:
on_stream = stream
off_stream = open(os.devnull, 'w')
self.stream = SpecOutputStream(on_stream, off_stream)
return self.stream
def __init__(self, stream):
logging.StreamHandler.__init__(self, stream)
if IS_WINDOWS:
stream = AnsiToWin32(stream)
self.stream = stream
self.record = None
def color_stream():
"""
Initializes sys.stdout with `colorama's <https://github.com/tartley/colorama>`_ formatting capabilities.
:return: a colorized output stream
:rtype: colorama.ansitowin32.StreamWrapper
"""
colorama_init(wrap=False)
return AnsiToWin32(sys.stdout).stream
def __init__(self, stream=sys.stderr, conf_file="colorunit_conf.ini"):
# init the colorama
init()
self.confReader = ConfReader(conf_file)
if os.name == 'nt':
#on windows, show color instead of ASCII sequences.
self.stream = AnsiToWin32(sys.stderr).stream
else:
self.stream = stream
def print_color(*args: Any, color: Text) -> None:
output = wrap_with_color(*args, color=color)
try:
# colorama is used to fix a regression where colors can not be printed on
# windows. https://github.com/RasaHQ/rasa/issues/7053
from colorama import AnsiToWin32
stream = AnsiToWin32(sys.stdout).stream
print(output, file=stream)
except ImportError:
print(output)
def colored_print(text, color):
#Check if we are running this on windows platform
is_windows = sys.platform.startswith('win')
if is_windows:
try:
import colorama
method = 1
except:
method = 2
if method == 1:
from colorama import init, AnsiToWin32
init(wrap=False)
stream = AnsiToWin32(sys.stderr).stream
print >> stream, color + text
elif method == 2:
if color == '\033[92m':
color = "G"
if color == '\033[93m':
color = "Y"
if color == '\033[94m':
color = "B"
if color == '\033[91m':
color = "R"
if color == '\033[0m':
color = "W"
def powershell_print(text, color):
color = color.upper()
powershell_colors = {
"B": "Blue",
"G": "Green",
"R": "Red",
"W": "White",
"Y": "Yellow"
}
import subprocess, sys
subprocess.Popen(
'powershell -command "write-host {{"{0}"}} -foreground "{1}" " '
.format(text, powershell_colors[color]),
stdout=sys.stdout)
powershell_print(text, color)
else:
print color + text + '\033[0m'
def setOutputStream(self, stream):
if platform.startswith('win'):
on_stream = AnsiToWin32(stream).stream
else:
on_stream = stream
off_stream = open(os.devnull, 'w')
if self.spec_file:
# Swapping outputs so nose will write it's normal output to the stream
# and spec output to the file
off_stream, on_stream = on_stream, open(self.spec_file, 'w')
self.stream = SpecOutputStream(on_stream, off_stream)
return self.stream
def printable(*args, **kwargs):
sep = kwargs.get('sep', ' ')
end = kwargs.get('end', '\n')
file = kwargs.get('file', sys.stderr)
flush = kwargs.get('flush', False)
show_color = file.isatty() and use_color
if is_windows and show_color:
file = AnsiToWin32(sys.stderr)
us = time_diff()
fmt = sep.join([repr(x) for x in args])
fmt = colored(fmt, us) if show_color else plain(fmt, us)
file.write(fmt + end)
if flush:
file.flush()
def find_exact_matches(df, city, street, sm_all_streets, sm_st_ed_dict, st_grid_st_list):
cprint("\nExact matching algorithm for %s \n" % (street), attrs=['underline'], file=AnsiToWin32(sys.stdout))
# Timer start
start = time.time()
# Bookkeeping
try:
post = '_' + street.split('_')[2].split('HN')[0]
except:
post = ''
num_records = len(df)
num_streets = len(df.groupby([street]).count())
basic_info = [post,num_records,num_streets]
# Check for exact matches between microdata and Steve Morse
df, exact_info_sm = find_exact_matches_sm(df, street, sm_all_streets, sm_st_ed_dict,basic_info)
# Check for exact matches between microdata and 1940 street grid
df, exact_info_stgrid = find_exact_matches_st_grid(df, street, st_grid_st_list, basic_info)
# Create final exact match variable
df['exact_match_bool'+post] = np.where(df['exact_match_sm_bool'+post] | df['exact_match_stgrid_bool'+post]==True,True,False)
df['exact_match_type'] = np.where(df['exact_match_stgrid_bool'+post] & ~df['exact_match_sm_bool'+post],'StGrid','')
df.loc[df['exact_match_sm_bool'+post],'exact_match_type'] = 'SM'
# Generate some information
num_exact_matches = df['exact_match_bool'+post].sum()
prop_exact_matches = float(num_exact_matches)/float(num_records)
cprint("Total cases with exact matches: "+str(num_exact_matches)+" of "+str(num_records)+" cases ("+str(round(100*prop_exact_matches, 1))+"%)", file=AnsiToWin32(sys.stdout))
df_exact_matches = df[df['exact_match_bool'+post]]
num_streets_exact = len(df_exact_matches.groupby([street]).count())
prop_exact_streets = float(num_streets_exact)/float(num_streets)
cprint("Total streets with exact matches: "+str(num_streets_exact)+" of "+str(num_streets)+" total streets pairs ("+str(round(100*prop_exact_streets, 1))+"%)\n", 'yellow', file=AnsiToWin32(sys.stdout))
# Timer stop
end = time.time()
exact_matching_time = round(float(end-start)/60, 1)
cprint("Exact matching for %s took %s\n" % (city, exact_matching_time), 'cyan', attrs=['dark'], file=AnsiToWin32(sys.stdout))
# Generate dashboard info
exact_info = [num_records, num_streets] + exact_info_sm + exact_info_stgrid + [exact_matching_time]
return df, exact_info
def stream(self, value):
if isinstance(value, string_types):
if value in self._stream_cache:
value = self._stream_cache[value]
else:
value = self._stream_cache[value] = open(value, 'a', buffering=0)
isatty = getattr(value, 'isatty', None)
if self.force_colors or (isatty and isatty() and os.name != 'java'):
self._stream = AnsiToWin32(value, strip=False)
self._tty = True
self.event_colors = EVENT_COLORS
self.code_colors = CODE_COLORS
else:
self._tty = False
self._stream = value
self.event_colors = NO_COLORS
self.code_colors = NO_COLORS
def stream(self, value):
if isinstance(value, STRING_TYPES):
if value in self._stream_cache:
value = self._stream_cache[value]
else:
value = self._stream_cache[value] = open(value, 'a', buffering=0)
isatty = getattr(value, 'isatty', None)
if self.force_colors or (isatty and isatty() and os.name != 'java'):
self._stream = AnsiToWin32(value, strip=False)
self._tty = True
self.event_colors = self.EVENT_COLORS
self.other_colors = self.OTHER_COLORS
else:
self._tty = False
self._stream = value
self.event_colors = {key: '' for key in self.EVENT_COLORS}
self.other_colors = {key: '' for key in self.OTHER_COLORS}
def extract_step1_labels(fields):
labels_step1 = []
pblks_labeled1 = []
for field in fields:
with arcpy.da.SearchCursor(block_file, ['pblk_id', field]) as cursor:
for row in cursor:
if row[1] != ' ':
labels_step1.append(row[1])
pblks_labeled1.append(row[0])
num_labels_step1 = len(labels_step1)
per_micro_blocks = round(100 * float(num_labels_step1) / num_micro_blocks,
1)
cprint("Physical blocks labeled in Step 1: " + str(num_labels_step1) +
" (" + str(per_micro_blocks) + r"% of microdata blocks)" + "\n",
'green',
attrs=['bold'],
file=AnsiToWin32(sys.stdout))
return labels_step1, pblks_labeled1, num_labels_step1
def initial():
import json
import sys
from colorama import init, AnsiToWin32, Fore, Back, Style
import math
# import datetime
init(wrap=False)
stream = AnsiToWin32(sys.stderr).stream
try:
regex = "Session A - [24 x 80]"
state_left = win32api.GetKeyState(
0x01) # Left button down = 0 or 1. Button up = -127 or -128
state_right = win32api.GetKeyState(
0x02) # Right button down = 0 or 1. Button up = -127 or -128
import os.path
secs_between_keys = 0.01
win = WindowMgr()
win_ins = win.find_window(regex)
filename = "images/vgm.png"
print(Style.RESET_ALL + '===Starting Process==', file=stream)
if win_ins == None:
print(Fore.RED + ('-Not found session : %s' % regex), file=stream)
print(Fore.RED + '-Please open CTCS program before execute..',
file=stream)
sys.exit()
win.Maximize(win_ins)
# (x,y,w,h) =win.set_onTop(win_ins)
(x, y, w, h) = win.set_onTop(win_ins)
vHeighScreen = h * 0.11
except Exception as e:
# except:
if hasattr(e, 'message'):
print(e.message)
else:
print(e)
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno),
type(e).__name__, e)
def __init__(self, stream, colorizer=Colorizer()):
"""
Initialize the :class:`StreamDisplay`.
:param stream: The stream to be attached too.
"""
super(StreamDisplay, self).__init__()
self.colorizer = colorizer
self.output_map = {}
if stream_has_color_support(stream):
self.stream = AnsiToWin32(stream).stream
else:
self.stream = stream
# Python 3 differentiates binary streams.
if hasattr(stream, 'buffer'):
self.binary_stream = stream.buffer
else:
self.binary_stream = stream
def _get_print(self):
if sys.platform == 'win32' and self._has_color:
from colorama import AnsiToWin32
return functools.partial(print,
file=AnsiToWin32(sys.stdout).stream)
return print
import string
import sys
import time
from colorama import init, Fore, AnsiToWin32, Style
"""
This file contains all output utilities which are needed throughout the project.
We decided to not wrap the functions into a class for readability of the resulting code.
"""
init(autoreset=True)
error_stream = AnsiToWin32(sys.stderr).stream
def print_countdown(wait_time: int,
prefix_text: string = "Waiting for") -> None:
"""
Prints a countdown, which updates itself every second. After the countdown has finished,
the complete text will be removed.
:param wait_time: The waiting time in seconds.
:param prefix_text: The text which should be displayed before the current seconds.
:return: None
"""
assert wait_time < 100
countdown_message = "\r" + prefix_text + " %2d seconds"
for remaining in range(wait_time, 0, -1):
sys.stdout.write(countdown_message % (remaining))
sys.stdout.flush()
time.sleep(1)
#!/usr/bin/python
# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
# Demonstrate the difference between colorama initialized with wrapping on and off.
# The point of the demonstration is to show how the ANSI wrapping on Windows can be disabled.
# The unwrapped cases will be interpreted with ANSI on Unix, but not on Windows.
from __future__ import print_function
import sys
import fixpath
from colorama import AnsiToWin32, init, Fore
init()
print('%sWrapped yellow going to stdout, via the default print function.' % Fore.YELLOW)
init(wrap=False)
print('%sUnwrapped CYAN going to stdout, via the default print function.' % Fore.CYAN)
print('%sUnwrapped CYAN, using the file parameter to write via colorama the AnsiToWin32 function.' % Fore.CYAN, file=AnsiToWin32(sys.stdout))
print('%sUnwrapped RED going to stdout, via the default print function.' % Fore.RED)
init()
print('%sWrapped RED going to stdout, via the default print function.' % Fore.RED)
def _get_print(self):
if sys.platform == 'win32' and self._has_color and AnsiToWin32 is not None:
return functools.partial(print,
file=AnsiToWin32(sys.stdout).stream)
return print
def Print(self,
message='',
verbosity=DEFAULT_VERBOSITY,
newlines=DEFAULT_NEWLINES,
indent_=DEFAULT_INDENT,
stderr=False):
'''
Prints a message to the output.
:param unicode|list(unicode) message: the message to print.
:param int verbosity:
The miminum verbosity value for this message to appear. See verbosity property.
:param int newlines:
The number of new-lines to append to the message.
:param int indent_:
The message indentation.
:param bool stderr:
By default we print to the standar output. If this flag is set we print to the standard
error.
'''
if self.verbosity < verbosity:
return
if stderr:
stream = self.__stderr
else:
stream = self.__stdout
if isinstance(message, (list, tuple)):
message = '\n'.join(map(six.text_type, message))
else:
message = six.text_type(message)
if self.color:
# `out` holds the stream of text we'll eventually output
# `stack` is the currently applied color codes
# `remaining` holds the still-unparsed part of message
# `match` is any <colorcode> or </> construct
out = ''
stack = []
remaining = message
match = self.MARKUP_RE.search(remaining)
while match:
# `token` is either 'colorcode' or '/'.
token = match.groups()[0]
out += remaining[:match.start()]
remaining = remaining[match.end():]
if token == '/':
if stack:
# Pull the last style off the stack.
# Emit a reset then reapply the remaining styles.
stack.pop()
out += self.COLOR_CODES['reset']
for name in stack:
out += self.COLOR_CODES[name]
else:
out += self.COLOR_CODES[token]
stack.append(token)
match = self.MARKUP_RE.search(remaining)
# Get any remaining text that doesn't have markup and
# reset the terminal if there are any unclosed color tags.
out += remaining
if stack:
out += self.COLOR_CODES['reset']
else:
# No color, just strip that information from the message
out = self.MARKUP_RE.sub('', message)
# Support for colors on Windows
assert isinstance(indent_, int)
assert isinstance(newlines, int)
from zerotk.text import indent
text = indent(out, indent_=indent_) + ('\n' * newlines)
# Encode text to the target (console) encoding.
if six.PY2 and isinstance(text, six.text_type) and (hasattr(
stream, 'encoding') and stream.encoding is None):
text = text.encode('ascii', 'replace')
if self.color and self.colorama:
from colorama import AnsiToWin32
ansi_to_win32 = AnsiToWin32(stream, strip=False, convert=True)
ansi_to_win32.write(text)
else:
stream.write(text)
stream.flush()
def wrap(stream):
from colorama import AnsiToWin32
return AnsiToWin32(stream, convert=True, strip=False, autoreset=False).stream
def Print(self,
message='',
verbosity=DEFAULT_VERBOSITY,
newlines=DEFAULT_NEWLINES,
indent=DEFAULT_INDENT,
stderr=False):
'''
Prints a message to the output.
:param int verbosity:
:param int newlines:
:param int indent:
:paran bool stderr:
'''
if self.verbosity < verbosity:
return
if stderr:
stream = self.__stderr
else:
stream = self.__stdout
message = str(message)
if self.color:
# `out` holds the stream of text we'll eventually output
# `stack` is the currently applied color codes
# `remaining` holds the still-unparsed part of message
# `match` is any <colorcode> or </> construct
out = ''
stack = []
remaining = message
match = self.MARKUP_RE.search(remaining)
while match:
# `token` is either 'colorcode' or '/'.
token = match.groups()[0]
out += remaining[:match.start()]
remaining = remaining[match.end():]
if token == '/':
if stack:
# Pull the last style off the stack.
# Emit a reset then reapply the remaining styles.
stack.pop()
out += self.COLOR_CODES['reset']
for name in stack:
out += self.COLOR_CODES[name]
else:
out += self.COLOR_CODES[token]
stack.append(token)
match = self.MARKUP_RE.search(remaining)
# Get any remaining text that doesn't have markup and
# reset the terminal if there are any unclosed color tags.
out += remaining
if stack:
out += self.COLOR_CODES['reset']
else:
# No color, just strip that information from the message
out = self.MARKUP_RE.sub('', message)
# Support for colors on Windows
assert isinstance(indent, int)
assert isinstance(newlines, int)
text = ' ' * indent + out + ('\n' * newlines)
if self.color and self.colorama:
from colorama import AnsiToWin32
AnsiToWin32(stream).write(text)
else:
stream.write(text)
stream.flush()
"""
lbl = f'{fn.__module__}..<{fn.__name__}>'
names = fn.__code__.co_freevars
cells = fn.__closure__
ns = dict(zip(names, cells or ()))
super().__init__(lbl, ns)
def __getitem__(self, name):
d, = self.dicts
try:
return d[name].cell_contents
except ValueError:
raise UnboundLocalError(name)
stderr = AnsiToWin32(sys.stderr).stream
def eprint(*things):
"""Print to stderr."""
print(*things, file=stderr)
def is_dataclass_type(cls):
"""Returns whether cls is a dataclass."""
return isinstance(cls, type) and hasattr(cls, '__dataclass_fields__')
def dataclass_methods(dc):
"""Returns a dataclass's method dictionary."""
return {name: getattr(dc, name)
