Exemplo n.º 1
0
    def temoa_checks(self):
        """Make sure Python 2.7 is used and that a suitable solver is available."""

        if version_info < (2, 7):
            msg = (
                "Temoa requires Python v2.7 to run.\n\n The model may not solve"
                "properly with another version.")
            raise SystemExit(msg)

        if self.options.neos is True:
            # Invoke NEOS solver manager if flag is specified in config file
            self.optimizer = pyomo.opt.SolverManagerFactory('neos')
        else:
            self.optimizer = SolverFactory(self.options.solver)

        if self.optimizer:
            pass
        elif self.options.solver != 'NONE':
            SE.write(
                "\nWarning: Unable to initialize solver interface for '{}'\n\n"
                .format(self.options.solver))
            if SE.isatty():
                SE.write("Please press enter to continue or Ctrl+C to quit.")
                if os.path.join(
                        'temoa_model',
                        'config_sample_myopic') not in options.file_location:
                    raw_input()
Exemplo n.º 2
0
 def process_file(self, file):
     # This should mirror PipelineStep, so an entire pipeline can function
     # as a pipeline step
     for step in self.steps:
         file.report["Errors"] = None
         try:
             file = step.process_file(file)
             assert file is not None
         except AbortPipelineForThisImage as exc:
             file.report.update({"PipelineAbortedMessage": str(exc)})
             print(f"\nAborting at {step.__class__.__name__}: {str(exc)}",
                   file=stderr)
             break
         except Exception as exc:
             path = file.filename
             if hasattr(file.fetcher, "pathondisk"):
                 path = file.fetcher.pathondisk
             print(
                 f"\n{exc.__class__.__name__}: {str(exc)} while processing '{path}'\n",
                 file=stderr)
             if stderr.isatty():
                 traceback.print_exc(file=stderr)
             file.report["Errors"] = f"{exc.__class__.__name__}: {str(exc)}"
             self.report.record(file.instant, **file.report)
             if isinstance(exc, FatalPipelineError):
                 raise
     self.report.record(file.instant, **file.report)
     return file
Exemplo n.º 3
0
def write_error(s):
    """
    Writes out an error message to stderr in red text. This is done so that the
    error messages from the Tasks system can be easily distinguished from the
    errors from the underlying commands being run. If we cannot change the text
    color (not supported by OS or redirecting to a file) then just the string is
    written.
    """
    from sys import stderr
    from os import name

    try:    is_tty = stderr.isatty()
    except: is_tty = False
    
    if is_tty:
        if name == "posix": stderr.write("\x1b[1;31m")
        elif name == "nt":
            from ctypes import windll, Structure, c_short, c_ushort, byref
            k32 = windll.kernel32
            handle = k32.GetStdHandle(-12)
            class COORD(Structure):      _fields_ = [("X", c_short), ("Y", c_short)]
            class SMALL_RECT(Structure): _fields_ = [("L", c_short), ("T", c_short), ("R", c_short), ("B", c_short)]
            class CONSOLE_SCREEN_BUFFER_INFO(Structure): _fields_ = [("Size", COORD), ("CursorPos", COORD), ("Color", c_ushort), ("Rect", SMALL_RECT), ("MaxSize", COORD)]
            csbi = CONSOLE_SCREEN_BUFFER_INFO()
            k32.GetConsoleScreenBufferInfo(handle, byref(csbi))
            prev = csbi.Color
            k32.SetConsoleTextAttribute(handle, 12)
    stderr.write(s)
    if is_tty:
        if name == "posix": stderr.write("\x1b[0m")
        elif name == "nt":  k32.SetConsoleTextAttribute(handle, prev)
    stderr.write("\n")
Exemplo n.º 4
0
 def _do_delete(self):
     if self.rm_script is not None:
         with open(self.rm_script, "a") as fh:
             cmd = "rm -vf" if self.mv_dest is None else f"mv -n -t {self.mv_dest}"
             for f in self.to_delete:
                 print(cmd, realpath(f), file=fh)
     else:
         if not self.force:
             if self.mv_dest is not None:
                 click.echo(
                     f"Will move the following files to {self.mv_dest}:")
             else:
                 click.echo("Will delete the following files:")
             for f in self.to_delete:
                 click.echo(f"\t{f}")
         if self.force or click.confirm("Is that OK?"):
             for f in self.to_delete:
                 try:
                     if self.mv_dest is None:
                         os.unlink(f)
                     else:
                         os.makedirs(self.mv_dest, exist_ok=True)
                         shutil.move(f, self.mv_dest)
                 except Exception as exc:
                     tqdm.write(f"Error deleting {f}: {str(exc)}")
                     if stderr.isatty():
                         traceback.print_exc(file=stderr)
             tqdm.write(f"Deleted {len(self.to_delete)} files")
     self.to_delete = []
Exemplo n.º 5
0
 def index(self, progress=True):
     if len(self._instants) == 0 or len(self._files) == 0:
         with FileLock(self._index_file, timeout=3600):
             pass
         try:
             if op.exists(
                     self._index_file
             ):  # TODO FIXME make this check if the index is stale
                 print("read index", self._index_file, file=stderr)
                 with open(self._index_file, "r") as fh:
                     self._files = {}
                     self._instants = {}
                     for line in tqdm(fh):
                         fetcher = Fetcher.from_json(json.loads(line))
                         self._files[fetcher.filename] = fetcher
                         self._instants[fetcher.instant] = fetcher
                     if len(self._instants) > 0 and len(self._files) > 0:
                         return
         except Exception as exc:
             print("Failed to load index file:", str(exc), file=stderr)
             if stderr.isatty():
                 traceback.print_exc(file=stderr)
         with FileLock(self._index_file):
             try:
                 itr = self.iter(tar_contents=False)
                 if progress:
                     itr = tqdm(itr)
                 if self.write_index:
                     with open(self._index_file, "w") as fh:
                         for f in itr:
                             self._instants[f.instant] = f.fetcher
                             print(json.dumps(f.fetcher.dict(),
                                              cls=PathAwareJsonEncoder),
                                   file=fh)
                 else:
                     for f in itr:
                         self._instants[f.instant] = f.fetcher
             except Exception as exc:
                 print("Failed to create timestream index file:",
                       str(exc),
                       file=stderr)
                 if stderr.isatty():
                     traceback.print_exc(file=stderr)
                 if op.exists(self._index_file):
                     os.unlink(self._index_file)
Exemplo n.º 6
0
def warn(warning: str):
	"""
	Logs a warning to stderr.
	"""
	from sys import stderr
	from time import ctime
	if stderr.isatty():
		print("\033[38;2;238;216;78mWW:", warning, "-\t", ctime(), "\033[m", file=stderr)
	else:
		print("WW:", warning, '-\t', ctime(), file=stderr)
Exemplo n.º 7
0
def main(code, args, end):
    for index in range(min(7, len(args))):
        atoms['³⁴⁵⁶⁷⁸⁹'[index]].call = lambda literal=args[index]: literal

    try:
        output(jamb_eval(code, args[:2]), end)
    except KeyboardInterrupt:
        if stderr.isatty():
            sys.stderr.write('\n')
        return 130
Exemplo n.º 8
0
def setup_color(color):
    enable_out = (False if color == 'never' else
                  True if color == 'always' else stdout.isatty())
    Out_Style.enable(enable_out)
    Out_Fore.enable(enable_out)

    enable_err = (False if color == 'never' else
                  True if color == 'always' else stderr.isatty())
    Err_Style.enable(enable_err)
    Err_Fore.enable(enable_err)
Exemplo n.º 9
0
def setup_color(color):
    enable_out = (False if color == 'never' else
                  True if color == 'always' else
                  stdout.isatty())
    Out_Style.enable(enable_out)
    Out_Fore.enable(enable_out)

    enable_err = (False if color == 'never' else
                  True if color == 'always' else
                  stderr.isatty())
    Err_Style.enable(enable_err)
    Err_Fore.enable(enable_err)
Exemplo n.º 10
0
def _terminal_has_colors() -> bool:
    """Initialize the terminal and check if it support colors.

    Returns:
        bool: True if successful False otherwise.
    """
    if not (hasattr(stderr, 'isatty') and stderr.isatty()):
        return False
    try:
        curses.setupterm()
        return curses.tigetnum('colors') > 0
    except Exception:  # pylint: disable=broad-except
        return False
Exemplo n.º 11
0
def error(err: Exception, fatal: int=False):
	"""
	Logs an error to stderr, then exits if fatal is a non-falsy value, using it as an exit code
	"""
	from sys import stderr
	from time import ctime
	if stderr.isatty():
		fmt = "\033[38;2;255;0;0mEE: %s:"
		print(fmt % type(err).__name__, "%s" % err, "-\t", ctime(), "\033[m", file=stderr)
	else:
		print("EE: %s:" % type(err).__name__, "%s" % err, "-\t", ctime(), file=stderr)
	if fatal:
		exit(int(fatal))
Exemplo n.º 12
0
 def write(self, file):
     if op.exists(self._index_file):
         with FileLock(self._index_file):
             try:
                 os.unlink(self._index_file)
             except Exception as exc:
                 print(str(exc), file=stderr)
                 if stderr.isatty():
                     traceback.print_exc(file=stderr)
     if self.name is None:
         raise RuntimeError("TSv2Stream not opened")
     if not isinstance(file, TimestreamFile):
         raise TypeError("file should be a TimestreamFile")
     subpath = self._timestream_path(file)
     if self.bundle == "none":
         outpath = op.join(self.path, subpath)
         os.makedirs(op.dirname(outpath), exist_ok=True)
         with FileLock(outpath):
             with open(outpath, 'wb') as fh:
                 fh.write(file.content)
     else:
         if self.bundle == "root":
             self.path = str(self.path)
             for ext in [".tar", ".zip", f".{file.format}"]:
                 if self.path.lower().endswith(ext):
                     self.path = self.path[:-len(ext)]
             self.path = Path(self.path)
         bundle = self._bundle_archive_path(file)
         bdir = op.dirname(bundle)
         if bdir:  # i.e. if not $PWD
             os.makedirs(bdir, exist_ok=True)
         with FileLock(bundle):
             with zipfile.ZipFile(bundle,
                                  mode="a",
                                  compression=zipfile.ZIP_STORED,
                                  allowZip64=True) as zip:
                 pathinzip = op.join(self.name, subpath)
                 if pathinzip not in zip.namelist():
                     zip.writestr(pathinzip, file.content)
                 else:
                     file_crc = zlib.crc32(file.content)
                     zip_crc = zip.getinfo(pathinzip).CRC
                     if file_crc != zip_crc:
                         raise RuntimeError(
                             f"ERROR: trying to overwrite file with different content: zip={bundle}, subpath={subpath}"
                         )
Exemplo n.º 13
0
	def temoa_checks(self):
		"""Make sure Python 2.7 is used and that a suitable solver is available."""
		
		if version_info < (2, 7):
			msg = ("Temoa requires Python v2.7 to run.\n\n The model may not solve"
				"properly with another version.")
			raise SystemExit( msg )
		
		self.optimizer = SolverFactory( self.options.solver )
		if self.optimizer:
			pass
		elif self.options.solver != 'NONE':
			SE.write( "\nWarning: Unable to initialize solver interface for '{}'\n\n"
				.format( self.options.solver ))
			if SE.isatty():
				SE.write( "Please press enter to continue or Ctrl+C to quit." )
				raw_input()
Exemplo n.º 14
0
Arquivo: io.py Projeto: gwk/pithy
def err_progress(iterator, label='progress', suffix='', frequency=0.1):
  '''
  For interactive terminals, return a generator that yields the elements of iterator
  and displays a progress indicator on std err.
  '''
  if not frequency or not stderr.isatty():
    return iterator

  from pithy.ansi import ERASE_LINE

  if label is None:
    label = str(iterator)
  pre = '◊ ' + label + ': '
  post = (suffix and ' ' + suffix) + '…\r'
  final = ' ' + suffix + '.' if suffix else '.'

  if isinstance(frequency, float):
    from time import time
    def err_progress_gen():
      prev_t = time()
      step = 1
      next_i = step
      for i, el in enumerate(iterator):
        if i == next_i:
          print(pre + str(i) + post, end='', file=stderr, flush=True)
          print(ERASE_LINE, end='', file=stderr, flush=False)
          t = time()
          d = t - prev_t
          step = max(1, int(step * frequency / d))
          prev_t = t
          next_i = i + step
        yield el
      print(pre + str(i) + final, file=stderr)


  else:
    def err_progress_gen():
      for i, el in enumerate(iterator):
        if i % frequency == 0:
          print(pre + str(i) + post, end='', file=stderr, flush=True)
          print(ERASE_LINE, end='', file=stderr, flush=False)
        yield el
      print(pre + str(i) + final, file=stderr)

  return err_progress_gen()
Exemplo n.º 15
0
 def field_map(match):
     indent = match.group(1)
     field = match.group(2)
     layer_def = layer.definition['layer']
     fields = layer_def['fields']
     if field not in fields:
         raise ValueError(f"Field '{field}' not found in the layer definition file")
     if 'values' not in fields[field]:
         raise ValueError(
             f"Field '{field}' has no values defined in the layer definition file")
     values = fields[field]['values']
     if not isinstance(values, dict):
         raise ValueError(f"Definition for {field}/values has to be a dictionary")
     conditions = []
     ignored = []
     for map_to, mapping in values.items():
         # mapping is a dictionary of input_fields -> (a value or a list of values)
         # If it is not a dictionary, skip it
         if not isinstance(mapping, dict):
             ignored.append(map_to)
             continue
         whens = []
         for in_fld, in_vals in mapping.items():
             if isinstance(in_vals, str):
                 in_vals = [in_vals]
             wildcards = [v for v in in_vals if '%' in v]
             in_vals = [v for v in in_vals if '%' not in v]
             if in_vals:
                 if len(in_vals) == 1:
                     expr = f"={sql_value(in_vals[0])}"
                 else:
                     expr = f" IN ({', '.join((sql_value(v) for v in in_vals))})"
                 whens.append(f'{sql_field(in_fld)}{expr}')
             for wildcard in wildcards:
                 whens.append(f'{sql_field(in_fld)} LIKE {sql_value(wildcard)}')
         if whens:
             cond = f'\n{indent}    OR '.join(whens) + \
                    (' ' if len(whens) == 1 else f'\n{indent}    ')
             conditions.append(f"WHEN {cond}THEN {sql_value(map_to)}")
         else:
             ignored.append(map_to)
     if ignored and not stderr.isatty():
         print(f"-- Assuming manual SQL handling of field '{field}' values "
               f"[{','.join(ignored)}] in layer {layer_def['id']}", file=stderr)
     return indent + f'\n{indent}'.join(conditions)
Exemplo n.º 16
0
def write_error(s):
    """
    Writes out an error message to stderr in red text. This is done so that the
    error messages from the Tasks system can be easily distinguished from the
    errors from the underlying commands being run. If we cannot change the text
    color (not supported by OS or redirecting to a file) then just the string is
    written.
    """
    from sys import stderr
    from os import name

    try:
        is_tty = stderr.isatty()
    except:
        is_tty = False

    if is_tty:
        if name == "posix": stderr.write("\x1b[1;31m")
        elif name == "nt":
            from ctypes import windll, Structure, c_short, c_ushort, byref
            k32 = windll.kernel32
            handle = k32.GetStdHandle(-12)

            class COORD(Structure):
                _fields_ = [("X", c_short), ("Y", c_short)]

            class SMALL_RECT(Structure):
                _fields_ = [("L", c_short), ("T", c_short), ("R", c_short),
                            ("B", c_short)]

            class CONSOLE_SCREEN_BUFFER_INFO(Structure):
                _fields_ = [("Size", COORD), ("CursorPos", COORD),
                            ("Color", c_ushort), ("Rect", SMALL_RECT),
                            ("MaxSize", COORD)]

            csbi = CONSOLE_SCREEN_BUFFER_INFO()
            k32.GetConsoleScreenBufferInfo(handle, byref(csbi))
            prev = csbi.Color
            k32.SetConsoleTextAttribute(handle, 12)
    stderr.write(s)
    if is_tty:
        if name == "posix": stderr.write("\x1b[0m")
        elif name == "nt": k32.SetConsoleTextAttribute(handle, prev)
    stderr.write("\n")
Exemplo n.º 17
0
def suggest_missing(miss=None, exclude=[]):
    global suggest
    sgs = dict(suggest)
    for exc in exclude:
        try:
            sgs.pop(exc)
        except KeyError:
            pass
    kamaki_docs = 'http://www.synnefo.org/docs/kamaki/latest'

    for k, v in (miss, sgs[miss]) if miss else sgs.items():
        if v['active'] and stderr.isatty():
            stderr.write('Suggestion: you may like to install %s\n' % k)
            stderr.write(
                ('%s\n' % v['description']).encode(pref_enc, 'replace'))
            stderr.write('\tIt is easy, here are the instructions:\n')
            stderr.write('\t%s/installation.html%s\n' % (
                kamaki_docs, v['url']))
            stderr.flush()
Exemplo n.º 18
0
 def parse(self):
     conditions = []
     ignored = []
     for map_to, mapping in self.field.values.items():
         # mapping is a dictionary of input_fields -> (a value or a list of values)
         # If it is not a dictionary, skip it
         if not isinstance(mapping, dict) and not isinstance(mapping, list):
             ignored.append(map_to)
             continue
         expr = self.to_expression(map_to, mapping)
         if expr:
             conditions.append(expr)
         else:
             ignored.append(map_to)
     if ignored and not stderr.isatty():
         print(f"-- Assuming manual SQL handling of field '{self.field.name}' "
               f"values [{','.join(ignored)}] in layer {self.layer.id}",
               file=stderr)
     return self.indent + f'\n{self.indent}'.join(conditions)
Exemplo n.º 19
0
def suggest_missing(miss=None, exclude=[]):
    global suggest
    sgs = dict(suggest)
    for exc in exclude:
        try:
            sgs.pop(exc)
        except KeyError:
            pass
    kamaki_docs = 'http://www.synnefo.org/docs/kamaki/latest'

    for k, v in (miss, sgs[miss]) if miss else sgs.items():
        if v['active'] and stderr.isatty():
            stderr.write('Suggestion: you may like to install %s\n' % k)
            stderr.write(
                ('%s\n' % v['description']).encode(pref_enc, 'replace'))
            stderr.write('\tIt is easy, here are the instructions:\n')
            stderr.write('\t%s/installation.html%s\n' %
                         (kamaki_docs, v['url']))
            stderr.flush()
Exemplo n.º 20
0
#!/usr/bin/env python

from sys import stdin, stdout, stderr

print "Piped input:", not stdin.isatty()
print "Piped output:", not stdout.isatty()
print "Piped error:", not stderr.isatty()
Exemplo n.º 21
0
        self._dict = defaultdict(str)

    def __getattr__(self, key):
        return self._dict[key]


Null_Style = Null_Fore = colorama_shim()

if stdout.isatty():
    Out_Style = Colo_Style
    Out_Fore = Colo_Fore
else:
    Out_Style = Null_Style
    Out_Fore = Null_Fore

if stderr.isatty():
    Err_Style = Colo_Style
    Err_Fore = Colo_Fore
else:
    Err_Style = Null_Style
    Err_Fore = Null_Fore


def info_main(*args):
    logger.info(''.join([Err_Style.BRIGHT, Err_Fore.GREEN] + list(args) +
                        [Err_Style.RESET_ALL, Err_Fore.RESET]))


def info_notify(s):
    info('{}{}{}{}'.format(Err_Style.BRIGHT, Err_Fore.LIGHTBLUE_EX, s,
                           Err_Style.RESET_ALL))
Exemplo n.º 22
0
#!/usr/bin/env python3
# Dedicated to the public domain under CC0: https://creativecommons.org/publicdomain/zero/1.0/.

from sys import stderr

exit(0 if stderr.isatty() else 1)
Exemplo n.º 23
0
#!/usr/bin/env python

from sys import stdin, stdout, stderr

print "Piped input:",  not stdin.isatty()
print "Piped output:", not stdout.isatty()
print "Piped error:",  not stderr.isatty()
Exemplo n.º 24
0
def scanimage(imgpath):
    # init all variables
    error = None
    pixel_mean = None
    codes = None
    qr_chamber = None
    qr_experiment = None
    file_size = None

    # Get size
    try:
        file_size = getsize(imgpath)
    except Exception as exc:
        if stderr.isatty():
            print(str(exc), imgpath, file=stderr)
            tb()
        error = "FileSize"

    # Load image
    try:
        pixels = loadimage(imgpath)
        if pixels.dtype == object:
            raise ValueError(
                "Bad image load: see https://github.com/python-pillow/Pillow/issues/3863"
            )
        pixel_mean = np.mean(pixels)
        codes = zbarlight.scan_codes('qrcode', Image.fromarray(pixels))
        if codes is not None:
            codes = ';'.join(sorted(x.decode('utf8') for x in codes))
            qr_chamber = re_chamber(codes)
            qr_experiment = re_experiment(codes)
    except Exception as exc:
        if stderr.isatty():
            print(str(exc), imgpath, file=stderr)
            tb()
        error = "ImageIO"

    # Metadata
    exif_time = None
    try:
        exif_time = get_exif_time(imgpath)
    except KeyError:
        pass
    except Exception as exc:
        if stderr.isatty():
            print(str(exc), imgpath, file=stderr)
            tb()
        error = "EXIF"

    dir_chamber = re_chamber(dirname(imgpath))
    dir_experiment = re_experiment(dirname(imgpath))

    fn_chamber = re_chamber(basename(imgpath))
    fn_experiment = re_experiment(basename(imgpath))
    fn_time = re_time(basename(imgpath))

    return {
        "imgpath": imgpath,
        "qr_chamber": qr_chamber,
        "qr_experiment": qr_experiment,
        "qr_codes": codes,
        "pixel_mean": pixel_mean,
        "exif_time": exif_time,
        "dir_chamber": dir_chamber,
        "dir_experiment": dir_experiment,
        "fn_chamber": fn_chamber,
        "fn_experiment": fn_experiment,
        "fn_time": fn_time,
        "file_size": file_size,
        "error": error,
    }
Exemplo n.º 25
0
    def __init__(self):
        self._dict = defaultdict(str)

    def __getattr__(self, key):
        return self._dict[key]

Null_Style = Null_Fore = colorama_shim()

if stdout.isatty():
    Out_Style = Colo_Style
    Out_Fore = Colo_Fore
else:
    Out_Style = Null_Style
    Out_Fore = Null_Fore

if stderr.isatty():
    Err_Style = Colo_Style
    Err_Fore = Colo_Fore
else:
    Err_Style = Null_Style
    Err_Fore = Null_Fore


def info_main(*args):
    logger.info(''.join([Err_Style.BRIGHT, Err_Fore.GREEN] + list(args) +
                        [Err_Style.RESET_ALL, Err_Fore.RESET]))


def info_notify(s):
    info('{}{}{}{}'.format(Err_Style.BRIGHT, Err_Fore.LIGHTBLUE_EX, s,
                           Err_Style.RESET_ALL))
Exemplo n.º 26
0
def temoa_solve(model_data):
    from sys import argv, version_info

    if version_info < (2, 7):
        msg = ("Temoa requires Python v2.7 to run.\n\nIf you've "
               "installed Coopr with Python 2.6 or less, you'll need to reinstall "
               'Coopr, taking care to install with a Python 2.7 (or greater) '
               'executable.')
        raise SystemExit(msg)

    from time import clock

    from coopr.opt import SolverFactory, SolverManagerFactory
    from coopr.pyomo import ModelData
    from utils import results_writer
    from pformat_results import pformat_results

    tee = False
    solver_manager = SolverManagerFactory('serial')

    options = parse_args()
    dot_dats = options.dot_dat

    opt = SolverFactory(options.solver)
    if opt:
        opt.keepFiles = options.keepPyomoLP
        opt.generateSymbolicLabels = options.useSymbolLabels
        if options.generateSolverLP:
            opt.options.wlp = path.basename(options.dot_dat[0])[:-4] + '.lp'
            SE.write('\nSolver will write file: {}\n\n'.format(opt.options.wlp))

    elif options.solver != 'NONE':
        SE.write("\nWarning: Unable to initialize solver interface for '{}'\n\n"
                 .format(options.solver))

    SE.write('[        ] Reading data files.')
    SE.flush()
    # Recreate the pyomo command's ability to specify multiple "dot dat" files
    # on the command line
    begin = clock()
    duration = lambda: clock() - begin

    mdata = ModelData()
    for f in dot_dats:
        if f[-4:] != '.dat':
            msg = "\n\nExpecting a dot dat (e.g., data.dat) file, found '{}'\n"
            raise SystemExit(msg.format(f))
        mdata.add(f)
    mdata.read(model_data.model)
    SE.write('\r[%8.2f\n' % duration())

    SE.write('[        ] Creating Temoa model instance.')
    SE.flush()
    # Now do the solve and ...
    model_data.instance = model_data.model.create(mdata)
    SE.write('\r[%8.2f\n' % duration())

    SE.write('[        ] Solving.')
    SE.flush()
    if opt:
        model_data.result = solver_manager.solve(model_data.instance, opt=opt, tee=tee,
                                  suffixes=['dual', 'rc'])
		# result = opt.solve(instance)
        SE.write('\r[%8.2f\n' % duration())
    else:
        SE.write('\r---------- Not solving: no available solver\n')
        raise SystemExit

    SE.write('[        ] Formatting results.')
    SE.flush()
    # ... print the easier-to-read/parse format
    results_writer(model_data.result, model_data.instance)
    # updated_results = instance.update_results(result)
    # formatted_results = pformat_results(instance, updated_results)
    SE.write('\r[%8.2f\n' % duration())
    # SO.write(formatted_results)

    if options.graph_format:
        SE.write('[        ] Creating Temoa model diagrams.')
        SE.flush()
        instance.load(result)
        CreateModelDiagrams(instance, options)
        SE.write('\r[%8.2f\n' % duration())

    if not (SO.isatty() or SE.isatty()):
        SO.write("\n\nNotice: You are not receiving 'standard error' messages."
                 "  Temoa uses the 'standard error' file to send meta information "
                 "on the progress of the solve.  If you aren't intentionally "
                 "ignoring standard error messages, you may correct the issue by "
                 "updating coopr/src/coopr.misc/coopr/misc/scripts.py as per this "
                 "coopr changeset: "
                 "https://software.sandia.gov/trac/coopr/changeset/5363\n")
Exemplo n.º 27
0
def verify(ephemerals, resource, informat, force_delete, rm_script, move_dest,
           pixel_distance, distance_file, only_check_exists):
    """
    Verify images from each of EPHEMERAL, ensuring images are in --resources.
    """
    resource_ts = TimeStream(resource, format=informat)
    decoder = DecodeImageFileStep()
    resource_ts.index()

    if rm_script is not None:
        with open(rm_script, "w") as fh:
            print("# rmscript for", *ephemerals, file=fh)

    with Removalist(rm_script=rm_script, mv_dest=move_dest,
                    force=force_delete) as rmer:
        if distance_file is not None:
            distance_file = open(distance_file, "w")
            print("ephemeral_image\tresource_image\tdistance",
                  file=distance_file)
        for ephemeral in ephemerals:
            click.echo(f"Crawling ephemeral timestream: {ephemeral}")
            ephemeral_ts = TimeStream(ephemeral, format=informat)
            try:
                for image in tqdm(ephemeral_ts, unit=" files"):
                    try:
                        res_img = resource_ts.getinstant(image.instant)
                        if not isinstance(image.fetcher, FileContentFetcher):
                            click.echo(
                                f"WARNING: can't delete {image.filename} as it is bundled",
                                err=True)
                            continue
                        if only_check_exists:
                            rmer.remove(image.fetcher.pathondisk)
                        elif pixel_distance is not None:
                            eimg = decoder.process_file(image)
                            rimg = decoder.process_file(res_img)
                            if eimg.pixels.shape != rimg.pixels.shape:
                                if distance_file is not None:
                                    print(basename(image.filename),
                                          basename(res_img.filename),
                                          "NA",
                                          file=distance_file)
                                continue
                            dist = np.mean(abs(eimg.pixels - rimg.pixels))
                            if distance_file is not None:
                                print(basename(image.filename),
                                      basename(res_img.filename),
                                      dist,
                                      file=distance_file)
                            if dist < pixel_distance:
                                rmer.remove(realpath(image.fetcher.pathondisk))
                        elif image.md5sum == res_img.md5sum:
                            rmer.remove(realpath(image.fetcher.pathondisk))
                    except KeyError:
                        tqdm.write(f"{image.instant} not in {resource}")
                        if distance_file is not None:
                            print(basename(image.filename),
                                  "",
                                  "",
                                  file=distance_file)
                    except Exception as exc:
                        click.echo(
                            f"WARNING: error in resources lookup of {image.filename}: {str(exc)}",
                            err=True)
                        if stderr.isatty():
                            traceback.print_exc(file=stderr)
            except KeyboardInterrupt:
                print("\n\nExiting cleanly", file=stderr)
                break
    if distance_file is not None:
        distance_file.close()
Exemplo n.º 28
0
def out(*args, **kwargs):
  if stderr.isatty():
    print(*args, file=stderr, **kwargs)
Exemplo n.º 29
0
def main():
    parser = argparse.ArgumentParser(description = __doc__)
    parser.add_argument('-m', '--metric', default=DEFAULT_METRIC,\
            help="a severity metric lambda for maximal repeat (c)ount and (l)ength" +\
            ", defaults to '%s'" % DEFAULT_METRIC)
    parser.add_argument("file", nargs='+', help="source file to analyze")
    args = parser.parse_args()
    filenames, metric = args.file, args.metric
    for c in metric:
        assert c in ALLOWED_METRIC_CHARS, "metric characters must be in '%s'" %\
                ''.join(list(ALLOWED_METRIC_CHARS))
    metric = eval("lambda c, l: " + metric, None, None)
    a = Analyzer(canonize)
    total = 0
    for filename in filenames:
        stderr.write("processing %s...\n" % filename)
        stderr.flush()
        node_count = 0
        for node_count in a.add(filename):
            if stderr.isatty():
                stderr.write('\r\t%d nodes processed' % node_count)
                stderr.flush()
        total += node_count
        stderr.write('\n')
        #stderr.write('\tsize of strie is now %d bytes\n' % a._trie.size)
        #stderr.flush()
    stderr.write("total nodes: %d\n" % total)
    stderr.write("finding maximal repeats (this may take a while)\n")
    repeats = a.get_sorted_repeats(metric)
    total_severity = 0
    for repeat in repeats:
        severity = metric(len(repeat.indices), repeat.length)
        contains = repeat.contains
        if contains is None:
            total_severity += severity
        else:
            total_severity += max(0,\
                    metric(len(repeat.indices), repeat.length - contains.length + 1))
        print colored_red(\
                'severity %d: %d repeats of length %d; id %d, contains %d' %\
                (severity, len(repeat.indices), repeat.length, id(repeat),\
                id(contains) if contains is not None else 0))
        indices = '@ '
        for index in repeat.indices:
            indices += '(%s,%d) ' % (basename(filenames[index[0]]),\
                    a.lineno_map[index[0]][index[1] - repeat.length + 1] + 1)
        print colored_red(indices)
        index = iter(repeat.indices).next()
        for i in xrange(index[1] - repeat.length + 1, index[1] + 1):
            print '\t', a.canonized[a.lines[index[0]][i]]
    print colored_red('%d/%d lines can be refactored' %\
            (total_severity, a.lines_count))
    try:
        with open(CUR_FILENAME, 'rb') as f:
            print colored_red('last run it was %s/%s lines' %\
                    tuple(f.read().strip().split('/')))
    except IOError as e:
        if e.errno != 2: # 2 is file does not exist
            raise
    with open(CUR_FILENAME, 'wb') as f:
        f.write('%d/%d' % (total_severity, a.lines_count))
Exemplo n.º 30
0
#!/usr/bin/env python
'''@package docstring
Some common python functions
'''

from re import sub
from sys import stdout, stderr
from os import getenv
from collections import namedtuple

_atty_out = stdout.isatty()
_atty_err = stderr.isatty()


def PInfo(module, msg, newline="\n"):
    ''' function to write to stdout'''
    if _atty_out:
        stdout.write('\033[0;32mINFO\033[0m    [%-40s]: %s%s' %
                     (module, msg, newline))
    else:
        stderr.write(
            'INFO    [%-40s]: %s%s' % (module, msg, newline)
        )  # redirect color-less output to stderr to maintain stream in log files


def PWarning(module, msg, newline="\n"):
    ''' function to write to stdout'''
    if _atty_out:
        stdout.write('\033[0;91mWARNING\033[0m [%-40s]: %s%s' %
                     (module, msg, newline))
    else:
Exemplo n.º 31
0
 def _tty_output(self, message, print_newline=True):
     if stderr.isatty():
         if print_newline:
             message += "\n"
         stderr.write(message)
Exemplo n.º 32
0
    args = argparser.parse_args(list(argv) or None)
    args.write_output(dodosc(args.file)(tuple(args.parse_input(args.input))))


def map_abs(inv):
    return map(abs, inv)


def read_stdin(inv):
    from sys import stdin
    yield from map_abs(inv)
    yield from stdin.buffer.read()


def write_bytes(outv):
    from sys import stdout
    stdout.buffer.write(bytes(map((256).__rmod__, outv)))


def write_nums(outv):
    for number in outv:
        print(number)


if __name__ == '__main__':
    try:
        dodos()
    except KeyboardInterrupt:
        if stderr.isatty(): stderr.write('\n')
        raise SystemExit(130)
Exemplo n.º 33
0
        return
  else:
    cb(('deny', "wrong otp"))


if __name__ == '__main__':
  path = os.path.expanduser(CONFIG)
  if not os.path.exists(path):
    out("no config found, performing initial setup of %s" % CONFIG)
    setup(path)
  else:
    queue = Queue()
    with open(path, 'rt') as fd:
      cfg = json.load(fd)

    if 'totp_secret' in cfg and stderr.isatty():
      t = Thread(target=check_totp,
                 kwargs=dict(secret=cfg['totp_secret'],
                 cb=queue.put))
      t.daemon = True
      t.start()

    if 'tokenizer' in cfg:
      from tokenizer import tokenizer
      t = Thread(target=tokenizer,
                 kwargs=dict(cfg=cfg['tokenizer'],
                 cb=queue.put))
      t.daemon = True
      t.start()

    try:
Exemplo n.º 34
0
def download_file(url: str,
                  file: Path,
                  resume=False,
                  rate_limit=0.0,
                  progress=False):
    """Throttled download with progress bar

    :param url: URL to download
    :param file: Output file
    :param resume: Append instead of overwrite
    :param rate_limit: Rate limit in MB/s
    :param progress: Show progress bar
    """

    if resume and file.exists():
        file_mode = 'ab'
        done_bytes = file.size
    else:
        file_mode = 'wb'
        done_bytes = 0

    # To be downloaded each second
    if rate_limit != 0:
        chunk_size = int(rate_limit * 1024 * 1024)
    else:
        # Default chunk size of 1 MB means we do not loose whole file if download gets aborted
        chunk_size = 1024 * 1024

    # Ask server to skip the first N bytes
    request = urllib.request.Request(url)
    request.add_header('Range', 'bytes={}-'.format(done_bytes))

    with urllib.request.urlopen(request) as response:
        if progress:
            # Get size of download
            total_bytes = int(response.headers['content-length']) + done_bytes
            # Avoid ZeroDivisionError and only print progress bar if we are in a terminal
            if total_bytes == 0 or not stderr.isatty():
                progress = False

        with file.open(file_mode) as f:
            while True:
                # Print a progress bar
                if progress:
                    percent = 100 * (done_bytes / total_bytes)
                    # Never more than 70 hash signs
                    bar = '#' * min(70 * done_bytes // total_bytes, 70)
                    ####----- (padded to 70 chars) NNN.N (padded to 5 chars) %
                    print('\r{:-<70} {: >5.1f}%'.format(bar, percent),
                          end='',
                          flush=True,
                          file=stderr)

                # Download and write a chunk
                started = time.time()
                chunk = response.read(chunk_size)
                done_bytes += len(chunk)
                if not chunk:
                    if progress:
                        print()  # newline when done
                    break
                f.write(chunk)

                if rate_limit:
                    try:
                        # Every chunk should take 1 second, sleep for the time that's left
                        time.sleep(1 + started - time.time())
                    except ValueError:
                        pass
Exemplo n.º 35
0
 def _tty_flush(self):
     if stderr.isatty():
         stderr.flush()
Exemplo n.º 36
0
# -*- coding: utf-8 -*-

from sys import stderr, stdout

_verbose = False

_warning_str = ('\033[0;101;37mWARNIG:\033[0m'
                if stderr.isatty() else "WARNING:")

_error_str = ('\033[0;41;37mERROR:\033[0m' if stderr.isatty() else "ERROR:")


def _flush_stdout():
    # This is needed before using stderr, since it is never buffered
    if not stdout.isatty():
        stdout.flush()


def turn_verbose_on():
    global _verbose
    _verbose = True


def normal(message=''):
    print(message)


def verbose(message=''):
    if _verbose:
        print(message)