def _init_console_encoding(): # utf-8, cp949, euc-kr등으로 자동감지하는 경우에는 그대로 사용하고, 그 외에는 utf-8로 통일! encoding = sys.stdout.encoding if encoding.replace('-', '').lower() not in ['utf8', 'cp949', 'euckr']: sys.stdout = codecs.getwriter('utf-8')(sys.stdout.detach()) sys.stderr = codecs.getwriter('utf-8')(sys.stderr.detach()) print('Warn! Current console encoding {} is not supported. Now, using UTF-8 encoding.'.format(encoding))
def unicode_output_stream(stream): """Get wrapper for given stream that writes any unicode without exception Characters that can't be coerced to the encoding of the stream, or 'ascii' if valid encoding is not found, will be replaced. The original stream may be returned in situations where a wrapper is determined unneeded. The wrapper only allows unicode to be written, not non-ascii bytestrings, which is a good thing to ensure sanity and sanitation. """ if sys.platform == "cli": # Best to never encode before writing in IronPython return stream try: writer = codecs.getwriter(stream.encoding or "") except (AttributeError, LookupError): # GZ 2010-06-16: Python 3 StringIO ends up here, but probably needs # different handling as it doesn't want bytestrings return codecs.getwriter("ascii")(stream, "replace") if writer.__module__.rsplit(".", 1)[1].startswith("utf"): # The current stream has a unicode encoding so no error handler is needed if sys.version_info > (3, 0): return stream return writer(stream) if sys.version_info > (3, 0): # Python 3 doesn't seem to make this easy, handle a common case try: return stream.__class__(stream.buffer, stream.encoding, "replace", stream.newlines, stream.line_buffering) except AttributeError: pass return writer(stream, "replace")
def parseArgs(self): global opts encoding = locale.getpreferredencoding() sys.stdout = codecs.getwriter(encoding)(sys.stdout, errors = "replace") sys.stderr = codecs.getwriter(encoding)(sys.stderr, errors = "replace") parser = utils.MyParser(version=__version__, usage="%prog [seatmlfile]", description=__doc__) #utils.addmanageropts(parser) #parser.add_option('-g', '--gui', dest='guimode', action="store_true", # default=False, # help='show file open dialog in GUI') parser.add_option('-f', '--config-file', dest='config_file', type="string", help='apply configuration file') parser.add_option('-n', '--name', dest='naming_format', type="string", help='set naming format' ) try: opts, args = parser.parse_args() except optparse.OptionError, e: print >>sys.stderr, 'OptionError:', e sys.exit(1)
def __call__(self): stdout_sio = codecs.getwriter(_encoding)(StringIO.StringIO()) stderr_sio = codecs.getwriter(_encoding)(StringIO.StringIO()) process = None exception_queue = squeue.Queue() print(repr(self.command)) process = subprocess.Popen(shlex.split(self.command.encode(_encoding)),bufsize=1,stdin=None,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=False) try: stdout_reader = StreamReader([forward(stdout_sio),self.stdout_callback],process.stdout,exception_queue) stderr_reader = StreamReader([forward(stderr_sio),self.stderr_callback],process.stderr,exception_queue) stdout_reader.start(); stderr_reader.start(); while process.poll() is None: try: exception = exception_queue.get(timeout=0.1) print(repr(exception)) except squeue.Empty: pass else: raise exception self.periodic_notifier() except: process.terminate(); raise return process.returncode,stdout_sio.getvalue(),stderr_sio.getvalue()
def __init__(self): super(Console, self).__init__() self._saved_ocp = ctypes.windll.kernel32.GetConsoleOutputCP() self._saved_icp = ctypes.windll.kernel32.GetConsoleCP() ctypes.windll.kernel32.SetConsoleOutputCP(65001) ctypes.windll.kernel32.SetConsoleCP(65001) # ANSI handling available through SetConsoleMode since Windows 10 v1511 # https://en.wikipedia.org/wiki/ANSI_escape_code#cite_note-win10th2-1 # if platform.release() == '10' and int(platform.version().split('.')[2]) > 10586: ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 import ctypes.wintypes as wintypes if not hasattr(wintypes, 'LPDWORD'): # PY2 wintypes.LPDWORD = ctypes.POINTER(wintypes.DWORD) SetConsoleMode = ctypes.windll.kernel32.SetConsoleMode GetConsoleMode = ctypes.windll.kernel32.GetConsoleMode GetStdHandle = ctypes.windll.kernel32.GetStdHandle mode = wintypes.DWORD() GetConsoleMode(GetStdHandle(-11), ctypes.byref(mode)) if (mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING) == 0: SetConsoleMode(GetStdHandle(-11), mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) self._saved_cm = mode self.output = codecs.getwriter('UTF-8')(Out(sys.stdout.fileno()), 'replace') # the change of the code page is not propagated to Python, manually fix it sys.stderr = codecs.getwriter('UTF-8')(Out(sys.stderr.fileno()), 'replace') sys.stdout = self.output self.output.encoding = 'UTF-8' # needed for input
def _write_with_fallback(s, write, fileobj): """Write the supplied string with the given write function like ``write(s)``, but use a writer for the locale's preferred encoding in case of a UnicodeEncodeError. Failing that attempt to write with 'utf-8' or 'latin-1'. """ try: write(s) return write except UnicodeEncodeError: # Let's try the next approach... pass enc = locale.getpreferredencoding() try: Writer = codecs.getwriter(enc) except LookupError: Writer = codecs.getwriter(_DEFAULT_ENCODING) f = Writer(fileobj) write = f.write try: write(s) return write except UnicodeEncodeError: Writer = codecs.getwriter('latin-1') f = Writer(fileobj) write = f.write # If this doesn't work let the exception bubble up; I'm out of ideas write(s) return write
def set_encoding(): # # XXX UTF-8 won't encode to latin-1/ISO8859-1: # UnicodeEncodeError: 'latin-1' codec can't encode character '\u2019' # # do PYTHONIOENCODING=utf8 equivalent # encoding = 'utf-8' if hasattr(sys.stdin, 'detach'): # >= 3.1 import io for s in ('stdin', 'stdout', 'stderr'): line_buffering = getattr(sys, s).line_buffering # print(s, line_buffering, file=sys.stderr) setattr(sys, s, io.TextIOWrapper(getattr(sys, s).detach(), encoding=encoding, line_buffering=line_buffering)) else: import codecs sys.stdin = codecs.getreader(encoding)(sys.stdin) sys.stdout = codecs.getwriter(encoding)(sys.stdout) sys.stderr = codecs.getwriter(encoding)(sys.stderr)
def main(): # log settings # log format #logging.basicConfig(level=logging.DEBUG, format='%(asctime)s [%(module)10s] [%(levelname)5s] %(message)s') sys.stdout = codecs.getwriter('utf8')(sys.stdout) sys.stderr = codecs.getwriter('utf8')(sys.stderr) config = init_config() if not config: return logger.log('[x] Sekolah Pokemon v2.0 by Bhagas', 'green') logger.log('[x] Menjalankan Konfigurasi', 'yellow') try: bot = PokemonGoBot(config) bot.start() logger.log('[x] Menjalankan PokemonGo Bot....', 'green') while (True): bot.take_step() except KeyboardInterrupt: logger.log('[x] Keluar PokemonGo Bot', 'red')
def __init__(self, output, termenc='UTF-8', scanner=DefaultScanner(), handler=DefaultHandler(), buffering=False): self.__termenc = termenc self.__scanner = scanner self.__handler = handler self._c1 = 0 if buffering: try: from cStringIO import StringIO self._output = codecs.getwriter(termenc)(StringIO()) except ImportError: try: from StringIO import StringIO self._output = codecs.getwriter(termenc)(StringIO()) except ImportError: from io import StringIO self._output = codecs.getwriter(termenc)(StringIO()) else: self._output = codecs.getwriter(termenc)(output) self._target_output = output self._buffering = buffering
def main(): logging.basicConfig(level=logging.INFO) logging.getLogger('suds.client').setLevel(logging.FATAL) sys.stdout = codecs.getwriter('utf-8')(sys.stdout) sys.stderr = codecs.getwriter('utf-8')(sys.stderr) parser = OptionParser() parser.add_option("", "--target_dir", dest="target_dir", help="HTML report target directory. The script will place the cnaf ggus report in the given directory.") (options, args) = parser.parse_args() print "cnaf-mon v. %s. " % __version__ if options.target_dir is None: print >> sys.stderr, "Please set the --target_dir option!" sys.exit(2) check_ticket_status(report_dir=options.target_dir)
def process_args(args): if not (args.ml or args.rb): args.rb = True if args.infile: ifp = io.open(args.infile, encoding='utf-8') else: if sys.version_info[0] >= 3: ifp = codecs.getreader('utf8')(sys.stdin.buffer) else: ifp = codecs.getreader('utf8')(sys.stdin) if args.outfile: ofp = io.open(args.outfile, mode='w', encoding='utf-8') else: if sys.version_info[0] >= 3: ofp = codecs.getwriter('utf8')(sys.stdout.buffer) else: ofp = codecs.getwriter('utf8')(sys.stdout) # initialize transliterator object trn = Transliterator(args.source, args.target, rb=args.rb, build_lookup=args.build_lookup) # transliterate text for line in ifp: tline = trn.convert(line) ofp.write(tline) # close files ifp.close() ofp.close()
def checkFiles(recurse, wrap, paths): # This method checks the input argument path(s) for existing files and # analyses them # Find existing files in the given input path(s) findFiles(recurse, paths) # If there are no valid input files then exit program checkNoInput(existingFiles) # Set encoding of the terminal to UTF-8 if config.PYTHON_VERSION.startswith(config.PYTHON_2): out = codecs.getwriter(config.UTF8_ENCODING)(sys.stdout) elif config.PYTHON_VERSION.startswith(config.PYTHON_3): out = codecs.getwriter(config.UTF8_ENCODING)(sys.stdout.buffer) # Wrap the xml output in <results> element, if wrapper flag is true if wrap: out.write("<?xml version='1.0' encoding='UTF-8'?>\n<results>\n") else: out.write("<?xml version='1.0' encoding='UTF-8'?>\n") # Process the input files for path in existingFiles: # Analyse file xmlElement = checkOneFile(path) # Write output to stdout writeElement(xmlElement,out) # Close </results> element if wrapper flag is true if wrap: out.write("</results>\n")
def main(): sys.stdin = codecs.getreader('utf-8')(sys.stdin) sys.stdout = codecs.getwriter('utf-8')(sys.stdout) sys.stderr = codecs.getwriter('utf-8')(sys.stderr) logging.basicConfig(level = logging.INFO) print '// Automatically generated by mozc' print '#ifndef MOZC_SESSION_QUALITY_MAIN_DATA_H_' print '#define MOZC_SESSION_QUALITY_MAIN_DATA_H_' print '' print 'namespace mozc {' print 'struct TestCase {' print ' const char* source;' print ' const char* expected_result;' print ' const char* hiragana_sentence;' print '};' print '' print 'static TestCase test_cases[] = {' for filename in sys.argv[1:]: convert_tsv(filename) print ' {NULL, NULL, NULL}' print '};' print '} // namespace mozc' print '#endif // MOZC_SESSION_QUALITY_MAIN_DATA_H_'
def cmd_expect(self, command, **kwargs): from pexpect import spawnu # prepare the environ, based on the system + our own env env = copy(environ) env.update(self.environ) # prepare the process kwargs.setdefault('env', env) kwargs.setdefault('show_output', self.log_level > 1) sensible = kwargs.pop('sensible', False) show_output = kwargs.pop('show_output') if show_output: if IS_PY3: kwargs['logfile'] = codecs.getwriter('utf8')(stdout.buffer) else: kwargs['logfile'] = codecs.getwriter('utf8')(stdout) if not sensible: self.debug('Run (expect) {0!r}'.format(command)) else: self.debug('Run (expect) {0!r} ...'.format(command.split()[0])) self.debug('Cwd {}'.format(kwargs.get('cwd'))) return spawnu(command, **kwargs)
def run(self, argv): """Runs mach with arguments provided from the command line.""" # If no encoding is defined, we default to UTF-8 because without this # Python 2.7 will assume the default encoding of ASCII. This will blow # up with UnicodeEncodeError as soon as it encounters a non-ASCII # character in a unicode instance. We simply install a wrapper around # the streams and restore once we have finished. orig_stdin = sys.stdin orig_stdout = sys.stdout orig_stderr = sys.stderr try: if sys.stdin.encoding is None: sys.stdin = codecs.getreader('utf-8')(sys.stdin) if sys.stdout.encoding is None: sys.stdout = codecs.getwriter('utf-8')(sys.stdout) if sys.stderr.encoding is None: sys.stderr = codecs.getwriter('utf-8')(sys.stderr) self._run(argv) finally: sys.stdin = orig_stdin sys.stdout = orig_stdout sys.stderr = orig_stderr
def setup_console(sys_enc="utf-8"): """ Set sys.defaultencoding to `sys_enc` and update stdout/stderr writers to corresponding encoding For Win32 the OEM console encoding will be used istead of `sys_enc` """ reload(sys) try: if sys.platform.startswith("win"): import ctypes enc = "cp%d" % ctypes.windll.kernel32.GetOEMCP() #TODO: win64/python64 implementation else: enc = (sys.stdout.encoding if sys.stdout.isatty() else sys.stderr.encoding if sys.stderr.isatty() else sys.getfilesystemencoding() or sys_enc) sys.setdefaultencoding(sys_enc) # redefine stdout/stderr in console if sys.stdout.isatty() and sys.stdout.encoding != enc: sys.stdout = codecs.getwriter(enc)(sys.stdout, 'replace') if sys.stderr.isatty() and sys.stderr.encoding != enc: sys.stderr = codecs.getwriter(enc)(sys.stderr, 'replace') except: pass print(sys.getdefaultencoding()) print(locale.getdefaultlocale())
def main(): # log settings # log format #logging.basicConfig(level=logging.DEBUG, format='%(asctime)s [%(module)10s] [%(levelname)5s] %(message)s') sys.stdout = codecs.getwriter('utf8')(sys.stdout) sys.stderr = codecs.getwriter('utf8')(sys.stderr) config = init_config() if not config: return logger.log('[x] PokemonGO Bot v1.0', 'green') logger.log('[x] Configuration initialized', 'yellow') try: bot = PokemonGoBot(config) bot.start() logger.log('[x] Starting PokemonGo Bot....', 'green') while True: bot.take_step() except KeyboardInterrupt: logger.log('[x] Exiting PokemonGo Bot', 'red')
def __init__(self, **kwargs): if sys.version_info > (3, 0): if sys.stdout.encoding != 'UTF-8': sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer, 'strict') if sys.stderr.encoding != 'UTF-8': sys.stderr = codecs.getwriter('utf-8')(sys.stderr.buffer, 'strict') else: if sys.stdout.encoding != 'UTF-8': sys.stdout = codecs.getwriter('utf-8')(sys.stdout, 'strict') if sys.stderr.encoding != 'UTF-8': sys.stderr = codecs.getwriter('utf-8')(sys.stderr, 'strict') self.aws_region = kwargs.get('aws_region') self.aws_access_key_id = kwargs.get('aws_access_key_id') self.aws_secret_access_key = kwargs.get('aws_secret_access_key') self.aws_session_token = kwargs.get('aws_session_token') self.log_group_name = kwargs.get('log_group_name') self.log_stream_name = kwargs.get('log_stream_name') self.filter_pattern = kwargs.get('filter_pattern') self.watch = kwargs.get('watch') self.color_enabled = kwargs.get('color_enabled') self.output_stream_enabled = kwargs.get('output_stream_enabled') self.output_group_enabled = kwargs.get('output_group_enabled') self.output_timestamp_enabled = kwargs.get('output_timestamp_enabled') self.output_ingestion_time_enabled = kwargs.get( 'output_ingestion_time_enabled') self.start = self.parse_datetime(kwargs.get('start')) self.end = self.parse_datetime(kwargs.get('end')) self.client = boto3.client( 'logs', aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, aws_session_token=self.aws_session_token, region_name=self.aws_region )
def wrap_fp(fp): if suffix == ".gz": fp = GzipFile(fileobj=fp, mode=mode) elif suffix == ".bz2": try: fp = BZ2File(fp, mode=mode) except TypeError: if sys.version_info < (3, 0, 0): raise NotImplementedError("built-in BZ2File is partially broken in python 2, install bz2file from pypi or use a compression setting other than 'bz2'") else: raise elif suffix == ".xz": fp = LZMAFile(fp, mode=mode) if (suffix or sys.version_info < (3,)) and "b" not in mode: # If mode is not binary (and we expect to be able to # write() str values, not bytes), need need to create # an additional encoding wrapper. That encoder can # probably use UTF-8 without any need for additional # configuration if "r" in mode and "w" in mode: fp = StreamReaderWriter(fp, codecs.getreader("utf-8"), codecs.getwriter("utf-8")) elif "w" in mode: fp = codecs.getwriter("utf-8")(fp) elif suffix: fp = codecs.getreader("utf-8")(fp) fp.realname = filename return fp
def main(): logging.basicConfig(level=logging.INFO) sys.stdout = codecs.getwriter('utf-8')(os.fdopen(sys.stdout.fileno(), 'w', 0)) sys.stderr = codecs.getwriter('utf-8')(os.fdopen(sys.stderr.fileno(), 'w', 0)) judgeenv.load_env(cli=True, testsuite=True) judgeenv.env['id'] = 'testsuite' executors.load_executors() executor_fail = not all(name in executors.executors for name in required_executors) if executor_fail: print(ansi_style('#ansi[A required executor failed to load.](red|bold)')) else: print(ansi_style('#ansi[All required executors loaded successfully.](green|bold)')) print() tester = Tester(judgeenv.problem_regex, judgeenv.case_regex) fails = tester.test_all() print() print('Test complete') if fails: print(ansi_style('#ansi[A total of %d case(s) failed](red|bold).') % fails) else: print(ansi_style('#ansi[All cases passed.](green|bold)')) raise SystemExit(int(executor_fail or fails != 0))
def setup_utf8(): """ We need to set the system encoding and stdout/stderr to utf-8. I already know that this is officially unsupported at least on Python 2.x installations, but many strings in Licorn® have UTF-8 characters inside. Our terminal emulators are all utf-8 natively, and outputing UTF-8 to log files never hurted anyone. Distros we support are all UTF-8 enabled at the lower level. If for some reason you would want another encoding, just define the PYTHONIOENCODING environment variable. This will probably hurt though, because many things in Licorn® assume a modern UTF-8 underlying OS. Some discussions: - http://drj11.wordpress.com/2007/05/14/python-how-is-sysstdoutencoding-chosen/ - http://www.haypocalc.com/wiki/Python_Unicode (in french) - http://stackoverflow.com/questions/1473577/writing-unicode-strings-via-sys-stdout-in-python - http://stackoverflow.com/questions/492483/setting-the-correct-encoding-when-piping-stdout-in-python - http://stackoverflow.com/questions/4374455/how-to-set-sys-stdout-encoding-in-python-3 """ # WARNING: 'UTF-8' is OK, 'utf-8' is not. It borks the ipython # shell prompt and readline() doesn't work anymore. default_encoding = os.getenv('PYTHONIOENCODING', 'UTF-8') if sys.getdefaultencoding() != default_encoding: reload(sys) sys.setdefaultencoding(default_encoding) if sys.stdout.encoding != default_encoding: sys.stdout = codecs.getwriter(default_encoding)(sys.stdout) if sys.stderr.encoding != default_encoding: sys.stderr = codecs.getwriter(default_encoding)(sys.stderr)
def setup_console(sys_enc="utf-8"): imp.reload(sys) try: # для win32 вызываем системную библиотечную функцию if sys.platform.startswith("win"): import ctypes enc = "cp%d" % ctypes.windll.kernel32.GetOEMCP() #TODO: проверить на win64/python64 else: # для Linux всё, кажется, есть и так enc = (sys.stdout.encoding if sys.stdout.isatty() else sys.stderr.encoding if sys.stderr.isatty() else sys.getfilesystemencoding() or sys_enc) # кодировка для sys sys.setdefaultencoding(sys_enc) # переопределяем стандартные потоки вывода, если они не перенаправлены if sys.stdout.isatty() and sys.stdout.encoding != enc: sys.stdout = codecs.getwriter(enc)(sys.stdout, 'replace') if sys.stderr.isatty() and sys.stderr.encoding != enc: sys.stderr = codecs.getwriter(enc)(sys.stderr, 'replace') except: pass # Ошибка? Всё равно какая - работаем по-старому...
def __init__(self, parent= None, description="A command line tool", **kwargs ): self.nErrors= 0 self.parent= parent self.config= self.defaultConfiguration() if self.parent: self.importConfiguration(parent.config) #Encoding is not set e.g. when piping. Maybe move this to Logger ? if self.config.outStream.encoding is None: self.config.outStream= getwriter("utf8")(self.config.outStream) if self.config.logStream.encoding is None: self.config.logStream= getwriter("utf8")(self.config.logStream) self.parser= argparse.ArgumentParser( add_help= False, description=description, **kwargs ) self.parser.add_argument("-h", "--help", action= 'store_true')
def xopen(filename, mode='r'): """ Replacement for the "open" function that can also open files that have been compressed with gzip. If the filename ends with .gz, the file is opened with gzip.open(). If it doesn't, the regular open() is used. If the filename is '-', standard output (mode 'w') or input (mode 'r') is returned. """ assert isinstance(filename, str) if filename == '-': return sys.stdin if 'r' in mode else sys.stdout if filename.endswith('.gz'): if sys.version_info[0] < 3: if 'r' in mode: return buffered_reader(gzip.open(filename, mode)) else: return gzip.open(filename, mode) else: if 'r' in mode: return getreader('ascii')(gzip.open(filename, mode)) else: return getwriter('ascii')(gzip.open(filename, mode)) elif filename.endswith('.bz2'): if sys.version_info[0] < 3: return bz2.BZ2File(filename, mode) else: if 'r' in mode: return getreader('ascii')(bz2.BZ2File(filename, mode)) else: return getwriter('ascii')(bz2.BZ2File(filename, mode)) else: return open(filename, mode)
def __init__(self, action_list=None, pipeline=None, stdout=None, stderr=None, recovery=None): self.action_list = action_list or [] self.pipeline = pipeline if pipeline is None: self.pipeline = json.loads(os.environ.get('PIPELINE', 'null')) if self.pipeline is None: self.pipeline = [] if stdout is None: stdout = codecs.getwriter('utf-8')(sys.stdout) if stderr is None: stderr = codecs.getwriter('utf-8')(sys.stderr) self.stdout = stdout self.stderr = stderr exec_pypeline.Pipeline.__init__( self, action_list or self.action_list, before_action=self.before_action, after_action=self.after_action, before_forward=self.before_forward, before_backward=self.before_backward, recovery=recovery ) store.Store.__init__(self, self.stdout, self.stderr) self._init_actions() self.notify_actions() self._failed_action = None self._failed_err = None
def resetOutputEncoding(): import sys import codecs if sys.stdout.encoding != 'cp850': sys.stdout = codecs.getwriter('cp850')(sys.stdout, 'strict') if sys.stderr.encoding != 'cp850': sys.stderr = codecs.getwriter('cp850')(sys.stderr, 'strict')
def main(self, *arguments): # Decode arguments. program = None import getopt options, arguments = getopt.getopt(arguments, 'de:hl:u') for option, value in options: if option == '-d': program = database.dump_database, elif option == '-e': program = database.erase_entries, value elif option == '-h': sys.stdout.write(__doc__) return elif option == '-l': program = database.load_entries, value elif option == '-u': program = database.undump_database, if program is None: if arguments: program = self.start_bot, arguments[0] arguments = arguments[1:] else: program = self.start_bot, os.environ['IRCSERVER'] assert not arguments, arguments sys.stdout = codecs.getwriter(file_encoding)( sys.stdout, 'backslashreplace') sys.stderr = codecs.getwriter(file_encoding)( sys.stderr, 'backslashreplace') program[0](*program[1:])
def main(): global dictionary global errwriter global filter_gramms global corpusHandler if "-utf" in sys.argv: errwriter = codecs.getwriter("utf-8")(sys.stderr, "xmlcharrefreplace") else: errwriter = codecs.getwriter("windows-1251")(sys.stderr, "xmlcharrefreplace") global WITHMONTHS WITHMONTHS = '-withmonths' in sys.argv filter_gramms = "-f" in sys.argv global replace_colon replace_colon = "--colon" in sys.argv corpusHandler = CorpusHandler() # assuming that if 4th argument doesn't start with '-', it's not some arbitrary option but # corpus name if len(sys.argv) > 4 and not sys.argv[4].startswith('-'): global CORPUS_NAME CORPUS_NAME = sys.argv[4] corpusHandler.load(sys.argv[1], "-utf" in sys.argv) convert_directory(os.path.abspath(sys.argv[2]), os.path.abspath(sys.argv[3]), os.path.abspath(sys.argv[2])) errwriter.close()
def main(argv): # Unicode I/O sys.stdin = codecs.getreader('UTF-8')(sys.stdin) sys.stdout = codecs.getwriter('UTF-8')(sys.stdout) sys.stderr = codecs.getwriter('UTF-8')(sys.stderr) def usage(): print >> sys.stderr, u'Create translation task. All input files should use UTF-8 encoding.' print >> sys.stderr, u'' print >> sys.stderr, u'Realtime post-editing task with feedback:' print >> sys.stderr, u'{0} realtime <source> <realtime.d> <task_name> <task_dir.data>'.format(argv[0]) print >> sys.stderr, u'' print >> sys.stderr, u'Realtime post-editing task (no adaptation from feedback):' print >> sys.stderr, u'{0} realtime-static <source> <realtime.d> <task_name> <task_dir.data>'.format(argv[0]) print >> sys.stderr, u'' print >> sys.stderr, u'Offline task (translations pre-generated):' print >> sys.stderr, u'{0} offline <source> <target> <task_name> <task_dir.data>'.format(argv[0]) print >> sys.stderr, u'' print >> sys.stderr, u'Place output task directory in data/tasks' sys.exit(2) task = argv[1] if len(argv) > 1 else '' if task == track.REALTIME and len(argv[2:]) == 4: new_realtime(*argv[2:], learn=True) elif task == track.REALTIME_STATIC and len(argv[2:]) == 4: new_realtime(*argv[2:], learn=False) elif task == track.OFFLINE and len(argv[2:]) == 4: new_offline(*argv[2:]) else: usage()
def main(): sys.stdout = codecs.getwriter('utf8')(sys.stdout) sys.stderr = codecs.getwriter('utf8')(sys.stderr) forum_db = ForumDB(); forum_db.connect_to_db(); print "*"*80 print "testing forum db input ..." print "*"*80 url = "http://www.heise.de/newsticker/foren/S-Apple-Patent-fuer-Notabschaltung-von-iPhone-Co/forum-184599/list/" print "storing %s ..." % (url) forum_db.store_forum_url(url) print "#"*80 print "" print "*"*80 print "testing forum db output ..." print "*"*80 max_forums_to_spider = 10; counter = 1 while 1: if counter > max_forums_to_spider: break; print "\tcounter: %i/%i ..." % (counter, max_forums_to_spider); url = forum_db.get_random_forum_url() if not url: print "\t\tgot no url ..." break print "\t\turl: " + url counter += 1 forum_db.disconnect_from_db() print "#"*80 print ""
#!/usr/bin/env python3.5 # Add gps coordinates to website # Usage: add.py?lat=-324.2343&lng=133.324324 import sys, codecs, cgi, cgitb, sqlite3, datetime web_cgi = codecs.getwriter('utf-8')(sys.stdout.buffer) HEADER = 'Content-Type:text/html;charset=utf-8' HEADER_ERR = 'Status: 400 Bad Request\n\nUsage: add.py?lat=-324.2343&lng=133.324324' def insert(lat, lng): timestamp = int(datetime.datetime.utcnow().timestamp()) try: conn = sqlite3.connect('activity.sqlite3') c = conn.cursor() c.execute( 'CREATE TABLE IF NOT EXISTS log (_id INTEGER PRIMARY KEY, lat REAL, lng REAL, timestamp INTEGER)' ) conn.commit() c.execute('INSERT INTO log (lat, lng, timestamp) VALUES (?, ?, ?)', (lat, lng, timestamp)) conn.commit() print("\n\nAdded.", file=web_cgi) except Exception as e: raise finally: c.close() conn.close() def main():
import codecs import time import datetime import sys from flask import Flask from flask_script import Manager, Option from sys import version_info from app.server import app, db from app.worker import backgroundWorker from app.models import User UTF8Writer = codecs.getwriter('utf8') sys.stdout = UTF8Writer(sys.stdout) app = Flask(__name__) manager = Manager(app, False) def getInput(text): py3 = version_info[0] > 2 #creates boolean value for test that Python major version > 2 if py3: return input(text) else: return raw_input(text) @manager.command
plt.grid(True) #plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=None, hspace=1) plt.show() #time.sleep(15) #plt.close('all') #def plot(self, data): logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG) locale.setlocale(locale.LC_NUMERIC, '') # change the output encoding to utf8 sys.stdout = codecs.getwriter('utf8')(sys.stdout) parser = argparse.ArgumentParser(description='Graph data') parser.add_argument('-price_min', dest='price_min', action='store', type=float, default=None, help='TODO') parser.add_argument('-price_max', dest='price_max', action='store', type=float, default=None, help='TODO')
import codecs import re from trans import translate try: import debug from pdb import set_trace as breakpoint except: pass debug_mode = False # debug_mode = True UTF8READER = codecs.getreader('utf8') UTF8WRITER = codecs.getwriter('utf8') EPILOG = """ Dependencies: python 2.6+ (maybe earlier). If the .md tag is used, install the markdown module (http://pythonhosted.org/Markdown/index.html). """ FORMAT_HELP = """PREPROCESSOR INPUT SYNTAX Notation: Grammar rules are of the form ELEMENT_TYPE -> GRAMMAR_EXPRESSION. A grammar expression may contain the following notation. {...} grouping, [...] optional, | or, +/* one/zero or more of the preceding form. ,.. non-empty comma-separated sequence of preceding form. Spaces are not significant in grammar rules. Sequences are non-empty. BOL/EOL beginning/end of line. EOF end of file.
def get_metrics(hparams, model_fn, ckpt=None): """Run inference and compute metrics.""" pred_estimator = tf.estimator.Estimator(model_fn=model_fn, model_dir=hparams.output_dir) benchmark_hook = BenchmarkHook(hparams.infer_batch_size) predictions = pred_estimator.predict(make_input_fn( hparams, tf.contrib.learn.ModeKeys.INFER), checkpoint_path=ckpt, hooks=[benchmark_hook]) translations = [] output_tokens = [] beam_id = 0 for prediction in predictions: # get the top translation. if beam_id == 0: for sent_id in range(hparams.infer_batch_size): if sent_id >= prediction["predictions"].shape[0]: break trans, output_length = nmt_utils.get_translation( prediction["predictions"], sent_id=sent_id, tgt_eos=hparams.eos, subword_option=hparams.subword_option) translations.append(trans) output_tokens.append(output_length) beam_id += 1 if beam_id == hparams.beam_width: beam_id = 0 trans_file = os.path.join( hparams.output_dir, "newstest2014_out_{}.tok.de".format( pred_estimator.get_variable_value(tf.GraphKeys.GLOBAL_STEP))) trans_dir = os.path.dirname(trans_file) if not tf.gfile.Exists(trans_dir): tf.gfile.MakeDirs(trans_dir) tf.logging.info("Writing to file %s" % trans_file) with codecs.getwriter("utf-8")(tf.gfile.GFile(trans_file, mode="wb")) as trans_f: trans_f.write("") # Write empty string to ensure file is created. for translation in translations: trans_f.write((translation + b"\n").decode("utf-8")) # Evaluation output_dir = os.path.join(pred_estimator.model_dir, "eval") tf.gfile.MakeDirs(output_dir) summary_writer = tf.summary.FileWriter(output_dir) ref_file = "%s.%s" % (hparams.test_prefix, hparams.tgt) # Hardcoded. metric = "bleu" score = get_sacrebleu(trans_file, hparams.detokenizer_file) misc_utils.print_out("bleu is %.5f" % score) with tf.Graph().as_default(): summaries = [] summaries.append(tf.Summary.Value(tag=metric, simple_value=score)) tf_summary = tf.Summary(value=list(summaries)) summary_writer.add_summary( tf_summary, pred_estimator.get_variable_value(tf.GraphKeys.GLOBAL_STEP)) summary_writer.close() return score, benchmark_hook.get_average_speed(), sum(output_tokens)
def main(): bot = False def handle_sigint(*args): raise SIGINTRecieved signal.signal(signal.SIGINT, handle_sigint) def initialize_task(bot, config): tree = TreeConfigBuilder(bot, config.raw_tasks).build() bot.workers = tree def initialize(config): from pokemongo_bot.datastore import Datastore ds = Datastore(conn_str='/data/{}.db'.format(config.username)) for directory in ['pokemongo_bot', 'pokemongo_bot/cell_workers']: ds.migrate(directory + '/migrations') bot = PokemonGoBot(ds.get_connection(), config) return bot def setup_logging(config): log_level = logging.ERROR if config.debug: log_level = logging.DEBUG logging.getLogger("requests").setLevel(log_level) logging.getLogger("websocket").setLevel(log_level) logging.getLogger("socketio").setLevel(log_level) logging.getLogger("engineio").setLevel(log_level) logging.getLogger("socketIO-client").setLevel(log_level) logging.getLogger("pgoapi").setLevel(log_level) logging.getLogger("rpc_api").setLevel(log_level) if config.logging: logging_format = '%(message)s' logging_format_options = '' if ('show_log_level' not in config.logging) or config.logging['show_log_level']: logging_format = '[%(levelname)s] ' + logging_format if ('show_process_name' not in config.logging) or config.logging['show_process_name']: logging_format = '[%(name)10s] ' + logging_format if ('show_thread_name' not in config.logging) or config.logging['show_thread_name']: logging_format = '[%(threadName)s] ' + logging_format if ('show_datetime' not in config.logging) or config.logging['show_datetime']: logging_format = '[%(asctime)s] ' + logging_format logging_format_options = '%Y-%m-%d %H:%M:%S' formatter = Formatter(logging_format,logging_format_options) for handler in logging.root.handlers[:]: handler.setFormatter(formatter) def start_bot(bot, config): bot.start() initialize_task(bot, config) bot.metrics.capture_stats() bot.health_record = BotEvent(config) return bot def get_commit_hash(): try: hash = subprocess.check_output(['git', 'rev-parse', 'HEAD'], stderr=subprocess.STDOUT) if all(c in string.hexdigits for c in hash[:-1]): with open('version', 'w') as f: f.write(hash) except: pass if not os.path.exists('version'): return 'unknown' with open('version') as f: return f.read()[:8] try: sys.stdout = codecs.getwriter('utf8')(sys.stdout) sys.stderr = codecs.getwriter('utf8')(sys.stderr) logger.info('PokemonGO Bot v1.0') logger.info('commit: ' + get_commit_hash()) config, config_file = init_config() if not config: return logger.info('Configuration initialized') health_record = BotEvent(config) health_record.login_success() setup_logging(config) finished = False while not finished: wait_time = config.reconnecting_timeout * 60 try: bot = initialize(config) bot = start_bot(bot, config) config_changed = check_mod(config_file) bot.event_manager.emit( 'bot_start', sender=bot, level='info', formatted='Starting bot...' ) while True: bot.tick() if config.live_config_update_enabled and config_changed(): logger.info('Config changed! Applying new config.') config, _ = init_config() if config.live_config_update_tasks_only: initialize_task(bot, config) else: bot = initialize(config) bot = start_bot(bot, config) except KeyboardInterrupt: bot.event_manager.emit( 'bot_exit', sender=bot, level='info', formatted='Exiting bot.' ) finished = True report_summary(bot) except NotLoggedInException: bot.event_manager.emit( 'api_error', sender=bot, level='info', formatted='Not logged in, reconnecting in {:d} seconds'.format(wait_time) ) time.sleep(wait_time) except ServerBusyOrOfflineException: bot.event_manager.emit( 'api_error', sender=bot, level='info', formatted='Server busy or offline' ) time.sleep(wait_time) except ServerSideRequestThrottlingException: bot.event_manager.emit( 'api_error', sender=bot, level='info', formatted='Server is throttling, reconnecting in 30 seconds' ) time.sleep(30) except PermaBannedException: bot.event_manager.emit( 'api_error', sender=bot, level='info', formatted='Probably permabanned, Game Over ! Play again at https://club.pokemon.com/us/pokemon-trainer-club/sign-up/' ) time.sleep(36000) except NoPlayerPositionSetException: bot.event_manager.emit( 'api_error', sender=bot, level='info', formatted='No player position set' ) time.sleep(wait_time) except GeocoderQuotaExceeded: raise Exception("Google Maps API key over requests limit.") except SIGINTRecieved: if bot: bot.event_manager.emit( 'bot_interrupted', sender=bot, level='info', formatted='Bot caught SIGINT. Shutting down.' ) report_summary(bot) except Exception as e: # always report session summary and then raise exception if bot: report_summary(bot) raise finally: # Cache here on SIGTERM, or Exception. Check data is available and worth caching. if bot: if len(bot.recent_forts) > 0 and bot.recent_forts[-1] is not None and bot.config.forts_cache_recent_forts: cached_forts_path = os.path.join( _base_dir, 'data', 'recent-forts-%s.json' % bot.config.username ) try: with open(cached_forts_path, 'w') as outfile: json.dump(bot.recent_forts, outfile) bot.event_manager.emit( 'cached_fort', sender=bot, level='debug', formatted='Forts cached.' ) except IOError as e: bot.event_manager.emit( 'error_caching_forts', sender=bot, level='debug', formatted='Error caching forts for {path}', data={'path': cached_forts_path} )
Return True if a sum is a simple. A sum is simple if its types have no fields, e.g. unaryop = Invert | Not | UAdd | USub """ for t in sum.types: if t.fields: return False return True #------------------------------------------------------------------------ # File Handling #------------------------------------------------------------------------ StreamWriter = codecs.getwriter('UTF-8') class FileAllocator(object): def __init__(self, output_dir=None): self.output_dir = output_dir # file_name -> file self.allocated_files = {} def open_sourcefile(self, name): "Allocate a file and save in in allocated_files" def close(self): for file in self.allocated_files.itervalues(): file.close()
def save_hparams(out_dir, hparams): """Save hparams.""" hparams_file = os.path.join(out_dir, "hparams") print_out(" saving hparams to %s" % hparams_file) with codecs.getwriter("utf-8")(tf.gfile.GFile(hparams_file, "wb")) as f: f.write(hparams.to_json())
try: with Image.open(tempfname) as img: return pytesseract.image_to_string(img) # f.write("============================\nIMAGE_STR\n============================\n") # f.write(pytesseract.image_to_string(img).encode('utf-8').decode('utf-8')) # f.write("\n\n============================\nBOUNDING BOXES\n============================\n") # f.write(pytesseract.image_to_boxes(img).encode('utf-8').decode('utf-8')) # f.write("\n\n============================\nIMAGE_DATA\n============================\n") # f.write(pytesseract.image_to_data(img).encode('utf-8').decode('utf-8')) # print("============================\nSCRIPT INFO\n============================") # print(pytesseract.image_to_osd(img).encode('utf-8').decode('utf-8')) except IOError as e: print('I/O error occurred: ' + e) finally: remove(tempfname) if __name__ == "__main__": if sys.stdout.encoding != 'cp850': sys.stdout = codecs.getwriter('cp850')(sys.stdout.buffer, 'strict') if sys.stderr.encoding != 'cp850': sys.stderr = codecs.getwriter('cp850')(sys.stderr.buffer, 'strict') parser = argparse.ArgumentParser() parser.add_argument("images", nargs='+', help="Image filename") args = parser.parse_args() runOCR(args.images)
def _print_figure(self, outfile, format, *, dpi, dsc_comments, orientation, papertype, bbox_inches_restore=None): """ Render the figure to a filesystem path or a file-like object. Parameters are as for `.print_figure`, except that *dsc_comments* is a all string containing Document Structuring Convention comments, generated from the *metadata* parameter to `.print_figure`. """ is_eps = format == 'eps' if not (isinstance(outfile, (str, os.PathLike)) or is_writable_file_like(outfile)): raise ValueError("outfile must be a path or a file-like object") # find the appropriate papertype width, height = self.figure.get_size_inches() if papertype == 'auto': papertype = _get_papertype( *orientation.swap_if_landscape((width, height))) paper_width, paper_height = orientation.swap_if_landscape( papersize[papertype]) if mpl.rcParams['ps.usedistiller']: # distillers improperly clip eps files if pagesize is too small if width > paper_width or height > paper_height: papertype = _get_papertype( *orientation.swap_if_landscape((width, height))) paper_width, paper_height = orientation.swap_if_landscape( papersize[papertype]) # center the figure on the paper xo = 72 * 0.5 * (paper_width - width) yo = 72 * 0.5 * (paper_height - height) llx = xo lly = yo urx = llx + self.figure.bbox.width ury = lly + self.figure.bbox.height rotation = 0 if orientation is _Orientation.landscape: llx, lly, urx, ury = lly, llx, ury, urx xo, yo = 72 * paper_height - yo, xo rotation = 90 bbox = (llx, lly, urx, ury) self._pswriter = StringIO() # mixed mode rendering ps_renderer = RendererPS(width, height, self._pswriter, imagedpi=dpi) renderer = MixedModeRenderer(self.figure, width, height, dpi, ps_renderer, bbox_inches_restore=bbox_inches_restore) self.figure.draw(renderer) def print_figure_impl(fh): # write the PostScript headers if is_eps: print("%!PS-Adobe-3.0 EPSF-3.0", file=fh) else: print( f"%!PS-Adobe-3.0\n" f"%%DocumentPaperSizes: {papertype}\n" f"%%Pages: 1\n", end="", file=fh) print( f"{dsc_comments}\n" f"%%Orientation: {orientation.name}\n" f"{get_bbox_header(bbox)[0]}\n" f"%%EndComments\n", end="", file=fh) Ndict = len(psDefs) print("%%BeginProlog", file=fh) if not mpl.rcParams['ps.useafm']: Ndict += len(ps_renderer._character_tracker.used) print("/mpldict %d dict def" % Ndict, file=fh) print("mpldict begin", file=fh) print("\n".join(psDefs), file=fh) if not mpl.rcParams['ps.useafm']: for font_path, chars \ in ps_renderer._character_tracker.used.items(): if not chars: continue fonttype = mpl.rcParams['ps.fonttype'] # Can't use more than 255 chars from a single Type 3 font. if len(chars) > 255: fonttype = 42 fh.flush() if fonttype == 3: fh.write(_font_to_ps_type3(font_path, chars)) else: # Type 42 only. _font_to_ps_type42(font_path, chars, fh) print("end", file=fh) print("%%EndProlog", file=fh) if not is_eps: print("%%Page: 1 1", file=fh) print("mpldict begin", file=fh) print("%s translate" % _nums_to_str(xo, yo), file=fh) if rotation: print("%d rotate" % rotation, file=fh) print("%s clipbox" % _nums_to_str(width * 72, height * 72, 0, 0), file=fh) # write the figure print(self._pswriter.getvalue(), file=fh) # write the trailer print("end", file=fh) print("showpage", file=fh) if not is_eps: print("%%EOF", file=fh) fh.flush() if mpl.rcParams['ps.usedistiller']: # We are going to use an external program to process the output. # Write to a temporary file. with TemporaryDirectory() as tmpdir: tmpfile = os.path.join(tmpdir, "tmp.ps") with open(tmpfile, 'w', encoding='latin-1') as fh: print_figure_impl(fh) if mpl.rcParams['ps.usedistiller'] == 'ghostscript': _try_distill(gs_distill, tmpfile, is_eps, ptype=papertype, bbox=bbox) elif mpl.rcParams['ps.usedistiller'] == 'xpdf': _try_distill(xpdf_distill, tmpfile, is_eps, ptype=papertype, bbox=bbox) _move_path_to_path_or_stream(tmpfile, outfile) else: # Write directly to outfile. with cbook.open_file_cm(outfile, "w", encoding="latin-1") as file: if not file_requires_unicode(file): file = codecs.getwriter("latin-1")(file) print_figure_impl(file)
from core import globalvar from core.utils import _ import wx import wx.lib.mixins.listctrl as listmix from grass.script import core as grass from core.gcmd import GMessage, GError, DecodeString, RunCommand from core.utils import GetListOfLocations, GetListOfMapsets from location_wizard.dialogs import RegionDef from gui_core.dialogs import TextEntryDialog from gui_core.widgets import GenericValidator, StaticWrapText from gui_core.wrap import Button, ListCtrl, StaticText, \ StaticBox, TextCtrl sys.stderr = codecs.getwriter('utf8')(sys.stderr) class GRASSStartup(wx.Frame): exit_success = 0 # 2 is file not found from python interpreter exit_user_requested = 5 """GRASS start-up screen""" def __init__(self, parent=None, id=wx.ID_ANY, style=wx.DEFAULT_FRAME_STYLE): # # GRASS variables #
# coding: utf-8 import os, re import json import requests import codecs, sys sys.stdout = codecs.getwriter("UTF-8")(sys.stdout) # this scrapes oibs64 for all the course data. # see data_spec.md for interpreting out_file. # be aware- spits a lot of output to stdout. out_file = "data.json" oibs_url = "https://oibs2.metu.edu.tr/View_Program_Course_Details_64/main.php" # stuff for department-izing course codes. # a course ID like 5720172 does not become aee172 on its own. prefixes = { '219': u'GENE', '956': u'OCEA', '450': u'FLE', '612': u'PERS', '451': u'TEFL', '810': u'GWS', '811': u'UPL', '814': u'SA', '815': u'ARS', '816': u'MCS', '817': u'FPSY', '453': u'PES',
#!/usr/bin/env python3 import cgi, codecs, sys sys.stdout = codecs.getwriter('utf8')(sys.stdout.buffer) # Support Chinese # Variables form = cgi.FieldStorage() template = open("../information.html", 'r') # An html template for information content = template.read() # Functions def getVal(form, attr): # Get value from html form if form.getvalue(attr): return form.getvalue(attr) return "" def arrayToString(arr): # Convert array to string if type(arr) is str: return arr string = "" for element in arr: string += element if element != arr[-1]: # Check if it is the last element of array string += ", " return string # Main function if __name__ == '__main__':
# -*- coding: utf-8 -*- # initialize Django API import os, sys, codecs os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'trimco.settings') import django django.setup() sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach( )) # for printing; parasol server for some curious reason uses latin-1 locale # import some models from info.models import Speaker from corpora.models import Recording import xml.etree.cElementTree as ET fields = ('string_id', 'sex', 'year_of_birth', 'language', 'education', 'location') # fields we want to include in an XML file if __name__ == '__main__': speakers = Speaker.objects.all() # get all speakers meta = ET.Element("meta") for speaker in speakers:
import pexpect import re import time import aggreg import switches import csv import tokenize import base64 import checkfunc from netaddr import IPNetwork, IPAddress import services t1 = time.time() opt_diag = None onu_status = None sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach()) script_name = 'check.py' ifload = [] # Переменная массива загрузки порта def checkpolicy(ip): print(ip) junroutes = '/var/www/check/cgi-bin/scripts/shaper_routes.txt' output = open(junroutes) policy = 'no ip policy INET_CHANNEL2' for route in output: net, gw = route.split()[2], route.split()[3] #gw = gw[:-1] #Check ip enter to network if IPAddress(ip) in IPNetwork(net): if gw == '10.10.10.10': #Gateway Cisco 4948 Bykova13
def main(): def usage(): sys.stderr.write( """usage: {0} [[-e|encoding <encoding>] -c|--clean|-s|--smudge] path """.format(sys.argv[0])) return # end of usage() # main function body try: opts, args = getopt.getopt(sys.argv[1:], "ce:hs", ['clean', 'encoding', 'smudge', 'help']) except getopt.GetoptErrror as err: sys.stderr.write(str(err)) usage() sys.exit(2) smudge = True enc = None hf = False for (o, a) in opts: if o in ('-c', '--clean'): smudge = False elif o in ('-e', '--encoding'): enc = a elif o in ('-h', '--help'): hf = True elif o in ('-s', '--smudge'): smudge = True else: assert False, "unhandled option {0}".format(o) if hf: usage() sys.exit(0) if len(args) != 1: sys.stderr.write('error:just one file path is needed') usage() sys.exit(1) if enc is not None: setdefaultencoding(enc) if isinstance(sys.stdin, io.TextIOWrapper): try: sys.stdin.reconfigure(encoding=_default_encoding, errors='surrogateescape') sys.stdout.reconfigure(encoding=_default_encoding, errors='surrogateescape') stdin = sys.stdin stdout = sys.stdout except AttributeError: stdin = codecs.getreader(_default_encoding)(sys.stdin.buffer, 'surrogateescape') stdout = codecs.getwriter(_default_encoding)(sys.stdout.buffer, 'surrogateescape') else: stdin = sys.stdin stdout = sys.stdout if smudge: substkw(stdin, stdout, expand_kwdic(args[0], read_conf(args[0]))) else: unsubstkw(stdin, stdout, read_conf(args[0]))
def delta(orgn, dest, searchdate, searchkey): if not DEV_LOCAL: db = customfunction.dbconnection() cursor = db.cursor() db.set_character_set('utf8') url = "http://www.delta.com/" searchid = str(searchkey) currentdatetime = datetime.datetime.now() stime = currentdatetime.strftime('%Y-%m-%d %H:%M:%S') sys.stdout = codecs.getwriter('utf-8')(sys.stdout) driver = webdriver.PhantomJS(service_args=[ '--ignore-ssl-errors=true', '--ssl-protocol=any', '--load-images=false' ], service_log_path='/tmp/ghostdriver.log') driver.set_window_size(1120, 1080) # driver = webdriver.Firefox() def storeFlag(searchkey, stime): if not DEV_LOCAL: cursor.execute( "INSERT INTO pexproject_flightdata (flighno,searchkeyid,scrapetime,stoppage,stoppage_station,origin,destination,duration,maincabin,maintax,firstclass,firsttax,business,businesstax,cabintype1,cabintype2,cabintype3,datasource,departdetails,arivedetails,planedetails,operatedby,economy_code,business_code,first_code) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);", ("flag", str(searchkey), stime, "flag", "test", "flag", "flag", "flag", "0", "0", "0", "0", "0", "0", "flag", "flag", "flag", "delta", "flag", "flag", "flag", "flag", "flag", "flag", "flag")) db.commit() driver.quit() try: driver.get(url) time.sleep(1) flg = 0 pageStatus = '' while flg < 15 and pageStatus != 'complete': time.sleep(1) print "flg", flg pageStatus = driver.execute_script('return document.readyState;') print "pageStatus", pageStatus flg = flg + 1 WebDriverWait(driver, 6).until( EC.presence_of_element_located((By.ID, "oneWayBtn"))) oneway = driver.find_element_by_id('oneWayBtn') #oneway.click() driver.execute_script("arguments[0].click();", oneway) driver.execute_script( "document.getElementById('originCity').setAttribute('value', '" + orgn.strip() + "')") # origin = driver.find_element_by_id("originCity") # origin.clear() # origin.send_keys(orgn.strip()) driver.execute_script( "document.getElementById('destinationCity').setAttribute('value', '" + dest.strip() + "')") # destination = driver.find_element_by_id("destinationCity") # destination.send_keys(dest.strip()) ddate = driver.find_element_by_id("departureDate") driver.execute_script( "document.getElementById('departureDate').setAttribute('value', '" + str(searchdate) + "')") milebtn = driver.find_element_by_id("milesBtn") milebtn.click() driver.find_element_by_id("findFlightsSubmit").send_keys(Keys.ENTER) except: print "before data page" storeFlag(searchkey, stime) return searchkey time.sleep(1) try: WebDriverWait(driver, 5).until( EC.presence_of_element_located((By.ID, "submitAdvanced"))) print "no data" storeFlag(searchkey, stime) return searchkey except: print "Data found" try: # driver.save_screenshot('/root/out_enter.png'); WebDriverWait(driver, 15).until( EC.presence_of_element_located((By.ID, "showAll-footer"))) print "More than one page" driver.execute_script(""" DWRHandler.currentPage = -1; var _shopInputDo=shopInputDo; shoppingUtil.scrollWindow("top"); FilterFunctions.showFilterMsg(); FlightUtil.emptyResults(); ResummarizeFlightResultsDWR.pageResults(DWRHandler.currentPage, _shopInputDo.currentSessionCheckSum, delta.airShopping.cacheKey, { async: true, callback: function(searchResults) { if (searchResults != null) { var jsonData = {}; jsonData['jsonobj'] = JSON.stringify(searchResults); var cabininfo = document.getElementsByClassName('tblHeadUp')[0].innerHTML; jsonData['cabinTypes'] = cabininfo; localStorage.setItem('deltaData', JSON.stringify(jsonData)); var element = document.createElement('div'); element.id = "submitAdvanced"; element.appendChild(document.createTextNode("text")); document.body.appendChild(element); throw new Error("Results found"); if (searchResults.errorFwdURL == null) { jsonResultPopulation(searchResults); paginationPopulation(searchResults); if (shoppingUtil.isIE8()) { if (DWRHandler.currentPage == -1) { ieTimeout = setTimeout("RenderTemplate.renderResult();FilterFunctions.hideFilterMsg();RenderTemplate.adjustHeight();", 200); } else { ieTimeout = setTimeout("RenderTemplate.renderResult();FilterFunctions.hideFilterMsg();RenderTemplate.adjustHeight();", 100); } } else { RenderTemplate.renderResult(); FilterFunctions.hideFilterMsg(); RenderTemplate.adjustHeight(); } if (DWRHandler.currentPage == -1) { $("#showAll").hide(); $("#showAll-footer").hide(); } contienuedOnload(false); if (searchResults.debugInfo != null && ((typeof(printRequestResponse) !== "undefined") && printRequestResponse == true)) { $("#requestXml").text(searchResults.debugInfo.itaRequest); $("#responceXml").text(searchResults.debugInfo.itaResponse); $("#reqRes").show(); } } else { window.location.replace(searchResults.errorFwdURL); } } else { FilterFunctions.errorHandling(); } $(".tableHeaderHolderFareBottom.return2Top").show(); }, exceptionHandler: FilterFunctions.errorHandling }); """) except: print "single page" try: driver.execute_script(""" var sortBy = "deltaScheduleAward" ; SearchFlightResultsDWR.searchResults(currentSessionCheckSum, sortBy[0], delta.airShopping.numberOfColumnsToRequest, delta.airShopping.cacheKey, { async: true, timeout: 65000, callback: function(searchResults) { var jsonData = {}; jsonData['jsonobj'] = JSON.stringify(searchResults); var cabininfo = document.getElementsByClassName('tblHeadUp')[0].innerHTML; jsonData['cabinTypes'] = cabininfo; localStorage.setItem('deltaData', JSON.stringify(jsonData)); var element = document.createElement('div'); element.id = "submitAdvanced"; element.appendChild(document.createTextNode("text")); document.body.appendChild(element); throw new Error("Results found"); if (searchResults.errorFwdURL == null || searchResults.errorFwdURL == "") { flightResultsObj.isDOMReady(searchResults, action, false); FilterFunctions.hideFilterMsg(); } else { flightResultsObj.isDOMReady(searchResults, false, true); } if (!action) { Wait.hide(); $(".tableHeaderHolderFareBottom").show(); $("#nextGenAirShopping .tableHeaderHolder").show(); } }, errorHandler: function(msg, exc) { shoppingUtil.errorHandler(msg, exc); }, exceptionHandler: function(msg, exc) { (action) ? FilterFunctions.hideFilterMsg(): ""; shoppingUtil.exceptionHandler(msg, exc); } }); """) except: storeFlag(searchkey, stime) return searchkey try: WebDriverWait(driver, 10).until( EC.presence_of_element_located((By.ID, "submitAdvanced"))) result = driver.execute_script( """ return localStorage.getItem('deltaData'); """) deltaObj = json.loads(result) a_file = open('1.json', 'w') a_file.write(json.dumps(deltaObj, indent=4)) # a_file.write(result.encode('utf8')) # print deltaObj, '@@@@@@' # return searchResult = json.loads(deltaObj['jsonobj']) # a_file = open('2.json', 'w') # a_file.write(json.dumps(searchResult, indent=4)) # return cabinhead = "<tr>" + deltaObj['cabinTypes'] + "</tr>" soup = BeautifulSoup(cabinhead, "xml") tds = soup.findAll("td") pricecol = '' pricecol = soup.findAll("a", {"class": "tblHeadBigtext lnkCabinName"}) if len(pricecol) < 1: pricecol = soup.findAll("label", {"class": "tblHeadBigtext"}) flightData = searchResult["itineraries"] except: # raise storeFlag(searchkey, stime) return searchkey values_string = [] for i in range(0, len(flightData)): totalFareDetails = flightData[i]['totalFare'] slicesDetails = flightData[i]['slices'] departDetail = [] ariveDetail = [] flightDetail = [] operatorDetail = [] SourceCOde = '' tripDuration = '' destinationCode = '' arivalTime = '' departTime = '' flightNo = '' for k in range(0, len(slicesDetails)): tripDuration = slicesDetails[k]['duration'] SourceCOde = slicesDetails[k]['sliceOrigin']['airportCode'] destinationCode = slicesDetails[k]['sliceDestination'][ 'airportCode'] arivalTime = slicesDetails[k]['sliceArrivalTime'] departTime = slicesDetails[k]['sliceDepartureTime'] layover = slicesDetails[k]['flights'] "**************************************** FLIGHT DETAILS ****************************************" FlightFlag = 0 for m in range(0, len(layover)): legData = layover[m]['legs'] for n in range(0, len(legData)): legdetail = legData[n] if legdetail: fromAirport = '' destAirport = '' "=========================LEG INFO==================================" if 'origin' in legdetail: orgnCode = legdetail['origin']['airportCode'] cityname = legdetail['origin']['nearByCities'][0][ 'name'] cityCode = legdetail['origin']['nearByCities'][0][ 'country']['region']['code'] fromAirport = orgnCode if 'destination' in legdetail: destCode = legdetail['destination']['airportCode'] destcityname = legdetail['destination'][ 'nearByCities'][0]['name'] destcityCode = legdetail['destination'][ 'nearByCities'][0]['country']['region']['code'] destAirport = destCode duration = legdetail['duration'] schedDepartureTime = legdetail['schedDepartureTime'] schedDepartureDate = legdetail['schedDepartureDate'] schedArrivalTime = legdetail['schedArrivalTime'] schedArrivalDate = legdetail['schedArrivalDate'] '@@@@@@@ departDetails format @@@@@@@' departinfo_time = schedDepartureDate + " " + schedDepartureTime departinfo_time = datetime.datetime.strptime( departinfo_time, '%a %b %d %Y %I:%M%p') departinfo_time = departinfo_time.strftime( '%Y/%m/%d %H:%M') if not DEV_LOCAL: fromAirport = customfunction.get_airport_detail( fromAirport) or fromAirport fromDetail = departinfo_time + " | from " + fromAirport departDetail.append(fromDetail) departinfo_time = schedArrivalDate + " " + schedArrivalTime departinfo_time = datetime.datetime.strptime( departinfo_time, '%a %b %d %Y %I:%M%p') departinfo_time = departinfo_time.strftime( '%Y/%m/%d %H:%M') if not DEV_LOCAL: destAirport = customfunction.get_airport_detail( destAirport) or destAirport toDetails = departinfo_time + " | at " + destAirport ariveDetail.append(toDetails) aircraft = legdetail['aircraft']['shortName'] airlineCode = legdetail['marketAirline']['airline'][ 'airlineCode'] flightNumber = legdetail['flightNumber'] if FlightFlag == 0: flightNo = airlineCode + " " + str(flightNumber) # --- NORM --- if aircraft[:3] == 'MD-': aircraft = 'McDonnell Douglas MD ' + aircraft[3:] elif aircraft[:3] == 'CRJ': aircraft = 'Bombardier ' + aircraft flightFormat = airlineCode + " " + str( flightNumber ) + " | " + aircraft + " (" + duration + ")" flightDetail.append(flightFormat) operatedby = legdetail['operatingAirline']['airline'][ 'airlineName'] operatorDetail.append(operatedby) FlightFlag = FlightFlag + 1 "====================Fare info =================================" fareFlag = 0 cabintype1 = '' cabintype2 = '' cabintype3 = '' ecofare = 0 echoTax = 0 bussfare = 0 busstax = 0 firstFare = 0 firsttax = 0 ecofareClass = '' bussFareClass = '' firstFareClass = '' eco_fare_code = '' bus_fare_code = '' first_fare_code = '' for j in range(0, len(totalFareDetails)): cabintype = '' miles = 0 taxes = 0 fareCode = [] if totalFareDetails[j]['cabinName'] != None: tax = 0 fareCodeHolder = totalFareDetails[j]['miscFlightInfos'] for c in range(0, len(fareCodeHolder)): fareCabin = fareCodeHolder[c]['cabinName'] bookingCode = fareCodeHolder[c]['displayBookingCode'] fareCode.append(bookingCode) bookingCode = bookingCode + " " + fareCabin cabinName = totalFareDetails[j]['cabinName'] miles = totalFareDetails[j]['totalAwardMiles'] if ',' in miles: miles = miles.replace(',', '') taxInt = totalFareDetails[j]['totalPriceLeft'] if ',' in taxInt: taxInt = taxInt.replace(',', '') taxFloat = totalFareDetails[j]['totalPriceRight'] if taxFloat == '' or taxFloat == None: taxFloat = 0 tax = float(taxInt) + float(taxFloat) currencyCode = totalFareDetails[j]['currencyCode'] if currencyCode and currencyCode != 'USD': currencychange = urllib.urlopen( "https://www.exchangerate-api.com/%s/%s/%f?k=e002a7b64cabe2535b57f764" % (currencyCode, "USD", float(tax))) taxes = currencychange.read() else: taxes = tax if len(pricecol) > 1: if j == 0: cabintype = "Economy" if j == 1 and 'First' not in pricecol[1].text: cabintype = "Business" if j == 2 and len( pricecol) > 2 and 'First' not in pricecol[2].text: cabintype = "Business" else: if len(pricecol) > 0 and len(pricecol) < 2: if 'Main Cabin' in pricecol[0].text: cabintype = "Economy" elif 'First' not in pricecol[0].text: cabintype = 'Business' if 'Economy' in cabintype: ecofare = miles echoTax = taxes cabintype1 = "Economy" if len(fareCode) > 0: eco_fare_code = ','.join(fareCode) ecofareClass = ' Economy@'.join(fareCode) + ' Economy' ecofareClass = ecofareClass elif 'Business' in cabintype: cabintype2 = "Business" bussfare = miles busstax = taxes if len(fareCode) > 0: bus_fare_code = ','.join(fareCode) bussFareClass = ' Business@'.join(fareCode) + ' Business' else: cabintype3 = "First" firstFare = miles firsttax = taxes if len(fareCode) > 0: first_fare_code = ','.join(fareCode) firstFareClass = ' First@'.join(fareCode) + ' First' departdetailtext = '@'.join(departDetail) ariveDetailtext = '@'.join(ariveDetail) flightDetailtext = '@'.join(flightDetail) operatorDetailtext = '@'.join(operatorDetail) stoppage = '' stop = int(len(departDetail) - 1) if stop == 0: stoppage = "NONSTOP" elif stop == 1: stoppage = "1 STOP" else: stoppage = str(stop) + " STOPS" arivalTime1 = (datetime.datetime.strptime(arivalTime, '%I:%M%p')) arivalTime = arivalTime1.strftime('%H:%M') departTime1 = (datetime.datetime.strptime(departTime, '%I:%M%p')) departTime = departTime1.strftime('%H:%M') if len(pricecol) > 1 and 'Delta Comfort+' in pricecol[1].text: bussFareClass = bussFareClass.replace('Business', 'Economy') values_string.append( (flightNo, str(searchkey), stime, stoppage, "test", SourceCOde, destinationCode, departTime, arivalTime, tripDuration, str(ecofare), str(echoTax), '0', '0', str(firstFare), str(firsttax), cabintype1, cabintype2, cabintype3, "delta", departdetailtext, ariveDetailtext, flightDetailtext, operatorDetailtext, ecofareClass, '', firstFareClass, eco_fare_code, '', first_fare_code)) values_string.append( (flightNo, str(searchkey), stime, stoppage, "test", SourceCOde, destinationCode, departTime, arivalTime, tripDuration, str(bussfare), str(busstax), '0', '0', '0', '0', cabintype1, cabintype2, cabintype3, "delta", departdetailtext, ariveDetailtext, flightDetailtext, operatorDetailtext, bussFareClass, '', '', bus_fare_code, '', '')) else: values_string.append( (flightNo, str(searchkey), stime, stoppage, "test", SourceCOde, destinationCode, departTime, arivalTime, tripDuration, str(ecofare), str(echoTax), str(bussfare), str(busstax), str(firstFare), str(firsttax), cabintype1, cabintype2, cabintype3, "delta", departdetailtext, ariveDetailtext, flightDetailtext, operatorDetailtext, ecofareClass, bussFareClass, firstFareClass, eco_fare_code, bus_fare_code, first_fare_code)) if len(values_string) > 50: if not DEV_LOCAL: cursor.executemany( "INSERT INTO pexproject_flightdata (flighno,searchkeyid,scrapetime,stoppage,stoppage_station,origin,destination,departure,arival,duration,maincabin,maintax,firstclass,firsttax,business,businesstax,cabintype1,cabintype2,cabintype3,datasource,departdetails,arivedetails,planedetails,operatedby,economy_code,business_code,first_code,eco_fare_code,business_fare_code,first_fare_code) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);", values_string) db.commit() else: print values_string values_string = [] if len(values_string) > 0: if not DEV_LOCAL: cursor.executemany( "INSERT INTO pexproject_flightdata (flighno,searchkeyid,scrapetime,stoppage,stoppage_station,origin,destination,departure,arival,duration,maincabin,maintax,firstclass,firsttax,business,businesstax,cabintype1,cabintype2,cabintype3,datasource,departdetails,arivedetails,planedetails,operatedby,economy_code,business_code,first_code,eco_fare_code,business_fare_code,first_fare_code) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s);", values_string) db.commit() else: print values_string storeFlag(searchkey, stime) return searchkey
def main(): from subprocess import check_output, CalledProcessError parser = ArgumentParser(epilog=__doc__, formatter_class=RawDescriptionHelpFormatter) task = parser.add_mutually_exclusive_group() task.add_argument('--install', action='store_true', help="""Install nbstripout in the current repository (set up the git filter and attributes)""") task.add_argument('--uninstall', action='store_true', help="""Uninstall nbstripout from the current repository (remove the git filter and attributes)""") task.add_argument('--is-installed', action='store_true', help='Check if nbstripout is installed in current repository') task.add_argument('--status', action='store_true', help='Print status of nbstripout installation in current repository and configuration summary if installed') parser.add_argument('--keep-count', action='store_true', help='Do not strip the execution count/prompt number') parser.add_argument('--keep-output', action='store_true', help='Do not strip output') parser.add_argument('--attributes', metavar='FILEPATH', help="""Attributes file to add the filter to (in combination with --install/--uninstall), defaults to .git/info/attributes""") parser.add_argument('--global', dest='_global', action='store_true', help='Use global git config (default is local config)') task.add_argument('--version', action='store_true', help='Print version') parser.add_argument('--force', '-f', action='store_true', help='Strip output also from files with non ipynb extension') parser.add_argument('--textconv', '-t', action='store_true', help='Prints stripped files to STDOUT') parser.add_argument('files', nargs='*', help='Files to strip output from') args = parser.parse_args() git_config = ['git', 'config'] + (['--global'] if args._global else []) if args.install: sys.exit(install(git_config, attrfile=args.attributes)) if args.uninstall: sys.exit(uninstall(git_config, attrfile=args.attributes)) if args.is_installed: sys.exit(status(git_config, verbose=False)) if args.status: sys.exit(status(git_config, verbose=True)) if args.version: print(__version__) sys.exit(0) try: extra_keys = check_output(git_config + ['filter.nbstripout.extrakeys']).strip() except CalledProcessError: extra_keys = '' input_stream = None if sys.version_info < (3, 0): import codecs # Use UTF8 reader/writer for stdin/stdout # http://stackoverflow.com/a/1169209 if sys.stdin: input_stream = codecs.getreader('utf8')(sys.stdin) output_stream = codecs.getwriter('utf8')(sys.stdout) else: # Wrap input/output stream in UTF-8 encoded text wrapper # https://stackoverflow.com/a/16549381 if sys.stdin: input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8') output_stream = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') for filename in args.files: if not (args.force or filename.endswith('.ipynb')): continue try: with io.open(filename, 'r', encoding='utf8') as f: nb = read(f, as_version=NO_CONVERT) nb = strip_output(nb, args.keep_output, args.keep_count, extra_keys) if args.textconv: write(nb, output_stream) output_stream.flush() else: with io.open(filename, 'w', encoding='utf8') as f: write(nb, f) except NotJSONError: print("'{}' is not a valid notebook".format(filename), file=sys.stderr) sys.exit(1) except Exception: # Ignore exceptions for non-notebook files. print("Could not strip '{}'".format(filename), file=sys.stderr) raise if not args.files and input_stream: try: nb = strip_output(read(input_stream, as_version=NO_CONVERT), args.keep_output, args.keep_count, extra_keys) write(nb, output_stream) output_stream.flush() except NotJSONError: print('No valid notebook detected', file=sys.stderr) sys.exit(1)
import argparse import codecs import os import re import sys import numpy as np parser = argparse.ArgumentParser(description='') parser.add_argument('inputf', type=str, metavar='', help='') A = parser.parse_args() if __name__ == '__main__': sys.stdout = codecs.getwriter('utf-8')(sys.stdout) sys.stderr = codecs.getwriter('utf-8')(sys.stderr) D = {} f = codecs.open(A.inputf, "r", "utf-8") for line in f: fields = line.strip().split() D[fields[0]] = np.array(map(float, fields[1:])) f.close() # for tag in D: # print tag, D[tag] POS_DICT = { 1: "NN", 2: "NNP", 3: "IN", 4: "DT",
from cam.sgnmt.predictors.grammar import RuleXtractPredictor from cam.sgnmt.predictors.length import WordCountPredictor, NBLengthPredictor, \ ExternalLengthPredictor, NgramCountPredictor from cam.sgnmt.predictors.misc import IdxmapPredictor, UnboundedIdxmapPredictor, \ UnboundedAltsrcPredictor, AltsrcPredictor, UnkvocabPredictor from cam.sgnmt.predictors.misc import UnkCountPredictor from cam.sgnmt.predictors.ngram import SRILMPredictor from cam.sgnmt.predictors.tokenization import Word2charPredictor, FSTTokPredictor from cam.sgnmt.tf.interface import tf_get_nmt_predictor, tf_get_nmt_vanilla_decoder, \ tf_get_rnnlm_predictor, tf_get_default_nmt_config, tf_get_rnnlm_prefix from cam.sgnmt.ui import get_args, get_parser, validate_args # UTF-8 support if sys.version_info < (3, 0): sys.stderr = codecs.getwriter('UTF-8')(sys.stderr) sys.stdout = codecs.getwriter('UTF-8')(sys.stdout) sys.stdin = codecs.getreader('UTF-8')(sys.stdin) # Load configuration from command line arguments or configuration file args = get_args() # Set up logger logger = logging.getLogger(__name__) logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') logging.getLogger().setLevel(logging.INFO) if args.verbosity == 'debug': logging.getLogger().setLevel(logging.DEBUG) elif args.verbosity == 'info': logging.getLogger().setLevel(logging.INFO) elif args.verbosity == 'warn':
# -*- coding: utf-8 -*- # Author: Santosh Rai # Version: 1.00 (20200520) # upload files to GCP with gsutil import platform import os import io import sys import codecs import csv import subprocess import datetime as dt sys.stdout = codecs.getwriter("utf-8")(sys.stdout) if sys.version_info[0] == 3: # for Python3 from tkinter import Frame, Label, Message, StringVar, Canvas, Button, Menu, DoubleVar import tkinter.filedialog as filedialogprompt from tkinter.ttk import Scrollbar from tkinter.constants import * import time from tkinter import ttk from tkinter import messagebox as message_box else: # for Python2 from Tkinter import Frame, Label, Message, StringVar, Canvas, Button, Menu, DoubleVar import tkFileDialog as filedialogprompt
"deprel." + args.lang, validate_langspec=True) # All relations available in DEPREL are also allowed in DEPS. # In addition, there might be relations that are only allowed in DEPS. # One of them, "ref", is universal and we currently list it directly in the code here, instead of creating a file "edeprel.ud". tagsets[DEPS] = tagsets[DEPREL] | {"ref"} | load_set( "deprel.ud", "edeprel." + args.lang, validate_enhanced=True) tagsets[FEATS] = load_set("feat_val.ud", "feat_val." + args.lang) tagsets[UPOS] = load_set("cpos.ud", None) tagsets[TOKENSWSPACE] = load_set("tokens_w_space.ud", "tokens_w_space." + args.lang) tagsets[TOKENSWSPACE] = [ re.compile(regex, re.U) for regex in tagsets[TOKENSWSPACE] ] #...turn into compiled regular expressions out = codecs.getwriter("utf-8")( sys.stdout) # hard-coding - does this ever need to be anything else? try: known_sent_ids = set() open_files = [] if args.input == []: args.input.append("-") for fname in args.input: if fname == "-": open_files.append(codecs.getreader("utf-8")(os.fdopen(0, "U"))) else: inp_raw = open(fname, mode="U") open_files.append(codecs.getreader("utf-8")(inp_raw)) for curr_fname, inp in zip(args.input, open_files): validate(inp, out, args, tagsets, known_sent_ids) except:
# You should have received a copy of the GNU General Public License # along with LilyPond. If not, see <http://www.gnu.org/licenses/>. import __main__ import codecs import gettext import glob import optparse import os import re import shutil import sys import time sys.stdin = codecs.getreader('utf8')(sys.stdin.detach()) sys.stdout = codecs.getwriter('utf8')(sys.stdout.detach()) sys.stderr = codecs.getwriter('utf8')(sys.stderr.detach()) # Lilylib globals. program_name = os.path.basename(sys.argv[0]) # Logging framework: We have the following output functions: # error # warning # progress # debug loglevels = { "NONE": 0, "ERROR": 1, "WARN": 2,
def main(prog_args=sys.argv[1:]): # in case we changed the location of the settings directory where the # config file lives, we need to parse this argument before we parse # the rest of the arguments (which can overwrite the options in the # config file) settings_parser = argparse.ArgumentParser(add_help=False) settings_parser.add_argument( '-S', '--settings', help='Path to settings, config and temp files directory ' '[Default=~/.spotify-ripper]') args, remaining_argv = settings_parser.parse_known_args(prog_args) init_util_globals(args) # load config file, overwriting any defaults defaults = { "bitrate": "320", "quality": "320", "comp": "10", "vbr": "0", "partial_check": "weak", } defaults = load_config(defaults) spotipy_envs = [ "SPOTIPY_CLIENT_ID", "SPOTIPY_CLIENT_SECRET", "SPOTIPY_REDIRECT_URI" ] for spotipy_env in spotipy_envs: if spotipy_env not in os.environ: value = defaults.get(spotipy_env.lower()) if value: os.environ[spotipy_env] = value parser = argparse.ArgumentParser( prog='spotify-ripper', description='Rips Spotify URIs to MP3s with ID3 tags and album covers', parents=[settings_parser], formatter_class=argparse.RawTextHelpFormatter, epilog='''Example usage: rip a single file: spotify-ripper -u user spotify:track:52xaypL0Kjzk0ngwv3oBPR rip entire playlist: spotify-ripper -u user spotify:user:username:playlist:4vkGNcsS8lRXj4q945NIA4 rip a list of URIs: spotify-ripper -u user list_of_uris.txt rip tracks from Spotify's charts: spotify-ripper -l spotify:charts:regional:global:weekly:latest search for tracks to rip: spotify-ripper -l -Q 160 -o "album:Rumours track:'the chain'" ''') # create group to prevent user from using both the -l and -u option is_user_set = defaults.get('user') is not None is_last_set = defaults.get('last') is True if is_user_set or is_last_set: if is_user_set and is_last_set: print("spotify-ripper: error: one of the arguments -u/--user " "-l/--last is required") sys.exit(1) else: group = parser.add_mutually_exclusive_group(required=False) else: group = parser.add_mutually_exclusive_group(required=True) encoding_group = parser.add_mutually_exclusive_group(required=False) # set defaults parser.set_defaults(**defaults) prog_version = pkg_resources.require("spotify-ripper")[0].version parser.add_argument( '-a', '--ascii', action='store_true', help='Convert the file name and the metadata tags to ASCII ' 'encoding [Default=utf-8]') encoding_group.add_argument( '--aac', action='store_true', help='Rip songs to AAC format with FreeAAC instead of MP3') encoding_group.add_argument( '--aiff', action='store_true', help='Rip songs to lossless AIFF encoding instead of MP3') encoding_group.add_argument( '--alac', action='store_true', help='Rip songs to Apple Lossless format instead of MP3') parser.add_argument( '--all-artists', action='store_true', help='Store all artists, rather than just the main artist, in the ' 'track\'s metadata tag') parser.add_argument( '--artist-album-type', help='Only load albums of specified types when passing a Spotify ' 'artist URI [Default=album,single,ep,compilation,appears_on]') parser.add_argument( '--artist-album-market', help='Only load albums with the specified ISO2 country code when ' 'passing a Spotify artist URI. You may get duplicate albums ' 'if not set. [Default=any]') parser.add_argument( '-A', '--ascii-path-only', action='store_true', help='Convert the file name (but not the metadata tags) to ASCII ' 'encoding [Default=utf-8]') parser.add_argument('-b', '--bitrate', help='CBR bitrate [Default=320]') parser.add_argument('-c', '--cbr', action='store_true', help='CBR encoding [Default=VBR]') parser.add_argument( '--comp', help='compression complexity for FLAC and Opus [Default=Max]') parser.add_argument( '--comment', help='Set comment metadata tag to all songs. Can include ' 'same tags as --format.') parser.add_argument( '--cover-file', help='Save album cover image to file name (e.g "cover.jpg") ' '[Default=embed]') parser.add_argument( '--cover-file-and-embed', metavar="COVER_FILE", help='Same as --cover-file but embeds the cover image too') parser.add_argument( '-d', '--directory', help='Base directory where ripped MP3s are saved [Default=cwd]') parser.add_argument('--fail-log', help="Logs the list of track URIs that failed to rip") encoding_group.add_argument( '--flac', action='store_true', help='Rip songs to lossless FLAC encoding instead of MP3') parser.add_argument( '-f', '--format', help='Save songs using this path and filename structure (see README)') parser.add_argument( '--format-case', choices=['upper', 'lower', 'capitalize'], help='Convert all words of the file name to upper-case, lower-case, ' 'or capitalized') parser.add_argument('--flat', action='store_true', help='Save all songs to a single directory ' '(overrides --format option)') parser.add_argument( '--flat-with-index', action='store_true', help='Similar to --flat [-f] but includes the playlist index at ' 'the start of the song file') parser.add_argument( '-g', '--genres', choices=['artist', 'album'], help='Attempt to retrieve genre information from Spotify\'s ' 'Web API [Default=skip]') parser.add_argument( '--grouping', help='Set grouping metadata tag to all songs. Can include ' 'same tags as --format.') encoding_group.add_argument( '--id3-v23', action='store_true', help='Store ID3 tags using version v2.3 [Default=v2.4]') parser.add_argument('-k', '--key', help='Path to Spotify application key file ' '[Default=Settings Directory]') group.add_argument('-u', '--user', help='Spotify username') parser.add_argument('-p', '--password', help='Spotify password [Default=ask interactively]') parser.add_argument( '--large-cover-art', action='store_true', help='Attempt to retrieve 640x640 cover art from Spotify\'s Web API ' '[Default=300x300]') group.add_argument('-l', '--last', action='store_true', help='Use last login credentials') parser.add_argument( '-L', '--log', help='Log in a log-friendly format to a file (use - to log to stdout)') encoding_group.add_argument( '--pcm', action='store_true', help='Saves a .pcm file with the raw PCM data instead of MP3') encoding_group.add_argument( '--mp4', action='store_true', help='Rip songs to MP4/M4A format with Fraunhofer FDK AAC codec ' 'instead of MP3') parser.add_argument('--normalize', action='store_true', help='Normalize volume levels of tracks') parser.add_argument('-na', '--normalized-ascii', action='store_true', help='Convert the file name to normalized ASCII with ' 'unicodedata.normalize (NFKD)') parser.add_argument('-o', '--overwrite', action='store_true', help='Overwrite existing MP3 files [Default=skip]') encoding_group.add_argument( '--opus', action='store_true', help='Rip songs to Opus encoding instead of MP3') parser.add_argument( '--partial-check', choices=['none', 'weak', 'strict'], help='Check for and overwrite partially ripped files. "weak" will ' 'err on the side of not re-ripping the file if it is unsure, ' 'whereas "strict" will re-rip the file [Default=weak]') parser.add_argument( '--play-token-resume', metavar="RESUME_AFTER", help='If the \'play token\' is lost to a different device using ' 'the same Spotify account, the script will wait a speficied ' 'amount of time before restarting. This argument takes the same ' 'values as --resume-after [Default=abort]') parser.add_argument('--playlist-m3u', action='store_true', help='create a m3u file when ripping a playlist') parser.add_argument('--playlist-wpl', action='store_true', help='create a wpl file when ripping a playlist') parser.add_argument( '--playlist-sync', action='store_true', help='Sync playlist songs (rename and remove old songs)') parser.add_argument( '--plus-pcm', action='store_true', help='Saves a .pcm file in addition to the encoded file (e.g. mp3)') parser.add_argument( '--plus-wav', action='store_true', help='Saves a .wav file in addition to the encoded file (e.g. mp3)') parser.add_argument( '-q', '--vbr', help='VBR quality setting or target bitrate for Opus [Default=0]') parser.add_argument('-Q', '--quality', choices=['160', '320', '96'], help='Spotify stream bitrate preference [Default=320]') parser.add_argument( '--remove-offline-cache', action='store_true', help='Remove libspotify\'s offline cache directory after the rip' 'is complete to save disk space') parser.add_argument( '--resume-after', help='Resumes script after a certain amount of time has passed ' 'after stopping (e.g. 1h30m). Alternatively, accepts a specific ' 'time in 24hr format to start after (e.g 03:30, 16:15). ' 'Requires --stop-after option to be set') parser.add_argument( '-R', '--replace', nargs="+", required=False, help='pattern to replace the output filename separated by "/". ' 'The following example replaces all spaces with "_" and all "-" ' 'with ".": spotify-ripper --replace " /_" "\-/." uri') parser.add_argument('-s', '--strip-colors', action='store_true', help='Strip coloring from output [Default=colors]') parser.add_argument( '--stereo-mode', choices=['j', 's', 'f', 'd', 'm', 'l', 'r'], help='Advanced stereo settings for Lame MP3 encoder only') parser.add_argument( '--stop-after', help='Stops script after a certain amount of time has passed ' '(e.g. 1h30m). Alternatively, accepts a specific time in 24hr ' 'format to stop after (e.g 03:30, 16:15)') parser.add_argument( '--timeout', type=int, help= 'Override the PySpotify timeout value in seconds (Default=10 seconds)') parser.add_argument('-V', '--version', action='version', version=prog_version) encoding_group.add_argument( '--wav', action='store_true', help='Rip songs to uncompressed WAV file instead of MP3') parser.add_argument('--windows-safe', action='store_true', help='Make filename safe for Windows file system ' '(truncate filename to 255 characters)') encoding_group.add_argument( '--vorbis', action='store_true', help='Rip songs to Ogg Vorbis encoding instead of MP3') parser.add_argument( '-r', '--remove-from-playlist', action='store_true', help='[WARNING: SPOTIFY IS NOT PROPROGATING PLAYLIST CHANGES TO ' 'THEIR SERVERS] Delete tracks from playlist after successful ' 'ripping [Default=no]') parser.add_argument( 'uri', nargs="+", help='One or more Spotify URI(s) (either URI, a file of URIs or a ' 'search query)') args = parser.parse_args(remaining_argv) init_util_globals(args) # kind of a hack to get colorama stripping to work when outputting # to a file instead of stdout. Taken from initialise.py in colorama def wrap_stream(stream, convert, strip, autoreset, wrap): if wrap: wrapper = AnsiToWin32(stream, convert=convert, strip=strip, autoreset=autoreset) if wrapper.should_wrap(): stream = wrapper.stream return stream args.has_log = args.log is not None if args.has_log: if args.log == "-": init(strip=True) else: encoding = "ascii" if args.ascii else "utf-8" log_file = codecs.open(enc_str(args.log), 'a', encoding) sys.stdout = wrap_stream(log_file, None, True, False, True) else: init(strip=True if args.strip_colors else None) if args.ascii_path_only is True: args.ascii = True # unless explicitly told not to, we are going to encode # for utf-8 by default if not args.ascii and sys.version_info < (3, 0): sys.stdout = codecs.getwriter('utf-8')(sys.stdout) # small sanity check on user option if args.user is not None and args.user == "USER": print(Fore.RED + "Please pass your username as --user " + "<YOUR_USER_NAME> instead of --user USER " + "<YOUR_USER_NAME>..." + Fore.RESET) sys.exit(1) # give warning for broken feature if args.remove_from_playlist: print(Fore.RED + "--REMOVE-FROM-PLAYLIST WARNING:") print("SPOTIFY HAS BROKEN libspotify") print("THE PLAYLIST WILL BE EMPTIED AT THE END USING THE WEB API") print( "CHECK THE GITHUB FOR INSTRUCTIONS ON AUTHENTICATING THE WEB API" + Fore.RESET) if args.wav: args.output_type = "wav" elif args.pcm: args.output_type = "pcm" elif args.flac: args.output_type = "flac" if args.comp == "10": args.comp = "8" elif args.vorbis: args.output_type = "ogg" if args.vbr == "0": args.vbr = "9" elif args.opus: args.output_type = "opus" if args.vbr == "0": args.vbr = "320" elif args.aac: args.output_type = "aac" if args.vbr == "0": args.vbr = "500" elif args.mp4: args.output_type = "m4a" if args.vbr == "0": args.vbr = "5" elif args.alac: args.output_type = "alac.m4a" else: args.output_type = "mp3" # check that encoder tool is available encoders = { "flac": ("flac", "flac"), "aiff": ("sox", "sox"), "aac": ("faac", "faac"), "ogg": ("oggenc", "vorbis-tools"), "opus": ("opusenc", "opus-tools"), "mp3": ("lame", "lame"), "m4a": ("fdkaac", "fdk-aac-encoder"), "alac.m4a": ("avconv", "libav-tools"), } if args.output_type in encoders.keys(): encoder = encoders[args.output_type][0] if which(encoder) is None: print(Fore.RED + "Missing dependency '" + encoder + "'. Please install and add to path..." + Fore.RESET) # assumes OS X or Ubuntu/Debian command_help = ("brew install " if sys.platform == "darwin" else "sudo apt-get install ") print("...try " + Fore.YELLOW + command_help + encoders[args.output_type][1] + Fore.RESET) sys.exit(1) # format string if args.flat: args.format = "{artist} - {track_name}.{ext}" elif args.flat_with_index: args.format = "{idx:3} - {artist} - {track_name}.{ext}" elif args.format is None: args.format = "{album_artist}/{album}/{artist} - {track_name}.{ext}" # print some settings print(Fore.GREEN + "Spotify Ripper - v" + prog_version + Fore.RESET) def encoding_output_str(): if args.output_type == "wav": return "WAV, Stereo 16bit 44100Hz" elif args.output_type == "pcm": return "Raw Headerless PCM, Stereo 16bit 44100Hz" else: if args.output_type == "flac": return "FLAC, Compression Level: " + args.comp elif args.output_type == "aiff": return "AIFF" elif args.output_type == "alac.m4a": return "Apple Lossless (ALAC)" elif args.output_type == "ogg": codec = "Ogg Vorbis" elif args.output_type == "opus": codec = "Opus" elif args.output_type == "mp3": codec = "MP3" elif args.output_type == "m4a": codec = "MPEG4 AAC" elif args.output_type == "aac": codec = "AAC" else: codec = "Unknown" if args.cbr: return codec + ", CBR " + args.bitrate + " kbps" else: return codec + ", VBR " + args.vbr print(Fore.YELLOW + " Encoding output:\t" + Fore.RESET + encoding_output_str()) print(Fore.YELLOW + " Spotify bitrate:\t" + Fore.RESET + args.quality + " kbps") def unicode_support_str(): if args.ascii_path_only: return "Unicode tags, ASCII file path" elif args.ascii: return "ASCII only" else: return "Yes" # check that --stop-after and --resume-after options are valid if args.stop_after is not None and \ parse_time_str(args.stop_after) is None: print(Fore.RED + "--stop-after option is not valid" + Fore.RESET) sys.exit(1) if args.resume_after is not None and \ parse_time_str(args.resume_after) is None: print(Fore.RED + "--resume-after option is not valid" + Fore.RESET) sys.exit(1) if args.play_token_resume is not None and \ parse_time_str(args.play_token_resume) is None: print(Fore.RED + "--play_token_resume option is not valid" + Fore.RESET) sys.exit(1) print(Fore.YELLOW + " Unicode support:\t" + Fore.RESET + unicode_support_str()) print(Fore.YELLOW + " Output directory:\t" + Fore.RESET + base_dir()) print(Fore.YELLOW + " Settings directory:\t" + Fore.RESET + settings_dir()) print(Fore.YELLOW + " Format String:\t" + Fore.RESET + args.format) print(Fore.YELLOW + " Overwrite files:\t" + Fore.RESET + ("Yes" if args.overwrite else "No")) # patch a bug when Python 3/MP4 if sys.version_info >= (3, 0) and \ (args.output_type == "m4a" or args.output_type == "alac.m4a"): patch_bug_in_mutagen() ripper = Ripper(args) ripper.start() # try to listen for terminal resize events # (needs to be called on main thread) if not args.has_log: ripper.progress.handle_resize() signal.signal(signal.SIGWINCH, ripper.progress.handle_resize) def hasStdinData(): return select.select([sys.stdin], [], [], 0) == ([sys.stdin], [], []) def abort(set_logged_in=False): ripper.abort_rip() if set_logged_in: ripper.ripper_continue.set() ripper.join() sys.exit(1) def skip(): if ripper.ripping.is_set(): ripper.skip.set() # check if we were passed a file name or search def check_uri_args(): if len(args.uri) == 1 and path_exists(args.uri[0]): encoding = "ascii" if args.ascii else "utf-8" args.uri = [ line.strip() for line in codecs.open(enc_str(args.uri[0]), 'r', encoding) if not line.strip().startswith("#") and len(line.strip()) > 0 ] elif len(args.uri) == 1 and not args.uri[0].startswith("spotify:"): args.uri = [list(ripper.search_query(args.uri[0]))] # login and uri_parse on main thread to catch any KeyboardInterrupt try: if not ripper.login(): print(Fore.RED + "Encountered issue while logging into " "Spotify, aborting..." + Fore.RESET) abort(set_logged_in=True) else: check_uri_args() ripper.ripper_continue.set() except (KeyboardInterrupt, Exception) as e: if not isinstance(e, KeyboardInterrupt): print(str(e)) print("\n" + Fore.RED + "Aborting..." + Fore.RESET) abort(set_logged_in=True) # wait for ripping thread to finish if not args.has_log: try: stdin_settings = termios.tcgetattr(sys.stdin) except termios.error: stdin_settings = None try: if not args.has_log and stdin_settings: tty.setcbreak(sys.stdin.fileno()) while ripper.isAlive(): schedule.run_pending() # check if the escape button was pressed if not args.has_log and hasStdinData(): c = sys.stdin.read(1) if c == '\x1b': skip() ripper.join(0.1) except (KeyboardInterrupt, Exception) as e: if not isinstance(e, KeyboardInterrupt): print(str(e)) print("\n" + Fore.RED + "Aborting..." + Fore.RESET) abort() finally: if not args.has_log and stdin_settings: termios.tcsetattr(sys.stdin, termios.TCSADRAIN, stdin_settings)
@lazy_property def cardnames() -> List[str]: return fetcher.catalog_cardnames() @lazy_property def pd_legal_cards() -> List[str]: print('Fetching http://pdmtgo.com/legal_cards.txt') return requests.get('http://pdmtgo.com/legal_cards.txt').text.split('\n') ALL_BUGS: List[Dict] = [] VERIFICATION_BY_ISSUE: Dict[int, str] = {} if sys.stdout.encoding != 'utf-8': sys.stdout = codecs.getwriter('utf-8')(sys.stdout.buffer) # type: ignore def main() -> None: if configuration.get('github_user') is None or configuration.get('github_password') is None: print('Invalid Config') sys.exit(1) verification_numbers() issues = repo.get_repo().get_issues() for issue in issues: print(issue.title) if issue.state == 'open': process_issue(issue) txt = open('bannable.txt', mode='w')
def save_jsonl_gz(data, filename): with gzip.GzipFile(filename, 'wb') as out_file: writer = codecs.getwriter('utf-8') for element in data: writer(out_file).write(json.dumps(element)) writer(out_file).write('\n')
def main(): usage = ("usage: pyinstrument [options] scriptfile [arg] ...") version_string = 'pyinstrument {v}, on Python {pyv[0]}.{pyv[1]}.{pyv[2]}'.format( v=pyinstrument.__version__, pyv=sys.version_info, ) parser = optparse.OptionParser(usage=usage, version=version_string) parser.allow_interspersed_args = False def dash_m_callback(option, opt, value, parser): parser.values.module_name = value # everything after the -m argument should be passed to that module parser.values.module_args = parser.rargs + parser.largs parser.rargs[:] = [] parser.largs[:] = [] parser.add_option('', '--load-prev', dest='load_prev', action='store', metavar='ID', help="instead of running a script, load a previous report") parser.add_option('-m', '', dest='module_name', action='callback', callback=dash_m_callback, type="str", help="run library module as a script, like 'python -m module'") parser.add_option('', '--from-path', dest='from_path', action='store_true', help="(POSIX only) instead of the working directory, look for scriptfile in the PATH environment variable") parser.add_option('-o', '--outfile', dest="outfile", action='store', help="save to <outfile>", default=None) parser.add_option('-r', '--renderer', dest='renderer', action='store', type='string', help=("how the report should be rendered. One of: 'text', 'html', 'json', or python " "import path to a renderer class"), default='text') parser.add_option('', '--html', dest="output_html", action='store_true', help=optparse.SUPPRESS_HELP, default=False) # deprecated shortcut for --renderer=html parser.add_option('-t', '--timeline', dest='timeline', action='store_true', help="render as a timeline - preserve ordering and don't condense repeated calls") parser.add_option('', '--hide', dest='hide_fnmatch', action='store', metavar='EXPR', help=("glob-style pattern matching the file paths whose frames to hide. Defaults to " "hiding non-application code")) parser.add_option('', '--hide-regex', dest='hide_regex', action='store', metavar='REGEX', help=("regex matching the file paths whose frames to hide. Useful if --hide doesn't give " "enough control.")) parser.add_option('', '--show', dest='show_fnmatch', action='store', metavar='EXPR', help=("glob-style pattern matching the file paths whose frames to " "show, regardless of --hide or --hide-regex. For example, use " "--show '*/<library>/*' to show frames within a library that " "would otherwise be hidden.")) parser.add_option('', '--show-regex', dest='show_regex', action='store', metavar='REGEX', help=("regex matching the file paths whose frames to always show. " "Useful if --show doesn't give enough control.")) parser.add_option('', '--show-all', dest='show_all', action='store_true', help="show everything", default=False) parser.add_option('', '--unicode', dest='unicode', action='store_true', help='(text renderer only) force unicode text output') parser.add_option('', '--no-unicode', dest='unicode', action='store_false', help='(text renderer only) force ascii text output') parser.add_option('', '--color', dest='color', action='store_true', help='(text renderer only) force ansi color text output') parser.add_option('', '--no-color', dest='color', action='store_false', help='(text renderer only) force no color text output') if not sys.argv[1:]: parser.print_help() sys.exit(2) options, args = parser.parse_args() if args == [] and options.module_name is None and options.load_prev is None: parser.print_help() sys.exit(2) if options.module_name is not None and options.from_path: parser.error("The options -m and --from-path are mutually exclusive.") if options.from_path and sys.platform == 'win32': parser.error('--from-path is not supported on Windows') if options.hide_fnmatch is not None and options.hide_regex is not None: parser.error('You can‘t specify both --hide and --hide-regex') if options.hide_fnmatch is not None: options.hide_regex = fnmatch.translate(options.hide_fnmatch) show_options_used = [options.show_fnmatch is not None, options.show_regex is not None, options.show_all] if show_options_used.count(True) > 1: parser.error('You can only specify one of --show, --show-regex and --show-all') if options.show_fnmatch is not None: options.show_regex = fnmatch.translate(options.show_fnmatch) if options.show_all: options.show_regex = r'.*' if options.load_prev: session = load_report(options.load_prev) else: if options.module_name is not None: if not (sys.path[0] and os.path.samefile(sys.path[0], '.')): # when called with '-m', search the cwd for that module sys.path[0] = os.path.abspath('.') sys.argv[:] = [options.module_name] + options.module_args code = "run_module(modname, run_name='__main__', alter_sys=True)" globs = { 'run_module': runpy.run_module, 'modname': options.module_name } else: sys.argv[:] = args if options.from_path: progname = shutil.which(args[0]) if progname is None: sys.exit('Error: program {} not found in PATH!'.format(args[0])) else: progname = args[0] if not os.path.exists(progname): sys.exit('Error: program {} not found!'.format(args[0])) # Make sure we overwrite the first entry of sys.path ('.') with directory of the program. sys.path[0] = os.path.dirname(progname) code = "run_path(progname, run_name='__main__')" globs = { 'run_path': runpy.run_path, 'progname': progname } profiler = Profiler() profiler.start() try: exec_(code, globs, None) except (SystemExit, KeyboardInterrupt): pass profiler.stop() session = profiler.last_session if options.output_html: options.renderer = 'html' output_to_temp_file = (options.renderer == 'html' and not options.outfile and file_is_a_tty(sys.stdout)) if options.outfile: f = codecs.open(options.outfile, 'w', 'utf-8') should_close_f_after_writing = True elif not output_to_temp_file: if PY2: f = codecs.getwriter('utf-8')(sys.stdout) else: f = sys.stdout should_close_f_after_writing = False renderer_kwargs = {'processor_options': { 'hide_regex': options.hide_regex, 'show_regex': options.show_regex, }} if options.timeline is not None: renderer_kwargs['timeline'] = options.timeline if options.renderer == 'text': unicode_override = options.unicode != None color_override = options.color != None unicode = options.unicode if unicode_override else file_supports_unicode(f) color = options.color if color_override else file_supports_color(f) renderer_kwargs.update({'unicode': unicode, 'color': color}) renderer_class = get_renderer_class(options.renderer) renderer = renderer_class(**renderer_kwargs) # remove this frame from the trace renderer.processors.append(remove_first_pyinstrument_frame_processor) if output_to_temp_file: output_filename = renderer.open_in_browser(session) print('stdout is a terminal, so saved profile output to %s' % output_filename) else: f.write(renderer.render(session)) if should_close_f_after_writing: f.close() if options.renderer == 'text': _, report_identifier = save_report(session) print('To view this report with different options, run:') print(' pyinstrument --load-prev %s [options]' % report_identifier) print('')
def extract_data(self): # create tmp_path folder if necessary if not os.path.exists( os.path.join(self.tmp_path, self.collection_name, 'data')): os.makedirs( os.path.join(self.tmp_path, self.collection_name, 'data')) if not os.path.exists( os.path.join(self.tmp_path, self.collection_name, 'rejected')): os.makedirs( os.path.join(self.tmp_path, self.collection_name, 'rejected')) # delete old tmp files if exists for old_file in glob.glob( os.path.join(self.tmp_path, self.collection_name, 'data', '*')): print "Deleting old file %s" % (old_file) os.remove(old_file) for old_file in glob.glob( os.path.join(self.tmp_path, self.collection_name, 'rejected', '*')): print "Deleting old file %s" % (old_file) os.remove(old_file) # some state variables part_num = 0 extract_file = None reject_part_num = 0 reject_file = None # start mongo client db = self.mongo_client[self.db_name] collection = db[self.collection_name] # turn query string into json if self.extract_query is not None: if 'ObjectId' in self.extract_query: # kinda hacky.. and dangerous! This is to evaluate an expression # like {"_id": {$gt:ObjectId("55401a60151a4b1a4f000001")}} from bson.objectid import ObjectId extract_query_json = eval(self.extract_query) else: extract_query_json = json.loads(self.extract_query) else: extract_query_json = None # query collection, sort by collection_sort_by_field for data in collection.find(extract_query_json).sort( self.collection_sort_by_field, 1): # track min and max id for auditing.. if self.sort_by_field_min == None: self.sort_by_field_min = data[self.collection_sort_by_field] self.sort_by_field_max = data[self.collection_sort_by_field] # open a new file if necessary if self.num_records_extracted % NUM_RECORDS_PER_PART == 0: if extract_file != None: extract_file.close() part_num += 1 extract_file_name = os.path.join(self.tmp_path, self.collection_name, 'data', str(part_num)) extract_file = open(extract_file_name, "w") extract_file_codec = codecs.getwriter("utf-8")(extract_file) self.extract_file_names.append(extract_file_name) print "Creating file %s" % extract_file_name # validate policies rejected = False for required_field_name, policy in self.required_fields.iteritems( ): if policy['required'] and jsonpath_get( data, required_field_name) is None: # -------------------------------------------------------- # document found that doesn't contain required fields. # -------------------------------------------------------- # open a new file if necessary if self.num_records_rejected % NUM_RECORDS_PER_PART == 0: if reject_file != None: reject_file.close() reject_part_num += 1 reject_file_name = os.path.join( self.tmp_path, self.collection_name, 'rejected', str(reject_part_num)) reject_file = open(reject_file_name, "w") reject_file_codec = codecs.getwriter("utf-8")( reject_file) self.reject_file_names.append(reject_file_name) print "Creating reject file %s" % reject_file_name self.num_records_rejected += 1 reject_file_codec.write("Rejected. Missing %s. Data: %s" % (required_field_name, dumps(data))) reject_file_codec.write('\n') rejected = True break if not rejected: self.num_records_extracted += 1 extract_file_codec.write(dumps(data)) extract_file_codec.write('\n') if extract_file != None: extract_file.close() if reject_file != None: reject_file.close()
# list of all the archive items that exist itemlist = 'what_cd-itemlist-2014-04-29.txt' # output file opfile = 'flaclist2.txt' # but write it zipped import gzip itemcount = 0 flaccount = 0 # Read the metadata of each item to create a list of artist album title length dir name with open(itemlist, 'r') as fp: # write directly to a gzip file, use codecs to translate to utf-8 on output with codecs.getwriter('utf-8')(gzip.open(opfile + '.gz', 'wb')) as opfp: # header opfp.write(('\t'.join( ['#artist', 'album', 'title', 'length', 'dir', 'name'])) + '\n') # body for item in fp: # Ignore any errors coming out of ia iacmd = ['ia', 'metadata', item] try: metadata = json.loads(subprocess.check_output(iacmd)) except: print "*** error running/parsing " + " ".join(iacmd) metadata = dict() # fake an empty result #print "Archive",item,"contains",metadata['files_count'],"files" if 'files' in metadata: for fileinfo in metadata['files']:
EXIT HELP LOOKUP REMOVE PARAM[ETER]S SHOW PARAM[ETER]S VERSION If a numeric value is entered, details of the node with that ID are displayed. Any other statements are executed as Cypher. """ if not PY3: _stdin = sys.stdin preferred_encoding = locale.getpreferredencoding() sys.stdin = codecs.getreader(preferred_encoding)(sys.stdin) sys.stdout = codecs.getwriter(preferred_encoding)(sys.stdout) sys.stderr = codecs.getwriter(preferred_encoding)(sys.stderr) class ResultWriter(object): def __init__(self, out=None): self.out = out or sys.stdout @classmethod def _stringify(cls, value, quoted=False): if value is None: if quoted: return "null" else: return "" elif isinstance(value, list):