def initbotscharsets(): '''set up right charset handling for specific charsets (UNOA, UNOB, UNOC, etc).''' codecs.register(codec_search_function) #tell python how to search a codec defined by bots. These are the codecs in usersys/charset botsglobal.botsreplacechar = unicode(botsglobal.ini.get('settings','botsreplacechar',u' ')) codecs.register_error('botsreplace', botscharsetreplace) #define the ' botsreplace' error handling for codecs/charsets. for key, value in botsglobal.ini.items('charsets'): #set aliases for charsets in bots.ini encodings.aliases.aliases[key]=value
def add_cp65001_codec(): if PY2: try: codecs.lookup("cp65001") except LookupError: codecs.register(lambda name: name == "cp65001" and codecs.lookup("utf-8") or None) return
def register_codec(): import codecs class Codec(codecs.Codec): def encode(self, input, errors='strict'): if not input: return b"", 0 output = encode(input) return b"".join(output), len(output) def decode(self, input, errors='strict'): if not input: return u"", 0 output = decode(input) return u"".join(output), len(output) class StreamWriter(Codec, codecs.StreamWriter): pass class StreamReader(Codec, codecs.StreamReader): pass def getregentry(encoding): if encoding != 'abicomp': return None return (Codec().encode, Codec().decode, StreamReader, StreamWriter) codecs.register(getregentry)
def register_codec(): class Codec(codecs.Codec): def decode(self, input, errors='strict'): return codecs.charmap_decode(input, errors, decoding_map) def encode(self, input, errors='strict'): return codecs.charmap_encode(input, errors, encoding_map) class StreamWriter(Codec, codecs.StreamWriter): pass class StreamReader(Codec, codecs.StreamReader): pass def getregentry(encoding): if encoding != 'latscii': return None return (Codec().encode, Codec().decode, StreamReader, StreamWriter) codecs.register(getregentry) def latscii_error(uerr): key = ord(uerr.object[uerr.start:uerr.end]) try: return unichr(decoding_map[key]), uerr.end except KeyError: handler = codecs.lookup_error('replace') return handler(uerr) codecs.register_error('replacelatscii', latscii_error)
def add_cp65001_codec(): try: codecs.lookup('cp65001') except LookupError: codecs.register( lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None) return
def find_encodings(enc=None, system=False): """Find functions for encoding translations for a specific codec. :param str enc: The codec to find translation functions for. It will be normalized by converting to lowercase, excluding everything which is not ascii, and hyphens will be converted to underscores. :param bool system: If True, find encodings based on the system's stdin encoding, otherwise assume utf-8. :raises: :exc:LookupError if the normalized codec, ``enc``, cannot be found in Python's encoding translation map. """ if not enc: enc = 'utf-8' if system: if getattr(sys.stdin, 'encoding', None) is None: enc = sys.stdin.encoding log.debug("Obtained encoding from stdin: %s" % enc) else: enc = 'ascii' ## have to have lowercase to work, see ## http://docs.python.org/dev/library/codecs.html#standard-encodings enc = enc.lower() codec_alias = encodings.normalize_encoding(enc) codecs.register(encodings.search_function) coder = codecs.lookup(codec_alias) return coder
def _mysql(self): ''' Configure db_connection for MySQL ''' logging.debug("Configured to use MySQL for a database") db_server, db_name, db_user, db_password = self._db_credentials() db_charset = "utf8mb4" codecs.register(lambda name: codecs.lookup('utf8') if name == 'utf8mb4' else None) __mysql = 'mysql://%s:%s@%s/%s?charset=%s' % ( db_user, db_password, db_server, db_name, db_charset ) __mysqlclient = 'mysql+mysqldb://%s:%s@%s/%s?charset=%s' % ( db_user, db_password, db_server, db_name, db_charset ) __pymysql = 'mysql+pymysql://%s:%s@%s/%s?charset=%s' % ( db_user, db_password, db_server, db_name, db_charset ) __mysqlconnector = 'mysql+mysqlconnector://%s:%s@%s/%s?charset=%s' % ( db_user, db_password, db_server, db_name, db_charset ) if self._test_connection(__mysql): return __mysql elif self._test_connection(__mysqlclient): return __mysqlclient elif self._test_connection(__pymysql): return __pymysql elif self._test_connection(__mysqlconnector): return __mysqlconnector else: logging.fatal("Cannot connect to database with any available driver. Verify correct username & password in rootthebox.cfg and db dependecies.") os._exit(1)
def register(): """Enable encodings of the form 'latex+x' where x describes another encoding. Unicode characters are translated to or from x when possible, otherwise expanded to latex. """ codecs.register(_registry)
def _create_mysql_tables(): codecs.register(lambda name: codecs.lookup('utf8') if name == 'utf8mb4' else None) _engine = getDatabaseEngine() BaseModel.metadata.create_all(_engine) # utf8mb4 _updateMessagePushTasksCharset(_engine) return
def main(): codecs.register( lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None ) app = QtGui.QApplication(sys.argv) ex = MainWindow() sys.exit(app.exec_())
def log(self, mes_type, message): # Remove all non-ASCII characters # message = removeNonAsciiDrop(message) codecs.register(lambda message: codecs.lookup('utf-8') if message == 'cp65001' else None) if not self.debug and mes_type == "DEBUG": return # Counter if mes_type == "ALERT": self.alerts += 1 if mes_type == "WARNING": self.warnings += 1 if mes_type == "NOTICE": self.notices += 1 if self.only_relevant: if mes_type not in ('ALERT', 'WARNING'): return # to file if not self.no_log_file: self.log_to_file(message, mes_type) # to stdout try: self.log_to_stdout(message.encode('ascii', errors='replace'), mes_type) except Exception, e: print "Cannot print certain characters to command line - see log file for full unicode encoded log line" self.log_to_stdout(removeNonAsciiDrop(message), mes_type)
def __init__(self, device, deviceStr, autodump=False, ignoreuiautomatorkilled=False): ''' Constructor @type device: Phone Object @param device: The device running the C{View server} to which this client will connect @type serialno: str @param serialno: the serial number of the device or emulator to connect to ''' self._device = device self._deviceStr = deviceStr self._autodump = autodump self._viewParser = ViewParser(self._deviceStr) self._ignoreuiautomatorkilled = ignoreuiautomatorkilled self._logger = logging.getLogger('') self._dumplogger = logging.getLogger('DUMPStats') # self._logger.debug('adb -s %s shell /system/bin/uiautomator server' % self._deviceStr) # host_exec('adb -s %s shell /system/bin/uiautomator server' % self._deviceStr) # # # time.sleep(3) ## A workaround to alias utf8 encoding to cp65001(Microsoft's utf8) try: codecs.lookup('cp65001') except: def cp65001(name): if name.lower() == 'cp65001': return codecs.lookup('utf-8') codecs.register(cp65001)
def register(): codecs.register(find_codecs) for name, (decode, encode) in codec_info.items(): if name in _codecs: continue _codecs[name] = CharmapCodec(decode, encode)
def init_db(): # 数据库包装这一层,存在大量的encode操作,使用的是database的charset,在数据库端指定了使用utf8mb4编码,那么外部模块与 # 数据库连接时,要使用相同的编码,修改数据库数据时也应该使用相同的编码。 # 而Python的codec模块不知道utf8mb4这种表述,所以需要使用别名。make python understand 'utf8mb4' as an alias for 'utf8'。 codecs.register(lambda name: codecs.lookup('utf8') if name == 'utf8mb4' else None) # 创建对应的数据库表。 engine = DBUtil.create_engine() DBUtil.__BASE_MODEL.metadata.create_all(engine)
def register(): '''Register this codec in the standard Python codec registry. Afterwards you can decode/encode strings using the codec name 'datev_ascii'.''' def is_datev(codec_name): if codec_name == 'datev_ascii': return getregentry() return None codecs.register(is_datev)
def register_codec_cp65001(): try: codecs.lookup('cp65001') except Exception: def lookup_func(name): if name.lower() == 'cp65001': return codecs.lookup('utf-8') codecs.register(lookup_func)
def RegisterCodecs(): """The example files that are distributed with the QTI specification contain a set of Chinese examples encoded using big5. However, the xml declarations on these files refer to the charset as "CN-BIG5" and this causes errors when parsing them as this is a non-standard way of refering to big5. QTI also requires use of the apple symbol font mapping for interpreting symbol-encoded maths text in questions.""" codecs.register(QTICodecSearch)
def fix_win_codec(): """Works around <http://bugs.python.org/issue6058>.""" # <http://msdn.microsoft.com/en-us/library/dd317756.aspx> try: codecs.lookup("cp65001") return False except LookupError: codecs.register(lambda name: name == "cp65001" and codecs.lookup("utf-8") or None) return True
def add_encoding_alias(encoding, alias, overwrite=False): try: codecs.lookup(alias) alias_exists = True except LookupError: alias_exists = False if overwrite or not alias_exists: codec = codecs.lookup(encoding) codecs.register(lambda x: codec if x == alias else None)
def register(self): """Register spec codec""" class StreamReader(utf_8.StreamReader): """Used by cPython to deal with a spec file""" def __init__(sr, stream, *args, **kwargs): codecs.StreamReader.__init__(sr, stream, *args, **kwargs) data = self.dealwith(sr.stream.readline) sr.stream = cStringIO.StringIO(data) def decode(text, *args): """Used by pypy and pylint to deal with a spec file""" buffered = cStringIO.StringIO(text) # Determine if we need to have imports for this string # It may be a fragment of the file has_spec = regexes['encoding_matcher'].search(buffered.readline()) no_imports = not has_spec buffered.seek(0) # Translate the text utf8 = encodings.search_function('utf8') # Assume utf8 encoding reader = utf8.streamreader(buffered) data = self.dealwith(reader.readline, no_imports=no_imports) # If nothing was changed, then we want to use the original file/line # Also have to replace indentation of original line with indentation of new line # To take into account nested describes if text and not regexes['only_whitespace'].match(text): if regexes['whitespace'].sub('', text) == regexes['whitespace'].sub('', data): bad_indentation = regexes['leading_whitespace'].search(text).groups()[0] good_indentation = regexes['leading_whitespace'].search(data).groups()[0] data = '%s%s' % (good_indentation, text[len(bad_indentation):]) # If text is empty and data isn't, then we should return text if len(text) == 0 and len(data) == 1: return unicode(text), 0 # Return translated version and it's length return unicode(data), len(data) def search_function(s): """Determine if a file is of spec encoding and return special CodecInfo if it is""" if s != 'spec': return None utf8 = encodings.search_function('utf8') # Assume utf8 encoding return codecs.CodecInfo( name='spec' , encode=utf8.encode , decode=decode , streamreader=StreamReader , streamwriter=utf8.streamwriter , incrementalencoder=utf8.incrementalencoder , incrementaldecoder=utf8.incrementaldecoder ) # Do the register codecs.register(search_function)
def add(self, encoding_name): """Adding encoding name to fake. :type encoding_name: lowercase plain string """ if not self._registered: codecs.register(self) self._registered = True if encoding_name is not None: self._enabled_encodings.add(encoding_name)
def register_cp65001(): """Register cp65001 so that PY2 knowns about it.""" import codecs try: codecs.lookup('cp65001') return False except LookupError: codecs.register( lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) return True
def register_sark_codecs(): import codecs from .encodings.hex_bytes import getregentry def sark_search_function(encoding): codec_info = getregentry() if encoding == codec_info.name: return codec_info codecs.register(sark_search_function)
def mock_mbcs_windows(): try: codecs.lookup('mbcs') except LookupError: ascii_codec = codecs.lookup('ascii') def lookup_func(name, encoding=ascii_codec): return {True: encoding}.get(name == 'mbcs') codecs.register(lookup_func)
def test_codecs_errors(self): # Error handling (encoding) self.assertRaises(UnicodeError, u"Andr\202 x".encode, "ascii") self.assertRaises(UnicodeError, u"Andr\202 x".encode, "ascii", "strict") self.assertEqual(u"Andr\202 x".encode("ascii", "ignore"), "Andr x") self.assertEqual(u"Andr\202 x".encode("ascii", "replace"), "Andr? x") # Error handling (decoding) self.assertRaises(UnicodeError, unicode, "Andr\202 x", "ascii") self.assertRaises(UnicodeError, unicode, "Andr\202 x", "ascii", "strict") self.assertEqual(unicode("Andr\202 x", "ascii", "ignore"), u"Andr x") self.assertEqual(unicode("Andr\202 x", "ascii", "replace"), u"Andr\uFFFD x") # Error handling (unknown character names) self.assertEqual("\\N{foo}xx".decode("unicode-escape", "ignore"), u"xx") # Error handling (truncated escape sequence) self.assertRaises(UnicodeError, "\\".decode, "unicode-escape") # Error handling (bad decoder return) def search_function(encoding): def decode1(input, errors="strict"): return 42 # not a tuple def encode1(input, errors="strict"): return 42 # not a tuple def encode2(input, errors="strict"): return (42, 42) # no unicode def decode2(input, errors="strict"): return (42, 42) # no unicode if encoding == "test.unicode1": return (encode1, decode1, None, None) elif encoding == "test.unicode2": return (encode2, decode2, None, None) else: return None codecs.register(search_function) self.assertRaises(TypeError, "hello".decode, "test.unicode1") self.assertRaises(TypeError, unicode, "hello", "test.unicode2") self.assertRaises(TypeError, u"hello".encode, "test.unicode1") self.assertRaises(TypeError, u"hello".encode, "test.unicode2") # executes PyUnicode_Encode() import imp self.assertRaises(ImportError, imp.find_module, "non-existing module", [u"non-existing dir"]) # Error handling (wrong arguments) self.assertRaises(TypeError, u"hello".encode, 42, 42, 42) # Error handling (PyUnicode_EncodeDecimal()) self.assertRaises(UnicodeError, int, u"\u0200")
def mbcs_work_around(): ''' work around for mbcs codec to make "bdist_wininst" work https://mail.python.org/pipermail/python-list/2012-February/620326.html ''' import codecs try: codecs.lookup('mbcs') except LookupError: ascii = codecs.lookup('ascii') codecs.register(lambda name: {True: ascii}.get(name == 'mbcs'))
def __init__(self, separate_tti, clear_uda, discard_user_data): self.separate_tti = separate_tti self.clear_uda = clear_uda self.discard_user_data = discard_user_data self.tti = [] # register all encodings that can be used for the Text Fields of the TTI blocks in STL codecs.register(iso6937().search) codecs.register(iso8859_5_stl().search) codecs.register(iso8859_6_stl().search) codecs.register(iso8859_7_stl().search) codecs.register(iso8859_8_stl().search)
def initbotscharsets(): '''set up right charset handling for specific charsets (UNOA, UNOB, UNOC, etc).''' #tell python how to search a codec defined by bots. Bots searches for this in usersys/charset codecs.register(codec_search_function) #syntax has parameters checkcharsetin or checkcharsetout. These can have value 'botsreplace' #eg: 'checkcharsetin':'botsreplace', #strict, ignore or botsreplace #in case of errors: the 'wrong' character is replaced with char as set in bots.ini. Default value in bots.ini is ' ' (space) botsglobal.botsreplacechar = unicode(botsglobal.ini.get('settings','botsreplacechar',' ')) codecs.register_error('botsreplace', botsreplacechar_handler) #need to register the handler for botsreplacechar #set aliases for the charsets in bots.ini for key, value in botsglobal.ini.items('charsets'): encodings.aliases.aliases[key] = value
def main(): #python dont know windows's cp65001 = utf-8 try: codecs.lookup('cp65001') except: def cp65001(name): if name.lower() == 'cp65001': return codecs.lookup('utf-8') codecs.register(cp65001) try: i = 0 for root, dirs, files in os.walk(FilePath): #print root filename = "" for fff in files: print os.path.join(root, fff) ff = codecs.open(os.path.join(root, fff), encoding='utf-16') for line in ff: try: i = i + 1 print i f = codecs.open(FilePath_Target +"PPT_PoliticMan_" + now.strftime("%Y%m%d%H%M") + "_" + str( os.getpid() )+ ".txt", "w+", "utf-8") page_url = line.rstrip() resp = urllib2.urlopen(page_url , timeout=10) soup = BeautifulSoup(resp) txtWrite = getpageContext(soup,page_url) f.writelines(txtWrite) resp.close() except: print "Unexpected error:", sys.exc_info()[0] print "Unexpected error:", sys.exc_info()[1] PrintException() pass ff.close() #delete file try: filename = os.path.join(root, fff) if os.path.isfile(filename): os.remove(filename) except: print "Unexpected error:", sys.exc_info()[0] print "Unexpected error:", sys.exc_info()[1] PrintException() pass except: #continue print "Unexpected error:", sys.exc_info()[0] print "Unexpected error:", sys.exc_info()[1] PrintException() pass
def __init__(self, url, depth=0, index=0): logging.basicConfig(filename='darts.log', level=logging.WARNING) assert isinstance(depth, int), "depth must be integer" self._index = _counter(index) self._depth = depth self._src_url = url self._base_url = "{0.scheme}://{0.netloc}/".format(urlsplit(url)) self._urls = self._get_internal_links(url) self._darts = set() self._clf_address = ClfAddress() for search_function in encoding.search_functions(): codecs.register(search_function)
def _real_main(argv=None): # Compatibility fixes for Windows if sys.platform == 'win32': # https://github.com/ytdl-org/youtube-dl/issues/820 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) workaround_optparse_bug9161() setproctitle('youtube-dl') parser, opts, args = parseOpts(argv) # Set user agent if opts.user_agent is not None: std_headers['User-Agent'] = opts.user_agent # Set referer if opts.referer is not None: std_headers['Referer'] = opts.referer # Custom HTTP headers if opts.headers is not None: for h in opts.headers: if ':' not in h: parser.error( 'wrong header formatting, it should be key:value, not "%s"' % h) key, value = h.split(':', 1) if opts.verbose: write_string( '[debug] Adding header from command line option %s:%s\n' % (key, value)) std_headers[key] = value # Dump user agent if opts.dump_user_agent: write_string(std_headers['User-Agent'] + '\n', out=sys.stdout) sys.exit(0) # Batch file verification batch_urls = [] if opts.batchfile is not None: try: if opts.batchfile == '-': batchfd = sys.stdin else: batchfd = io.open(expand_path(opts.batchfile), 'r', encoding='utf-8', errors='ignore') batch_urls = read_batch_urls(batchfd) if opts.verbose: write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n') except IOError: sys.exit('ERROR: batch file %s could not be read' % opts.batchfile) all_urls = batch_urls + [ url.strip() for url in args ] # batch_urls are already striped in read_batch_urls _enc = preferredencoding() all_urls = [ url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls ] if opts.list_extractors: for ie in list_extractors(opts.age_limit): write_string(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '') + '\n', out=sys.stdout) matchedUrls = [url for url in all_urls if ie.suitable(url)] for mu in matchedUrls: write_string(' ' + mu + '\n', out=sys.stdout) sys.exit(0) if opts.list_extractor_descriptions: for ie in list_extractors(opts.age_limit): if not ie._WORKING: continue desc = getattr(ie, 'IE_DESC', ie.IE_NAME) if desc is False: continue if hasattr(ie, 'SEARCH_KEY'): _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow') _COUNTS = ('', '5', '10', 'all') desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES)) write_string(desc + '\n', out=sys.stdout) sys.exit(0) if opts.ap_list_mso: table = [[mso_id, mso_info['name']] for mso_id, mso_info in MSO_INFO.items()] write_string('Supported TV Providers:\n' + render_table(['mso', 'mso name'], table) + '\n', out=sys.stdout) sys.exit(0) # Conflicting, missing and erroneous options if opts.usenetrc and (opts.username is not None or opts.password is not None): parser.error('using .netrc conflicts with giving username/password') if opts.password is not None and opts.username is None: parser.error('account username missing\n') if opts.ap_password is not None and opts.ap_username is None: parser.error('TV Provider account username missing\n') if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid): parser.error( 'using output template conflicts with using title, video ID or auto number' ) if opts.autonumber_size is not None: if opts.autonumber_size <= 0: parser.error('auto number size must be positive') if opts.autonumber_start is not None: if opts.autonumber_start < 0: parser.error('auto number start must be positive or 0') if opts.usetitle and opts.useid: parser.error('using title conflicts with using video ID') if opts.username is not None and opts.password is None: opts.password = compat_getpass( 'Type account password and press [Return]: ') if opts.ap_username is not None and opts.ap_password is None: opts.ap_password = compat_getpass( 'Type TV provider account password and press [Return]: ') if opts.ratelimit is not None: numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) if numeric_limit is None: parser.error('invalid rate limit specified') opts.ratelimit = numeric_limit if opts.min_filesize is not None: numeric_limit = FileDownloader.parse_bytes(opts.min_filesize) if numeric_limit is None: parser.error('invalid min_filesize specified') opts.min_filesize = numeric_limit if opts.max_filesize is not None: numeric_limit = FileDownloader.parse_bytes(opts.max_filesize) if numeric_limit is None: parser.error('invalid max_filesize specified') opts.max_filesize = numeric_limit if opts.sleep_interval is not None: if opts.sleep_interval < 0: parser.error('sleep interval must be positive or 0') if opts.max_sleep_interval is not None: if opts.max_sleep_interval < 0: parser.error('max sleep interval must be positive or 0') if opts.sleep_interval is None: parser.error( 'min sleep interval must be specified, use --min-sleep-interval' ) if opts.max_sleep_interval < opts.sleep_interval: parser.error( 'max sleep interval must be greater than or equal to min sleep interval' ) else: opts.max_sleep_interval = opts.sleep_interval if opts.ap_mso and opts.ap_mso not in MSO_INFO: parser.error( 'Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers' ) def parse_retries(retries): if retries in ('inf', 'infinite'): parsed_retries = float('inf') else: try: parsed_retries = int(retries) except (TypeError, ValueError): parser.error('invalid retry count specified') return parsed_retries if opts.retries is not None: opts.retries = parse_retries(opts.retries) if opts.fragment_retries is not None: opts.fragment_retries = parse_retries(opts.fragment_retries) if opts.buffersize is not None: numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize) if numeric_buffersize is None: parser.error('invalid buffer size specified') opts.buffersize = numeric_buffersize if opts.http_chunk_size is not None: numeric_chunksize = FileDownloader.parse_bytes(opts.http_chunk_size) if not numeric_chunksize: parser.error('invalid http chunk size specified') opts.http_chunk_size = numeric_chunksize if opts.playliststart <= 0: raise ValueError('Playlist start must be positive') if opts.playlistend not in ( -1, None) and opts.playlistend < opts.playliststart: raise ValueError('Playlist end must be greater than playlist start') if opts.extractaudio: if opts.audioformat not in [ 'best', 'aac', 'flac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav' ]: parser.error('invalid audio format specified') if opts.audioquality: opts.audioquality = opts.audioquality.strip('k').strip('K') if not opts.audioquality.isdigit(): parser.error('invalid audio quality specified') if opts.recodevideo is not None: if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'avi']: parser.error('invalid video recode format specified') if opts.convertsubtitles is not None: if opts.convertsubtitles not in ['srt', 'vtt', 'ass', 'lrc']: parser.error('invalid subtitle format specified') if opts.date is not None: date = DateRange.day(opts.date) else: date = DateRange(opts.dateafter, opts.datebefore) # Do not download videos when there are audio-only formats if opts.extractaudio and not opts.keepvideo and opts.format is None: opts.format = 'bestaudio/best' # --all-sub automatically sets --write-sub if --write-auto-sub is not given # this was the old behaviour if only --all-sub was given. if opts.allsubtitles and not opts.writeautomaticsub: opts.writesubtitles = True outtmpl = ((opts.outtmpl is not None and opts.outtmpl) or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s') or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s') or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s') or (opts.usetitle and '%(title)s-%(id)s.%(ext)s') or (opts.useid and '%(id)s.%(ext)s') or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s') or DEFAULT_OUTTMPL) if not os.path.splitext(outtmpl)[1] and opts.extractaudio: parser.error('Cannot download a video and extract audio into the same' ' file! Use "{0}.%(ext)s" instead of "{0}" as the output' ' template'.format(outtmpl)) any_getting = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json any_printing = opts.print_json download_archive_fn = expand_path( opts.download_archive ) if opts.download_archive is not None else opts.download_archive # PostProcessors postprocessors = [] if opts.metafromtitle: postprocessors.append({ 'key': 'MetadataFromTitle', 'titleformat': opts.metafromtitle }) if opts.extractaudio: postprocessors.append({ 'key': 'FFmpegExtractAudio', 'preferredcodec': opts.audioformat, 'preferredquality': opts.audioquality, 'nopostoverwrites': opts.nopostoverwrites, }) if opts.recodevideo: postprocessors.append({ 'key': 'FFmpegVideoConvertor', 'preferedformat': opts.recodevideo, }) # FFmpegMetadataPP should be run after FFmpegVideoConvertorPP and # FFmpegExtractAudioPP as containers before conversion may not support # metadata (3gp, webm, etc.) # And this post-processor should be placed before other metadata # manipulating post-processors (FFmpegEmbedSubtitle) to prevent loss of # extra metadata. By default ffmpeg preserves metadata applicable for both # source and target containers. From this point the container won't change, # so metadata can be added here. if opts.addmetadata: postprocessors.append({'key': 'FFmpegMetadata'}) if opts.convertsubtitles: postprocessors.append({ 'key': 'FFmpegSubtitlesConvertor', 'format': opts.convertsubtitles, }) if opts.embedsubtitles: postprocessors.append({ 'key': 'FFmpegEmbedSubtitle', }) if opts.embedthumbnail: already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails postprocessors.append({ 'key': 'EmbedThumbnail', 'already_have_thumbnail': already_have_thumbnail }) if not already_have_thumbnail: opts.writethumbnail = True # XAttrMetadataPP should be run after post-processors that may change file # contents if opts.xattrs: postprocessors.append({'key': 'XAttrMetadata'}) # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way. # So if the user is able to remove the file before your postprocessor runs it might cause a few problems. if opts.exec_cmd: postprocessors.append({ 'key': 'ExecAfterDownload', 'exec_cmd': opts.exec_cmd, }) external_downloader_args = None if opts.external_downloader_args: external_downloader_args = compat_shlex_split( opts.external_downloader_args) postprocessor_args = None if opts.postprocessor_args: postprocessor_args = compat_shlex_split(opts.postprocessor_args) match_filter = (None if opts.match_filter is None else match_filter_func( opts.match_filter)) ydl_opts = { 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, 'twofactor': opts.twofactor, 'videopassword': opts.videopassword, 'ap_mso': opts.ap_mso, 'ap_username': opts.ap_username, 'ap_password': opts.ap_password, 'quiet': (opts.quiet or any_getting or any_printing), 'no_warnings': opts.no_warnings, 'forceurl': opts.geturl, 'forcetitle': opts.gettitle, 'forceid': opts.getid, 'forcethumbnail': opts.getthumbnail, 'forcedescription': opts.getdescription, 'forceduration': opts.getduration, 'forcefilename': opts.getfilename, 'forceformat': opts.getformat, 'forcejson': opts.dumpjson or opts.print_json, 'dump_single_json': opts.dump_single_json, 'simulate': opts.simulate or any_getting, 'skip_download': opts.skip_download, 'format': opts.format, 'listformats': opts.listformats, 'outtmpl': outtmpl, 'autonumber_size': opts.autonumber_size, 'autonumber_start': opts.autonumber_start, 'restrictfilenames': opts.restrictfilenames, 'ignoreerrors': opts.ignoreerrors, 'force_generic_extractor': opts.force_generic_extractor, 'ratelimit': opts.ratelimit, 'nooverwrites': opts.nooverwrites, 'retries': opts.retries, 'fragment_retries': opts.fragment_retries, 'skip_unavailable_fragments': opts.skip_unavailable_fragments, 'keep_fragments': opts.keep_fragments, 'buffersize': opts.buffersize, 'noresizebuffer': opts.noresizebuffer, 'http_chunk_size': opts.http_chunk_size, 'continuedl': opts.continue_dl, 'noprogress': opts.noprogress, 'progress_with_newline': opts.progress_with_newline, 'playliststart': opts.playliststart, 'playlistend': opts.playlistend, 'playlistreverse': opts.playlist_reverse, 'playlistrandom': opts.playlist_random, 'noplaylist': opts.noplaylist, 'logtostderr': opts.outtmpl == '-', 'consoletitle': opts.consoletitle, 'nopart': opts.nopart, 'updatetime': opts.updatetime, 'writedescription': opts.writedescription, 'writeannotations': opts.writeannotations, 'writeinfojson': opts.writeinfojson, 'writethumbnail': opts.writethumbnail, 'write_all_thumbnails': opts.write_all_thumbnails, 'writesubtitles': opts.writesubtitles, 'writeautomaticsub': opts.writeautomaticsub, 'allsubtitles': opts.allsubtitles, 'listsubtitles': opts.listsubtitles, 'subtitlesformat': opts.subtitlesformat, 'subtitleslangs': opts.subtitleslangs, 'matchtitle': decodeOption(opts.matchtitle), 'rejecttitle': decodeOption(opts.rejecttitle), 'max_downloads': opts.max_downloads, 'prefer_free_formats': opts.prefer_free_formats, 'verbose': opts.verbose, 'dump_intermediate_pages': opts.dump_intermediate_pages, 'write_pages': opts.write_pages, 'test': opts.test, 'keepvideo': opts.keepvideo, 'min_filesize': opts.min_filesize, 'max_filesize': opts.max_filesize, 'min_views': opts.min_views, 'max_views': opts.max_views, 'daterange': date, 'cachedir': opts.cachedir, 'youtube_print_sig_code': opts.youtube_print_sig_code, 'age_limit': opts.age_limit, 'download_archive': download_archive_fn, 'cookiefile': opts.cookiefile, 'nocheckcertificate': opts.no_check_certificate, 'prefer_insecure': opts.prefer_insecure, 'proxy': opts.proxy, 'socket_timeout': opts.socket_timeout, 'bidi_workaround': opts.bidi_workaround, 'debug_printtraffic': opts.debug_printtraffic, 'prefer_ffmpeg': opts.prefer_ffmpeg, 'include_ads': opts.include_ads, 'default_search': opts.default_search, 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest, 'encoding': opts.encoding, 'extract_flat': opts.extract_flat, 'mark_watched': opts.mark_watched, 'merge_output_format': opts.merge_output_format, 'postprocessors': postprocessors, 'fixup': opts.fixup, 'source_address': opts.source_address, 'call_home': opts.call_home, 'sleep_interval': opts.sleep_interval, 'max_sleep_interval': opts.max_sleep_interval, 'external_downloader': opts.external_downloader, 'list_thumbnails': opts.list_thumbnails, 'playlist_items': opts.playlist_items, 'xattr_set_filesize': opts.xattr_set_filesize, 'match_filter': match_filter, 'no_color': opts.no_color, 'ffmpeg_location': opts.ffmpeg_location, 'hls_prefer_native': opts.hls_prefer_native, 'hls_use_mpegts': opts.hls_use_mpegts, 'external_downloader_args': external_downloader_args, 'postprocessor_args': postprocessor_args, 'cn_verification_proxy': opts.cn_verification_proxy, 'geo_verification_proxy': opts.geo_verification_proxy, 'config_location': opts.config_location, 'geo_bypass': opts.geo_bypass, 'geo_bypass_country': opts.geo_bypass_country, 'geo_bypass_ip_block': opts.geo_bypass_ip_block, # just for deprecation check 'autonumber': opts.autonumber if opts.autonumber is True else None, 'usetitle': opts.usetitle if opts.usetitle is True else None, } with YoutubeDL(ydl_opts) as ydl: # Update version if opts.update_self: update_self(ydl.to_screen, opts.verbose, ydl._opener) # Remove cache dir if opts.rm_cachedir: ydl.cache.remove() # Maybe do nothing if (len(all_urls) < 1) and (opts.load_info_filename is None): if opts.update_self or opts.rm_cachedir: sys.exit() ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv) parser.error( 'You must provide at least one URL.\n' 'Type youtube-dl --help to see a list of all options.') try: if opts.load_info_filename is not None: retcode = ydl.download_with_info_file( expand_path(opts.load_info_filename)) else: retcode = ydl.download(all_urls) except MaxDownloadsReached: ydl.to_screen('--max-download limit reached, aborting.') retcode = 101
def get_version(): """ extract version from lib/datax-core-<version>.jar package """ core_jar = glob(os.path.join(DATAX_HOME, "lib", "datax-core-*.jar")) if not core_jar: return "" else: return os.path.basename(core_jar[0])[11:-4] DATAX_HOME = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) DATAX_VERSION = 'DataX ' + get_version() if isWindows(): codecs.register( lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None) CLASS_PATH = "{1}{0}lib{0}*".format(os.sep, DATAX_HOME) else: CLASS_PATH = "{1}{0}lib{0}*:.".format(os.sep, DATAX_HOME) LOGBACK_FILE = "{1}{0}conf{0}logback.xml".format(os.sep, DATAX_HOME) DEFAULT_JVM = "-Xms64m -Xmx2g -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={}".format( DATAX_HOME) DEFAULT_PROPERTY_CONF = "-Dfile.encoding=UTF-8 -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \ -Djava.security.egd=file:///dev/urandom -Ddatax.home=%s -Dlogback.configurationFile=%s " % \ (DATAX_HOME, LOGBACK_FILE) ENGINE_COMMAND = "java -server ${jvm} %s -classpath %s ${params} com.wgzhao.datax.core.Engine \ -mode ${mode} -jobid ${jobid} -job ${job}" % \ (DEFAULT_PROPERTY_CONF, "/etc/hbase/conf:" + CLASS_PATH) REMOTE_DEBUG_CONFIG = "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=9999" RET_STATE = {
# Log errors from the Opbeat module to the console (recommended) 'opbeat.errors': { 'level': 'ERROR', 'handlers': ['console'], 'propagate': False, }, }, } # FILE_UPLOAD_HANDLERS = ("django.core.files.uploadhandler.TemporaryFileUploadHandler",) WSGI_APPLICATION = 'config.wsgi.application' # Database # https://docs.djangoproject.com/en/1.8/ref/settings/#databases codecs.register(lambda name: codecs.lookup('utf8') if name == 'utf8mb4' else None) if not PRODUCTION: DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 'ATOMIC_REQUESTS': True } } else: DATABASES = { 'default': { 'NAME': os.environ.get('MYSQL_INSTANCE_NAME', 'xxx'), 'ENGINE': 'django.db.backends.mysql', 'USER': os.environ.get("MYSQL_USERNAME", 'xxx'), 'PASSWORD': os.environ.get("MYSQL_PASSWORD", 'xxx'),
def _register_utf8_codec(): codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
def _real_main(argv=None): # Compatibility fixes for Windows if sys.platform == 'win32': # https://github.com/rg3/youtube-dl/issues/820 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) workaround_optparse_bug9161() setproctitle('ananse') parser, opts, args = parseOpts(argv) # Set user agent if opts.user_agent is not None: std_headers['User-Agent'] = opts.user_agent # Set referer if opts.referer is not None: std_headers['Referer'] = opts.referer # Custom HTTP headers if opts.headers is not None: for h in opts.headers: if h.find(':', 1) < 0: parser.error( 'wrong header formatting, it should be key:value, not "%s"' % h) key, value = h.split(':', 2) if opts.verbose: write_string( '[debug] Adding header from command line option %s:%s\n' % (key, value)) std_headers[key] = value # Dump user agent if opts.dump_user_agent: compat_print(std_headers['User-Agent']) sys.exit(0) # Batch file verification batch_urls = [] if opts.batchfile is not None: try: if opts.batchfile == '-': batchfd = sys.stdin else: batchfd = io.open(opts.batchfile, 'r', encoding='utf-8', errors='ignore') batch_urls = read_batch_urls(batchfd) if opts.verbose: write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n') except IOError: sys.exit('ERROR: batch file could not be read') all_urls = batch_urls + args all_urls = [url.strip() for url in all_urls] _enc = preferredencoding() all_urls = [ url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls ] extractors = gen_extractors() if opts.list_extractors: for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()): compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) matchedUrls = [url for url in all_urls if ie.suitable(url)] for mu in matchedUrls: compat_print(' ' + mu) sys.exit(0) if opts.list_extractor_descriptions: for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()): if not ie._WORKING: continue desc = getattr(ie, 'IE_DESC', ie.IE_NAME) if desc is False: continue if hasattr(ie, 'SEARCH_KEY'): _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny') _COUNTS = ('', '5', '10', 'all') desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES)) compat_print(desc) sys.exit(0) # Conflicting, missing and erroneous options if opts.usenetrc and (opts.username is not None or opts.password is not None): parser.error('using .netrc conflicts with giving username/password') if opts.password is not None and opts.username is None: parser.error('account username missing\n') if opts.outtmpl is not None and (opts.usetitle or opts.autonumber or opts.useid): parser.error( 'using output template conflicts with using title, video ID or auto number' ) if opts.usetitle and opts.useid: parser.error('using title conflicts with using video ID') if opts.username is not None and opts.password is None: opts.password = compat_getpass( 'Type account password and press [Return]: ') if opts.ratelimit is not None: numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) if numeric_limit is None: parser.error('invalid rate limit specified') opts.ratelimit = numeric_limit if opts.min_filesize is not None: numeric_limit = FileDownloader.parse_bytes(opts.min_filesize) if numeric_limit is None: parser.error('invalid min_filesize specified') opts.min_filesize = numeric_limit if opts.max_filesize is not None: numeric_limit = FileDownloader.parse_bytes(opts.max_filesize) if numeric_limit is None: parser.error('invalid max_filesize specified') opts.max_filesize = numeric_limit if opts.retries is not None: try: opts.retries = int(opts.retries) except (TypeError, ValueError): parser.error('invalid retry count specified') if opts.buffersize is not None: numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize) if numeric_buffersize is None: parser.error('invalid buffer size specified') opts.buffersize = numeric_buffersize if opts.playliststart <= 0: raise ValueError('Playlist start must be positive') if opts.playlistend not in ( -1, None) and opts.playlistend < opts.playliststart: raise ValueError('Playlist end must be greater than playlist start') if opts.extractaudio: if opts.audioformat not in [ 'best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav' ]: parser.error('invalid audio format specified') if opts.audioquality: opts.audioquality = opts.audioquality.strip('k').strip('K') if not opts.audioquality.isdigit(): parser.error('invalid audio quality specified') if opts.recodevideo is not None: if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']: parser.error('invalid video recode format specified') if opts.date is not None: date = DateRange.day(opts.date) else: date = DateRange(opts.dateafter, opts.datebefore) # Do not download videos when there are audio-only formats if opts.extractaudio and not opts.keepvideo and opts.format is None: opts.format = 'bestaudio/best' # --all-sub automatically sets --write-sub if --write-auto-sub is not given # this was the old behaviour if only --all-sub was given. if opts.allsubtitles and not opts.writeautomaticsub: opts.writesubtitles = True if sys.version_info < (3, ): # In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems) if opts.outtmpl is not None: opts.outtmpl = opts.outtmpl.decode(preferredencoding()) outtmpl = ((opts.outtmpl is not None and opts.outtmpl) or (opts.format == '-1' and opts.usetitle and '%(title)s-%(id)s-%(format)s.%(ext)s') or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s') or (opts.usetitle and opts.autonumber and '%(autonumber)s-%(title)s-%(id)s.%(ext)s') or (opts.usetitle and '%(title)s-%(id)s.%(ext)s') or (opts.useid and '%(id)s.%(ext)s') or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s') or DEFAULT_OUTTMPL) if not os.path.splitext(outtmpl)[1] and opts.extractaudio: parser.error('Cannot download a video and extract audio into the same' ' file! Use "{0}.%(ext)s" instead of "{0}" as the output' ' template'.format(outtmpl)) any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json download_archive_fn = compat_expanduser( opts.download_archive ) if opts.download_archive is not None else opts.download_archive # PostProcessors postprocessors = [] # Add the metadata pp first, the other pps will copy it if opts.addmetadata: postprocessors.append({'key': 'FFmpegMetadata'}) if opts.extractaudio: postprocessors.append({ 'key': 'FFmpegExtractAudio', 'preferredcodec': opts.audioformat, 'preferredquality': opts.audioquality, 'nopostoverwrites': opts.nopostoverwrites, }) if opts.recodevideo: postprocessors.append({ 'key': 'FFmpegVideoConvertor', 'preferedformat': opts.recodevideo, }) if opts.embedsubtitles: postprocessors.append({ 'key': 'FFmpegEmbedSubtitle', 'subtitlesformat': opts.subtitlesformat, }) if opts.xattrs: postprocessors.append({'key': 'XAttrMetadata'}) if opts.embedthumbnail: if not opts.addmetadata: postprocessors.append({'key': 'FFmpegAudioFix'}) postprocessors.append({'key': 'AtomicParsley'}) # Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way. # So if the user is able to remove the file before your postprocessor runs it might cause a few problems. if opts.exec_cmd: postprocessors.append({ 'key': 'ExecAfterDownload', 'verboseOutput': opts.verbose, 'exec_cmd': opts.exec_cmd, }) adl_opts = { 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, 'twofactor': opts.twofactor, 'videopassword': opts.videopassword, 'quiet': (opts.quiet or any_printing), 'no_warnings': opts.no_warnings, 'forceurl': opts.geturl, 'forcetitle': opts.gettitle, 'forceid': opts.getid, 'forcethumbnail': opts.getthumbnail, 'forcedescription': opts.getdescription, 'forceduration': opts.getduration, 'forcefilename': opts.getfilename, 'forceformat': opts.getformat, 'forcejson': opts.dumpjson, 'dump_single_json': opts.dump_single_json, 'simulate': opts.simulate or any_printing, 'skip_download': opts.skip_download, 'format': opts.format, 'format_limit': opts.format_limit, 'listformats': opts.listformats, 'outtmpl': outtmpl, 'autonumber_size': opts.autonumber_size, 'restrictfilenames': opts.restrictfilenames, 'ignoreerrors': opts.ignoreerrors, 'ratelimit': opts.ratelimit, 'nooverwrites': opts.nooverwrites, 'retries': opts.retries, 'buffersize': opts.buffersize, 'noresizebuffer': opts.noresizebuffer, 'continuedl': opts.continue_dl, 'noprogress': opts.noprogress, 'progress_with_newline': opts.progress_with_newline, 'playliststart': opts.playliststart, 'playlistend': opts.playlistend, 'playlistreverse': opts.playlist_reverse, 'noplaylist': opts.noplaylist, 'logtostderr': opts.outtmpl == '-', 'consoletitle': opts.consoletitle, 'nopart': opts.nopart, 'updatetime': opts.updatetime, 'writedescription': opts.writedescription, 'writeannotations': opts.writeannotations, 'writeinfojson': opts.writeinfojson, 'writethumbnail': opts.writethumbnail, 'writesubtitles': opts.writesubtitles, 'writeautomaticsub': opts.writeautomaticsub, 'allsubtitles': opts.allsubtitles, 'listsubtitles': opts.listsubtitles, 'subtitlesformat': opts.subtitlesformat, 'subtitleslangs': opts.subtitleslangs, 'matchtitle': decodeOption(opts.matchtitle), 'rejecttitle': decodeOption(opts.rejecttitle), 'max_downloads': opts.max_downloads, 'prefer_free_formats': opts.prefer_free_formats, 'verbose': opts.verbose, 'dump_intermediate_pages': opts.dump_intermediate_pages, 'write_pages': opts.write_pages, 'test': opts.test, 'keepvideo': opts.keepvideo, 'min_filesize': opts.min_filesize, 'max_filesize': opts.max_filesize, 'min_views': opts.min_views, 'max_views': opts.max_views, 'daterange': date, 'cachedir': opts.cachedir, 'youtube_print_sig_code': opts.youtube_print_sig_code, 'age_limit': opts.age_limit, 'download_archive': download_archive_fn, 'cookiefile': opts.cookiefile, 'nocheckcertificate': opts.no_check_certificate, 'prefer_insecure': opts.prefer_insecure, 'proxy': opts.proxy, 'socket_timeout': opts.socket_timeout, 'bidi_workaround': opts.bidi_workaround, 'debug_printtraffic': opts.debug_printtraffic, 'prefer_ffmpeg': opts.prefer_ffmpeg, 'include_ads': opts.include_ads, 'default_search': opts.default_search, 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest, 'encoding': opts.encoding, 'exec_cmd': opts.exec_cmd, 'extract_flat': opts.extract_flat, 'postprocessors': postprocessors, } with AnanseDl(adl_opts) as adl: # Update version if opts.update_self: update_self(adl.to_screen, opts.verbose) # Remove cache dir if opts.rm_cachedir: adl.cache.remove() # Maybe do nothing if (len(all_urls) < 1) and (opts.load_info_filename is None): if opts.update_self or opts.rm_cachedir: sys.exit() adl.warn_if_short_id(sys.argv[1:] if argv is None else argv) parser.error('you must provide at least one URL') try: if opts.load_info_filename is not None: retcode = adl.download_with_info_file(opts.load_info_filename) else: retcode = adl.download(all_urls) except MaxDownloadsReached: adl.to_screen('--max-download limit reached, aborting.') retcode = 101 sys.exit(retcode)
def register_pdf_encodings(): codecs.register(winansi.WinAnsiCodec.search) codecs.register(standard.StandardCodec.search) codecs.register(macroman.MacRomanCodec.search) codecs.register(pdfdoc.PdfDocCodec.search)
decodes.append(c) else: results.append(c) results.append(process(decodes)) return ''.join(results), len(s) def encode_mb64(s): s_utf7 = s.encode('utf-7') return s_utf7[1:-1].replace('/', ',') def decode_mb64(s): s_utf7 = '+' + s.replace(',', '/') + '-' return s_utf7.decode('utf-7') codecPack = encode, decode, StreamReader, StreamWriter try: codecPack = codecs.CodecInfo(*codecPack, name=CODEC_NAME) except AttributeError: # pragma: no cover pass def utf_7_imap4(codecName): if codecName == CODEC_NAME: return codecPack codecs.register(utf_7_imap4)
def main(argv=None): if argv is None: argv = sys.argv[1:] formats = [ ("auto", "(default) detect file type automatically"), ("pe", "Windows PE file"), ("sc32", "32-bit shellcode"), ("sc64", "64-bit shellcode"), ("freeze", "features previously frozen by capa"), ] format_help = ", ".join(["%s: %s" % (f[0], f[1]) for f in formats]) desc = "The FLARE team's open-source tool to identify capabilities in executable files." epilog = textwrap.dedent(""" By default, capa uses a default set of embedded rules. You can see the rule set here: https://github.com/fireeye/capa-rules To provide your own rule set, use the `-r` flag: capa --rules /path/to/rules suspicious.exe capa -r /path/to/rules suspicious.exe examples: identify capabilities in a binary capa suspicious.exe identify capabilities in 32-bit shellcode, see `-f` for all supported formats capa -f sc32 shellcode.bin report match locations capa -v suspicious.exe report all feature match details capa -vv suspicious.exe filter rules by meta fields, e.g. rule name or namespace capa -t "create TCP socket" suspicious.exe """) parser = argparse.ArgumentParser( description=desc, epilog=epilog, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("sample", type=str, help="path to sample to analyze") parser.add_argument("--version", action="version", version="%(prog)s {:s}".format( capa.version.__version__)) parser.add_argument( "-r", "--rules", type=str, default=RULES_PATH_DEFAULT_STRING, help="path to rule file or directory, use embedded rules by default", ) parser.add_argument("-f", "--format", choices=[f[0] for f in formats], default="auto", help="select sample format, %s" % format_help) parser.add_argument("-t", "--tag", type=str, help="filter on rule meta field values") parser.add_argument("-j", "--json", action="store_true", help="emit JSON instead of text") parser.add_argument( "-v", "--verbose", action="store_true", help="enable verbose result document (no effect with --json)") parser.add_argument( "-vv", "--vverbose", action="store_true", help="enable very verbose result document (no effect with --json)") parser.add_argument("-d", "--debug", action="store_true", help="enable debugging output on STDERR") parser.add_argument("-q", "--quiet", action="store_true", help="disable all output but errors") parser.add_argument( "--color", type=str, choices=("auto", "always", "never"), default="auto", help= "enable ANSI color codes in results, default: only during interactive session", ) args = parser.parse_args(args=argv) if args.quiet: logging.basicConfig(level=logging.WARNING) logging.getLogger().setLevel(logging.WARNING) elif args.debug: logging.basicConfig(level=logging.DEBUG) logging.getLogger().setLevel(logging.DEBUG) else: logging.basicConfig(level=logging.INFO) logging.getLogger().setLevel(logging.INFO) # disable vivisect-related logging, it's verbose and not relevant for capa users set_vivisect_log_level(logging.CRITICAL) try: taste = get_file_taste(args.sample) except IOError as e: logger.error("%s", str(e)) return -1 # py2 doesn't know about cp65001, which is a variant of utf-8 on windows # tqdm bails when trying to render the progress bar in this setup. # because cp65001 is utf-8, we just map that codepage to the utf-8 codec. # see #380 and: https://stackoverflow.com/a/3259271/87207 import codecs codecs.register(lambda name: codecs.lookup("utf-8") if name == "cp65001" else None) if args.rules == RULES_PATH_DEFAULT_STRING: logger.debug("-" * 80) logger.debug(" Using default embedded rules.") logger.debug( " To provide your own rules, use the form `capa.exe -r ./path/to/rules/ /path/to/mal.exe`." ) logger.debug(" You can see the current default rule set here:") logger.debug(" https://github.com/fireeye/capa-rules") logger.debug("-" * 80) if hasattr(sys, "frozen") and hasattr(sys, "_MEIPASS"): logger.debug("detected running under PyInstaller") rules_path = os.path.join(sys._MEIPASS, "rules") logger.debug("default rule path (PyInstaller method): %s", rules_path) else: logger.debug("detected running from source") rules_path = os.path.join(os.path.dirname(__file__), "..", "rules") logger.debug("default rule path (source method): %s", rules_path) if not os.path.exists(rules_path): # when a users installs capa via pip, # this pulls down just the source code - not the default rules. # i'm not sure the default rules should even be written to the library directory, # so in this case, we require the user to use -r to specify the rule directory. logger.error( "default embedded rules not found! (maybe you installed capa as a library?)" ) logger.error("provide your own rule set via the `-r` option.") return -1 else: rules_path = args.rules logger.debug("using rules path: %s", rules_path) try: rules = get_rules(rules_path, disable_progress=args.quiet) rules = capa.rules.RuleSet(rules) logger.debug("successfully loaded %s rules", len(rules)) if args.tag: rules = rules.filter_rules_by_meta(args.tag) logger.debug("selected %s rules", len(rules)) for i, r in enumerate(rules.rules, 1): # TODO don't display subscope rules? logger.debug(" %d. %s", i, r) except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e: logger.error("%s", str(e)) return -1 if (args.format == "freeze") or (args.format == "auto" and capa.features.freeze.is_freeze(taste)): format = "freeze" with open(args.sample, "rb") as f: extractor = capa.features.freeze.load(f.read()) else: format = args.format try: extractor = get_extractor(args.sample, args.format, disable_progress=args.quiet) except UnsupportedFormatError: logger.error("-" * 80) logger.error(" Input file does not appear to be a PE file.") logger.error(" ") logger.error( " capa currently only supports analyzing PE files (or shellcode, when using --format sc32|sc64)." ) logger.error( " If you don't know the input file type, you can try using the `file` utility to guess it." ) logger.error("-" * 80) return -1 except UnsupportedRuntimeError: logger.error("-" * 80) logger.error(" Unsupported runtime or Python interpreter.") logger.error(" ") logger.error( " capa supports running under Python 2.7 using Vivisect for binary analysis." ) logger.error( " It can also run within IDA Pro, using either Python 2.7 or 3.5+." ) logger.error(" ") logger.error( " If you're seeing this message on the command line, please ensure you're running Python 2.7." ) logger.error("-" * 80) return -1 meta = collect_metadata(argv, args.sample, args.rules, format, extractor) capabilities, counts = find_capabilities(rules, extractor, disable_progress=args.quiet) meta["analysis"].update(counts) if has_file_limitation(rules, capabilities): # bail if capa encountered file limitation e.g. a packed binary # do show the output in verbose mode, though. if not (args.verbose or args.vverbose or args.json): return -1 if args.color == "always": colorama.init(strip=False) elif args.color == "auto": # colorama will detect: # - when on Windows console, and fixup coloring, and # - when not an interactive session, and disable coloring # renderers should use coloring and assume it will be stripped out if necessary. colorama.init() elif args.color == "never": colorama.init(strip=True) else: raise RuntimeError("unexpected --color value: " + args.color) if args.json: print(capa.render.render_json(meta, rules, capabilities)) elif args.vverbose: print(capa.render.render_vverbose(meta, rules, capabilities)) elif args.verbose: print(capa.render.render_verbose(meta, rules, capabilities)) else: print(capa.render.render_default(meta, rules, capabilities)) colorama.deinit() logger.debug("done.") return 0
import gnureadline as readline except ImportError: try: import gnureadline as readline except ImportError: readline_error = True pass # Set default encoding _reload_module(sys) #sys.setdefaultencoding(settings.DEFAULT_ENCODING) if settings.IS_WINDOWS: import codecs # Reference: https://github.com/nodejs/node/issues/12786#issuecomment-298652440 codecs.register(lambda name: codecs.lookup("utf-8") if name == "cp65001" else None) # Use Colorama to make Termcolor work on Windows too :) init() """ Define HTTP User-Agent header. """ def user_agent_header(): # Check if defined "--random-agent" option. if menu.options.random_agent: if (menu.options.agent != settings.DEFAULT_USER_AGENT) or menu.options.mobile: err_msg = "The option '--random-agent' is incompatible with option '--user-agent' or switch '--mobile'." print(settings.print_critical_msg(err_msg)) raise SystemExit() else: if settings.VERBOSITY_LEVEL >= 1:
# (C)2002-2015 Chris Liechti <*****@*****.**> # # SPDX-License-Identifier: BSD-3-Clause import codecs import os import sys import threading import serial from serial.tools.list_ports import comports from serial.tools import hexlify_codec # pylint: disable=wrong-import-order,wrong-import-position codecs.register(lambda c: hexlify_codec.getregentry() if c == 'hexlify' else None) try: raw_input except NameError: # pylint: disable=redefined-builtin,invalid-name raw_input = input # in python3 it's "raw" unichr = chr def key_description(character): """generate a readable description for a key""" ascii_code = ord(character) if ascii_code < 32: return 'Ctrl+{:c}'.format(ord('@') + ascii_code) else:
def dcp_block(block_data, verbose): bauds = ['Undefined', '100', '300', '1200'] platforms = ['CS1', 'CS2'] modulation_indicies = ['Unknown', 'Normal', 'High', 'Low'] scids = ['Unknown', 'GOES-East', 'GOES-West', 'GOES-Central', 'GOES-Test'] # Spacecraft ID's blk_len = int.from_bytes(block_data[0x01:0x03], byteorder='little') seq_num = int.from_bytes(block_data[0x03:0x06], byteorder='little') flags = block_data[0x06] baud = bauds[(flags & 0b111)] # B0-B2 of flags defines the DCP Data Rate platform = platforms[(flags & 0b1000) >> 3] # B3 is the platform type rx_parity = bool( (flags & 0b10000) >> 4) # Presence of Parity Errors in DCP Data arm_flags = block_data[0x07] # Abnormal Received Messages Flags arm_text = parse_arm(arm_flags) corrected_addr = hex( int.from_bytes(block_data[0x08:0x0C], byteorder='little')) carr_start = block_data[0x0C:0x13] msg_end = block_data[0x13:0x1A] sig_strength = int.from_bytes(block_data[0x1A:0x1C], byteorder='little') & 0x03FF freq_offset = int.from_bytes(block_data[0x1C:0x1E], byteorder='little') & 0x3FFF if freq_offset > 8191: # 2's complement conversion freq_offset = freq_offset - 16384 phs_noise = int.from_bytes(block_data[0x1E:0x20], byteorder='little') & 0x01FFF mod_index = modulation_indicies[ (int.from_bytes(block_data[0x1E:0x20], byteorder='little') & 0x0C000) >> 14] good_phs = int.from_bytes(block_data[0x20:0x21], byteorder='little') channel = int.from_bytes(block_data[0x21:0x23], byteorder='little') & 0x03FF spacecraft = int.from_bytes(block_data[0x21:0x23], byteorder='little') >> 12 source_code = block_data[0x23:0x25].decode('ascii') source_sec = block_data[0x25:0x27].decode('ascii') codecs.register(pseudo_search_func) dcp_data_pseudo = codecs.decode(block_data[0x27:-2], encoding='pseudo-binary') dcp_crc16 = int.from_bytes(block_data[-2:], byteorder='little') calc_crc = binascii.crc_hqx(block_data[:-2], 0xFFFF) print('\n----------[ DCP Block ]----------') if verbose: print('Header:') print(' Size: {} Bytes ({} Bytes of Data)'.format( blk_len, blk_len - 41)) # 39 Bytes for Header, 2 Bytes for CRC print(' Seqeuence Number: {}'.format(seq_num)) print(' Flags:') print(' Data Rate: {} Baud'.format(baud)) print(' Platform: {}'.format(platform)) print(' Parity Error? {}'.format(rx_parity)) print(' ARM Flags: {}'.format(arm_text)) print(' Corrected Address: {}'.format(corrected_addr)) print(' Carrier Start: {}'.format(bcd_to_date(carr_start))) print(' Message End: {}'.format(bcd_to_date(msg_end))) print(' Signal Strength: {}dBm EIRP'.format(sig_strength / 10)) print(' Frequency Offset: {}Hz'.format(freq_offset / 10)) print(' Phase Noise: {}° RMS'.format(phs_noise / 100)) print(' Modulation Index: {}'.format(mod_index)) print(' Good Phase: {}%'.format(good_phs / 2)) print(' Channel: {}'.format(channel)) print(' Spacecraft: {}'.format(scids[spacecraft])) print(' Source Code: {}'.format(source_code)) print(' Source Secondary: {}'.format(source_sec)) if dcp_crc16 == calc_crc: print('Block CRC: OK\n') else: print('CRC: FAILED\n') else: print('Corrected Address: {}'.format(corrected_addr)) print('Data (Pseudo-Binary): \n{}'.format(dcp_data_pseudo))
def win32_unicode_console(): import codecs from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_int from ctypes.wintypes import BOOL, HANDLE, DWORD, LPWSTR, LPCWSTR, LPVOID original_stderr = sys.stderr # Output exceptions in this code to original_stderr, so that we can at least see them def _complain(message): original_stderr.write( message if isinstance(message, str) else repr(message)) original_stderr.write('\n') codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) try: GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(("GetStdHandle", windll.kernel32)) STD_OUTPUT_HANDLE = DWORD(-11) STD_ERROR_HANDLE = DWORD(-12) GetFileType = WINFUNCTYPE(DWORD, DWORD)(("GetFileType", windll.kernel32)) FILE_TYPE_CHAR = 0x0002 FILE_TYPE_REMOTE = 0x8000 GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))( ("GetConsoleMode", windll.kernel32)) INVALID_HANDLE_VALUE = DWORD(-1).value def not_a_console(handle): if handle == INVALID_HANDLE_VALUE or handle is None: return True return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or GetConsoleMode(handle, byref(DWORD())) == 0) old_stdout_fileno = None old_stderr_fileno = None if hasattr(sys.stdout, 'fileno'): old_stdout_fileno = sys.stdout.fileno() if hasattr(sys.stderr, 'fileno'): old_stderr_fileno = sys.stderr.fileno() STDOUT_FILENO = 1 STDERR_FILENO = 2 real_stdout = (old_stdout_fileno == STDOUT_FILENO) real_stderr = (old_stderr_fileno == STDERR_FILENO) if real_stdout: hStdout = GetStdHandle(STD_OUTPUT_HANDLE) if not_a_console(hStdout): real_stdout = False if real_stderr: hStderr = GetStdHandle(STD_ERROR_HANDLE) if not_a_console(hStderr): real_stderr = False if real_stdout or real_stderr: WriteConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID)( ("WriteConsoleW", windll.kernel32)) class UnicodeOutput: def __init__(self, hConsole, stream, fileno, name): self._hConsole = hConsole self._stream = stream self._fileno = fileno self.closed = False self.softspace = False self.mode = 'w' self.encoding = 'utf-8' self.name = name self.flush() def isatty(self): return False def close(self): # don't really close the handle, that would only cause problems self.closed = True def fileno(self): return self._fileno def flush(self): if self._hConsole is None: try: self._stream.flush() except Exception as e: _complain("%s.flush: %r from %r" % (self.name, e, self._stream)) raise def write(self, text): try: if self._hConsole is None: if not PY3K and isinstance(text, unicode): text = text.encode('utf-8') elif PY3K and isinstance(text, str): text = text.encode('utf-8') self._stream.write(text) else: if not PY3K and not isinstance(text, unicode): text = str(text).decode('utf-8') elif PY3K and not isinstance(text, str): text = text.decode('utf-8') remaining = len(text) while remaining: n = DWORD(0) # There is a shorter-than-documented limitation on the # length of the string passed to WriteConsoleW (see # <http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1232>. retval = WriteConsoleW(self._hConsole, text, min(remaining, 10000), byref(n), None) if retval == 0 or n.value == 0: raise IOError( "WriteConsoleW returned %r, n.value = %r" % (retval, n.value)) remaining -= n.value if not remaining: break text = text[n.value:] except Exception as e: _complain("%s.write: %r" % (self.name, e)) raise def writelines(self, lines): try: for line in lines: self.write(line) except Exception as e: _complain("%s.writelines: %r" % (self.name, e)) raise if real_stdout: sys.stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO, '<Unicode console stdout>') else: sys.stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno, '<Unicode redirected stdout>') if real_stderr: sys.stderr = UnicodeOutput(hStderr, None, STDERR_FILENO, '<Unicode console stderr>') else: sys.stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno, '<Unicode redirected stderr>') except Exception as e: _complain("exception %r while fixing up sys.stdout and sys.stderr" % (e, ))
else: r.append(modified_unbase64(''.join(decode[1:]))) decode = [] elif decode: decode.append(c) else: r.append(c) if decode: r.append(modified_unbase64(''.join(decode[1:]))) bin_str = ''.join(r) return bin_str, len(s) class StreamReader(codecs.StreamReader): def decode(self, s, errors='strict'): return decoder(s) class StreamWriter(codecs.StreamWriter): def decode(self, s, errors='strict'): return encoder(s) def imap4_utf_7(name): if name == 'imap4-utf-7': return encoder, decoder, StreamReader, StreamWriter codecs.register(imap4_utf_7)
import struct import subprocess import sys if sys.version_info < (2, 7): sys.stderr.write('You need Python 2.7 or later\n') sys.exit(1) # Setup imports depending on whether IronPython or CPython if sys.platform == 'win32' \ or sys.platform == 'cli': from _winreg import * # Fix "Unknown encoding" error when running on Powershell # https://stackoverflow.com/a/3259271/3354920 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) def bytetohex(data): if sys.version_info > (3, 0): # Python 3 code in this block return "".join("{:02X} ".format(c) for c in data) else: # Python 2 code in this block return "".join("{:02X} ".format(ord(c)) for c in data) def joinpath(folder, filename): return os.path.join(folder, filename) def printkey(i, offset, smc_key, smc_data):
# aliases) try: codecaliases = mod.getaliases() except AttributeError: pass else: for alias in codecaliases: if alias not in _aliases: _aliases[alias] = modname # Return the registry entry return entry # Register the search_function in the Python codec registry codecs.register(search_function) if sys.platform == 'win32': def _alias_mbcs(encoding): try: import _winapi ansi_code_page = "cp%s" % _winapi.GetACP() if encoding == ansi_code_page: import encodings.mbcs return encodings.mbcs.getregentry() except ImportError: # Imports may fail while we are shutting down pass codecs.register(_alias_mbcs)
def _real_main(argv=None): # Compatibility fixes for Windows if sys.platform == 'win32': # https://github.com/ytdl-org/youtube-dl/issues/820 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) workaround_optparse_bug9161() setproctitle('yt-dlp') parser, opts, args = parseOpts(argv) warnings = [] # Set user agent if opts.user_agent is not None: std_headers['User-Agent'] = opts.user_agent # Set referer if opts.referer is not None: std_headers['Referer'] = opts.referer # Custom HTTP headers std_headers.update(opts.headers) # Dump user agent if opts.dump_user_agent: write_string(std_headers['User-Agent'] + '\n', out=sys.stdout) sys.exit(0) # Batch file verification batch_urls = [] if opts.batchfile is not None: try: if opts.batchfile == '-': batchfd = sys.stdin else: batchfd = io.open(expand_path(opts.batchfile), 'r', encoding='utf-8', errors='ignore') batch_urls = read_batch_urls(batchfd) if opts.verbose: write_string('[debug] Batch file urls: ' + repr(batch_urls) + '\n') except IOError: sys.exit('ERROR: batch file %s could not be read' % opts.batchfile) all_urls = batch_urls + [ url.strip() for url in args ] # batch_urls are already striped in read_batch_urls _enc = preferredencoding() all_urls = [ url.decode(_enc, 'ignore') if isinstance(url, bytes) else url for url in all_urls ] if opts.list_extractors: for ie in list_extractors(opts.age_limit): write_string(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '') + '\n', out=sys.stdout) matchedUrls = [url for url in all_urls if ie.suitable(url)] for mu in matchedUrls: write_string(' ' + mu + '\n', out=sys.stdout) sys.exit(0) if opts.list_extractor_descriptions: for ie in list_extractors(opts.age_limit): if not ie._WORKING: continue desc = getattr(ie, 'IE_DESC', ie.IE_NAME) if desc is False: continue if hasattr(ie, 'SEARCH_KEY'): _SEARCHES = ('cute kittens', 'slithering pythons', 'falling cat', 'angry poodle', 'purple fish', 'running tortoise', 'sleeping bunny', 'burping cow') _COUNTS = ('', '5', '10', 'all') desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY, random.choice(_COUNTS), random.choice(_SEARCHES)) write_string(desc + '\n', out=sys.stdout) sys.exit(0) if opts.ap_list_mso: table = [[mso_id, mso_info['name']] for mso_id, mso_info in MSO_INFO.items()] write_string('Supported TV Providers:\n' + render_table(['mso', 'mso name'], table) + '\n', out=sys.stdout) sys.exit(0) # Conflicting, missing and erroneous options if opts.usenetrc and (opts.username is not None or opts.password is not None): parser.error('using .netrc conflicts with giving username/password') if opts.password is not None and opts.username is None: parser.error('account username missing\n') if opts.ap_password is not None and opts.ap_username is None: parser.error('TV Provider account username missing\n') if opts.autonumber_size is not None: if opts.autonumber_size <= 0: parser.error('auto number size must be positive') if opts.autonumber_start is not None: if opts.autonumber_start < 0: parser.error('auto number start must be positive or 0') if opts.username is not None and opts.password is None: opts.password = compat_getpass( 'Type account password and press [Return]: ') if opts.ap_username is not None and opts.ap_password is None: opts.ap_password = compat_getpass( 'Type TV provider account password and press [Return]: ') if opts.ratelimit is not None: numeric_limit = FileDownloader.parse_bytes(opts.ratelimit) if numeric_limit is None: parser.error('invalid rate limit specified') opts.ratelimit = numeric_limit if opts.min_filesize is not None: numeric_limit = FileDownloader.parse_bytes(opts.min_filesize) if numeric_limit is None: parser.error('invalid min_filesize specified') opts.min_filesize = numeric_limit if opts.max_filesize is not None: numeric_limit = FileDownloader.parse_bytes(opts.max_filesize) if numeric_limit is None: parser.error('invalid max_filesize specified') opts.max_filesize = numeric_limit if opts.sleep_interval is not None: if opts.sleep_interval < 0: parser.error('sleep interval must be positive or 0') if opts.max_sleep_interval is not None: if opts.max_sleep_interval < 0: parser.error('max sleep interval must be positive or 0') if opts.sleep_interval is None: parser.error( 'min sleep interval must be specified, use --min-sleep-interval' ) if opts.max_sleep_interval < opts.sleep_interval: parser.error( 'max sleep interval must be greater than or equal to min sleep interval' ) else: opts.max_sleep_interval = opts.sleep_interval if opts.sleep_interval_subtitles is not None: if opts.sleep_interval_subtitles < 0: parser.error('subtitles sleep interval must be positive or 0') if opts.sleep_interval_requests is not None: if opts.sleep_interval_requests < 0: parser.error('requests sleep interval must be positive or 0') if opts.ap_mso and opts.ap_mso not in MSO_INFO: parser.error( 'Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers' ) if opts.overwrites: # --yes-overwrites implies --no-continue opts.continue_dl = False if opts.concurrent_fragment_downloads <= 0: raise ValueError('Concurrent fragments must be positive') def parse_retries(retries, name=''): if retries in ('inf', 'infinite'): parsed_retries = float('inf') else: try: parsed_retries = int(retries) except (TypeError, ValueError): parser.error('invalid %sretry count specified' % name) return parsed_retries if opts.retries is not None: opts.retries = parse_retries(opts.retries) if opts.fragment_retries is not None: opts.fragment_retries = parse_retries(opts.fragment_retries, 'fragment ') if opts.extractor_retries is not None: opts.extractor_retries = parse_retries(opts.extractor_retries, 'extractor ') if opts.buffersize is not None: numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize) if numeric_buffersize is None: parser.error('invalid buffer size specified') opts.buffersize = numeric_buffersize if opts.http_chunk_size is not None: numeric_chunksize = FileDownloader.parse_bytes(opts.http_chunk_size) if not numeric_chunksize: parser.error('invalid http chunk size specified') opts.http_chunk_size = numeric_chunksize if opts.playliststart <= 0: raise ValueError('Playlist start must be positive') if opts.playlistend not in ( -1, None) and opts.playlistend < opts.playliststart: raise ValueError('Playlist end must be greater than playlist start') if opts.extractaudio: if opts.audioformat not in ['best'] + list( FFmpegExtractAudioPP.SUPPORTED_EXTS): parser.error('invalid audio format specified') if opts.audioquality: opts.audioquality = opts.audioquality.strip('k').strip('K') if not opts.audioquality.isdigit(): parser.error('invalid audio quality specified') if opts.recodevideo is not None: opts.recodevideo = opts.recodevideo.replace(' ', '') if not re.match(FFmpegVideoConvertorPP.FORMAT_RE, opts.recodevideo): parser.error('invalid video remux format specified') if opts.remuxvideo is not None: opts.remuxvideo = opts.remuxvideo.replace(' ', '') if not re.match(FFmpegVideoRemuxerPP.FORMAT_RE, opts.remuxvideo): parser.error('invalid video remux format specified') if opts.convertsubtitles is not None: if opts.convertsubtitles not in FFmpegSubtitlesConvertorPP.SUPPORTED_EXTS: parser.error('invalid subtitle format specified') if opts.convertthumbnails is not None: if opts.convertthumbnails not in FFmpegThumbnailsConvertorPP.SUPPORTED_EXTS: parser.error('invalid thumbnail format specified') if opts.date is not None: date = DateRange.day(opts.date) else: date = DateRange(opts.dateafter, opts.datebefore) def parse_compat_opts(): parsed_compat_opts, compat_opts = set(), opts.compat_opts[::-1] while compat_opts: actual_opt = opt = compat_opts.pop().lower() if opt == 'youtube-dl': compat_opts.extend(['-multistreams', 'all']) elif opt == 'youtube-dlc': compat_opts.extend( ['-no-youtube-channel-redirect', '-no-live-chat', 'all']) elif opt == 'all': parsed_compat_opts.update(all_compat_opts) elif opt == '-all': parsed_compat_opts = set() else: if opt[0] == '-': opt = opt[1:] parsed_compat_opts.discard(opt) else: parsed_compat_opts.update([opt]) if opt not in all_compat_opts: parser.error('Invalid compatibility option %s' % actual_opt) return parsed_compat_opts all_compat_opts = [ 'filename', 'format-sort', 'abort-on-error', 'format-spec', 'no-playlist-metafiles', 'multistreams', 'no-live-chat', 'playlist-index', 'list-formats', 'no-direct-merge', 'no-youtube-channel-redirect', 'no-youtube-unavailable-videos', 'no-attach-info-json', 'embed-thumbnail-atomicparsley', ] compat_opts = parse_compat_opts() def _unused_compat_opt(name): if name not in compat_opts: return False compat_opts.discard(name) compat_opts.update(['*%s' % name]) return True def set_default_compat(compat_name, opt_name, default=True, remove_compat=False): attr = getattr(opts, opt_name) if compat_name in compat_opts: if attr is None: setattr(opts, opt_name, not default) return True else: if remove_compat: _unused_compat_opt(compat_name) return False elif attr is None: setattr(opts, opt_name, default) return None set_default_compat('abort-on-error', 'ignoreerrors') set_default_compat('no-playlist-metafiles', 'allow_playlist_files') if 'format-sort' in compat_opts: opts.format_sort.extend(InfoExtractor.FormatSort.ytdl_default) _video_multistreams_set = set_default_compat( 'multistreams', 'allow_multiple_video_streams', False, remove_compat=False) _audio_multistreams_set = set_default_compat( 'multistreams', 'allow_multiple_audio_streams', False, remove_compat=False) if _video_multistreams_set is False and _audio_multistreams_set is False: _unused_compat_opt('multistreams') outtmpl_default = opts.outtmpl.get('default') if 'filename' in compat_opts: if outtmpl_default is None: outtmpl_default = '%(title)s.%(id)s.%(ext)s' opts.outtmpl.update({'default': outtmpl_default}) else: _unused_compat_opt('filename') def validate_outtmpl(tmpl, msg): err = YoutubeDL.validate_outtmpl(tmpl) if err: parser.error('invalid %s %r: %s' % (msg, tmpl, error_to_compat_str(err))) for k, tmpl in opts.outtmpl.items(): validate_outtmpl(tmpl, '%s output template' % k) for tmpl in opts.forceprint: validate_outtmpl(tmpl, 'print template') if opts.extractaudio and not opts.keepvideo and opts.format is None: opts.format = 'bestaudio/best' if outtmpl_default is not None and not os.path.splitext( outtmpl_default)[1] and opts.extractaudio: parser.error('Cannot download a video and extract audio into the same' ' file! Use "{0}.%(ext)s" instead of "{0}" as the output' ' template'.format(outtmpl_default)) for f in opts.format_sort: if re.match(InfoExtractor.FormatSort.regex, f) is None: parser.error('invalid format sort string "%s" specified' % f) if opts.metafromfield is None: opts.metafromfield = [] if opts.metafromtitle is not None: opts.metafromfield.append('title:%s' % opts.metafromtitle) for f in opts.metafromfield: if re.match(MetadataFromFieldPP.regex, f) is None: parser.error( 'invalid format string "%s" specified for --parse-metadata' % f) any_getting = opts.forceprint or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json any_printing = opts.print_json download_archive_fn = expand_path( opts.download_archive ) if opts.download_archive is not None else opts.download_archive # If JSON is not printed anywhere, but comments are requested, save it to file printing_json = opts.dumpjson or opts.print_json or opts.dump_single_json if opts.getcomments and not printing_json: opts.writeinfojson = True def report_conflict(arg1, arg2): warnings.append('%s is ignored since %s was given' % (arg2, arg1)) if opts.remuxvideo and opts.recodevideo: report_conflict('--recode-video', '--remux-video') opts.remuxvideo = False if opts.sponskrub_cut and opts.split_chapters and opts.sponskrub is not False: report_conflict('--split-chapter', '--sponskrub-cut') opts.sponskrub_cut = False if opts.allow_unplayable_formats: if opts.extractaudio: report_conflict('--allow-unplayable-formats', '--extract-audio') opts.extractaudio = False if opts.remuxvideo: report_conflict('--allow-unplayable-formats', '--remux-video') opts.remuxvideo = False if opts.recodevideo: report_conflict('--allow-unplayable-formats', '--recode-video') opts.recodevideo = False if opts.addmetadata: report_conflict('--allow-unplayable-formats', '--add-metadata') opts.addmetadata = False if opts.embedsubtitles: report_conflict('--allow-unplayable-formats', '--embed-subs') opts.embedsubtitles = False if opts.embedthumbnail: report_conflict('--allow-unplayable-formats', '--embed-thumbnail') opts.embedthumbnail = False if opts.xattrs: report_conflict('--allow-unplayable-formats', '--xattrs') opts.xattrs = False if opts.fixup and opts.fixup.lower() not in ('never', 'ignore'): report_conflict('--allow-unplayable-formats', '--fixup') opts.fixup = 'never' if opts.sponskrub: report_conflict('--allow-unplayable-formats', '--sponskrub') opts.sponskrub = False # PostProcessors postprocessors = [] if opts.metafromfield: postprocessors.append({ 'key': 'MetadataFromField', 'formats': opts.metafromfield, # Run this immediately after extraction is complete 'when': 'pre_process' }) if opts.convertsubtitles: postprocessors.append({ 'key': 'FFmpegSubtitlesConvertor', 'format': opts.convertsubtitles, # Run this before the actual video download 'when': 'before_dl' }) if opts.convertthumbnails: postprocessors.append({ 'key': 'FFmpegThumbnailsConvertor', 'format': opts.convertthumbnails, # Run this before the actual video download 'when': 'before_dl' }) if opts.extractaudio: postprocessors.append({ 'key': 'FFmpegExtractAudio', 'preferredcodec': opts.audioformat, 'preferredquality': opts.audioquality, 'nopostoverwrites': opts.nopostoverwrites, }) if opts.remuxvideo: postprocessors.append({ 'key': 'FFmpegVideoRemuxer', 'preferedformat': opts.remuxvideo, }) if opts.recodevideo: postprocessors.append({ 'key': 'FFmpegVideoConvertor', 'preferedformat': opts.recodevideo, }) # FFmpegMetadataPP should be run after FFmpegVideoConvertorPP and # FFmpegExtractAudioPP as containers before conversion may not support # metadata (3gp, webm, etc.) # And this post-processor should be placed before other metadata # manipulating post-processors (FFmpegEmbedSubtitle) to prevent loss of # extra metadata. By default ffmpeg preserves metadata applicable for both # source and target containers. From this point the container won't change, # so metadata can be added here. if opts.addmetadata: postprocessors.append({'key': 'FFmpegMetadata'}) if opts.embedsubtitles: already_have_subtitle = opts.writesubtitles postprocessors.append({ 'key': 'FFmpegEmbedSubtitle', # already_have_subtitle = True prevents the file from being deleted after embedding 'already_have_subtitle': already_have_subtitle }) if not already_have_subtitle: opts.writesubtitles = True # --all-sub automatically sets --write-sub if --write-auto-sub is not given # this was the old behaviour if only --all-sub was given. if opts.allsubtitles and not opts.writeautomaticsub: opts.writesubtitles = True # This should be above EmbedThumbnail since sponskrub removes the thumbnail attachment # but must be below EmbedSubtitle and FFmpegMetadata # See https://github.com/yt-dlp/yt-dlp/issues/204 , https://github.com/faissaloo/SponSkrub/issues/29 # If opts.sponskrub is None, sponskrub is used, but it silently fails if the executable can't be found if opts.sponskrub is not False: postprocessors.append({ 'key': 'SponSkrub', 'path': opts.sponskrub_path, 'args': opts.sponskrub_args, 'cut': opts.sponskrub_cut, 'force': opts.sponskrub_force, 'ignoreerror': opts.sponskrub is None, }) if opts.embedthumbnail: already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails postprocessors.append({ 'key': 'EmbedThumbnail', # already_have_thumbnail = True prevents the file from being deleted after embedding 'already_have_thumbnail': already_have_thumbnail }) if not already_have_thumbnail: opts.writethumbnail = True if opts.split_chapters: postprocessors.append({'key': 'FFmpegSplitChapters'}) # XAttrMetadataPP should be run after post-processors that may change file contents if opts.xattrs: postprocessors.append({'key': 'XAttrMetadata'}) # ExecAfterDownload must be the last PP if opts.exec_cmd: postprocessors.append({ 'key': 'ExecAfterDownload', 'exec_cmd': opts.exec_cmd, # Run this only after the files have been moved to their final locations 'when': 'after_move' }) def report_args_compat(arg, name): warnings.append( '%s given without specifying name. The arguments will be given to all %s' % (arg, name)) if 'default' in opts.external_downloader_args: report_args_compat('--downloader-args', 'external downloaders') if 'default-compat' in opts.postprocessor_args and 'default' not in opts.postprocessor_args: report_args_compat('--post-processor-args', 'post-processors') opts.postprocessor_args.setdefault('sponskrub', []) opts.postprocessor_args['default'] = opts.postprocessor_args[ 'default-compat'] final_ext = (opts.recodevideo if opts.recodevideo in FFmpegVideoConvertorPP.SUPPORTED_EXTS else opts.remuxvideo if opts.remuxvideo in FFmpegVideoRemuxerPP.SUPPORTED_EXTS else opts.audioformat if (opts.extractaudio and opts.audioformat != 'best') else None) match_filter = (None if opts.match_filter is None else match_filter_func( opts.match_filter)) ydl_opts = { 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, 'twofactor': opts.twofactor, 'videopassword': opts.videopassword, 'ap_mso': opts.ap_mso, 'ap_username': opts.ap_username, 'ap_password': opts.ap_password, 'quiet': (opts.quiet or any_getting or any_printing), 'no_warnings': opts.no_warnings, 'forceurl': opts.geturl, 'forcetitle': opts.gettitle, 'forceid': opts.getid, 'forcethumbnail': opts.getthumbnail, 'forcedescription': opts.getdescription, 'forceduration': opts.getduration, 'forcefilename': opts.getfilename, 'forceformat': opts.getformat, 'forceprint': opts.forceprint, 'forcejson': opts.dumpjson or opts.print_json, 'dump_single_json': opts.dump_single_json, 'force_write_download_archive': opts.force_write_download_archive, 'simulate': opts.simulate or any_getting, 'skip_download': opts.skip_download, 'format': opts.format, 'allow_unplayable_formats': opts.allow_unplayable_formats, 'ignore_no_formats_error': opts.ignore_no_formats_error, 'format_sort': opts.format_sort, 'format_sort_force': opts.format_sort_force, 'allow_multiple_video_streams': opts.allow_multiple_video_streams, 'allow_multiple_audio_streams': opts.allow_multiple_audio_streams, 'check_formats': opts.check_formats, 'listformats': opts.listformats, 'listformats_table': opts.listformats_table, 'outtmpl': opts.outtmpl, 'outtmpl_na_placeholder': opts.outtmpl_na_placeholder, 'paths': opts.paths, 'autonumber_size': opts.autonumber_size, 'autonumber_start': opts.autonumber_start, 'restrictfilenames': opts.restrictfilenames, 'windowsfilenames': opts.windowsfilenames, 'ignoreerrors': opts.ignoreerrors, 'force_generic_extractor': opts.force_generic_extractor, 'ratelimit': opts.ratelimit, 'overwrites': opts.overwrites, 'retries': opts.retries, 'fragment_retries': opts.fragment_retries, 'extractor_retries': opts.extractor_retries, 'skip_unavailable_fragments': opts.skip_unavailable_fragments, 'keep_fragments': opts.keep_fragments, 'concurrent_fragment_downloads': opts.concurrent_fragment_downloads, 'buffersize': opts.buffersize, 'noresizebuffer': opts.noresizebuffer, 'http_chunk_size': opts.http_chunk_size, 'continuedl': opts.continue_dl, 'noprogress': opts.noprogress, 'progress_with_newline': opts.progress_with_newline, 'playliststart': opts.playliststart, 'playlistend': opts.playlistend, 'playlistreverse': opts.playlist_reverse, 'playlistrandom': opts.playlist_random, 'noplaylist': opts.noplaylist, 'logtostderr': outtmpl_default == '-', 'consoletitle': opts.consoletitle, 'nopart': opts.nopart, 'updatetime': opts.updatetime, 'writedescription': opts.writedescription, 'writeannotations': opts.writeannotations, 'writeinfojson': opts.writeinfojson, 'allow_playlist_files': opts.allow_playlist_files, 'clean_infojson': opts.clean_infojson, 'getcomments': opts.getcomments, 'writethumbnail': opts.writethumbnail, 'write_all_thumbnails': opts.write_all_thumbnails, 'writelink': opts.writelink, 'writeurllink': opts.writeurllink, 'writewebloclink': opts.writewebloclink, 'writedesktoplink': opts.writedesktoplink, 'writesubtitles': opts.writesubtitles, 'writeautomaticsub': opts.writeautomaticsub, 'allsubtitles': opts.allsubtitles, 'listsubtitles': opts.listsubtitles, 'subtitlesformat': opts.subtitlesformat, 'subtitleslangs': opts.subtitleslangs, 'matchtitle': decodeOption(opts.matchtitle), 'rejecttitle': decodeOption(opts.rejecttitle), 'max_downloads': opts.max_downloads, 'prefer_free_formats': opts.prefer_free_formats, 'trim_file_name': opts.trim_file_name, 'verbose': opts.verbose, 'dump_intermediate_pages': opts.dump_intermediate_pages, 'write_pages': opts.write_pages, 'test': opts.test, 'keepvideo': opts.keepvideo, 'min_filesize': opts.min_filesize, 'max_filesize': opts.max_filesize, 'min_views': opts.min_views, 'max_views': opts.max_views, 'daterange': date, 'cachedir': opts.cachedir, 'youtube_print_sig_code': opts.youtube_print_sig_code, 'age_limit': opts.age_limit, 'download_archive': download_archive_fn, 'break_on_existing': opts.break_on_existing, 'break_on_reject': opts.break_on_reject, 'skip_playlist_after_errors': opts.skip_playlist_after_errors, 'cookiefile': opts.cookiefile, 'nocheckcertificate': opts.no_check_certificate, 'prefer_insecure': opts.prefer_insecure, 'proxy': opts.proxy, 'socket_timeout': opts.socket_timeout, 'bidi_workaround': opts.bidi_workaround, 'debug_printtraffic': opts.debug_printtraffic, 'prefer_ffmpeg': opts.prefer_ffmpeg, 'include_ads': opts.include_ads, 'default_search': opts.default_search, 'dynamic_mpd': opts.dynamic_mpd, 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest, 'youtube_include_hls_manifest': opts.youtube_include_hls_manifest, 'encoding': opts.encoding, 'extract_flat': opts.extract_flat, 'mark_watched': opts.mark_watched, 'merge_output_format': opts.merge_output_format, 'final_ext': final_ext, 'postprocessors': postprocessors, 'fixup': opts.fixup, 'source_address': opts.source_address, 'call_home': opts.call_home, 'sleep_interval_requests': opts.sleep_interval_requests, 'sleep_interval': opts.sleep_interval, 'max_sleep_interval': opts.max_sleep_interval, 'sleep_interval_subtitles': opts.sleep_interval_subtitles, 'external_downloader': opts.external_downloader, 'list_thumbnails': opts.list_thumbnails, 'playlist_items': opts.playlist_items, 'xattr_set_filesize': opts.xattr_set_filesize, 'match_filter': match_filter, 'no_color': opts.no_color, 'ffmpeg_location': opts.ffmpeg_location, 'hls_prefer_native': opts.hls_prefer_native, 'hls_use_mpegts': opts.hls_use_mpegts, 'hls_split_discontinuity': opts.hls_split_discontinuity, 'external_downloader_args': opts.external_downloader_args, 'postprocessor_args': opts.postprocessor_args, 'cn_verification_proxy': opts.cn_verification_proxy, 'geo_verification_proxy': opts.geo_verification_proxy, 'geo_bypass': opts.geo_bypass, 'geo_bypass_country': opts.geo_bypass_country, 'geo_bypass_ip_block': opts.geo_bypass_ip_block, 'warnings': warnings, 'compat_opts': compat_opts, # just for deprecation check 'autonumber': opts.autonumber or None, 'usetitle': opts.usetitle or None, 'useid': opts.useid or None, } with YoutubeDL(ydl_opts) as ydl: actual_use = len(all_urls) or opts.load_info_filename # Remove cache dir if opts.rm_cachedir: ydl.cache.remove() # Update version if opts.update_self: # If updater returns True, exit. Required for windows if run_update(ydl): if actual_use: sys.exit( 'ERROR: The program must exit for the update to complete' ) sys.exit() # Maybe do nothing if not actual_use: if opts.update_self or opts.rm_cachedir: sys.exit() ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv) parser.error('You must provide at least one URL.\n' 'Type yt-dlp --help to see a list of all options.') try: if opts.load_info_filename is not None: retcode = ydl.download_with_info_file( expand_path(opts.load_info_filename)) else: retcode = ydl.download(all_urls) except (MaxDownloadsReached, ExistingVideoReached, RejectedVideoReached): ydl.to_screen('Aborting remaining downloads') retcode = 101 sys.exit(retcode)
return safe_encode(input, self.errors)[0] class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): return safe_decode(input, self.errors)[0] class StreamWriter(Codec, codecs.StreamWriter): pass class StreamReader(Codec, codecs.StreamReader): pass def getregentry(name): if name == 'safe-utf-8': return codecs.CodecInfo( name=name, encode=safe_encode, decode=safe_decode, incrementalencoder=IncrementalEncoder, incrementaldecoder=IncrementalDecoder, streamreader=StreamReader, streamwriter=StreamWriter, ) codecs.register(getregentry)
def initialize(): global done import sys if sys.platform != "win32" or done: return True done = True import codecs, re from ctypes import WINFUNCTYPE, WinError, windll, POINTER, byref, c_int, get_last_error from ctypes.wintypes import BOOL, HANDLE, DWORD, UINT, LPWSTR, LPCWSTR, LPVOID from allmydata.util import log from allmydata.util.encodingutil import canonical_encoding # <https://msdn.microsoft.com/en-us/library/ms680621%28VS.85%29.aspx> SetErrorMode = WINFUNCTYPE(UINT, UINT, use_last_error=True)( ("SetErrorMode", windll.kernel32)) SEM_FAILCRITICALERRORS = 0x0001 SEM_NOOPENFILEERRORBOX = 0x8000 SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX) original_stderr = sys.stderr # If any exception occurs in this code, we'll probably try to print it on stderr, # which makes for frustrating debugging if stderr is directed to our wrapper. # So be paranoid about catching errors and reporting them to original_stderr, # so that we can at least see them. def _complain(message): print(isinstance(message, str) and message or repr(message), file=original_stderr) log.msg(message, level=log.WEIRD) # Work around <http://bugs.python.org/issue6058>. codecs.register( lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None) # Make Unicode console output work independently of the current code page. # This also fixes <http://bugs.python.org/issue1602>. # Credit to Michael Kaplan <https://blogs.msdn.com/b/michkap/archive/2010/04/07/9989346.aspx> # and TZOmegaTZIOY # <http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/1432462#1432462>. try: # <https://msdn.microsoft.com/en-us/library/ms683231(VS.85).aspx> # HANDLE WINAPI GetStdHandle(DWORD nStdHandle); # returns INVALID_HANDLE_VALUE, NULL, or a valid handle # # <https://msdn.microsoft.com/en-us/library/aa364960(VS.85).aspx> # DWORD WINAPI GetFileType(DWORD hFile); # # <https://msdn.microsoft.com/en-us/library/ms683167(VS.85).aspx> # BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode); GetStdHandle = WINFUNCTYPE(HANDLE, DWORD, use_last_error=True)( ("GetStdHandle", windll.kernel32)) STD_OUTPUT_HANDLE = DWORD(-11) STD_ERROR_HANDLE = DWORD(-12) GetFileType = WINFUNCTYPE(DWORD, DWORD, use_last_error=True)( ("GetFileType", windll.kernel32)) FILE_TYPE_CHAR = 0x0002 FILE_TYPE_REMOTE = 0x8000 GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD), use_last_error=True)( ("GetConsoleMode", windll.kernel32)) INVALID_HANDLE_VALUE = DWORD(-1).value def not_a_console(handle): if handle == INVALID_HANDLE_VALUE or handle is None: return True return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or GetConsoleMode(handle, byref(DWORD())) == 0) old_stdout_fileno = None old_stderr_fileno = None if hasattr(sys.stdout, 'fileno'): old_stdout_fileno = sys.stdout.fileno() if hasattr(sys.stderr, 'fileno'): old_stderr_fileno = sys.stderr.fileno() STDOUT_FILENO = 1 STDERR_FILENO = 2 real_stdout = (old_stdout_fileno == STDOUT_FILENO) real_stderr = (old_stderr_fileno == STDERR_FILENO) if real_stdout: hStdout = GetStdHandle(STD_OUTPUT_HANDLE) if not_a_console(hStdout): real_stdout = False if real_stderr: hStderr = GetStdHandle(STD_ERROR_HANDLE) if not_a_console(hStderr): real_stderr = False if real_stdout or real_stderr: # <https://msdn.microsoft.com/en-us/library/windows/desktop/ms687401%28v=vs.85%29.aspx> # BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars, # LPDWORD lpCharsWritten, LPVOID lpReserved); WriteConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID, use_last_error=True)( ("WriteConsoleW", windll.kernel32)) class UnicodeOutput(object): def __init__(self, hConsole, stream, fileno, name): self._hConsole = hConsole self._stream = stream self._fileno = fileno self.closed = False self.softspace = False self.mode = 'w' self.encoding = 'utf-8' self.name = name if hasattr(stream, 'encoding') and canonical_encoding( stream.encoding) != 'utf-8': log.msg( "%s: %r had encoding %r, but we're going to write UTF-8 to it" % (name, stream, stream.encoding), level=log.CURIOUS) self.flush() def isatty(self): return False def close(self): # don't really close the handle, that would only cause problems self.closed = True def fileno(self): return self._fileno def flush(self): if self._hConsole is None: try: self._stream.flush() except Exception as e: _complain("%s.flush: %r from %r" % (self.name, e, self._stream)) raise def write(self, text): try: if self._hConsole is None: if isinstance(text, unicode): text = text.encode('utf-8') self._stream.write(text) else: if not isinstance(text, unicode): text = str(text).decode('utf-8') remaining = len(text) while remaining > 0: n = DWORD(0) # There is a shorter-than-documented limitation on the length of the string # passed to WriteConsoleW (see #1232). retval = WriteConsoleW(self._hConsole, text, min(remaining, 10000), byref(n), None) if retval == 0: raise IOError( "WriteConsoleW failed with WinError: %s" % (WinError(get_last_error()), )) if n.value == 0: raise IOError( "WriteConsoleW returned %r, n.value = 0" % (retval, )) remaining -= n.value if remaining == 0: break text = text[n.value:] except Exception as e: _complain("%s.write: %r" % (self.name, e)) raise def writelines(self, lines): try: for line in lines: self.write(line) except Exception as e: _complain("%s.writelines: %r" % (self.name, e)) raise if real_stdout: sys.stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO, '<Unicode console stdout>') else: sys.stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno, '<Unicode redirected stdout>') if real_stderr: sys.stderr = UnicodeOutput(hStderr, None, STDERR_FILENO, '<Unicode console stderr>') else: sys.stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno, '<Unicode redirected stderr>') except Exception as e: _complain("exception %r while fixing up sys.stdout and sys.stderr" % (e, )) # This works around <http://bugs.python.org/issue2128>. # <https://msdn.microsoft.com/en-us/library/windows/desktop/ms683156%28v=vs.85%29.aspx> GetCommandLineW = WINFUNCTYPE(LPWSTR, use_last_error=True)( ("GetCommandLineW", windll.kernel32)) # <https://msdn.microsoft.com/en-us/library/windows/desktop/bb776391%28v=vs.85%29.aspx> CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int), use_last_error=True)( ("CommandLineToArgvW", windll.shell32)) argc = c_int(0) argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) if argv_unicode is None: raise WinError(get_last_error()) # Because of <http://bugs.python.org/issue8775> (and similar limitations in # twisted), the 'bin/tahoe' script cannot invoke us with the actual Unicode arguments. # Instead it "mangles" or escapes them using \x7F as an escape character, which we # unescape here. def unmangle(s): return re.sub( u'\\x7F[0-9a-fA-F]*\\;', # type ignored for 'unichr' (Python 2 only) lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore s, ) try: argv = [ unmangle(argv_unicode[i]).encode('utf-8') for i in xrange(0, argc.value) ] except Exception as e: _complain( "%s: could not unmangle Unicode arguments.\n%r" % (sys.argv[0], [argv_unicode[i] for i in xrange(0, argc.value)])) raise # Take only the suffix with the same number of arguments as sys.argv. # This accounts for anything that can cause initial arguments to be stripped, # for example, the Python interpreter or any options passed to it, or runner # scripts such as 'coverage run'. It works even if there are no such arguments, # as in the case of a frozen executable created by bb-freeze or similar. sys.argv = argv[-len(sys.argv):] if sys.argv[0].endswith('.pyscript'): sys.argv[0] = sys.argv[0][:-9]
def _from_response(self, rsp): self.version = rsp.hpm_1_version self.components = [] for i in range(8): if rsp.component_present & (1 << i): self.components.append(i) def __str__(self): string = [] string.append("Target Upgrade Capabilities") string.append(" HPM.1 version: %s" % self.version) string.append(" Components: %s" % self.components) return "\n".join(string) codecs.register(bcd_search) class ComponentProperty(object): def __init__(self, data=None): if (data): self._from_rsp_data(data) @staticmethod def from_data(component_id, data): if isinstance(data, str): data = [ord(c) for c in data] if component_id is PROPERTY_GENERAL_PROPERTIES: return ComponentPropertyGeneral(data) elif component_id is PROPERTY_CURRENT_VERSION:
from __future__ import absolute_import, print_function from setuptools.command.build_py import build_py # Work around mbcs bug in distutils. # http://bugs.python.org/issue10945 import codecs try: codecs.lookup('mbcs') except LookupError: ascii = codecs.lookup('ascii') func = lambda name, enc=ascii: {True: enc}.get(name == 'mbcs') codecs.register(func) from setuptools import setup, Extension import glob, os, shutil, fnmatch, platform, sys version = '2.4.7' def generate_content(): # generate the file content... from generator import mavgen, mavparse # path to message_definitions directory if os.getenv("MDEF", None) is not None: mdef_paths = [os.getenv("MDEF")] else: mdef_paths = [ os.path.join('..', 'message_definitions'), os.path.join('mavlink', 'message_definitions'), os.path.join('..', 'mavlink', 'message_definitions'),
inner = next(self.tokens) if inner.string in pydoc.Helper.keywords: self.output.append( tokenize.TokenInfo( type=tokenize.STRING, string=repr(inner.string), start=inner.start, end=inner.end, line=inner.line, ) ) else: self.output.append(inner) def encode(string, errors="strict"): raise NotImplemented("this is nonsense, right?") def decode(byteslike, errors="strict"): code_bytes = bytes(byteslike) decoder = HelpEncoding( tokenize.tokenize(io.BytesIO(b"(" + code_bytes + b")").readline) ) decoder.parse() return (tokenize.untokenize(decoder.output[2:-2]), len(code_bytes)) codec_info = codecs.CodecInfo(encode, decode, name="halp") codecs.register({"halp": codec_info}.get)
def register_cp65001(): """Register codecs cp65001 as utf-8.""" # Work around <http://bugs.python.org/issue6058>. codecs.register(lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None)
) pass # This is the updated MBCS Encoding Bypass for making MBCS encodings work on Linux - NovaCygni try: codecs.lookup('mbcs') except LookupError: ascii_encoding = codecs.lookup('latin-1') def mbcs_bypass(name, encoding=ascii_encoding): if name == "mbcs": return encoding codecs.register(mbcs_bypass) # Colours W = "\033[0m" R = "\033[31m" G = "\033[32m" O = "\033[33m" B = "\033[34m" subprocess.call("clear", shell=True) arg_end = "--" arg_eva = "+" colMax = 60 # Change this at your will endsub = 1 gets = 0 timeout = 8
if buildCython: cmdclass["build_ext"] = setup_cython.CythonBuildExtension setup_cython.registerCythonModules() else: print("Skipping build of CYTHON modules.") ext_modules = setup_cython.ext_modules extraKeywords = {} # Workaround for mbcs codec bug in distutils # http://bugs.python.org/issue10945 import codecs try: codecs.lookup("mbcs") except LookupError: codecs.register(lambda name: codecs.lookup("ascii") if name == "mbcs" else None) # Create list of scripts. Depends on OS. scripts = [ "awlsim-gui", "awlsim-client", "awlsim-server", "awlsim-symtab", "awlsim-proupgrade", "awlsim-test", ] if isWindows or fullBuild: scripts.append("awlsim-win.cmd") if not isWindows or fullBuild: scripts.append("awlsim-linuxcnc-hal")
def find_invertcaps(encoding): """Return the codec for 'invertcaps'. """ if encoding == 'invertcaps': return codecs.CodecInfo( name='invertcaps', encode=InvertCapsCodec().encode, decode=InvertCapsCodec().decode, incrementalencoder=InvertCapsIncrementalEncoder, incrementaldecoder=InvertCapsIncrementalDecoder, streamreader=InvertCapsStreamReader, streamwriter=InvertCapsStreamWriter, ) return None codecs.register(find_invertcaps) if __name__ == '__main__': # Stateless encoder/decoder encoder = codecs.getencoder('invertcaps') text = 'abc.DEF' encoded_text, consumed = encoder(text) print 'Encoder converted "{}" to "{}", consuming {} characters'.format( text, encoded_text, consumed) # Stream writer import sys writer = codecs.getwriter('invertcaps')(sys.stdout) print 'StreamWriter for stdout: ', writer.write('abc.DEF')
# GNU General Public License for more details. # # # # You should have received a copy of the GNU General Public License # # along with this program; if not, write to the Free Software # # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # # MA 02110-1301, USA. # ##################################################################### """ Main game executable. """ # Register the latin-1 encoding import codecs import encodings.iso8859_1 import encodings.utf_8 codecs.register(lambda encoding: encodings.iso8859_1.getregentry()) codecs.register(lambda encoding: encodings.utf_8.getregentry()) assert codecs.lookup("iso-8859-1") assert codecs.lookup("utf-8") from GameEngine import GameEngine from MainMenu import MainMenu import Log import Config import Version import getopt import sys import os import codecs import Resource
def fix_broken_win_console(): '''Fix broken Unicode support in Windows console. This disables ANSI color support. It is only necessary to run this if the UTF-8 cp65001 code page is active. This code is lifted from Daira Hopwood's answer on SO: http://stackoverflow.com/a/3259271/1583598 ''' import codecs from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_int from ctypes.wintypes import BOOL, HANDLE, DWORD, LPWSTR, LPCWSTR, LPVOID original_stderr = sys.stderr # If any exception occurs in this code, we'll probably try to print it on stderr, # which makes for frustrating debugging if stderr is directed to our wrapper. # So be paranoid about catching errors and reporting them to original_stderr, # so that we can at least see them. def _complain(message): print(message if isinstance(message, str) else repr(message), file=original_stderr) # Work around <http://bugs.python.org/issue6058>. codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) # Make Unicode console output work independently of the current code page. # This also fixes <http://bugs.python.org/issue1602>. # Credit to Michael Kaplan <http://www.siao2.com/2010/04/07/9989346.aspx> # and TZOmegaTZIOY # <http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/1432462#1432462>. try: # <http://msdn.microsoft.com/en-us/library/ms683231(VS.85).aspx> # HANDLE WINAPI GetStdHandle(DWORD nStdHandle); # returns INVALID_HANDLE_VALUE, NULL, or a valid handle # # <http://msdn.microsoft.com/en-us/library/aa364960(VS.85).aspx> # DWORD WINAPI GetFileType(DWORD hFile); # # <http://msdn.microsoft.com/en-us/library/ms683167(VS.85).aspx> # BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode); GetStdHandle = WINFUNCTYPE(HANDLE, DWORD)(("GetStdHandle", windll.kernel32)) STD_OUTPUT_HANDLE = DWORD(-11) STD_ERROR_HANDLE = DWORD(-12) GetFileType = WINFUNCTYPE(DWORD, DWORD)(("GetFileType", windll.kernel32)) FILE_TYPE_CHAR = 0x0002 FILE_TYPE_REMOTE = 0x8000 GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))( ("GetConsoleMode", windll.kernel32)) INVALID_HANDLE_VALUE = DWORD(-1).value def not_a_console(handle): if handle == INVALID_HANDLE_VALUE or handle is None: return True return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or GetConsoleMode(handle, byref(DWORD())) == 0) old_stdout_fileno = None old_stderr_fileno = None if hasattr(sys.stdout, 'fileno'): old_stdout_fileno = sys.stdout.fileno() if hasattr(sys.stderr, 'fileno'): old_stderr_fileno = sys.stderr.fileno() STDOUT_FILENO = 1 STDERR_FILENO = 2 real_stdout = (old_stdout_fileno == STDOUT_FILENO) real_stderr = (old_stderr_fileno == STDERR_FILENO) if real_stdout: hStdout = GetStdHandle(STD_OUTPUT_HANDLE) if not_a_console(hStdout): real_stdout = False if real_stderr: hStderr = GetStdHandle(STD_ERROR_HANDLE) if not_a_console(hStderr): real_stderr = False if real_stdout or real_stderr: # BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars, # LPDWORD lpCharsWritten, LPVOID lpReserved); WriteConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD, POINTER(DWORD), LPVOID)( ("WriteConsoleW", windll.kernel32)) class UnicodeOutput: def __init__(self, hConsole, stream, fileno, name): self._hConsole = hConsole self._stream = stream self._fileno = fileno self.closed = False self.softspace = False self.mode = 'w' self.encoding = 'utf-8' self.name = name self.flush() def isatty(self): return False def close(self): # don't really close the handle, that would only cause problems self.closed = True def fileno(self): return self._fileno def flush(self): if self._hConsole is None: try: self._stream.flush() except Exception as e: _complain("%s.flush: %r from %r" % (self.name, e, self._stream)) raise def write(self, text): try: if self._hConsole is None: if isinstance(text, str): text = text.encode('utf-8') self._stream.write(text) else: if not isinstance(text, str): text = str(text).decode('utf-8') remaining = len(text) while remaining: n = DWORD(0) # There is a shorter-than-documented limitation on the # length of the string passed to WriteConsoleW (see # <http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1232>. retval = WriteConsoleW(self._hConsole, text, min(remaining, 10000), byref(n), None) if retval == 0 or n.value == 0: raise IOError( "WriteConsoleW returned %r, n.value = %r" % (retval, n.value)) remaining -= n.value if not remaining: break text = text[n.value:] except Exception as e: _complain("%s.write: %r" % (self.name, e)) raise def writelines(self, lines): try: for line in lines: self.write(line) except Exception as e: _complain("%s.writelines: %r" % (self.name, e)) raise if real_stdout: sys.stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO, '<Unicode console stdout>') else: sys.stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno, '<Unicode redirected stdout>') if real_stderr: sys.stderr = UnicodeOutput(hStderr, None, STDERR_FILENO, '<Unicode console stderr>') else: sys.stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno, '<Unicode redirected stderr>') except Exception as e: _complain("exception %r while fixing up sys.stdout and sys.stderr" % (e, )) # While we're at it, let's unmangle the command-line arguments: # This works around <http://bugs.python.org/issue2128>. GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( ("CommandLineToArgvW", windll.shell32)) argc = c_int(0) argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) argv = [argv_unicode[i].encode('utf-8') for i in range(0, argc.value)] if not hasattr(sys, 'frozen'): # If this is an executable produced by py2exe or bbfreeze, then it will # have been invoked directly. Otherwise, unicode_argv[0] is the Python # interpreter, so skip that. argv = argv[1:] # Also skip option arguments to the Python interpreter. while len(argv) > 0: arg = argv[0] if not arg.startswith("-") or arg == "-": break argv = argv[1:] if arg == '-m': # sys.argv[0] should really be the absolute path of the module source, # but never mind break if arg == '-c': argv[0] = '-c' break # if you like: sys.argv = argv
not hasattr(entry[1], '__call__') or \ (entry[2] is not None and not hasattr(entry[2], '__call__')) or \ (entry[3] is not None and not hasattr(entry[3], '__call__')) or \ (len(entry) > 4 and entry[4] is not None and not hasattr(entry[4], '__call__')) or \ (len(entry) > 5 and entry[5] is not None and not hasattr(entry[5], '__call__')): raise CodecRegistryError('incompatible codecs in module "%s" (%s)' % (mod.__name__, mod.__file__)) if len(entry)<7 or entry[6] is None: entry += (None,)*(6-len(entry)) + (mod.__name__.split(".", 1)[1],) entry = codecs.CodecInfo(*entry) # Cache the codec registry entry _cache[encoding] = entry # Register its aliases (without overwriting previously registered # aliases) try: codecaliases = mod.getaliases() except AttributeError: pass else: for alias in codecaliases: if alias not in _aliases: _aliases[alias] = modname # Return the registry entry return entry # Register the search_function in the Python codec registry codecs.register(search_function)
#!/usr/bin/env python3 from setuptools import setup # Work around mbcs bug in distutils. # http://bugs.python.org/issue10945 import codecs try: codecs.lookup('mbcs') except LookupError: ascii = codecs.lookup('ascii') codecs.register(lambda name, enc=ascii: {True: enc}.get(name == 'mbcs')) VERSION = '0.2.0' URL = "https://github.com/bitshares/python-bitshares" setup( name='bitshares', version=VERSION, description='Python library for bitshares', long_description=open('README.md').read(), download_url='{}/tarball/{}'.format(URL, VERSION), author='Fabian Schuh', author_email='*****@*****.**', maintainer='Fabian Schuh', maintainer_email='*****@*****.**', url=URL, keywords=['bitshares', 'library', 'api', 'rpc'], packages=[ "bitshares", "bitsharesapi",