def initbotscharsets():
'''set up right charset handling for specific charsets (UNOA, UNOB, UNOC, etc).'''
codecs.register(codec_search_function) #tell python how to search a codec defined by bots. These are the codecs in usersys/charset
botsglobal.botsreplacechar = unicode(botsglobal.ini.get('settings','botsreplacechar',u' '))
codecs.register_error('botsreplace', botscharsetreplace) #define the ' botsreplace' error handling for codecs/charsets.
for key, value in botsglobal.ini.items('charsets'): #set aliases for charsets in bots.ini
encodings.aliases.aliases[key]=value
def add_cp65001_codec():
if PY2:
try:
codecs.lookup("cp65001")
except LookupError:
codecs.register(lambda name: name == "cp65001" and codecs.lookup("utf-8") or None)
return
def register_codec():
import codecs
class Codec(codecs.Codec):
def encode(self, input, errors='strict'):
if not input:
return b"", 0
output = encode(input)
return b"".join(output), len(output)
def decode(self, input, errors='strict'):
if not input:
return u"", 0
output = decode(input)
return u"".join(output), len(output)
class StreamWriter(Codec, codecs.StreamWriter):
pass
class StreamReader(Codec, codecs.StreamReader):
pass
def getregentry(encoding):
if encoding != 'abicomp':
return None
return (Codec().encode,
Codec().decode,
StreamReader,
StreamWriter)
codecs.register(getregentry)
def register_codec():
class Codec(codecs.Codec):
def decode(self, input, errors='strict'):
return codecs.charmap_decode(input, errors, decoding_map)
def encode(self, input, errors='strict'):
return codecs.charmap_encode(input, errors, encoding_map)
class StreamWriter(Codec, codecs.StreamWriter):
pass
class StreamReader(Codec, codecs.StreamReader):
pass
def getregentry(encoding):
if encoding != 'latscii':
return None
return (Codec().encode,
Codec().decode,
StreamReader,
StreamWriter)
codecs.register(getregentry)
def latscii_error(uerr):
key = ord(uerr.object[uerr.start:uerr.end])
try:
return unichr(decoding_map[key]), uerr.end
except KeyError:
handler = codecs.lookup_error('replace')
return handler(uerr)
codecs.register_error('replacelatscii', latscii_error)
def add_cp65001_codec():
try:
codecs.lookup('cp65001')
except LookupError:
codecs.register(
lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None)
return
def find_encodings(enc=None, system=False):
"""Find functions for encoding translations for a specific codec.
:param str enc: The codec to find translation functions for. It will be
normalized by converting to lowercase, excluding
everything which is not ascii, and hyphens will be
converted to underscores.
:param bool system: If True, find encodings based on the system's stdin
encoding, otherwise assume utf-8.
:raises: :exc:LookupError if the normalized codec, ``enc``, cannot be
found in Python's encoding translation map.
"""
if not enc:
enc = 'utf-8'
if system:
if getattr(sys.stdin, 'encoding', None) is None:
enc = sys.stdin.encoding
log.debug("Obtained encoding from stdin: %s" % enc)
else:
enc = 'ascii'
## have to have lowercase to work, see
## http://docs.python.org/dev/library/codecs.html#standard-encodings
enc = enc.lower()
codec_alias = encodings.normalize_encoding(enc)
codecs.register(encodings.search_function)
coder = codecs.lookup(codec_alias)
return coder
def _mysql(self):
''' Configure db_connection for MySQL '''
logging.debug("Configured to use MySQL for a database")
db_server, db_name, db_user, db_password = self._db_credentials()
db_charset = "utf8mb4"
codecs.register(lambda name: codecs.lookup('utf8') if name == 'utf8mb4' else None)
__mysql = 'mysql://%s:%s@%s/%s?charset=%s' % (
db_user, db_password, db_server, db_name, db_charset
)
__mysqlclient = 'mysql+mysqldb://%s:%s@%s/%s?charset=%s' % (
db_user, db_password, db_server, db_name, db_charset
)
__pymysql = 'mysql+pymysql://%s:%s@%s/%s?charset=%s' % (
db_user, db_password, db_server, db_name, db_charset
)
__mysqlconnector = 'mysql+mysqlconnector://%s:%s@%s/%s?charset=%s' % (
db_user, db_password, db_server, db_name, db_charset
)
if self._test_connection(__mysql):
return __mysql
elif self._test_connection(__mysqlclient):
return __mysqlclient
elif self._test_connection(__pymysql):
return __pymysql
elif self._test_connection(__mysqlconnector):
return __mysqlconnector
else:
logging.fatal("Cannot connect to database with any available driver. Verify correct username & password in rootthebox.cfg and db dependecies.")
os._exit(1)
def register():
"""Enable encodings of the form 'latex+x' where x describes another encoding.
Unicode characters are translated to or from x when possible, otherwise
expanded to latex.
"""
codecs.register(_registry)
def _create_mysql_tables():
codecs.register(lambda name: codecs.lookup('utf8') if name == 'utf8mb4' else None)
_engine = getDatabaseEngine()
BaseModel.metadata.create_all(_engine)
# utf8mb4
_updateMessagePushTasksCharset(_engine)
return
def main():
codecs.register(
lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None
)
app = QtGui.QApplication(sys.argv)
ex = MainWindow()
sys.exit(app.exec_())
def log(self, mes_type, message):
# Remove all non-ASCII characters
# message = removeNonAsciiDrop(message)
codecs.register(lambda message: codecs.lookup('utf-8') if message == 'cp65001' else None)
if not self.debug and mes_type == "DEBUG":
return
# Counter
if mes_type == "ALERT":
self.alerts += 1
if mes_type == "WARNING":
self.warnings += 1
if mes_type == "NOTICE":
self.notices += 1
if self.only_relevant:
if mes_type not in ('ALERT', 'WARNING'):
return
# to file
if not self.no_log_file:
self.log_to_file(message, mes_type)
# to stdout
try:
self.log_to_stdout(message.encode('ascii', errors='replace'), mes_type)
except Exception, e:
print "Cannot print certain characters to command line - see log file for full unicode encoded log line"
self.log_to_stdout(removeNonAsciiDrop(message), mes_type)
def __init__(self, device, deviceStr, autodump=False, ignoreuiautomatorkilled=False):
'''
Constructor
@type device: Phone Object
@param device: The device running the C{View server} to which this client will connect
@type serialno: str
@param serialno: the serial number of the device or emulator to connect to
'''
self._device = device
self._deviceStr = deviceStr
self._autodump = autodump
self._viewParser = ViewParser(self._deviceStr)
self._ignoreuiautomatorkilled = ignoreuiautomatorkilled
self._logger = logging.getLogger('')
self._dumplogger = logging.getLogger('DUMPStats')
# self._logger.debug('adb -s %s shell /system/bin/uiautomator server' % self._deviceStr)
# host_exec('adb -s %s shell /system/bin/uiautomator server' % self._deviceStr)
#
#
# time.sleep(3)
## A workaround to alias utf8 encoding to cp65001(Microsoft's utf8)
try:
codecs.lookup('cp65001')
except:
def cp65001(name):
if name.lower() == 'cp65001':
return codecs.lookup('utf-8')
codecs.register(cp65001)
def register():
codecs.register(find_codecs)
for name, (decode, encode) in codec_info.items():
if name in _codecs:
continue
_codecs[name] = CharmapCodec(decode, encode)
def init_db():
# 数据库包装这一层，存在大量的encode操作，使用的是database的charset，在数据库端指定了使用utf8mb4编码，那么外部模块与
# 数据库连接时，要使用相同的编码，修改数据库数据时也应该使用相同的编码。
# 而Python的codec模块不知道utf8mb4这种表述，所以需要使用别名。make python understand 'utf8mb4' as an alias for 'utf8'。
codecs.register(lambda name: codecs.lookup('utf8') if name == 'utf8mb4' else None)
# 创建对应的数据库表。
engine = DBUtil.create_engine()
DBUtil.__BASE_MODEL.metadata.create_all(engine)
def register():
'''Register this codec in the standard Python codec registry. Afterwards you
can decode/encode strings using the codec name 'datev_ascii'.'''
def is_datev(codec_name):
if codec_name == 'datev_ascii':
return getregentry()
return None
codecs.register(is_datev)
def register_codec_cp65001():
try:
codecs.lookup('cp65001')
except Exception:
def lookup_func(name):
if name.lower() == 'cp65001':
return codecs.lookup('utf-8')
codecs.register(lookup_func)
def RegisterCodecs():
"""The example files that are distributed with the QTI specification contain
a set of Chinese examples encoded using big5. However, the xml declarations
on these files refer to the charset as "CN-BIG5" and this causes errors when
parsing them as this is a non-standard way of refering to big5.
QTI also requires use of the apple symbol font mapping for interpreting
symbol-encoded maths text in questions."""
codecs.register(QTICodecSearch)
def fix_win_codec():
"""Works around <http://bugs.python.org/issue6058>."""
# <http://msdn.microsoft.com/en-us/library/dd317756.aspx>
try:
codecs.lookup("cp65001")
return False
except LookupError:
codecs.register(lambda name: name == "cp65001" and codecs.lookup("utf-8") or None)
return True
def add_encoding_alias(encoding, alias, overwrite=False):
try:
codecs.lookup(alias)
alias_exists = True
except LookupError:
alias_exists = False
if overwrite or not alias_exists:
codec = codecs.lookup(encoding)
codecs.register(lambda x: codec if x == alias else None)
def register(self):
"""Register spec codec"""
class StreamReader(utf_8.StreamReader):
"""Used by cPython to deal with a spec file"""
def __init__(sr, stream, *args, **kwargs):
codecs.StreamReader.__init__(sr, stream, *args, **kwargs)
data = self.dealwith(sr.stream.readline)
sr.stream = cStringIO.StringIO(data)
def decode(text, *args):
"""Used by pypy and pylint to deal with a spec file"""
buffered = cStringIO.StringIO(text)
# Determine if we need to have imports for this string
# It may be a fragment of the file
has_spec = regexes['encoding_matcher'].search(buffered.readline())
no_imports = not has_spec
buffered.seek(0)
# Translate the text
utf8 = encodings.search_function('utf8') # Assume utf8 encoding
reader = utf8.streamreader(buffered)
data = self.dealwith(reader.readline, no_imports=no_imports)
# If nothing was changed, then we want to use the original file/line
# Also have to replace indentation of original line with indentation of new line
# To take into account nested describes
if text and not regexes['only_whitespace'].match(text):
if regexes['whitespace'].sub('', text) == regexes['whitespace'].sub('', data):
bad_indentation = regexes['leading_whitespace'].search(text).groups()[0]
good_indentation = regexes['leading_whitespace'].search(data).groups()[0]
data = '%s%s' % (good_indentation, text[len(bad_indentation):])
# If text is empty and data isn't, then we should return text
if len(text) == 0 and len(data) == 1:
return unicode(text), 0
# Return translated version and it's length
return unicode(data), len(data)
def search_function(s):
"""Determine if a file is of spec encoding and return special CodecInfo if it is"""
if s != 'spec': return None
utf8 = encodings.search_function('utf8') # Assume utf8 encoding
return codecs.CodecInfo(
name='spec'
, encode=utf8.encode
, decode=decode
, streamreader=StreamReader
, streamwriter=utf8.streamwriter
, incrementalencoder=utf8.incrementalencoder
, incrementaldecoder=utf8.incrementaldecoder
)
# Do the register
codecs.register(search_function)
def add(self, encoding_name):
"""Adding encoding name to fake.
:type encoding_name: lowercase plain string
"""
if not self._registered:
codecs.register(self)
self._registered = True
if encoding_name is not None:
self._enabled_encodings.add(encoding_name)
def register_cp65001():
"""Register cp65001 so that PY2 knowns about it."""
import codecs
try:
codecs.lookup('cp65001')
return False
except LookupError:
codecs.register(
lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
return True
def register_sark_codecs():
import codecs
from .encodings.hex_bytes import getregentry
def sark_search_function(encoding):
codec_info = getregentry()
if encoding == codec_info.name:
return codec_info
codecs.register(sark_search_function)
def mock_mbcs_windows():
try:
codecs.lookup('mbcs')
except LookupError:
ascii_codec = codecs.lookup('ascii')
def lookup_func(name, encoding=ascii_codec):
return {True: encoding}.get(name == 'mbcs')
codecs.register(lookup_func)
def test_codecs_errors(self):
# Error handling (encoding)
self.assertRaises(UnicodeError, u"Andr\202 x".encode, "ascii")
self.assertRaises(UnicodeError, u"Andr\202 x".encode, "ascii", "strict")
self.assertEqual(u"Andr\202 x".encode("ascii", "ignore"), "Andr x")
self.assertEqual(u"Andr\202 x".encode("ascii", "replace"), "Andr? x")
# Error handling (decoding)
self.assertRaises(UnicodeError, unicode, "Andr\202 x", "ascii")
self.assertRaises(UnicodeError, unicode, "Andr\202 x", "ascii", "strict")
self.assertEqual(unicode("Andr\202 x", "ascii", "ignore"), u"Andr x")
self.assertEqual(unicode("Andr\202 x", "ascii", "replace"), u"Andr\uFFFD x")
# Error handling (unknown character names)
self.assertEqual("\\N{foo}xx".decode("unicode-escape", "ignore"), u"xx")
# Error handling (truncated escape sequence)
self.assertRaises(UnicodeError, "\\".decode, "unicode-escape")
# Error handling (bad decoder return)
def search_function(encoding):
def decode1(input, errors="strict"):
return 42 # not a tuple
def encode1(input, errors="strict"):
return 42 # not a tuple
def encode2(input, errors="strict"):
return (42, 42) # no unicode
def decode2(input, errors="strict"):
return (42, 42) # no unicode
if encoding == "test.unicode1":
return (encode1, decode1, None, None)
elif encoding == "test.unicode2":
return (encode2, decode2, None, None)
else:
return None
codecs.register(search_function)
self.assertRaises(TypeError, "hello".decode, "test.unicode1")
self.assertRaises(TypeError, unicode, "hello", "test.unicode2")
self.assertRaises(TypeError, u"hello".encode, "test.unicode1")
self.assertRaises(TypeError, u"hello".encode, "test.unicode2")
# executes PyUnicode_Encode()
import imp
self.assertRaises(ImportError, imp.find_module, "non-existing module", [u"non-existing dir"])
# Error handling (wrong arguments)
self.assertRaises(TypeError, u"hello".encode, 42, 42, 42)
# Error handling (PyUnicode_EncodeDecimal())
self.assertRaises(UnicodeError, int, u"\u0200")
def mbcs_work_around():
'''
work around for mbcs codec to make "bdist_wininst" work
https://mail.python.org/pipermail/python-list/2012-February/620326.html
'''
import codecs
try:
codecs.lookup('mbcs')
except LookupError:
ascii = codecs.lookup('ascii')
codecs.register(lambda name: {True: ascii}.get(name == 'mbcs'))
def __init__(self, separate_tti, clear_uda, discard_user_data):
self.separate_tti = separate_tti
self.clear_uda = clear_uda
self.discard_user_data = discard_user_data
self.tti = []
# register all encodings that can be used for the Text Fields of the TTI blocks in STL
codecs.register(iso6937().search)
codecs.register(iso8859_5_stl().search)
codecs.register(iso8859_6_stl().search)
codecs.register(iso8859_7_stl().search)
codecs.register(iso8859_8_stl().search)
def initbotscharsets():
'''set up right charset handling for specific charsets (UNOA, UNOB, UNOC, etc).'''
#tell python how to search a codec defined by bots. Bots searches for this in usersys/charset
codecs.register(codec_search_function)
#syntax has parameters checkcharsetin or checkcharsetout. These can have value 'botsreplace'
#eg: 'checkcharsetin':'botsreplace', #strict, ignore or botsreplace
#in case of errors: the 'wrong' character is replaced with char as set in bots.ini. Default value in bots.ini is ' ' (space)
botsglobal.botsreplacechar = unicode(botsglobal.ini.get('settings','botsreplacechar',' '))
codecs.register_error('botsreplace', botsreplacechar_handler) #need to register the handler for botsreplacechar
#set aliases for the charsets in bots.ini
for key, value in botsglobal.ini.items('charsets'):
encodings.aliases.aliases[key] = value
def main():
#python dont know windows's cp65001 = utf-8
try:
codecs.lookup('cp65001')
except:
def cp65001(name):
if name.lower() == 'cp65001':
return codecs.lookup('utf-8')
codecs.register(cp65001)
try:
i = 0
for root, dirs, files in os.walk(FilePath):
#print root
filename = ""
for fff in files:
print os.path.join(root, fff)
ff = codecs.open(os.path.join(root, fff), encoding='utf-16')
for line in ff:
try:
i = i + 1
print i
f = codecs.open(FilePath_Target +"PPT_PoliticMan_" + now.strftime("%Y%m%d%H%M") + "_" + str( os.getpid() )+ ".txt", "w+", "utf-8")
page_url = line.rstrip()
resp = urllib2.urlopen(page_url , timeout=10)
soup = BeautifulSoup(resp)
txtWrite = getpageContext(soup,page_url)
f.writelines(txtWrite)
resp.close()
except:
print "Unexpected error:", sys.exc_info()[0]
print "Unexpected error:", sys.exc_info()[1]
PrintException()
pass
ff.close()
#delete file
try:
filename = os.path.join(root, fff)
if os.path.isfile(filename):
os.remove(filename)
except:
print "Unexpected error:", sys.exc_info()[0]
print "Unexpected error:", sys.exc_info()[1]
PrintException()
pass
except:
#continue
print "Unexpected error:", sys.exc_info()[0]
print "Unexpected error:", sys.exc_info()[1]
PrintException()
pass
def __init__(self, url, depth=0, index=0):
logging.basicConfig(filename='darts.log', level=logging.WARNING)
assert isinstance(depth, int), "depth must be integer"
self._index = _counter(index)
self._depth = depth
self._src_url = url
self._base_url = "{0.scheme}://{0.netloc}/".format(urlsplit(url))
self._urls = self._get_internal_links(url)
self._darts = set()
self._clf_address = ClfAddress()
for search_function in encoding.search_functions():
codecs.register(search_function)
def _real_main(argv=None):
# Compatibility fixes for Windows
if sys.platform == 'win32':
# https://github.com/ytdl-org/youtube-dl/issues/820
codecs.register(lambda name: codecs.lookup('utf-8')
if name == 'cp65001' else None)
workaround_optparse_bug9161()
setproctitle('youtube-dl')
parser, opts, args = parseOpts(argv)
# Set user agent
if opts.user_agent is not None:
std_headers['User-Agent'] = opts.user_agent
# Set referer
if opts.referer is not None:
std_headers['Referer'] = opts.referer
# Custom HTTP headers
if opts.headers is not None:
for h in opts.headers:
if ':' not in h:
parser.error(
'wrong header formatting, it should be key:value, not "%s"'
% h)
key, value = h.split(':', 1)
if opts.verbose:
write_string(
'[debug] Adding header from command line option %s:%s\n' %
(key, value))
std_headers[key] = value
# Dump user agent
if opts.dump_user_agent:
write_string(std_headers['User-Agent'] + '\n', out=sys.stdout)
sys.exit(0)
# Batch file verification
batch_urls = []
if opts.batchfile is not None:
try:
if opts.batchfile == '-':
batchfd = sys.stdin
else:
batchfd = io.open(expand_path(opts.batchfile),
'r',
encoding='utf-8',
errors='ignore')
batch_urls = read_batch_urls(batchfd)
if opts.verbose:
write_string('[debug] Batch file urls: ' + repr(batch_urls) +
'\n')
except IOError:
sys.exit('ERROR: batch file %s could not be read' % opts.batchfile)
all_urls = batch_urls + [
url.strip() for url in args
] # batch_urls are already striped in read_batch_urls
_enc = preferredencoding()
all_urls = [
url.decode(_enc, 'ignore') if isinstance(url, bytes) else url
for url in all_urls
]
if opts.list_extractors:
for ie in list_extractors(opts.age_limit):
write_string(ie.IE_NAME +
(' (CURRENTLY BROKEN)' if not ie._WORKING else '') +
'\n',
out=sys.stdout)
matchedUrls = [url for url in all_urls if ie.suitable(url)]
for mu in matchedUrls:
write_string(' ' + mu + '\n', out=sys.stdout)
sys.exit(0)
if opts.list_extractor_descriptions:
for ie in list_extractors(opts.age_limit):
if not ie._WORKING:
continue
desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
if desc is False:
continue
if hasattr(ie, 'SEARCH_KEY'):
_SEARCHES = ('cute kittens', 'slithering pythons',
'falling cat', 'angry poodle', 'purple fish',
'running tortoise', 'sleeping bunny',
'burping cow')
_COUNTS = ('', '5', '10', 'all')
desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY,
random.choice(_COUNTS),
random.choice(_SEARCHES))
write_string(desc + '\n', out=sys.stdout)
sys.exit(0)
if opts.ap_list_mso:
table = [[mso_id, mso_info['name']]
for mso_id, mso_info in MSO_INFO.items()]
write_string('Supported TV Providers:\n' +
render_table(['mso', 'mso name'], table) + '\n',
out=sys.stdout)
sys.exit(0)
# Conflicting, missing and erroneous options
if opts.usenetrc and (opts.username is not None
or opts.password is not None):
parser.error('using .netrc conflicts with giving username/password')
if opts.password is not None and opts.username is None:
parser.error('account username missing\n')
if opts.ap_password is not None and opts.ap_username is None:
parser.error('TV Provider account username missing\n')
if opts.outtmpl is not None and (opts.usetitle or opts.autonumber
or opts.useid):
parser.error(
'using output template conflicts with using title, video ID or auto number'
)
if opts.autonumber_size is not None:
if opts.autonumber_size <= 0:
parser.error('auto number size must be positive')
if opts.autonumber_start is not None:
if opts.autonumber_start < 0:
parser.error('auto number start must be positive or 0')
if opts.usetitle and opts.useid:
parser.error('using title conflicts with using video ID')
if opts.username is not None and opts.password is None:
opts.password = compat_getpass(
'Type account password and press [Return]: ')
if opts.ap_username is not None and opts.ap_password is None:
opts.ap_password = compat_getpass(
'Type TV provider account password and press [Return]: ')
if opts.ratelimit is not None:
numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
if numeric_limit is None:
parser.error('invalid rate limit specified')
opts.ratelimit = numeric_limit
if opts.min_filesize is not None:
numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
if numeric_limit is None:
parser.error('invalid min_filesize specified')
opts.min_filesize = numeric_limit
if opts.max_filesize is not None:
numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
if numeric_limit is None:
parser.error('invalid max_filesize specified')
opts.max_filesize = numeric_limit
if opts.sleep_interval is not None:
if opts.sleep_interval < 0:
parser.error('sleep interval must be positive or 0')
if opts.max_sleep_interval is not None:
if opts.max_sleep_interval < 0:
parser.error('max sleep interval must be positive or 0')
if opts.sleep_interval is None:
parser.error(
'min sleep interval must be specified, use --min-sleep-interval'
)
if opts.max_sleep_interval < opts.sleep_interval:
parser.error(
'max sleep interval must be greater than or equal to min sleep interval'
)
else:
opts.max_sleep_interval = opts.sleep_interval
if opts.ap_mso and opts.ap_mso not in MSO_INFO:
parser.error(
'Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers'
)
def parse_retries(retries):
if retries in ('inf', 'infinite'):
parsed_retries = float('inf')
else:
try:
parsed_retries = int(retries)
except (TypeError, ValueError):
parser.error('invalid retry count specified')
return parsed_retries
if opts.retries is not None:
opts.retries = parse_retries(opts.retries)
if opts.fragment_retries is not None:
opts.fragment_retries = parse_retries(opts.fragment_retries)
if opts.buffersize is not None:
numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
if numeric_buffersize is None:
parser.error('invalid buffer size specified')
opts.buffersize = numeric_buffersize
if opts.http_chunk_size is not None:
numeric_chunksize = FileDownloader.parse_bytes(opts.http_chunk_size)
if not numeric_chunksize:
parser.error('invalid http chunk size specified')
opts.http_chunk_size = numeric_chunksize
if opts.playliststart <= 0:
raise ValueError('Playlist start must be positive')
if opts.playlistend not in (
-1, None) and opts.playlistend < opts.playliststart:
raise ValueError('Playlist end must be greater than playlist start')
if opts.extractaudio:
if opts.audioformat not in [
'best', 'aac', 'flac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav'
]:
parser.error('invalid audio format specified')
if opts.audioquality:
opts.audioquality = opts.audioquality.strip('k').strip('K')
if not opts.audioquality.isdigit():
parser.error('invalid audio quality specified')
if opts.recodevideo is not None:
if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv', 'avi']:
parser.error('invalid video recode format specified')
if opts.convertsubtitles is not None:
if opts.convertsubtitles not in ['srt', 'vtt', 'ass', 'lrc']:
parser.error('invalid subtitle format specified')
if opts.date is not None:
date = DateRange.day(opts.date)
else:
date = DateRange(opts.dateafter, opts.datebefore)
# Do not download videos when there are audio-only formats
if opts.extractaudio and not opts.keepvideo and opts.format is None:
opts.format = 'bestaudio/best'
# --all-sub automatically sets --write-sub if --write-auto-sub is not given
# this was the old behaviour if only --all-sub was given.
if opts.allsubtitles and not opts.writeautomaticsub:
opts.writesubtitles = True
outtmpl = ((opts.outtmpl is not None and opts.outtmpl)
or (opts.format == '-1' and opts.usetitle
and '%(title)s-%(id)s-%(format)s.%(ext)s')
or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s')
or (opts.usetitle and opts.autonumber
and '%(autonumber)s-%(title)s-%(id)s.%(ext)s')
or (opts.usetitle and '%(title)s-%(id)s.%(ext)s')
or (opts.useid and '%(id)s.%(ext)s')
or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s')
or DEFAULT_OUTTMPL)
if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
parser.error('Cannot download a video and extract audio into the same'
' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
' template'.format(outtmpl))
any_getting = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
any_printing = opts.print_json
download_archive_fn = expand_path(
opts.download_archive
) if opts.download_archive is not None else opts.download_archive
# PostProcessors
postprocessors = []
if opts.metafromtitle:
postprocessors.append({
'key': 'MetadataFromTitle',
'titleformat': opts.metafromtitle
})
if opts.extractaudio:
postprocessors.append({
'key': 'FFmpegExtractAudio',
'preferredcodec': opts.audioformat,
'preferredquality': opts.audioquality,
'nopostoverwrites': opts.nopostoverwrites,
})
if opts.recodevideo:
postprocessors.append({
'key': 'FFmpegVideoConvertor',
'preferedformat': opts.recodevideo,
})
# FFmpegMetadataPP should be run after FFmpegVideoConvertorPP and
# FFmpegExtractAudioPP as containers before conversion may not support
# metadata (3gp, webm, etc.)
# And this post-processor should be placed before other metadata
# manipulating post-processors (FFmpegEmbedSubtitle) to prevent loss of
# extra metadata. By default ffmpeg preserves metadata applicable for both
# source and target containers. From this point the container won't change,
# so metadata can be added here.
if opts.addmetadata:
postprocessors.append({'key': 'FFmpegMetadata'})
if opts.convertsubtitles:
postprocessors.append({
'key': 'FFmpegSubtitlesConvertor',
'format': opts.convertsubtitles,
})
if opts.embedsubtitles:
postprocessors.append({
'key': 'FFmpegEmbedSubtitle',
})
if opts.embedthumbnail:
already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails
postprocessors.append({
'key': 'EmbedThumbnail',
'already_have_thumbnail': already_have_thumbnail
})
if not already_have_thumbnail:
opts.writethumbnail = True
# XAttrMetadataPP should be run after post-processors that may change file
# contents
if opts.xattrs:
postprocessors.append({'key': 'XAttrMetadata'})
# Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
# So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
if opts.exec_cmd:
postprocessors.append({
'key': 'ExecAfterDownload',
'exec_cmd': opts.exec_cmd,
})
external_downloader_args = None
if opts.external_downloader_args:
external_downloader_args = compat_shlex_split(
opts.external_downloader_args)
postprocessor_args = None
if opts.postprocessor_args:
postprocessor_args = compat_shlex_split(opts.postprocessor_args)
match_filter = (None if opts.match_filter is None else match_filter_func(
opts.match_filter))
ydl_opts = {
'usenetrc': opts.usenetrc,
'username': opts.username,
'password': opts.password,
'twofactor': opts.twofactor,
'videopassword': opts.videopassword,
'ap_mso': opts.ap_mso,
'ap_username': opts.ap_username,
'ap_password': opts.ap_password,
'quiet': (opts.quiet or any_getting or any_printing),
'no_warnings': opts.no_warnings,
'forceurl': opts.geturl,
'forcetitle': opts.gettitle,
'forceid': opts.getid,
'forcethumbnail': opts.getthumbnail,
'forcedescription': opts.getdescription,
'forceduration': opts.getduration,
'forcefilename': opts.getfilename,
'forceformat': opts.getformat,
'forcejson': opts.dumpjson or opts.print_json,
'dump_single_json': opts.dump_single_json,
'simulate': opts.simulate or any_getting,
'skip_download': opts.skip_download,
'format': opts.format,
'listformats': opts.listformats,
'outtmpl': outtmpl,
'autonumber_size': opts.autonumber_size,
'autonumber_start': opts.autonumber_start,
'restrictfilenames': opts.restrictfilenames,
'ignoreerrors': opts.ignoreerrors,
'force_generic_extractor': opts.force_generic_extractor,
'ratelimit': opts.ratelimit,
'nooverwrites': opts.nooverwrites,
'retries': opts.retries,
'fragment_retries': opts.fragment_retries,
'skip_unavailable_fragments': opts.skip_unavailable_fragments,
'keep_fragments': opts.keep_fragments,
'buffersize': opts.buffersize,
'noresizebuffer': opts.noresizebuffer,
'http_chunk_size': opts.http_chunk_size,
'continuedl': opts.continue_dl,
'noprogress': opts.noprogress,
'progress_with_newline': opts.progress_with_newline,
'playliststart': opts.playliststart,
'playlistend': opts.playlistend,
'playlistreverse': opts.playlist_reverse,
'playlistrandom': opts.playlist_random,
'noplaylist': opts.noplaylist,
'logtostderr': opts.outtmpl == '-',
'consoletitle': opts.consoletitle,
'nopart': opts.nopart,
'updatetime': opts.updatetime,
'writedescription': opts.writedescription,
'writeannotations': opts.writeannotations,
'writeinfojson': opts.writeinfojson,
'writethumbnail': opts.writethumbnail,
'write_all_thumbnails': opts.write_all_thumbnails,
'writesubtitles': opts.writesubtitles,
'writeautomaticsub': opts.writeautomaticsub,
'allsubtitles': opts.allsubtitles,
'listsubtitles': opts.listsubtitles,
'subtitlesformat': opts.subtitlesformat,
'subtitleslangs': opts.subtitleslangs,
'matchtitle': decodeOption(opts.matchtitle),
'rejecttitle': decodeOption(opts.rejecttitle),
'max_downloads': opts.max_downloads,
'prefer_free_formats': opts.prefer_free_formats,
'verbose': opts.verbose,
'dump_intermediate_pages': opts.dump_intermediate_pages,
'write_pages': opts.write_pages,
'test': opts.test,
'keepvideo': opts.keepvideo,
'min_filesize': opts.min_filesize,
'max_filesize': opts.max_filesize,
'min_views': opts.min_views,
'max_views': opts.max_views,
'daterange': date,
'cachedir': opts.cachedir,
'youtube_print_sig_code': opts.youtube_print_sig_code,
'age_limit': opts.age_limit,
'download_archive': download_archive_fn,
'cookiefile': opts.cookiefile,
'nocheckcertificate': opts.no_check_certificate,
'prefer_insecure': opts.prefer_insecure,
'proxy': opts.proxy,
'socket_timeout': opts.socket_timeout,
'bidi_workaround': opts.bidi_workaround,
'debug_printtraffic': opts.debug_printtraffic,
'prefer_ffmpeg': opts.prefer_ffmpeg,
'include_ads': opts.include_ads,
'default_search': opts.default_search,
'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
'encoding': opts.encoding,
'extract_flat': opts.extract_flat,
'mark_watched': opts.mark_watched,
'merge_output_format': opts.merge_output_format,
'postprocessors': postprocessors,
'fixup': opts.fixup,
'source_address': opts.source_address,
'call_home': opts.call_home,
'sleep_interval': opts.sleep_interval,
'max_sleep_interval': opts.max_sleep_interval,
'external_downloader': opts.external_downloader,
'list_thumbnails': opts.list_thumbnails,
'playlist_items': opts.playlist_items,
'xattr_set_filesize': opts.xattr_set_filesize,
'match_filter': match_filter,
'no_color': opts.no_color,
'ffmpeg_location': opts.ffmpeg_location,
'hls_prefer_native': opts.hls_prefer_native,
'hls_use_mpegts': opts.hls_use_mpegts,
'external_downloader_args': external_downloader_args,
'postprocessor_args': postprocessor_args,
'cn_verification_proxy': opts.cn_verification_proxy,
'geo_verification_proxy': opts.geo_verification_proxy,
'config_location': opts.config_location,
'geo_bypass': opts.geo_bypass,
'geo_bypass_country': opts.geo_bypass_country,
'geo_bypass_ip_block': opts.geo_bypass_ip_block,
# just for deprecation check
'autonumber': opts.autonumber if opts.autonumber is True else None,
'usetitle': opts.usetitle if opts.usetitle is True else None,
}
with YoutubeDL(ydl_opts) as ydl:
# Update version
if opts.update_self:
update_self(ydl.to_screen, opts.verbose, ydl._opener)
# Remove cache dir
if opts.rm_cachedir:
ydl.cache.remove()
# Maybe do nothing
if (len(all_urls) < 1) and (opts.load_info_filename is None):
if opts.update_self or opts.rm_cachedir:
sys.exit()
ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
parser.error(
'You must provide at least one URL.\n'
'Type youtube-dl --help to see a list of all options.')
try:
if opts.load_info_filename is not None:
retcode = ydl.download_with_info_file(
expand_path(opts.load_info_filename))
else:
retcode = ydl.download(all_urls)
except MaxDownloadsReached:
ydl.to_screen('--max-download limit reached, aborting.')
retcode = 101
def get_version():
"""
extract version from lib/datax-core-<version>.jar package
"""
core_jar = glob(os.path.join(DATAX_HOME, "lib", "datax-core-*.jar"))
if not core_jar:
return ""
else:
return os.path.basename(core_jar[0])[11:-4]
DATAX_HOME = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATAX_VERSION = 'DataX ' + get_version()
if isWindows():
codecs.register(
lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None)
CLASS_PATH = "{1}{0}lib{0}*".format(os.sep, DATAX_HOME)
else:
CLASS_PATH = "{1}{0}lib{0}*:.".format(os.sep, DATAX_HOME)
LOGBACK_FILE = "{1}{0}conf{0}logback.xml".format(os.sep, DATAX_HOME)
DEFAULT_JVM = "-Xms64m -Xmx2g -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath={}".format(
DATAX_HOME)
DEFAULT_PROPERTY_CONF = "-Dfile.encoding=UTF-8 -Dlogback.statusListenerClass=ch.qos.logback.core.status.NopStatusListener \
-Djava.security.egd=file:///dev/urandom -Ddatax.home=%s -Dlogback.configurationFile=%s " % \
(DATAX_HOME, LOGBACK_FILE)
ENGINE_COMMAND = "java -server ${jvm} %s -classpath %s ${params} com.wgzhao.datax.core.Engine \
-mode ${mode} -jobid ${jobid} -job ${job}" % \
(DEFAULT_PROPERTY_CONF, "/etc/hbase/conf:" + CLASS_PATH)
REMOTE_DEBUG_CONFIG = "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=9999"
RET_STATE = {
# Log errors from the Opbeat module to the console (recommended)
'opbeat.errors': {
'level': 'ERROR',
'handlers': ['console'],
'propagate': False,
},
},
}
# FILE_UPLOAD_HANDLERS = ("django.core.files.uploadhandler.TemporaryFileUploadHandler",)
WSGI_APPLICATION = 'config.wsgi.application'
# Database
# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
codecs.register(lambda name: codecs.lookup('utf8')
if name == 'utf8mb4' else None)
if not PRODUCTION:
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
'ATOMIC_REQUESTS': True
}
}
else:
DATABASES = {
'default': {
'NAME': os.environ.get('MYSQL_INSTANCE_NAME', 'xxx'),
'ENGINE': 'django.db.backends.mysql',
'USER': os.environ.get("MYSQL_USERNAME", 'xxx'),
'PASSWORD': os.environ.get("MYSQL_PASSWORD", 'xxx'),
def _register_utf8_codec():
codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
def _real_main(argv=None):
# Compatibility fixes for Windows
if sys.platform == 'win32':
# https://github.com/rg3/youtube-dl/issues/820
codecs.register(lambda name: codecs.lookup('utf-8')
if name == 'cp65001' else None)
workaround_optparse_bug9161()
setproctitle('ananse')
parser, opts, args = parseOpts(argv)
# Set user agent
if opts.user_agent is not None:
std_headers['User-Agent'] = opts.user_agent
# Set referer
if opts.referer is not None:
std_headers['Referer'] = opts.referer
# Custom HTTP headers
if opts.headers is not None:
for h in opts.headers:
if h.find(':', 1) < 0:
parser.error(
'wrong header formatting, it should be key:value, not "%s"'
% h)
key, value = h.split(':', 2)
if opts.verbose:
write_string(
'[debug] Adding header from command line option %s:%s\n' %
(key, value))
std_headers[key] = value
# Dump user agent
if opts.dump_user_agent:
compat_print(std_headers['User-Agent'])
sys.exit(0)
# Batch file verification
batch_urls = []
if opts.batchfile is not None:
try:
if opts.batchfile == '-':
batchfd = sys.stdin
else:
batchfd = io.open(opts.batchfile,
'r',
encoding='utf-8',
errors='ignore')
batch_urls = read_batch_urls(batchfd)
if opts.verbose:
write_string('[debug] Batch file urls: ' + repr(batch_urls) +
'\n')
except IOError:
sys.exit('ERROR: batch file could not be read')
all_urls = batch_urls + args
all_urls = [url.strip() for url in all_urls]
_enc = preferredencoding()
all_urls = [
url.decode(_enc, 'ignore') if isinstance(url, bytes) else url
for url in all_urls
]
extractors = gen_extractors()
if opts.list_extractors:
for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
compat_print(ie.IE_NAME +
(' (CURRENTLY BROKEN)' if not ie._WORKING else ''))
matchedUrls = [url for url in all_urls if ie.suitable(url)]
for mu in matchedUrls:
compat_print(' ' + mu)
sys.exit(0)
if opts.list_extractor_descriptions:
for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()):
if not ie._WORKING:
continue
desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
if desc is False:
continue
if hasattr(ie, 'SEARCH_KEY'):
_SEARCHES = ('cute kittens', 'slithering pythons',
'falling cat', 'angry poodle', 'purple fish',
'running tortoise', 'sleeping bunny')
_COUNTS = ('', '5', '10', 'all')
desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY,
random.choice(_COUNTS),
random.choice(_SEARCHES))
compat_print(desc)
sys.exit(0)
# Conflicting, missing and erroneous options
if opts.usenetrc and (opts.username is not None
or opts.password is not None):
parser.error('using .netrc conflicts with giving username/password')
if opts.password is not None and opts.username is None:
parser.error('account username missing\n')
if opts.outtmpl is not None and (opts.usetitle or opts.autonumber
or opts.useid):
parser.error(
'using output template conflicts with using title, video ID or auto number'
)
if opts.usetitle and opts.useid:
parser.error('using title conflicts with using video ID')
if opts.username is not None and opts.password is None:
opts.password = compat_getpass(
'Type account password and press [Return]: ')
if opts.ratelimit is not None:
numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
if numeric_limit is None:
parser.error('invalid rate limit specified')
opts.ratelimit = numeric_limit
if opts.min_filesize is not None:
numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
if numeric_limit is None:
parser.error('invalid min_filesize specified')
opts.min_filesize = numeric_limit
if opts.max_filesize is not None:
numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
if numeric_limit is None:
parser.error('invalid max_filesize specified')
opts.max_filesize = numeric_limit
if opts.retries is not None:
try:
opts.retries = int(opts.retries)
except (TypeError, ValueError):
parser.error('invalid retry count specified')
if opts.buffersize is not None:
numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
if numeric_buffersize is None:
parser.error('invalid buffer size specified')
opts.buffersize = numeric_buffersize
if opts.playliststart <= 0:
raise ValueError('Playlist start must be positive')
if opts.playlistend not in (
-1, None) and opts.playlistend < opts.playliststart:
raise ValueError('Playlist end must be greater than playlist start')
if opts.extractaudio:
if opts.audioformat not in [
'best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav'
]:
parser.error('invalid audio format specified')
if opts.audioquality:
opts.audioquality = opts.audioquality.strip('k').strip('K')
if not opts.audioquality.isdigit():
parser.error('invalid audio quality specified')
if opts.recodevideo is not None:
if opts.recodevideo not in ['mp4', 'flv', 'webm', 'ogg', 'mkv']:
parser.error('invalid video recode format specified')
if opts.date is not None:
date = DateRange.day(opts.date)
else:
date = DateRange(opts.dateafter, opts.datebefore)
# Do not download videos when there are audio-only formats
if opts.extractaudio and not opts.keepvideo and opts.format is None:
opts.format = 'bestaudio/best'
# --all-sub automatically sets --write-sub if --write-auto-sub is not given
# this was the old behaviour if only --all-sub was given.
if opts.allsubtitles and not opts.writeautomaticsub:
opts.writesubtitles = True
if sys.version_info < (3, ):
# In Python 2, sys.argv is a bytestring (also note http://bugs.python.org/issue2128 for Windows systems)
if opts.outtmpl is not None:
opts.outtmpl = opts.outtmpl.decode(preferredencoding())
outtmpl = ((opts.outtmpl is not None and opts.outtmpl)
or (opts.format == '-1' and opts.usetitle
and '%(title)s-%(id)s-%(format)s.%(ext)s')
or (opts.format == '-1' and '%(id)s-%(format)s.%(ext)s')
or (opts.usetitle and opts.autonumber
and '%(autonumber)s-%(title)s-%(id)s.%(ext)s')
or (opts.usetitle and '%(title)s-%(id)s.%(ext)s')
or (opts.useid and '%(id)s.%(ext)s')
or (opts.autonumber and '%(autonumber)s-%(id)s.%(ext)s')
or DEFAULT_OUTTMPL)
if not os.path.splitext(outtmpl)[1] and opts.extractaudio:
parser.error('Cannot download a video and extract audio into the same'
' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
' template'.format(outtmpl))
any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
download_archive_fn = compat_expanduser(
opts.download_archive
) if opts.download_archive is not None else opts.download_archive
# PostProcessors
postprocessors = []
# Add the metadata pp first, the other pps will copy it
if opts.addmetadata:
postprocessors.append({'key': 'FFmpegMetadata'})
if opts.extractaudio:
postprocessors.append({
'key': 'FFmpegExtractAudio',
'preferredcodec': opts.audioformat,
'preferredquality': opts.audioquality,
'nopostoverwrites': opts.nopostoverwrites,
})
if opts.recodevideo:
postprocessors.append({
'key': 'FFmpegVideoConvertor',
'preferedformat': opts.recodevideo,
})
if opts.embedsubtitles:
postprocessors.append({
'key': 'FFmpegEmbedSubtitle',
'subtitlesformat': opts.subtitlesformat,
})
if opts.xattrs:
postprocessors.append({'key': 'XAttrMetadata'})
if opts.embedthumbnail:
if not opts.addmetadata:
postprocessors.append({'key': 'FFmpegAudioFix'})
postprocessors.append({'key': 'AtomicParsley'})
# Please keep ExecAfterDownload towards the bottom as it allows the user to modify the final file in any way.
# So if the user is able to remove the file before your postprocessor runs it might cause a few problems.
if opts.exec_cmd:
postprocessors.append({
'key': 'ExecAfterDownload',
'verboseOutput': opts.verbose,
'exec_cmd': opts.exec_cmd,
})
adl_opts = {
'usenetrc': opts.usenetrc,
'username': opts.username,
'password': opts.password,
'twofactor': opts.twofactor,
'videopassword': opts.videopassword,
'quiet': (opts.quiet or any_printing),
'no_warnings': opts.no_warnings,
'forceurl': opts.geturl,
'forcetitle': opts.gettitle,
'forceid': opts.getid,
'forcethumbnail': opts.getthumbnail,
'forcedescription': opts.getdescription,
'forceduration': opts.getduration,
'forcefilename': opts.getfilename,
'forceformat': opts.getformat,
'forcejson': opts.dumpjson,
'dump_single_json': opts.dump_single_json,
'simulate': opts.simulate or any_printing,
'skip_download': opts.skip_download,
'format': opts.format,
'format_limit': opts.format_limit,
'listformats': opts.listformats,
'outtmpl': outtmpl,
'autonumber_size': opts.autonumber_size,
'restrictfilenames': opts.restrictfilenames,
'ignoreerrors': opts.ignoreerrors,
'ratelimit': opts.ratelimit,
'nooverwrites': opts.nooverwrites,
'retries': opts.retries,
'buffersize': opts.buffersize,
'noresizebuffer': opts.noresizebuffer,
'continuedl': opts.continue_dl,
'noprogress': opts.noprogress,
'progress_with_newline': opts.progress_with_newline,
'playliststart': opts.playliststart,
'playlistend': opts.playlistend,
'playlistreverse': opts.playlist_reverse,
'noplaylist': opts.noplaylist,
'logtostderr': opts.outtmpl == '-',
'consoletitle': opts.consoletitle,
'nopart': opts.nopart,
'updatetime': opts.updatetime,
'writedescription': opts.writedescription,
'writeannotations': opts.writeannotations,
'writeinfojson': opts.writeinfojson,
'writethumbnail': opts.writethumbnail,
'writesubtitles': opts.writesubtitles,
'writeautomaticsub': opts.writeautomaticsub,
'allsubtitles': opts.allsubtitles,
'listsubtitles': opts.listsubtitles,
'subtitlesformat': opts.subtitlesformat,
'subtitleslangs': opts.subtitleslangs,
'matchtitle': decodeOption(opts.matchtitle),
'rejecttitle': decodeOption(opts.rejecttitle),
'max_downloads': opts.max_downloads,
'prefer_free_formats': opts.prefer_free_formats,
'verbose': opts.verbose,
'dump_intermediate_pages': opts.dump_intermediate_pages,
'write_pages': opts.write_pages,
'test': opts.test,
'keepvideo': opts.keepvideo,
'min_filesize': opts.min_filesize,
'max_filesize': opts.max_filesize,
'min_views': opts.min_views,
'max_views': opts.max_views,
'daterange': date,
'cachedir': opts.cachedir,
'youtube_print_sig_code': opts.youtube_print_sig_code,
'age_limit': opts.age_limit,
'download_archive': download_archive_fn,
'cookiefile': opts.cookiefile,
'nocheckcertificate': opts.no_check_certificate,
'prefer_insecure': opts.prefer_insecure,
'proxy': opts.proxy,
'socket_timeout': opts.socket_timeout,
'bidi_workaround': opts.bidi_workaround,
'debug_printtraffic': opts.debug_printtraffic,
'prefer_ffmpeg': opts.prefer_ffmpeg,
'include_ads': opts.include_ads,
'default_search': opts.default_search,
'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
'encoding': opts.encoding,
'exec_cmd': opts.exec_cmd,
'extract_flat': opts.extract_flat,
'postprocessors': postprocessors,
}
with AnanseDl(adl_opts) as adl:
# Update version
if opts.update_self:
update_self(adl.to_screen, opts.verbose)
# Remove cache dir
if opts.rm_cachedir:
adl.cache.remove()
# Maybe do nothing
if (len(all_urls) < 1) and (opts.load_info_filename is None):
if opts.update_self or opts.rm_cachedir:
sys.exit()
adl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
parser.error('you must provide at least one URL')
try:
if opts.load_info_filename is not None:
retcode = adl.download_with_info_file(opts.load_info_filename)
else:
retcode = adl.download(all_urls)
except MaxDownloadsReached:
adl.to_screen('--max-download limit reached, aborting.')
retcode = 101
sys.exit(retcode)
def register_pdf_encodings():
codecs.register(winansi.WinAnsiCodec.search)
codecs.register(standard.StandardCodec.search)
codecs.register(macroman.MacRomanCodec.search)
codecs.register(pdfdoc.PdfDocCodec.search)
decodes.append(c)
else:
results.append(c)
results.append(process(decodes))
return ''.join(results), len(s)
def encode_mb64(s):
s_utf7 = s.encode('utf-7')
return s_utf7[1:-1].replace('/', ',')
def decode_mb64(s):
s_utf7 = '+' + s.replace(',', '/') + '-'
return s_utf7.decode('utf-7')
codecPack = encode, decode, StreamReader, StreamWriter
try:
codecPack = codecs.CodecInfo(*codecPack, name=CODEC_NAME)
except AttributeError: # pragma: no cover
pass
def utf_7_imap4(codecName):
if codecName == CODEC_NAME:
return codecPack
codecs.register(utf_7_imap4)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
formats = [
("auto", "(default) detect file type automatically"),
("pe", "Windows PE file"),
("sc32", "32-bit shellcode"),
("sc64", "64-bit shellcode"),
("freeze", "features previously frozen by capa"),
]
format_help = ", ".join(["%s: %s" % (f[0], f[1]) for f in formats])
desc = "The FLARE team's open-source tool to identify capabilities in executable files."
epilog = textwrap.dedent("""
By default, capa uses a default set of embedded rules.
You can see the rule set here:
https://github.com/fireeye/capa-rules
To provide your own rule set, use the `-r` flag:
capa --rules /path/to/rules suspicious.exe
capa -r /path/to/rules suspicious.exe
examples:
identify capabilities in a binary
capa suspicious.exe
identify capabilities in 32-bit shellcode, see `-f` for all supported formats
capa -f sc32 shellcode.bin
report match locations
capa -v suspicious.exe
report all feature match details
capa -vv suspicious.exe
filter rules by meta fields, e.g. rule name or namespace
capa -t "create TCP socket" suspicious.exe
""")
parser = argparse.ArgumentParser(
description=desc,
epilog=epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("sample", type=str, help="path to sample to analyze")
parser.add_argument("--version",
action="version",
version="%(prog)s {:s}".format(
capa.version.__version__))
parser.add_argument(
"-r",
"--rules",
type=str,
default=RULES_PATH_DEFAULT_STRING,
help="path to rule file or directory, use embedded rules by default",
)
parser.add_argument("-f",
"--format",
choices=[f[0] for f in formats],
default="auto",
help="select sample format, %s" % format_help)
parser.add_argument("-t",
"--tag",
type=str,
help="filter on rule meta field values")
parser.add_argument("-j",
"--json",
action="store_true",
help="emit JSON instead of text")
parser.add_argument(
"-v",
"--verbose",
action="store_true",
help="enable verbose result document (no effect with --json)")
parser.add_argument(
"-vv",
"--vverbose",
action="store_true",
help="enable very verbose result document (no effect with --json)")
parser.add_argument("-d",
"--debug",
action="store_true",
help="enable debugging output on STDERR")
parser.add_argument("-q",
"--quiet",
action="store_true",
help="disable all output but errors")
parser.add_argument(
"--color",
type=str,
choices=("auto", "always", "never"),
default="auto",
help=
"enable ANSI color codes in results, default: only during interactive session",
)
args = parser.parse_args(args=argv)
if args.quiet:
logging.basicConfig(level=logging.WARNING)
logging.getLogger().setLevel(logging.WARNING)
elif args.debug:
logging.basicConfig(level=logging.DEBUG)
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
logging.getLogger().setLevel(logging.INFO)
# disable vivisect-related logging, it's verbose and not relevant for capa users
set_vivisect_log_level(logging.CRITICAL)
try:
taste = get_file_taste(args.sample)
except IOError as e:
logger.error("%s", str(e))
return -1
# py2 doesn't know about cp65001, which is a variant of utf-8 on windows
# tqdm bails when trying to render the progress bar in this setup.
# because cp65001 is utf-8, we just map that codepage to the utf-8 codec.
# see #380 and: https://stackoverflow.com/a/3259271/87207
import codecs
codecs.register(lambda name: codecs.lookup("utf-8")
if name == "cp65001" else None)
if args.rules == RULES_PATH_DEFAULT_STRING:
logger.debug("-" * 80)
logger.debug(" Using default embedded rules.")
logger.debug(
" To provide your own rules, use the form `capa.exe -r ./path/to/rules/ /path/to/mal.exe`."
)
logger.debug(" You can see the current default rule set here:")
logger.debug(" https://github.com/fireeye/capa-rules")
logger.debug("-" * 80)
if hasattr(sys, "frozen") and hasattr(sys, "_MEIPASS"):
logger.debug("detected running under PyInstaller")
rules_path = os.path.join(sys._MEIPASS, "rules")
logger.debug("default rule path (PyInstaller method): %s",
rules_path)
else:
logger.debug("detected running from source")
rules_path = os.path.join(os.path.dirname(__file__), "..", "rules")
logger.debug("default rule path (source method): %s", rules_path)
if not os.path.exists(rules_path):
# when a users installs capa via pip,
# this pulls down just the source code - not the default rules.
# i'm not sure the default rules should even be written to the library directory,
# so in this case, we require the user to use -r to specify the rule directory.
logger.error(
"default embedded rules not found! (maybe you installed capa as a library?)"
)
logger.error("provide your own rule set via the `-r` option.")
return -1
else:
rules_path = args.rules
logger.debug("using rules path: %s", rules_path)
try:
rules = get_rules(rules_path, disable_progress=args.quiet)
rules = capa.rules.RuleSet(rules)
logger.debug("successfully loaded %s rules", len(rules))
if args.tag:
rules = rules.filter_rules_by_meta(args.tag)
logger.debug("selected %s rules", len(rules))
for i, r in enumerate(rules.rules, 1):
# TODO don't display subscope rules?
logger.debug(" %d. %s", i, r)
except (IOError, capa.rules.InvalidRule, capa.rules.InvalidRuleSet) as e:
logger.error("%s", str(e))
return -1
if (args.format
== "freeze") or (args.format == "auto"
and capa.features.freeze.is_freeze(taste)):
format = "freeze"
with open(args.sample, "rb") as f:
extractor = capa.features.freeze.load(f.read())
else:
format = args.format
try:
extractor = get_extractor(args.sample,
args.format,
disable_progress=args.quiet)
except UnsupportedFormatError:
logger.error("-" * 80)
logger.error(" Input file does not appear to be a PE file.")
logger.error(" ")
logger.error(
" capa currently only supports analyzing PE files (or shellcode, when using --format sc32|sc64)."
)
logger.error(
" If you don't know the input file type, you can try using the `file` utility to guess it."
)
logger.error("-" * 80)
return -1
except UnsupportedRuntimeError:
logger.error("-" * 80)
logger.error(" Unsupported runtime or Python interpreter.")
logger.error(" ")
logger.error(
" capa supports running under Python 2.7 using Vivisect for binary analysis."
)
logger.error(
" It can also run within IDA Pro, using either Python 2.7 or 3.5+."
)
logger.error(" ")
logger.error(
" If you're seeing this message on the command line, please ensure you're running Python 2.7."
)
logger.error("-" * 80)
return -1
meta = collect_metadata(argv, args.sample, args.rules, format, extractor)
capabilities, counts = find_capabilities(rules,
extractor,
disable_progress=args.quiet)
meta["analysis"].update(counts)
if has_file_limitation(rules, capabilities):
# bail if capa encountered file limitation e.g. a packed binary
# do show the output in verbose mode, though.
if not (args.verbose or args.vverbose or args.json):
return -1
if args.color == "always":
colorama.init(strip=False)
elif args.color == "auto":
# colorama will detect:
# - when on Windows console, and fixup coloring, and
# - when not an interactive session, and disable coloring
# renderers should use coloring and assume it will be stripped out if necessary.
colorama.init()
elif args.color == "never":
colorama.init(strip=True)
else:
raise RuntimeError("unexpected --color value: " + args.color)
if args.json:
print(capa.render.render_json(meta, rules, capabilities))
elif args.vverbose:
print(capa.render.render_vverbose(meta, rules, capabilities))
elif args.verbose:
print(capa.render.render_verbose(meta, rules, capabilities))
else:
print(capa.render.render_default(meta, rules, capabilities))
colorama.deinit()
logger.debug("done.")
return 0
import gnureadline as readline
except ImportError:
try:
import gnureadline as readline
except ImportError:
readline_error = True
pass
# Set default encoding
_reload_module(sys)
#sys.setdefaultencoding(settings.DEFAULT_ENCODING)
if settings.IS_WINDOWS:
import codecs
# Reference: https://github.com/nodejs/node/issues/12786#issuecomment-298652440
codecs.register(lambda name: codecs.lookup("utf-8") if name == "cp65001" else None)
# Use Colorama to make Termcolor work on Windows too :)
init()
"""
Define HTTP User-Agent header.
"""
def user_agent_header():
# Check if defined "--random-agent" option.
if menu.options.random_agent:
if (menu.options.agent != settings.DEFAULT_USER_AGENT) or menu.options.mobile:
err_msg = "The option '--random-agent' is incompatible with option '--user-agent' or switch '--mobile'."
print(settings.print_critical_msg(err_msg))
raise SystemExit()
else:
if settings.VERBOSITY_LEVEL >= 1:
# (C)2002-2015 Chris Liechti <*****@*****.**>
#
# SPDX-License-Identifier: BSD-3-Clause
import codecs
import os
import sys
import threading
import serial
from serial.tools.list_ports import comports
from serial.tools import hexlify_codec
# pylint: disable=wrong-import-order,wrong-import-position
codecs.register(lambda c: hexlify_codec.getregentry() if c == 'hexlify' else None)
try:
raw_input
except NameError:
# pylint: disable=redefined-builtin,invalid-name
raw_input = input # in python3 it's "raw"
unichr = chr
def key_description(character):
"""generate a readable description for a key"""
ascii_code = ord(character)
if ascii_code < 32:
return 'Ctrl+{:c}'.format(ord('@') + ascii_code)
else:
def dcp_block(block_data, verbose):
bauds = ['Undefined', '100', '300', '1200']
platforms = ['CS1', 'CS2']
modulation_indicies = ['Unknown', 'Normal', 'High', 'Low']
scids = ['Unknown', 'GOES-East', 'GOES-West', 'GOES-Central',
'GOES-Test'] # Spacecraft ID's
blk_len = int.from_bytes(block_data[0x01:0x03], byteorder='little')
seq_num = int.from_bytes(block_data[0x03:0x06], byteorder='little')
flags = block_data[0x06]
baud = bauds[(flags & 0b111)] # B0-B2 of flags defines the DCP Data Rate
platform = platforms[(flags & 0b1000) >> 3] # B3 is the platform type
rx_parity = bool(
(flags & 0b10000) >> 4) # Presence of Parity Errors in DCP Data
arm_flags = block_data[0x07] # Abnormal Received Messages Flags
arm_text = parse_arm(arm_flags)
corrected_addr = hex(
int.from_bytes(block_data[0x08:0x0C], byteorder='little'))
carr_start = block_data[0x0C:0x13]
msg_end = block_data[0x13:0x1A]
sig_strength = int.from_bytes(block_data[0x1A:0x1C],
byteorder='little') & 0x03FF
freq_offset = int.from_bytes(block_data[0x1C:0x1E],
byteorder='little') & 0x3FFF
if freq_offset > 8191: # 2's complement conversion
freq_offset = freq_offset - 16384
phs_noise = int.from_bytes(block_data[0x1E:0x20],
byteorder='little') & 0x01FFF
mod_index = modulation_indicies[
(int.from_bytes(block_data[0x1E:0x20], byteorder='little') & 0x0C000)
>> 14]
good_phs = int.from_bytes(block_data[0x20:0x21], byteorder='little')
channel = int.from_bytes(block_data[0x21:0x23],
byteorder='little') & 0x03FF
spacecraft = int.from_bytes(block_data[0x21:0x23],
byteorder='little') >> 12
source_code = block_data[0x23:0x25].decode('ascii')
source_sec = block_data[0x25:0x27].decode('ascii')
codecs.register(pseudo_search_func)
dcp_data_pseudo = codecs.decode(block_data[0x27:-2],
encoding='pseudo-binary')
dcp_crc16 = int.from_bytes(block_data[-2:], byteorder='little')
calc_crc = binascii.crc_hqx(block_data[:-2], 0xFFFF)
print('\n----------[ DCP Block ]----------')
if verbose:
print('Header:')
print(' Size: {} Bytes ({} Bytes of Data)'.format(
blk_len, blk_len - 41)) # 39 Bytes for Header, 2 Bytes for CRC
print(' Seqeuence Number: {}'.format(seq_num))
print(' Flags:')
print(' Data Rate: {} Baud'.format(baud))
print(' Platform: {}'.format(platform))
print(' Parity Error? {}'.format(rx_parity))
print(' ARM Flags: {}'.format(arm_text))
print(' Corrected Address: {}'.format(corrected_addr))
print(' Carrier Start: {}'.format(bcd_to_date(carr_start)))
print(' Message End: {}'.format(bcd_to_date(msg_end)))
print(' Signal Strength: {}dBm EIRP'.format(sig_strength / 10))
print(' Frequency Offset: {}Hz'.format(freq_offset / 10))
print(' Phase Noise: {}° RMS'.format(phs_noise / 100))
print(' Modulation Index: {}'.format(mod_index))
print(' Good Phase: {}%'.format(good_phs / 2))
print(' Channel: {}'.format(channel))
print(' Spacecraft: {}'.format(scids[spacecraft]))
print(' Source Code: {}'.format(source_code))
print(' Source Secondary: {}'.format(source_sec))
if dcp_crc16 == calc_crc:
print('Block CRC: OK\n')
else:
print('CRC: FAILED\n')
else:
print('Corrected Address: {}'.format(corrected_addr))
print('Data (Pseudo-Binary): \n{}'.format(dcp_data_pseudo))
def win32_unicode_console():
import codecs
from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_int
from ctypes.wintypes import BOOL, HANDLE, DWORD, LPWSTR, LPCWSTR, LPVOID
original_stderr = sys.stderr
# Output exceptions in this code to original_stderr, so that we can at least see them
def _complain(message):
original_stderr.write(
message if isinstance(message, str) else repr(message))
original_stderr.write('\n')
codecs.register(lambda name: codecs.lookup('utf-8')
if name == 'cp65001' else None)
try:
GetStdHandle = WINFUNCTYPE(HANDLE,
DWORD)(("GetStdHandle", windll.kernel32))
STD_OUTPUT_HANDLE = DWORD(-11)
STD_ERROR_HANDLE = DWORD(-12)
GetFileType = WINFUNCTYPE(DWORD,
DWORD)(("GetFileType", windll.kernel32))
FILE_TYPE_CHAR = 0x0002
FILE_TYPE_REMOTE = 0x8000
GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))(
("GetConsoleMode", windll.kernel32))
INVALID_HANDLE_VALUE = DWORD(-1).value
def not_a_console(handle):
if handle == INVALID_HANDLE_VALUE or handle is None:
return True
return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
or GetConsoleMode(handle, byref(DWORD())) == 0)
old_stdout_fileno = None
old_stderr_fileno = None
if hasattr(sys.stdout, 'fileno'):
old_stdout_fileno = sys.stdout.fileno()
if hasattr(sys.stderr, 'fileno'):
old_stderr_fileno = sys.stderr.fileno()
STDOUT_FILENO = 1
STDERR_FILENO = 2
real_stdout = (old_stdout_fileno == STDOUT_FILENO)
real_stderr = (old_stderr_fileno == STDERR_FILENO)
if real_stdout:
hStdout = GetStdHandle(STD_OUTPUT_HANDLE)
if not_a_console(hStdout):
real_stdout = False
if real_stderr:
hStderr = GetStdHandle(STD_ERROR_HANDLE)
if not_a_console(hStderr):
real_stderr = False
if real_stdout or real_stderr:
WriteConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD,
POINTER(DWORD), LPVOID)(
("WriteConsoleW", windll.kernel32))
class UnicodeOutput:
def __init__(self, hConsole, stream, fileno, name):
self._hConsole = hConsole
self._stream = stream
self._fileno = fileno
self.closed = False
self.softspace = False
self.mode = 'w'
self.encoding = 'utf-8'
self.name = name
self.flush()
def isatty(self):
return False
def close(self):
# don't really close the handle, that would only cause problems
self.closed = True
def fileno(self):
return self._fileno
def flush(self):
if self._hConsole is None:
try:
self._stream.flush()
except Exception as e:
_complain("%s.flush: %r from %r" %
(self.name, e, self._stream))
raise
def write(self, text):
try:
if self._hConsole is None:
if not PY3K and isinstance(text, unicode):
text = text.encode('utf-8')
elif PY3K and isinstance(text, str):
text = text.encode('utf-8')
self._stream.write(text)
else:
if not PY3K and not isinstance(text, unicode):
text = str(text).decode('utf-8')
elif PY3K and not isinstance(text, str):
text = text.decode('utf-8')
remaining = len(text)
while remaining:
n = DWORD(0)
# There is a shorter-than-documented limitation on the
# length of the string passed to WriteConsoleW (see
# <http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1232>.
retval = WriteConsoleW(self._hConsole, text,
min(remaining, 10000),
byref(n), None)
if retval == 0 or n.value == 0:
raise IOError(
"WriteConsoleW returned %r, n.value = %r"
% (retval, n.value))
remaining -= n.value
if not remaining:
break
text = text[n.value:]
except Exception as e:
_complain("%s.write: %r" % (self.name, e))
raise
def writelines(self, lines):
try:
for line in lines:
self.write(line)
except Exception as e:
_complain("%s.writelines: %r" % (self.name, e))
raise
if real_stdout:
sys.stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO,
'<Unicode console stdout>')
else:
sys.stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno,
'<Unicode redirected stdout>')
if real_stderr:
sys.stderr = UnicodeOutput(hStderr, None, STDERR_FILENO,
'<Unicode console stderr>')
else:
sys.stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno,
'<Unicode redirected stderr>')
except Exception as e:
_complain("exception %r while fixing up sys.stdout and sys.stderr" %
(e, ))
else:
r.append(modified_unbase64(''.join(decode[1:])))
decode = []
elif decode:
decode.append(c)
else:
r.append(c)
if decode:
r.append(modified_unbase64(''.join(decode[1:])))
bin_str = ''.join(r)
return bin_str, len(s)
class StreamReader(codecs.StreamReader):
def decode(self, s, errors='strict'):
return decoder(s)
class StreamWriter(codecs.StreamWriter):
def decode(self, s, errors='strict'):
return encoder(s)
def imap4_utf_7(name):
if name == 'imap4-utf-7':
return encoder, decoder, StreamReader, StreamWriter
codecs.register(imap4_utf_7)
import struct
import subprocess
import sys
if sys.version_info < (2, 7):
sys.stderr.write('You need Python 2.7 or later\n')
sys.exit(1)
# Setup imports depending on whether IronPython or CPython
if sys.platform == 'win32' \
or sys.platform == 'cli':
from _winreg import *
# Fix "Unknown encoding" error when running on Powershell
# https://stackoverflow.com/a/3259271/3354920
codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
def bytetohex(data):
if sys.version_info > (3, 0):
# Python 3 code in this block
return "".join("{:02X} ".format(c) for c in data)
else:
# Python 2 code in this block
return "".join("{:02X} ".format(ord(c)) for c in data)
def joinpath(folder, filename):
return os.path.join(folder, filename)
def printkey(i, offset, smc_key, smc_data):
# aliases)
try:
codecaliases = mod.getaliases()
except AttributeError:
pass
else:
for alias in codecaliases:
if alias not in _aliases:
_aliases[alias] = modname
# Return the registry entry
return entry
# Register the search_function in the Python codec registry
codecs.register(search_function)
if sys.platform == 'win32':
def _alias_mbcs(encoding):
try:
import _winapi
ansi_code_page = "cp%s" % _winapi.GetACP()
if encoding == ansi_code_page:
import encodings.mbcs
return encodings.mbcs.getregentry()
except ImportError:
# Imports may fail while we are shutting down
pass
codecs.register(_alias_mbcs)
def _real_main(argv=None):
# Compatibility fixes for Windows
if sys.platform == 'win32':
# https://github.com/ytdl-org/youtube-dl/issues/820
codecs.register(lambda name: codecs.lookup('utf-8')
if name == 'cp65001' else None)
workaround_optparse_bug9161()
setproctitle('yt-dlp')
parser, opts, args = parseOpts(argv)
warnings = []
# Set user agent
if opts.user_agent is not None:
std_headers['User-Agent'] = opts.user_agent
# Set referer
if opts.referer is not None:
std_headers['Referer'] = opts.referer
# Custom HTTP headers
std_headers.update(opts.headers)
# Dump user agent
if opts.dump_user_agent:
write_string(std_headers['User-Agent'] + '\n', out=sys.stdout)
sys.exit(0)
# Batch file verification
batch_urls = []
if opts.batchfile is not None:
try:
if opts.batchfile == '-':
batchfd = sys.stdin
else:
batchfd = io.open(expand_path(opts.batchfile),
'r',
encoding='utf-8',
errors='ignore')
batch_urls = read_batch_urls(batchfd)
if opts.verbose:
write_string('[debug] Batch file urls: ' + repr(batch_urls) +
'\n')
except IOError:
sys.exit('ERROR: batch file %s could not be read' % opts.batchfile)
all_urls = batch_urls + [
url.strip() for url in args
] # batch_urls are already striped in read_batch_urls
_enc = preferredencoding()
all_urls = [
url.decode(_enc, 'ignore') if isinstance(url, bytes) else url
for url in all_urls
]
if opts.list_extractors:
for ie in list_extractors(opts.age_limit):
write_string(ie.IE_NAME +
(' (CURRENTLY BROKEN)' if not ie._WORKING else '') +
'\n',
out=sys.stdout)
matchedUrls = [url for url in all_urls if ie.suitable(url)]
for mu in matchedUrls:
write_string(' ' + mu + '\n', out=sys.stdout)
sys.exit(0)
if opts.list_extractor_descriptions:
for ie in list_extractors(opts.age_limit):
if not ie._WORKING:
continue
desc = getattr(ie, 'IE_DESC', ie.IE_NAME)
if desc is False:
continue
if hasattr(ie, 'SEARCH_KEY'):
_SEARCHES = ('cute kittens', 'slithering pythons',
'falling cat', 'angry poodle', 'purple fish',
'running tortoise', 'sleeping bunny',
'burping cow')
_COUNTS = ('', '5', '10', 'all')
desc += ' (Example: "%s%s:%s" )' % (ie.SEARCH_KEY,
random.choice(_COUNTS),
random.choice(_SEARCHES))
write_string(desc + '\n', out=sys.stdout)
sys.exit(0)
if opts.ap_list_mso:
table = [[mso_id, mso_info['name']]
for mso_id, mso_info in MSO_INFO.items()]
write_string('Supported TV Providers:\n' +
render_table(['mso', 'mso name'], table) + '\n',
out=sys.stdout)
sys.exit(0)
# Conflicting, missing and erroneous options
if opts.usenetrc and (opts.username is not None
or opts.password is not None):
parser.error('using .netrc conflicts with giving username/password')
if opts.password is not None and opts.username is None:
parser.error('account username missing\n')
if opts.ap_password is not None and opts.ap_username is None:
parser.error('TV Provider account username missing\n')
if opts.autonumber_size is not None:
if opts.autonumber_size <= 0:
parser.error('auto number size must be positive')
if opts.autonumber_start is not None:
if opts.autonumber_start < 0:
parser.error('auto number start must be positive or 0')
if opts.username is not None and opts.password is None:
opts.password = compat_getpass(
'Type account password and press [Return]: ')
if opts.ap_username is not None and opts.ap_password is None:
opts.ap_password = compat_getpass(
'Type TV provider account password and press [Return]: ')
if opts.ratelimit is not None:
numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
if numeric_limit is None:
parser.error('invalid rate limit specified')
opts.ratelimit = numeric_limit
if opts.min_filesize is not None:
numeric_limit = FileDownloader.parse_bytes(opts.min_filesize)
if numeric_limit is None:
parser.error('invalid min_filesize specified')
opts.min_filesize = numeric_limit
if opts.max_filesize is not None:
numeric_limit = FileDownloader.parse_bytes(opts.max_filesize)
if numeric_limit is None:
parser.error('invalid max_filesize specified')
opts.max_filesize = numeric_limit
if opts.sleep_interval is not None:
if opts.sleep_interval < 0:
parser.error('sleep interval must be positive or 0')
if opts.max_sleep_interval is not None:
if opts.max_sleep_interval < 0:
parser.error('max sleep interval must be positive or 0')
if opts.sleep_interval is None:
parser.error(
'min sleep interval must be specified, use --min-sleep-interval'
)
if opts.max_sleep_interval < opts.sleep_interval:
parser.error(
'max sleep interval must be greater than or equal to min sleep interval'
)
else:
opts.max_sleep_interval = opts.sleep_interval
if opts.sleep_interval_subtitles is not None:
if opts.sleep_interval_subtitles < 0:
parser.error('subtitles sleep interval must be positive or 0')
if opts.sleep_interval_requests is not None:
if opts.sleep_interval_requests < 0:
parser.error('requests sleep interval must be positive or 0')
if opts.ap_mso and opts.ap_mso not in MSO_INFO:
parser.error(
'Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers'
)
if opts.overwrites: # --yes-overwrites implies --no-continue
opts.continue_dl = False
if opts.concurrent_fragment_downloads <= 0:
raise ValueError('Concurrent fragments must be positive')
def parse_retries(retries, name=''):
if retries in ('inf', 'infinite'):
parsed_retries = float('inf')
else:
try:
parsed_retries = int(retries)
except (TypeError, ValueError):
parser.error('invalid %sretry count specified' % name)
return parsed_retries
if opts.retries is not None:
opts.retries = parse_retries(opts.retries)
if opts.fragment_retries is not None:
opts.fragment_retries = parse_retries(opts.fragment_retries,
'fragment ')
if opts.extractor_retries is not None:
opts.extractor_retries = parse_retries(opts.extractor_retries,
'extractor ')
if opts.buffersize is not None:
numeric_buffersize = FileDownloader.parse_bytes(opts.buffersize)
if numeric_buffersize is None:
parser.error('invalid buffer size specified')
opts.buffersize = numeric_buffersize
if opts.http_chunk_size is not None:
numeric_chunksize = FileDownloader.parse_bytes(opts.http_chunk_size)
if not numeric_chunksize:
parser.error('invalid http chunk size specified')
opts.http_chunk_size = numeric_chunksize
if opts.playliststart <= 0:
raise ValueError('Playlist start must be positive')
if opts.playlistend not in (
-1, None) and opts.playlistend < opts.playliststart:
raise ValueError('Playlist end must be greater than playlist start')
if opts.extractaudio:
if opts.audioformat not in ['best'] + list(
FFmpegExtractAudioPP.SUPPORTED_EXTS):
parser.error('invalid audio format specified')
if opts.audioquality:
opts.audioquality = opts.audioquality.strip('k').strip('K')
if not opts.audioquality.isdigit():
parser.error('invalid audio quality specified')
if opts.recodevideo is not None:
opts.recodevideo = opts.recodevideo.replace(' ', '')
if not re.match(FFmpegVideoConvertorPP.FORMAT_RE, opts.recodevideo):
parser.error('invalid video remux format specified')
if opts.remuxvideo is not None:
opts.remuxvideo = opts.remuxvideo.replace(' ', '')
if not re.match(FFmpegVideoRemuxerPP.FORMAT_RE, opts.remuxvideo):
parser.error('invalid video remux format specified')
if opts.convertsubtitles is not None:
if opts.convertsubtitles not in FFmpegSubtitlesConvertorPP.SUPPORTED_EXTS:
parser.error('invalid subtitle format specified')
if opts.convertthumbnails is not None:
if opts.convertthumbnails not in FFmpegThumbnailsConvertorPP.SUPPORTED_EXTS:
parser.error('invalid thumbnail format specified')
if opts.date is not None:
date = DateRange.day(opts.date)
else:
date = DateRange(opts.dateafter, opts.datebefore)
def parse_compat_opts():
parsed_compat_opts, compat_opts = set(), opts.compat_opts[::-1]
while compat_opts:
actual_opt = opt = compat_opts.pop().lower()
if opt == 'youtube-dl':
compat_opts.extend(['-multistreams', 'all'])
elif opt == 'youtube-dlc':
compat_opts.extend(
['-no-youtube-channel-redirect', '-no-live-chat', 'all'])
elif opt == 'all':
parsed_compat_opts.update(all_compat_opts)
elif opt == '-all':
parsed_compat_opts = set()
else:
if opt[0] == '-':
opt = opt[1:]
parsed_compat_opts.discard(opt)
else:
parsed_compat_opts.update([opt])
if opt not in all_compat_opts:
parser.error('Invalid compatibility option %s' %
actual_opt)
return parsed_compat_opts
all_compat_opts = [
'filename',
'format-sort',
'abort-on-error',
'format-spec',
'no-playlist-metafiles',
'multistreams',
'no-live-chat',
'playlist-index',
'list-formats',
'no-direct-merge',
'no-youtube-channel-redirect',
'no-youtube-unavailable-videos',
'no-attach-info-json',
'embed-thumbnail-atomicparsley',
]
compat_opts = parse_compat_opts()
def _unused_compat_opt(name):
if name not in compat_opts:
return False
compat_opts.discard(name)
compat_opts.update(['*%s' % name])
return True
def set_default_compat(compat_name,
opt_name,
default=True,
remove_compat=False):
attr = getattr(opts, opt_name)
if compat_name in compat_opts:
if attr is None:
setattr(opts, opt_name, not default)
return True
else:
if remove_compat:
_unused_compat_opt(compat_name)
return False
elif attr is None:
setattr(opts, opt_name, default)
return None
set_default_compat('abort-on-error', 'ignoreerrors')
set_default_compat('no-playlist-metafiles', 'allow_playlist_files')
if 'format-sort' in compat_opts:
opts.format_sort.extend(InfoExtractor.FormatSort.ytdl_default)
_video_multistreams_set = set_default_compat(
'multistreams',
'allow_multiple_video_streams',
False,
remove_compat=False)
_audio_multistreams_set = set_default_compat(
'multistreams',
'allow_multiple_audio_streams',
False,
remove_compat=False)
if _video_multistreams_set is False and _audio_multistreams_set is False:
_unused_compat_opt('multistreams')
outtmpl_default = opts.outtmpl.get('default')
if 'filename' in compat_opts:
if outtmpl_default is None:
outtmpl_default = '%(title)s.%(id)s.%(ext)s'
opts.outtmpl.update({'default': outtmpl_default})
else:
_unused_compat_opt('filename')
def validate_outtmpl(tmpl, msg):
err = YoutubeDL.validate_outtmpl(tmpl)
if err:
parser.error('invalid %s %r: %s' %
(msg, tmpl, error_to_compat_str(err)))
for k, tmpl in opts.outtmpl.items():
validate_outtmpl(tmpl, '%s output template' % k)
for tmpl in opts.forceprint:
validate_outtmpl(tmpl, 'print template')
if opts.extractaudio and not opts.keepvideo and opts.format is None:
opts.format = 'bestaudio/best'
if outtmpl_default is not None and not os.path.splitext(
outtmpl_default)[1] and opts.extractaudio:
parser.error('Cannot download a video and extract audio into the same'
' file! Use "{0}.%(ext)s" instead of "{0}" as the output'
' template'.format(outtmpl_default))
for f in opts.format_sort:
if re.match(InfoExtractor.FormatSort.regex, f) is None:
parser.error('invalid format sort string "%s" specified' % f)
if opts.metafromfield is None:
opts.metafromfield = []
if opts.metafromtitle is not None:
opts.metafromfield.append('title:%s' % opts.metafromtitle)
for f in opts.metafromfield:
if re.match(MetadataFromFieldPP.regex, f) is None:
parser.error(
'invalid format string "%s" specified for --parse-metadata' %
f)
any_getting = opts.forceprint or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson or opts.dump_single_json
any_printing = opts.print_json
download_archive_fn = expand_path(
opts.download_archive
) if opts.download_archive is not None else opts.download_archive
# If JSON is not printed anywhere, but comments are requested, save it to file
printing_json = opts.dumpjson or opts.print_json or opts.dump_single_json
if opts.getcomments and not printing_json:
opts.writeinfojson = True
def report_conflict(arg1, arg2):
warnings.append('%s is ignored since %s was given' % (arg2, arg1))
if opts.remuxvideo and opts.recodevideo:
report_conflict('--recode-video', '--remux-video')
opts.remuxvideo = False
if opts.sponskrub_cut and opts.split_chapters and opts.sponskrub is not False:
report_conflict('--split-chapter', '--sponskrub-cut')
opts.sponskrub_cut = False
if opts.allow_unplayable_formats:
if opts.extractaudio:
report_conflict('--allow-unplayable-formats', '--extract-audio')
opts.extractaudio = False
if opts.remuxvideo:
report_conflict('--allow-unplayable-formats', '--remux-video')
opts.remuxvideo = False
if opts.recodevideo:
report_conflict('--allow-unplayable-formats', '--recode-video')
opts.recodevideo = False
if opts.addmetadata:
report_conflict('--allow-unplayable-formats', '--add-metadata')
opts.addmetadata = False
if opts.embedsubtitles:
report_conflict('--allow-unplayable-formats', '--embed-subs')
opts.embedsubtitles = False
if opts.embedthumbnail:
report_conflict('--allow-unplayable-formats', '--embed-thumbnail')
opts.embedthumbnail = False
if opts.xattrs:
report_conflict('--allow-unplayable-formats', '--xattrs')
opts.xattrs = False
if opts.fixup and opts.fixup.lower() not in ('never', 'ignore'):
report_conflict('--allow-unplayable-formats', '--fixup')
opts.fixup = 'never'
if opts.sponskrub:
report_conflict('--allow-unplayable-formats', '--sponskrub')
opts.sponskrub = False
# PostProcessors
postprocessors = []
if opts.metafromfield:
postprocessors.append({
'key': 'MetadataFromField',
'formats': opts.metafromfield,
# Run this immediately after extraction is complete
'when': 'pre_process'
})
if opts.convertsubtitles:
postprocessors.append({
'key': 'FFmpegSubtitlesConvertor',
'format': opts.convertsubtitles,
# Run this before the actual video download
'when': 'before_dl'
})
if opts.convertthumbnails:
postprocessors.append({
'key': 'FFmpegThumbnailsConvertor',
'format': opts.convertthumbnails,
# Run this before the actual video download
'when': 'before_dl'
})
if opts.extractaudio:
postprocessors.append({
'key': 'FFmpegExtractAudio',
'preferredcodec': opts.audioformat,
'preferredquality': opts.audioquality,
'nopostoverwrites': opts.nopostoverwrites,
})
if opts.remuxvideo:
postprocessors.append({
'key': 'FFmpegVideoRemuxer',
'preferedformat': opts.remuxvideo,
})
if opts.recodevideo:
postprocessors.append({
'key': 'FFmpegVideoConvertor',
'preferedformat': opts.recodevideo,
})
# FFmpegMetadataPP should be run after FFmpegVideoConvertorPP and
# FFmpegExtractAudioPP as containers before conversion may not support
# metadata (3gp, webm, etc.)
# And this post-processor should be placed before other metadata
# manipulating post-processors (FFmpegEmbedSubtitle) to prevent loss of
# extra metadata. By default ffmpeg preserves metadata applicable for both
# source and target containers. From this point the container won't change,
# so metadata can be added here.
if opts.addmetadata:
postprocessors.append({'key': 'FFmpegMetadata'})
if opts.embedsubtitles:
already_have_subtitle = opts.writesubtitles
postprocessors.append({
'key': 'FFmpegEmbedSubtitle',
# already_have_subtitle = True prevents the file from being deleted after embedding
'already_have_subtitle': already_have_subtitle
})
if not already_have_subtitle:
opts.writesubtitles = True
# --all-sub automatically sets --write-sub if --write-auto-sub is not given
# this was the old behaviour if only --all-sub was given.
if opts.allsubtitles and not opts.writeautomaticsub:
opts.writesubtitles = True
# This should be above EmbedThumbnail since sponskrub removes the thumbnail attachment
# but must be below EmbedSubtitle and FFmpegMetadata
# See https://github.com/yt-dlp/yt-dlp/issues/204 , https://github.com/faissaloo/SponSkrub/issues/29
# If opts.sponskrub is None, sponskrub is used, but it silently fails if the executable can't be found
if opts.sponskrub is not False:
postprocessors.append({
'key': 'SponSkrub',
'path': opts.sponskrub_path,
'args': opts.sponskrub_args,
'cut': opts.sponskrub_cut,
'force': opts.sponskrub_force,
'ignoreerror': opts.sponskrub is None,
})
if opts.embedthumbnail:
already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails
postprocessors.append({
'key': 'EmbedThumbnail',
# already_have_thumbnail = True prevents the file from being deleted after embedding
'already_have_thumbnail': already_have_thumbnail
})
if not already_have_thumbnail:
opts.writethumbnail = True
if opts.split_chapters:
postprocessors.append({'key': 'FFmpegSplitChapters'})
# XAttrMetadataPP should be run after post-processors that may change file contents
if opts.xattrs:
postprocessors.append({'key': 'XAttrMetadata'})
# ExecAfterDownload must be the last PP
if opts.exec_cmd:
postprocessors.append({
'key': 'ExecAfterDownload',
'exec_cmd': opts.exec_cmd,
# Run this only after the files have been moved to their final locations
'when': 'after_move'
})
def report_args_compat(arg, name):
warnings.append(
'%s given without specifying name. The arguments will be given to all %s'
% (arg, name))
if 'default' in opts.external_downloader_args:
report_args_compat('--downloader-args', 'external downloaders')
if 'default-compat' in opts.postprocessor_args and 'default' not in opts.postprocessor_args:
report_args_compat('--post-processor-args', 'post-processors')
opts.postprocessor_args.setdefault('sponskrub', [])
opts.postprocessor_args['default'] = opts.postprocessor_args[
'default-compat']
final_ext = (opts.recodevideo if opts.recodevideo
in FFmpegVideoConvertorPP.SUPPORTED_EXTS else opts.remuxvideo
if opts.remuxvideo in FFmpegVideoRemuxerPP.SUPPORTED_EXTS else
opts.audioformat if
(opts.extractaudio and opts.audioformat != 'best') else None)
match_filter = (None if opts.match_filter is None else match_filter_func(
opts.match_filter))
ydl_opts = {
'usenetrc': opts.usenetrc,
'username': opts.username,
'password': opts.password,
'twofactor': opts.twofactor,
'videopassword': opts.videopassword,
'ap_mso': opts.ap_mso,
'ap_username': opts.ap_username,
'ap_password': opts.ap_password,
'quiet': (opts.quiet or any_getting or any_printing),
'no_warnings': opts.no_warnings,
'forceurl': opts.geturl,
'forcetitle': opts.gettitle,
'forceid': opts.getid,
'forcethumbnail': opts.getthumbnail,
'forcedescription': opts.getdescription,
'forceduration': opts.getduration,
'forcefilename': opts.getfilename,
'forceformat': opts.getformat,
'forceprint': opts.forceprint,
'forcejson': opts.dumpjson or opts.print_json,
'dump_single_json': opts.dump_single_json,
'force_write_download_archive': opts.force_write_download_archive,
'simulate': opts.simulate or any_getting,
'skip_download': opts.skip_download,
'format': opts.format,
'allow_unplayable_formats': opts.allow_unplayable_formats,
'ignore_no_formats_error': opts.ignore_no_formats_error,
'format_sort': opts.format_sort,
'format_sort_force': opts.format_sort_force,
'allow_multiple_video_streams': opts.allow_multiple_video_streams,
'allow_multiple_audio_streams': opts.allow_multiple_audio_streams,
'check_formats': opts.check_formats,
'listformats': opts.listformats,
'listformats_table': opts.listformats_table,
'outtmpl': opts.outtmpl,
'outtmpl_na_placeholder': opts.outtmpl_na_placeholder,
'paths': opts.paths,
'autonumber_size': opts.autonumber_size,
'autonumber_start': opts.autonumber_start,
'restrictfilenames': opts.restrictfilenames,
'windowsfilenames': opts.windowsfilenames,
'ignoreerrors': opts.ignoreerrors,
'force_generic_extractor': opts.force_generic_extractor,
'ratelimit': opts.ratelimit,
'overwrites': opts.overwrites,
'retries': opts.retries,
'fragment_retries': opts.fragment_retries,
'extractor_retries': opts.extractor_retries,
'skip_unavailable_fragments': opts.skip_unavailable_fragments,
'keep_fragments': opts.keep_fragments,
'concurrent_fragment_downloads': opts.concurrent_fragment_downloads,
'buffersize': opts.buffersize,
'noresizebuffer': opts.noresizebuffer,
'http_chunk_size': opts.http_chunk_size,
'continuedl': opts.continue_dl,
'noprogress': opts.noprogress,
'progress_with_newline': opts.progress_with_newline,
'playliststart': opts.playliststart,
'playlistend': opts.playlistend,
'playlistreverse': opts.playlist_reverse,
'playlistrandom': opts.playlist_random,
'noplaylist': opts.noplaylist,
'logtostderr': outtmpl_default == '-',
'consoletitle': opts.consoletitle,
'nopart': opts.nopart,
'updatetime': opts.updatetime,
'writedescription': opts.writedescription,
'writeannotations': opts.writeannotations,
'writeinfojson': opts.writeinfojson,
'allow_playlist_files': opts.allow_playlist_files,
'clean_infojson': opts.clean_infojson,
'getcomments': opts.getcomments,
'writethumbnail': opts.writethumbnail,
'write_all_thumbnails': opts.write_all_thumbnails,
'writelink': opts.writelink,
'writeurllink': opts.writeurllink,
'writewebloclink': opts.writewebloclink,
'writedesktoplink': opts.writedesktoplink,
'writesubtitles': opts.writesubtitles,
'writeautomaticsub': opts.writeautomaticsub,
'allsubtitles': opts.allsubtitles,
'listsubtitles': opts.listsubtitles,
'subtitlesformat': opts.subtitlesformat,
'subtitleslangs': opts.subtitleslangs,
'matchtitle': decodeOption(opts.matchtitle),
'rejecttitle': decodeOption(opts.rejecttitle),
'max_downloads': opts.max_downloads,
'prefer_free_formats': opts.prefer_free_formats,
'trim_file_name': opts.trim_file_name,
'verbose': opts.verbose,
'dump_intermediate_pages': opts.dump_intermediate_pages,
'write_pages': opts.write_pages,
'test': opts.test,
'keepvideo': opts.keepvideo,
'min_filesize': opts.min_filesize,
'max_filesize': opts.max_filesize,
'min_views': opts.min_views,
'max_views': opts.max_views,
'daterange': date,
'cachedir': opts.cachedir,
'youtube_print_sig_code': opts.youtube_print_sig_code,
'age_limit': opts.age_limit,
'download_archive': download_archive_fn,
'break_on_existing': opts.break_on_existing,
'break_on_reject': opts.break_on_reject,
'skip_playlist_after_errors': opts.skip_playlist_after_errors,
'cookiefile': opts.cookiefile,
'nocheckcertificate': opts.no_check_certificate,
'prefer_insecure': opts.prefer_insecure,
'proxy': opts.proxy,
'socket_timeout': opts.socket_timeout,
'bidi_workaround': opts.bidi_workaround,
'debug_printtraffic': opts.debug_printtraffic,
'prefer_ffmpeg': opts.prefer_ffmpeg,
'include_ads': opts.include_ads,
'default_search': opts.default_search,
'dynamic_mpd': opts.dynamic_mpd,
'youtube_include_dash_manifest': opts.youtube_include_dash_manifest,
'youtube_include_hls_manifest': opts.youtube_include_hls_manifest,
'encoding': opts.encoding,
'extract_flat': opts.extract_flat,
'mark_watched': opts.mark_watched,
'merge_output_format': opts.merge_output_format,
'final_ext': final_ext,
'postprocessors': postprocessors,
'fixup': opts.fixup,
'source_address': opts.source_address,
'call_home': opts.call_home,
'sleep_interval_requests': opts.sleep_interval_requests,
'sleep_interval': opts.sleep_interval,
'max_sleep_interval': opts.max_sleep_interval,
'sleep_interval_subtitles': opts.sleep_interval_subtitles,
'external_downloader': opts.external_downloader,
'list_thumbnails': opts.list_thumbnails,
'playlist_items': opts.playlist_items,
'xattr_set_filesize': opts.xattr_set_filesize,
'match_filter': match_filter,
'no_color': opts.no_color,
'ffmpeg_location': opts.ffmpeg_location,
'hls_prefer_native': opts.hls_prefer_native,
'hls_use_mpegts': opts.hls_use_mpegts,
'hls_split_discontinuity': opts.hls_split_discontinuity,
'external_downloader_args': opts.external_downloader_args,
'postprocessor_args': opts.postprocessor_args,
'cn_verification_proxy': opts.cn_verification_proxy,
'geo_verification_proxy': opts.geo_verification_proxy,
'geo_bypass': opts.geo_bypass,
'geo_bypass_country': opts.geo_bypass_country,
'geo_bypass_ip_block': opts.geo_bypass_ip_block,
'warnings': warnings,
'compat_opts': compat_opts,
# just for deprecation check
'autonumber': opts.autonumber or None,
'usetitle': opts.usetitle or None,
'useid': opts.useid or None,
}
with YoutubeDL(ydl_opts) as ydl:
actual_use = len(all_urls) or opts.load_info_filename
# Remove cache dir
if opts.rm_cachedir:
ydl.cache.remove()
# Update version
if opts.update_self:
# If updater returns True, exit. Required for windows
if run_update(ydl):
if actual_use:
sys.exit(
'ERROR: The program must exit for the update to complete'
)
sys.exit()
# Maybe do nothing
if not actual_use:
if opts.update_self or opts.rm_cachedir:
sys.exit()
ydl.warn_if_short_id(sys.argv[1:] if argv is None else argv)
parser.error('You must provide at least one URL.\n'
'Type yt-dlp --help to see a list of all options.')
try:
if opts.load_info_filename is not None:
retcode = ydl.download_with_info_file(
expand_path(opts.load_info_filename))
else:
retcode = ydl.download(all_urls)
except (MaxDownloadsReached, ExistingVideoReached,
RejectedVideoReached):
ydl.to_screen('Aborting remaining downloads')
retcode = 101
sys.exit(retcode)
return safe_encode(input, self.errors)[0]
class IncrementalDecoder(codecs.IncrementalDecoder):
def decode(self, input, final=False):
return safe_decode(input, self.errors)[0]
class StreamWriter(Codec, codecs.StreamWriter):
pass
class StreamReader(Codec, codecs.StreamReader):
pass
def getregentry(name):
if name == 'safe-utf-8':
return codecs.CodecInfo(
name=name,
encode=safe_encode,
decode=safe_decode,
incrementalencoder=IncrementalEncoder,
incrementaldecoder=IncrementalDecoder,
streamreader=StreamReader,
streamwriter=StreamWriter,
)
codecs.register(getregentry)
def initialize():
global done
import sys
if sys.platform != "win32" or done:
return True
done = True
import codecs, re
from ctypes import WINFUNCTYPE, WinError, windll, POINTER, byref, c_int, get_last_error
from ctypes.wintypes import BOOL, HANDLE, DWORD, UINT, LPWSTR, LPCWSTR, LPVOID
from allmydata.util import log
from allmydata.util.encodingutil import canonical_encoding
# <https://msdn.microsoft.com/en-us/library/ms680621%28VS.85%29.aspx>
SetErrorMode = WINFUNCTYPE(UINT, UINT, use_last_error=True)(
("SetErrorMode", windll.kernel32))
SEM_FAILCRITICALERRORS = 0x0001
SEM_NOOPENFILEERRORBOX = 0x8000
SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOOPENFILEERRORBOX)
original_stderr = sys.stderr
# If any exception occurs in this code, we'll probably try to print it on stderr,
# which makes for frustrating debugging if stderr is directed to our wrapper.
# So be paranoid about catching errors and reporting them to original_stderr,
# so that we can at least see them.
def _complain(message):
print(isinstance(message, str) and message or repr(message),
file=original_stderr)
log.msg(message, level=log.WEIRD)
# Work around <http://bugs.python.org/issue6058>.
codecs.register(
lambda name: name == 'cp65001' and codecs.lookup('utf-8') or None)
# Make Unicode console output work independently of the current code page.
# This also fixes <http://bugs.python.org/issue1602>.
# Credit to Michael Kaplan <https://blogs.msdn.com/b/michkap/archive/2010/04/07/9989346.aspx>
# and TZOmegaTZIOY
# <http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/1432462#1432462>.
try:
# <https://msdn.microsoft.com/en-us/library/ms683231(VS.85).aspx>
# HANDLE WINAPI GetStdHandle(DWORD nStdHandle);
# returns INVALID_HANDLE_VALUE, NULL, or a valid handle
#
# <https://msdn.microsoft.com/en-us/library/aa364960(VS.85).aspx>
# DWORD WINAPI GetFileType(DWORD hFile);
#
# <https://msdn.microsoft.com/en-us/library/ms683167(VS.85).aspx>
# BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode);
GetStdHandle = WINFUNCTYPE(HANDLE, DWORD, use_last_error=True)(
("GetStdHandle", windll.kernel32))
STD_OUTPUT_HANDLE = DWORD(-11)
STD_ERROR_HANDLE = DWORD(-12)
GetFileType = WINFUNCTYPE(DWORD, DWORD, use_last_error=True)(
("GetFileType", windll.kernel32))
FILE_TYPE_CHAR = 0x0002
FILE_TYPE_REMOTE = 0x8000
GetConsoleMode = WINFUNCTYPE(BOOL,
HANDLE,
POINTER(DWORD),
use_last_error=True)(
("GetConsoleMode", windll.kernel32))
INVALID_HANDLE_VALUE = DWORD(-1).value
def not_a_console(handle):
if handle == INVALID_HANDLE_VALUE or handle is None:
return True
return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
or GetConsoleMode(handle, byref(DWORD())) == 0)
old_stdout_fileno = None
old_stderr_fileno = None
if hasattr(sys.stdout, 'fileno'):
old_stdout_fileno = sys.stdout.fileno()
if hasattr(sys.stderr, 'fileno'):
old_stderr_fileno = sys.stderr.fileno()
STDOUT_FILENO = 1
STDERR_FILENO = 2
real_stdout = (old_stdout_fileno == STDOUT_FILENO)
real_stderr = (old_stderr_fileno == STDERR_FILENO)
if real_stdout:
hStdout = GetStdHandle(STD_OUTPUT_HANDLE)
if not_a_console(hStdout):
real_stdout = False
if real_stderr:
hStderr = GetStdHandle(STD_ERROR_HANDLE)
if not_a_console(hStderr):
real_stderr = False
if real_stdout or real_stderr:
# <https://msdn.microsoft.com/en-us/library/windows/desktop/ms687401%28v=vs.85%29.aspx>
# BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars,
# LPDWORD lpCharsWritten, LPVOID lpReserved);
WriteConsoleW = WINFUNCTYPE(BOOL,
HANDLE,
LPWSTR,
DWORD,
POINTER(DWORD),
LPVOID,
use_last_error=True)(
("WriteConsoleW", windll.kernel32))
class UnicodeOutput(object):
def __init__(self, hConsole, stream, fileno, name):
self._hConsole = hConsole
self._stream = stream
self._fileno = fileno
self.closed = False
self.softspace = False
self.mode = 'w'
self.encoding = 'utf-8'
self.name = name
if hasattr(stream, 'encoding') and canonical_encoding(
stream.encoding) != 'utf-8':
log.msg(
"%s: %r had encoding %r, but we're going to write UTF-8 to it"
% (name, stream, stream.encoding),
level=log.CURIOUS)
self.flush()
def isatty(self):
return False
def close(self):
# don't really close the handle, that would only cause problems
self.closed = True
def fileno(self):
return self._fileno
def flush(self):
if self._hConsole is None:
try:
self._stream.flush()
except Exception as e:
_complain("%s.flush: %r from %r" %
(self.name, e, self._stream))
raise
def write(self, text):
try:
if self._hConsole is None:
if isinstance(text, unicode):
text = text.encode('utf-8')
self._stream.write(text)
else:
if not isinstance(text, unicode):
text = str(text).decode('utf-8')
remaining = len(text)
while remaining > 0:
n = DWORD(0)
# There is a shorter-than-documented limitation on the length of the string
# passed to WriteConsoleW (see #1232).
retval = WriteConsoleW(self._hConsole, text,
min(remaining, 10000),
byref(n), None)
if retval == 0:
raise IOError(
"WriteConsoleW failed with WinError: %s"
% (WinError(get_last_error()), ))
if n.value == 0:
raise IOError(
"WriteConsoleW returned %r, n.value = 0"
% (retval, ))
remaining -= n.value
if remaining == 0: break
text = text[n.value:]
except Exception as e:
_complain("%s.write: %r" % (self.name, e))
raise
def writelines(self, lines):
try:
for line in lines:
self.write(line)
except Exception as e:
_complain("%s.writelines: %r" % (self.name, e))
raise
if real_stdout:
sys.stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO,
'<Unicode console stdout>')
else:
sys.stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno,
'<Unicode redirected stdout>')
if real_stderr:
sys.stderr = UnicodeOutput(hStderr, None, STDERR_FILENO,
'<Unicode console stderr>')
else:
sys.stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno,
'<Unicode redirected stderr>')
except Exception as e:
_complain("exception %r while fixing up sys.stdout and sys.stderr" %
(e, ))
# This works around <http://bugs.python.org/issue2128>.
# <https://msdn.microsoft.com/en-us/library/windows/desktop/ms683156%28v=vs.85%29.aspx>
GetCommandLineW = WINFUNCTYPE(LPWSTR, use_last_error=True)(
("GetCommandLineW", windll.kernel32))
# <https://msdn.microsoft.com/en-us/library/windows/desktop/bb776391%28v=vs.85%29.aspx>
CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR),
LPCWSTR,
POINTER(c_int),
use_last_error=True)(
("CommandLineToArgvW",
windll.shell32))
argc = c_int(0)
argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))
if argv_unicode is None:
raise WinError(get_last_error())
# Because of <http://bugs.python.org/issue8775> (and similar limitations in
# twisted), the 'bin/tahoe' script cannot invoke us with the actual Unicode arguments.
# Instead it "mangles" or escapes them using \x7F as an escape character, which we
# unescape here.
def unmangle(s):
return re.sub(
u'\\x7F[0-9a-fA-F]*\\;',
# type ignored for 'unichr' (Python 2 only)
lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore
s,
)
try:
argv = [
unmangle(argv_unicode[i]).encode('utf-8')
for i in xrange(0, argc.value)
]
except Exception as e:
_complain(
"%s: could not unmangle Unicode arguments.\n%r" %
(sys.argv[0], [argv_unicode[i] for i in xrange(0, argc.value)]))
raise
# Take only the suffix with the same number of arguments as sys.argv.
# This accounts for anything that can cause initial arguments to be stripped,
# for example, the Python interpreter or any options passed to it, or runner
# scripts such as 'coverage run'. It works even if there are no such arguments,
# as in the case of a frozen executable created by bb-freeze or similar.
sys.argv = argv[-len(sys.argv):]
if sys.argv[0].endswith('.pyscript'):
sys.argv[0] = sys.argv[0][:-9]
def _from_response(self, rsp):
self.version = rsp.hpm_1_version
self.components = []
for i in range(8):
if rsp.component_present & (1 << i):
self.components.append(i)
def __str__(self):
string = []
string.append("Target Upgrade Capabilities")
string.append(" HPM.1 version: %s" % self.version)
string.append(" Components: %s" % self.components)
return "\n".join(string)
codecs.register(bcd_search)
class ComponentProperty(object):
def __init__(self, data=None):
if (data):
self._from_rsp_data(data)
@staticmethod
def from_data(component_id, data):
if isinstance(data, str):
data = [ord(c) for c in data]
if component_id is PROPERTY_GENERAL_PROPERTIES:
return ComponentPropertyGeneral(data)
elif component_id is PROPERTY_CURRENT_VERSION:
from __future__ import absolute_import, print_function
from setuptools.command.build_py import build_py
# Work around mbcs bug in distutils.
# http://bugs.python.org/issue10945
import codecs
try:
codecs.lookup('mbcs')
except LookupError:
ascii = codecs.lookup('ascii')
func = lambda name, enc=ascii: {True: enc}.get(name == 'mbcs')
codecs.register(func)
from setuptools import setup, Extension
import glob, os, shutil, fnmatch, platform, sys
version = '2.4.7'
def generate_content():
# generate the file content...
from generator import mavgen, mavparse
# path to message_definitions directory
if os.getenv("MDEF", None) is not None:
mdef_paths = [os.getenv("MDEF")]
else:
mdef_paths = [
os.path.join('..', 'message_definitions'),
os.path.join('mavlink', 'message_definitions'),
os.path.join('..', 'mavlink', 'message_definitions'),
inner = next(self.tokens)
if inner.string in pydoc.Helper.keywords:
self.output.append(
tokenize.TokenInfo(
type=tokenize.STRING,
string=repr(inner.string),
start=inner.start,
end=inner.end,
line=inner.line,
)
)
else:
self.output.append(inner)
def encode(string, errors="strict"):
raise NotImplemented("this is nonsense, right?")
def decode(byteslike, errors="strict"):
code_bytes = bytes(byteslike)
decoder = HelpEncoding(
tokenize.tokenize(io.BytesIO(b"(" + code_bytes + b")").readline)
)
decoder.parse()
return (tokenize.untokenize(decoder.output[2:-2]), len(code_bytes))
codec_info = codecs.CodecInfo(encode, decode, name="halp")
codecs.register({"halp": codec_info}.get)
def register_cp65001():
"""Register codecs cp65001 as utf-8."""
# Work around <http://bugs.python.org/issue6058>.
codecs.register(lambda name: name == 'cp65001'
and codecs.lookup('utf-8') or None)
)
pass
# This is the updated MBCS Encoding Bypass for making MBCS encodings work on Linux - NovaCygni
try:
codecs.lookup('mbcs')
except LookupError:
ascii_encoding = codecs.lookup('latin-1')
def mbcs_bypass(name, encoding=ascii_encoding):
if name == "mbcs":
return encoding
codecs.register(mbcs_bypass)
# Colours
W = "\033[0m"
R = "\033[31m"
G = "\033[32m"
O = "\033[33m"
B = "\033[34m"
subprocess.call("clear", shell=True)
arg_end = "--"
arg_eva = "+"
colMax = 60 # Change this at your will
endsub = 1
gets = 0
timeout = 8
if buildCython:
cmdclass["build_ext"] = setup_cython.CythonBuildExtension
setup_cython.registerCythonModules()
else:
print("Skipping build of CYTHON modules.")
ext_modules = setup_cython.ext_modules
extraKeywords = {}
# Workaround for mbcs codec bug in distutils
# http://bugs.python.org/issue10945
import codecs
try:
codecs.lookup("mbcs")
except LookupError:
codecs.register(lambda name: codecs.lookup("ascii")
if name == "mbcs" else None)
# Create list of scripts. Depends on OS.
scripts = [
"awlsim-gui",
"awlsim-client",
"awlsim-server",
"awlsim-symtab",
"awlsim-proupgrade",
"awlsim-test",
]
if isWindows or fullBuild:
scripts.append("awlsim-win.cmd")
if not isWindows or fullBuild:
scripts.append("awlsim-linuxcnc-hal")
def find_invertcaps(encoding):
"""Return the codec for 'invertcaps'.
"""
if encoding == 'invertcaps':
return codecs.CodecInfo(
name='invertcaps',
encode=InvertCapsCodec().encode,
decode=InvertCapsCodec().decode,
incrementalencoder=InvertCapsIncrementalEncoder,
incrementaldecoder=InvertCapsIncrementalDecoder,
streamreader=InvertCapsStreamReader,
streamwriter=InvertCapsStreamWriter,
)
return None
codecs.register(find_invertcaps)
if __name__ == '__main__':
# Stateless encoder/decoder
encoder = codecs.getencoder('invertcaps')
text = 'abc.DEF'
encoded_text, consumed = encoder(text)
print 'Encoder converted "{}" to "{}", consuming {} characters'.format(
text, encoded_text, consumed)
# Stream writer
import sys
writer = codecs.getwriter('invertcaps')(sys.stdout)
print 'StreamWriter for stdout: ',
writer.write('abc.DEF')
# GNU General Public License for more details. #
# #
# You should have received a copy of the GNU General Public License #
# along with this program; if not, write to the Free Software #
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, #
# MA 02110-1301, USA. #
#####################################################################
"""
Main game executable.
"""
# Register the latin-1 encoding
import codecs
import encodings.iso8859_1
import encodings.utf_8
codecs.register(lambda encoding: encodings.iso8859_1.getregentry())
codecs.register(lambda encoding: encodings.utf_8.getregentry())
assert codecs.lookup("iso-8859-1")
assert codecs.lookup("utf-8")
from GameEngine import GameEngine
from MainMenu import MainMenu
import Log
import Config
import Version
import getopt
import sys
import os
import codecs
import Resource
def fix_broken_win_console():
'''Fix broken Unicode support in Windows console.
This disables ANSI color support. It is only necessary to run this if the UTF-8 cp65001 code page
is active.
This code is lifted from Daira Hopwood's answer on SO: http://stackoverflow.com/a/3259271/1583598
'''
import codecs
from ctypes import WINFUNCTYPE, windll, POINTER, byref, c_int
from ctypes.wintypes import BOOL, HANDLE, DWORD, LPWSTR, LPCWSTR, LPVOID
original_stderr = sys.stderr
# If any exception occurs in this code, we'll probably try to print it on stderr,
# which makes for frustrating debugging if stderr is directed to our wrapper.
# So be paranoid about catching errors and reporting them to original_stderr,
# so that we can at least see them.
def _complain(message):
print(message if isinstance(message, str) else repr(message),
file=original_stderr)
# Work around <http://bugs.python.org/issue6058>.
codecs.register(lambda name: codecs.lookup('utf-8')
if name == 'cp65001' else None)
# Make Unicode console output work independently of the current code page.
# This also fixes <http://bugs.python.org/issue1602>.
# Credit to Michael Kaplan <http://www.siao2.com/2010/04/07/9989346.aspx>
# and TZOmegaTZIOY
# <http://stackoverflow.com/questions/878972/windows-cmd-encoding-change-causes-python-crash/1432462#1432462>.
try:
# <http://msdn.microsoft.com/en-us/library/ms683231(VS.85).aspx>
# HANDLE WINAPI GetStdHandle(DWORD nStdHandle);
# returns INVALID_HANDLE_VALUE, NULL, or a valid handle
#
# <http://msdn.microsoft.com/en-us/library/aa364960(VS.85).aspx>
# DWORD WINAPI GetFileType(DWORD hFile);
#
# <http://msdn.microsoft.com/en-us/library/ms683167(VS.85).aspx>
# BOOL WINAPI GetConsoleMode(HANDLE hConsole, LPDWORD lpMode);
GetStdHandle = WINFUNCTYPE(HANDLE,
DWORD)(("GetStdHandle", windll.kernel32))
STD_OUTPUT_HANDLE = DWORD(-11)
STD_ERROR_HANDLE = DWORD(-12)
GetFileType = WINFUNCTYPE(DWORD,
DWORD)(("GetFileType", windll.kernel32))
FILE_TYPE_CHAR = 0x0002
FILE_TYPE_REMOTE = 0x8000
GetConsoleMode = WINFUNCTYPE(BOOL, HANDLE, POINTER(DWORD))(
("GetConsoleMode", windll.kernel32))
INVALID_HANDLE_VALUE = DWORD(-1).value
def not_a_console(handle):
if handle == INVALID_HANDLE_VALUE or handle is None:
return True
return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR
or GetConsoleMode(handle, byref(DWORD())) == 0)
old_stdout_fileno = None
old_stderr_fileno = None
if hasattr(sys.stdout, 'fileno'):
old_stdout_fileno = sys.stdout.fileno()
if hasattr(sys.stderr, 'fileno'):
old_stderr_fileno = sys.stderr.fileno()
STDOUT_FILENO = 1
STDERR_FILENO = 2
real_stdout = (old_stdout_fileno == STDOUT_FILENO)
real_stderr = (old_stderr_fileno == STDERR_FILENO)
if real_stdout:
hStdout = GetStdHandle(STD_OUTPUT_HANDLE)
if not_a_console(hStdout):
real_stdout = False
if real_stderr:
hStderr = GetStdHandle(STD_ERROR_HANDLE)
if not_a_console(hStderr):
real_stderr = False
if real_stdout or real_stderr:
# BOOL WINAPI WriteConsoleW(HANDLE hOutput, LPWSTR lpBuffer, DWORD nChars,
# LPDWORD lpCharsWritten, LPVOID lpReserved);
WriteConsoleW = WINFUNCTYPE(BOOL, HANDLE, LPWSTR, DWORD,
POINTER(DWORD), LPVOID)(
("WriteConsoleW", windll.kernel32))
class UnicodeOutput:
def __init__(self, hConsole, stream, fileno, name):
self._hConsole = hConsole
self._stream = stream
self._fileno = fileno
self.closed = False
self.softspace = False
self.mode = 'w'
self.encoding = 'utf-8'
self.name = name
self.flush()
def isatty(self):
return False
def close(self):
# don't really close the handle, that would only cause problems
self.closed = True
def fileno(self):
return self._fileno
def flush(self):
if self._hConsole is None:
try:
self._stream.flush()
except Exception as e:
_complain("%s.flush: %r from %r" %
(self.name, e, self._stream))
raise
def write(self, text):
try:
if self._hConsole is None:
if isinstance(text, str):
text = text.encode('utf-8')
self._stream.write(text)
else:
if not isinstance(text, str):
text = str(text).decode('utf-8')
remaining = len(text)
while remaining:
n = DWORD(0)
# There is a shorter-than-documented limitation on the
# length of the string passed to WriteConsoleW (see
# <http://tahoe-lafs.org/trac/tahoe-lafs/ticket/1232>.
retval = WriteConsoleW(self._hConsole, text,
min(remaining, 10000),
byref(n), None)
if retval == 0 or n.value == 0:
raise IOError(
"WriteConsoleW returned %r, n.value = %r"
% (retval, n.value))
remaining -= n.value
if not remaining:
break
text = text[n.value:]
except Exception as e:
_complain("%s.write: %r" % (self.name, e))
raise
def writelines(self, lines):
try:
for line in lines:
self.write(line)
except Exception as e:
_complain("%s.writelines: %r" % (self.name, e))
raise
if real_stdout:
sys.stdout = UnicodeOutput(hStdout, None, STDOUT_FILENO,
'<Unicode console stdout>')
else:
sys.stdout = UnicodeOutput(None, sys.stdout, old_stdout_fileno,
'<Unicode redirected stdout>')
if real_stderr:
sys.stderr = UnicodeOutput(hStderr, None, STDERR_FILENO,
'<Unicode console stderr>')
else:
sys.stderr = UnicodeOutput(None, sys.stderr, old_stderr_fileno,
'<Unicode redirected stderr>')
except Exception as e:
_complain("exception %r while fixing up sys.stdout and sys.stderr" %
(e, ))
# While we're at it, let's unmangle the command-line arguments:
# This works around <http://bugs.python.org/issue2128>.
GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32))
CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))(
("CommandLineToArgvW", windll.shell32))
argc = c_int(0)
argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc))
argv = [argv_unicode[i].encode('utf-8') for i in range(0, argc.value)]
if not hasattr(sys, 'frozen'):
# If this is an executable produced by py2exe or bbfreeze, then it will
# have been invoked directly. Otherwise, unicode_argv[0] is the Python
# interpreter, so skip that.
argv = argv[1:]
# Also skip option arguments to the Python interpreter.
while len(argv) > 0:
arg = argv[0]
if not arg.startswith("-") or arg == "-":
break
argv = argv[1:]
if arg == '-m':
# sys.argv[0] should really be the absolute path of the module source,
# but never mind
break
if arg == '-c':
argv[0] = '-c'
break
# if you like:
sys.argv = argv
def register():
"""Enable encodings of the form 'latex+x' where x describes another encoding.
Unicode characters are translated to or from x when possible, otherwise
expanded to latex. """
codecs.register(_registry)
not hasattr(entry[1], '__call__') or \
(entry[2] is not None and not hasattr(entry[2], '__call__')) or \
(entry[3] is not None and not hasattr(entry[3], '__call__')) or \
(len(entry) > 4 and entry[4] is not None and not hasattr(entry[4], '__call__')) or \
(len(entry) > 5 and entry[5] is not None and not hasattr(entry[5], '__call__')):
raise CodecRegistryError('incompatible codecs in module "%s" (%s)'
% (mod.__name__, mod.__file__))
if len(entry)<7 or entry[6] is None:
entry += (None,)*(6-len(entry)) + (mod.__name__.split(".", 1)[1],)
entry = codecs.CodecInfo(*entry)
# Cache the codec registry entry
_cache[encoding] = entry
# Register its aliases (without overwriting previously registered
# aliases)
try:
codecaliases = mod.getaliases()
except AttributeError:
pass
else:
for alias in codecaliases:
if alias not in _aliases:
_aliases[alias] = modname
# Return the registry entry
return entry
# Register the search_function in the Python codec registry
codecs.register(search_function)
#!/usr/bin/env python3
from setuptools import setup
# Work around mbcs bug in distutils.
# http://bugs.python.org/issue10945
import codecs
try:
codecs.lookup('mbcs')
except LookupError:
ascii = codecs.lookup('ascii')
codecs.register(lambda name, enc=ascii: {True: enc}.get(name == 'mbcs'))
VERSION = '0.2.0'
URL = "https://github.com/bitshares/python-bitshares"
setup(
name='bitshares',
version=VERSION,
description='Python library for bitshares',
long_description=open('README.md').read(),
download_url='{}/tarball/{}'.format(URL, VERSION),
author='Fabian Schuh',
author_email='*****@*****.**',
maintainer='Fabian Schuh',
maintainer_email='*****@*****.**',
url=URL,
keywords=['bitshares', 'library', 'api', 'rpc'],
packages=[
"bitshares",
"bitsharesapi",
