def text_lines(location, demarkup=False):
"""
Return a text lines iterator from file at `location`. Return an empty
iterator if no text content is extractible. Text extraction is based on
detected file type.
if `demarkup` is True, attempt to detect if a file contains HTML/XML-like
markup and cleanup this markup.
Note: For testing or building from strings, location can be a is a list of
unicode line strings.
"""
# TODO: add support for "wide" UTF-16-like strings where each char is
# followed by a zero as is often found in some Windows binaries. Do this for
# binaries only. This is in direct conflict with "strings" extraction as
# currently implemented
if not location:
return iter([])
if not isinstance(location, basestring):
# not a path: wrap an iterator on location which should be a sequence
# of lines
return iter(location)
T = typecode.get_type(location)
if not T.contains_text:
return iter([])
# Should we read this as some markup, pdf office doc, text or binary?
if T.is_pdf:
return unicode_text_lines_from_pdf(location)
# lightweight markup stripping support
if demarkup and markup.is_markup(location):
try:
return markup.demarkup(location)
except:
# try again later with as plain text
pass
# TODO: handle Office-like documents, RTF, etc
# if T.is_doc:
# return unicode_text_lines_from_doc(location)
if T.is_text:
return unicode_text_lines(location)
# DO NOT introspect media, archives and compressed files
# if not T.contains_text:
# return iter([])
if T.is_binary:
# fall back to binary
return unicode_text_lines_from_binary(location)
else:
# if neither text, text-like nor binary: treat as binary
# this should never happen
# fall back to binary
return unicode_text_lines_from_binary(location)
def numbered_text_lines(
location,
demarkup=False,
plain_text=False,
start_line=1,
):
"""
Yield tuples of (line number, text line) from the file at `location`. Return
an empty iterator if no text content is extractible. Text extraction is
based on detected file type. Long lines are broken down in chunks, therefore
two items can have the same line number.
line numbers start at ``start_line`` which is 1-based by default.
If `demarkup` is True, attempt to detect if a file contains HTML/XML-like
markup and cleanup this markup.
If `plain_text` is True treat the file as a plain text file and do not
attempt to detect its type and extract it's content with special procedures.
This is used mostly when loading license texts and rules.
Note: For testing or building from strings, location can be a is a list of
unicode line strings.
"""
if not location:
return iter([])
if not isinstance(location, str):
# not a path: wrap an iterator on location which should be a sequence
# of lines
if TRACE:
logger_debug('numbered_text_lines:', 'location is not a file')
return enumerate(iter(location), start_line)
if plain_text:
if TRACE:
logger_debug('numbered_text_lines:', 'plain_text')
return enumerate(unicode_text_lines(location), start_line)
T = typecode.get_type(location)
if TRACE:
logger_debug('numbered_text_lines: T.filetype_file:', T.filetype_file)
logger_debug('numbered_text_lines: T.is_text_with_long_lines:',
T.is_text_with_long_lines)
logger_debug('numbered_text_lines: T.is_binary:', T.is_binary)
# TODO: we should have a command line to force digging inside binaries
if not T.contains_text:
return iter([])
# Should we read this as some markup, pdf office doc, text or binary?
if T.is_pdf and T.is_pdf_with_text:
if TRACE:
logger_debug('numbered_text_lines:', 'is_pdf')
return enumerate(unicode_text_lines_from_pdf(location), start_line)
if T.filetype_file.startswith('Spline Font Database'):
if TRACE:
logger_debug('numbered_text_lines:', 'Spline Font Database')
return enumerate(
(as_unicode(l) for l in sfdb.get_text_lines(location)),
start_line,
)
# lightweight markup stripping support
if demarkup and markup.is_markup(location):
try:
lines = list(enumerate(markup.demarkup(location), start_line))
if TRACE:
logger_debug('numbered_text_lines:', 'demarkup')
return lines
except:
# try again later with as plain text
pass
if T.is_js_map:
try:
lines = list(enumerate(js_map_sources_lines(location), start_line))
if TRACE:
logger_debug('numbered_text_lines:', 'js_map')
return lines
except:
# try again later with as plain text otherwise
pass
if T.is_text:
numbered_lines = enumerate(unicode_text_lines(location), start_line)
# text with very long lines such minified JS, JS map files or large JSON
if (not location.endswith('package.json')
and (T.is_text_with_long_lines or T.is_compact_js
or T.filetype_file == 'data' or 'locale' in location)):
numbered_lines = break_numbered_unicode_text_lines(numbered_lines)
if TRACE:
logger_debug('numbered_text_lines:',
'break_numbered_unicode_text_lines')
return numbered_lines
# TODO: handle Office-like documents, RTF, etc
# if T.is_doc:
# return unicode_text_lines_from_doc(location)
# TODO: add support for "wide" UTF-16-like strings where each char is
# followed by a zero as is often found in some Windows binaries. Do this for
# binaries only. This is may conflicting with "strings" extraction as
# currently implemented
if T.is_binary:
# fall back to binary
if TRACE:
logger_debug('numbered_text_lines:', 'is_binary')
return enumerate(unicode_text_lines_from_binary(location), start_line)
return iter([])
def test_jsp_is_markup(self):
test_file = self.get_test_loc(u'markup/java.jsp')
assert markup.is_markup(test_file)
def text_lines(location, demarkup=False):
"""
Return a text lines iterator from file at `location`. Return an empty
iterator if no text content is extractible. Text extraction is based on
detected file type.
if `demarkup` is True, attempt to detect if a file contains HTML/XML-like
markup and cleanup this markup.
Note: For testing or building from strings, location can be a is a list of
unicode line strings.
"""
# TODO: add support for "wide" UTF-16-like strings where each char is
# followed by a zero as is often found in some Windows binaries. Do this for
# binaries only. This is in direct conflict with "strings" extraction as
# currently implemented
if not location:
return iter([])
if not isinstance(location, basestring):
# not a path: wrap an iterator on location which should be a sequence
# of lines
return iter(location)
T = typecode.get_type(location)
if not T.contains_text:
return iter([])
# Should we read this as some markup, pdf office doc, text or binary?
if T.is_pdf:
return unicode_text_lines_from_pdf(location)
# lightweight markup stripping support
if demarkup and markup.is_markup(location):
try:
return markup.demarkup(location)
except:
# try again later with as plain text
pass
# TODO: handle Office-like documents, RTF, etc
# if T.is_doc:
# return unicode_text_lines_from_doc(location)
if T.is_text:
return unicode_text_lines(location)
# DO NOT introspect media, archives and compressed files
# if not T.contains_text:
# return iter([])
if T.is_binary:
# fall back to binary
return unicode_text_lines_from_binary(location)
else:
# if neither text, text-like nor binary: treat as binary
# this should never happen
# fall back to binary
return unicode_text_lines_from_binary(location)