def sentences_from_sources(sources, log_errors=False):
if len(sources) > 0:
for source in sources:
for sentence in sentences_from_source(source, log_errors):
yield sentence
else:
for sentence in sentences_from_source(sys.stdin, log_errors):
yield sentence
def sentences_from_sources(sources, log_errors=False):
if len(sources) > 0:
for source in sources:
for sentence in sentences_from_source(source, log_errors):
yield sentence
else:
for sentence in sentences_from_source(sys.stdin, log_errors):
yield sentence
def sentences_from_sources(sources, log_errors=False):
if len(sources) > 0:
for source in sources:
try:
for sentence in sentences_from_source(source, log_errors):
yield sentence
except:
logging.exception("Unexpected failure with source {}; continuing".format(source))
else:
for sentence in sentences_from_source(sys.stdin, log_errors):
yield sentence
def burst(source, dest, verbose):
""" Takes large AIS files and splits them up by sender. """
if not dest:
dest = source
writers = {}
fname, ext = os.path.splitext(dest)
for sentence in sentences_from_source(source, log_errors=verbose):
mmsi = sentence['mmsi']
if not mmsi:
mmsi = 'other'
if mmsi not in writers:
writers[mmsi] = open("{}-{}{}".format(fname, mmsi, ext), "wt")
print_sentence_source(sentence, writers[mmsi])
for writer in writers.values():
writer.close()
def burst(source, dest, verbose):
""" Takes large AIS files and splits them up by sender. """
if not dest:
dest = source
writers = {}
fname, ext = os.path.splitext(dest)
for sentence in sentences_from_source(source, log_errors=verbose):
mmsi = sentence['mmsi']
if not mmsi:
mmsi = 'other'
if mmsi not in writers:
writers[mmsi] = open("{}-{}{}".format(fname, mmsi, ext), "wt")
print_sentence_source(sentence, writers[mmsi])
for writer in writers.values():
writer.close()
def real_thing():
global t2
with Timer() as t2:
for sentence in sentences_from_source(filename):
counter.count(sentence['mmsi'])
