def sentences_from_sources(sources, log_errors=False): if len(sources) > 0: for source in sources: for sentence in sentences_from_source(source, log_errors): yield sentence else: for sentence in sentences_from_source(sys.stdin, log_errors): yield sentence
def sentences_from_sources(sources, log_errors=False): if len(sources) > 0: for source in sources: try: for sentence in sentences_from_source(source, log_errors): yield sentence except: logging.exception("Unexpected failure with source {}; continuing".format(source)) else: for sentence in sentences_from_source(sys.stdin, log_errors): yield sentence
def burst(source, dest, verbose): """ Takes large AIS files and splits them up by sender. """ if not dest: dest = source writers = {} fname, ext = os.path.splitext(dest) for sentence in sentences_from_source(source, log_errors=verbose): mmsi = sentence['mmsi'] if not mmsi: mmsi = 'other' if mmsi not in writers: writers[mmsi] = open("{}-{}{}".format(fname, mmsi, ext), "wt") print_sentence_source(sentence, writers[mmsi]) for writer in writers.values(): writer.close()
def real_thing(): global t2 with Timer() as t2: for sentence in sentences_from_source(filename): counter.count(sentence['mmsi'])