def open(self, file_name, mode): self.source = self.source_class(file_name, mode).open() # skip offset number of lines logger.info("Skipping %d lines in [%s]." % (self.offset, file_name)) start = time.perf_counter() io.skip_lines(file_name, self.source, self.offset) end = time.perf_counter() logger.info("Skipping %d lines took %f s." % (self.offset, end - start)) return self
def open(self, file_name, mode, bulk_size): self.bulk_size = bulk_size self.source = self.source_class(file_name, mode).open() self.logger.info("Will read [%d] lines from [%s] starting from line [%d] with bulk size [%d].", self.number_of_lines, file_name, self.offset, self.bulk_size) start = time.perf_counter() io.skip_lines(file_name, self.source, self.offset) end = time.perf_counter() self.logger.debug("Skipping [%d] lines took [%f] s.", self.offset, end - start) return self
def __enter__(self): self.f = self.file_source.open(self.data_file, 'rt') # skip offset number of lines logger.info("Skipping %d lines in [%s]." % (self.offset, self.data_file)) start = time.perf_counter() io.skip_lines(self.data_file, self.f, self.offset) end = time.perf_counter() logger.info("Skipping %d lines took %f s." % (self.offset, end - start)) return self