class CompressedData: def __init__(self, data_file, is_open_stream = False): """Extracts meta data. If is_open_stream is True data_file is treated as a readable binary stream.""" self.data_stream = File(data_file, is_open_stream) magic_number = self.data_stream.read(len(MAGIC_NUMBER)) if magic_number != MAGIC_NUMBER.encode("utf-8"): raise NotCorrectMagicNumber("Compressed file should start with\ {}".format(MAGIC_NUMBER)) meta = self._get_meta(self.data_stream) self.orig_length = meta["orig_length"] self.word_length = meta["word_length"] self.encode_dict = meta["encode_dict"] self.data_index = meta["data_index"] def _get_meta(self, data_stream): """Returns a dict containing original data length, the encode table and the compressed data""" meta = dict() orig_length = int.from_bytes(self.data_stream.read(8), "big") word_length = int.from_bytes(self.data_stream.read(8), "big") encode_dict_temp = BytesIO() while True: c = self.data_stream.read(1) encode_dict_temp.write(c) if c == b"}": break data_start = self.data_stream.tell() + len("DICT_END") encode_dict_temp.seek(0) encode_dict = encode_dict_temp.read() encode_dict = encode_dict[encode_dict.index(b"{") + 1: encode_dict.index(b"}")] meta["orig_length"] = orig_length meta["word_length"] = word_length meta["encode_dict"] = encode_dict meta["data_index"] = data_start return meta
def __init__(self, input_file, output_file, debug=False): self.time_start = time.time() self.time_finish = None self.time_execution = None self.input_file = input_file self.output_file = output_file self.input_data = File.read(input_file) self.output_data = [] self.debug = debug
def create_config_file(self,host=None, jid=None, name=None, ip=None): try: filee = File() supervisord_config_template = filee.read(SUPERVISORD["CONF_TEMPLATE_DIR"]) except Exception as e: self.logger.error(str(e)) base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) supervisord_config = supervisord_config_template.replace('{name}', name) supervisord_config = supervisord_config.replace('{base_dir}', base_dir) supervisord_config = supervisord_config.replace('{host}', host) supervisord_config = supervisord_config.replace('{jid}', str(jid)) supervisord_config = supervisord_config.replace('{ip}', ip) return supervisord_config
class RawChunk(): def __init__(self, filename, bounds=Bounds(0,-1)): self._file = File(filename) self._bounds = bounds def close(self): self._file.close() @property def file(self): return self._file @property def bounds(self): return self._bounds @property def valid(self): return self._bounds.apply(self._file.length).size > 0 @property def size(self): return self._bounds.apply(self._file.length).size def __repr__(self): return '<RawChunk %s%s>' % (self._file.name, self._bounds) def read(self, limit=256): return self._file.read(self._bounds, limit) def readhex(self, limit=256): return self._file.readhex(self._bounds, limit) def readhexs(self, bounds, bytesperline=16): return self._file.readhexs(self._bounds, bytesperline) def crc32(self): return self._file.crc32(self._bounds) def adler32(self): return self._file.adler32(self._bounds) def md5(self): return self._file.md5(self._bounds) def _split(self, position): # print 'D: cutting %s at %d' % (self._bounds, position) if not self.bounds.contains(position): before, after = (RawChunk(self._file.name, self._bounds), None) else: before = RawChunk(self._file.name, Bounds(self._bounds.start, position - 1)) after = RawChunk(self._file.name, Bounds(position, self._bounds.end)) before = before.valid and before or None after = after.valid and after or None self.close() return (before, after) def chop(self, bounds): before, remains = self._split(bounds.start) if remains is None: chopped, after = (None, None) else: chopped, after = remains._split(bounds.end + 1) return (before, chopped, after) def cutat(self, position): return self._split(position) def debit(self, count, chopsize): chops = [] remains = self for i in range(0, count): if remains is not None and remains.valid: head, remains = remains._split(remains.bounds.start + chopsize) chops.append(head) chops.append(remains) return chops def divide(self, count): chopsize = max(self.size / count, 1) return self.debit(count, chopsize) # r = RawChunk("test/data") # print r # print r.bounds.size # print r.size # print [ r.read() for r in r.debit(1, -1) ]