class CompressedData:
def __init__(self, data_file, is_open_stream = False):
"""Extracts meta data.
If is_open_stream is True data_file is treated as a readable
binary stream."""
self.data_stream = File(data_file, is_open_stream)
magic_number = self.data_stream.read(len(MAGIC_NUMBER))
if magic_number != MAGIC_NUMBER.encode("utf-8"):
raise NotCorrectMagicNumber("Compressed file should start with\
{}".format(MAGIC_NUMBER))
meta = self._get_meta(self.data_stream)
self.orig_length = meta["orig_length"]
self.word_length = meta["word_length"]
self.encode_dict = meta["encode_dict"]
self.data_index = meta["data_index"]
def _get_meta(self, data_stream):
"""Returns a dict containing original data length,
the encode table and the compressed data"""
meta = dict()
orig_length = int.from_bytes(self.data_stream.read(8), "big")
word_length = int.from_bytes(self.data_stream.read(8), "big")
encode_dict_temp = BytesIO()
while True:
c = self.data_stream.read(1)
encode_dict_temp.write(c)
if c == b"}":
break
data_start = self.data_stream.tell() + len("DICT_END")
encode_dict_temp.seek(0)
encode_dict = encode_dict_temp.read()
encode_dict = encode_dict[encode_dict.index(b"{") + 1: encode_dict.index(b"}")]
meta["orig_length"] = orig_length
meta["word_length"] = word_length
meta["encode_dict"] = encode_dict
meta["data_index"] = data_start
return meta
def __init__(self, input_file, output_file, debug=False):
self.time_start = time.time()
self.time_finish = None
self.time_execution = None
self.input_file = input_file
self.output_file = output_file
self.input_data = File.read(input_file)
self.output_data = []
self.debug = debug
def create_config_file(self,host=None, jid=None, name=None, ip=None):
try:
filee = File()
supervisord_config_template = filee.read(SUPERVISORD["CONF_TEMPLATE_DIR"])
except Exception as e:
self.logger.error(str(e))
base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
supervisord_config = supervisord_config_template.replace('{name}', name)
supervisord_config = supervisord_config.replace('{base_dir}', base_dir)
supervisord_config = supervisord_config.replace('{host}', host)
supervisord_config = supervisord_config.replace('{jid}', str(jid))
supervisord_config = supervisord_config.replace('{ip}', ip)
return supervisord_config
class RawChunk():
def __init__(self, filename, bounds=Bounds(0,-1)):
self._file = File(filename)
self._bounds = bounds
def close(self):
self._file.close()
@property
def file(self):
return self._file
@property
def bounds(self):
return self._bounds
@property
def valid(self):
return self._bounds.apply(self._file.length).size > 0
@property
def size(self):
return self._bounds.apply(self._file.length).size
def __repr__(self):
return '<RawChunk %s%s>' % (self._file.name, self._bounds)
def read(self, limit=256):
return self._file.read(self._bounds, limit)
def readhex(self, limit=256):
return self._file.readhex(self._bounds, limit)
def readhexs(self, bounds, bytesperline=16):
return self._file.readhexs(self._bounds, bytesperline)
def crc32(self):
return self._file.crc32(self._bounds)
def adler32(self):
return self._file.adler32(self._bounds)
def md5(self):
return self._file.md5(self._bounds)
def _split(self, position):
# print 'D: cutting %s at %d' % (self._bounds, position)
if not self.bounds.contains(position):
before, after = (RawChunk(self._file.name, self._bounds), None)
else:
before = RawChunk(self._file.name, Bounds(self._bounds.start, position - 1))
after = RawChunk(self._file.name, Bounds(position, self._bounds.end))
before = before.valid and before or None
after = after.valid and after or None
self.close()
return (before, after)
def chop(self, bounds):
before, remains = self._split(bounds.start)
if remains is None:
chopped, after = (None, None)
else:
chopped, after = remains._split(bounds.end + 1)
return (before, chopped, after)
def cutat(self, position):
return self._split(position)
def debit(self, count, chopsize):
chops = []
remains = self
for i in range(0, count):
if remains is not None and remains.valid:
head, remains = remains._split(remains.bounds.start + chopsize)
chops.append(head)
chops.append(remains)
return chops
def divide(self, count):
chopsize = max(self.size / count, 1)
return self.debit(count, chopsize)
# r = RawChunk("test/data")
# print r
# print r.bounds.size
# print r.size
# print [ r.read() for r in r.debit(1, -1) ]
