def __encrypt_file(self): """ Core method to encrypt a file. """ fs_input = common.get_file_size(self.__file_input) fs_output = fs_input + self.__obfuscate_enc if self.__reverse_bytes > fs_input: common.exception("The reverse byte value must not be greater " \ "than the input file size.") if self.__existing_key: fs_key = common.get_file_size(self.__file_key) if (fs_key - self.__obfuscate_key) < \ (fs_input - self.__obfuscate_enc): common.exception("The given key file is too small.") else: fs_key = fs_input + self.__obfuscate_key common.build_task_file(self.__task_id, self.__file_input, fs_input, self.__file_key, fs_key, self.__file_output, fs_output, "encryption") keyfile.build_file_key(self.__file_key, fs_input, self.__buffer_size, self.__obfuscate_key, self.__existing_key, self.__dev_random, self.__fortuna, self.__overwrite, 1) self.__file_size = fs_input self.__erfr_core() if int(self.__obfuscate_enc) > 0: obfuscator.add_random_bytes(self.__file_output, self.__buffer_size, self.__obfuscate_enc, self.__dev_random, self.__fortuna)
def compare_files(file_input, directory, ignore_read_errors=True, obfuscate_enc=0, obfuscate_key=0): """ Compare files to find out which key fits to an encrypted file and vice versa. """ pv.path(file_input, "input", True, True) pv.path(directory, "compare", False, True) pv.intvalue(obfuscate_enc, "encrypted file obfuscation", True, True, \ False) pv.intvalue(obfuscate_key, "key file obfuscation", True, True, False) obfuscate_enc = int(obfuscate_enc) obfuscate_key = int(obfuscate_key) file_input = os.path.abspath(file_input) file_input_size = int(common.get_file_size(file_input)) directory = os.path.abspath(directory) list_files = [] for item in os.listdir(directory): path = os.path.join(directory, item) if os.path.isfile(path): if path == file_input: continue file_enc_size1 = int(common.get_file_size(path) + obfuscate_enc) file_enc_size2 = int(common.get_file_size(path) - obfuscate_enc) file_key_size1 = int(common.get_file_size(path) + obfuscate_key) file_key_size2 = int(common.get_file_size(path) - obfuscate_key) try: if file_input_size == file_enc_size1 or \ file_input_size == file_enc_size2 or \ file_input_size == file_key_size1 or \ file_input_size == file_key_size2: list_files.append(path) except Exception as e: if not ignore_read_errors: raise Exception(e) else: pass list_files.sort() return list_files
def merge_key(file_output, buffer_size=4096, overwrite=False): """ Merge multiple key file parts to a single key file. """ if not overwrite: pv.path(file_output, "output key", True, False) else: pv.path(file_output, "key", True, True) part_id = 0 fh_output = open(file_output, "wb") while True: part_id += 1 file_key = file_output + "." + str(part_id).rjust(3, "0") if not os.path.exists(file_key): break fh_key = open(file_key, "rb") file_size = common.get_file_size(file_key) byte_blocks = int(file_size / buffer_size) byte_remainder = file_size % buffer_size for block in range(byte_blocks): fh_output.write(fh_key.read(buffer_size)) if byte_remainder > 0: fh_output.write(fh_key.read(byte_remainder)) fh_key.close() fh_output.close()
def open(self, file_path): with open(file_path, 'rb') as f: file_size = common.get_file_size(f) # Read magic header magic_number = f.read(4) if magic_number != 'BIND': raise Exception('Not a BIND file! Missing BIND header id.') # Read "size" byte size # This determines what's the "size" of the size variable itself self.size_byte_size = common.read_uint16(f) if self.size_byte_size not in (1, 2, 4): raise Exception('Illegal BIND size byte size: %s' % self.index_size) # Read number of entries number_of_entries = common.read_uint16(f) # Read block size self.block_size = common.read_uint32(f) # Read header size header_size = common.read_uint32(f) # The header size is: # (16 + number of entries * self.size_byte_size) aligned to self.block_size # Load the entry sizes processed_entries = [] previous_entry_end = header_size for i in xrange(0, number_of_entries): # Calculate the offset of this entry based on the end of the previous entry entry_offset = previous_entry_end # Read the entry size entry_size = 0 if self.size_byte_size == 1: entry_size = common.read_uint8(f) elif self.size_byte_size == 2: entry_size = common.read_uint16(f) elif self.size_byte_size == 4: entry_size = common.read_uint32(f) # Calculate how much memory this entry takes entry_padded_size = common.align_size(entry_size, self.block_size) # Update previous_entry_end for the next iteration previous_entry_end += entry_padded_size # Add the entry to processed entries processed_entries.append((entry_offset, entry_size)) # Read entries for (entry_offset, entry_size) in processed_entries: f.seek(entry_offset) content = f.read(entry_size) new_entry = self.add_entry(content)
def main(argv): args = read_parameter(argv) if args.info: logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") if args.debug: logging.basicConfig(level=logging.DEBUG, format="%(levelname)s: %(message)s") else: logging.basicConfig(level=logging.WARNING, format="%(message)s") logging.info('Input folder: %s' % args.folder) tar_file = create_tar(args.folder, args.compress) logging.info('Created TAR file: %s' % tar_file) size = get_file_size(tar_file) logging.info('File size: %s (%i bytes)' % (sizeof_fmt(size), size)) tree_hash_hex = get_tree_hash_of_file(tar_file) logging.info('Hash (SHA-256 treehash): %s' % tree_hash_hex) description = "files from %s" % re.sub(r'[^\x00-\x7F]+','', args.folder) # remove non ASCII chars archive_id = upload_to_glacier(tar_file, size, description, args.vault, tree_hash_hex) delete_temp_file(tar_file) logging.info('Removed temporary file') now = datetime.datetime.now() print "%s\t%s\t%s\t%s\t%s" % (now, args.folder, args.vault, archive_id, tree_hash_hex)
def __decrypt_file(self): """ Core method to decrypt a file. """ fs_input = common.get_file_size(self.__file_input) fs_key = common.get_file_size(self.__file_key) fs_output = fs_input - self.__obfuscate_enc if self.__reverse_bytes > fs_input: common.exception("The reverse byte value must not be greater " \ "than the input file size.") common.build_task_file(self.__task_id, self.__file_input, fs_input, self.__file_key, fs_key, self.__file_output, fs_output, "decryption") self.__file_size = fs_output self.__erfr_core()
def save(self, file_path): with open(file_path, 'wb') as f: file_size = common.get_file_size(f) # Write magic header f.write('.EVS') # Write the number of entries common.write_uint32(f, len(self.entries)) # Write entry offsets # Size of header + size of entry table previous_entry_end = 8 + 4 * len(self.entries) converted_entries = [] for (entry_type, entry_parameters, entry_content) in self.entries: # Write offset common.write_uint32(f, previous_entry_end) # Calculate the size of this entry, # so that we know when the next entry starts # Convert UTF8 to Shift-JIS entry_content = common.to_eva_sjis(entry_content) # Calculate the entry_size # parameters + content entry_size = 4 * len(entry_parameters) + len(entry_content) # Update previous_entry_end for the next iteration # Add 4 for the entry_type and entry_size fields, along with padding previous_entry_end += common.align_size(4 + entry_size, 4) # Add the entry to the converted entries converted_entries.append( (entry_type, entry_size, entry_parameters, entry_content)) # Loop through entries for (entry_type, entry_size, entry_parameters, entry_content) in converted_entries: # Write entry type common.write_uint16(f, entry_type) # Write entry size common.write_uint16(f, entry_size) # Write the parameters for entry_parameter in entry_parameters: common.write_uint32(f, entry_parameter) # Write the content f.write(entry_content) # Add padding entry_padding = (common.align_size(entry_size, 4) - entry_size) f.write('\0' * entry_padding)
def index(self): current_blobs = [] for infile in os.listdir(self.box_path): if os.path.isdir(infile) or infile == '.meta': continue # Get file full path full_path = os.path.join(self.box_path, infile) checksum = hash_file(full_path) timestamp = get_file_timetamp(full_path) size = get_file_size(full_path) # Just to ease debugging. data = '%s:%s:%s:%s\n' % (checksum, infile, timestamp, size) blobpath = os.path.join(self.registry, checksum) # Append to current blobs current_blobs.append({ 'checksum': checksum, 'filename': blobpath }) # Write the blob if we don't already have it. if not os.path.isfile(blobpath): try: # Todo, write much more information to the meta-blob file. # Last modified, which server modified, .. print("Writing blob %s (%s)" % (checksum, infile,)) blobfile = open(blobpath, 'w') blobfile.write(data) blobfile.close() except IOError: raise Exception("Writing a blob failed") else: print("Blob %s exists" % checksum) # Clear orphaned files. for checksum in os.listdir(self.registry): blob_path = os.path.join(self.registry, checksum) content = open(blob_path, 'r').read() found = False for current in current_blobs: if current['checksum'] == checksum: found = True break if not found: print "Found obsolete %s blob" % checksum os.unlink(blob_path)
def split_key(file_input, parts, buffer_size=4096, overwrite=False): """ Split a key file in a user-defined number of parts. """ if not overwrite: pv.path(file_input, "key", True, True) pv.path(file_input + ".001", "key part", True, False) else: pv.path(file_input, "key", True, True) buffer_size = int(buffer_size) parts = int(parts) if parts < 2: common.exception("The number of key parts must be greater than 1.") elif parts > 999: common.exception("The number of key parts must be less than 1000.") file_size = common.get_file_size(file_input) part_id = 0 part_size = int(file_size / parts) part_last = file_size - (part_size * (parts - 1)) fh_input = open(file_input, "rb") for part in range(parts): part_id += 1 file_key = file_input + "." + str(part_id).rjust(3, "0") if part_id < parts: file_size = int(part_size) else: file_size = int(part_last) fh_output = open(file_key, "wb") byte_blocks = int(file_size / buffer_size) byte_remainder = file_size % buffer_size for block in range(byte_blocks): fh_output.write(fh_input.read(buffer_size)) if byte_remainder > 0: fh_output.write(fh_input.read(byte_remainder)) fh_output.close() fh_input.close()
def open(self, file_path): with open(file_path, 'rb') as f: archive_size = common.get_file_size(f) while f.tell() < archive_size: magic_number = f.read(4).decode('ascii', 'ignore') if magic_number != 'RIFF': raise Exception('Not a WAVE entry!') wave_size = common.read_uint32(f) + 8 f.seek(-8, os.SEEK_CUR) content = f.read(wave_size) new_entry = self.add_entry(content) # Find how many blocks are needed to contain wave_size # and skip that amount of remaining bytes block_aligned_wave_size = common.align_size( wave_size, WaveArchive.BLOCK_SIZE) f.seek(block_aligned_wave_size - wave_size, os.SEEK_CUR)
def obfuscate_file(task_id, file_path, buffer_size=4096, bytes_random=0, dev_random=False, fortuna=False): """ Create a task file first, then add given amount of random bytes to the given file. """ pv.path(file_path, "target", True, True) pv.intvalue(buffer_size, "buffer size", True, False, False) pv.intvalue(bytes_random, "random bytes", True, False, False) buffer_size = int(buffer_size) bytes_random = int(bytes_random) file_path = os.path.abspath(file_path) file_size = common.get_file_size(file_path) + int(bytes_random) common.build_task_file(task_id, file_path, file_size, None, 0, None, 0, "file obfuscation") add_random_bytes(file_path, buffer_size, bytes_random, dev_random, fortuna)
def upload(self, local_file, remote_path): # 上传文件 """ 小的单文件上传 :param local_file: 本地文件路径 :param remote_path: 远程路径(带文件名) :return: None """ at = self.__get_access_token() headers = {'Authorization': 'bearer ' + at} file_size = common.get_file_size(local_file) logging.debug('文件大小:{0}'.format(file_size)) url = self.app_url + '_api/v2.0/me/drive/root:/' + remote_path + ':/content' file = open(local_file, 'rb') r = requests.put(url, headers=headers, data=file) r.encoding = "utf-8" if r.status_code == 200 or r.status_code == 201: # 返回json格式的数据 print(r.json()) else: logging.debug('接口访问失败')
def open(self, file_path): with open(file_path, 'rb') as f: file_size = common.get_file_size(f) # Read magic header magic_number = f.read(4).decode('ascii', 'ignore') if magic_number != 'TEXT': raise Exception('Not a TEXT file!') # Read number of entries number_of_entries = common.read_uint32(f) # Read size of header header_size = common.read_uint32(f) if (header_size != 16): msg = '# Warning: Non-standard TEXT header size: %s' % header_size self.warn(msg) # Read content start offset content_start_offset = common.read_uint32(f) # There should be no post-header padding, # but check for it anyways pre_header_skip_position = f.tell() # Skip the header f.seek(header_size) # Complete the header padding calculation self.header_padding = f.tell() - pre_header_skip_position # Read entries # Don't create them yet, # since we want to store the string index instead of the offset # and we can't figure out the string index until we've seen all the strings entry_unknowns = [0] * number_of_entries entry_string_offsets = [0] * number_of_entries for i in range(0, number_of_entries): entry_unknowns[i] = common.read_uint32(f) entry_string_offsets[i] = common.read_uint32(f) # There should be no post-header padding, # but check for it anyways pre_entry_skip_position = f.tell() # Skip to the content start f.seek(content_start_offset) # Complete the header padding calculation self.entry_padding = f.tell() - pre_entry_skip_position # Read the actual strings (which are used multiple times per entry) # Figure out the string index by sorting string offsets from low to high string_offset_index_map = {} for index, offset in enumerate(sorted(set(entry_string_offsets))): # Update the string offset map string_offset_index_map[offset] = index # Go to the string offset f.seek(offset) # If offset is at the end, skip if (offset >= file_size): self.warn( 'Out of bounds string in TEXT file, using nulls instead' ) self.strings.append((None, None, None)) continue # Read the unknowns unknown_first = common.read_uint32(f) unknown_second = common.read_uint32(f) # Read content until null-terminator (but aligned to 32-bit boundaries) raw_string_content = b'' while True: raw_string_content += f.read(4) if raw_string_content[-1] == 0: break # Convert Shift-JIS to unicode string_content = common.from_eva_sjis(raw_string_content) # Strip trailing \0's string_content = string_content.rstrip('\0') # Add this string to the array self.strings.append( (unknown_first, unknown_second, string_content)) # Finally create the entries now that we know the string offset to index mapping for i in range(0, number_of_entries): self.entries.append( (entry_unknowns[i], string_offset_index_map[entry_string_offsets[i]]))
def save(self, file_path): with open(file_path, 'wb') as f: file_size = common.get_file_size(f) # Write magic header f.write(b'.EVS') # Write the number of entries common.write_uint32(f, len(self.entries)) # Write entry offsets # Size of header + size of entry table previous_entry_end = 8 + 4 * len(self.entries) converted_entries = [] for (entry_type, entry_parameters, entry_content) in self.entries: has_content_section = entry_type in HAS_CONTENT_SECTION # Write offset common.write_uint32(f, previous_entry_end) # Calculate the size of this entry, # so that we know when the next entry starts # Calculate how much memory this string takes if has_content_section: # Convert UTF8 to Shift-JIS raw_entry_content = common.to_eva_sjis(entry_content) # We add a single null terminator and only count the single null terminator, # but later we include the extra alignment padding in the size calculation string_terminated_size = len(raw_entry_content) + 1 raw_entry_content = common.zero_pad_and_align_string( raw_entry_content) else: string_terminated_size = 0 raw_entry_content = b'' # Calculate the entry_size # parameters + content entry_size = 4 * len(entry_parameters) + string_terminated_size # Update previous_entry_end for the next iteration # Add 4 for the entry_type (omitted in the entry_size calculation) previous_entry_end += common.align_size(4 + entry_size, 4) # Add the entry to the converted entries converted_entries.append((entry_type, entry_size, entry_parameters, raw_entry_content)) # Loop through entries for (entry_type, entry_size, entry_parameters, raw_entry_content) in converted_entries: # Write entry type common.write_uint16(f, entry_type) # Write entry size common.write_uint16(f, entry_size) # Write the parameters for entry_parameter in entry_parameters: common.write_uint32(f, entry_parameter) # Write the content f.write(raw_entry_content)
def open(self, file_path): with open(file_path, 'rb') as f: file_size = common.get_file_size(f) # Read magic header magic_number = f.read(4) if magic_number != 'HGPT': raise Exception('Not an HGPT file! Missing HGPT header id.') # Read pp offset pp_offset = common.read_uint16(f) if pp_offset < 0x10: raise Exception( 'PP offset less than 0x10, PS2 variant not supported!') # Read if it has an extended header has_extended_header = common.read_uint16(f) if has_extended_header not in (0, 1): raise Exception('Unknown has_extended_header value: %s' % has_extended_header) self.has_extended_header = has_extended_header = ( has_extended_header == 1) # Read number of divisions number_of_divisions = common.read_uint16(f) # Read unknowns unknown_one = common.read_uint16(f) if unknown_one != 0x0001: raise Exception('First unknown is not 0x0001: %08X' % unknown_one) # ff ff ff ff in pictures with extended header # 00 00 00 00 in pictures w/o extended header unknown_two = common.read_uint32(f) # Load divisions if has_extended_header: # Read number of divisions (again) number_of_divisions_repeat = common.read_uint16(f) if number_of_divisions != number_of_divisions_repeat: raise Exception( 'Number of divisions and its repeat don\'t match: %s != %s' % (number_of_divisions, number_of_divisions_repeat)) # Read unknown # 0x0013 is the most common # 0x0014 is the other occurence unknown_three = common.read_uint16(f) if unknown_three != 0x0013: print '# Warning: UnknownThree (0x%X) != 0x0013' % unknown_three # Read division name self.division_name = (f.read(8) + '\0' * 8)[0:8] # Read divisions for i in xrange(0, number_of_divisions): division_start_x = common.read_uint16(f) division_start_y = common.read_uint16(f) division_width = common.read_uint16(f) division_height = common.read_uint16(f) self.divisions.append((division_start_x, division_start_y, division_width, division_height)) # Calculate and skip zero padding divisions_size = 12 + 8 * number_of_divisions divisions_padded_size = common.align_size(divisions_size, 16) divisions_padding = divisions_padded_size - divisions_size f.seek(divisions_padding, os.SEEK_CUR) # Check that it's the correct pp_offset if f.tell() != pp_offset: raise Exception('Incorrect pp offset') # Read pp header pp_header = common.read_uint32(f) if pp_header & 0x0000FFFF != 0x00007070: raise Exception('Missing pp header! %08X' % pp_header) # Decipher pp format pp_format = (pp_header >> 16) & 0xFFFF if pp_format not in (0x13, 0x14, 0x8800): raise Exception('PP format (0x%X) is unknown' % pp_format) # Calculate values that depend on the pp format bytes_per_pixel = 1 bytes_per_pixel_ppd_size = bytes_per_pixel tile_width = 16 if pp_format == 0x8800: bytes_per_pixel = 4 bytes_per_pixel_ppd_size = 1 tile_width = 4 elif pp_format == 0x13: bytes_per_pixel = 1 bytes_per_pixel_ppd_size = bytes_per_pixel tile_width = 16 elif pp_format == 0x14: bytes_per_pixel = 0.5 bytes_per_pixel_ppd_size = bytes_per_pixel tile_width = 32 # Read picture display dimensions self.width = pp_display_width = common.read_uint16(f) self.height = pp_display_height = common.read_uint16(f) # Skip zero padding f.seek(2 * 4, os.SEEK_CUR) # Read ppd header ppd_header = common.read_uint32(f) if ppd_header & 0x00FFFFFF != 0x00647070: raise Exception('Missing ppd header!') # Decipher ppd format ppd_format = (ppd_header >> 24) & 0xFF if ppd_format != (pp_format & 0xFF): raise Exception( 'PPD format (0x%X) does not match PP format (0x%X)' % (ppd_format, pp_format & 0xFF)) # Read ppd display resolution ppd_display_width = common.read_uint16(f) ppd_display_height = common.read_uint16(f) if pp_display_width != ppd_display_width: raise Exception( 'PP display width (%s) != PPD display width (%s)' % (pp_display_width, ppd_display_width)) if pp_display_height != ppd_display_height: raise Exception( 'PP display height (%s) != PPD display height (%s)' % (pp_display_height, ppd_display_height)) # Skip zero padding f.seek(4, os.SEEK_CUR) # Read ppd sixteenths resolution ppd_sixteenths_width = common.read_uint16(f) ppd_sixteenths_height = common.read_uint16(f) # Calculate ppd sixteenths resolution (using the pp_display resolution) calculated_sixteenths_width = common.align_size( pp_display_width, 16) calculated_sixteenths_height = common.align_size( pp_display_height, 8) if (calculated_sixteenths_width != ppd_sixteenths_width) or ( calculated_sixteenths_height != ppd_sixteenths_height): raise Exception( 'PPD sixteenths resolution (%s x %s) doesn\'t match the calculated eights resolution (%s x %s)' % (ppd_sixteenths_width, ppd_sixteenths_height, calculated_sixteenths_width, calculated_sixteenths_height)) # Calculate storage resolution (using the pp_display resolution) # This is the resolution that ties in with the ppd_size calculated_storage_width = common.align_size( pp_display_width, tile_width) calculated_storage_height = common.align_size(pp_display_height, 8) number_of_pixels = calculated_storage_width * calculated_storage_height # Read ppd_size which is the size of the ppd header + number of pixels # The ppd header which is 0x20 bytes in size ppd_size = common.read_uint32(f) calculated_ppd_size = int( number_of_pixels * bytes_per_pixel_ppd_size) + 0x20 if (calculated_ppd_size != ppd_size): raise Exception( 'PPD size %s does not match the calculated ppd size %s' % (ppd_size, calculated_ppd_size)) # Calculate the number of bytes number_of_bytes = int(number_of_pixels * bytes_per_pixel) # Skip padding f.seek(3 * 4, os.SEEK_CUR) # Read the tiled image data # The image data is stored in a scrambled tiled format, so we'll have to reprocess it tiled_image_data = [0] * number_of_pixels cache_last_pixel = None for i in xrange(0, number_of_pixels): if number_of_bytes <= 0: break if pp_format == 0x13: tiled_image_data[i] = common.read_uint8(f) number_of_bytes -= 1 elif pp_format == 0x14: if (i & 1) == 0: # Even read cache_last_pixel = common.read_uint8(f) else: # Odd read number_of_bytes -= 1 tiled_image_data[i] = cache_last_pixel & 0xF cache_last_pixel = cache_last_pixel >> 4 elif pp_format == 0x8800: # Read full RGBA tiled_image_data[i] = (common.read_uint8(f), common.read_uint8(f), common.read_uint8(f), decode_alpha(common.read_uint8(f))) number_of_bytes -= 4 # Skip unread bytes f.seek(number_of_bytes, os.SEEK_CUR) # Un-tile and store the information as the content self.content = [0] * (pp_display_width * pp_display_height) tile_height = 8 tile_size = tile_width * tile_height tile_row = tile_size * int(calculated_storage_width / tile_width) for y in xrange(0, pp_display_height): for x in xrange(0, pp_display_width): tile_y = int(y / tile_height) tile_x = int(x / tile_width) tile_sub_y = y % tile_height tile_sub_x = x % tile_width self.content[y * pp_display_width + x] = tiled_image_data[tile_y * tile_row + tile_x * tile_size + tile_sub_y * tile_width + tile_sub_x] # Check if optional ppc header exists if pp_format == 0x8800: self.palette = [] else: # Read ppc header ppc_header = common.read_uint32(f) if ppc_header & 0xFFFFFFFF != 0x00637070: raise Exception('Missing ppc header!') # Skip zero padding f.seek(2, os.SEEK_CUR) # Read the number of palette entries # It needs to be multiplied by 8 to get the correct total palette_total = common.read_uint16(f) * 8 # Skip zero padding f.seek(2 * 4, os.SEEK_CUR) # Read palette self.palette = [(common.read_uint8(f), common.read_uint8(f), common.read_uint8(f), decode_alpha(common.read_uint8(f))) for i in xrange(0, palette_total)]
def open(self, file_path): with open(file_path, 'rb') as f: file_size = common.get_file_size(f) # Read magic header magic_number = f.read(4).decode('ascii', 'ignore') if magic_number != '.EVS': raise Exception('Not an EVS file!') # Read the number of entries number_of_entries = common.read_uint32(f) # Read entry offsets entry_offsets = [0] * number_of_entries for i in range(0, number_of_entries): entry_offsets[i] = common.read_uint32(f) # Loop through entries for i in range(0, number_of_entries): f.seek(entry_offsets[i]) # Read entry type entry_type = common.read_uint16(f) # Read entry size entry_size = common.read_uint16(f) # Read entry number_of_parameters = get_number_of_parameters(entry_type) has_content_section = entry_type in HAS_CONTENT_SECTION if number_of_parameters is None: # Commands that I've yet to document the parameter count for raise Exception( 'Unknown number of parameters for entry type: 0x%X, size: %s' % (entry_type, entry_size)) # Read the parameters entry_parameters = [] for j in range(0, number_of_parameters): entry_parameters.append(common.read_uint32(f)) # Whatever remains after the parameters is the content parameter_size = 4 * number_of_parameters remaining_bytes = entry_size - parameter_size if remaining_bytes < 0: raise Exception( 'Number of parameters overshoots total entry size; entry type: 0x%X, size: %s' % (entry_type, entry_size)) if has_content_section: raw_entry_content = f.read(remaining_bytes) # Convert encoding entry_content = common.from_eva_sjis(raw_entry_content) # Strip trailing \0's entry_content = entry_content.rstrip('\0') else: entry_content = None if not has_content_section and (entry_size != parameter_size): raise Exception( 'Extra unannounced content in entry type: 0x%X, size: %s' % (entry_type, entry_size)) # Add entry self.entries.append( (entry_type, entry_parameters, entry_content))
def __monitor_file(task_file, file_path, file_size, description, delay, progress): """ Monitor the file size of the given file. """ file_name = os.path.basename(file_path) file_dir = __remove_duplicate_chars( \ file_path.rstrip(file_name).rstrip(os.path.sep), os.path.sep) file_size = int(file_size) file_size_init = 0 file_size_current = 0 file_size_perc = 0 chars_running = ["-", "\\", "|", "/"] chars_stalled = ["?", " "] chars_missing = ["X", " "] delay_running = 0.1 delay_stalled = 0.6 progress_chars = chars_running progress_count = 0 stalled = False wait = delay_running display_file_info = \ bool(int(common.global_config(["KeyGenerator", "Monitor"], ["display_file_info"], "1"))) if display_file_info: print("%s:" % description).ljust(16, " ") + file_name print("File path:").ljust(16, " ") + file_dir else: print "%s" % description if file_size < 1000: print("File size:").ljust(16, " ") + ("%s bytes total" % file_size) else: size_round = __format_size(file_size) print ("File size:").ljust(16, " ") + \ ("%s (%s bytes total)" % (size_round, file_size)) if not progress: return try: file_size_init = file_size file_size_current = common.get_file_size(file_path) file_size_perc = int((file_size_current * 100) / file_size) except: pass count = 0 while file_size_current < file_size: try: file_size_current = common.get_file_size(file_path) except: pass if file_size_current == file_size: break file_exists_task = common.file_exists(task_file) file_exists_input = common.file_exists(file_path) if not file_exists_task or not file_exists_input: if not file_exists_input: progress_chars = chars_missing else: progress_chars = chars_stalled stalled = True wait = delay_stalled else: progress_chars = chars_running wait = delay_running if stalled: dict_contents = __read_content(task_file) if not int(dict_contents["file_input_size"]) == \ file_size_init: print "-" * 78 common.exception("Task mismatch. Process cancelled.") stalled = False progress_count += 1 if progress_count >= len(progress_chars): progress_count = 0 if delay == 0: __progress(file_size_perc, None, True) return if delay > 0: if file_size_perc < 100: __progress( \ file_size_perc, progress_chars[progress_count], False) time.sleep(wait) if count < delay: count += 0.1 continue else: count = 0 try: file_size_current = common.get_file_size(file_path) if not stalled: file_size_perc = int((file_size_current * 100) / file_size) except: pass __progress(100, " ", True)