class ChunkChannel(object): """ Reader/writer for chunked data. .. note:: logs at DEBUG level """ max_chunk_size = 65535 def __init__(self, sock): self.socket = sock self.address = Address(*sock.getpeername()) self.raw = BytesIO() self.output_buffer = [] self.output_size = 0 self._recv_buffer = b"" def write(self, b): """ Write some bytes, splitting into chunks if necessary. """ max_chunk_size = self.max_chunk_size output_buffer = self.output_buffer while b: size = len(b) future_size = self.output_size + size if future_size >= max_chunk_size: end = max_chunk_size - self.output_size output_buffer.append(b[:end]) self.output_size = max_chunk_size b = b[end:] self.flush() else: output_buffer.append(b) self.output_size = future_size b = b"" def flush(self, end_of_message=False): """ Flush everything written since the last chunk to the stream, followed by a zero-chunk if required. """ output_buffer = self.output_buffer if output_buffer: lines = [struct_pack(">H", self.output_size)] + output_buffer else: lines = [] if end_of_message: lines.append(b"\x00\x00") if lines: self.raw.writelines(lines) self.raw.flush() del output_buffer[:] self.output_size = 0 def send(self): """ Send all queued messages to the server. """ data = self.raw.getvalue() log_debug("C: b%r", data) self.socket.sendall(data) self.raw.seek(self.raw.truncate(0))
def dump_message(file, headers, props=None, content=None): msg = Message() for (name, value) in headers: msg[name] = value payload = BytesIO() if props is not None: start = payload.tell() for (key, value) in props.items(): payload.write("K {}\n".format(len(key)).encode("ascii")) payload.writelines((key.encode("ascii"), b"\n")) payload.write("V {}\n".format(len(value)).encode("ascii")) payload.writelines((value.encode("ascii"), b"\n")) payload.write(b"PROPS-END\n") msg["Prop-content-length"] = format(payload.tell() - start) if content is not None: msg["Text-content-length"] = format(len(content)) payload.write(content) if props is not None or content is not None: payload = payload.getvalue() msg["Content-length"] = format(len(payload)) # Workaround for Python issue 18324, "set_payload does not handle # binary payloads correctly", http://bugs.python.org/issue18324 msg.set_payload(payload.decode("ascii", "surrogateescape")) BytesGenerator(file, mangle_from_=False).flatten(msg)
def fetch(self): """ Receive exactly one message from the server. """ if self.closed: raise ProtocolError("Cannot read from a closed connection") if self.defunct: raise ProtocolError("Cannot read from a defunct connection") raw = BytesIO() unpack = Unpacker(raw).unpack try: raw.writelines(self.channel.chunk_reader()) except ProtocolError: self.defunct = True self.close() raise # Unpack from the raw byte stream and call the relevant message handler(s) raw.seek(0) response = self.responses[0] for signature, fields in unpack(): if __debug__: log_info("S: %s %s", message_names[signature], " ".join(map(repr, fields))) if signature in SUMMARY: response.complete = True self.responses.popleft() if signature == FAILURE: self.acknowledge_failure() handler_name = "on_%s" % message_names[signature].lower() try: handler = getattr(response, handler_name) except AttributeError: pass else: handler(*fields) raw.close()
def __init__(self): bio = BytesIO() bio.write("==='ByteIO' demo===\n".encode('utf8')) print("round 1:", bio.getvalue().decode('utf8')) bio.writelines([f"line{i}\n".encode('utf8') for i in range(10)]) print("round 2:", bio.getvalue().decode('utf8')) bio.close()
def _read(self): """Return contents of object.""" stream = self.native_obj.as_stream() content = BytesIO() content.writelines(stream) content.seek(0) return content.read()
def get_csv_stub(key: str, s3_client, config: Mapping[str, Mapping[str, str]]) -> IO: """ Get a file-like object containing the first config['PredicTrip'].getint('csv_stub_bytes') bytes of an S3 object. :param key: S3 key of the file object :param s3_client: Boto3 S3 Client instance with which to download stub :param config: mapping of predictrip configuration items :return: File-like object, open and ready to read """ file = BytesIO() # byte range is inclusive and numbered from 0. # (see https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.get_object # and https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35) resp = s3_client.get_object( Bucket=config['AWS']['s3_bucket_name'], Key=key, Range='bytes={}-{}'.format( 0, config['PredicTrip'].getint('csv_stub_bytes') - 1)) # HTTP status code 206 represents successful request for partial content if resp['ResponseMetadata']['HTTPStatusCode'] != 206: raise Exception( 'Unexpected HTTP status code received when requesting file stub: {}. ' .format(resp['ResponseMetadata']['HTTPStatusCode']) + '(Expected 206.) S3 object key: ' + key) # TODO (future): verify that resp['ContentType'] == 'text/csv', as canary of potential reporting changes file.writelines(resp['Body']) file.seek(0) return file
def fetch_next(self): """ Receive exactly one message from the server. """ raw = BytesIO() unpack = Unpacker(raw).unpack raw.writelines(self.channel.chunk_reader()) # Unpack from the raw byte stream and call the relevant message handler(s) raw.seek(0) response = self.responses[0] for signature, fields in unpack(): if __debug__: log_info("S: %s %s", message_names[signature], " ".join(map(repr, fields))) handler_name = "on_%s" % message_names[signature].lower() try: handler = getattr(response, handler_name) except AttributeError: pass else: handler(*fields) if signature in SUMMARY: response.complete = True self.responses.popleft() if signature == FAILURE: self.append(ACK_FAILURE, response=AckFailureResponse(self)) raw.close()
def get_file_object_config(): res_fp = BytesIO() txt_lines = get_text_config_lines() print txt_lines res_fp.writelines(txt_lines) res_fp.seek(0) return res_fp
def upload_object_via_stream(self, iterator, container, object_name, verify_hash=False, extra=None, ex_use_lease=False, ex_blob_type=None, ex_page_blob_size=None): """ @inherits: :class:`StorageDriver.upload_object_via_stream` Note that if ``iterator`` does not support ``seek``, the entire generator will be buffered in memory. :param ex_blob_type: Storage class :type ex_blob_type: ``str`` :param ex_page_blob_size: The maximum size to which the page blob can grow to :type ex_page_blob_size: ``int`` :param ex_use_lease: Indicates if we must take a lease before upload :type ex_use_lease: ``bool`` """ if ex_blob_type is None: ex_blob_type = self.ex_blob_type """ Azure requires that for page blobs that a maximum size that the page can grow to. For block blobs, it is required that the Content-Length header be set. The size of the block blob will be the total size of the stream minus the current position, so in this case ex_page_blob_size should be 0 (and will be checked in self._check_values). Source: https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob """ self._check_values(ex_blob_type, ex_page_blob_size) if ex_blob_type == "BlockBlob": try: iterator.seek(0, os.SEEK_END) except AttributeError: buffer = BytesIO() buffer.writelines(iterator) iterator = buffer blob_size = iterator.tell() iterator.seek(0) else: blob_size = ex_page_blob_size return self._put_object(container=container, object_name=object_name, object_size=blob_size, extra=extra, verify_hash=verify_hash, blob_type=ex_blob_type, use_lease=ex_use_lease, stream=iterator)
def get_bytes(chunk_iterator, compression): """ Read a (possibly compressed) file and return a BytesIO object with the contents. """ buffer = BytesIO() buffer.writelines(decompress_stream(chunk_iterator, compression)) buffer.seek(0) return buffer
class ChunkWriter(object): """ Writer for chunked data. """ max_chunk_size = 65535 def __init__(self): self.raw = BytesIO() self.output_buffer = [] self.output_size = 0 def write(self, b): """ Write some bytes, splitting into chunks if necessary. """ max_chunk_size = self.max_chunk_size output_buffer = self.output_buffer while b: size = len(b) future_size = self.output_size + size if future_size >= max_chunk_size: end = max_chunk_size - self.output_size output_buffer.append(b[:end]) self.output_size = max_chunk_size b = b[end:] self.flush() else: output_buffer.append(b) self.output_size = future_size b = b"" def flush(self, zero_chunk=False): """ Flush everything written since the last chunk to the stream, followed by a zero-chunk if required. """ output_buffer = self.output_buffer if output_buffer: lines = [struct.pack(">H", self.output_size)] + output_buffer else: lines = [] if zero_chunk: lines.append(b"\x00\x00") if lines: self.raw.writelines(lines) self.raw.flush() del output_buffer[:] self.output_size = 0 def to_bytes(self): """ Extract the written data as bytes. """ return self.raw.getvalue() def close(self, zero_chunk=False): """ Close the stream. """ self.flush(zero_chunk=zero_chunk) self.raw.close()
def setUp(self): super(TestBasicSvc, self).setUp() request.environ['HTTP_TRANSFER_ENCODING'] = 'chunked' in_data = BytesIO() in_data.writelines('{}') in_data.flush() in_data.seek(0) request.environ['wsgi.input'] = in_data
def consume_app(app, env): body = BytesIO() start = [] def start_response(status, headers, exc_info=None): start.append((status, headers, exc_info)) return body.write extra_content = list(app(env, start_response)) body.writelines(extra_content) return start[0], body.getvalue()
def process_item(self, item, spider): external_id = item.get("external_id") if external_id is None: return item buf = BytesIO() buf.writelines(self.encoder.encode(dict(item)) + '\n') buf.seek(0) path = '{}.json'.format(external_id) self.store.persist_file(path, buf, None) return item
def _get_document_bytes(query, data): bytes_ = BytesIO() res = list( map(lambda e: json.dumps(e, indent=None, ensure_ascii=False), data)) h = hashlib.md5(query.encode('utf8')) bytes_.name = "receipt_%s_%s.txt" % (datetime.date.today(), h.hexdigest()[0:8]) bytes_.writelines(map(lambda e: e.encode('utf8'), res)) bytes_.seek(0) return bytes_
def test_export_tarball_generator(self): wt = self.make_branch_and_tree('.') self.build_tree(['a']) wt.add(["a"]) wt.commit("1", timestamp=42) target = BytesIO() with wt.lock_read(): target.writelines(tarball_generator(wt, "bar")) # Ball should now be closed. target.seek(0) ball2 = tarfile.open(None, "r", target) self.addCleanup(ball2.close) self.assertEqual(["bar/a"], ball2.getnames())
async def exportkudos(self, ctx: commands.Context): scores = kudos.get_highest_kudos(100000) file = BytesIO() file.writelines(f"{member_id},{points}\n".encode() for member_id, points in scores) file.seek(0) await ctx.send( f"User kudos totals as of {datetime.utcnow().date().isoformat()}", file=nextcord.File( file, filename= f"member-kudos-{datetime.utcnow().date().isoformat().replace('-', '')}.csv", ), )
def add_file_edge_hashes(self, tree, file_ids): """Update to reflect the hashes for files in the tree. :param tree: The tree containing the files. :param file_ids: A list of file_ids to perform the updates for. """ desired_files = [(tree.id2path(f), f) for f in file_ids] with ui_factory.nested_progress_bar() as task: for num, (file_id, contents) in enumerate( tree.iter_files_bytes(desired_files)): task.update(gettext('Calculating hashes'), num, len(file_ids)) s = BytesIO() s.writelines(contents) s.seek(0) self.add_edge_hashes(s.readlines(), file_id)
def GenCapInf(self): self.CapInfFileName = os.path.join(GenFdsGlobalVariable.FvDir, self.UiCapsuleName + "_Cap" + '.inf') CapInfFile = BytesIO() #open (self.CapInfFileName , 'w+') CapInfFile.writelines("[options]" + TAB_LINE_BREAK) for Item in self.TokensDict: CapInfFile.writelines("EFI_" + \ Item + \ ' = ' + \ self.TokensDict[Item] + \ TAB_LINE_BREAK) return CapInfFile
def store_data(providers, curr_version=0): """ Store validated provider configs into the config file :param providers: List of sorted/validated (by domain) providers :param curr_version: version of the generated file :return: path of the file which was created """ log.info("Storing data into the config files") files_path = [] version = curr_version + 1 temp_file_path = os.path.join(os.path.dirname(__file__), 'temp_files') timestamp = datetime.datetime.now().strftime("%m-%d_%H") templates = aws_s3_processing() for file_name, file_template in templates.iteritems(): try: buffer_template = BytesIO() buffer_template.writelines(file_template) record_template = get_record_template(file_template) with BytesIO() as data_buffer: for provider in sorted(providers, key=lambda k: k['provider_id']): filled_template = str(record_template % (provider)) data_buffer.writelines(filled_template) tp = Template(buffer_template.getvalue()) output = tp.render(version=version, provider_data=data_buffer.getvalue()) file_name, extension = file_name.split('.') new_file_name = '.'.join([file_name + timestamp, extension]) file_name_location = os.path.join(temp_file_path, new_file_name) with open(file_name_location, 'wb') as conf_file: conf_file.write(output) log.info("Data was successfully stored into the file: {filename}". format(filename=new_file_name)) files_path.append(file_name_location) except Exception as e: log.info( "Error occurred while storing data into the file: {filename}\n{err}" .format(filename=file_name, err=e)) return files_path
class TestHeadMiddleware(tests.TestCase): def _trap_start_response(self, status, response_headers, exc_info=None): self._write_buffer = BytesIO() self._start_response_passed = (status, response_headers, exc_info) return self._write_buffer.write def _consume_app(self, app, request_method): environ = {'REQUEST_METHOD': request_method} value = list(app(environ, self._trap_start_response)) self._write_buffer.writelines(value) def _verify_get_passthrough(self, app): app = http_head.HeadMiddleware(app) self._consume_app(app, 'GET') self.assertEqual(('200 OK', headers, None), self._start_response_passed) self.assertEqualDiff(b''.join(content), self._write_buffer.getvalue()) def _verify_head_no_body(self, app): app = http_head.HeadMiddleware(app) self._consume_app(app, 'HEAD') self.assertEqual(('200 OK', headers, None), self._start_response_passed) self.assertEqualDiff(b'', self._write_buffer.getvalue()) def test_get_passthrough_yielding(self): self._verify_get_passthrough(yielding_app) def test_head_passthrough_yielding(self): self._verify_head_no_body(yielding_app) def test_get_passthrough_list(self): self._verify_get_passthrough(list_app) def test_head_passthrough_list(self): self._verify_head_no_body(list_app) def test_get_passthrough_writer(self): self._verify_get_passthrough(writer_app) def test_head_passthrough_writer(self): self._verify_head_no_body(writer_app)
def set_element_data_file_header(mhd_file, raw_file_name): # Read file lines into list f_content = mhd_file.readlines() # Replace line with new ElementDataFile name for i, line in enumerate(f_content): if b"ElementDataFile" in line: f_content[i] = "ElementDataFile = {}\n".format( raw_file_name).encode() # Write lines into new file and return new_file = BytesIO() new_file.writelines(f_content) new_file.seek(0) return InMemoryUploadedFile( new_file, "ImageField", mhd_file.name, "application/octet-stream", None, sys.getsizeof(new_file), )
def set_element_data_file_header(mhd_file, raw_file_name): # Read file lines into list f_content = mhd_file.readlines() # Replace line with new ElementDataFile name for i, line in enumerate(f_content): if b"ElementDataFile" in line: f_content[i] = "ElementDataFile = {}\n".format( raw_file_name ).encode() # Write lines into new file and return new_file = BytesIO() new_file.writelines(f_content) new_file.seek(0) return InMemoryUploadedFile( new_file, "ImageField", mhd_file.name, "application/octet-stream", None, sys.getsizeof(new_file), )
def get(self, request, *args, **kwargs): if request.GET.get("download") == "ical": ical = ScheduleExport(user=self.user).as_ical() buffer = BytesIO() buffer.writelines([line.encode() for line in ical]) buffer.seek(0) return FileResponse( buffer, as_attachment=True, filename=f"coursist_export_{self.user}_{arrow.utcnow()}.ics", ) elif request.GET.get("download") == "json": event_json = ScheduleExport(user=self.user).as_dict() buffer = BytesIO() buffer.write(json.dumps(event_json, indent=4, sort_keys=True, cls=DjangoJSONEncoder).encode()) buffer.seek(0) return FileResponse( buffer, as_attachment=True, filename=f"coursist_export_{self.user}_{arrow.utcnow()}.json", ) else: response = super().get(request, *args, **kwargs) if self.user.is_authenticated and SCHEDULE_COOKIE in self.request.COOKIES: # safe to delete the cookie here, because super has already called # get_context_data, which already used this cookie response.delete_cookie(SCHEDULE_COOKIE) return response
class DataIO(object): """ This class simply wraps a binary file or a bytes string and implements both the file and bytes interface. It allows an input to be provided as files of bytes and manipulated indifferently as a file or a bytes object. """ def __init__(self, f): if isinstance(f, bytes): from io import BytesIO self.f = BytesIO(f) else: self.f = f self.view = dataView(dataio=self) def __getitem__(self, i): stay = self.f.tell() sta = i.start if sta is None: sta = stay self.f.seek(sta, 0) if i.stop is None: data = self.f.read() else: data = self.f.read(i.stop - sta) self.f.seek(stay, 0) return data def size(self): stay = self.f.tell() self.f.seek(0, 2) sz = self.f.tell() self.f.seek(stay, 0) return sz def read(self, size=-1): return self.f.read(size) def readline(self, size=-1): return self.f.readline(size) def readlines(self, size=-1): return self.f.readlines(size) def xreadlines(self, size=-1): return self.f.xreadlines(size) def write(self, s): return self.f.write(s) def writelines(self, l): return self.f.writelines(l) def seek(self, offset, whence=0): return self.f.seek(offset, whence) def tell(self): return self.f.tell() def flush(self): return self.f.flush() def fileno(self): return self.f.fileno() def isatty(self): return self.f.isatty() def next(self): return self.f.next() def truncate(self, size=0): return self.f.truncate(size) def close(self): return self.f.close() @property def closed(self): return self.f.closed @property def encoding(self): return self.f.encoding @property def errors(self): return self.f.errors @property def mode(self): return self.f.mode @property def name(self): try: return self.f.name except AttributeError: s = bytes(self.f.getvalue()) return "(sc-%s...)" % ("".join(["%02x" % x for x in s])[:8]) filename = name @property def newlines(self): return self.f.newlines @property def softspace(self): return self.f.softspace
class ChunkChannel(object): """ Reader/writer for chunked data. .. note:: logs at DEBUG level """ max_chunk_size = 65535 def __init__(self, sock): self.socket = sock self.raw = BytesIO() self.output_buffer = [] self.output_size = 0 self._recv_buffer = b"" def write(self, b): """ Write some bytes, splitting into chunks if necessary. """ max_chunk_size = self.max_chunk_size output_buffer = self.output_buffer while b: size = len(b) future_size = self.output_size + size if future_size >= max_chunk_size: end = max_chunk_size - self.output_size output_buffer.append(b[:end]) self.output_size = max_chunk_size b = b[end:] self.flush() else: output_buffer.append(b) self.output_size = future_size b = b"" def flush(self, end_of_message=False): """ Flush everything written since the last chunk to the stream, followed by a zero-chunk if required. """ output_buffer = self.output_buffer if output_buffer: lines = [struct_pack(">H", self.output_size)] + output_buffer else: lines = [] if end_of_message: lines.append(b"\x00\x00") if lines: self.raw.writelines(lines) self.raw.flush() del output_buffer[:] self.output_size = 0 def send(self): """ Send all queued messages to the server. """ data = self.raw.getvalue() if __debug__: log_debug("C: %s", ":".join(map(hex2, data))) self.socket.sendall(data) self.raw.seek(self.raw.truncate(0)) def _recv(self, size): # If data is needed, keep reading until all bytes have been received remaining = size - len(self._recv_buffer) ready_to_read = None while remaining > 0: # Read up to the required amount remaining b = self.socket.recv(8192) if b: if __debug__: log_debug("S: %s", ":".join(map(hex2, b))) else: if ready_to_read is not None: raise ProtocolError("Server closed connection") remaining -= len(b) self._recv_buffer += b # If more is required, wait for available network data if remaining > 0: ready_to_read, _, _ = select((self.socket,), (), (), 0) while not ready_to_read: ready_to_read, _, _ = select((self.socket,), (), (), 0) # Split off the amount of data required and keep the rest in the buffer data, self._recv_buffer = self._recv_buffer[:size], self._recv_buffer[size:] return data def chunk_reader(self): chunk_size = -1 while chunk_size != 0: chunk_header = self._recv(2) chunk_size, = struct_unpack_from(">H", chunk_header) if chunk_size > 0: data = self._recv(chunk_size) yield data def close(self): """ Shut down and close the connection. """ if __debug__: log_info("~~ [CLOSE]") socket = self.socket socket.shutdown(SHUT_RDWR) socket.close()
class FV(FvClassObject): ## The constructor # # @param self The object pointer # def __init__(self): FvClassObject.__init__(self) self.FvInfFile = None self.FvAddressFile = None self.BaseAddress = None self.InfFileName = None self.FvAddressFileName = None self.CapsuleName = None self.FvBaseAddress = None self.FvForceRebase = None self.FvRegionInFD = None self.UsedSizeEnable = False ## AddToBuffer() # # Generate Fv and add it to the Buffer # # @param self The object pointer # @param Buffer The buffer generated FV data will be put # @param BaseAddress base address of FV # @param BlockSize block size of FV # @param BlockNum How many blocks in FV # @param ErasePolarity Flash erase polarity # @param VtfDict VTF objects # @param MacroDict macro value pair # @retval string Generated FV file path # def AddToBuffer(self, Buffer, BaseAddress=None, BlockSize=None, BlockNum=None, ErasePloarity='1', VtfDict=None, MacroDict={}, Flag=False): if BaseAddress is None and self.UiFvName.upper( ) + 'fv' in GenFdsGlobalVariable.ImageBinDict: return GenFdsGlobalVariable.ImageBinDict[self.UiFvName.upper() + 'fv'] # # Check whether FV in Capsule is in FD flash region. # If yes, return error. Doesn't support FV in Capsule image is also in FD flash region. # if self.CapsuleName is not None: for FdObj in GenFdsGlobalVariable.FdfParser.Profile.FdDict.values( ): for RegionObj in FdObj.RegionList: if RegionObj.RegionType == BINARY_FILE_TYPE_FV: for RegionData in RegionObj.RegionDataList: if RegionData.endswith(".fv"): continue elif RegionData.upper( ) + 'fv' in GenFdsGlobalVariable.ImageBinDict: continue elif self.UiFvName.upper() == RegionData.upper(): GenFdsGlobalVariable.ErrorLogger( "Capsule %s in FD region can't contain a FV %s in FD region." % (self.CapsuleName, self.UiFvName.upper())) if not Flag: GenFdsGlobalVariable.InfLogger("\nGenerating %s FV" % self.UiFvName) GenFdsGlobalVariable.LargeFileInFvFlags.append(False) FFSGuid = None if self.FvBaseAddress is not None: BaseAddress = self.FvBaseAddress if not Flag: self.__InitializeInf__(BaseAddress, BlockSize, BlockNum, ErasePloarity, VtfDict) # # First Process the Apriori section # MacroDict.update(self.DefineVarDict) GenFdsGlobalVariable.VerboseLogger('First generate Apriori file !') FfsFileList = [] for AprSection in self.AprioriSectionList: FileName = AprSection.GenFfs(self.UiFvName, MacroDict, IsMakefile=Flag) FfsFileList.append(FileName) # Add Apriori file name to Inf file if not Flag: self.FvInfFile.writelines("EFI_FILE_NAME = " + \ FileName + \ TAB_LINE_BREAK) # Process Modules in FfsList for FfsFile in self.FfsList: if Flag: if isinstance(FfsFile, FfsFileStatement.FileStatement): continue if GenFdsGlobalVariable.EnableGenfdsMultiThread and GenFdsGlobalVariable.ModuleFile and GenFdsGlobalVariable.ModuleFile.Path.find( os.path.normpath(FfsFile.InfFileName)) == -1: continue FileName = FfsFile.GenFfs(MacroDict, FvParentAddr=BaseAddress, IsMakefile=Flag, FvName=self.UiFvName) FfsFileList.append(FileName) if not Flag: self.FvInfFile.writelines("EFI_FILE_NAME = " + \ FileName + \ TAB_LINE_BREAK) if not Flag: SaveFileOnChange(self.InfFileName, self.FvInfFile.getvalue(), False) self.FvInfFile.close() # # Call GenFv tool # FvOutputFile = os.path.join(GenFdsGlobalVariable.FvDir, self.UiFvName) FvOutputFile = FvOutputFile + '.Fv' # BUGBUG: FvOutputFile could be specified from FDF file (FV section, CreateFile statement) if self.CreateFileName is not None: FvOutputFile = self.CreateFileName if Flag: GenFdsGlobalVariable.ImageBinDict[self.UiFvName.upper() + 'fv'] = FvOutputFile return FvOutputFile FvInfoFileName = os.path.join(GenFdsGlobalVariable.FfsDir, self.UiFvName + '.inf') if not Flag: CopyLongFilePath(GenFdsGlobalVariable.FvAddressFileName, FvInfoFileName) OrigFvInfo = None if os.path.exists(FvInfoFileName): OrigFvInfo = open(FvInfoFileName, 'r').read() if GenFdsGlobalVariable.LargeFileInFvFlags[-1]: FFSGuid = GenFdsGlobalVariable.EFI_FIRMWARE_FILE_SYSTEM3_GUID GenFdsGlobalVariable.GenerateFirmwareVolume( FvOutputFile, [self.InfFileName], AddressFile=FvInfoFileName, FfsList=FfsFileList, ForceRebase=self.FvForceRebase, FileSystemGuid=FFSGuid) NewFvInfo = None if os.path.exists(FvInfoFileName): NewFvInfo = open(FvInfoFileName, 'r').read() if NewFvInfo is not None and NewFvInfo != OrigFvInfo: FvChildAddr = [] AddFileObj = open(FvInfoFileName, 'r') AddrStrings = AddFileObj.readlines() AddrKeyFound = False for AddrString in AddrStrings: if AddrKeyFound: #get base address for the inside FvImage FvChildAddr.append(AddrString) elif AddrString.find("[FV_BASE_ADDRESS]") != -1: AddrKeyFound = True AddFileObj.close() if FvChildAddr != []: # Update Ffs again for FfsFile in self.FfsList: FileName = FfsFile.GenFfs(MacroDict, FvChildAddr, BaseAddress, IsMakefile=Flag, FvName=self.UiFvName) if GenFdsGlobalVariable.LargeFileInFvFlags[-1]: FFSGuid = GenFdsGlobalVariable.EFI_FIRMWARE_FILE_SYSTEM3_GUID #Update GenFv again GenFdsGlobalVariable.GenerateFirmwareVolume( FvOutputFile, [self.InfFileName], AddressFile=FvInfoFileName, FfsList=FfsFileList, ForceRebase=self.FvForceRebase, FileSystemGuid=FFSGuid) # # Write the Fv contents to Buffer # if os.path.isfile( FvOutputFile) and os.path.getsize(FvOutputFile) >= 0x48: FvFileObj = open(FvOutputFile, 'rb') # PI FvHeader is 0x48 byte FvHeaderBuffer = FvFileObj.read(0x48) Signature = FvHeaderBuffer[0x28:0x32] if Signature and Signature.startswith('_FVH'): GenFdsGlobalVariable.VerboseLogger( "\nGenerate %s FV Successfully" % self.UiFvName) GenFdsGlobalVariable.SharpCounter = 0 FvFileObj.seek(0) Buffer.write(FvFileObj.read()) # FV alignment position. FvAlignmentValue = 1 << (ord(FvHeaderBuffer[0x2E]) & 0x1F) if FvAlignmentValue >= 0x400: if FvAlignmentValue >= 0x100000: if FvAlignmentValue >= 0x1000000: #The max alignment supported by FFS is 16M. self.FvAlignment = "16M" else: self.FvAlignment = str( FvAlignmentValue / 0x100000) + "M" else: self.FvAlignment = str( FvAlignmentValue / 0x400) + "K" else: # FvAlignmentValue is less than 1K self.FvAlignment = str(FvAlignmentValue) FvFileObj.close() GenFdsGlobalVariable.ImageBinDict[self.UiFvName.upper() + 'fv'] = FvOutputFile GenFdsGlobalVariable.LargeFileInFvFlags.pop() else: GenFdsGlobalVariable.ErrorLogger("Invalid FV file %s." % self.UiFvName) else: GenFdsGlobalVariable.ErrorLogger( "Failed to generate %s FV file." % self.UiFvName) return FvOutputFile ## _GetBlockSize() # # Calculate FV's block size # Inherit block size from FD if no block size specified in FV # def _GetBlockSize(self): if self.BlockSizeList: return True for FdObj in GenFdsGlobalVariable.FdfParser.Profile.FdDict.values(): for RegionObj in FdObj.RegionList: if RegionObj.RegionType != BINARY_FILE_TYPE_FV: continue for RegionData in RegionObj.RegionDataList: # # Found the FD and region that contain this FV # if self.UiFvName.upper() == RegionData.upper(): RegionObj.BlockInfoOfRegion(FdObj.BlockSizeList, self) if self.BlockSizeList: return True return False ## __InitializeInf__() # # Initilize the inf file to create FV # # @param self The object pointer # @param BaseAddress base address of FV # @param BlockSize block size of FV # @param BlockNum How many blocks in FV # @param ErasePolarity Flash erase polarity # @param VtfDict VTF objects # def __InitializeInf__(self, BaseAddress=None, BlockSize=None, BlockNum=None, ErasePloarity='1', VtfDict=None): # # Create FV inf file # self.InfFileName = os.path.join(GenFdsGlobalVariable.FvDir, self.UiFvName + '.inf') self.FvInfFile = BytesIO() # # Add [Options] # self.FvInfFile.writelines("[options]" + TAB_LINE_BREAK) if BaseAddress is not None: self.FvInfFile.writelines("EFI_BASE_ADDRESS = " + \ BaseAddress + \ TAB_LINE_BREAK) if BlockSize is not None: self.FvInfFile.writelines("EFI_BLOCK_SIZE = " + \ '0x%X' %BlockSize + \ TAB_LINE_BREAK) if BlockNum is not None: self.FvInfFile.writelines("EFI_NUM_BLOCKS = " + \ ' 0x%X' %BlockNum + \ TAB_LINE_BREAK) else: if self.BlockSizeList == []: if not self._GetBlockSize(): #set default block size is 1 self.FvInfFile.writelines("EFI_BLOCK_SIZE = 0x1" + TAB_LINE_BREAK) for BlockSize in self.BlockSizeList: if BlockSize[0] is not None: self.FvInfFile.writelines("EFI_BLOCK_SIZE = " + \ '0x%X' %BlockSize[0] + \ TAB_LINE_BREAK) if BlockSize[1] is not None: self.FvInfFile.writelines("EFI_NUM_BLOCKS = " + \ ' 0x%X' %BlockSize[1] + \ TAB_LINE_BREAK) if self.BsBaseAddress is not None: self.FvInfFile.writelines('EFI_BOOT_DRIVER_BASE_ADDRESS = ' + \ '0x%X' %self.BsBaseAddress) if self.RtBaseAddress is not None: self.FvInfFile.writelines('EFI_RUNTIME_DRIVER_BASE_ADDRESS = ' + \ '0x%X' %self.RtBaseAddress) # # Add attribute # self.FvInfFile.writelines("[attributes]" + TAB_LINE_BREAK) self.FvInfFile.writelines("EFI_ERASE_POLARITY = " + \ ' %s' %ErasePloarity + \ TAB_LINE_BREAK) if not (self.FvAttributeDict is None): for FvAttribute in self.FvAttributeDict.keys(): if FvAttribute == "FvUsedSizeEnable": if self.FvAttributeDict[FvAttribute].upper() in ('TRUE', '1'): self.UsedSizeEnable = True continue self.FvInfFile.writelines("EFI_" + \ FvAttribute + \ ' = ' + \ self.FvAttributeDict[FvAttribute] + \ TAB_LINE_BREAK ) if self.FvAlignment is not None: self.FvInfFile.writelines("EFI_FVB2_ALIGNMENT_" + \ self.FvAlignment.strip() + \ " = TRUE" + \ TAB_LINE_BREAK) # # Generate FV extension header file # if not self.FvNameGuid: if len(self.FvExtEntryType) > 0 or self.UsedSizeEnable: GenFdsGlobalVariable.ErrorLogger( "FV Extension Header Entries declared for %s with no FvNameGuid declaration." % (self.UiFvName)) else: TotalSize = 16 + 4 Buffer = '' if self.UsedSizeEnable: TotalSize += (4 + 4) ## define EFI_FV_EXT_TYPE_USED_SIZE_TYPE 0x03 #typedef struct # { # EFI_FIRMWARE_VOLUME_EXT_ENTRY Hdr; # UINT32 UsedSize; # } EFI_FIRMWARE_VOLUME_EXT_ENTRY_USED_SIZE_TYPE; Buffer += pack('HHL', 8, 3, 0) if self.FvNameString == 'TRUE': # # Create EXT entry for FV UI name # This GUID is used: A67DF1FA-8DE8-4E98-AF09-4BDF2EFFBC7C # FvUiLen = len(self.UiFvName) TotalSize += (FvUiLen + 16 + 4) Guid = FV_UI_EXT_ENTY_GUID.split('-') # # Layout: # EFI_FIRMWARE_VOLUME_EXT_ENTRY : size 4 # GUID : size 16 # FV UI name # Buffer += (pack('HH', (FvUiLen + 16 + 4), 0x0002) + PackGUID(Guid) + self.UiFvName) for Index in range(0, len(self.FvExtEntryType)): if self.FvExtEntryType[Index] == 'FILE': # check if the path is absolute or relative if os.path.isabs(self.FvExtEntryData[Index]): FileFullPath = os.path.normpath( self.FvExtEntryData[Index]) else: FileFullPath = os.path.normpath( os.path.join(GenFdsGlobalVariable.WorkSpaceDir, self.FvExtEntryData[Index])) # check if the file path exists or not if not os.path.isfile(FileFullPath): GenFdsGlobalVariable.ErrorLogger( "Error opening FV Extension Header Entry file %s." % (self.FvExtEntryData[Index])) FvExtFile = open(FileFullPath, 'rb') FvExtFile.seek(0, 2) Size = FvExtFile.tell() if Size >= 0x10000: GenFdsGlobalVariable.ErrorLogger( "The size of FV Extension Header Entry file %s exceeds 0x10000." % (self.FvExtEntryData[Index])) TotalSize += (Size + 4) FvExtFile.seek(0) Buffer += pack('HH', (Size + 4), int(self.FvExtEntryTypeValue[Index], 16)) Buffer += FvExtFile.read() FvExtFile.close() if self.FvExtEntryType[Index] == 'DATA': ByteList = self.FvExtEntryData[Index].split(',') Size = len(ByteList) if Size >= 0x10000: GenFdsGlobalVariable.ErrorLogger( "The size of FV Extension Header Entry data %s exceeds 0x10000." % (self.FvExtEntryData[Index])) TotalSize += (Size + 4) Buffer += pack('HH', (Size + 4), int(self.FvExtEntryTypeValue[Index], 16)) for Index1 in range(0, Size): Buffer += pack('B', int(ByteList[Index1], 16)) Guid = self.FvNameGuid.split('-') Buffer = PackGUID(Guid) + pack('=L', TotalSize) + Buffer # # Generate FV extension header file if the total size is not zero # if TotalSize > 0: FvExtHeaderFileName = os.path.join(GenFdsGlobalVariable.FvDir, self.UiFvName + '.ext') FvExtHeaderFile = BytesIO() FvExtHeaderFile.write(Buffer) Changed = SaveFileOnChange(FvExtHeaderFileName, FvExtHeaderFile.getvalue(), True) FvExtHeaderFile.close() if Changed: if os.path.exists(self.InfFileName): os.remove(self.InfFileName) self.FvInfFile.writelines("EFI_FV_EXT_HEADER_FILE_NAME = " + \ FvExtHeaderFileName + \ TAB_LINE_BREAK) # # Add [Files] # self.FvInfFile.writelines("[files]" + TAB_LINE_BREAK) if VtfDict and self.UiFvName in VtfDict: self.FvInfFile.writelines("EFI_FILE_NAME = " + \ VtfDict[self.UiFvName] + \ TAB_LINE_BREAK)
class BCDataStream: def __init__(self, data=None): self.data = BytesIO(data) @property def is_at_beginning(self): return self.data.tell() == 0 def reset(self): self.data.seek(0) def get_bytes(self): return self.data.getvalue() def read(self, size): return self.data.read(size) def write(self, data): self.data.write(data) def write_many(self, many): self.data.writelines(many) def read_string(self): return self.read(self.read_compact_size()) def write_string(self, s): self.write_compact_size(len(s)) self.write(s) def read_compact_size(self): size = self.read_uint8() if size < 253: return size if size == 253: return self.read_uint16() elif size == 254: return self.read_uint32() elif size == 255: return self.read_uint64() def write_compact_size(self, size): if size < 253: self.write_uint8(size) elif size <= 0xFFFF: self.write_uint8(253) self.write_uint16(size) elif size <= 0xFFFFFFFF: self.write_uint8(254) self.write_uint32(size) else: self.write_uint8(255) self.write_uint64(size) def read_boolean(self): return self.read_uint8() != 0 def write_boolean(self, val): return self.write_uint8(1 if val else 0) int8 = struct.Struct('b') uint8 = struct.Struct('B') int16 = struct.Struct('<h') uint16 = struct.Struct('<H') int32 = struct.Struct('<i') uint32 = struct.Struct('<I') int64 = struct.Struct('<q') uint64 = struct.Struct('<Q') def _read_struct(self, fmt): value = self.read(fmt.size) if len(value) > 0: return fmt.unpack(value)[0] def read_int8(self): return self._read_struct(self.int8) def read_uint8(self): return self._read_struct(self.uint8) def read_int16(self): return self._read_struct(self.int16) def read_uint16(self): return self._read_struct(self.uint16) def read_int32(self): return self._read_struct(self.int32) def read_uint32(self): return self._read_struct(self.uint32) def read_int64(self): return self._read_struct(self.int64) def read_uint64(self): return self._read_struct(self.uint64) def write_int8(self, val): self.write(self.int8.pack(val)) def write_uint8(self, val): self.write(self.uint8.pack(val)) def write_int16(self, val): self.write(self.int16.pack(val)) def write_uint16(self, val): self.write(self.uint16.pack(val)) def write_int32(self, val): self.write(self.int32.pack(val)) def write_uint32(self, val): self.write(self.uint32.pack(val)) def write_int64(self, val): self.write(self.int64.pack(val)) def write_uint64(self, val): self.write(self.uint64.pack(val))
class DataIO(BinFormat): """This class wraps a binary file or a string of bytes and provides both the file and bytes API. """ def __init__(self, f): if isinstance(f,bytes): self.f=BytesIO(f) else: self.f=f def __getitem__(self,i): stay = self.f.tell() sta = i.start or stay self.f.seek(sta,0) if i.stop is None: data = self.f.read() else: data = self.f.read(i.stop-sta) self.f.seek(stay,0) return data def read(self,size=-1): return self.f.read(size) def readline(self,size=-1): return self.f.readline(size) def readlines(self,size=-1): return self.f.readlines(size) def xreadlines(self,size=-1): return self.f.xreadlines(size) def write(self,s): return self.f.write(s) def writelines(self,l): return self.f.writelines(l) def seek(self,offset,whence=0): return self.f.seek(offset,whence) def tell(self): return self.f.tell() def flush(self): return self.f.flush() def fileno(self): return self.f.fileno() def isatty(self): return self.f.isatty() def next(self): return self.f.next() def truncate(self,size=0): return self.f.truncate(size) def close(self): return self.f.close() @property def closed(self): return self.f.closed @property def encoding(self): return self.f.encoding @property def errors(self): return self.f.errors @property def mode(self): return self.f.mode @property def name(self): try: return self.f.name except AttributeError: try: from builtins import bytes except ImportError: pass s = bytes(self.f.getvalue()) return '(sc-%s...)'%(''.join(["%02x"%x for x in s])[:8]) filename = name @property def newlines(self): return self.f.newlines @property def softspace(self): return self.f.softspace
class ChunkChannel(object): """ Reader/writer for chunked data. .. note:: logs at DEBUG level """ max_chunk_size = 65535 def __init__(self, sock): self.socket = sock self.raw = BytesIO() self.output_buffer = [] self.output_size = 0 self._recv_buffer = b"" def write(self, b): """ Write some bytes, splitting into chunks if necessary. """ max_chunk_size = self.max_chunk_size output_buffer = self.output_buffer while b: size = len(b) future_size = self.output_size + size if future_size >= max_chunk_size: end = max_chunk_size - self.output_size output_buffer.append(b[:end]) self.output_size = max_chunk_size b = b[end:] self.flush() else: output_buffer.append(b) self.output_size = future_size b = b"" def flush(self, end_of_message=False): """ Flush everything written since the last chunk to the stream, followed by a zero-chunk if required. """ output_buffer = self.output_buffer if output_buffer: lines = [struct_pack(">H", self.output_size)] + output_buffer else: lines = [] if end_of_message: lines.append(b"\x00\x00") if lines: self.raw.writelines(lines) self.raw.flush() del output_buffer[:] self.output_size = 0 def send(self): """ Send all queued messages to the server. """ data = self.raw.getvalue() if __debug__: log_debug("C: %s", ":".join(map(hex2, data))) self.socket.sendall(data) self.raw.seek(self.raw.truncate(0)) def _recv(self, size): # If data is needed, keep reading until all bytes have been received remaining = size - len(self._recv_buffer) ready_to_read = None while remaining > 0: # Read up to the required amount remaining b = self.socket.recv(8192) if b: if __debug__: log_debug("S: %s", ":".join(map(hex2, b))) else: if ready_to_read is not None: raise ProtocolError("Server closed connection") remaining -= len(b) self._recv_buffer += b # If more is required, wait for available network data if remaining > 0: ready_to_read, _, _ = select((self.socket,), (), (), 0) while not ready_to_read: ready_to_read, _, _ = select((self.socket,), (), (), 0) # Split off the amount of data required and keep the rest in the buffer data, self._recv_buffer = self._recv_buffer[:size], self._recv_buffer[size:] return data def chunk_reader(self): chunk_size = -1 while chunk_size != 0: chunk_header = self._recv(2) chunk_size, = struct_unpack_from(">H", chunk_header) if chunk_size > 0: data = self._recv(chunk_size) yield data
class GitBranchBuilder(object): def __init__(self, stream=None): if not FastimportFeature.available(): raise tests.UnavailableFeature(FastimportFeature) self.commit_info = [] self.orig_stream = stream if stream is None: self.stream = BytesIO() else: self.stream = stream self._counter = 0 self._branch = b'refs/heads/master' def set_branch(self, branch): """Set the branch we are committing.""" self._branch = branch def _write(self, text): self.stream.write(text) def _writelines(self, lines): self.stream.writelines(lines) def _create_blob(self, content): self._counter += 1 from fastimport.commands import BlobCommand blob = BlobCommand(b'%d' % self._counter, content) self._write(bytes(blob) + b"\n") return self._counter def set_symlink(self, path, content): """Create or update symlink at a given path.""" mark = self._create_blob(self._encode_path(content)) mode = b'120000' self.commit_info.append(b'M %s :%d %s\n' % (mode, mark, self._encode_path(path))) def set_submodule(self, path, commit_sha): """Create or update submodule at a given path.""" mode = b'160000' self.commit_info.append(b'M %s %s %s\n' % (mode, commit_sha, self._encode_path(path))) def set_file(self, path, content, executable): """Create or update content at a given path.""" mark = self._create_blob(content) if executable: mode = b'100755' else: mode = b'100644' self.commit_info.append(b'M %s :%d %s\n' % (mode, mark, self._encode_path(path))) def delete_entry(self, path): """This will delete files or symlinks at the given location.""" self.commit_info.append(b'D %s\n' % (self._encode_path(path), )) @staticmethod def _encode_path(path): if isinstance(path, bytes): return path if '\n' in path or path[0] == '"': path = path.replace('\\', '\\\\') path = path.replace('\n', '\\n') path = path.replace('"', '\\"') path = '"' + path + '"' return path.encode('utf-8') # TODO: Author # TODO: Author timestamp+timezone def commit(self, committer, message, timestamp=None, timezone=b'+0000', author=None, merge=None, base=None): """Commit the new content. :param committer: The name and address for the committer :param message: The commit message :param timestamp: The timestamp for the commit :param timezone: The timezone of the commit, such as '+0000' or '-1000' :param author: The name and address of the author (if different from committer) :param merge: A list of marks if this should merge in another commit :param base: An id for the base revision (primary parent) if that is not the last commit. :return: A mark which can be used in the future to reference this commit. """ self._counter += 1 mark = b'%d' % (self._counter, ) if timestamp is None: timestamp = int(time.time()) self._write(b'commit %s\n' % (self._branch, )) self._write(b'mark :%s\n' % (mark, )) self._write(b'committer %s %ld %s\n' % (committer, timestamp, timezone)) if not isinstance(message, bytes): message = message.encode('UTF-8') self._write(b'data %d\n' % (len(message), )) self._write(message) self._write(b'\n') if base is not None: self._write(b'from :%s\n' % (base, )) if merge is not None: for m in merge: self._write(b'merge :%s\n' % (m, )) self._writelines(self.commit_info) self._write(b'\n') self.commit_info = [] return mark def reset(self, ref=None, mark=None): """Create or recreate the named branch. :param ref: branch name, defaults to the current branch. :param mark: commit the branch will point to. """ if ref is None: ref = self._branch self._write(b'reset %s\n' % (ref, )) if mark is not None: self._write(b'from :%s\n' % mark) self._write(b'\n') def finish(self): """We are finished building, close the stream, get the id mapping""" self.stream.seek(0) if self.orig_stream is None: from dulwich.repo import Repo r = Repo(".") from dulwich.fastexport import GitImportProcessor importer = GitImportProcessor(r) return importer.import_stream(self.stream)
class _BaseBinaryWrapper: def __init__(self, stream: Union[typing.BinaryIO, bytes] = b""): if isinstance(stream, bytes) or isinstance(stream, bytearray): self.stream = BytesIO(stream) else: self.stream = stream # Wrappings: def close(self) -> None: return self.stream.close() def flush(self) -> None: return self.stream.flush() def read(self, n: int = -1) -> AnyStr: return self.stream.read(n) def readable(self) -> bool: return self.stream.readable() def readline(self, limit: int = -1) -> AnyStr: return self.stream.readline(limit) def readlines(self, hint: int = -1) -> List[AnyStr]: return self.stream.readlines(hint) def write(self, s: Union[bytes, bytearray]) -> int: return self.stream.write(s) def writable(self) -> bool: return self.stream.writable() def writelines(self, lines: Iterable[AnyStr]) -> None: self.stream.writelines(lines) def seek(self, offset: int, whence: int = 0) -> int: return self.stream.seek(offset, whence) def seekable(self) -> bool: return self.stream.seekable() def tell(self) -> int: return self.stream.tell() def fileno(self) -> int: return self.stream.fileno() def __enter__(self): self.stream.__enter__() return self def __exit__(self, exc_type, exc_val, exc_tb): self.stream.__exit__(exc_type, exc_val, exc_tb) # helper functions def readall(self): self.stream.seek(0) return self.stream.read() def getvalue(self): if isinstance(self.stream, BytesIO): return self.stream.getvalue() pos = self.stream.tell() ret = self.readall() self.stream.seek(pos) return ret def align(self, alignment=4): if offset := (self.tell() % alignment): self.seek(self.tell() + alignment - offset)
def _write_refs(self, refs): f = BytesIO() f.writelines(write_info_refs(refs, self.store)) self.scon.put_object(self.filename, f)
class FV (FvClassObject): ## The constructor # # @param self The object pointer # def __init__(self): FvClassObject.__init__(self) self.FvInfFile = None self.FvAddressFile = None self.BaseAddress = None self.InfFileName = None self.FvAddressFileName = None self.CapsuleName = None self.FvBaseAddress = None self.FvForceRebase = None self.FvRegionInFD = None self.UsedSizeEnable = False ## AddToBuffer() # # Generate Fv and add it to the Buffer # # @param self The object pointer # @param Buffer The buffer generated FV data will be put # @param BaseAddress base address of FV # @param BlockSize block size of FV # @param BlockNum How many blocks in FV # @param ErasePolarity Flash erase polarity # @param VtfDict VTF objects # @param MacroDict macro value pair # @retval string Generated FV file path # def AddToBuffer (self, Buffer, BaseAddress=None, BlockSize= None, BlockNum=None, ErasePloarity='1', VtfDict=None, MacroDict = {}, Flag=False) : if BaseAddress is None and self.UiFvName.upper() + 'fv' in GenFdsGlobalVariable.ImageBinDict: return GenFdsGlobalVariable.ImageBinDict[self.UiFvName.upper() + 'fv'] # # Check whether FV in Capsule is in FD flash region. # If yes, return error. Doesn't support FV in Capsule image is also in FD flash region. # if self.CapsuleName is not None: for FdObj in GenFdsGlobalVariable.FdfParser.Profile.FdDict.values(): for RegionObj in FdObj.RegionList: if RegionObj.RegionType == BINARY_FILE_TYPE_FV: for RegionData in RegionObj.RegionDataList: if RegionData.endswith(".fv"): continue elif RegionData.upper() + 'fv' in GenFdsGlobalVariable.ImageBinDict: continue elif self.UiFvName.upper() == RegionData.upper(): GenFdsGlobalVariable.ErrorLogger("Capsule %s in FD region can't contain a FV %s in FD region." % (self.CapsuleName, self.UiFvName.upper())) if not Flag: GenFdsGlobalVariable.InfLogger( "\nGenerating %s FV" %self.UiFvName) GenFdsGlobalVariable.LargeFileInFvFlags.append(False) FFSGuid = None if self.FvBaseAddress is not None: BaseAddress = self.FvBaseAddress if not Flag: self.__InitializeInf__(BaseAddress, BlockSize, BlockNum, ErasePloarity, VtfDict) # # First Process the Apriori section # MacroDict.update(self.DefineVarDict) GenFdsGlobalVariable.VerboseLogger('First generate Apriori file !') FfsFileList = [] for AprSection in self.AprioriSectionList: FileName = AprSection.GenFfs (self.UiFvName, MacroDict, IsMakefile=Flag) FfsFileList.append(FileName) # Add Apriori file name to Inf file if not Flag: self.FvInfFile.writelines("EFI_FILE_NAME = " + \ FileName + \ TAB_LINE_BREAK) # Process Modules in FfsList for FfsFile in self.FfsList : if Flag: if isinstance(FfsFile, FfsFileStatement.FileStatement): continue if GenFdsGlobalVariable.EnableGenfdsMultiThread and GenFdsGlobalVariable.ModuleFile and GenFdsGlobalVariable.ModuleFile.Path.find(os.path.normpath(FfsFile.InfFileName)) == -1: continue FileName = FfsFile.GenFfs(MacroDict, FvParentAddr=BaseAddress, IsMakefile=Flag, FvName=self.UiFvName) FfsFileList.append(FileName) if not Flag: self.FvInfFile.writelines("EFI_FILE_NAME = " + \ FileName + \ TAB_LINE_BREAK) if not Flag: SaveFileOnChange(self.InfFileName, self.FvInfFile.getvalue(), False) self.FvInfFile.close() # # Call GenFv tool # FvOutputFile = os.path.join(GenFdsGlobalVariable.FvDir, self.UiFvName) FvOutputFile = FvOutputFile + '.Fv' # BUGBUG: FvOutputFile could be specified from FDF file (FV section, CreateFile statement) if self.CreateFileName is not None: FvOutputFile = self.CreateFileName if Flag: GenFdsGlobalVariable.ImageBinDict[self.UiFvName.upper() + 'fv'] = FvOutputFile return FvOutputFile FvInfoFileName = os.path.join(GenFdsGlobalVariable.FfsDir, self.UiFvName + '.inf') if not Flag: CopyLongFilePath(GenFdsGlobalVariable.FvAddressFileName, FvInfoFileName) OrigFvInfo = None if os.path.exists (FvInfoFileName): OrigFvInfo = open(FvInfoFileName, 'r').read() if GenFdsGlobalVariable.LargeFileInFvFlags[-1]: FFSGuid = GenFdsGlobalVariable.EFI_FIRMWARE_FILE_SYSTEM3_GUID GenFdsGlobalVariable.GenerateFirmwareVolume( FvOutputFile, [self.InfFileName], AddressFile=FvInfoFileName, FfsList=FfsFileList, ForceRebase=self.FvForceRebase, FileSystemGuid=FFSGuid ) NewFvInfo = None if os.path.exists (FvInfoFileName): NewFvInfo = open(FvInfoFileName, 'r').read() if NewFvInfo is not None and NewFvInfo != OrigFvInfo: FvChildAddr = [] AddFileObj = open(FvInfoFileName, 'r') AddrStrings = AddFileObj.readlines() AddrKeyFound = False for AddrString in AddrStrings: if AddrKeyFound: #get base address for the inside FvImage FvChildAddr.append (AddrString) elif AddrString.find ("[FV_BASE_ADDRESS]") != -1: AddrKeyFound = True AddFileObj.close() if FvChildAddr != []: # Update Ffs again for FfsFile in self.FfsList : FileName = FfsFile.GenFfs(MacroDict, FvChildAddr, BaseAddress, IsMakefile=Flag, FvName=self.UiFvName) if GenFdsGlobalVariable.LargeFileInFvFlags[-1]: FFSGuid = GenFdsGlobalVariable.EFI_FIRMWARE_FILE_SYSTEM3_GUID; #Update GenFv again GenFdsGlobalVariable.GenerateFirmwareVolume( FvOutputFile, [self.InfFileName], AddressFile=FvInfoFileName, FfsList=FfsFileList, ForceRebase=self.FvForceRebase, FileSystemGuid=FFSGuid ) # # Write the Fv contents to Buffer # if os.path.isfile(FvOutputFile): FvFileObj = open(FvOutputFile, 'rb') GenFdsGlobalVariable.VerboseLogger("\nGenerate %s FV Successfully" % self.UiFvName) GenFdsGlobalVariable.SharpCounter = 0 Buffer.write(FvFileObj.read()) FvFileObj.seek(0) # PI FvHeader is 0x48 byte FvHeaderBuffer = FvFileObj.read(0x48) # FV alignment position. FvAlignmentValue = 1 << (ord(FvHeaderBuffer[0x2E]) & 0x1F) if FvAlignmentValue >= 0x400: if FvAlignmentValue >= 0x100000: if FvAlignmentValue >= 0x1000000: #The max alignment supported by FFS is 16M. self.FvAlignment = "16M" else: self.FvAlignment = str(FvAlignmentValue / 0x100000) + "M" else: self.FvAlignment = str(FvAlignmentValue / 0x400) + "K" else: # FvAlignmentValue is less than 1K self.FvAlignment = str (FvAlignmentValue) FvFileObj.close() GenFdsGlobalVariable.ImageBinDict[self.UiFvName.upper() + 'fv'] = FvOutputFile GenFdsGlobalVariable.LargeFileInFvFlags.pop() else: GenFdsGlobalVariable.ErrorLogger("Failed to generate %s FV file." %self.UiFvName) return FvOutputFile ## _GetBlockSize() # # Calculate FV's block size # Inherit block size from FD if no block size specified in FV # def _GetBlockSize(self): if self.BlockSizeList: return True for FdObj in GenFdsGlobalVariable.FdfParser.Profile.FdDict.values(): for RegionObj in FdObj.RegionList: if RegionObj.RegionType != BINARY_FILE_TYPE_FV: continue for RegionData in RegionObj.RegionDataList: # # Found the FD and region that contain this FV # if self.UiFvName.upper() == RegionData.upper(): RegionObj.BlockInfoOfRegion(FdObj.BlockSizeList, self) if self.BlockSizeList: return True return False ## __InitializeInf__() # # Initilize the inf file to create FV # # @param self The object pointer # @param BaseAddress base address of FV # @param BlockSize block size of FV # @param BlockNum How many blocks in FV # @param ErasePolarity Flash erase polarity # @param VtfDict VTF objects # def __InitializeInf__ (self, BaseAddress = None, BlockSize= None, BlockNum = None, ErasePloarity='1', VtfDict=None) : # # Create FV inf file # self.InfFileName = os.path.join(GenFdsGlobalVariable.FvDir, self.UiFvName + '.inf') self.FvInfFile = BytesIO() # # Add [Options] # self.FvInfFile.writelines("[options]" + TAB_LINE_BREAK) if BaseAddress is not None : self.FvInfFile.writelines("EFI_BASE_ADDRESS = " + \ BaseAddress + \ TAB_LINE_BREAK) if BlockSize is not None: self.FvInfFile.writelines("EFI_BLOCK_SIZE = " + \ '0x%X' %BlockSize + \ TAB_LINE_BREAK) if BlockNum is not None: self.FvInfFile.writelines("EFI_NUM_BLOCKS = " + \ ' 0x%X' %BlockNum + \ TAB_LINE_BREAK) else: if self.BlockSizeList == []: if not self._GetBlockSize(): #set default block size is 1 self.FvInfFile.writelines("EFI_BLOCK_SIZE = 0x1" + TAB_LINE_BREAK) for BlockSize in self.BlockSizeList : if BlockSize[0] is not None: self.FvInfFile.writelines("EFI_BLOCK_SIZE = " + \ '0x%X' %BlockSize[0] + \ TAB_LINE_BREAK) if BlockSize[1] is not None: self.FvInfFile.writelines("EFI_NUM_BLOCKS = " + \ ' 0x%X' %BlockSize[1] + \ TAB_LINE_BREAK) if self.BsBaseAddress is not None: self.FvInfFile.writelines('EFI_BOOT_DRIVER_BASE_ADDRESS = ' + \ '0x%X' %self.BsBaseAddress) if self.RtBaseAddress is not None: self.FvInfFile.writelines('EFI_RUNTIME_DRIVER_BASE_ADDRESS = ' + \ '0x%X' %self.RtBaseAddress) # # Add attribute # self.FvInfFile.writelines("[attributes]" + TAB_LINE_BREAK) self.FvInfFile.writelines("EFI_ERASE_POLARITY = " + \ ' %s' %ErasePloarity + \ TAB_LINE_BREAK) if not (self.FvAttributeDict is None): for FvAttribute in self.FvAttributeDict.keys() : if FvAttribute == "FvUsedSizeEnable": if self.FvAttributeDict[FvAttribute].upper() in ('TRUE', '1') : self.UsedSizeEnable = True continue self.FvInfFile.writelines("EFI_" + \ FvAttribute + \ ' = ' + \ self.FvAttributeDict[FvAttribute] + \ TAB_LINE_BREAK ) if self.FvAlignment is not None: self.FvInfFile.writelines("EFI_FVB2_ALIGNMENT_" + \ self.FvAlignment.strip() + \ " = TRUE" + \ TAB_LINE_BREAK) # # Generate FV extension header file # if not self.FvNameGuid: if len(self.FvExtEntryType) > 0 or self.UsedSizeEnable: GenFdsGlobalVariable.ErrorLogger("FV Extension Header Entries declared for %s with no FvNameGuid declaration." % (self.UiFvName)) else: TotalSize = 16 + 4 Buffer = '' if self.UsedSizeEnable: TotalSize += (4 + 4) ## define EFI_FV_EXT_TYPE_USED_SIZE_TYPE 0x03 #typedef struct # { # EFI_FIRMWARE_VOLUME_EXT_ENTRY Hdr; # UINT32 UsedSize; # } EFI_FIRMWARE_VOLUME_EXT_ENTRY_USED_SIZE_TYPE; Buffer += pack('HHL', 8, 3, 0) if self.FvNameString == 'TRUE': # # Create EXT entry for FV UI name # This GUID is used: A67DF1FA-8DE8-4E98-AF09-4BDF2EFFBC7C # FvUiLen = len(self.UiFvName) TotalSize += (FvUiLen + 16 + 4) Guid = FV_UI_EXT_ENTY_GUID.split('-') # # Layout: # EFI_FIRMWARE_VOLUME_EXT_ENTRY : size 4 # GUID : size 16 # FV UI name # Buffer += (pack('HH', (FvUiLen + 16 + 4), 0x0002) + PackGUID(Guid) + self.UiFvName) for Index in range (0, len(self.FvExtEntryType)): if self.FvExtEntryType[Index] == 'FILE': # check if the path is absolute or relative if os.path.isabs(self.FvExtEntryData[Index]): FileFullPath = os.path.normpath(self.FvExtEntryData[Index]) else: FileFullPath = os.path.normpath(os.path.join(GenFdsGlobalVariable.WorkSpaceDir, self.FvExtEntryData[Index])) # check if the file path exists or not if not os.path.isfile(FileFullPath): GenFdsGlobalVariable.ErrorLogger("Error opening FV Extension Header Entry file %s." % (self.FvExtEntryData[Index])) FvExtFile = open (FileFullPath, 'rb') FvExtFile.seek(0, 2) Size = FvExtFile.tell() if Size >= 0x10000: GenFdsGlobalVariable.ErrorLogger("The size of FV Extension Header Entry file %s exceeds 0x10000." % (self.FvExtEntryData[Index])) TotalSize += (Size + 4) FvExtFile.seek(0) Buffer += pack('HH', (Size + 4), int(self.FvExtEntryTypeValue[Index], 16)) Buffer += FvExtFile.read() FvExtFile.close() if self.FvExtEntryType[Index] == 'DATA': ByteList = self.FvExtEntryData[Index].split(',') Size = len (ByteList) if Size >= 0x10000: GenFdsGlobalVariable.ErrorLogger("The size of FV Extension Header Entry data %s exceeds 0x10000." % (self.FvExtEntryData[Index])) TotalSize += (Size + 4) Buffer += pack('HH', (Size + 4), int(self.FvExtEntryTypeValue[Index], 16)) for Index1 in range (0, Size): Buffer += pack('B', int(ByteList[Index1], 16)) Guid = self.FvNameGuid.split('-') Buffer = PackGUID(Guid) + pack('=L', TotalSize) + Buffer # # Generate FV extension header file if the total size is not zero # if TotalSize > 0: FvExtHeaderFileName = os.path.join(GenFdsGlobalVariable.FvDir, self.UiFvName + '.ext') FvExtHeaderFile = BytesIO() FvExtHeaderFile.write(Buffer) Changed = SaveFileOnChange(FvExtHeaderFileName, FvExtHeaderFile.getvalue(), True) FvExtHeaderFile.close() if Changed: if os.path.exists (self.InfFileName): os.remove (self.InfFileName) self.FvInfFile.writelines("EFI_FV_EXT_HEADER_FILE_NAME = " + \ FvExtHeaderFileName + \ TAB_LINE_BREAK) # # Add [Files] # self.FvInfFile.writelines("[files]" + TAB_LINE_BREAK) if VtfDict and self.UiFvName in VtfDict: self.FvInfFile.writelines("EFI_FILE_NAME = " + \ VtfDict[self.UiFvName] + \ TAB_LINE_BREAK)