def close_output_stream(self, stream: BinaryIO) -> None: for s3key, bio in self.key_stream_pairs.items(): if bio is not stream: continue obj = self.bucket.Object(s3key) if s3key.endswith('.json'): content_type = 'application/json' elif s3key.endswith('.html'): content_type = 'text/html' elif s3key.endswith('.txt'): content_type = 'text/plain' else: content_type = 'application/octet-stream' logger.info( f'put s3://{self.bucket.name}/{s3key}, ' f'content_type={content_type}' ) bio.seek(0) obj.upload_fileobj( stream, ExtraArgs={'ContentType': content_type}, ) stream.close() return raise ValueError('could not put a stream object to S3')
def backup_dfu(file: BinaryIO) -> None: """Backs up device data via DFU. Args: file: file where firmware (MCU flash memory) will be saved """ try: # TODO: implement this using pydfu raise NoBackendError except NoBackendError: # if libusb was not found, try using dfu-util command line tool with _get_dfu_util() as dfu_util: file.close() # dfu-util won't overwrite existing files so we have to do that first os.remove(file.name) exit( call([ dfu_util, "--device", f",{SPIKE_PRIME_DEVICE},{MINDSTORMS_INVENTOR_DEVICE}", "--alt", "0", "--dfuse-address", f"{FIRMWARE_ADDRESS}:{FIRMWARE_SIZE}", "--upload", file.name, ]))
def publish_input_data(self, expected_input_id: str, metadata: InputMetadata, input_data_stream: BinaryIO) -> None: input_file_path = self.input_file(expected_input_id) if os.path.exists(input_file_path): input_data_stream.close() return file_hash = hashlib.sha256() fd, temp_file_path = tempfile.mkstemp(dir=self.temp_data_dir) try: with os.fdopen(fd, 'wb') as f: bytes_read = 0 while True: data = input_data_stream.read(READ_BUFFER_SIZE) bytes_read += len(data) log.debug(f'{bytes_read} bytes of input read') if not data: break f.write(data) file_hash.update(data) input_id = file_hash.hexdigest() if input_id != expected_input_id: raise IncorrectInputIDException() os.rename(temp_file_path, input_file_path) if metadata.has_all_args(): self._store_input_id(metadata, input_id) return except Exception: os.remove(temp_file_path) raise
def restore_dfu(file: BinaryIO) -> None: """Restores flash memory from a file (raw data, not .dfu file). Args: file: the file that contains the firmware data """ file.seek(0, os.SEEK_END) size = file.tell() file.seek(0, os.SEEK_SET) if size < 512: raise ValueError("File is too small to be a valid firmware file") try: # TODO: implement this using pydfu raise NoBackendError except NoBackendError: # if libusb was not found, try using dfu-util command line tool with _get_dfu_util() as dfu_util: file.close() exit( call([ dfu_util, "--device", f",{SPIKE_PRIME_DEVICE},{MINDSTORMS_INVENTOR_DEVICE}", "--alt", "0", "--dfuse-address", f"{FIRMWARE_ADDRESS}", "--download", file.name, ]))
def apply_window( window: Any, sbuf: bytes = sbuf, target_stream: BinaryIO = target_stream ): if window is None: target_stream.close() return # Last call patch = delta.apply_txdelta_window(sbuf, window) target_stream.write(patch)
def download_from(self, payload_id: payload_types.PayloadId, blob_name: str, dest_obj: BinaryIO = None, dest_path: str = None, timeout=None) -> payload_types.PayloadFileDetails: """ Downloads a blob, identified by "blob_name", from a payload, identified by its "payload_id", from Clara. Data downloaded from the payload will be written to 'dest_obj' a passed in BinaryIO object with write privileges Args: payload_id (payload_types.PayloadId): Unique identifier of the payload. blob_name (str): The name, or path, of the blob in the payload. dest_obj (BinaryIO): Target stream object to write to with write privileges dest_path (str): Alternative to passing in BinaryIO object to download to, and rather passing in path for a file """ if (self._channel is None) or (self._stub is None): raise Exception("Connection is currently closed. Please run reconnect() to reopen connection") if (payload_id.value is None) or (payload_id.value == ""): raise Exception("Payload identifier argument must be initialized with non-null instance") if (blob_name is None) or (blob_name == ""): raise Exception("Name of source blob must be initialized with non-null string") file_path_used = False if dest_obj is None: if dest_path is None: raise Exception("Destination object for upload must be initialized with non-null BinaryIO object") else: dest_obj = open(dest_path, 'wb') file_path_used = True request = payloads_pb2.PayloadsDownloadRequest( header=self.get_request_header(), name=blob_name, payload_id=payload_id.to_grpc_value() ) responses = self._stub.Download(request, timeout=timeout) result = None for resp in responses: if result is None: self.check_response_header(header=resp.header) result = payload_types.PayloadFileDetails( mode=resp.details.mode, name=resp.details.name, size=resp.details.size ) dest_obj.write(resp.data) if file_path_used: dest_obj.close() return result
def test_encodedfile_writelines(tmpfile: BinaryIO) -> None: ef = capture.EncodedFile(tmpfile, "utf-8") with pytest.raises(AttributeError): ef.writelines([b"line1", b"line2"]) # type: ignore[list-item] # noqa: F821 assert ef.writelines(["line1", "line2"]) is None # type: ignore[func-returns-value] # noqa: F821 tmpfile.seek(0) assert tmpfile.read() == b"line1line2" tmpfile.close() with pytest.raises(ValueError): ef.read()
def dump(self, fp: BinaryIO) -> None: """ Serialize crypto object as binary stream to `fp` (a `.write()`-supporting file-like object). :param fp: the output file pointer. Must be set in binary mode (mode='wb') :return: None """ pem = self._private_key.private_bytes(Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption()) # type: ignore fp.write(pem) fp.close()
def stream_out(stream: typing.BinaryIO) -> typing.Generator[None, bytes, None]: data = yield while data: stream.write(data) data = yield stream.close() yield
def test_encodedfile_writelines(tmpfile: BinaryIO) -> None: ef = capture.EncodedFile(tmpfile, encoding="utf-8") with pytest.raises(TypeError): ef.writelines([b"line1", b"line2"]) assert ef.writelines(["line3", "line4"]) is None # type: ignore[func-returns-value] # noqa: F821 ef.flush() tmpfile.seek(0) assert tmpfile.read() == b"line3line4" tmpfile.close() with pytest.raises(ValueError): ef.read()
def stream_in( stream: typing.BinaryIO, buffer_size: int = 1 << 20) -> typing.Generator[bytes, None, None]: try: data = stream.read(buffer_size) while data: yield data data = stream.read(buffer_size) finally: stream.close()
def upload(self, payload_id: payload_types.PayloadId, blob_name: str, file_object: BinaryIO = None, file_path: str = None, timeout=None) -> payload_types.PayloadFileDetails: """ Uploads a blob from "file_object", to a Clara Payload identified by "payload_id". Each uploaded blob must be have a unique "blob_name" value within a given payload. Args: payload_id (payload_types.PayloadId): Unique identifier of the payload. blob_name (str): The name, or path, of the blob in the payload. file_object (BinaryIO): stream to read from and upload with read privileges file_path (str): Alternative to passing in BinaryIO object for upload, and rather passing in path for a file """ if (self._channel is None) or (self._stub is None): raise Exception("Connection is currently closed. Please run reconnect() to reopen connection") if (payload_id.value is None) or (payload_id.value == ""): raise Exception("Payload identifier argument must be initialized with non-null instance") if (blob_name is None) or (blob_name == ""): raise Exception("Name of destination blob must be initialized with non-null string") file_path_used = False if file_object is None: if file_path is None: raise Exception("File_object of file for upload must be initialized with non-null BinaryIO object") else: file_object = open(file_path, 'rb') file_path_used = True requests = self.upload_request_iterator( payload_id=payload_id, file_name=blob_name, source_object=file_object ) response = self._stub.Upload( requests, timeout=timeout ) self.check_response_header(header=response.header) result = payload_types.PayloadFileDetails(other=response.details) if file_path_used: file_object.close() return result
def download( obj: TrezorConnection, output: BinaryIO, version: str, skip_check: bool, fingerprint: str, beta: bool, bitcoin_only: bool, ) -> None: """Download and save the firmware image. Validation is done by default, can be omitted by "-s" or "--skip-check". When fingerprint or output file are not set, take them from SL servers. """ # When a version is specified, we do not even need the client connection # (and we will not be checking device when validating) if version: url, fp = find_specified_firmware_version(version=version, beta=beta, bitcoin_only=bitcoin_only) bootloader_onev2 = None trezor_major_version = None else: with obj.client_context() as client: url, fp = find_best_firmware_version(client=client, version=version, beta=beta, bitcoin_only=bitcoin_only) bootloader_onev2 = _is_bootloader_onev2(client) trezor_major_version = client.features.major_version firmware_data = download_firmware_data(url) if not fingerprint: fingerprint = fp if not skip_check: validate_firmware( firmware_data=firmware_data, fingerprint=fingerprint, bootloader_onev2=bootloader_onev2, trezor_major_version=trezor_major_version, ) if not output: output = open(_get_file_name_from_url(url), "wb") output.write(firmware_data) output.close() click.echo(f"Firmware saved under {output.name}.")
def load(cls, file: BinaryIO): """ Decode and return a Protocol object. """ master = struct.unpack(b'>L>L', file.read(8)) if master[0] != cls.MAGIC: raise ValueError('Attempt to decode corrupted or non-RVLT data') length = struct.unpack(b'>Q', file.read(8)) # header length header = file.read(length[0]) concrete = cls.lookup(master[1]) protocol = concrete._decode(header) protocol._body = file if protocol.body is None: file.close() return protocol
def _textChunkGenerator(self, streamIn: BinaryIO, logDir: str, logFile: str) -> Generator[TextChunk, None, None]: ''' Take an input stream with binary data, produced a generator of decoded "text chunks", and also save the original binary data in a log file. ''' os.makedirs(logDir, exist_ok=True) logF = open(os.path.join(logDir, logFile), 'wb') for bChunk in iter(streamIn.readline, b''): logF.write(bChunk) # Write to log file immediately tChunk = self._decodeBinaryChunk(bChunk) # decode if tChunk is not None: yield tChunk # TODO: split into actual text lines # At the end... streamIn.close() # Close the stream logF.close() # Close the output file
def save(self, destination: BinaryIO, buffer_size: int = 16384) -> None: """Save the file to the destination. Arguments: destination: A filename (str) or file object to write to. buffer_size: Buffer size as used as length in :func:`shutil.copyfileobj`. """ close_destination = False if isinstance(destination, str): destination = open(destination, 'wb') close_destination = True try: copyfileobj(self.stream, destination, buffer_size) finally: if close_destination: destination.close()
def write_file(self, file: typing.BinaryIO, remote_file: str, file_len: int = -1, run_after: FTCompleteOptions = FTCompleteOptions.DONT_RUN, linked_filename: Optional[str] = None, linked_vid: int_str = 'afros', compress: bool = False, **kwargs): if file_len < 0: file_len = file.seek(0, 2) file.seek(0, 0) display_name = remote_file if hasattr(file, 'name'): display_name = f'{remote_file} ({Path(file.name).name})' if compress and self.can_compress: file, file_len = compress_file(file, file_len) if self.is_wireless and file_len > 0x25000: confirm(f'You\'re about to upload {file_len} bytes wirelessly. This could take some time, and you should ' f'consider uploading directly with a wire.', abort=True, default=False) crc32 = self.VEX_CRC32.compute(file.read(file_len)) file.seek(0, 0) addr = kwargs.get('addr', 0x03800000) logger(__name__).info('Transferring {} ({} bytes) to the V5 from {}'.format(remote_file, file_len, file)) ft_meta = self.ft_initialize(remote_file, function='upload', length=file_len, crc=crc32, **kwargs) if linked_filename is not None: logger(__name__).debug('Setting file link') self.ft_set_link(linked_filename, vid=linked_vid) assert ft_meta['file_size'] >= file_len if len(remote_file) > 24: logger(__name__).info('Truncating {} to {} due to length'.format(remote_file, remote_file[:24])) remote_file = remote_file[:24] max_packet_size = int(ft_meta['max_packet_size'] / 2) with ui.progressbar(length=file_len, label='Uploading {}'.format(display_name)) as progress: for i in range(0, file_len, max_packet_size): packet_size = max_packet_size if i + max_packet_size > file_len: packet_size = file_len - i logger(__name__).debug('Writing {} bytes at 0x{:02X}'.format(packet_size, addr + i)) self.ft_write(addr + i, file.read(packet_size)) progress.update(packet_size) logger(__name__).debug('Completed {} of {} bytes'.format(i + packet_size, file_len)) logger(__name__).debug('Data transfer complete, sending ft complete') if compress and self.status['system_version'] in Spec('>=1.0.5'): logger(__name__).info('Closing gzip file') file.close() self.ft_complete(options=run_after)
def encode(self, obj: Union[dict, list, tuple, int, bytes, str], buff: BinaryIO = None) -> Optional[bytes]: """ 把对象bencode编码 :param obj: :param buff: :return: buff为None时, 返回bytes, 否则不返回, 数据写入传入buff """ has_init_buff = buff is not None if not has_init_buff: buff = BytesIO() self.__write_value(buff, obj) if not has_init_buff: bs = buff.getvalue() buff.close() return bs
def handle( # type: ignore[override] self, pub: typing.BinaryIO, ca: CertificateAuthority, **options: typing.Any) -> None: pub_data = pub.read() # close reader objects (otherwise we get a ResourceWarning) pub.close() # load public key try: pub_loaded = x509.load_pem_x509_certificate( pub_data, default_backend()) except Exception: # pylint: disable=broad-except try: pub_loaded = x509.load_der_x509_certificate( pub_data, default_backend()) except Exception as ex: raise CommandError("Unable to load public key.") from ex cert = Certificate(ca=ca) cert.update_certificate(pub_loaded) cert.save()
def reader(self, stream: BinaryIO, context: SimpleNamespace) -> None: ''' Read lines from subprocess output stream and write the information to a specified callable or to sys.stderr. @param stream: Popen.stdout readable byte stream object as returned by open() @param context: The information for the virtual environment creation request being processed. ''' progress = self.progress while True: s = stream.readline() if not s: break if progress is not None: progress(s, context) else: if not self.verbose: sys.stderr.write('.') else: sys.stderr.write(s.decode('utf-8')) sys.stderr.flush() stream.close()
def merge_upload( self, request, fileobj: BinaryIO, conflicts: str, author_name: Optional[str] = None, author_email: Optional[str] = None, method: str = "translate", fuzzy: str = "", ): """Top level handler for file uploads.""" from weblate.accounts.models import AuditLog # Optionally set authorship orig_user = None if author_email: from weblate.auth.models import User orig_user = request.user request.user, created = User.objects.get_or_create( email=author_email, defaults={ "username": author_email, "full_name": author_name or author_email, }, ) if created: AuditLog.objects.create( request.user, request, "autocreated", ) try: if method == "replace": return self.handle_replace(request, fileobj) if method == "source": return self.handle_source(request, fileobj) filecopy = fileobj.read() fileobj.close() # Strip possible UTF-8 BOM if filecopy[:3] == codecs.BOM_UTF8: filecopy = filecopy[3:] # Load backend file store = try_load( fileobj.name, filecopy, self.component.file_format_cls, self.component.template_store, ) # Check valid plural forms if hasattr(store.store, "parseheader"): header = store.store.parseheader() try: number, formula = Plural.parse_plural_forms(header["Plural-Forms"]) if not self.plural.same_plural(number, formula): raise PluralFormsMismatch() except (ValueError, KeyError): # Formula wrong or missing pass if method in ("translate", "fuzzy", "approve"): # Merge on units level with self.component.repository.lock: return self.merge_translations( request, store, conflicts, method, fuzzy ) elif method == "add": return self.handle_add_upload(request, store, fuzzy=fuzzy) # Add as sugestions return self.merge_suggestions(request, store, fuzzy) finally: if orig_user: request.user = orig_user
def read_output(out_stream: BinaryIO, out_queue: Queue) -> None: for line in iter(out_stream.readline, b""): out_queue.put(line.decode("utf-8")) out_stream.close()
def close(self, file_descriptor: BinaryIO): ''' Closes a file descriptor as returned by self.open() ''' file_descriptor.close()
def calculate_molecule_matches( uploaded_molecules_file: BinaryIO) -> MoleculeSet: """ Calculate molecule matches of all SMARTS in the database given a molecule file, and store the Molecule and Match instances in the database. :param uploaded_molecules_file: An open file handle to a molecule file to match """ import tempfile import sys from smartsexplore.util import run_process moleculefile, smartsfile, moleculematchfile = [None] * 3 # get all SMARTS patterns in file mol_set = None try: session = get_session() mol_set = create_molecules_from_smiles_file(uploaded_molecules_file) moleculefile, _ = molecules_to_temporary_smiles_file(mol_set.molecules) try: smartsfile = write_smarts_to_tempfile() except NoSMARTSException: session.commit() return mol_set # Run moleculematch on the temporary SMARTS file, and write the # stdout to a new temporary result output file. moleculematchfile = tempfile.NamedTemporaryFile(mode='w+') match_cmd = [ current_app.config['MATCHTOOL_PATH'], '-i', '2', '-m', moleculefile.name, '-s', smartsfile.name ] run_process(match_cmd, stdout=moleculematchfile, stderr=sys.stderr, reraise_exceptions=True) moleculematchfile.seek(0) # Parse the moleculematch output parse_iterator = parse_moleculematch(moleculematchfile) # --- Code to store results in the database starts here --- for (smartsid, moleculeid) in parse_iterator: mmol = session.query(Molecule).get(moleculeid) msmarts = session.query(SMARTS).get(smartsid) newmatch = Match(molecule=mmol, smarts=msmarts) session.add(newmatch) # Commit the session session.commit() return mol_set except Exception as e: if mol_set is not None: # clean up molset if exception occurred session = get_session() session.delete(mol_set) session.commit() raise e finally: # close all open file handles if uploaded_molecules_file: uploaded_molecules_file.close() if moleculefile: moleculefile.close() if smartsfile: smartsfile.close() if moleculematchfile: moleculematchfile.close()
def close_output_stream(self, stream: BinaryIO) -> None: stream.close()
def handle( # type: ignore[override] self, name: str, key: typing.BinaryIO, pem: typing.BinaryIO, parent: typing.Optional[CertificateAuthority], password: typing.Optional[bytes], import_password: typing.Optional[bytes], issuer_alternative_name: typing.Optional[str], issuer_url: typing.Optional[str], **options: typing.Any ) -> None: if not os.path.exists(ca_settings.CA_DIR): try: os.makedirs(ca_settings.CA_DIR) except PermissionError as ex: pem.close() key.close() raise CommandError( "%s: Could not create CA_DIR: Permission denied." % ca_settings.CA_DIR ) from ex # FileNotFoundError shouldn't happen, whole point of this block is to create it pem_data = pem.read() key_data = key.read() crl_url = "\n".join(options["crl_url"]) # close reader objects (otherwise we get a ResourceWarning) key.close() pem.close() if issuer_alternative_name is None: # pragma: no branch - no CA sets this issuer_alternative_name = "" ca = CertificateAuthority( name=name, parent=parent, issuer_url=issuer_url, issuer_alt_name=issuer_alternative_name, crl_url=crl_url, ) # load public key try: pem_loaded = x509.load_pem_x509_certificate(pem_data, default_backend()) except Exception: # pylint: disable=broad-except try: pem_loaded = x509.load_der_x509_certificate(pem_data, default_backend()) except Exception as ex: raise CommandError("Unable to load public key.") from ex ca.update_certificate(pem_loaded) ca.private_key_path = ca_storage.generate_filename("%s.key" % ca.serial.replace(":", "")) # load private key try: key_loaded = serialization.load_pem_private_key(key_data, import_password, default_backend()) except Exception: # pylint: disable=broad-except try: key_loaded = serialization.load_der_private_key(key_data, import_password, default_backend()) except Exception as ex: raise CommandError("Unable to load private key.") from ex if password is None: encryption: serialization.KeySerializationEncryption = serialization.NoEncryption() else: encryption = serialization.BestAvailableEncryption(password) # write private key to file pem_as_bytes = key_loaded.private_bytes( encoding=Encoding.PEM, format=PrivateFormat.TraditionalOpenSSL, encryption_algorithm=encryption ) try: ca_storage.save(ca.private_key_path, ContentFile(pem_as_bytes)) except PermissionError as ex: raise CommandError( "%s: Permission denied: Could not open file for writing" % ca.private_key_path ) from ex # Only save CA to database if we loaded all data and copied private key ca.save()
def parse_jpg(self, bytes_: BinaryIO) -> None: """Method to extract metadata from a JPEG image ``bytes_`` needed to embed this image in a PDF document. This method creates this instance's attibute ``pdf_obj``, containing a dict that can be added to a :class:`pdfme.base.PDFBase` instance as a PDF Stream object that represents this image. Args: bytes_ (BinaryIO): A file-like object containing the image. """ try: while True: markerHigh, markerLow = struct.unpack('BB', bytes_.read(2)) if markerHigh != 0xFF or markerLow < 0xC0: raise SyntaxError('No JPEG marker found') elif markerLow == 0xDA: # SOS raise SyntaxError('No JPEG SOF marker found') elif (markerLow == 0xC8 or # JPG (markerLow >= 0xD0 and markerLow <= 0xD9) or # RSTx (markerLow >= 0xF0 and markerLow <= 0xFD)): # JPGx continue else: data_size, = struct.unpack('>H', bytes_.read(2)) data = bytes_.read(data_size - 2) if data_size > 2 else b'' if ((markerLow >= 0xC0 and markerLow <= 0xC3) or #SOF0-SOF3 (markerLow >= 0xC5 and markerLow <= 0xC7) or #SOF4-SOF7 (markerLow >= 0xC9 and markerLow <= 0xCB) or #SOF9-SOF11 (markerLow >= 0xCD and markerLow <= 0xCF) #SOF13-SOF15 ): depth, h, w, layers = struct.unpack_from('>BHHB', data) if layers == 3: colspace = b'/DeviceRGB' elif layers == 4: colspace = b'/DeviceCMYK' else: colspace = b'/DeviceGray' break except Exception: traceback.print_exc() raise ValueError("Couldn't process image: {}".format( self.image_name)) bytes_.seek(0) image_data = bytes_.read() bytes_.close() self.width = int(w) self.height = int(h) self.pdf_obj = { 'Type': b'/XObject', 'Subtype': b'/Image', 'Width': self.width, 'Height': self.height, 'ColorSpace': colspace, 'BitsPerComponent': int(depth), 'Filter': b'/DCTDecode', '__skip_filter__': True, '__stream__': image_data }
def _fpclose(self, fp: BinaryIO) -> None: assert self._fileRefCnt > 0 self._fileRefCnt -= 1 if not self._fileRefCnt and not self._filePassed: fp.close()