def close_output_stream(self, stream: BinaryIO) -> None:
for s3key, bio in self.key_stream_pairs.items():
if bio is not stream:
continue
obj = self.bucket.Object(s3key)
if s3key.endswith('.json'):
content_type = 'application/json'
elif s3key.endswith('.html'):
content_type = 'text/html'
elif s3key.endswith('.txt'):
content_type = 'text/plain'
else:
content_type = 'application/octet-stream'
logger.info(
f'put s3://{self.bucket.name}/{s3key}, '
f'content_type={content_type}'
)
bio.seek(0)
obj.upload_fileobj(
stream,
ExtraArgs={'ContentType': content_type},
)
stream.close()
return
raise ValueError('could not put a stream object to S3')
def backup_dfu(file: BinaryIO) -> None:
"""Backs up device data via DFU.
Args:
file:
file where firmware (MCU flash memory) will be saved
"""
try:
# TODO: implement this using pydfu
raise NoBackendError
except NoBackendError:
# if libusb was not found, try using dfu-util command line tool
with _get_dfu_util() as dfu_util:
file.close()
# dfu-util won't overwrite existing files so we have to do that first
os.remove(file.name)
exit(
call([
dfu_util,
"--device",
f",{SPIKE_PRIME_DEVICE},{MINDSTORMS_INVENTOR_DEVICE}",
"--alt",
"0",
"--dfuse-address",
f"{FIRMWARE_ADDRESS}:{FIRMWARE_SIZE}",
"--upload",
file.name,
]))
def publish_input_data(self, expected_input_id: str,
metadata: InputMetadata,
input_data_stream: BinaryIO) -> None:
input_file_path = self.input_file(expected_input_id)
if os.path.exists(input_file_path):
input_data_stream.close()
return
file_hash = hashlib.sha256()
fd, temp_file_path = tempfile.mkstemp(dir=self.temp_data_dir)
try:
with os.fdopen(fd, 'wb') as f:
bytes_read = 0
while True:
data = input_data_stream.read(READ_BUFFER_SIZE)
bytes_read += len(data)
log.debug(f'{bytes_read} bytes of input read')
if not data:
break
f.write(data)
file_hash.update(data)
input_id = file_hash.hexdigest()
if input_id != expected_input_id:
raise IncorrectInputIDException()
os.rename(temp_file_path, input_file_path)
if metadata.has_all_args():
self._store_input_id(metadata, input_id)
return
except Exception:
os.remove(temp_file_path)
raise
def restore_dfu(file: BinaryIO) -> None:
"""Restores flash memory from a file (raw data, not .dfu file).
Args:
file: the file that contains the firmware data
"""
file.seek(0, os.SEEK_END)
size = file.tell()
file.seek(0, os.SEEK_SET)
if size < 512:
raise ValueError("File is too small to be a valid firmware file")
try:
# TODO: implement this using pydfu
raise NoBackendError
except NoBackendError:
# if libusb was not found, try using dfu-util command line tool
with _get_dfu_util() as dfu_util:
file.close()
exit(
call([
dfu_util,
"--device",
f",{SPIKE_PRIME_DEVICE},{MINDSTORMS_INVENTOR_DEVICE}",
"--alt",
"0",
"--dfuse-address",
f"{FIRMWARE_ADDRESS}",
"--download",
file.name,
]))
def apply_window(
window: Any, sbuf: bytes = sbuf, target_stream: BinaryIO = target_stream
):
if window is None:
target_stream.close()
return # Last call
patch = delta.apply_txdelta_window(sbuf, window)
target_stream.write(patch)
def download_from(self, payload_id: payload_types.PayloadId, blob_name: str, dest_obj: BinaryIO = None,
dest_path: str = None, timeout=None) -> payload_types.PayloadFileDetails:
"""
Downloads a blob, identified by "blob_name", from a payload, identified by its "payload_id", from Clara.
Data downloaded from the payload will be written to 'dest_obj' a passed in BinaryIO object with write privileges
Args:
payload_id (payload_types.PayloadId): Unique identifier of the payload.
blob_name (str): The name, or path, of the blob in the payload.
dest_obj (BinaryIO): Target stream object to write to with write privileges
dest_path (str): Alternative to passing in BinaryIO object to download to, and rather passing in path for a file
"""
if (self._channel is None) or (self._stub is None):
raise Exception("Connection is currently closed. Please run reconnect() to reopen connection")
if (payload_id.value is None) or (payload_id.value == ""):
raise Exception("Payload identifier argument must be initialized with non-null instance")
if (blob_name is None) or (blob_name == ""):
raise Exception("Name of source blob must be initialized with non-null string")
file_path_used = False
if dest_obj is None:
if dest_path is None:
raise Exception("Destination object for upload must be initialized with non-null BinaryIO object")
else:
dest_obj = open(dest_path, 'wb')
file_path_used = True
request = payloads_pb2.PayloadsDownloadRequest(
header=self.get_request_header(),
name=blob_name,
payload_id=payload_id.to_grpc_value()
)
responses = self._stub.Download(request, timeout=timeout)
result = None
for resp in responses:
if result is None:
self.check_response_header(header=resp.header)
result = payload_types.PayloadFileDetails(
mode=resp.details.mode,
name=resp.details.name,
size=resp.details.size
)
dest_obj.write(resp.data)
if file_path_used:
dest_obj.close()
return result
def test_encodedfile_writelines(tmpfile: BinaryIO) -> None:
ef = capture.EncodedFile(tmpfile, "utf-8")
with pytest.raises(AttributeError):
ef.writelines([b"line1", b"line2"]) # type: ignore[list-item] # noqa: F821
assert ef.writelines(["line1", "line2"]) is None # type: ignore[func-returns-value] # noqa: F821
tmpfile.seek(0)
assert tmpfile.read() == b"line1line2"
tmpfile.close()
with pytest.raises(ValueError):
ef.read()
def dump(self, fp: BinaryIO) -> None:
"""
Serialize crypto object as binary stream to `fp` (a `.write()`-supporting file-like object).
:param fp: the output file pointer. Must be set in binary mode (mode='wb')
:return: None
"""
pem = self._private_key.private_bytes(Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption()) # type: ignore
fp.write(pem)
fp.close()
def stream_out(stream: typing.BinaryIO) -> typing.Generator[None, bytes, None]:
data = yield
while data:
stream.write(data)
data = yield
stream.close()
yield
def test_encodedfile_writelines(tmpfile: BinaryIO) -> None:
ef = capture.EncodedFile(tmpfile, encoding="utf-8")
with pytest.raises(TypeError):
ef.writelines([b"line1", b"line2"])
assert ef.writelines(["line3", "line4"]) is None # type: ignore[func-returns-value] # noqa: F821
ef.flush()
tmpfile.seek(0)
assert tmpfile.read() == b"line3line4"
tmpfile.close()
with pytest.raises(ValueError):
ef.read()
def stream_in(
stream: typing.BinaryIO,
buffer_size: int = 1 << 20) -> typing.Generator[bytes, None, None]:
try:
data = stream.read(buffer_size)
while data:
yield data
data = stream.read(buffer_size)
finally:
stream.close()
def upload(self, payload_id: payload_types.PayloadId, blob_name: str, file_object: BinaryIO = None,
file_path: str = None,
timeout=None) -> payload_types.PayloadFileDetails:
"""
Uploads a blob from "file_object", to a Clara Payload identified by "payload_id".
Each uploaded blob must be have a unique "blob_name" value within a given payload.
Args:
payload_id (payload_types.PayloadId): Unique identifier of the payload.
blob_name (str): The name, or path, of the blob in the payload.
file_object (BinaryIO): stream to read from and upload with read privileges
file_path (str): Alternative to passing in BinaryIO object for upload, and rather passing in path for a file
"""
if (self._channel is None) or (self._stub is None):
raise Exception("Connection is currently closed. Please run reconnect() to reopen connection")
if (payload_id.value is None) or (payload_id.value == ""):
raise Exception("Payload identifier argument must be initialized with non-null instance")
if (blob_name is None) or (blob_name == ""):
raise Exception("Name of destination blob must be initialized with non-null string")
file_path_used = False
if file_object is None:
if file_path is None:
raise Exception("File_object of file for upload must be initialized with non-null BinaryIO object")
else:
file_object = open(file_path, 'rb')
file_path_used = True
requests = self.upload_request_iterator(
payload_id=payload_id,
file_name=blob_name,
source_object=file_object
)
response = self._stub.Upload(
requests,
timeout=timeout
)
self.check_response_header(header=response.header)
result = payload_types.PayloadFileDetails(other=response.details)
if file_path_used:
file_object.close()
return result
def download(
obj: TrezorConnection,
output: BinaryIO,
version: str,
skip_check: bool,
fingerprint: str,
beta: bool,
bitcoin_only: bool,
) -> None:
"""Download and save the firmware image.
Validation is done by default, can be omitted by "-s" or "--skip-check".
When fingerprint or output file are not set, take them from SL servers.
"""
# When a version is specified, we do not even need the client connection
# (and we will not be checking device when validating)
if version:
url, fp = find_specified_firmware_version(version=version,
beta=beta,
bitcoin_only=bitcoin_only)
bootloader_onev2 = None
trezor_major_version = None
else:
with obj.client_context() as client:
url, fp = find_best_firmware_version(client=client,
version=version,
beta=beta,
bitcoin_only=bitcoin_only)
bootloader_onev2 = _is_bootloader_onev2(client)
trezor_major_version = client.features.major_version
firmware_data = download_firmware_data(url)
if not fingerprint:
fingerprint = fp
if not skip_check:
validate_firmware(
firmware_data=firmware_data,
fingerprint=fingerprint,
bootloader_onev2=bootloader_onev2,
trezor_major_version=trezor_major_version,
)
if not output:
output = open(_get_file_name_from_url(url), "wb")
output.write(firmware_data)
output.close()
click.echo(f"Firmware saved under {output.name}.")
def load(cls, file: BinaryIO):
"""
Decode and return a Protocol object.
"""
master = struct.unpack(b'>L>L', file.read(8))
if master[0] != cls.MAGIC:
raise ValueError('Attempt to decode corrupted or non-RVLT data')
length = struct.unpack(b'>Q', file.read(8)) # header length
header = file.read(length[0])
concrete = cls.lookup(master[1])
protocol = concrete._decode(header)
protocol._body = file
if protocol.body is None:
file.close()
return protocol
def _textChunkGenerator(self, streamIn: BinaryIO, logDir: str,
logFile: str) -> Generator[TextChunk, None, None]:
'''
Take an input stream with binary data, produced a generator of decoded
"text chunks", and also save the original binary data in a log file.
'''
os.makedirs(logDir, exist_ok=True)
logF = open(os.path.join(logDir, logFile), 'wb')
for bChunk in iter(streamIn.readline, b''):
logF.write(bChunk) # Write to log file immediately
tChunk = self._decodeBinaryChunk(bChunk) # decode
if tChunk is not None:
yield tChunk # TODO: split into actual text lines
# At the end...
streamIn.close() # Close the stream
logF.close() # Close the output file
def save(self, destination: BinaryIO, buffer_size: int = 16384) -> None:
"""Save the file to the destination.
Arguments:
destination: A filename (str) or file object to write to.
buffer_size: Buffer size as used as length in
:func:`shutil.copyfileobj`.
"""
close_destination = False
if isinstance(destination, str):
destination = open(destination, 'wb')
close_destination = True
try:
copyfileobj(self.stream, destination, buffer_size)
finally:
if close_destination:
destination.close()
def write_file(self, file: typing.BinaryIO, remote_file: str, file_len: int = -1,
run_after: FTCompleteOptions = FTCompleteOptions.DONT_RUN, linked_filename: Optional[str] = None,
linked_vid: int_str = 'afros', compress: bool = False, **kwargs):
if file_len < 0:
file_len = file.seek(0, 2)
file.seek(0, 0)
display_name = remote_file
if hasattr(file, 'name'):
display_name = f'{remote_file} ({Path(file.name).name})'
if compress and self.can_compress:
file, file_len = compress_file(file, file_len)
if self.is_wireless and file_len > 0x25000:
confirm(f'You\'re about to upload {file_len} bytes wirelessly. This could take some time, and you should '
f'consider uploading directly with a wire.', abort=True, default=False)
crc32 = self.VEX_CRC32.compute(file.read(file_len))
file.seek(0, 0)
addr = kwargs.get('addr', 0x03800000)
logger(__name__).info('Transferring {} ({} bytes) to the V5 from {}'.format(remote_file, file_len, file))
ft_meta = self.ft_initialize(remote_file, function='upload', length=file_len, crc=crc32, **kwargs)
if linked_filename is not None:
logger(__name__).debug('Setting file link')
self.ft_set_link(linked_filename, vid=linked_vid)
assert ft_meta['file_size'] >= file_len
if len(remote_file) > 24:
logger(__name__).info('Truncating {} to {} due to length'.format(remote_file, remote_file[:24]))
remote_file = remote_file[:24]
max_packet_size = int(ft_meta['max_packet_size'] / 2)
with ui.progressbar(length=file_len, label='Uploading {}'.format(display_name)) as progress:
for i in range(0, file_len, max_packet_size):
packet_size = max_packet_size
if i + max_packet_size > file_len:
packet_size = file_len - i
logger(__name__).debug('Writing {} bytes at 0x{:02X}'.format(packet_size, addr + i))
self.ft_write(addr + i, file.read(packet_size))
progress.update(packet_size)
logger(__name__).debug('Completed {} of {} bytes'.format(i + packet_size, file_len))
logger(__name__).debug('Data transfer complete, sending ft complete')
if compress and self.status['system_version'] in Spec('>=1.0.5'):
logger(__name__).info('Closing gzip file')
file.close()
self.ft_complete(options=run_after)
def encode(self,
obj: Union[dict, list, tuple, int, bytes, str],
buff: BinaryIO = None) -> Optional[bytes]:
"""
把对象bencode编码
:param obj:
:param buff:
:return: buff为None时, 返回bytes, 否则不返回, 数据写入传入buff
"""
has_init_buff = buff is not None
if not has_init_buff:
buff = BytesIO()
self.__write_value(buff, obj)
if not has_init_buff:
bs = buff.getvalue()
buff.close()
return bs
def handle( # type: ignore[override]
self, pub: typing.BinaryIO, ca: CertificateAuthority,
**options: typing.Any) -> None:
pub_data = pub.read()
# close reader objects (otherwise we get a ResourceWarning)
pub.close()
# load public key
try:
pub_loaded = x509.load_pem_x509_certificate(
pub_data, default_backend())
except Exception: # pylint: disable=broad-except
try:
pub_loaded = x509.load_der_x509_certificate(
pub_data, default_backend())
except Exception as ex:
raise CommandError("Unable to load public key.") from ex
cert = Certificate(ca=ca)
cert.update_certificate(pub_loaded)
cert.save()
def reader(self, stream: BinaryIO, context: SimpleNamespace) -> None:
'''
Read lines from subprocess output stream and write the information
to a specified callable or to sys.stderr.
@param stream: Popen.stdout readable byte stream object as returned
by open()
@param context: The information for the virtual environment
creation request being processed.
'''
progress = self.progress
while True:
s = stream.readline()
if not s:
break
if progress is not None:
progress(s, context)
else:
if not self.verbose:
sys.stderr.write('.')
else:
sys.stderr.write(s.decode('utf-8'))
sys.stderr.flush()
stream.close()
def merge_upload(
self,
request,
fileobj: BinaryIO,
conflicts: str,
author_name: Optional[str] = None,
author_email: Optional[str] = None,
method: str = "translate",
fuzzy: str = "",
):
"""Top level handler for file uploads."""
from weblate.accounts.models import AuditLog
# Optionally set authorship
orig_user = None
if author_email:
from weblate.auth.models import User
orig_user = request.user
request.user, created = User.objects.get_or_create(
email=author_email,
defaults={
"username": author_email,
"full_name": author_name or author_email,
},
)
if created:
AuditLog.objects.create(
request.user,
request,
"autocreated",
)
try:
if method == "replace":
return self.handle_replace(request, fileobj)
if method == "source":
return self.handle_source(request, fileobj)
filecopy = fileobj.read()
fileobj.close()
# Strip possible UTF-8 BOM
if filecopy[:3] == codecs.BOM_UTF8:
filecopy = filecopy[3:]
# Load backend file
store = try_load(
fileobj.name,
filecopy,
self.component.file_format_cls,
self.component.template_store,
)
# Check valid plural forms
if hasattr(store.store, "parseheader"):
header = store.store.parseheader()
try:
number, formula = Plural.parse_plural_forms(header["Plural-Forms"])
if not self.plural.same_plural(number, formula):
raise PluralFormsMismatch()
except (ValueError, KeyError):
# Formula wrong or missing
pass
if method in ("translate", "fuzzy", "approve"):
# Merge on units level
with self.component.repository.lock:
return self.merge_translations(
request, store, conflicts, method, fuzzy
)
elif method == "add":
return self.handle_add_upload(request, store, fuzzy=fuzzy)
# Add as sugestions
return self.merge_suggestions(request, store, fuzzy)
finally:
if orig_user:
request.user = orig_user
def read_output(out_stream: BinaryIO, out_queue: Queue) -> None:
for line in iter(out_stream.readline, b""):
out_queue.put(line.decode("utf-8"))
out_stream.close()
def close(self, file_descriptor: BinaryIO):
'''
Closes a file descriptor as returned by self.open()
'''
file_descriptor.close()
def calculate_molecule_matches(
uploaded_molecules_file: BinaryIO) -> MoleculeSet:
"""
Calculate molecule matches of all SMARTS in the database given a molecule file,
and store the Molecule and Match instances in the database.
:param uploaded_molecules_file: An open file handle to a molecule file to match
"""
import tempfile
import sys
from smartsexplore.util import run_process
moleculefile, smartsfile, moleculematchfile = [None] * 3
# get all SMARTS patterns in file
mol_set = None
try:
session = get_session()
mol_set = create_molecules_from_smiles_file(uploaded_molecules_file)
moleculefile, _ = molecules_to_temporary_smiles_file(mol_set.molecules)
try:
smartsfile = write_smarts_to_tempfile()
except NoSMARTSException:
session.commit()
return mol_set
# Run moleculematch on the temporary SMARTS file, and write the
# stdout to a new temporary result output file.
moleculematchfile = tempfile.NamedTemporaryFile(mode='w+')
match_cmd = [
current_app.config['MATCHTOOL_PATH'], '-i', '2', '-m',
moleculefile.name, '-s', smartsfile.name
]
run_process(match_cmd,
stdout=moleculematchfile,
stderr=sys.stderr,
reraise_exceptions=True)
moleculematchfile.seek(0)
# Parse the moleculematch output
parse_iterator = parse_moleculematch(moleculematchfile)
# --- Code to store results in the database starts here ---
for (smartsid, moleculeid) in parse_iterator:
mmol = session.query(Molecule).get(moleculeid)
msmarts = session.query(SMARTS).get(smartsid)
newmatch = Match(molecule=mmol, smarts=msmarts)
session.add(newmatch)
# Commit the session
session.commit()
return mol_set
except Exception as e:
if mol_set is not None: # clean up molset if exception occurred
session = get_session()
session.delete(mol_set)
session.commit()
raise e
finally: # close all open file handles
if uploaded_molecules_file:
uploaded_molecules_file.close()
if moleculefile:
moleculefile.close()
if smartsfile:
smartsfile.close()
if moleculematchfile:
moleculematchfile.close()
def close_output_stream(self, stream: BinaryIO) -> None:
stream.close()
def handle( # type: ignore[override]
self,
name: str,
key: typing.BinaryIO,
pem: typing.BinaryIO,
parent: typing.Optional[CertificateAuthority],
password: typing.Optional[bytes],
import_password: typing.Optional[bytes],
issuer_alternative_name: typing.Optional[str],
issuer_url: typing.Optional[str],
**options: typing.Any
) -> None:
if not os.path.exists(ca_settings.CA_DIR):
try:
os.makedirs(ca_settings.CA_DIR)
except PermissionError as ex:
pem.close()
key.close()
raise CommandError(
"%s: Could not create CA_DIR: Permission denied." % ca_settings.CA_DIR
) from ex
# FileNotFoundError shouldn't happen, whole point of this block is to create it
pem_data = pem.read()
key_data = key.read()
crl_url = "\n".join(options["crl_url"])
# close reader objects (otherwise we get a ResourceWarning)
key.close()
pem.close()
if issuer_alternative_name is None: # pragma: no branch - no CA sets this
issuer_alternative_name = ""
ca = CertificateAuthority(
name=name,
parent=parent,
issuer_url=issuer_url,
issuer_alt_name=issuer_alternative_name,
crl_url=crl_url,
)
# load public key
try:
pem_loaded = x509.load_pem_x509_certificate(pem_data, default_backend())
except Exception: # pylint: disable=broad-except
try:
pem_loaded = x509.load_der_x509_certificate(pem_data, default_backend())
except Exception as ex:
raise CommandError("Unable to load public key.") from ex
ca.update_certificate(pem_loaded)
ca.private_key_path = ca_storage.generate_filename("%s.key" % ca.serial.replace(":", ""))
# load private key
try:
key_loaded = serialization.load_pem_private_key(key_data, import_password, default_backend())
except Exception: # pylint: disable=broad-except
try:
key_loaded = serialization.load_der_private_key(key_data, import_password, default_backend())
except Exception as ex:
raise CommandError("Unable to load private key.") from ex
if password is None:
encryption: serialization.KeySerializationEncryption = serialization.NoEncryption()
else:
encryption = serialization.BestAvailableEncryption(password)
# write private key to file
pem_as_bytes = key_loaded.private_bytes(
encoding=Encoding.PEM, format=PrivateFormat.TraditionalOpenSSL, encryption_algorithm=encryption
)
try:
ca_storage.save(ca.private_key_path, ContentFile(pem_as_bytes))
except PermissionError as ex:
raise CommandError(
"%s: Permission denied: Could not open file for writing" % ca.private_key_path
) from ex
# Only save CA to database if we loaded all data and copied private key
ca.save()
def parse_jpg(self, bytes_: BinaryIO) -> None:
"""Method to extract metadata from a JPEG image ``bytes_`` needed to
embed this image in a PDF document.
This method creates this instance's attibute ``pdf_obj``, containing
a dict that can be added to a :class:`pdfme.base.PDFBase` instance as
a PDF Stream object that represents this image.
Args:
bytes_ (BinaryIO): A file-like object containing the
image.
"""
try:
while True:
markerHigh, markerLow = struct.unpack('BB', bytes_.read(2))
if markerHigh != 0xFF or markerLow < 0xC0:
raise SyntaxError('No JPEG marker found')
elif markerLow == 0xDA: # SOS
raise SyntaxError('No JPEG SOF marker found')
elif (markerLow == 0xC8 or # JPG
(markerLow >= 0xD0 and markerLow <= 0xD9) or # RSTx
(markerLow >= 0xF0 and markerLow <= 0xFD)): # JPGx
continue
else:
data_size, = struct.unpack('>H', bytes_.read(2))
data = bytes_.read(data_size - 2) if data_size > 2 else b''
if ((markerLow >= 0xC0 and markerLow <= 0xC3)
or #SOF0-SOF3
(markerLow >= 0xC5
and markerLow <= 0xC7) or #SOF4-SOF7
(markerLow >= 0xC9 and markerLow <= 0xCB)
or #SOF9-SOF11
(markerLow >= 0xCD and markerLow <= 0xCF) #SOF13-SOF15
):
depth, h, w, layers = struct.unpack_from('>BHHB', data)
if layers == 3: colspace = b'/DeviceRGB'
elif layers == 4: colspace = b'/DeviceCMYK'
else: colspace = b'/DeviceGray'
break
except Exception:
traceback.print_exc()
raise ValueError("Couldn't process image: {}".format(
self.image_name))
bytes_.seek(0)
image_data = bytes_.read()
bytes_.close()
self.width = int(w)
self.height = int(h)
self.pdf_obj = {
'Type': b'/XObject',
'Subtype': b'/Image',
'Width': self.width,
'Height': self.height,
'ColorSpace': colspace,
'BitsPerComponent': int(depth),
'Filter': b'/DCTDecode',
'__skip_filter__': True,
'__stream__': image_data
}
def _fpclose(self, fp: BinaryIO) -> None:
assert self._fileRefCnt > 0
self._fileRefCnt -= 1
if not self._fileRefCnt and not self._filePassed:
fp.close()
