def stream_compress(instr: io.BufferedReader, outstr: io.BufferedWriter, chunk_size=DEFAULT_MAX_CHUNK): """ A stream processor that call compress on bytes available in instr And write them into outstr :param instr: buffered reader :param outstr: buffered writer :param chunk: the sizeof chunk to read at one time. if 0 attempt to read as much as possible. :returns: original consumed data size, compressed data size """ orig_data_size: int = 0 comp_data_size: int = 0 inbytes: bytes = instr.read(chunk_size) while inbytes: data_comp = compress(inbytes) # we prepend with uncompressed data chunk size # to be used later for random access orig_data_size += len(inbytes) # '>H' is unsigned short format, fit on 2 bytes. output = struct.pack('>H', len(inbytes)) + struct.pack( '>H', len(data_comp)) + data_comp # we need to include the chunk indexes in the compressed size comp_data_size += len(output) outstr.write(output) # keep consuming data, in case more is available... inbytes = instr.read(chunk_size) return orig_data_size, comp_data_size
def consecutive_download(node_id: str, file: io.BufferedWriter, **kwargs): """Keyword args: write_callback""" r = BackOffRequest.get(get_content_url() + 'nodes/' + node_id + '/content', stream=True) if r.status_code not in OK_CODES: raise RequestError(r.status_code, r.text) write_callback = kwargs.get('write_callback', None) total_ln = int(r.headers.get('content-length')) length = kwargs.get('length', None) if length and total_ln != length: logging.info('Length mismatch: argument %d, content %d' % (length, total_ln)) pgo = progress.Progress() curr_ln = 0 try: for chunk in r.iter_content(chunk_size=FS_RW_CHUNK_SZ): if chunk: # filter out keep-alive new chunks file.write(chunk) file.flush() if write_callback: write_callback(chunk) curr_ln += len(chunk) pgo.print_progress(total_ln, curr_ln) except (ConnectionError, ReadTimeoutError) as e: raise RequestError(RequestError.CODE.READ_TIMEOUT, '[acd_cli] Timeout. ' + e.__str__()) print() # break progress line r.close() return
class PANDAUploadBackend(AbstractUploadBackend): """ Customized backend to handle AJAX uploads. """ def update_filename(self, request, filename): """ Verify that the filename is unique, if it isn't append and iterate a counter until it is. """ self._original_filename = filename filename = self._original_filename root, ext = os.path.splitext(self._original_filename) path = os.path.join(settings.MEDIA_ROOT, filename) i = 1 while os.path.exists(path): filename = '%s%i%s' % (root, i, ext) path = os.path.join(settings.MEDIA_ROOT, filename) i += 1 return filename def setup(self, filename): """ Open the destination file for writing. """ self._path = os.path.join(settings.MEDIA_ROOT, filename) try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._dest = BufferedWriter(FileIO(self._path, "w")) def upload_chunk(self, chunk): """ Write a chunk of data to the destination. """ self._dest.write(chunk) def upload_complete(self, request, filename): """ Close the destination file and create an Upload object in the database recording its existence. """ self._dest.close() root, ext = os.path.splitext(filename) path = os.path.join(settings.MEDIA_ROOT, filename) size = os.path.getsize(path) upload = Upload.objects.create( filename=filename, original_filename=self._original_filename, size=size) return { 'id': upload.id }
def save_bits(file_name: str, bool_array: nparray, packed=True) -> int: ''' Save bits to a file from a bool array. Parameters ---------- file_name: string The name of the file to save. bool_array: numpy.array The bool array. packed: bool Whether to pack the bits into bytes. Defaults to True. Returns the number of bytes saved. ''' with open(file_name, 'wb') as bit_file: writer = BufferedWriter(bit_file) count = 0 if packed: for byte in pack_bools_to_bytes(bool_array): writer.write(byte) count += 1 else: for byte in bools_to_bytes(bool_array): writer.write(byte) count += 1 writer.flush() return count
def chunked_download(self, node_id: str, file: io.BufferedWriter, **kwargs): """:param kwargs: offset (int): byte offset -- start byte for ranged request length (int): total file length[!], equal to end + 1 write_callbacks (list[function]) """ ok_codes = [http.PARTIAL_CONTENT] write_callbacks = kwargs.get('write_callbacks', []) chunk_start = kwargs.get('offset', 0) length = kwargs.get('length', 100 * 1024**4) retries = 0 while chunk_start < length: chunk_end = chunk_start + CHUNK_SIZE - 1 if chunk_end >= length: chunk_end = length - 1 if retries >= CHUNK_MAX_RETRY: raise RequestError( RequestError.CODE.FAILED_SUBREQUEST, '[acd_api] Downloading chunk failed multiple times.') r = self.BOReq.get( self.content_url + 'nodes/' + node_id + '/content', stream=True, acc_codes=ok_codes, headers={'Range': 'bytes=%d-%d' % (chunk_start, chunk_end)}) logger.debug('Range %d-%d' % (chunk_start, chunk_end)) # this should only happen at the end of unknown-length downloads if r.status_code == http.REQUESTED_RANGE_NOT_SATISFIABLE: r.close() logger.debug('Invalid byte range requested %d-%d' % (chunk_start, chunk_end)) break if r.status_code not in ok_codes: r.close() retries += 1 logging.debug('Chunk [%d-%d], retry %d.' % (chunk_start, chunk_end, retries)) continue curr_ln = 0 try: for chunk in r.iter_content(chunk_size=FS_RW_CHUNK_SZ): if chunk: # filter out keep-alive new chunks file.write(chunk) file.flush() for wcb in write_callbacks: wcb(chunk) curr_ln += len(chunk) finally: r.close() chunk_start += CHUNK_SIZE retries = 0 return
class LocalUploadBackend(AbstractUploadBackend): UPLOAD_DIR = ajaxuploader_settings.UPLOAD_DIRECTORY # TODO: allow this to be overridden per-widget/view def setup(self, filename): self._relative_path = os.path.normpath( os.path.join( force_unicode( datetime.datetime.now().strftime( # allow %Y, %s, etc smart_str(self.UPLOAD_DIR))), filename)) self._path = os.path.join(settings.MEDIA_ROOT, self._relative_path) try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._dest = BufferedWriter(FileIO(self._path, "w")) def upload_chunk(self, chunk): self._dest.write(chunk) def upload_complete(self, request, filename): self._dest.close() return {"path": self._relative_path} def update_filename(self, request, filename): return ajaxuploader_settings.SANITIZE_FILENAME(filename)
def send_packet(writer: io.BufferedWriter, cmd: int, flags: int = 0, data: bytes = b'') -> None: packet = Packet(cmd, flags, len(data), data, protocol_utils.Formats.HEADER) writer.write(packet.to_bytes()) writer.flush()
class LocalUploadBackend(AbstractUploadBackend): UPLOAD_DIR = getattr(settings, "UPLOAD_DIR", "uploads") def setup(self, filename, *args, **kwargs): self._path = os.path.join(settings.MEDIA_ROOT, self.UPLOAD_DIR, filename) try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._dest = BufferedWriter(FileIO(self._path, "w")) def upload_chunk(self, chunk, *args, **kwargs): self._dest.write(chunk) def upload_complete(self, request, filename, *args, **kwargs): path = settings.MEDIA_URL + self.UPLOAD_DIR + "/" + filename self._dest.close() return {"path": path} def update_filename(self, request, filename, *args, **kwargs): """ Returns a new name for the file being uploaded. Ensure file with name doesn't exist, and if it does, create a unique filename to avoid overwriting """ filename = os.path.basename(filename) self._dir = os.path.join(settings.MEDIA_ROOT, self.UPLOAD_DIR) unique_filename = False filename_suffix = 0 # Check if file at filename exists if os.path.isfile(os.path.join(self._dir, filename)): while not unique_filename: try: if filename_suffix == 0: open(os.path.join(self._dir, filename)) else: filename_no_extension, extension = os.path.splitext( filename) open( os.path.join( self._dir, filename_no_extension + str(filename_suffix) + extension)) filename_suffix += 1 except IOError: unique_filename = True if filename_suffix == 0: return filename else: return filename_no_extension + str(filename_suffix) + extension @property def path(self): """ Return a path of file uploaded """ return self._path
class LocalUploadBackend(AbstractUploadBackend): UPLOAD_DIR = getattr(settings, "UPLOAD_DIR", "uploads") def setup(self, filename, *args, **kwargs): self._path = os.path.join( settings.MEDIA_ROOT, self.UPLOAD_DIR, filename) try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._dest = BufferedWriter(FileIO(self._path, "w")) def upload_chunk(self, chunk, *args, **kwargs): self._dest.write(chunk) def upload_complete(self, request, filename, *args, **kwargs): path = settings.MEDIA_URL + self.UPLOAD_DIR + "/" + filename self._dest.close() return {"path": path} def update_filename(self, request, filename, *args, **kwargs): """ Returns a new name for the file being uploaded. Ensure file with name doesn't exist, and if it does, create a unique filename to avoid overwriting """ filename = os.path.basename(filename) self._dir = os.path.join( settings.MEDIA_ROOT, self.UPLOAD_DIR) unique_filename = False filename_suffix = 0 # Check if file at filename exists if os.path.isfile(os.path.join(self._dir, filename)): while not unique_filename: try: if filename_suffix == 0: open(os.path.join(self._dir, filename)) else: filename_no_extension, extension = os.path.splitext(filename) open(os.path.join(self._dir, filename_no_extension + str(filename_suffix) + extension)) filename_suffix += 1 except IOError: unique_filename = True if filename_suffix == 0: return filename else: return filename_no_extension + str(filename_suffix) + extension @property def path(self): """ Return a path of file uploaded """ return self._path
def chunked_download(self, node_id: str, file: io.BufferedWriter, **kwargs): """:param kwargs: offset (int): byte offset -- start byte for ranged request length (int): total file length[!], equal to end + 1 write_callbacks (list[function]) """ ok_codes = [http.PARTIAL_CONTENT] write_callbacks = kwargs.get("write_callbacks", []) chunk_start = kwargs.get("offset", 0) length = kwargs.get("length", 100 * 1024 ** 4) retries = 0 while chunk_start < length: chunk_end = chunk_start + CHUNK_SIZE - 1 if chunk_end >= length: chunk_end = length - 1 if retries >= CHUNK_MAX_RETRY: raise RequestError( RequestError.CODE.FAILED_SUBREQUEST, "[acd_api] Downloading chunk failed multiple times." ) r = self.BOReq.get( self.content_url + "nodes/" + node_id + "/content", stream=True, acc_codes=ok_codes, headers={"Range": "bytes=%d-%d" % (chunk_start, chunk_end)}, ) logger.debug("Range %d-%d" % (chunk_start, chunk_end)) # this should only happen at the end of unknown-length downloads if r.status_code == http.REQUESTED_RANGE_NOT_SATISFIABLE: logger.debug("Invalid byte range requested %d-%d" % (chunk_start, chunk_end)) break if r.status_code not in ok_codes: r.close() retries += 1 logging.debug("Chunk [%d-%d], retry %d." % (chunk_start, chunk_end, retries)) continue curr_ln = 0 try: for chunk in r.iter_content(chunk_size=FS_RW_CHUNK_SZ): if chunk: # filter out keep-alive new chunks file.write(chunk) file.flush() for wcb in write_callbacks: wcb(chunk) curr_ln += len(chunk) finally: r.close() chunk_start += CHUNK_SIZE retries = 0 return
def chunked_download(node_id: str, file: io.BufferedWriter, **kwargs): """Keyword args: offset: byte offset length: total length, equal to end - 1 write_callback """ ok_codes = [http.PARTIAL_CONTENT] write_callback = kwargs.get('write_callback', None) length = kwargs.get('length', 100 * 1024 ** 4) pgo = progress.Progress() chunk_start = kwargs.get('offset', 0) retries = 0 while chunk_start < length: chunk_end = chunk_start + CHUNK_SIZE - 1 if chunk_end >= length: chunk_end = length - 1 if retries >= CHUNK_MAX_RETRY: raise RequestError(RequestError.CODE.FAILED_SUBREQUEST, '[acd_cli] Downloading chunk failed multiple times.') r = BackOffRequest.get(get_content_url() + 'nodes/' + node_id + '/content', stream=True, acc_codes=ok_codes, headers={'Range': 'bytes=%d-%d' % (chunk_start, chunk_end)}) logger.debug('Range %d-%d' % (chunk_start, chunk_end)) # this should only happen at the end of unknown-length downloads if r.status_code == http.REQUESTED_RANGE_NOT_SATISFIABLE: logger.debug('Invalid byte range requested %d-%d' % (chunk_start, chunk_end)) break if r.status_code not in ok_codes: r.close() retries += 1 logging.debug('Chunk [%d-%d], retry %d.' % (retries, chunk_start, chunk_end)) continue try: curr_ln = 0 for chunk in r.iter_content(chunk_size=FS_RW_CHUNK_SZ): if chunk: # filter out keep-alive new chunks file.write(chunk) file.flush() if write_callback: write_callback(chunk) curr_ln += len(chunk) pgo.print_progress(length, curr_ln + chunk_start) chunk_start += CHUNK_SIZE retries = 0 r.close() except (ConnectionError, ReadTimeoutError) as e: file.close() raise RequestError(RequestError.CODE.READ_TIMEOUT, '[acd_cli] Timeout. ' + e.__str__()) print() # break progress line return
def _encode(cls, file: BufferedReader, archive_file: BufferedWriter, file_path: str): encoding_dictionary = cls.get_encoding_dictionary(file) archive_file.write(cls._get_file_path_data(file_path)) archive_file.write(cls._get_dictionary_data(encoding_dictionary)) archive_file.write(cls._compose_data(cls._get_control_sum(file))) cls._write_encoded_file_data(file, archive_file, encoding_dictionary)
def chunked_download(self, node_id: str, file: io.BufferedWriter, **kwargs): """:param kwargs: offset (int): byte offset -- start byte for ranged request length (int): total file length[!], equal to end + 1 write_callbacks (list[function]) """ ok_codes = [http.PARTIAL_CONTENT] write_callbacks = kwargs.get('write_callbacks', []) chunk_start = kwargs.get('offset', 0) length = kwargs.get('length', 100 * 1024 ** 4) dl_chunk_sz = self._conf.getint('transfer', 'dl_chunk_size') retries = 0 while chunk_start < length: chunk_end = chunk_start + dl_chunk_sz - 1 if chunk_end >= length: chunk_end = length - 1 if retries >= self._conf.getint('transfer', 'chunk_retries'): raise RequestError(RequestError.CODE.FAILED_SUBREQUEST, '[acd_api] Downloading chunk failed multiple times.') r = self.BOReq.get(self.content_url + 'nodes/' + node_id + '/content', stream=True, acc_codes=ok_codes, headers={'Range': 'bytes=%d-%d' % (chunk_start, chunk_end)}) logger.debug('Node "%s", range %d-%d' % (node_id, chunk_start, chunk_end)) # this should only happen at the end of unknown-length downloads if r.status_code == http.REQUESTED_RANGE_NOT_SATISFIABLE: r.close() logger.debug('Invalid byte range requested %d-%d' % (chunk_start, chunk_end)) break if r.status_code not in ok_codes: r.close() retries += 1 logging.debug('Chunk [%d-%d], retry %d.' % (chunk_start, chunk_end, retries)) continue curr_ln = 0 try: for chunk in r.iter_content(chunk_size=self._conf.getint('transfer', 'fs_chunk_size')): if chunk: # filter out keep-alive new chunks file.write(chunk) file.flush() for wcb in write_callbacks: wcb(chunk) curr_ln += len(chunk) finally: r.close() chunk_start = file.tell() retries = 0 return
def _extract(self, frame:ImageFrame, dstfile:io.BufferedWriter, imgfile:io.BufferedReader, **kwargs): """override optional (if no encryption/compression)""" imgfile.seek(frame.offset) dstdata = imgfile.read(frame.length) if len(dstdata) != frame.length: raise ValueError('Image Frame data length does not match, expected {0.length!r} bytes, not {1!r} bytes'.format(frame, len(dstdata))) if dstfile is None: return dstdata else: dstfile.write(dstdata)
class LocalUploadBackend(AbstractUploadBackend): UPLOAD_DIR = "uploads" def setup(self, filename): self._path = os.path.join( settings.MEDIA_ROOT, self.UPLOAD_DIR, filename) try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._dest = BufferedWriter(FileIO(self._path, "w")) def upload_chunk(self, chunk): self._dest.write(chunk) def upload_complete(self, request, filename): path = settings.MEDIA_URL + self.UPLOAD_DIR + "/" + filename self._dest.close() return {"path": path} def update_filename(self, request, filename): """ Returns a new name for the file being uploaded. Ensure file with name doesn't exist, and if it does, create a unique filename to avoid overwriting """ self._dir = os.path.join( settings.MEDIA_ROOT, self.UPLOAD_DIR) unique_filename = False filename_suffix = 0 print "orig filename: " + os.path.join(self._dir, filename) # Check if file at filename exists if os.path.isfile(os.path.join(self._dir, filename)): while not unique_filename: try: if filename_suffix == 0: open(os.path.join(self._dir, filename)) else: filename_no_extension, extension = os.path.splitext(filename) print "filename all ready exists. Trying " + filename_no_extension + str(filename_suffix) + extension open(os.path.join(self._dir, filename_no_extension + str(filename_suffix) + extension)) filename_suffix += 1 except IOError: unique_filename = True if filename_suffix == 0: print "using filename: " + os.path.join(self._dir, filename) return filename else: print "using filename: " + filename_no_extension + str(filename_suffix) + extension return filename_no_extension + str(filename_suffix) + extension
def _extract(self, entry: ArchiveEntry, dstfile: io.BufferedWriter, arcfile: io.BufferedReader, **kwargs): """override optional (if no encryption/compression)""" arcfile.seek(entry.offset) dstdata = arcfile.read(entry.length) if len(dstdata) != entry.length: raise ValueError( 'Archive Entry data length does not match, expected {0.length!r} bytes, not {1!r} bytes' .format(entry, len(dstdata))) if dstfile is None: return dstdata else: dstfile.write(dstdata)
def chunked_download(node_id: str, file: io.BufferedWriter, **kwargs): """Keyword args: offset (int): byte offset -- start byte for ranged request length (int): total file length[!], equal to end + 1 write_callbacks: (list[function]) """ ok_codes = [http.PARTIAL_CONTENT] write_callbacks = kwargs.get('write_callbacks', []) chunk_start = kwargs.get('offset', 0) length = kwargs.get('length', 100 * 1024 ** 4) retries = 0 while chunk_start < length: chunk_end = chunk_start + CHUNK_SIZE - 1 if chunk_end >= length: chunk_end = length - 1 if retries >= CHUNK_MAX_RETRY: raise RequestError(RequestError.CODE.FAILED_SUBREQUEST, '[acd_cli] Downloading chunk failed multiple times.') r = BackOffRequest.get(get_content_url() + 'nodes/' + node_id + '/content', stream=True, acc_codes=ok_codes, headers={'Range': 'bytes=%d-%d' % (chunk_start, chunk_end)}) logger.debug('Range %d-%d' % (chunk_start, chunk_end)) # this should only happen at the end of unknown-length downloads if r.status_code == http.REQUESTED_RANGE_NOT_SATISFIABLE: logger.debug('Invalid byte range requested %d-%d' % (chunk_start, chunk_end)) break if r.status_code not in ok_codes: r.close() retries += 1 logging.debug('Chunk [%d-%d], retry %d.' % (chunk_start, chunk_end, retries)) continue curr_ln = 0 # connection exceptions occur here for chunk in r.iter_content(chunk_size=FS_RW_CHUNK_SZ): if chunk: # filter out keep-alive new chunks file.write(chunk) file.flush() for wcb in write_callbacks: wcb(chunk) curr_ln += len(chunk) chunk_start += CHUNK_SIZE retries = 0 r.close() return
def download(url: str, fhandle: BufferedWriter, on_progress: Callable = None): """ Download a file to a specific target. Inspired from Patrick Massot's code in leanproject. :param url: HTTP(s) url to download file from (GET request) :param path: File path on local filesystem :param on_progress: callback(idx,count, progress) to monitor download progress. :return: the sha1 checksum of the downloaded file """ # TODO(florian): better error handling ? # -> ConnectionError raised by requests.get # -> HTTPError raised by raise_for_status sha1 = hashlib.sha1() response = requests.get(url, stream=True) response.raise_for_status() # Raise HTTPError if any tot_len = response.headers.get("content-length", 0) if not tot_len: fhandle.write(response.content) sha1.update(response.content) else: dl_size = 0 tot_len = int(tot_len) progress = 0 progress_prev = 0 for chunk in response.iter_content(chunk_size=CHUNK_SIZE): dl_size += len(chunk) fhandle.write(chunk) sha1.update(chunk) # Compute and display progress if /10 progress_prev = progress progress = (100 * (dl_size / tot_len)) if int(progress) % 10 == 0 and int(progress) != int(progress_prev): log.info(_("Progress : {:03d}%").format(int(progress))) # Trigger progress callback if on_progress is not None: on_progress(dl_size, tot_len, progress) return sha1.hexdigest()
class PANDAAbstractUploadBackend(AbstractUploadBackend): """ Customized backend to handle AJAX uploads. """ def update_filename(self, request, filename): """ Verify that the filename is unique, if it isn't append and iterate a counter until it is. """ self._original_filename = filename filename = self._original_filename root, ext = os.path.splitext(self._original_filename) path = os.path.join(settings.MEDIA_ROOT, filename) i = 1 while os.path.exists(path): filename = '%s%i%s' % (root, i, ext) path = os.path.join(settings.MEDIA_ROOT, filename) i += 1 return filename def setup(self, filename): """ Open the destination file for writing. """ self._path = os.path.join(settings.MEDIA_ROOT, filename) try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._dest = BufferedWriter(FileIO(self._path, "w")) def upload_chunk(self, chunk): """ Write a chunk of data to the destination. """ self._dest.write(chunk) def upload_complete(self, request, filename): """ Close the destination file. """ self._dest.close()
def stream_decompress(instr: io.BufferedReader, outstr: io.BufferedWriter, chunk_size=DEFAULT_MAX_CHUNK): """ A stream processor that call decompress on bytes available in instr And write them into outstr :param instr: buffered reader :param outstr: buffered writer :param chunk: the sizeof chunk to read at one time. if 0 attempt to read as much as possible. :returns: compressed data size, original consumed data size """ orig_data_size: int = 0 decomp_data_size: int = 0 inbytes: bytes = instr.read(chunk_size) # we find chunk indexes # Note: we dont care about next_ori_chunk_idx for decompressing everything # next_ori_chunk_idx = struct.unpack('>H', inbytes[0:2]) if inbytes else None next_comp_chunk_idx: int = struct.unpack('>H', inbytes[2:4]) if inbytes else None # careful : next_ori_chunk_idx is the location *after* decompression (ie. in the original uncompressed sequence) cur_chunk_idx = 4 while inbytes: decomp_data = bytearray() while len(inbytes) > next_comp_chunk_idx: # if next chunk index is already in range, we can already uncompress this chunk decomp_data += decompress( inbytes[cur_chunk_idx:next_comp_chunk_idx]) # find next chunk cur_chunk_idx = next_comp_chunk_idx next_comp_chunk_idx = inbytes[next_comp_chunk_idx] orig_data_size += len(inbytes) decomp_data_size += len(decomp_data) outstr.write(bytes(decomp_data)) # correct the next chunk index value next_comp_chunk_idx = next_comp_chunk_idx - len(inbytes) cur_chunk_idx = 0 # read more data in case it is now available inbytes = instr.read(chunk_size) return orig_data_size, decomp_data_size
def send_generic(msg: KQMLPerformative, out: BufferedWriter): """Basic send mechanism copied (more or less) from pykqml. Writes the msg as a string to the output buffer then flushes it. Args: msg (KQMLPerformative): Message to be sent out (BufferedWriter): The output to write to, needed for sending to Companions and sending on our own port. """ LOGGER.debug('Sending: %s', msg) try: msg.write(out) except IOError: LOGGER.error('IOError during message sending') out.write(b'\n') out.flush()
def pandas_write_hdf5_buffered(df: pd.DataFrame, key: str, buf: io.BufferedWriter): """ Write a Pandas dataframe in HDF5 format to a buffer. """ ## I am getting ## HDF5ExtError("Unable to open/create file '/dev/null'") ## unable to truncate a file which is already open with write_lock: with pd.HDFStore("/dev/null", mode="w", driver="H5FD_CORE", driver_core_backing_store=0) as store: store["results"] = df buf.write(store._handle.get_file_image())
def assemble_script(self, writer: io.BufferedWriter) -> NoReturn: if not isinstance(writer, StructIO): writer = StructIO(writer) # header: if self.signature not in (self.SIGNATURE_ENCRYPTED, self.SIGNATURE_DECRYPTED): raise Exception( f'{self.__class__.__name__} signature must be {self.SIGNATURE_ENCRYPTED.decode("cp932")!r} or {self.SIGNATURE_DECRYPTED.decode("cp932")!r}, not {self.signature.decode("cp932")!r}' ) writer.pack('<16sIII', self.signature, self.main_offset, self.line_count, len(self.functions)) is_encrypted: bool = (self.signature == self.SIGNATURE_ENCRYPTED) assert (is_encrypted ^ (self.signature in (self.SIGNATURE_DECRYPTED, self.SIGNATURE_PLAIN))) # functions table: for fn in self.functions: writer.pack('<II', *fn) # fn.name_hash, fn.offset # bytecode: writer.pack('<I', self.bytecode_size) # initialize full-length of bytecode ahead of time (is this actually efficient in Python?) ms: io.BytesIO = io.BytesIO(bytes(self.bytecode_size)) self.assemble_bytecode(StructIO(ms)) ms.flush() bytecode: bytes = ms.getvalue() if is_encrypted: bytecode = crypt.crypt32(bytecode) # encrypt bytecode written_size = writer.write(bytecode) assert (written_size == self.bytecode_size)
class LocalUploadBackend(AbstractUploadBackend): UPLOAD_DIR = "uploads" def setup(self, filename): self._path = os.path.join(settings.MEDIA_ROOT, self.UPLOAD_DIR, filename) try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._dest = BufferedWriter(FileIO(self._path, "w")) def upload_chunk(self, chunk): self._dest.write(chunk) def upload_complete(self, request, filename): path = settings.MEDIA_URL + self.UPLOAD_DIR + "/" + filename return {"path": path}
def _dump(self, data: TLM, file: BufferedWriter, *args, **kwargs): size_x, size_y = (0, 0) if data.lod_images: size_x, size_y = data.lod_images[0].size file.write( pack('<11I', data.unknown_00, data.created_at.year, data.created_at.month, data.created_at.day, data.created_at.hour, data.created_at.minute, data.created_at.second, data.created_at.microsecond, data.unknown_01, size_x, size_y)) for lod_image in data.lod_images: lod_array = np.array(lod_image.getdata(), dtype=np.uint8) lod_array = lod_array.reshape(lod_image.size[0], lod_image.size[1], 4) lod_array = self._swap_rb(lod_array) file.write(lod_array.tobytes())
class LocalUploadBackend(AbstractUploadBackend): UPLOAD_DIR = "uploads" def setup(self, filename): self._path = os.path.join( settings.MEDIA_ROOT, self.UPLOAD_DIR, filename) try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._dest = BufferedWriter(FileIO(self._path, "wb")) def upload_chunk(self, chunk): self._dest.write(chunk) def upload_complete(self, request, filename): path = settings.MEDIA_URL + self.UPLOAD_DIR + "/" + filename return {"path": path}
def write_to_stream(self, writer_stream: BufferedWriter): if self._is_operation_result(): with open(self._file_path, "rb", -1) as file: while True: buffer = file.read(io.DEFAULT_BUFFER_SIZE) writer_stream.write(buffer) if len(buffer) < io.DEFAULT_BUFFER_SIZE: break writer_stream.close() self._logger.info( "Writing file at {tmp_file_path} to writer stream".format( tmp_file_path=self._file_path)) else: self._logger.error( "Invalid use of write_to_stream(). Method invoked on FileRef instance which does not point to an operation " "result") raise AttributeError( "Method write_to_stream() only allowed on operation results")
def main( image1: BufferedReader, image2: BufferedReader, url: str, output: BufferedWriter, verbose: bool, display: bool, ): """ Thrash detector API client. """ files = {"image1": image1.read(), "image2": image2.read()} url_path = url.rstrip("/") + "/api/thrash/json" if verbose: click.echo(f"Querying thrash detector API on {url_path}") click.echo(f"Image 1: {image1.name}") click.echo(f"Image 2: {image2.name}") response = requests.post(url_path, files=files) if response.status_code != 200: click.echo( message=f"API request failed with code {response.status_code}.", err=True) click.echo(message=f"{response.json()['error']}", err=True) sys.exit(response.status_code) data = response.json() image = data["encoded_img"]["bytes"] bounds = data["bounds"] click.echo(f"Bounding box: {bounds}") if verbose: click.echo(f"Save bounding box image to {output.name}") image = base64.decodebytes(image.encode("ascii")) output.write(image) if display: res = cv2.imdecode(np.frombuffer(image, np.uint8), -1) cv2.imshow("bounding box", res) click.echo("Press any key to close the image") cv2.waitKey(0)
def download_file(self, path: str, stream: io.BufferedWriter) -> Response: """ Calls the gRPC DownloadFile function :param path: (str) The location of the file which needs to be downloaded :param stream: (io.BufferedWriter) A BufferedWriter to write to :return: (Response) The response object containing values corresponding to the request """ meta = self._make_meta() download_file_request = server_pb2.DownloadFileRequest(path=path, meta=meta) for response in self.stub.DownloadFile(download_file_request): if response.status == 200: stream.write(response.payload) else: return Response( server_pb2.Response(status=response.status, error=response.error)) return Response(server_pb2.Response(status=200))
def doExport(self, file: BufferedWriter) -> None: cardIds = self.cardIds() data = [] for id, flds, tags in self.col.db.execute(""" select guid, flds, tags from notes where id in (select nid from cards where cards.id in %s)""" % ids2str(cardIds)): row = [] # note id if self.includeID: row.append(str(id)) # fields row.extend([self.processText(f) for f in splitFields(flds)]) # tags if self.includeTags: row.append(tags.strip()) data.append("\t".join(row)) self.count = len(data) out = "\n".join(data) file.write(out.encode("utf-8"))
def download_document(outfile): headers = {'user-agent': I_USERAGENT} conn = http.client.HTTPSConnection(I_HOST) try: conn.request('GET', I_TEX_URL, headers=headers) rsp = conn.getresponse() CHUNK_SIZE = 1024 * 10 reader = BufferedReader(rsp) with open(outfile, 'wb') as outf: writer = BufferedWriter(outf) while True: data = reader.read(CHUNK_SIZE) if len(data) > 0: writer.write(data) else: break finally: conn.close()
def _write_data(outfile: io.BufferedWriter, data): """ Writes the 86p data section to the outfile Args: outfile - Outfile to write to. data - Data buffer """ checksum = 0 # Header struct format: # H -> Always 12 (0x0c00) # H -> length of the variable data # B -> Data Type ID # B -> Length of the variable name # 8s -> Variable name, padding w/ space characters # H -> length of the variable data (again) vname = bytes('MiniRPG', 'ascii') # We add 4 to the data len to account for the program header. # So many headers. dlen = len(data) + 4 entry_header = pack('H H B B 8s H', 12, dlen, 18, 7, vname, dlen) outfile.write(entry_header) checksum += reduce(lambda x, y: x + y, entry_header) # Write the ASM program into the fileformat prog_header = pack('H B B', len(data), 0x8e, 0x28) outfile.write(prog_header) checksum += reduce(lambda x, y: x + y, prog_header) outfile.write(bytes(data)) checksum += reduce(lambda x, y: x + y, data) return checksum
class LocalUploadBackend(AbstractUploadBackend): UPLOAD_DIR = 'tmp' def update_filename(self, request, filename): name, ext = os.path.splitext(filename) return slughifi(name) + ext def setup(self, filename): self._path = os.path.join(self.UPLOAD_DIR, filename) self.path = default_storage.save(self._path, ContentFile('')) self._abs_path = default_storage.path(self.path) self._dest = BufferedWriter(FileIO(self._abs_path, "w")) def upload_chunk(self, chunk): self._dest.write(chunk) def upload_complete(self, request, filename): self._dest.close() context = {'thumbnail_path': self._path, 'file_name': filename, } thumbnail = render_to_string('ajaxupload/includes/thumbnail.html', context) return {"path": self._path, 'thumbnail': thumbnail}
def add_garbage(file: BufferedWriter, total_bytes: int) -> None: one_mb_of_data = b"*" * 1024 * 1024 written = 0 file.write(b"\n<!-- ") while written < total_bytes: file.write(one_mb_of_data) written += len(one_mb_of_data) file.write(b" -->\n")
def _dump(self, data: THM, file: BufferedWriter, *args, **kwargs): size_x, size_y = (0, 0) if data.lod_images: size_x, size_y = data.lod_images[0].size file.write(pack( '<13I', data.unknown_01, data.created_at.year, data.created_at.month, data.created_at.day, data.created_at.hour, data.created_at.minute, data.created_at.second, data.created_at.microsecond, data.unknown_01, data.unknown_02, data.unknown_03, size_x, size_y )) for lod_image in data.lod_images: file.write(lod_image.tobytes())
def dump(self, writer: io.BufferedWriter, *args, strict: bool = STRICT, encoding: str = ENCODING, **kwargs) -> None: _TABLE, _FRAME, _FILESPEC, _END = self._TABLE, self._FRAME, self._FILESPEC, self._END if strict and not len(self): raise ValueError('Strict: Cannot write CGList with 0 tables') for i, table in enumerate(self): frameSizes = [(_FRAME.size + len(f) * _FILESPEC.size) for f in table] tableSize = _TABLE.size + sum(frameSizes) tableNext = tableSize if (i + 1 < len(self)) else 0 writer.write(_TABLE.pack(tableNext, table.number, len(table))) for j, frame in enumerate(table): # frameNext is non-zero until the final frame of the final table frameNext = frameSizes[j] if (tableNext or j + 1 < len(table)) else 0 if tableNext and j + 1 == len( table): # Account for distance across table headers. ii = i + 1 while ii < len(self) and not len(self[ii]): ii += 1 if ii == len(self): frameNext = 0 # No more frames after this, it's all empty tables from here. else: frameNext += _TABLE.size * ( ii - i ) # Number of empty tables and final non-empty table. writer.write( _FRAME.pack(frameNext, frame.name.encode(encoding), len(frame))) for k, file in enumerate(frame): writer.write(_FILESPEC.pack(file.encode(encoding))) # File always ends with dummy empty table header. # No idea why, when we use find-by-offset for tables... writer.write(_TABLE.pack(*_END))
def stream_noop(instr: io.BufferedReader, outstr: io.BufferedWriter, chunk_size=None): orig_data_size: int = 0 dest_data_size: int = 0 data = instr.read(chunk_size) while data: orig_data_size += len(data) # no transformation dest_data_size += len(data) written = outstr.write(data) outstr.flush() # detect early if we couldnt write everything assert written == len(data) # read more data in case it is now available data = instr.read(chunk_size) return orig_data_size, dest_data_size
def _write_header(outfile: io.BufferedWriter, datasize: int): """ Writes the 86p header to the outfile. Args: outfile - Outfile to write to. datasize - Size of the ASM program. checksum - ASM program checksum """ # Pack the signature, **TI86** plus 3 bytes of magic number. out = pack('8s B B B', bytes('**TI86**', 'ascii'), 0x1a, 0x0a, 0x00) outfile.write(out) # Pack a comment (up to 42 bytes) out = pack('42s', bytes('Packed by pack86.py', 'ascii')) outfile.write(out) # Length of program + 16 for the data header + 4 for the program header. outfile.write(pack('H', datasize + 16 + 4))
def _extract(self, entry: ArchiveEntry, dstfile: io.BufferedWriter, arcfile: io.BufferedReader, *, buffersize: int = 0x4000, **kwargs): if self._cipher is None or entry.length == 0: return super()._extract(entry, dstfile, arcfile) # arcfile.seek(entry.offset) # cipher = self._cipher length = entry.length # tmp_len = (buffersize & ~0x7) if buffersize else 0x4000 if tmp_len <= length: tmp_buf = bytearray(length) read_len = arcfile.readinto(tmp_buf) if read_len < length: raise ValueError( 'Unexpected end of entry data {0.name!r}'.format(entry)) self._cipher.decrypt_buffer(tmp_buf, 0, length & ~0x7) dstfile.write(tmp_buf) else: tmp_buf = bytearray(tmp_len) read_len = arcfile.readinto(tmp_buf) while read_len == tmp_len and length >= tmp_len: self._cipher.decrypt_buffer(tmp_buf, 0, min(read_len, length) & ~0x7) dstfile.write(tmp_buf) length -= read_len read_len = arcfile.readinto(tmp_buf) if read_len < length: raise ValueError( 'Unexpected end of entry data {0.name!r}'.format(entry)) if length > 0: self._cipher.decrypt_buffer(tmp_buf, 0, length & ~0x7) dstfile.write(tmp_buf[:length])
def _write_encoded_file_data(cls, file: BufferedReader, archive_file: BufferedWriter, encoding_dictionary): bit_buffer = bitarray() while True: byte = file.read(1) if not byte: break bit_buffer.extend(encoding_dictionary[byte[0]]) while len(bit_buffer) >= cls.data_unit_size * cls.byte: archive_file.write(bytes([cls.data_unit_size])) archive_file.write(bit_buffer[:cls.data_unit_size * cls.byte:].tobytes()) bit_buffer = bit_buffer[cls.data_unit_size * cls.byte::] if len(bit_buffer) != 0: empty_bits_count = cls.byte - len(bit_buffer) % cls.byte if empty_bits_count == cls.byte: empty_bits_count = 0 byte_buffer = bytearray(bit_buffer.tobytes()) byte_buffer.append(empty_bits_count) archive_file.write(cls._compose_data(bytes(byte_buffer)))
class LocalUploadBackend(AbstractUploadBackend): #UPLOAD_DIR = "uploads" # The below key must be synchronized with the implementing project # Used to store an array of unclaimed file_pks in the django session # So they can be claimed later when the anon user authenticates #SESSION_UNCLAIMED_FILES_KEY = KarmaSettings.SESSION_UNCLAIMED_FILES_KEY # When a file is uploaded anonymously, # What username should we assign ownership to? # This is important because File.save # behavior will not set awarded_karma to True # until an owner is assigned who has username != this #DEFAULT_UPLOADER_USERNAME = KarmaSettings.DEFAULT_UPLOADER_USERNAME def setup(self, filename): self._path = os.path.join( settings.MEDIA_ROOT, filename) try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._dest = BufferedWriter(FileIO(self._path, "w")) def upload_chunk(self, chunk): self._dest.write(chunk) def upload(self, uploaded, filename, raw_data): """ :raw_data: is 0/1 """ try: if raw_data: # File was uploaded via ajax, and is streaming in. chunk = uploaded.read(self.BUFFER_SIZE) while len(chunk) > 0: self.upload_chunk(chunk) chunk = uploaded.read(self.BUFFER_SIZE) else: # File was uploaded via a POST, and is here. for chunk in uploaded.chunks(): self.upload_chunk(chunk) return True except: # things went badly. return False def upload_complete(self, request, filename, upload): path = settings.MEDIA_URL + "/" + filename self._dest.close() self._dir = settings.MEDIA_ROOT # Avoid File.objects.create, as this will try to make # Another file copy at FileField's 'upload_to' dir print "creating note" note = Note() note.name = filename note.note_file = os.path.join(self._dir, filename) note.course_id = request.GET['course_id'] note.draft = True # Pending approval from user print "saving note" note.save() # FIXME: Make get or create print "setting up session vars" #import ipdb; ipdb.set_trace() if 'uploaded_files' in request.session: request.session['uploaded_files'].append(note.pk) else: request.session['uploaded_files'] = [note.pk] # Asynchronously process document with Google Documents API print "upload_complete, firing task" tasks.process_document.delay(note) return {'note_url': note.get_absolute_url()} def update_filename(self, request, filename): """ Returns a new name for the file being uploaded. Ensure file with name doesn't exist, and if it does, create a unique filename to avoid overwriting """ self._dir = settings.MEDIA_ROOT unique_filename = False filename_suffix = 0 # Check if file at filename exists if os.path.isfile(os.path.join(self._dir, filename)): while not unique_filename: try: if filename_suffix == 0: open(os.path.join(self._dir, filename)) else: filename_no_extension, extension = os.path.splitext(filename) #print "filename all ready exists. Trying " + filename_no_extension + str(filename_suffix) + extension open(os.path.join(self._dir, filename_no_extension + str(filename_suffix) + extension)) filename_suffix += 1 except IOError: unique_filename = True if filename_suffix == 0: #print "using filename: " + os.path.join(self._dir, filename) return filename else: #print "using filename: " + filename_no_extension + str(filename_suffix) + extension return filename_no_extension + str(filename_suffix) + extension
def write(self, b): ret = BufferedWriter.write(self, b) self.flush() return ret
class MyBaseUploadBackend(AbstractUploadBackend): def __init__(self, dirname, **kwargs): super(MyBaseUploadBackend, self).__init__(**kwargs) self.report_id = None def set_report_id(self, report_id): self.report_id = report_id try_number = 1 while True: try: self.quast_session = QuastSession.objects.get(report_id=self.report_id) return True except QuastSession.DoesNotExist: logger.error('No quast session with report_id=%s' % self.report_id) return False except OperationalError: logger.error(traceback.format_exc()) try_number += 1 logger.error('Retrying. Try number ' + str(try_number)) def setup(self, filename): dirpath = self.quast_session.get_contigs_dirpath() logger.info('filename is %s' % filename) logger.info('contigs dirpath is %s' % dirpath) if not os.path.exists(dirpath): logger.error("contigs directory doesn't exist") return False fpath = os.path.join(dirpath, filename) self._path = fpath self._dest = BufferedWriter(FileIO(self._path, 'w')) return True def upload_chunk(self, chunk): self._dest.write(chunk) def upload_complete(self, request, filename): self._dest.close() file_index = "%x" % random.getrandbits(128) c_fn = ContigsFile(fname=filename, file_index=file_index) c_fn.save() qc = QuastSession_ContigsFile(contigs_file=c_fn, quast_session=self.quast_session) qc.save() logger.info('%s' % filename) return { 'file_index': file_index, } def update_filename(self, request, filename): dirpath = self.quast_session.get_contigs_dirpath() logger.info('contigs dirpath is %s' % dirpath) fpath = os.path.join(dirpath, filename) logger.info('file path is %s' % fpath) i = 2 base_fpath = fpath base_filename = filename while os.path.isfile(fpath): fpath = str(base_fpath) + '__' + str(i) filename = str(base_filename) + '__' + str(i) i += 1 return filename def remove(self, request): if 'fileIndex' not in request.GET: logger.error('Request.GET must contain "fileIndex"') return False, 'Request.GET must contain "fileIndex"' file_index = request.GET['fileIndex'] try: contigs_file = self.quast_session.contigs_files.get(file_index=file_index) except ContigsFile.DoesNotExist: logger.error('No file with such index %d in this quast_session' % file_index) return False, 'No file with such index' success, msg = self.__remove(contigs_file) return success, msg # if contigs_file.user_session != self.user_session: # logger.error('This file (%s) belongs to session %s, this session is %s' # % (fname, str(contigs_file.user_session ), str(self.user_session.session_key))) # return False, 'This file does not belong to this session' def __remove(self, contigs_file): fname = contigs_file.fname contigs_fpath = os.path.join(self.quast_session.get_contigs_dirpath(), fname) if os.path.isfile(contigs_fpath): try: os.remove(contigs_fpath) except IOError as e: logger.error('IOError when removing "%s", fileIndex=%d": %s' % (fname, file_index, e.message)) return False, 'Cannot remove file' try: contigs_file.delete() except DatabaseError as e: logger.warn('DatabaseError when removing "%s", fileIndex=%d: %s' % (fname, file_index, e.message)) return False, 'Data base error when removing file' except Exception as e: logger.error('Exception when removing "%s", fileIndex=%d: %s' % (fname, file_index, e.message)) return False, 'Data base exception when removing file' return True, '' def remove_all(self, request): # if 'fileNames' not in request.GET: # logger.error('remove_all: Request.GET must contain "fileNames"') # return False # # file_names_one_string = request.GET['fileNames'] # file_names = file_names_one_string.split('\n')[:-1] # this_user_contigs_files = ContigsFile.objects.filter(user_session=self.user_session) logger.info('uploader_backend.remove_all') for c_f in self.quast_session.contigs_files.all(): success, msg = self.__remove(c_f) return True def get_uploads(self, request): contigs_files = self.quast_session.contigs_files.all() return [{"fileName": c_f.fname, "fileIndex": c_f.file_index, "file_index": c_f.file_index, # "fileSize": c_f.file_size if c_f.file_size else None, } for c_f in contigs_files]
class UploadStorage(object): BUFFER_SIZE = 10485760 # 10MB def __init__(self, **kwargs): self.__dict__.update(kwargs) self.size = None def set_size(self, width, height): self.size = int(width), int(height) #logger.debug(self.size) def setup(self, filename, upload_to): """ Creates the filename on the system, along with the required folder structure. """ self.filename = filename self.upload_to = upload_to #logger.debug('File: '+self.filename) self._path = self.update_filename() #logger.debug('Dir: '+self._dir) #logger.debug('Path: '+self._path) #logger.debug(os.path.realpath(os.path.dirname(self._path))) try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._dest = BufferedWriter(FileIO(self._path, "w")) def upload_chunk(self, chunk, *args, **kwargs): self._dest.write(chunk) def upload_complete(self): path = settings.MEDIA_URL + "/" + self.upload_to + "/" + self.filename self._dest.close() return {"path": path} def update_filename(self): """ Returns a new name for the file being uploaded. Ensure file with name doesn't exist, and if it does, create a unique filename to avoid overwriting """ unique_filename = False filename_suffix = 0 # remove trailing current folder if given if self.upload_to[:2] == './': self.upload_to = self.upload_to[2:] # format upload path with date formats self.upload_to = time.strftime(self.upload_to) self._dir = os.path.join(settings.MEDIA_ROOT, self.upload_to) #logger.debug('Upload to: '+self._dir) # Check if file at filename exists) if os.path.isfile(os.path.join(self._dir, self.filename)): #logger.debug('this file already exists') while not unique_filename: try: if filename_suffix == 0: open(os.path.join(self._dir, self.filename)) else: filename_no_extension, extension = os.path.splitext(self.filename) open(os.path.join(self._dir, "{}_{}{}".format(filename_no_extension, str(filename_suffix), extension))) filename_suffix += 1 except IOError: unique_filename = True if filename_suffix > 0: self.filename = "{}_{}{}".format(filename_no_extension, str(filename_suffix), extension) return os.path.join(self._dir, self.filename) def upload(self, uploaded, raw_data): try: if raw_data: # File was uploaded via ajax, and is streaming in. chunk = uploaded.read(self.BUFFER_SIZE) while len(chunk) > 0: self.upload_chunk(chunk) chunk = uploaded.read(self.BUFFER_SIZE) else: # File was uploaded via a POST, and is here. for chunk in uploaded.chunks(): self.upload_chunk(chunk) # make sure the file is closed self._dest.close() # file has been uploaded self.filename = os.path.join(settings.MEDIA_URL, self.upload_to, self.filename) image = Image.open(self._path) #logger.debug("{} {} {}".format(image.format, image.size, image.mode)) # resize image if self.size: #logger.debug(self.size) image = image.convert('RGBA') # the image is resized using the minimum dimension width, height = self.size if image.size[0] < image.size[1]: # the height is bigger than the width # we set the maximum height to the original height, # so that the image fits the width height = image.size[1] elif image.size[0] > image.size[1]: # the width is bigger than the height # we set the maximum width to the original width, # so that the image fits the height width = image.size[0] else: # we have a square pass image.thumbnail((width, height), Image.ANTIALIAS) # if the image is not a square, we crop the exceeding width/length as required, # to fit the square if image.size[0] != image.size[1]: # we crop in the middle of the image if self.size[0] == image.size[0]: # the width fits the container, center the height x0 = 0 y0 = (image.size[1] / 2) - (self.size[1] / 2) x1 = self.size[0] y1 = y0 + self.size[1] else: # center the width x0 = (image.size[0] / 2) - (self.size[0] / 2) y0 = 0 x1 = x0 + self.size[0] y1 = self.size[1] box = (x0, y0, x1, y1) region = image.crop(box) background = Image.new('RGBA', size = self.size, color = (255, 255, 255, 0)) background.paste(region, (0, 0)) #logger.debug("{} {} {}".format(background.format, background.size, background.mode)) background.save(self._path) else: image.save(self._path) return True except Exception as e: logger.error(e) return False
class LocalUploadBackend(AbstractUploadBackend): UPLOAD_DIR = "uploads" def setup(self, filename, *args, **kwargs): self._path = os.path.join( settings.MEDIA_ROOT, self.UPLOAD_DIR, filename) try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._dest = BufferedWriter(FileIO(self._path, "w")) def upload_chunk(self, chunk, *args, **kwargs): self._dest.write(chunk) def upload_complete(self, request, filename, *args, **kwargs): path = settings.MEDIA_URL + self.UPLOAD_DIR + "/" + filename self._dest.close() return {"path": path} def update_filename(self, request, filename, *args, **kwargs): """ Returns a new name for the file being uploaded. Ensure file with name doesn't exist, and if it does, create a unique filename to avoid overwriting """ self._dir = os.path.join( settings.MEDIA_ROOT, self.UPLOAD_DIR) unique_filename = False filename_suffix = 0 # Check if file at filename exists if os.path.isfile(os.path.join(self._dir, filename)): while not unique_filename: try: if filename_suffix == 0: open(os.path.join(self._dir, filename)) else: filename_no_extension, extension = os.path.splitext(filename) open(os.path.join(self._dir, filename_no_extension + str(filename_suffix) + extension)) filename_suffix += 1 except IOError: unique_filename = True if filename_suffix == 0: return filename else: return filename_no_extension + str(filename_suffix) + extension def resize_for_display(self, filename, width, height): upload_dir_path = os.path.join(settings.MEDIA_ROOT, self.UPLOAD_DIR) + "/" original_path = upload_dir_path + filename filename_no_extension, extension = os.path.splitext(filename) need_ratio = float(width) / float(height) im = Image.open(original_path) real_width, real_height = [float(x) for x in im.size] real_ratio = real_width / real_height if real_width > width or real_height > height: if real_ratio > need_ratio: displayed_width = width displayed_height = int(width / real_ratio) else: displayed_height = height displayed_width = int(height * real_ratio) resized_im = im.resize((displayed_width, displayed_height)) displayed_filename = '%s_displayed%s' % (filename_no_extension, extension) resized_im.save(upload_dir_path + displayed_filename) displayed_path = settings.MEDIA_URL + self.UPLOAD_DIR + "/" + displayed_filename else: displayed_path = settings.MEDIA_URL + self.UPLOAD_DIR + "/" + filename return {'displayed_path': displayed_path, 'true_size': im.size}
class BinaryWriter: """ Small utility class to write binary data. Also creates a "Memory Stream" if necessary """ def __init__(self, stream=None): if not stream: stream = BytesIO() self.writer = BufferedWriter(stream) self.written_count = 0 # region Writing # "All numbers are written as little endian." |> Source: https://core.telegram.org/mtproto def write_byte(self, value): """Writes a single byte value""" self.writer.write(pack('B', value)) self.written_count += 1 def write_int(self, value, signed=True): """Writes an integer value (4 bytes), which can or cannot be signed""" self.writer.write( int.to_bytes( value, length=4, byteorder='little', signed=signed)) self.written_count += 4 def write_long(self, value, signed=True): """Writes a long integer value (8 bytes), which can or cannot be signed""" self.writer.write( int.to_bytes( value, length=8, byteorder='little', signed=signed)) self.written_count += 8 def write_float(self, value): """Writes a floating point value (4 bytes)""" self.writer.write(pack('<f', value)) self.written_count += 4 def write_double(self, value): """Writes a floating point value (8 bytes)""" self.writer.write(pack('<d', value)) self.written_count += 8 def write_large_int(self, value, bits, signed=True): """Writes a n-bits long integer value""" self.writer.write( int.to_bytes( value, length=bits // 8, byteorder='little', signed=signed)) self.written_count += bits // 8 def write(self, data): """Writes the given bytes array""" self.writer.write(data) self.written_count += len(data) # endregion # region Telegram custom writing def tgwrite_bytes(self, data): """Write bytes by using Telegram guidelines""" if len(data) < 254: padding = (len(data) + 1) % 4 if padding != 0: padding = 4 - padding self.write(bytes([len(data)])) self.write(data) else: padding = len(data) % 4 if padding != 0: padding = 4 - padding self.write(bytes([254])) self.write(bytes([len(data) % 256])) self.write(bytes([(len(data) >> 8) % 256])) self.write(bytes([(len(data) >> 16) % 256])) self.write(data) self.write(bytes(padding)) def tgwrite_string(self, string): """Write a string by using Telegram guidelines""" self.tgwrite_bytes(string.encode('utf-8')) def tgwrite_bool(self, boolean): """Write a boolean value by using Telegram guidelines""" # boolTrue boolFalse self.write_int(0x997275b5 if boolean else 0xbc799737, signed=False) def tgwrite_date(self, datetime): """Converts a Python datetime object into Unix time (used by Telegram) and writes it""" value = 0 if datetime is None else int(datetime.timestamp()) self.write_int(value) def tgwrite_object(self, tlobject): """Writes a Telegram object""" tlobject.on_send(self) def tgwrite_vector(self, vector): """Writes a vector of Telegram objects""" self.write_int(0x1cb5c415, signed=False) # Vector's constructor ID self.write_int(len(vector)) for item in vector: self.tgwrite_object(item) # endregion def flush(self): """Flush the current stream to "update" changes""" self.writer.flush() def close(self): """Close the current stream""" self.writer.close() def get_bytes(self, flush=True): """Get the current bytes array content from the buffer, optionally flushing first""" if flush: self.writer.flush() return self.writer.raw.getvalue() def get_written_bytes_count(self): """Gets the count of bytes written in the buffer. This may NOT be equal to the stream length if one was provided when initializing the writer""" return self.written_count # with block def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close()
def write(data, wfile: io.BufferedWriter) -> None: wfile.write(json.dumps(data).encode() + b"\n") wfile.flush()
class AjaxUploader(object): BUFFER_SIZE = 10485760 # 10MB def __init__(self, filetype='file', upload_dir='files', size_limit=10485760): self._upload_dir = os.path.join(settings.MEDIA_ROOT, upload_dir, get_date_directory()) self._filetype = filetype if filetype == 'image': self._save_format = setting('IMAGE_UPLOAD_FORMAT', 'JPEG') else: self._save_format = None self._size_limit = size_limit def max_size(self): """ Checking file max size """ if int(self._destination.tell()) > self._size_limit: self._destination.close() os.remove(self._path) return True def setup(self, filename): ext = os.path.splitext(filename)[1] self._filename = md5(filename.encode('utf8')).hexdigest() + ext self._path = os.path.join(self._upload_dir, self._filename) # noinspection PyBroadException try: os.makedirs(os.path.realpath(os.path.dirname(self._path))) except: pass self._destination = BufferedWriter(FileIO(self._path, "w")) def handle_upload(self, request): is_raw = True if request.FILES: is_raw = False if len(request.FILES) == 1: upload = request.FILES.values()[0] else: return dict(success=False, error=_("Bad upload.")) filename = upload.name else: # the file is stored raw in the request upload = request # get file size try: filename = request.GET['qqfile'] except KeyError as aerr: return dict(success=False, error=_("Can't read file name")) self.setup(filename) # noinspection PyBroadException try: if is_raw: # File was uploaded via ajax, and is streaming in. chunk = upload.read(self.BUFFER_SIZE) while len(chunk) > 0: self._destination.write(chunk) if self.max_size(): raise IOError chunk = upload.read(self.BUFFER_SIZE) else: # File was uploaded via a POST, and is here. for chunk in upload.chunks(): self._destination.write(chunk) if self.max_size(): raise IOError except: # things went badly. return dict(success=False, error=_("Upload error")) self._destination.close() if self._filetype == 'image': # noinspection PyBroadException try: i = Image.open(self._path) except: os.remove(self._path) return dict(success=False, error=_("File is not image format")) f_name, f_ext = os.path.splitext(self._filename) f_without_ext = os.path.splitext(self._path)[0] new_path = ".".join([f_without_ext, self._save_format.lower()]) if setting('IMAGE_STORE_ORIGINAL', False): # TODO need change the extension orig_path = ".".join([f_without_ext + '_orig', self._save_format.lower()]) shutil.copy2(self._path, orig_path) i.thumbnail((1200, 1200), Image.ANTIALIAS) # noinspection PyBroadException try: if self._path == new_path: i.save(self._path, self._save_format) else: i.save(new_path, self._save_format) os.remove(self._path) self._path = new_path except: # noinspection PyBroadException try: os.remove(self._path) os.remove(new_path) except: pass return dict(success=False, error=_("Error saving image")) self._filename = ".".join([f_name, self._save_format.lower()]) return dict(success=True, fullpath=self._path, path=os.path.relpath(self._path, '/' + settings.MEDIA_ROOT), old_filename=filename, filename=self._filename)
def convert_with_google_drive(note): """ Upload a local note and download HTML using Google Drive :note: a File model instance # FIXME """ # TODO: set the permission of the file to permissive so we can use the # gdrive_url to serve files directly to users # Get file_type and encoding of uploaded file # i.e: file_type = 'text/plain', encoding = None (file_type, encoding) = mimetypes.guess_type(note.note_file.path) if file_type != None: media = MediaFileUpload(note.note_file.path, mimetype=file_type, chunksize=1024*1024, resumable=True) else: media = MediaFileUpload(note.note_file.path, chunksize=1024*1024, resumable=True) auth = DriveAuth.objects.filter(email=GOOGLE_USER).all()[0] creds = auth.transform_to_cred() creds, auth = check_and_refresh(creds, auth) service, http = build_api_service(creds) # get the file extension filename, extension = os.path.splitext(note.note_file.path) file_dict = upload_to_gdrive(service, media, filename, extension) content_dict = download_from_gdrive(file_dict, http, extension) # Get a new copy of the file from the database with the new metadata from filemeta new_note = Note.objects.get(id=note.id) if extension.lower() == '.pdf': new_note.file_type = 'pdf' elif extension.lower() in ['.ppt', '.pptx']: new_note.file_type = 'ppt' now = datetime.datetime.utcnow() # create a folder path to store the ppt > pdf file with year and month folders nonce_path = '/ppt_pdf/%s/%s/' % (now.year, now.month) _path = filename + '.pdf' try: # If those folders don't exist, create them os.makedirs(os.path.realpath(os.path.dirname(_path))) except: print "we failed to create those directories" _writer = BufferedWriter(FileIO(_path, "w")) _writer.write(content_dict['pdf']) _writer.close() new_note.pdf_file = _path else: # PPT files do not have this export ability new_note.gdrive_url = file_dict[u'exportLinks']['application/vnd.oasis.opendocument.text'] new_note.html = content_dict['html'] new_note.text = content_dict['text'] # before we save new html, sanitize a tags in note.html #new_note.sanitize_html(save=False) #FIXME: ^^^ disabled until we can get html out of an Etree html element # Finally, save whatever data we got back from google new_note.save()
def write(self, strng): BufferedWriter.write(self,strng.encode('utf-8'))