def rejpg_argparse(args): patterns = pipeable.input_many(args.patterns, skip_blank=True, strip=True) files = spinal.walk(recurse=args.recurse, glob_filenames=patterns) files = [f.absolute_path for f in files] bytes_saved = 0 remaining_size = 0 for filename in files: print(''.join(c for c in filename if c in string.printable)) bytesio = io.BytesIO() i = PIL.Image.open(filename) i = imagetools.rotate_by_exif(i) i.save(bytesio, format='jpeg', quality=args.quality) bytesio.seek(0) new_bytes = bytesio.read() old_size = os.path.getsize(filename) new_size = len(new_bytes) remaining_size += new_size if new_size < old_size: bytes_saved += (old_size - new_size) f = open(filename, 'wb') f.write(new_bytes) f.close() print('Saved', bytestring.bytestring(bytes_saved)) print('Remaining are', bytestring.bytestring(remaining_size))
def reserve_disk_space_argparse(args): try: status = reserve_disk_space(reserve=bytestring.parsebytes( args.reserve), drive=args.drive) free = bytestring.bytestring(status.free) reserve = bytestring.bytestring(status.reserve) log.info('There is %s available out of %s.', free, reserve) return 0 except NotEnoughSpace as exc: free = bytestring.bytestring(exc.free) reserve = bytestring.bytestring(exc.reserve) log.fatal('Only %s available out of %s.', free, reserve) return 1
def kbps(time=None, size=None, kbps=None): if [time, size, kbps].count(None) != 1: raise ValueError('Incorrect number of unknowns') if size is None: seconds = hms_s(time) kibs = int(kbps) / 8 size = kibs * 1024 size *= seconds out = bytestring.bytestring(size) return out if time is None: size = bytestring.parsebytes(size) kilobits = size / 128 time = kilobits / int(kbps) return s_hms(time) if kbps is None: seconds = hms_s(time) size = bytestring.parsebytes(size) kibs = size / 1024 kilobits = kibs * 8 kbps = kilobits / seconds kbps = '%d kbps' % int(round(kbps)) return kbps
def step(self, bytes_downloaded): if g.start is None: g.start = time.time() percent = 100 * bytes_downloaded / self.bytes_total percent = '%07.3f%%:' % percent chunk = bytes_downloaded - g.total g.total = bytes_downloaded METER.digest(chunk) METER_2.digest(chunk) now = round(time.time(), 1) if now > g.last or (bytes_downloaded >= self.bytes_total): g.last = now percent = percent.rjust(9, ' ') rate = bytestring.bytestring(METER.report()[2]).rjust(15, ' ') rate2 = bytestring.bytestring(METER_2.report()[2]).rjust(15, ' ') elapsed = str(round(now - g.start, 1)).rjust(10, ' ') print(percent, rate, rate2, elapsed, end='\r', flush=True)
def __init__(self, total_bytes): self.limiter = ratelimiter.Ratelimiter(allowance=8, mode='reject') self.limiter.balance = 1 self.total_bytes = max(1, total_bytes) self.divisor = bytestring.get_appropriate_divisor(total_bytes) self.total_format = bytestring.bytestring(total_bytes, force_unit=self.divisor) self.downloaded_format = '{:>%d}' % len(self.total_format) self.blank_char = ' ' self.solid_char = '█'
def table_row(self, display_name=None, shaded=False): form = '<tr style="background-color:#{bg}"><td style="width:90%">{anchor}</td><td>{size}</td></tr>' size = self.size if size is None: size = '' else: size = bytestring.bytestring(size) bg = 'ddd' if shaded else 'fff'; row = form.format( bg=bg, anchor=self.anchor(display_name=display_name), size=size, ) return row
def table_row(self, display_name=None, shaded=False): form = '<tr style="background-color:#{bg}"><td style="width:90%">{anchor}</td><td>{size}</td></tr>' size = self.size if size is None: size = '' else: size = bytestring.bytestring(size) bg = 'ddd' if shaded else 'fff' row = form.format( bg=bg, anchor=self.anchor(display_name=display_name), size=size, ) return row
def encrypt_file(aes, input_handle, output_handle): last_byte = 0 done = False bytes_read = 0 while not done: chunk = input_handle.read(BLOCK_SIZE) if len(chunk) > 0: last_byte = chunk[-1] if len(chunk) < BLOCK_SIZE: pad_byte = (last_byte + 1) % 256 pad_byte = bytes([pad_byte]) chunk += pad_byte * (BLOCK_SIZE - len(chunk)) done = True bytes_read += len(chunk) if bytes_read % bytestring.MIBIBYTE == 0: print(bytestring.bytestring(bytes_read)) chunk = aes.encrypt(chunk) output_handle.write(chunk)
def decrypt_file(aes, input_handle, output_handle): current_pos = input_handle.tell() input_size = input_handle.seek(0, SEEK_END) - current_pos input_handle.seek(current_pos) bytes_read = 0 while True: chunk = input_handle.read(BLOCK_SIZE) if len(chunk) == 0: break bytes_read += len(chunk) chunk = aes.decrypt(chunk) if bytes_read == input_size: last_byte = chunk[-1] while chunk and chunk[-1] == last_byte: chunk = chunk[:-1] if bytes_read % bytestring.MIBIBYTE == 0: print(bytestring.bytestring(bytes_read)) output_handle.write(chunk)
def decrypt_file(aes, input_handle, output_handle): current_pos = input_handle.tell() input_size = input_handle.seek(0, SEEK_END) - current_pos input_handle.seek(current_pos) bytes_read = 0 while True: chunk = input_handle.read(BLOCK_SIZE) if len(chunk) == 0: break bytes_read += len(chunk) chunk = aes.decrypt(chunk) if bytes_read == input_size: last_byte = chunk[-1] while chunk and chunk[-1] == last_byte: chunk = chunk[:-1] if bytes_read % bytestring.MIBIBYTE == 0: print(bytestring.bytestring(bytes_read)) output_handle.write(chunk)
def encrypt_file(aes, input_handle, output_handle): last_byte = 0 done = False bytes_read = 0 while not done: chunk = input_handle.read(BLOCK_SIZE) if len(chunk) > 0: last_byte = chunk[-1] if len(chunk) < BLOCK_SIZE: pad_byte = (last_byte + 1) % 256 pad_byte = bytes([pad_byte]) chunk += pad_byte * (BLOCK_SIZE - len(chunk)) done = True bytes_read += len(chunk) if bytes_read % bytestring.MIBIBYTE == 0: print(bytestring.bytestring(bytes_read)) chunk = aes.encrypt(chunk) output_handle.write(chunk)
def table_row(path, display_name=None, shaded=False): form = ''' <tr style="background-color:#{bg}"> <td class="column_name">{anchor}</td> <td class="column_size">{size}</td></tr> '''.replace('\n', ' ') if path.is_file: size = bytestring.bytestring(path.size) else: size = '' bg = 'ddd' if shaded else 'fff' anchor = atag(path, display_name=display_name) row = form.format( bg=bg, anchor=anchor, size=size, ) return row
def step(self, bytes_downloaded): #print(self.limiter.balance) percent = bytes_downloaded / self.total_bytes percent = min(1, percent) if self.limiter.limit(1) is False and percent < 1: return downloaded_string = bytestring.bytestring(bytes_downloaded, force_unit=self.divisor) downloaded_string = self.downloaded_format.format(downloaded_string) block_count = 50 solid_blocks = self.solid_char * int(block_count * percent) statusbar = solid_blocks.ljust(block_count, self.blank_char) statusbar = self.solid_char + statusbar + self.solid_char end = '\n' if percent == 1 else '' message = '\r{bytes_downloaded} {statusbar} {total_bytes}' message = message.format( bytes_downloaded=downloaded_string, total_bytes=self.total_format, statusbar=statusbar, ) print(message, end=end, flush=True)
def list_basenames(databasename, output_filename=None): ''' Print the Enabled entries in order of the file basenames. This makes it easier to find interesting titles without worrying about what directory they're in. ''' sql = sqlite3.connect(databasename) cur = sql.cursor() cur.execute('SELECT * FROM urls WHERE do_download == 1') items = cur.fetchall() longest = max(items, key=lambda x: len(x[SQL_BASENAME])) longest = len(longest[SQL_BASENAME]) items.sort(key=lambda x: x[SQL_BASENAME].lower()) if output_filename is not None: output_file = open(output_filename, 'w', encoding='utf-8') else: output_file = None form = '{basename:<%ds} : {url} : {size}' % longest for item in items: size = item[SQL_CONTENT_LENGTH] if size is None: size = '' else: size = bytestring.bytestring(size) line = form.format( basename=item[SQL_BASENAME], url=item[SQL_URL], size=size, ) write(line, output_file) if output_file: output_file.close()
def list_basenames(databasename, output_filename=None): ''' Print the Enabled entries in order of the file basenames. This makes it easier to find interesting titles without worrying about what directory they're in. ''' sql = sqlite3.connect(databasename) cur = sql.cursor() cur.execute('SELECT * FROM urls WHERE do_download == 1') items = cur.fetchall() longest = max(items, key=lambda x: len(x[SQL_BASENAME])) longest = len(longest[SQL_BASENAME]) items.sort(key=lambda x: x[SQL_BASENAME].lower()) if output_filename is not None: output_file = open(output_filename, 'w', encoding='utf-8') else: output_file = None form = '{basename:<%ds} : {url} : {size}' % longest for item in items: size = item[SQL_CONTENT_LENGTH] if size is None: size = '' else: size = bytestring.bytestring(size) line = form.format( basename=item[SQL_BASENAME], url=item[SQL_URL], size=size, ) write(line, output_file) if output_file: output_file.close()
def recursive_print_node(node, depth=0, use_html=False, header=None, footer=None): ''' Given a tree node (presumably the root), print it and all of its children. use_html: Generate a neat HTML page instead of plain text. header: This text goes at the top of the file, or just below the <body> tag. footer: This text goes at the end of the file, or just above the </body> tag. ''' if depth == 0: if use_html: yield '<!DOCTYPE html>\n<html>' yield HTML_TREE_HEAD yield '<body>' if header is not None: yield header size = node.size if size is None: size = '???' else: size = bytestring.bytestring(size) if use_html: css_class = 'directory_even' if depth % 2 == 0 else 'directory_odd' if node.item_type == 'directory': directory_url = node.path directory_anchor = '<a href="{url}">→</a>' if directory_url else '' directory_anchor = directory_anchor.format(url=directory_url) line = HTML_FORMAT_DIRECTORY.format( css=css_class, directory_anchor=directory_anchor, name=node.display_name, size=size, ) else: line = HTML_FORMAT_FILE.format( name=node.display_name, size=size, url=node.path, ) else: line = '{space}{bar}{name} : ({size})' line = line.format( space='| ' * (depth-1), bar='|---' if depth > 0 else '', name=node.display_name, size=size ) yield line # Sort by type (directories first) then subsort by lowercase path customsort = lambda node: ( node.item_type == 'file', node.path.lower(), ) for child in node.list_children(sort=customsort): yield from recursive_print_node(child, depth=depth+1, use_html=use_html) if node.item_type == 'directory': if use_html: # Close the directory div yield '</div>' else: # This helps put some space between sibling directories yield '| ' * (depth) if depth == 0: if footer is not None: yield footer if use_html: yield '</body>\n</html>'
walker = spinal.walk_generator() files = list(walker) files = [f.absolute_path for f in files] else: files = os.listdir() files = [f for f in files if os.path.isfile(f)] files = [f for f in files if any(ext in f.lower() for ext in ['.jpg', '.jpeg'])] bytes_saved = 0 remaining_size = 0 for filename in files: print(''.join(c for c in filename if c in string.printable)) bytesio = io.BytesIO() i = PIL.Image.open(filename) i.save(bytesio, format='jpeg', quality=80) bytesio.seek(0) new_bytes = bytesio.read() old_size = os.path.getsize(filename) new_size = len(new_bytes) remaining_size += new_size if new_size < old_size: bytes_saved += (old_size - new_size) f = open(filename, 'wb') f.write(new_bytes) f.close() print('Saved', bytestring.bytestring(bytes_saved)) print('Remaining are', bytestring.bytestring(remaining_size))
def measure(databasename, fullscan=False, new_only=False, threads=4): ''' Given a database, print the sum of all Content-Lengths. URLs will be HEAD requested if: `new_only` is True and the file has no stored content length, or `fullscan` is True and `new_only` is False ''' if isinstance(fullscan, str): fullscan = bool(fullscan) sql = sqlite3.connect(databasename) cur = sql.cursor() if new_only: cur.execute('SELECT * FROM urls WHERE do_download == 1 AND content_length IS NULL') else: cur.execute('SELECT * FROM urls WHERE do_download == 1') items = cur.fetchall() filecount = len(items) totalsize = 0 unmeasured_file_count = 0 if threads is None: threads = 1 thread_queue = threadqueue.ThreadQueue(threads) try: for fetch in items: size = fetch[SQL_CONTENT_LENGTH] if fullscan or new_only: url = fetch[SQL_URL] thread_queue.add(do_head, url, raise_for_status=False) elif size is None: # Unmeasured and no intention to measure. unmeasured_file_count += 1 else: totalsize += size for head in thread_queue.run(): fetch = smart_insert(sql, cur, head=head, commit=False) size = fetch[SQL_CONTENT_LENGTH] if size is None: write('"%s" is not revealing Content-Length' % url) size = 0 totalsize += size except (Exception, KeyboardInterrupt): sql.commit() raise sql.commit() size_string = bytestring.bytestring(totalsize) totalsize_string = '{size_short} ({size_exact:,} bytes) in {filecount:,} files' totalsize_string = totalsize_string.format( size_short=size_string, size_exact=totalsize, filecount=filecount, ) write(totalsize_string) if unmeasured_file_count > 0: write(UNMEASURED_WARNING % unmeasured_file_count) return totalsize
files = [f.absolute_path for f in files] else: files = os.listdir() files = [f for f in files if os.path.isfile(f)] files = [ f for f in files if any(ext in f.lower() for ext in ['.jpg', '.jpeg']) ] bytes_saved = 0 remaining_size = 0 for filename in files: print(''.join(c for c in filename if c in string.printable)) bytesio = io.BytesIO() i = PIL.Image.open(filename) i.save(bytesio, format='jpeg', quality=80) bytesio.seek(0) new_bytes = bytesio.read() old_size = os.path.getsize(filename) new_size = len(new_bytes) remaining_size += new_size if new_size < old_size: bytes_saved += (old_size - new_size) f = open(filename, 'wb') f.write(new_bytes) f.close() print('Saved', bytestring.bytestring(bytes_saved)) print('Remaining are', bytestring.bytestring(remaining_size))
def measure(databasename, fullscan=False, new_only=False, threads=4): ''' Given a database, print the sum of all Content-Lengths. URLs will be HEAD requested if: `new_only` is True and the file has no stored content length, or `fullscan` is True and `new_only` is False ''' if isinstance(fullscan, str): fullscan = bool(fullscan) sql = sqlite3.connect(databasename) cur = sql.cursor() if new_only: cur.execute('SELECT * FROM urls WHERE do_download == 1 AND content_length IS NULL') else: cur.execute('SELECT * FROM urls WHERE do_download == 1') items = cur.fetchall() filecount = len(items) totalsize = 0 unmeasured_file_count = 0 if threads is None: threads = 1 thread_queue = threadqueue.ThreadQueue(threads) try: for fetch in items: size = fetch[SQL_CONTENT_LENGTH] if fullscan or new_only: url = fetch[SQL_URL] thread_queue.add(do_head, url, raise_for_status=False) elif size is None: # Unmeasured and no intention to measure. unmeasured_file_count += 1 else: totalsize += size for head in thread_queue.run(): fetch = smart_insert(sql, cur, head=head, commit=False) size = fetch[SQL_CONTENT_LENGTH] if size is None: write('"%s" is not revealing Content-Length' % url) size = 0 totalsize += size except (Exception, KeyboardInterrupt): sql.commit() raise sql.commit() size_string = bytestring.bytestring(totalsize) totalsize_string = '{size_short} ({size_exact:,} bytes) in {filecount:,} files' totalsize_string = totalsize_string.format( size_short=size_string, size_exact=totalsize, filecount=filecount, ) write(totalsize_string) if unmeasured_file_count > 0: write(UNMEASURED_WARNING % unmeasured_file_count) return totalsize