def getimagesize(url): """ Attempts to determine an image's width and height, and returns a string suitable for use in an <img> tag, or None in case of failure. Requires that PIL is installed. >>> getimagesize("http://www.google.com/intl/en_ALL/images/logo.gif") ... #doctest: +ELLIPSIS, +SKIP 'width="..." height="..."' """ from PIL import ImageFile try: p = ImageFile.Parser() f = urlopen(url) while True: s = f.read(1024) if not s: break p.feed(s) if p.image: return 'width="%i" height="%i"' % p.image.size except (OSError, ValueError): return None
def generate_public_uti_map(): from lxml import etree from polyglot.urllib import urlopen from html5_parser import parse raw = urlopen( 'https://developer.apple.com/library/ios/documentation/Miscellaneous/Reference/UTIRef/Articles/System-DeclaredUniformTypeIdentifiers.html').read() root = parse(raw) tables = root.xpath('//table')[0::2] data = {} for table in tables: for tr in table.xpath('descendant::tr')[1:]: td = tr.xpath('descendant::td') identifier = etree.tostring(td[0], method='text', encoding=unicode_type).strip() tags = etree.tostring(td[2], method='text', encoding=unicode_type).strip() identifier = identifier.split()[0].replace('\u200b', '') exts = [x.strip()[1:].lower() for x in tags.split(',') if x.strip().startswith('.')] for ext in exts: data[ext] = identifier lines = ['PUBLIC_UTI_MAP = {'] for ext in sorted(data): r = ("'" + ext + "':").ljust(16) lines.append((' ' * 4) + r + "'" + data[ext] + "',") lines.append('}') with open(__file__, 'r+b') as f: raw = f.read() f.seek(0) nraw = re.sub(r'^PUBLIC_UTI_MAP = .+?}', '\n'.join(lines), raw, flags=re.MULTILINE | re.DOTALL) f.truncate(), f.write(nraw)
def getimagesize(url): """ Attempts to determine an image's width and height, and returns a string suitable for use in an <img> tag, or None in case of failure. Requires that PIL is installed. >>> getimagesize("http://www.google.com/intl/en_ALL/images/logo.gif") ... #doctest: +ELLIPSIS, +SKIP 'width="..." height="..."' """ try: from PIL import ImageFile except ImportError: try: import ImageFile except ImportError: return None try: p = ImageFile.Parser() f = urlopen(url) while True: s = f.read(1024) if not s: break p.feed(s) if p.image: return 'width="%i" height="%i"' % p.image.size except (IOError, ValueError): return None
def download_one(tdir, timeout, progress_report, data_uri_map, url): try: purl = urlparse(url) data_url_key = None with NamedTemporaryFile(dir=tdir, delete=False) as df: if purl.scheme == 'file': path = unquote(purl.path) if iswindows and path.startswith('/'): path = path[1:] src = lopen(path, 'rb') filename = os.path.basename(path) sz = (src.seek(0, os.SEEK_END), src.tell(), src.seek(0))[1] elif purl.scheme == 'data': prefix, payload = purl.path.split(',', 1) parts = prefix.split(';') if parts and parts[-1].lower() == 'base64': payload = re.sub(r'\s+', '', payload) payload = from_base64_bytes(payload) else: payload = payload.encode('utf-8') seen_before = data_uri_map.get(payload) if seen_before is not None: return True, (url, filename, seen_before, guess_type(seen_before)) data_url_key = payload src = BytesIO(payload) sz = len(payload) ext = 'unknown' for x in parts: if '=' not in x and '/' in x: exts = mimetypes.guess_all_extensions(x) if exts: ext = exts[0] break filename = 'data-uri.' + ext else: src = urlopen(url, timeout=timeout) filename = get_filename(purl, src) sz = get_content_length(src) progress_report(url, 0, sz) dest = ProgressTracker(df, url, sz, progress_report) with closing(src): shutil.copyfileobj(src, dest) if data_url_key is not None: data_uri_map[data_url_key] = dest.name filename = sanitize_file_name(filename) mt = guess_type(filename) if mt in OEB_DOCS: raise ValueError( 'The external resource {} looks like a HTML document ({})'. format(url, filename)) if not mt or mt == 'application/octet-stream' or '.' not in filename: raise ValueError( 'The external resource {} is not of a known type'.format( url)) return True, (url, filename, dest.name, mt) except Exception as err: return False, (url, as_unicode(err))
def download_one(tdir, timeout, progress_report, data_uri_map, url): try: purl = urlparse(url) data_url_key = None with NamedTemporaryFile(dir=tdir, delete=False) as df: if purl.scheme == 'file': src = lopen(purl.path, 'rb') filename = os.path.basename(src) sz = (src.seek(0, os.SEEK_END), src.tell(), src.seek(0))[1] elif purl.scheme == 'data': prefix, payload = purl.path.split(',', 1) parts = prefix.split(';') if parts and parts[-1].lower() == 'base64': payload = re.sub(r'\s+', '', payload) payload = from_base64_bytes(payload) else: payload = payload.encode('utf-8') seen_before = data_uri_map.get(payload) if seen_before is not None: return True, (url, filename, seen_before, guess_type(seen_before)) data_url_key = payload src = BytesIO(payload) sz = len(payload) ext = 'unknown' for x in parts: if '=' not in x and '/' in x: exts = mimetypes.guess_all_extensions(x) if exts: ext = exts[0] break filename = 'data-uri.' + ext else: src = urlopen(url, timeout=timeout) filename = get_filename(purl, src) sz = get_content_length(src) progress_report(url, 0, sz) dest = ProgressTracker(df, url, sz, progress_report) with closing(src): shutil.copyfileobj(src, dest) if data_url_key is not None: data_uri_map[data_url_key] = dest.name filename = sanitize_file_name(filename) mt = guess_type(filename) if mt in OEB_DOCS: raise ValueError('The external resource {} looks like a HTML document ({})'.format(url, filename)) if not mt or mt == 'application/octet-stream' or '.' not in filename: raise ValueError('The external resource {} is not of a known type'.format(url)) return True, (url, filename, dest.name, mt) except Exception as err: return False, (url, as_unicode(err))
def request(path, data=None): r = Request('https://api.fosshub.com/rest/' + path.lstrip('/'), headers={ 'Content-Type': 'application/json', 'X-auth-key': api_key, 'User-Agent': 'calibre' }) res = urlopen(r, data=data) ans = json.loads(res.read()) if ans.get('error'): raise SystemExit(ans['error']) if res.getcode() != 200: raise SystemExit('Request to {} failed with response code: {}'.format(path, res.getcode())) # from pprint import pprint # pprint(ans) return ans['status'] if 'status' in ans else ans['data']
def import_from_launchpad(url): f = open('/tmp/launchpad_export.tar.gz', 'wb') shutil.copyfileobj(urlopen(url), f) f.close() tf = tarfile.open('/tmp/launchpad_export.tar.gz', 'r:gz') next = tf.next() while next is not None: if next.isfile() and next.name.endswith('.po'): try: po = re.search(r'-([a-z]{2,3}\.po)', next.name).group(1) except: next = tf.next() continue out = os.path.abspath(os.path.join('.', os.path.basename(po))) print('Updating', '%6s'%po, '-->', out) open(out, 'wb').write(tf.extractfile(next).read()) next = tf.next() check_for_critical_bugs() path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) print(path) subprocess.check_call('python setup.py translations'.split(), cwd=path) return 0
def upload_to_fosshub(): # fosshub has no API to do partial uploads, so we always upload all files. print('Sending upload request to fosshub...') files = set(installers()) entries = [] for fname in files: desc = installer_description(fname) url = 'https://download.calibre-ebook.com/%s/%s' % ( __version__, os.path.basename(fname) ) entries.append({ 'url': url, 'type': desc, 'version': __version__, }) jq = { 'software': 'Calibre', 'apiKey': get_fosshub_data(), 'upload': entries, 'delete': [{ 'type': '*', 'version': '*', 'name': '*' }] } # print(json.dumps(jq, indent=2)) rq = urlopen( 'https://www.fosshub.com/JSTools/uploadJson', urlencode({ 'content': json.dumps(jq) }) ) resp = rq.read() if rq.getcode() != 200: raise SystemExit( 'Failed to upload to fosshub, with HTTP error code: %d and response: %s' % (rq.getcode(), resp) )