def _font_url(path, only_path=False, cache_buster=True, inline=False): filepath = StringValue(path).value path = None if callable(config.STATIC_ROOT): try: _file, _storage = list(config.STATIC_ROOT(filepath))[0] d_obj = _storage.modified_time(_file) filetime = int(time.mktime(d_obj.timetuple())) if inline: path = _storage.open(_file) except: filetime = 'NA' else: _path = os.path.join(config.STATIC_ROOT, filepath.strip('/')) if os.path.exists(_path): filetime = int(os.path.getmtime(_path)) if inline: path = open(_path, 'rb') else: filetime = 'NA' BASE_URL = config.STATIC_URL if path and inline: mime_type = mimetypes.guess_type(filepath)[0] url = 'data:' + mime_type + ';base64,' + base64.b64encode(path.read()) else: url = '%s%s' % (BASE_URL, filepath) if cache_buster: url = add_cache_buster(url, filetime) if not only_path: url = 'url("%s")' % escape(url) return StringValue(url)
def encrypt_file(self, file_name): with open(file_name, 'rb') as path: plainttext = path.read() enc = self.encrypt(plainttext,self.key) with open(file_name + ".enc", 'wb') as path: path.write(enc) os.remove(file_name)
def from_csv(cls, path, column_names=None, column_types=None, row_names=None, header=True, sniff_limit=0, encoding='utf-8', **kwargs): """ Create a new table from a CSV. This method uses agate's builtin CSV reader, which supplies encoding support for both Python 2 and Python 3. :code:`kwargs` will be passed through to the CSV reader. :param path: Filepath or file-like object from which to read CSV data. :param column_names: See :meth:`.Table.__init__`. :param column_types: See :meth:`.Table.__init__`. :param row_names: See :meth:`.Table.__init__`. :param header: If `True`, the first row of the CSV is assumed to contains headers and will be skipped. If `header` and `column_names` are both specified then a row will be skipped, but `column_names` will be used. :param sniff_limit: Limit CSV dialect sniffing to the specified number of bytes. Set to None to sniff the entire file. Defaults to 0 or no sniffing. :param encoding: Character encoding of the CSV file. Note: if passing in a file handle it is assumed you have already opened it with the correct encoding specified. """ if hasattr(path, 'read'): contents = path.read() else: with io.open(path, encoding=encoding) as f: contents = f.read() if sniff_limit is None: kwargs['dialect'] = csv.Sniffer().sniff(contents) elif sniff_limit > 0: kwargs['dialect'] = csv.Sniffer().sniff(contents[:sniff_limit]) if six.PY2: contents = contents.encode('utf-8') rows = list(csv.reader(six.StringIO(contents), header=header, **kwargs)) if header: if column_names is None: column_names = rows.pop(0) else: rows.pop(0) return Table(rows, column_names, column_types, row_names=row_names)
def test_download_url_to_file_if_not_exists__when_file_exists( url, file_name, tmpdir): path = tmpdir.join(file_name) path.write('content') path_str = str(path) assert os.path.isfile(path_str) download_url_to_file_if_not_exists(url, path_str) assert path.read() == 'content'
def setup_module(mod): mod.db = he.Database(str(py.test.ensuretemp('test_search.db'))) mod.db.add_attr_index('@title', he.ESTIDXATTRSTR) for path in TESTDATA.listdir(): if path.basename.startswith('rfc'): doc = he.Document(unicode(path.basename)) stat = path.lstat() doc['@cdate'] = unicode(formattime(stat.ctime)) doc['@mdate'] = unicode(formattime(stat.mtime)) doc['@size'] = unicode(stat.size) doc['@title'] = unicode(path.basename) doc.add_text(unicode(path.read(), 'utf-8')) mod.db.put_doc(doc) db.flush() db.sync() db.optimize()
def save(path, real_parent): """Save a virtual path to the disk. path -- (Path) real_parent -- (str) directory """ full = os.path.join(real_parent, path.name) if type(path) is Directory: if not os.path.exists(full): mkdir(full) for child in path.children.values(): save(child, full) else: real_file = open(full, 'w') real_file.write(path.read()) real_file.close()
def from_csv(cls, path, column_names=None, column_types=None, row_names=None, header=True, sniff_limit=0, **kwargs): """ Create a new table for a CSV. This method uses agate's builtin CSV reader, which supports unicode on both Python 2 and Python 3. `kwargs` will be passed through to the CSV reader. :param path: Filepath or file-like object from which to read CSV data. :param column_names: See :meth:`Table.__init__`. :param column_types: See :meth:`Table.__init__`. :param row_names: See :meth:`Table.__init__`. :param header: If `True`, the first row of the CSV is assumed to contains headers and will be skipped. If `header` and `column_names` are both specified then a row will be skipped, but `column_names` will be used. :param sniff_limit: Limit CSV dialect sniffing to the specified number of bytes. Set to None to sniff the entire file. Defaults to 0 or no sniffing. """ if hasattr(path, 'read'): contents = path.read() else: with open(path) as f: contents = f.read() if sniff_limit is None: kwargs['dialect'] = csv.Sniffer().sniff(contents) elif sniff_limit > 0: kwargs['dialect'] = csv.Sniffer().sniff(contents[:sniff_limit]) rows = list(csv.reader(StringIO(contents), header=header, **kwargs)) if header: if column_names is None: column_names = rows.pop(0) else: rows.pop(0) return Table(rows, column_names, column_types, row_names=row_names)
def _image_url(path, only_path=False, cache_buster=True, dst_color=None, src_color=None, inline=False, mime_type=None, spacing=None, collapse_x=None, collapse_y=None): """ src_color - a list of or a single color to be replaced by each corresponding dst_color colors spacing - spaces to be added to the image collapse_x, collapse_y - collapsable (layered) image of the given size (x, y) """ if inline or dst_color or spacing: if not Image: raise Exception("Images manipulation require PIL") filepath = String.unquoted(path).value fileext = os.path.splitext(filepath)[1].lstrip('.').lower() if mime_type: mime_type = String.unquoted(mime_type).value if not mime_type: mime_type = mimetypes.guess_type(filepath)[0] if not mime_type: mime_type = 'image/%s' % fileext path = None IMAGES_ROOT = _images_root() if callable(IMAGES_ROOT): try: _file, _storage = list(IMAGES_ROOT(filepath))[0] except IndexError: filetime = None else: filetime = getmtime(_file, _storage) if filetime is None: filetime = 'NA' elif inline or dst_color or spacing: path = _storage.open(_file) else: _path = os.path.join(IMAGES_ROOT.rstrip('/'), filepath.strip('/')) filetime = getmtime(_path) if filetime is None: filetime = 'NA' elif inline or dst_color or spacing: path = open(_path, 'rb') BASE_URL = config.IMAGES_URL or config.STATIC_URL if path: dst_colors = [ list(Color(v).value[:3]) for v in List.from_maybe(dst_color) if v ] src_color = Color.from_name( 'black') if src_color is None else src_color src_colors = [ tuple(Color(v).value[:3]) for v in List.from_maybe(src_color) ] len_colors = max(len(dst_colors), len(src_colors)) dst_colors = (dst_colors * len_colors)[:len_colors] src_colors = (src_colors * len_colors)[:len_colors] spacing = Number(0) if spacing is None else spacing spacing = [int(Number(v).value) for v in List.from_maybe(spacing)] spacing = (spacing * 4)[:4] file_name, file_ext = os.path.splitext( os.path.normpath(filepath).replace('\\', '_').replace('/', '_')) key = (filetime, src_color, dst_color, spacing) key = file_name + '-' + base64.urlsafe_b64encode( hashlib.md5(repr(key)).digest()).rstrip('=').replace('-', '_') asset_file = key + file_ext ASSETS_ROOT = config.ASSETS_ROOT or os.path.join( config.STATIC_ROOT, 'assets') asset_path = os.path.join(ASSETS_ROOT, asset_file) if os.path.exists(asset_path): filepath = asset_file BASE_URL = config.ASSETS_URL if inline: path = open(asset_path, 'rb') url = 'data:' + mime_type + ';base64,' + base64.b64encode( path.read()) else: url = '%s%s' % (BASE_URL, filepath) if cache_buster: filetime = getmtime(asset_path) url = add_cache_buster(url, filetime) else: simply_process = False image = None if fileext in ('cur', ): simply_process = True else: try: image = Image.open(path) except IOError: if not collapse_x and not collapse_y and not dst_colors: simply_process = True if simply_process: if inline: url = 'data:' + mime_type + ';base64,' + base64.b64encode( path.read()) else: url = '%s%s' % (BASE_URL, filepath) if cache_buster: filetime = getmtime(asset_path) url = add_cache_buster(url, filetime) else: width, height = collapse_x or image.size[ 0], collapse_y or image.size[1] new_image = Image.new(mode='RGBA', size=(width + spacing[1] + spacing[3], height + spacing[0] + spacing[2]), color=(0, 0, 0, 0)) for i, dst_color in enumerate(dst_colors): src_color = src_colors[i] pixdata = image.load() for _y in xrange(image.size[1]): for _x in xrange(image.size[0]): pixel = pixdata[_x, _y] if pixel[:3] == src_color: pixdata[_x, _y] = tuple( [int(c) for c in dst_color] + [pixel[3] if len(pixel) == 4 else 255]) iwidth, iheight = image.size if iwidth != width or iheight != height: cy = 0 while cy < iheight: cx = 0 while cx < iwidth: cropped_image = image.crop( (cx, cy, cx + width, cy + height)) new_image.paste(cropped_image, (int(spacing[3]), int(spacing[0])), cropped_image) cx += width cy += height else: new_image.paste(image, (int(spacing[3]), int(spacing[0]))) if not inline: try: new_image.save(asset_path) filepath = asset_file BASE_URL = config.ASSETS_URL if cache_buster: filetime = getmtime(asset_path) except IOError: log.exception("Error while saving image") inline = True # Retry inline version url = os.path.join(config.ASSETS_URL.rstrip('/'), asset_file.lstrip('/')) if cache_buster: url = add_cache_buster(url, filetime) if inline: output = six.BytesIO() new_image.save(output, format='PNG') contents = output.getvalue() output.close() url = 'data:' + mime_type + ';base64,' + base64.b64encode( contents) else: url = os.path.join(BASE_URL.rstrip('/'), filepath.lstrip('/')) if cache_buster and filetime != 'NA': url = add_cache_buster(url, filetime) if not only_path: url = 'url(%s)' % escape(url) return String.unquoted(url)
def _image_url(path, only_path=False, cache_buster=True, dst_color=None, src_color=None, inline=False, mime_type=None, spacing=None, collapse_x=None, collapse_y=None): """ src_color - a list of or a single color to be replaced by each corresponding dst_color colors spacing - spaces to be added to the image collapse_x, collapse_y - collapsable (layered) image of the given size (x, y) """ if inline or dst_color or spacing: if not Image: raise Exception("Images manipulation require PIL") filepath = String.unquoted(path).value fileext = os.path.splitext(filepath)[1].lstrip('.').lower() if mime_type: mime_type = String.unquoted(mime_type).value if not mime_type: mime_type = mimetypes.guess_type(filepath)[0] if not mime_type: mime_type = 'image/%s' % fileext path = None IMAGES_ROOT = _images_root() if callable(IMAGES_ROOT): try: _file, _storage = list(IMAGES_ROOT(filepath))[0] except IndexError: filetime = None else: filetime = getmtime(_file, _storage) if filetime is None: filetime = 'NA' elif inline or dst_color or spacing: path = _storage.open(_file) else: _path = os.path.join(IMAGES_ROOT.rstrip(os.sep), filepath.strip('\\/')) filetime = getmtime(_path) if filetime is None: filetime = 'NA' elif inline or dst_color or spacing: path = open(_path, 'rb') BASE_URL = config.IMAGES_URL or config.STATIC_URL if path: dst_colors = [list(Color(v).value[:3]) for v in List.from_maybe(dst_color) if v] src_color = Color.from_name('black') if src_color is None else src_color src_colors = [tuple(Color(v).value[:3]) for v in List.from_maybe(src_color)] len_colors = max(len(dst_colors), len(src_colors)) dst_colors = (dst_colors * len_colors)[:len_colors] src_colors = (src_colors * len_colors)[:len_colors] spacing = Number(0) if spacing is None else spacing spacing = [int(Number(v).value) for v in List.from_maybe(spacing)] spacing = (spacing * 4)[:4] file_name, file_ext = os.path.splitext(os.path.normpath(filepath).replace(os.sep, '_')) key = (filetime, src_color, dst_color, spacing) asset_file = file_name + '-' + make_filename_hash(key) + file_ext ASSETS_ROOT = config.ASSETS_ROOT or os.path.join(config.STATIC_ROOT, 'assets') asset_path = os.path.join(ASSETS_ROOT, asset_file) if os.path.exists(asset_path): filepath = asset_file BASE_URL = config.ASSETS_URL if inline: path = open(asset_path, 'rb') url = make_data_url(mime_type, path.read()) else: url = '%s%s' % (BASE_URL, filepath) if cache_buster: filetime = getmtime(asset_path) url = add_cache_buster(url, filetime) else: simply_process = False image = None if fileext in ('cur',): simply_process = True else: try: image = Image.open(path) except IOError: if not collapse_x and not collapse_y and not dst_colors: simply_process = True if simply_process: if inline: url = make_data_url(mime_type, path.read()) else: url = '%s%s' % (BASE_URL, filepath) if cache_buster: filetime = getmtime(asset_path) url = add_cache_buster(url, filetime) else: width, height = collapse_x or image.size[0], collapse_y or image.size[1] new_image = Image.new( mode='RGBA', size=(width + spacing[1] + spacing[3], height + spacing[0] + spacing[2]), color=(0, 0, 0, 0) ) for i, dst_color in enumerate(dst_colors): src_color = src_colors[i] pixdata = image.load() for _y in xrange(image.size[1]): for _x in xrange(image.size[0]): pixel = pixdata[_x, _y] if pixel[:3] == src_color: pixdata[_x, _y] = tuple([int(c) for c in dst_color] + [pixel[3] if len(pixel) == 4 else 255]) iwidth, iheight = image.size if iwidth != width or iheight != height: cy = 0 while cy < iheight: cx = 0 while cx < iwidth: cropped_image = image.crop((cx, cy, cx + width, cy + height)) new_image.paste(cropped_image, (int(spacing[3]), int(spacing[0])), cropped_image) cx += width cy += height else: new_image.paste(image, (int(spacing[3]), int(spacing[0]))) if not inline: try: new_image.save(asset_path) filepath = asset_file BASE_URL = config.ASSETS_URL if cache_buster: filetime = getmtime(asset_path) except IOError: log.exception("Error while saving image") inline = True # Retry inline version url = os.path.join(config.ASSETS_URL.rstrip(os.sep), asset_file.lstrip(os.sep)) if cache_buster: url = add_cache_buster(url, filetime) if inline: output = six.BytesIO() new_image.save(output, format='PNG') contents = output.getvalue() output.close() url = make_data_url(mime_type, contents) else: url = os.path.join(BASE_URL.rstrip('/'), filepath.lstrip('\\/')) if cache_buster and filetime != 'NA': url = add_cache_buster(url, filetime) if not os.sep == '/': url = url.replace(os.sep, '/') if only_path: return String.unquoted(url) else: return Url.unquoted(url)
def _replace(self, path, what, to): with open(path, 'r') as p: data = p.read() data = data.replace(what, to) with open(path, 'w') as p: p.write(data)
def _image_url(path, only_path=False, cache_buster=True, dst_color=None, src_color=None, inline=False, mime_type=None, spacing=None, collapse_x=None, collapse_y=None): """ src_color - a list of or a single color to be replaced by each corresponding dst_color colors spacing - spaces to be added to the image collapse_x, collapse_y - collapsable (layered) image of the given size (x, y) """ if inline or dst_color or spacing: if not Image: raise Exception("Images manipulation require PIL") filepath = StringValue(path).value mime_type = inline and (StringValue(mime_type).value or mimetypes.guess_type(filepath)[0]) path = None if callable(config.STATIC_ROOT): try: _file, _storage = list(config.STATIC_ROOT(filepath))[0] d_obj = _storage.modified_time(_file) filetime = int(time.mktime(d_obj.timetuple())) if inline or dst_color or spacing: path = _storage.open(_file) except: filetime = 'NA' else: _path = os.path.join(config.STATIC_ROOT, filepath.strip('/')) if os.path.exists(_path): filetime = int(os.path.getmtime(_path)) if inline or dst_color or spacing: path = open(_path, 'rb') else: filetime = 'NA' BASE_URL = config.STATIC_URL if path: dst_colors = [list(ColorValue(v).value[:3]) for v in List.from_maybe(dst_color) if v] src_colors = src_color src_colors = [tuple(ColorValue(v).value[:3]) if v else (0, 0, 0) for v in List.from_maybe(src_color)] len_colors = max(len(dst_colors), len(src_colors)) dst_colors = (dst_colors * len_colors)[:len_colors] src_colors = (src_colors * len_colors)[:len_colors] spacing = [int(NumberValue(v).value) if v else 0 for v in List.from_maybe(spacing)] spacing = (spacing * 4)[:4] file_name, file_ext = os.path.splitext(os.path.normpath(filepath).replace('\\', '_').replace('/', '_')) key = (filetime, src_color, dst_color, spacing) key = file_name + '-' + base64.urlsafe_b64encode(hashlib.md5(repr(key)).digest()).rstrip('=').replace('-', '_') asset_file = key + file_ext ASSETS_ROOT = config.ASSETS_ROOT or os.path.join(config.STATIC_ROOT, 'assets') asset_path = os.path.join(ASSETS_ROOT, asset_file) if os.path.exists(asset_path): filepath = asset_file BASE_URL = config.ASSETS_URL if inline: path = open(asset_path, 'rb') url = 'data:' + mime_type + ';base64,' + base64.b64encode(path.read()) else: url = '%s%s' % (BASE_URL, filepath) if cache_buster: filetime = int(os.path.getmtime(asset_path)) url = add_cache_buster(url, filetime) else: image = Image.open(path) width, height = collapse_x or image.size[0], collapse_y or image.size[1] new_image = Image.new( mode='RGBA', size=(width + spacing[1] + spacing[3], height + spacing[0] + spacing[2]), color=(0, 0, 0, 0) ) for i, dst_color in enumerate(dst_colors): src_color = src_colors[i] pixdata = image.load() for _y in xrange(image.size[1]): for _x in xrange(image.size[0]): pixel = pixdata[_x, _y] if pixel[:3] == src_color: pixdata[_x, _y] = tuple([int(c) for c in dst_color] + [pixel[3] if len(pixel) == 4 else 255]) iwidth, iheight = image.size if iwidth != width or iheight != height: cy = 0 while cy < iheight: cx = 0 while cx < iwidth: cropped_image = image.crop((cx, cy, cx + width, cy + height)) new_image.paste(cropped_image, (int(spacing[3]), int(spacing[0])), cropped_image) cx += width cy += height else: new_image.paste(image, (int(spacing[3]), int(spacing[0]))) if not inline: try: new_image.save(asset_path) filepath = asset_file BASE_URL = config.ASSETS_URL if cache_buster: filetime = int(os.path.getmtime(asset_path)) except IOError: log.exception("Error while saving image") inline = True # Retry inline version url = '%s%s' % (config.ASSETS_URL, asset_file) if cache_buster: url = add_cache_buster(url, filetime) if inline: output = six.BytesIO() new_image.save(output, format='PNG') contents = output.getvalue() output.close() url = 'data:' + mime_type + ';base64,' + base64.b64encode(contents) else: url = '%s%s' % (BASE_URL, filepath) if cache_buster: url = add_cache_buster(url, filetime) if not only_path: url = 'url("%s")' % escape(url) return StringValue(url)
def _image_url( path, only_path=False, cache_buster=True, dst_color=None, src_color=None, inline=False, mime_type=None, spacing=None, collapse_x=None, collapse_y=None, ): """ src_color - a list of or a single color to be replaced by each corresponding dst_color colors spacing - spaces to be added to the image collapse_x, collapse_y - collapsable (layered) image of the given size (x, y) """ if inline or dst_color or spacing: if not Image: raise Exception("Images manipulation require PIL") filepath = String.unquoted(path).value fileext = os.path.splitext(filepath)[1].lstrip(".").lower() if mime_type: mime_type = String.unquoted(mime_type).value if not mime_type: mime_type = mimetypes.guess_type(filepath)[0] if not mime_type: mime_type = "image/%s" % fileext path = None IMAGES_ROOT = _images_root() if callable(IMAGES_ROOT): try: _file, _storage = list(IMAGES_ROOT(filepath))[0] d_obj = _storage.modified_time(_file) filetime = int(time.mktime(d_obj.timetuple())) if inline or dst_color or spacing: path = _storage.open(_file) except: filetime = "NA" else: _path = os.path.join(IMAGES_ROOT.rstrip("/"), filepath.strip("/")) if os.path.exists(_path): filetime = int(os.path.getmtime(_path)) if inline or dst_color or spacing: path = open(_path, "rb") else: filetime = "NA" BASE_URL = config.IMAGES_URL or config.STATIC_URL if path: dst_colors = [list(Color(v).value[:3]) for v in List.from_maybe(dst_color) if v] src_color = Color.from_name("black") if src_color is None else src_color src_colors = [tuple(Color(v).value[:3]) for v in List.from_maybe(src_color)] len_colors = max(len(dst_colors), len(src_colors)) dst_colors = (dst_colors * len_colors)[:len_colors] src_colors = (src_colors * len_colors)[:len_colors] spacing = Number(0) if spacing is None else spacing spacing = [int(Number(v).value) for v in List.from_maybe(spacing)] spacing = (spacing * 4)[:4] file_name, file_ext = os.path.splitext(os.path.normpath(filepath).replace("\\", "_").replace("/", "_")) key = (filetime, src_color, dst_color, spacing) key = file_name + "-" + base64.urlsafe_b64encode(hashlib.md5(repr(key)).digest()).rstrip("=").replace("-", "_") asset_file = key + file_ext ASSETS_ROOT = config.ASSETS_ROOT or os.path.join(config.STATIC_ROOT, "assets") asset_path = os.path.join(ASSETS_ROOT, asset_file) if os.path.exists(asset_path): filepath = asset_file BASE_URL = config.ASSETS_URL if inline: path = open(asset_path, "rb") url = "data:" + mime_type + ";base64," + base64.b64encode(path.read()) else: url = "%s%s" % (BASE_URL, filepath) if cache_buster: filetime = int(os.path.getmtime(asset_path)) url = add_cache_buster(url, filetime) else: simply_process = False image = None if fileext in ("cur",): simply_process = True else: try: image = Image.open(path) except IOError: if not collapse_x and not collapse_y and not dst_colors: simply_process = True if simply_process: if inline: url = "data:" + mime_type + ";base64," + base64.b64encode(path.read()) else: url = "%s%s" % (BASE_URL, filepath) if cache_buster: filetime = int(os.path.getmtime(asset_path)) url = add_cache_buster(url, filetime) else: width, height = collapse_x or image.size[0], collapse_y or image.size[1] new_image = Image.new( mode="RGBA", size=(width + spacing[1] + spacing[3], height + spacing[0] + spacing[2]), color=(0, 0, 0, 0), ) for i, dst_color in enumerate(dst_colors): src_color = src_colors[i] pixdata = image.load() for _y in xrange(image.size[1]): for _x in xrange(image.size[0]): pixel = pixdata[_x, _y] if pixel[:3] == src_color: pixdata[_x, _y] = tuple( [int(c) for c in dst_color] + [pixel[3] if len(pixel) == 4 else 255] ) iwidth, iheight = image.size if iwidth != width or iheight != height: cy = 0 while cy < iheight: cx = 0 while cx < iwidth: cropped_image = image.crop((cx, cy, cx + width, cy + height)) new_image.paste(cropped_image, (int(spacing[3]), int(spacing[0])), cropped_image) cx += width cy += height else: new_image.paste(image, (int(spacing[3]), int(spacing[0]))) if not inline: try: new_image.save(asset_path) filepath = asset_file BASE_URL = config.ASSETS_URL if cache_buster: filetime = int(os.path.getmtime(asset_path)) except IOError: log.exception("Error while saving image") inline = True # Retry inline version url = os.path.join(config.ASSETS_URL.rstrip("/"), asset_file.lstrip("/")) if cache_buster: url = add_cache_buster(url, filetime) if inline: output = six.BytesIO() new_image.save(output, format="PNG") contents = output.getvalue() output.close() url = "data:" + mime_type + ";base64," + base64.b64encode(contents) else: url = os.path.join(BASE_URL.rstrip("/"), filepath.lstrip("/")) if cache_buster and filetime != "NA": url = add_cache_buster(url, filetime) if not only_path: url = "url(%s)" % escape(url) return String.unquoted(url)