def put(environ, start_response): """ Put a bag to the server, meaning the description and policy of the bag, if policy allows. """ bag_name = web.get_route_value(environ, 'bag_name') bag_name = web.handle_extension(environ, bag_name) bag = Bag(bag_name) store = environ['tiddlyweb.store'] length = environ['CONTENT_LENGTH'] usersign = environ['tiddlyweb.usersign'] try: bag = store.get(bag) bag.policy.allows(usersign, 'manage') except NoBagError: create_policy_check(environ, 'bag', usersign) try: serialize_type = web.get_serialize_type(environ)[0] serializer = Serializer(serialize_type, environ) serializer.object = bag content = environ['wsgi.input'].read(int(length)) serializer.from_string(content.decode('utf-8')) bag.policy.owner = usersign['name'] _validate_bag(environ, bag) store.put(bag) except BagFormatError, exc: raise HTTP400('unable to put bag: %s' % exc)
def put(environ, start_response): """ Put a bag to the server, meaning the description and policy of the bag, if policy allows. """ bag_name = _determine_bag_name(environ) bag_name = web.handle_extension(environ, bag_name) bag = Bag(bag_name) store = environ["tiddlyweb.store"] length = environ["CONTENT_LENGTH"] usersign = environ["tiddlyweb.usersign"] try: bag = store.get(bag) bag.policy.allows(usersign, "manage") except NoBagError: create_policy_check(environ, "bag", usersign) try: serialize_type = web.get_serialize_type(environ)[0] serializer = Serializer(serialize_type, environ) serializer.object = bag content = environ["wsgi.input"].read(int(length)) serializer.from_string(content.decode("utf-8")) bag.policy.owner = usersign["name"] _validate_bag(environ, bag) store.put(bag) except BagFormatError, exc: raise HTTP400("unable to put bag: %s" % exc)
def test_bad_string_raises(): serializer = Serializer('text') foobar = Tiddler('foobar') serializer.object = foobar with pytest.raises(TiddlerFormatError): serializer.from_string(bad_string)
def _process_request_body(environ, tiddler): """ Read request body to set tiddler content. If a serializer exists for the content type, use it, otherwise treat the content as binary or pseudo-binary tiddler. """ length, content_type = content_length_and_type(environ) content = read_request_body(environ, length) try: try: serialize_type = get_serialize_type(environ)[0] serializer = Serializer(serialize_type, environ) # Short circuit de-serialization attempt to avoid # decoding content multiple times. if hasattr(serializer.serialization, 'as_tiddler'): serializer.object = tiddler try: serializer.from_string(content.decode('utf-8')) except TiddlerFormatError as exc: raise HTTP400('unable to put tiddler: %s' % exc) else: raise NoSerializationError() except NoSerializationError: tiddler.type = content_type if pseudo_binary(tiddler.type): tiddler.text = content.decode('utf-8') else: tiddler.text = content except UnicodeDecodeError as exc: raise HTTP400('unable to decode tiddler, utf-8 expected: %s' % exc)
def post_request(environ, start_response): length = int(environ["CONTENT_LENGTH"]) post_content = environ["wsgi.input"].read(length) serializer = Serializer("json") # TODO: use content-type to determine serialization rev = Tiddler("untitled") # N.B.: Serializations need not contain title. serializer.object = rev serializer.from_string(post_content.decode("utf-8")) query = environ["tiddlyweb.query"] store = environ["tiddlyweb.store"] rev1_id = query.get("rev1", [None])[0] rev2_id = query.get("rev2", [None])[0] try: if not rev1_id: rev1 = rev rev2 = _get_tiddler(rev2_id, store) elif not rev2_id: rev1 = _get_tiddler(rev1_id, store) rev2 = rev else: raise HTTP400("ambiguous request") except AttributeError: raise HTTP400("missing revision parameter") format = query.get("format", [None])[0] content = compare_tiddlers(rev1, rev2, format) return _generate_response(content, environ, start_response)
def import_plugin(bag, url): """ Import one plugin, at svn url, into bag, retrieving both the .js and .js.meta files. If there is no meta file, then set title and tags to something appropriate before de-serializing. """ meta_url = "%s.meta" % url plugin_content = get_url(url) try: meta_content = _get_url(meta_url) except HTTPError: meta_content = "title: %s\ntags: systemConfig\n" % _strip_extension(url, ".js").split("/")[-1] title = [line for line in meta_content.split("\n") if line.startswith("title:")][0] title = title.split(":", 1)[1].lstrip().rstrip() tiddler_meta = "\n".join([line for line in meta_content.split("\n") if not line.startswith("title:")]) tiddler_meta.rstrip() tiddler_text = "%s\n\n%s" % (tiddler_meta, plugin_content) tiddler = Tiddler(title, bag) serializer = Serializer("text") serializer.object = tiddler serializer.from_string(tiddler_text) _store().put(tiddler)
def test_json_recipe(): """ JSON serializer roundtrips. """ recipe = Recipe('other') recipe.set_recipe([['bagbuzz', '']]) recipe.policy.manage = ['a'] recipe.policy.read = ['b'] recipe.policy.create = ['c'] recipe.policy.delete = ['d'] recipe.policy.owner = 'e' serializer = Serializer('json') serializer.object = recipe string = serializer.to_string() other_recipe = Recipe('other') serializer.object = other_recipe serializer.from_string(string) assert recipe == other_recipe serializer.object = other_recipe other_string = serializer.to_string() assert string == other_string
def put(environ, start_response): """ Put a bag to the server, meaning the description and policy of the bag, if policy allows. """ bag_name = web.get_route_value(environ, 'bag_name') bag_name = web.handle_extension(environ, bag_name) bag = Bag(bag_name) store = environ['tiddlyweb.store'] length, _ = web.content_length_and_type(environ) usersign = environ['tiddlyweb.usersign'] try: bag = store.get(bag) bag.policy.allows(usersign, 'manage') except NoBagError: create_policy_check(environ, 'bag', usersign) try: serialize_type = web.get_serialize_type(environ)[0] serializer = Serializer(serialize_type, environ) serializer.object = bag content = web.read_request_body(environ, length) serializer.from_string(content.decode('utf-8')) bag.policy.owner = usersign['name'] _validate_bag(environ, bag) store.put(bag) except BagFormatError, exc: raise HTTP400('unable to put bag: %s' % exc)
def test_tiddler_from_json(): serializer = Serializer('json') tiddler = Tiddler('test tiddler') serializer.object = tiddler serializer.from_string(expected_json_string) assert tiddler.title == 'test tiddler' assert tiddler.text == "Hello, I'm the content."
def process_tid_tiddler(tiddler, content): """ Deserialize a tid. """ serializer = Serializer("text") serializer.object = tiddler serializer.from_string(content) return tiddler
def _put(entity, content, serialization): """ Put entity to store, by serializing content using the named serialization. """ serializer = Serializer(serialization) serializer.object = entity serializer.from_string(content) _store().put(entity)
def save_tiddler(store, tiddler): title = tiddler['title'] bag = tiddler['bag'] tiddler_json = simplejson.dumps(tiddler) tid = Tiddler(title) s = Serializer('json') s.object = tid s.from_string(tiddler_json) tid.bag = bag store.put(tid)
def setup_module(module): init(config) module.environ = { 'tiddlyweb.config': config } serializer = Serializer('json') module.tiddlers = Tiddlers() for title, json in test_tiddlers: tiddler = Tiddler(title) serializer.object = tiddler serializer.from_string(json) module.tiddlers.add(tiddler)
def _from_text(title, content): """ Generates a tiddler from an RFC822-style string This corresponds to TiddlyWeb's text serialization of TiddlerS. """ tiddler = Tiddler(title) serializer = Serializer('text') serializer.object = tiddler serializer.from_string(content) return tiddler
def _from_text(title, content): """ generates Tiddler from an RFC822-style string This corresponds to TiddlyWeb's text serialization of TiddlerS. """ tiddler = Tiddler(title) serializer = Serializer("text") serializer.object = tiddler serializer.from_string(content) return tiddler
def test_json_to_bag(): serializer = Serializer('json') json_string = simplejson.dumps(dict(policy=dict(read=['user1'], manage=['NONE']), desc='simply the best')) newbag = Bag('bagho') serializer.object = newbag serializer.from_string(json_string) assert newbag.name == 'bagho' assert newbag.policy.read == ['user1'] assert newbag.policy.manage == ['NONE'] assert newbag.desc == 'simply the best'
def test_json_to_bag(): serializer = Serializer("json") json_string = simplejson.dumps(dict(policy=dict(read=["user1"], manage=["NONE"]), desc="simply the best")) newbag = Bag("bagho") serializer.object = newbag serializer.from_string(json_string) assert newbag.name == "bagho" assert newbag.policy.read == ["user1"] assert newbag.policy.manage == ["NONE"] assert newbag.desc == "simply the best"
def test_old_text(): """ Send in text without a description and make sure we are able to accept it. """ recipe = Recipe('other') serializer = Serializer('text') serializer.object = recipe serializer.from_string(no_desc) output = serializer.to_string() assert output == expected_string
def test_wrong_class(): class Foo(object): pass foo = Foo() serializer = Serializer('text') serializer.object = foo string = 'haha' with py.test.raises(AttributeError): serializer.to_string() with py.test.raises(AttributeError): serializer.from_string(string)
def put(environ, start_response): """ Handle ``PUT`` on a single recipe URI. Put a :py:class:`recipe <tiddlyweb.model.recipe.Recipe>` to the server, meaning the description, policy and recipe list of the recipe, if :py:class:`policy <tiddlyweb.model.policy.Policy>` allows. """ recipe_name = web.get_route_value(environ, 'recipe_name') recipe_name = web.handle_extension(environ, recipe_name) recipe = Recipe(recipe_name) store = environ['tiddlyweb.store'] length, _ = web.content_length_and_type(environ) usersign = environ['tiddlyweb.usersign'] try: recipe = store.get(recipe) recipe.policy.allows(usersign, 'manage') except NoRecipeError: create_policy_check(environ, 'recipe', usersign) try: serialize_type = web.get_serialize_type(environ)[0] except TypeError: raise HTTP400('Content-type header required') try: serializer = Serializer(serialize_type, environ) serializer.object = recipe content = web.read_request_body(environ, length) serializer.from_string(content.decode('utf-8')) recipe.policy.owner = usersign['name'] _validate_recipe(environ, recipe) store.put(recipe) except RecipeFormatError as exc: raise HTTP400('unable to put recipe: %s' % exc) except TypeError as exc: raise HTTP400('malformed input: %s' % exc) except NoSerializationError: raise HTTP415('Content type %s not supported' % serialize_type) start_response("204 No Content", [('Location', web.recipe_url(environ, recipe))]) return []
def setup_module(module): csv_config = merge_config(config, { 'serializers': { 'text/csv': ['tiddlywebplugins.csv', 'text/csv; charset=UTF-8'] }, 'extension_types': { 'csv': 'text/csv' } }) module.environ = { 'tiddlyweb.config': csv_config } serializer = Serializer('json') module.tiddlers = Tiddlers() for title, json in test_tiddlers: tiddler = Tiddler(title) serializer.object = tiddler serializer.from_string(json) module.tiddlers.add(tiddler)
def _put_tiddler(environ, start_response, tiddler): """ The guts of putting a tiddler into the store. """ store = environ['tiddlyweb.store'] length = environ['CONTENT_LENGTH'] content_type = environ['tiddlyweb.type'] if content_type != 'text/plain' and content_type != 'application/json': tiddler.type = content_type try: bag = Bag(tiddler.bag) try: try: revision = store.list_tiddler_revisions(tiddler)[0] except StoreMethodNotImplemented: revision = 1 tiddler.revision = revision # These both next will raise exceptions if # the contraints don't match. _check_bag_constraint(environ, bag, 'write') _validate_tiddler(environ, tiddler) except NoTiddlerError: _check_bag_constraint(environ, bag, 'create') content = environ['wsgi.input'].read(int(length)) if not tiddler.type: serialize_type = web.get_serialize_type(environ)[0] serializer = Serializer(serialize_type, environ) serializer.object = tiddler serializer.from_string(content.decode('UTF-8')) else: tiddler.text = content user = environ['tiddlyweb.usersign']['name'] if not user == 'GUEST': tiddler.modifier = user store.put(tiddler) except NoBagError, exc: raise HTTP409("Unable to put tiddler, %s. There is no bag named: " \ "%s (%s). Create the bag." % (tiddler.title, tiddler.bag, exc))
def put(environ, start_response): """ Put a bag to the server, meaning the description and policy of the bag, if policy allows. """ bag_name = _determine_bag_name(environ) bag_name = web.handle_extension(environ, bag_name) bag = Bag(bag_name) store = environ['tiddlyweb.store'] length = environ['CONTENT_LENGTH'] usersign = environ['tiddlyweb.usersign'] try: bag.skinny = True bag = store.get(bag) bag.policy.allows(usersign, 'manage') try: delattr(bag, 'skinny') except AttributeError: pass except NoBagError: create_policy_check(environ, 'bag', usersign) try: serialize_type = web.get_serialize_type(environ)[0] serializer = Serializer(serialize_type, environ) serializer.object = bag content = environ['wsgi.input'].read(int(length)) serializer.from_string(content.decode('utf-8')) bag.policy.owner = usersign['name'] _validate_bag(environ, bag) store.put(bag) except TypeError: raise HTTP400('Content-type header required') except NoSerializationError: raise HTTP415('Content type not supported: %s' % serialize_type) start_response("204 No Content", [('Location', web.bag_url(environ, bag))]) return []
def put(environ, start_response): """ Handle ``PUT`` on a single bag URI. Put a :py:class:`bag <tiddlyweb.model.bag.Bag>` to the server, meaning the description and policy of the bag, if :py:class:`policy <tiddlyweb.model.policy.Policy>` allows. """ bag_name = web.get_route_value(environ, 'bag_name') bag_name = web.handle_extension(environ, bag_name) bag = Bag(bag_name) store = environ['tiddlyweb.store'] length, _ = web.content_length_and_type(environ) usersign = environ['tiddlyweb.usersign'] try: bag = store.get(bag) bag.policy.allows(usersign, 'manage') except NoBagError: create_policy_check(environ, 'bag', usersign) try: serialize_type = web.get_serialize_type(environ)[0] serializer = Serializer(serialize_type, environ) serializer.object = bag content = web.read_request_body(environ, length) serializer.from_string(content.decode('utf-8')) bag.policy.owner = usersign['name'] _validate_bag(environ, bag) store.put(bag) except BagFormatError as exc: raise HTTP400('unable to put bag: %s' % exc) except TypeError: raise HTTP400('Content-type header required') except NoSerializationError: raise HTTP415('Content type not supported: %s' % serialize_type) start_response("204 No Content", [('Location', web.bag_url(environ, bag))]) return []
def test_tiddler_fields_as_json(): tiddler = Tiddler('feebles', bag='bag0') tiddler = store.get(tiddler) serializer = Serializer('json') serializer.object = tiddler json_string = serializer.to_string() tiddler_info = simplejson.loads(json_string) assert tiddler_info['fields']['field1'] == 'value1' assert tiddler_info['fields']['field2'] == 'value2' assert tiddler_info['bag'] == 'bag0' tiddler = Tiddler('new feebles', bag='bag0') serializer.object = tiddler serializer.from_string(json_string) assert tiddler.fields['field1'] == 'value1' assert tiddler.fields['field2'] == 'value2' assert tiddler.bag == 'bag0'
def _process_json_tiddler(environ, content, uri): """ Transmute JSON content into a Tiddler. """ content = content.decode('utf-8') data = simplejson.loads(content) tiddler = RemoteTiddler(data['title'], uri) serializer = Serializer('json', environ) serializer.object = tiddler return serializer.from_string(content)
def _store_tiddler_revisions(environ, content, tiddler): """ Given JSON revisions in content, store them as a revision history to tiddler. """ try: json_tiddlers = simplejson.loads(content) except ValueError as exc: raise HTTP409('unable to handle json: %s' % exc) store = environ['tiddlyweb.store'] serializer = Serializer('json', environ) serializer.object = tiddler try: for json_tiddler in reversed(json_tiddlers): json_string = simplejson.dumps(json_tiddler) serializer.from_string(json_string) store.put(tiddler) except NoTiddlerError as exc: raise HTTP400('Unable to store tiddler revisions: %s', exc)
def _store_tiddler_revisions(environ, content, tiddler): """ Given JSON revisions in content, store them as a revision history to tiddler. """ try: json_tiddlers = simplejson.loads(content) except ValueError as exc: raise HTTP409('unable to handle json: %s' % exc) store = environ['tiddlyweb.store'] serializer = Serializer('json', environ) serializer.object = tiddler try: for json_tiddler in reversed(json_tiddlers): json_string = simplejson.dumps(json_tiddler) serializer.from_string(json_string) store.put(tiddler) except NoTiddlerError as exc: raise HTTP400('Unable to store tiddler revisions: %s' % exc)
def test_simple_recipe(): recipe = Recipe('other') recipe.set_recipe([('bagbuzz', '')]) recipe.policy.manage = ['a'] recipe.policy.read = ['b'] recipe.policy.create = ['c'] recipe.policy.delete = ['d'] recipe.policy.owner = 'e' serializer = Serializer('text') serializer.object = recipe string = serializer.to_string() new_recipe = Recipe('other') serializer.object = new_recipe serializer.from_string(string) assert recipe == new_recipe, 'recipe and new_recipe have equality' recipe = Recipe('other') recipe.set_recipe([('bagboom', '')]) assert recipe != new_recipe, 'modified recipe not equal new_recipe'
def _process_request_body(environ, tiddler): """ Read request body to set tiddler.text. """ length, content_type = content_length_and_type(environ) content = read_request_body(environ, length) try: try: serialize_type = get_serialize_type(environ)[0] serializer = Serializer(serialize_type, environ) # Short circuit de-serialization attempt to avoid # decoding content multiple times. if hasattr(serializer.serialization, 'as_tiddler'): serializer.object = tiddler try: serializer.from_string(content.decode('utf-8')) except TiddlerFormatError, exc: raise HTTP400('unable to put tiddler: %s' % exc) else: raise NoSerializationError()
def test_simple_recipe(): recipe = Recipe("other") recipe.set_recipe([("bagbuzz", "")]) recipe.policy.manage = ["a"] recipe.policy.read = ["b"] recipe.policy.create = ["c"] recipe.policy.delete = ["d"] recipe.policy.owner = "e" serializer = Serializer("text") serializer.object = recipe string = serializer.to_string() new_recipe = Recipe("other") serializer.object = new_recipe serializer.from_string(string) assert recipe.get_recipe() == new_recipe.get_recipe() recipe = Recipe("other") recipe.set_recipe([("bagboom", "")]) assert recipe != new_recipe
def test_tiddler_json_base64(): serializer = Serializer('json', environ={'tiddlyweb.config': config}) tiddler = Tiddler('binarytiddler') tiddler.bag = u'foo' with open('test/peermore.png', 'rb') as image_file: tiddler.text = image_file.read() bininfo = tiddler.text b64expected = b64encode(bininfo) tiddler.type = 'image/png' serializer.object = tiddler string = serializer.to_string() info = simplejson.loads(string) assert info['text'] == b64expected.decode('utf-8') tiddler = serializer.from_string(string) assert tiddler.text == bininfo info['text'] = '..badbinary..' string = simplejson.dumps(info) with pytest.raises(TiddlerFormatError): serializer.from_string(string)
def put(environ, start_response): """ Put a new recipe to the server. """ recipe_name = environ["wsgiorg.routing_args"][1]["recipe_name"] recipe_name = urllib.unquote(recipe_name) recipe_name = unicode(recipe_name, "utf-8") recipe_name = web.handle_extension(environ, recipe_name) recipe = Recipe(recipe_name) store = environ["tiddlyweb.store"] length = environ["CONTENT_LENGTH"] usersign = environ["tiddlyweb.usersign"] try: recipe = store.get(recipe) recipe.policy.allows(usersign, "manage") except NoRecipeError: create_policy_check(environ, "recipe", usersign) try: serialize_type = web.get_serialize_type(environ)[0] serializer = Serializer(serialize_type, environ) serializer.object = recipe content = environ["wsgi.input"].read(int(length)) serializer.from_string(content.decode("utf-8")) recipe.policy.owner = usersign["name"] _validate_recipe(environ, recipe) store.put(recipe) except TypeError: raise HTTP400("Content-type header required") except NoSerializationError: raise HTTP415("Content type %s not supported" % serialize_type) start_response("204 No Content", [("Location", web.recipe_url(environ, recipe))]) return []
def import_wiki(environ, start_response): """ Accept a tiddlywiki as POST and using it as the source parse it for tiddlers to be stored in the named bag. """ bag_name = _determine_bag_name(environ) bag = _get_bag(environ, bag_name, True) length = environ["CONTENT_LENGTH"] content = environ["wsgi.input"].read(int(length)) bag.policy.allows(environ["tiddlyweb.usersign"], "create") try: serialize_type, mime_type = web.get_serialize_type(environ) serializer = Serializer(serialize_type, environ) serializer.object = bag serializer.from_string(content) except NoSerializationError: raise HTTP415("Content type not supported: %s" % mime_type) except AttributeError, exc: raise HTTP400("Content malformed: %s" % exc)
def put(environ, start_response): """ Put a new recipe to the server. """ recipe_name = environ['wsgiorg.routing_args'][1]['recipe_name'] recipe_name = urllib.unquote(recipe_name) recipe_name = unicode(recipe_name, 'utf-8') recipe_name = web.handle_extension(environ, recipe_name) recipe = Recipe(recipe_name) store = environ['tiddlyweb.store'] length = environ['CONTENT_LENGTH'] usersign = environ['tiddlyweb.usersign'] try: recipe = store.get(recipe) recipe.policy.allows(usersign, 'manage') except NoRecipeError: create_policy_check(environ, 'recipe', usersign) try: serialize_type = web.get_serialize_type(environ)[0] except TypeError: raise HTTP400('Content-type header required') try: serializer = Serializer(serialize_type, environ) serializer.object = recipe content = environ['wsgi.input'].read(int(length)) serializer.from_string(content.decode('utf-8')) recipe.policy.owner = usersign['name'] _validate_recipe(environ, recipe) store.put(recipe) except RecipeFormatError, exc: raise HTTP400('unable to put recipe: %s' % exc)
def test_json_recipe(): """ JSON serializer roundtrips. """ recipe = Recipe('other') recipe.set_recipe([('bagbuzz', '')]) recipe.policy.manage = ['a'] recipe.policy.read = ['b'] recipe.policy.create = ['c'] recipe.policy.delete = ['d'] recipe.policy.owner = 'e' serializer = Serializer('json') serializer.object = recipe string = serializer.to_string() other_recipe = Recipe('other') serializer.object = other_recipe serializer.from_string(string) serializer.object = other_recipe other_string = serializer.to_string() assert string == other_string
def test_tiddler_json_base64(): serializer = Serializer('json') tiddler = Tiddler('binarytiddler') tiddler.bag = u'foo' tiddler.text = file('test/peermore.png', 'rb').read() bininfo = tiddler.text b64expected = b64encode(tiddler.text) tiddler.type = 'image/png' serializer.object = tiddler string = serializer.to_string() info = simplejson.loads(string) assert info['text'] == b64expected tiddler = serializer.from_string(string) assert tiddler.text == bininfo
def edit(args): """Edit a tiddler in a bag in the store. As text. <bag> <tiddler>""" try: bag, title = args[0:2] except IndexError: print >> sys.stderr, 'You must provide a bag name and tiddler title' sys.exit(1) tiddler = Tiddler(title, bag) store = get_store(config) serializer = Serializer('text') serializer.object = tiddler #check for bag try: bagobject = Bag(bag) bagobject = store.get(bagobject) except NoBagError: print >> sys.stderr, 'You must provide the name of a bag that exists.' sys.exit(1) try: tiddler = store.get(tiddler) tiddler_content = '%s' % serializer except NoTiddlerError: tiddler_content = u'' fd = NamedTemporaryFile(delete=False) fd.write(tiddler_content.encode('utf-8')) _edit(fd) try: fd = open(fd.name) tiddler_content = fd.read() tiddler = serializer.from_string(tiddler_content.decode('utf-8')) except (IOError, TiddlerFormatError), exc: print 'something went wrong:\n', exc print 'your edits are in file %s' % fd.name sys.exit(1)
class Store(StorageInterface): """ Replicate the StorageInterface calls as a collection of http requests to some other TiddlyWeb server. Since this is just a series of GETs and PUTs of some JSON data, what we need to do here is establish what URL we can to go to, and then package whatever data we might want. Much of the HTTP handling can be abstracted away. """ recipes_url = '/recipes' bags_url = '/bags' recipe_url = '/recipes/%s' recipe_tiddlers_url = '/recipes/%s/tiddlers' bag_url = '/bags/%s' bag_tiddlers_url = '/bags/%s/tiddlers' tiddler_url = '/bags/%s/tiddlers/%s' revisions_url = '/bags/%s/tiddlers/%s/revisions' revision_url = '/bags/%s/tiddlers/%s/revisions/%s' search_url = '/search?q=%s' def __init__(self, store_config=None, environ=None): super(Store, self).__init__(store_config, environ) server_info = self.store_config self._base = server_info['server_base'] user = server_info.get('user', None) password = server_info.get('password', None) cache = server_info.get('use_cache', False) if cache: self.http = httplib2.Http('.cache') else: self.http = httplib2.Http() self.authorization = None if user and password: self.authorization = b64encode('%s:%s' % (user, password)) self.serializer = Serializer('json', environ) def _request(self, method, url, data=None): headers = {} if method == 'GET': headers = {'Accept': 'application/json'} else: headers = {'Content-Type': 'application/json'} if self.authorization: # httplib2's authorization handling is triggered only # in response to 40x. We don't want that, we want to # force authentication, so we just set the header. headers['Authorization'] = 'Basic %s' % self.authorization url = self._base + url return self.http.request(url, method=method, headers=headers, body=data) def _is_success(self, response): return response['status'].startswith( '20') or response['status'] == '304' def _any_delete(self, url, target_object): response, content = self._request('DELETE', url) if not self._is_success(response): raise TiddlyWebWebError('%s: %s' % (response['status'], content)) def _any_get(self, url, target_object): response, content = self._request('GET', url) # Presence of response.previous indicates a redirect to # challenger if not response.previous and self._is_success(response): if response['content-type'].startswith('application/json'): self.serializer.object = target_object self.serializer.from_string(content) else: # XXX got a binary tiddler pass elif response['status'] == '401' or response.previous: raise UserRequiredError('you do not have permission on %s' % target_object) else: raise TiddlyWebWebError('%s: %s' % (response['status'], content)) def _any_put(self, url, target_object): self.serializer.object = target_object data = self.serializer.to_string() response, content = self._request('PUT', url, data) if not self._is_success(response): raise TiddlyWebWebError('%s: %s' % (response['status'], content)) def doit(self, url, object, method, exception): try: method(url, object) except TiddlyWebWebError, e: raise exception(e)
class Store(StorageInterface): """ A :py:class:`StorageInterface <tiddlyweb.stores.StorageInterface>` which stores text-based representations in a collection of directories and files. Some of the entities are serialized to and from text by the :py:class:`text <tiddlyweb.serializations.text.Serialization>` :py:class:`Serializer <tiddlyweb.serializer.Serializer>`. """ def __init__(self, store_config=None, environ=None): super(Store, self).__init__(store_config, environ) self.serializer = Serializer('text') self._root = self._fixup_root(store_config['store_root']) self._init_store() def _fixup_root(self, path): """ Adjust the ``store_root`` path so it is absolute. This is required in some web serving environments. """ if not os.path.isabs(path): path = os.path.join(self.environ['tiddlyweb.config']. get('root_dir', ''), path) return path def _init_store(self): """ Make sure the data storage directory and structure is present. """ if not os.path.exists(self._store_root()): os.mkdir(self._store_root()) for name in ['bags', 'recipes', 'users']: path = os.path.join(self._store_root(), name) if not os.path.exists(path): os.mkdir(path) def recipe_delete(self, recipe): """ Remove a :py:class:`recipe <tiddlyweb.model.recipe.Recipe>`, irrevocably, from the system. No impact on :py:class:`tiddlers <tiddlyweb.model.tiddler.Tiddler>`. """ try: recipe_path = self._recipe_path(recipe) if not os.path.exists(recipe_path): raise NoRecipeError('%s not present' % recipe_path) os.remove(recipe_path) except (NoRecipeError, StoreEncodingError) as exc: raise NoRecipeError(exc) except Exception as exc: raise IOError('unable to delete recipe %s: %s' % (recipe.name, exc)) def recipe_get(self, recipe): """ Fill :py:class:`recipe <tiddlyweb.model.recipe.Recipe>` with data in the store. """ try: recipe_path = self._recipe_path(recipe) self.serializer.object = recipe recipe_string = read_utf8_file(recipe_path) except StoreEncodingError as exc: raise NoRecipeError(exc) except IOError as exc: raise NoRecipeError('unable to get recipe %s: %s' % (recipe.name, exc)) return self.serializer.from_string(recipe_string) def recipe_put(self, recipe): """ Put :py:class:`recipe <tiddlyweb.model.recipe.Recipe>` into the store. """ try: recipe_path = self._recipe_path(recipe) self.serializer.object = recipe write_utf8_file(recipe_path, self.serializer.to_string()) except StoreEncodingError as exc: raise NoRecipeError(exc) def bag_delete(self, bag): """ Delete :py:class:`bag <tiddlyweb.model.bag.Bag>` **and** the :py:class:`tiddlers <tiddlyweb.model.tiddler.Tiddler>` within from the system. """ bag_path = self._bag_path(bag.name) try: if not os.path.exists(bag_path): raise NoBagError('%s not present' % bag_path) shutil.rmtree(bag_path) except NoBagError: raise except Exception as exc: raise IOError('unable to delete bag %s: %s' % (bag.name, exc)) def bag_get(self, bag): """ Fill :py:class:`bag <tiddlyweb.model.bag.Bag>` with data from the store. """ bag_path = self._bag_path(bag.name) try: bag.desc = self._read_bag_description(bag_path) bag.policy = self._read_policy(bag_path) except IOError as exc: raise NoBagError( 'unable to read policy or description at %s: %s' % (bag_path, exc)) return bag def bag_put(self, bag): """ Put :py:class:`bag <tiddlyweb.model.bag.Bag>` into the store. """ bag_path = self._bag_path(bag.name) tiddlers_dir = self._tiddlers_dir(bag.name) if not os.path.exists(bag_path): os.mkdir(bag_path) if not os.path.exists(tiddlers_dir): os.mkdir(tiddlers_dir) self._write_bag_description(bag.desc, bag_path) self._write_policy(bag.policy, bag_path) def tiddler_delete(self, tiddler): """ Irrevocably remove :py:class:`tiddler <tiddlyweb.model.tiddler.Tiddler>` from the filesystem. """ try: tiddler_base_filename = self._tiddler_base_filename(tiddler) if not os.path.exists(tiddler_base_filename): raise NoTiddlerError('%s not present' % tiddler_base_filename) shutil.rmtree(tiddler_base_filename) except NoTiddlerError: raise except Exception as exc: raise IOError('unable to delete %s: %s' % (tiddler.title, exc)) def tiddler_get(self, tiddler): """ Fill :py:class:`tiddler <tiddlyweb.model.tiddler.Tiddler>` with data from the store. """ try: # read in the desired tiddler tiddler = self._read_tiddler_revision(tiddler) # now make another tiddler to get created time first_rev = Tiddler(tiddler.title) first_rev.bag = tiddler.bag first_rev = self._read_tiddler_revision(first_rev, index=-1) # set created on new tiddler from modified on first_rev # (might be the same) tiddler.created = first_rev.modified tiddler.creator = first_rev.modifier return tiddler except IOError as exc: raise NoTiddlerError('no tiddler for %s: %s' % (tiddler.title, exc)) def tiddler_put(self, tiddler): """ Write a :py:class:`tiddler <tiddlyweb.model.tiddler.Tiddler>` into the store. We only write if the tiddler's :py:class:`bag <tiddlyweb.model.bag.Bag>` already exists. Bag creation is a separate action. """ tiddler_base_filename = self._tiddler_base_filename(tiddler) if not os.path.exists(tiddler_base_filename): try: os.mkdir(tiddler_base_filename) except OSError as exc: raise NoTiddlerError('unable to put tiddler: %s' % exc) locked = 0 lock_attempts = 0 while not locked: try: lock_attempts = lock_attempts + 1 write_lock(tiddler_base_filename) locked = 1 except LockError as exc: if lock_attempts > 4: raise StoreLockError(exc) time.sleep(.1) # Protect against incoming tiddlers that have revision # set. Since we are putting a new one, we want the system # to calculate. tiddler.revision = None revision = self._tiddler_revision_filename(tiddler) + 1 tiddler_filename = self._tiddler_full_filename(tiddler, revision) representation = self.serializer.serialization.tiddler_as(tiddler, omit_empty=True, omit_members=['creator']) write_utf8_file(tiddler_filename, representation) write_unlock(tiddler_base_filename) tiddler.revision = revision def user_delete(self, user): """ Delete :py:class:`user <tiddlyweb.model.user.User>` from the store. """ try: user_path = self._user_path(user) if not os.path.exists(user_path): raise NoUserError('%s not present' % user_path) os.unlink(user_path) except NoUserError: raise except Exception as exc: raise IOError('unable to delete %s: %s' % (user.usersign, exc)) def user_get(self, user): """ Fill :py:class:`user <tiddlyweb.model.user.User>` with data from the store. """ try: user_path = self._user_path(user) user_info = read_utf8_file(user_path) user_data = simplejson.loads(user_info) for key, value in user_data.items(): if key == 'roles': user.roles = set(value) continue if key == 'password': key = '_password' user.__setattr__(key, value) return user except IOError as exc: raise NoUserError('unable to get user %s: %s' % (user.usersign, exc)) def user_put(self, user): """ Put :py:class:`user <tiddlyweb.model.user.User>` data into the store. """ user_path = self._user_path(user) user_dict = {} for key in ['usersign', 'note', '_password', 'roles']: value = user.__getattribute__(key) if key == 'roles': user_dict[key] = list(value) continue if key == '_password': key = 'password' user_dict[key] = value user_info = simplejson.dumps(user_dict, indent=0) write_utf8_file(user_path, user_info) def list_recipes(self): """ List all the :py:class:`recipes <tiddlyweb.model.recipe.Recipe>` in the store. """ path = os.path.join(self._store_root(), 'recipes') recipes = self._files_in_dir(path) return (Recipe(unquote(recipe)) for recipe in recipes) def list_bags(self): """ List all the :py:class:`bags <tiddlyweb.model.bag.Bag>` in the store. """ bags = self._bag_filenames() return (Bag(unquote(bag)) for bag in bags) def list_bag_tiddlers(self, bag): """ List all the :py:class:`tiddlers <tiddlyweb.model.tiddler.Tiddler>` in the provided :py:class:`bag <tiddlyweb.model.bag.Bag>`. """ tiddlers_dir = self._tiddlers_dir(bag.name) try: tiddlers = (filename for filename in self._files_in_dir(tiddlers_dir) if os.path.isdir(os.path.join(tiddlers_dir, filename))) except (IOError, OSError) as exc: raise NoBagError('unable to list tiddlers in bag: %s' % exc) for title in tiddlers: title = unquote(title) tiddler = Tiddler(title, bag.name) yield tiddler def list_users(self): """ List all the :py:class:`users <tiddlyweb.model.user.User>` in the store. """ path = os.path.join(self._store_root(), 'users') users = self._files_in_dir(path) return (User(unquote(user)) for user in users) def list_tiddler_revisions(self, tiddler): """ List all the revisions of one :py:class:`tiddler <tiddlyweb.model.tiddler.Tiddler>`, returning a list of ints. """ tiddler_base_filename = self._tiddler_base_filename(tiddler) try: revisions = sorted( int(x) for x in self._numeric_files_in_dir(tiddler_base_filename)) except OSError as exc: raise NoTiddlerError('unable to list revisions in tiddler: %s' % exc) revisions.reverse() return revisions def search(self, search_query): """ Search in the store for :py:class:`tiddlers <tiddlyweb.model.tiddler.Tiddler>` that match ``search_query``. This is intentionally implemented as a simple and limited grep through files. """ bag_filenames = self._bag_filenames() query = search_query.lower() for bagname in bag_filenames: bagname = unquote(bagname) tiddler_dir = self._tiddlers_dir(bagname) tiddler_files = self._files_in_dir(tiddler_dir) for tiddler_name in tiddler_files: tiddler = Tiddler(title=unquote(tiddler_name), bag=bagname) try: revision_id = self.list_tiddler_revisions(tiddler)[0] if query in tiddler.title.lower(): yield tiddler continue with codecs.open( self._tiddler_full_filename(tiddler, revision_id), encoding='utf-8') as tiddler_file: for line in tiddler_file: if query in line.lower(): yield tiddler break except (OSError, NoTiddlerError) as exc: LOGGER.warn('malformed tiddler during search: %s:%s, %s', bagname, tiddler_name, exc) return def _bag_filenames(self): """ List the filenames that are bags. """ path = os.path.join(self._store_root(), 'bags') return self._files_in_dir(path) def _bag_path(self, bag_name): """ Return a string that is the path to a bag. """ try: return os.path.join(self._store_root(), 'bags', _encode_filename(bag_name)) except (AttributeError, StoreEncodingError) as exc: raise NoBagError('No bag name: %s' % exc) def _files_in_dir(self, path): """ List the filenames in a dir that do not start with . """ return (x for x in os.listdir(path)) def _numeric_files_in_dir(self, path): """ List the filenames in a dir that are made up of digits. """ return (x for x in self._files_in_dir(path) if x.isdigit()) def _read_tiddler_file(self, tiddler, tiddler_filename): """ Read a tiddler file from the disk, returning a tiddler object. """ tiddler_string = read_utf8_file(tiddler_filename) self.serializer.object = tiddler self.serializer.from_string(tiddler_string) return tiddler def _read_tiddler_revision(self, tiddler, index=0): """ Read a specific revision of a tiddler from disk. """ tiddler_revision = self._tiddler_revision_filename(tiddler, index=index) tiddler_filename = self._tiddler_full_filename(tiddler, tiddler_revision) tiddler = self._read_tiddler_file(tiddler, tiddler_filename) tiddler.revision = tiddler_revision return tiddler def _read_bag_description(self, bag_path): """ Read and return the description of a bag. """ desc_filename = os.path.join(bag_path, 'description') if not os.path.exists(desc_filename): return '' desc = read_utf8_file(desc_filename) return desc def _read_policy(self, bag_path): """ Read and return a bag's policy file, return the Policy object. """ policy_filename = os.path.join(bag_path, 'policy') policy = read_utf8_file(policy_filename) policy_data = simplejson.loads(policy) policy = Policy() for key, value in policy_data.items(): policy.__setattr__(key, value) return policy def _recipe_path(self, recipe): """ Return a string representing the pathname of a recipe. """ return os.path.join(self._store_root(), 'recipes', _encode_filename(recipe.name)) def _store_root(self): """ Return a string which is the path to the root of the store. """ return self._root def _tiddler_base_filename(self, tiddler): """ Return the string that is the pathname to a tiddler's directory. """ # should we get a Bag or a name here? bag_name = tiddler.bag store_dir = self._tiddlers_dir(bag_name) if not os.path.exists(store_dir): raise NoBagError('%s does not exist' % store_dir) try: return os.path.join(store_dir, _encode_filename(tiddler.title)) except StoreEncodingError as exc: raise NoTiddlerError(exc) def _tiddler_full_filename(self, tiddler, revision): """ Return the full path to the respective tiddler file. """ return os.path.join(self._tiddlers_dir(tiddler.bag), _encode_filename(tiddler.title), str(revision)) def _tiddlers_dir(self, bag_name): """ Return the string that is the pathname of the tiddlers directory in a bag. """ return os.path.join(self._bag_path(bag_name), 'tiddlers') def _tiddler_revision_filename(self, tiddler, index=0): """ Calculate the revision filename for the tiddler revision we want. """ revision = 0 if tiddler.revision: revision = tiddler.revision else: revisions = self.list_tiddler_revisions(tiddler) if revisions: revision = revisions[index] try: revision = int(revision) except ValueError: raise NoTiddlerError('%s is not a valid revision id' % revision) return revision def _user_path(self, user): """ Return the pathname for a user in the store. """ return os.path.join(self._store_root(), 'users', _encode_filename(user.usersign)) def _write_bag_description(self, desc, bag_path): """ Write the description of a bag to disk. """ desc_filename = os.path.join(bag_path, 'description') write_utf8_file(desc_filename, desc) def _write_policy(self, policy, bag_path): """ Write the policy of a bad to disk. """ policy_dict = {} for key in Policy.attributes: policy_dict[key] = policy.__getattribute__(key) policy_string = simplejson.dumps(policy_dict) policy_filename = os.path.join(bag_path, 'policy') write_utf8_file(policy_filename, policy_string)
if environ['REQUEST_METHOD'] == 'PUT': length, content_type = _length_and_type(environ) content = environ['wsgi.input'].read(int(length)) try: try: # XXX HACK! We don't want to decode content unless # the serializer has a as_tiddler. We should be able # to just rely on NoSerializationError, but we need # to call the method to do that, and to call the method we # need to decode the string... serialize_type = web.get_serialize_type(environ)[0] serializer = Serializer(serialize_type, environ) serializer.object = tiddler try: serializer.from_string(content.decode('utf-8')) except TiddlerFormatError, exc: raise HTTP400('unable to put tiddler: %s' % exc) except NoSerializationError: tiddler.type = content_type if pseudo_binary(tiddler.type): tiddler.text = content.decode('utf-8') else: tiddler.text = content except UnicodeDecodeError, exc: raise HTTP400('unable to decode tiddler, utf-8 expected: %s', exc) try: recipe_name = web.get_route_value(environ, 'recipe_name') recipe = Recipe(recipe_name) try: