def handle_describechannels_for_missing_tip(self):
test_id = u"tests/1"
test_channel = u"test_channel"
test_protocol = u"test_protocol"
gref = Gref(self.station.station.store, test_channel, test_id)
root_obj = RootObject(test_id, test_channel, test_protocol)
root_oid = self.station.station.write(root_obj.as_object())
current_oid = [root_oid]
for i in xrange(10):
update_obj = UpdateObject([current_oid.pop()], "loldata")
current_oid.append(self.station.station.write(update_obj.as_object()))
update_obj = UpdateObject([current_oid.pop()], "loldata")
current_oid.append(binascii.hexlify(pygit2.hash(update_obj.as_object())))
self.station.station.update_gref(gref, [Tip(root_oid, "")])
self.assertEqual([root_oid], gref.tips())
current_oid = current_oid.pop()
self.station.payload = _payload(update_obj, gref, [current_oid])
handle_describechannels(self.station)
self.assertEqual([current_oid], gref.tips())
def handle_describechannels_for_missing_tip(self):
test_id = u"tests/1"
test_channel = u"test_channel"
test_protocol = u"test_protocol"
gref = Gref(self.station.station.store, test_channel, test_id)
root_obj = RootObject(test_id, test_channel, test_protocol)
root_oid = self.station.station.write(root_obj.as_object())
current_oid = [root_oid]
for i in xrange(10):
update_obj = UpdateObject([current_oid.pop()], "loldata")
current_oid.append(self.station.station.write(update_obj.as_object()))
update_obj = UpdateObject([current_oid.pop()], "loldata")
current_oid.append(binascii.hexlify(pygit2.hash(update_obj.as_object())))
self.station.station.update_gref(gref, [root_oid])
self.assertEqual([root_oid], gref.tips())
current_oid = current_oid.pop()
self.station.payload = _payload(update_obj, gref, [current_oid])
handle_describechannels(self.station)
self.assertEqual([current_oid], gref.tips())
def test_get_signature(self):
gref = Gref(self.repo, "testchannel", "test_get_signature")
root = self.create_root_object(gref)
oid = self.repo.create_blob(root.as_object())
signature = (17 ** 23,)
gref.write_tip(oid, signature)
self.assertEqual(signature, gref.get_signature(oid))
def test_get_signature(self):
gref = Gref(self.repo, "testchannel", "test_get_signature")
root = self.create_root_object(gref)
oid = self.repo.create_blob(root.as_object())
signature = (17**23, )
gref.write_tip(oid, signature)
self.assertEqual(signature, gref.get_signature(oid))
def test_marshalls_untrusted_signatures(self):
gref = Gref(self.repo, "testchannel", "test_untrusted_signature")
adaptor = RSAAdaptor({"valid": crypto_fixture.passphrase_pubkey})
private_adaptor = RSAPrivateAdaptor(crypto_fixture.valid_key)
root = root_object("test", "test_channel", "test")
root_oid = self.repo.create_blob(root.as_object())
update = update_object("test object", [root_oid])
update_oid = self.repo.create_blob(update.as_object())
gref.write_tip(update_oid, private_adaptor.sign(update_oid))
marshalled = gref.marshall(crypto_adaptor=adaptor)
self.assertFalse(marshalled["signatures"][update_oid])
def test_parents(self):
gref = Gref(self.repo, "testchannel", "test_write_tip")
root = self.create_root_object(gref)
current = root
parents = []
for i in xrange(10):
oid = self.repo.create_blob(current.as_object())
parents.append(oid)
current = self.create_update_object([oid], "lol data")
else:
oid = self.repo.create_blob(current.as_object())
gref.write_tip(oid, "")
our_parents = gref.parents()
for parent in our_parents:
self.assertIn(parent, parents)
def test_parents(self):
gref = Gref(self.repo, "testchannel", "test_write_tip")
root = self.create_root_object(gref)
current = root
parents = []
for i in xrange(10):
oid = self.repo.create_blob(current.as_object())
parents.append(oid)
current = self.create_update_object([oid], "lol data")
else:
oid = self.repo.create_blob(current.as_object())
gref.write_tip(oid, "")
our_parents = gref.parents()
for parent in our_parents:
self.assertIn(parent, parents)
def test_direct_parents(self):
gref = Gref(self.repo, "testchannel", "test_write_tip")
root = self.create_root_object(gref)
root_oid = self.repo.create_blob(root.as_object())
first_tier = []
for i in xrange(5):
obj = self.create_update_object([root_oid], "test_%i")
oid = self.repo.create_blob(obj.as_object())
first_tier.append(oid)
final = self.create_update_object(first_tier, "final object")
final_oid = self.repo.create_blob(final.as_object())
gref.write_tip(final_oid, "")
self.assertEqual(gref.direct_parents(final_oid), first_tier)
def test_marshalls_tips(self):
gref = Gref(self.repo, "testchannel", "test_write_tip")
objects = []
root = root_object("test", "test_channel", "test")
root_oid = self.repo.create_blob(root.as_object())
for i in xrange(5):
update = update_object("test %i" % i, [root_oid])
update_oid = self.repo.create_blob(update.as_object())
objects.append(update_oid)
gref.write_tip(update_oid, "")
marshalled = gref.marshall()
for i in objects:
self.assertIn(i, marshalled["tips"])
def test_direct_parents(self):
gref = Gref(self.repo, "testchannel", "test_write_tip")
root = self.create_root_object(gref)
root_oid = self.repo.create_blob(root.as_object())
first_tier = []
for i in xrange(5):
obj = self.create_update_object([root_oid], "test_%i")
oid = self.repo.create_blob(obj.as_object())
first_tier.append(oid)
final = self.create_update_object(first_tier, "final object")
final_oid = self.repo.create_blob(final.as_object())
gref.write_tip(final_oid, "")
self.assertEqual(gref.direct_parents(final_oid), first_tier)
def test_marshalls_tips(self):
gref = Gref(self.repo, "testchannel", "test_write_tip")
objects = []
root = root_object("test", "test_channel", "test")
root_oid = self.repo.create_blob(root.as_object())
for i in xrange(5):
update = update_object("test %i" % i, [root_oid])
update_oid = self.repo.create_blob(update.as_object())
objects.append(update_oid)
gref.write_tip(update_oid, "")
marshalled = gref.marshall()
for i in objects:
self.assertIn(i, marshalled["tips"])
def test_marshalls_untrusted_signatures(self):
gref = Gref(self.repo, "testchannel", "test_untrusted_signature")
adaptor = RSAAdaptor({
"valid": crypto_fixture.passphrase_pubkey
})
private_adaptor = RSAPrivateAdaptor(crypto_fixture.valid_key)
root = root_object("test", "test_channel", "test")
root_oid = self.repo.create_blob(root.as_object())
update = update_object("test object", [root_oid])
update_oid = self.repo.create_blob(update.as_object())
gref.write_tip(update_oid, private_adaptor.sign(update_oid))
marshalled = gref.marshall(crypto_adaptor=adaptor)
self.assertFalse(marshalled["signatures"][update_oid])
def fetch_gref(channel, identifier):
crypto_adaptor = station.get_crypto_adaptor()
adaptor = GithubReadAdaptor(station, channel)
gref = Gref(station.store, channel, identifier)
log.info("Trying to fetch channel: %s identifier: %s" %
(channel, identifier))
marshalled_thread = adaptor.get_issue(gref,
crypto_adaptor=crypto_adaptor)
root_obj = marshalled_thread["roots"].pop()
root = root_obj.as_json()
root["hash"] = oid2hex(pygit2.hash(root_obj.as_object()))
response = []
while marshalled_thread["thread"]:
node = marshalled_thread["thread"].pop()
data = json.loads(node.data)
data["parents"] = list(node.parents)
data["hash"] = oid2hex(pygit2.hash(node.as_object()))
response.append(data)
return jsonate(
{
"content": response,
"root": root,
"tips": marshalled_thread["tips"],
"signatures": marshalled_thread["signatures"]
}, False)
def create_gref(channel):
def _write_object(obj):
return station.write(obj.as_object())
name = request.form["name"]
protocol = request.form["protocol"]
user = request.form["user"]
body = request.form["body"]
title = request.form["title"]
gref = Gref(station.store, channel, name)
root = RootObject(name, channel, protocol)
root_oid = _write_object(root)
_title = UpdateObject([root_oid],
json.dumps({
"type": "title",
"id": None,
"body": title,
"user": user
}))
title_oid = _write_object(_title)
_body = UpdateObject([title_oid],
json.dumps({
"type": "body",
"id": None,
"body": body
}))
body_oid = _write_object(_body)
_update_gref(gref, [Tip(body_oid, "")], [])
return ""
def grefs(self, channel):
channel_path = os.path.join(self.store.gref_path(), channel)
if not groundstation.utils.is_dir(channel_path):
raise NonExistantChannel()
grefs = []
for id in groundstation.utils.find_leaf_dirs(channel_path, True):
grefs.append(Gref(self.store, channel, id))
return grefs
def test_marshalls_roots(self):
gref = Gref(self.repo, "testchannel", "test_write_root")
roots = []
for i in xrange(5):
root = root_object("test", "test_channel", "test%i" % i)
root_oid = self.repo.create_blob(root.as_object())
root.sha1 = root_oid
roots.append(root_oid)
# Create an update object for valid tip
update = update_object("test %i" % i, roots)
update_oid = self.repo.create_blob(update.as_object())
gref.write_tip(update_oid, "")
marshalled = gref.marshall()
root_hashes = [i.sha1 for i in marshalled["roots"]]
for i in roots:
self.assertIn(i, root_hashes)
def test_marshalls_roots(self):
gref = Gref(self.repo, "testchannel", "test_write_root")
roots = []
for i in xrange(5):
root = root_object("test", "test_channel", "test%i" % i)
root_oid = self.repo.create_blob(root.as_object())
root.sha1 = root_oid
roots.append(root_oid)
# Create an update object for valid tip
update = update_object("test %i" % i, roots)
update_oid = self.repo.create_blob(update.as_object())
gref.write_tip(update_oid, "")
marshalled = gref.marshall()
root_hashes = [i.sha1 for i in marshalled["roots"]]
for i in roots:
self.assertIn(i, root_hashes)
def handle_newgreftip(self):
proto_channels = groundstation.proto.channel_list_pb2.ChannelList()
proto_channels.ParseFromString(self.payload)
for channel in proto_channels.channels:
for gref in channel.grefs:
_gref = Gref(self.station.store, channel.channelname, gref.identifier)
# Create a tip object to avoid upsetting fileutils
tips = [tip.tip for tip in gref.tips]
self.station.update_gref(_gref, tips, True)
def test_remove_tip(self):
gref = Gref(self.repo, "testchannel", "test_write_tip")
gref.write_tip("foobarbaz", "")
gref.write_tip("lulzbutts", "")
gref.write_tip("buttslols", "")
gref.remove_tip("lulzbutts")
self.assertTrue("foobarbaz" in list(gref))
self.assertTrue("buttslols" in list(gref))
self.assertFalse("lulzbutts" in list(gref))
def update_gref(channel, identifier):
# adaptor = github_protocol.GithubWriteAdaptor(station, channel)
gref = Gref(station.store, channel, identifier)
# Ugly type coercion
user = request.form["user"]
body = request.form["body"]
parents = map(str, json.loads(request.form["parents"]))
payload = {"type": "comment", "id": None, "body": body, "user": user}
update_object = UpdateObject(parents, json.dumps(payload))
oid = station.write(update_object.as_object())
_update_gref(gref, [Tip(oid, "")], parents)
return jsonate({"response": "ok"}, False)
def test_remove_tip(self):
gref = Gref(self.repo, "testchannel", "test_write_tip")
gref.write_tip("foobarbaz", "")
gref.write_tip("lulzbutts", "")
gref.write_tip("buttslols", "")
gref.remove_tip("lulzbutts")
self.assertTrue("foobarbaz" in list(gref))
self.assertTrue("buttslols" in list(gref))
self.assertFalse("lulzbutts" in list(gref))
def start_listening(self):
header_byte_count, header_bytes, header_length = reset_header()
payload = reset_payload()
listen.setup_processes(self.tones)
for char in listen.start_analysing_stream():
char_value = ord(char)
if char_value & 0b10000000: # Header byte
log.info("Header byte: %s" % (bin(char_value)))
if header_byte_count == 0: # First header byte
header_bytes[0] |= char_value & 0xf
header_byte_count += 1
elif header_byte_count == 1:
header_bytes[0] |= (char_value & 0xf) << 4
header_byte_count += 1
elif header_byte_count == 2:
header_bytes[1] |= (char_value & 0xf)
header_byte_count += 1
elif header_byte_count == 3:
header_bytes[1] |= (char_value & 0xf) << 4
header_byte_count += 1
header_length = struct.unpack(
">h", ''.join(map(chr, header_bytes)))[0]
log.info("Reading %d bytes of payload" % header_length)
payload = reset_payload()
elif header_byte_count == 4:
log.warning(
"Got header bytes while still expecting payload")
payload.append(char)
else: # Payload byte
if header_length == 0:
log.warning("Got payload bytes when expecting header")
payload.append(char)
if len(payload) % 8 == 0:
log.info("Got %d bytes so far" % (len(payload)))
if len(payload) == header_length:
log.info("Got all of object")
decoded = base64.b64decode(''.join(payload))
log.info("Writing payload to DB")
oid = self.station.write(decoded)
log.info("Wrote object, got: %s" % (repr(oid)))
Gref(self.station.store, "soundstation", "demo")
try:
self.station.update_gref(g, [Tip(oid, "")], []) # yolo
except:
pass
header_byte_count, header_bytes, header_length = reset_header(
)
payload = reset_payload()
def handle_describechannels(self):
if not self.payload:
log.info(
"station %s sent empty DESCRIBECHANNELS payload - new database?" %
(str(self.origin)))
return
proto_channels = groundstation.proto.channel_list_pb2.ChannelList()
proto_channels.ParseFromString(self.payload)
for channel in proto_channels.channels:
for gref in channel.grefs:
_gref = Gref(self.station.store, channel.channelname,
gref.identifier)
# Create a tip object to avoid upsetting fileutils
tips = [Tip(tip.tip, tip.signature) for tip in gref.tips]
self.station.update_gref(_gref, tips, True)
def test_handle_listallchannels(self):
test_id = u"tests/1"
test_channel = u"test_channel"
test_protocol = u"test_protocol"
obj = RootObject(test_id, test_channel, test_protocol)
gref = Gref(self.station.station.store, test_channel, test_id)
oid = self.station.station.write(obj.as_object())
self.station.station.update_gref(gref, [Tip(oid, "")], [])
handle_listallchannels(self.station)
response = self.station.stream.pop()
serialized_response = response.SerializeToString()
self.assertIsInstance(serialized_response, str)
assert len(self.station.stream) == 0, "Someone is leaving objects lyind around"
channel_description = ChannelList()
channel_description.ParseFromString(response.payload)
self.assertEqual(channel_description.channels[0].channelname, test_channel)
self.assertEqual(channel_description.channels[0].grefs[0].identifier, test_id)
self.assertEqual(channel_description.channels[0].grefs[0].tips[0].tip, oid)
def test_channels(self):
gref = Gref(self.station.store, "test_channel", "test_id")
self.station.update_gref(gref, [Tip("foobar", "")], "")
self.assertEqual(self.station.channels(), ["test_channel"])
# Creates a thread in the current groundstation context
import uuid
import stricken
from groundstation.node import Node
from groundstation.station import Station
from groundstation.objects.root_object import RootObject
from groundstation.objects.update_object import UpdateObject
from groundstation.gref import Gref
node = Node()
station = Station.from_env(node)
CHANNEL = "messages"
thread_id = str(uuid.uuid1())
gref = Gref(station.store, CHANNEL, thread_id)
root = RootObject(thread_id, CHANNEL, stricken.PROTOCOL)
oid = station.write(root.as_object())
print("Root id: %s" % (oid))
update = UpdateObject([oid], "Post content")
oid = station.write(update.as_object())
print("Update id: %s" % (oid))
gref.write_tip(oid, "")
def issue_gref(self, issue):
return Gref(self.station.store, self.channel, self._issue_id(issue))
def test_write_tip(self):
gref = Gref(self.repo, "testchannel", "test_write_tip")
gref.write_tip("foobarbaz", "")
self.assertEqual(list(gref), ["foobarbaz"])
def gref(self, channel, identifier):
return Gref(self, channel, identifier)
def test_write_tip(self):
gref = Gref(self.repo, "testchannel", "test_write_tip")
gref.write_tip("foobarbaz", "")
self.assertEqual(list(gref), ["foobarbaz"])