def LoadFromTurtle(self, stream, volume_arn):
data = streams.ReadAll(stream)
g = rdflib.Graph()
g.parse(data=data, format="turtle")
for urn, attr, value in g:
urn = utils.SmartUnicode(urn)
attr = utils.SmartUnicode(attr)
serialized_value = value
if isinstance(value, rdflib.URIRef):
value = rdfvalue.URN(utils.SmartUnicode(serialized_value))
elif value.datatype in registry.RDF_TYPE_MAP:
dt = value.datatype
value = registry.RDF_TYPE_MAP[value.datatype](
serialized_value)
else:
# Default to a string literal.
value = rdfvalue.XSDString(value)
if attr == rdfvalue.URN(lexicon.AFF4_TYPE) and value == rdfvalue.URN(lexicon.AFF4_IMAGE_TYPE):
self.Add(lexicon.transient_graph, urn, lexicon.AFF4_STORED, volume_arn)
self.Add(volume_arn, urn, attr, value)
# look for the AFF4 namespace defined in the turtle
for (_, b) in g.namespace_manager.namespaces():
if (str(b) == lexicon.AFF4_NAMESPACE or
str(b) == lexicon.AFF4_LEGACY_NAMESPACE):
self.aff4NS = b
def DumpToTurtle(self, zipcontainer, ):
infoARN = escaping.urn_from_member_name(u"information.turtle", zipcontainer.urn, zipcontainer.version)
mode = self.GetUnique(lexicon.transient_graph, zipcontainer.backing_store_urn, lexicon.AFF4_STREAM_WRITE_MODE)
if mode == "random":
# random mode is used for appending to encrypted streams, where the stream size changes
# snapshot mode creates the situation where we have multiple versions of the stream object
# mashed together, and we cant tell the most recent
turtle_append_mode="latest"
else:
# in append mode, we assume that each time we append, we are adding to the container, rather
# than modifying any existing objects in the container. Because of this, we get to save multiple
# independent copies of the turtle from each run, and join them together as text for efficiency
turtle_append_mode="snapshot"
if not zipcontainer.ContainsMember(infoARN):
with zipcontainer.CreateZipSegment(u"information.turtle") as turtle_segment:
turtle_segment.compression_method = ZIP_STORED
result = self._DumpToTurtle(zipcontainer.urn)
turtle_segment.write(utils.SmartStr(result))
turtle_segment.Flush()
turtle_segment.Close()
else:
# append to an existng container
self.invalidateCachedMetadata(zipcontainer)
if turtle_append_mode == "latest":
zipcontainer.RemoveMember(infoARN)
with zipcontainer.CreateZipSegment(u"information.turtle") as turtle_segment:
turtle_segment.compression_method = ZIP_STORED
result = self._DumpToTurtle(zipcontainer.urn)
turtle_segment.write(utils.SmartStr(result))
turtle_segment.Flush()
turtle_segment.Close()
return
explodedTurtleDirectivesARN = escaping.urn_from_member_name(u"information.turtle/directives", zipcontainer.urn, zipcontainer.version)
if not zipcontainer.ContainsMember(explodedTurtleDirectivesARN):
# this is the first append operation. Create the chunked turtle structures
with zipcontainer.OpenZipSegment(u"information.turtle") as turtle_segment:
currentTurtleBytes= streams.ReadAll(turtle_segment)
currentturtle = utils.SmartUnicode(currentTurtleBytes)
#hexdump.hexdump(currentTurtleBytes)
(directives_txt, triples_txt) = turtle.toDirectivesAndTripes(currentturtle)
with zipcontainer.CreateZipSegment(u"information.turtle/directives") as directives_segment:
directives_segment.compression_method = ZIP_DEFLATE
directives_segment.write(utils.SmartStr(directives_txt))
directives_segment.Flush()
directives_segment.Close()
with zipcontainer.CreateZipSegment(u"information.turtle/%08d" % 0) as turtle_chunk_segment:
turtle_chunk_segment.compression_method = ZIP_DEFLATE
turtle_chunk_segment.write(utils.SmartStr(triples_txt))
turtle_chunk_segment.Flush()
self.Close(turtle_chunk_segment)
turtle_segment.Close()
(current_directives_txt, current_triples_txt) = turtle.toDirectivesAndTripes(utils.SmartUnicode(self._DumpToTurtle(zipcontainer.urn)))
directives_difference = turtle.difference(directives_txt, current_directives_txt)
if not len(directives_difference) == 0:
directives_txt = directives_txt + u"\r\n" + directives_difference
with zipcontainer.CreateZipSegment(u"information.turtle/directives") as directives_segment:
directives_segment.compression_method = ZIP_DEFLATE
directives_segment.write(utils.SmartStr(directives_txt))
directives_segment.Flush()
directives_segment.Close()
current_turtle_chunk_arn = rdfvalue.URN(u"%s/information.turtle/%08d" % (zipcontainer.urn, 1))
with zipcontainer.CreateMember(current_turtle_chunk_arn) as turtle_segment:
turtle_segment.compression_method = ZIP_DEFLATE
turtle_segment.write(utils.SmartStr(current_triples_txt))
turtle_segment.Flush()
self.Close(turtle_segment)
zipcontainer.RemoveSegment(u"information.turtle")
with zipcontainer.CreateZipSegment(u"information.turtle") as turtle_segment:
turtle_segment.compression_method = ZIP_STORED
turtle_segment.write(utils.SmartStr(directives_txt + "\r\n\r\n"))
turtleContainerIndex = 0
while True:
current_turtle_chunk_arn = rdfvalue.URN(u"%s/information.turtle/%08d" % (zipcontainer.urn, turtleContainerIndex))
if zipcontainer.ContainsMember(current_turtle_chunk_arn):
with zipcontainer.OpenMember(current_turtle_chunk_arn) as turtle_chunk_segment:
turtle_chunk_txt = utils.SmartUnicode(streams.ReadAll(turtle_chunk_segment))
turtle_segment.write(utils.SmartStr(turtle_chunk_txt + u"\r\n"))
turtleContainerIndex += 1
else:
break
turtle_segment.Flush()
turtle_segment.Close()
else:
# more than one append as already occurred
turtleContainerIndex = 0
while True:
turtleARN = escaping.urn_from_member_name(u"information.turtle/%08d" % turtleContainerIndex,
zipcontainer.urn, zipcontainer.version)
if not zipcontainer.ContainsMember(turtleARN):
break
turtleContainerIndex = turtleContainerIndex + 1
with zipcontainer.OpenZipSegment(u"information.turtle/directives") as directives_segment:
directives_txt = utils.SmartUnicode(streams.ReadAll(directives_segment))
(current_directives_txt, current_triples_txt) = turtle.toDirectivesAndTripes(utils.SmartUnicode(self._DumpToTurtle(zipcontainer.urn)))
directives_difference = turtle.difference(directives_txt, current_directives_txt)
if len(directives_difference) > 0:
directives_txt = directives_txt + u"\r\n" + u"\r\n".join(directives_difference)
with zipcontainer.CreateZipSegment(u"information.turtle/directives") as directives_segment:
directives_segment.compression_method = ZIP_DEFLATE
directives_segment.write(utils.SmartStr(directives_txt))
directives_segment.Flush()
directives_segment.Close()
with zipcontainer.CreateZipSegment(u"information.turtle/%08d" % turtleContainerIndex) as turtle_segment:
turtle_segment.compression_method = ZIP_DEFLATE
turtle_segment.write(utils.SmartStr(current_triples_txt))
turtle_segment.Flush()
turtle_segment.Close()
with zipcontainer.CreateZipSegment(u"information.turtle") as turtle_segment:
turtle_segment.compression_method = ZIP_DEFLATE
turtle_segment.write(utils.SmartStr(directives_txt + u"\r\n\r\n"))
turtleContainerIndex = 0
while True:
turtleARN = escaping.urn_from_member_name(u"information.turtle/%08d" % turtleContainerIndex,
zipcontainer.urn, zipcontainer.version)
if zipcontainer.ContainsMember(turtleARN):
with zipcontainer.OpenZipSegment(
u"information.turtle/%08d" % turtleContainerIndex) as turtle_chunk_segment:
turtle_chunk_txt = utils.SmartUnicode(streams.ReadAll(turtle_chunk_segment))
turtle_segment.write(utils.SmartStr(turtle_chunk_txt + u"\r\n"))
turtleContainerIndex += 1
else:
break
turtle_segment.Flush()
turtle_segment.Close()
def DumpToTurtle(self, zipcontainer):
infoARN = escaping.urn_from_member_name(u"information.turtle", zipcontainer.urn, zipcontainer.version)
if not zipcontainer.ContainsMember(infoARN):
with zipcontainer.CreateZipSegment(u"information.turtle") as turtle_segment:
turtle_segment.compression_method = ZIP_DEFLATE
result = self._DumpToTurtle(zipcontainer.urn)
turtle_segment.write(utils.SmartStr(result))
turtle_segment.Flush()
turtle_segment.Close()
else:
# append to an existng container
self.invalidateCachedMetadata(zipcontainer)
explodedTurtleDirectivesARN = escaping.urn_from_member_name(u"information.turtle/directives", zipcontainer.urn, zipcontainer.version)
if not zipcontainer.ContainsMember(explodedTurtleDirectivesARN):
# this is the first append operation. Create the chunked turtle structures
with zipcontainer.OpenZipSegment(u"information.turtle") as turtle_segment:
currentturtle = utils.SmartUnicode(streams.ReadAll(turtle_segment))
(directives_txt, triples_txt) = turtle.toDirectivesAndTripes(currentturtle)
with zipcontainer.CreateZipSegment(u"information.turtle/directives") as directives_segment:
directives_segment.compression_method = ZIP_DEFLATE
directives_segment.write(utils.SmartStr(directives_txt))
directives_segment.Flush()
directives_segment.Close()
with zipcontainer.CreateZipSegment(u"information.turtle/%08d" % 0) as turtle_chunk_segment:
turtle_chunk_segment.compression_method = ZIP_DEFLATE
turtle_chunk_segment.write(utils.SmartStr(triples_txt))
turtle_chunk_segment.Flush()
self.Close(turtle_chunk_segment)
turtle_segment.Close()
(current_directives_txt, current_triples_txt) = turtle.toDirectivesAndTripes(utils.SmartUnicode(self._DumpToTurtle(zipcontainer.urn)))
directives_difference = turtle.difference(directives_txt, current_directives_txt)
if not len(directives_difference) == 0:
directives_txt = directives_txt + u"\r\n" + directives_difference
with zipcontainer.CreateZipSegment(u"information.turtle/directives") as directives_segment:
directives_segment.compression_method = ZIP_DEFLATE
directives_segment.write(utils.SmartStr(directives_txt))
directives_segment.Flush()
directives_segment.Close()
current_turtle_chunk_arn = rdfvalue.URN(u"%s/information.turtle/%08d" % (zipcontainer.urn, 1))
with zipcontainer.CreateMember(current_turtle_chunk_arn) as turtle_segment:
turtle_segment.compression_method = ZIP_DEFLATE
turtle_segment.write(utils.SmartStr(current_triples_txt))
turtle_segment.Flush()
self.Close(turtle_segment)
with zipcontainer.CreateZipSegment(u"information.turtle") as turtle_segment:
turtle_segment.compression_method = ZIP_DEFLATE
turtle_segment.write(utils.SmartStr(directives_txt + "\r\n\r\n"))
turtleContainerIndex = 0
while True:
current_turtle_chunk_arn = rdfvalue.URN(u"%s/information.turtle/%08d" % (zipcontainer.urn, turtleContainerIndex))
if zipcontainer.ContainsMember(current_turtle_chunk_arn):
with zipcontainer.OpenMember(current_turtle_chunk_arn) as turtle_chunk_segment:
turtle_chunk_txt = utils.SmartUnicode(streams.ReadAll(turtle_chunk_segment))
turtle_segment.write(utils.SmartStr(turtle_chunk_txt + u"\r\n"))
turtleContainerIndex += 1
else:
break
turtle_segment.Flush()
turtle_segment.Close()
else:
# more than one append as already occurred
turtleContainerIndex = 0
while True:
turtleARN = escaping.urn_from_member_name(u"information.turtle/%08d" % turtleContainerIndex,
zipcontainer.urn, zipcontainer.version)
if not zipcontainer.ContainsMember(turtleARN):
break
turtleContainerIndex = turtleContainerIndex + 1
with zipcontainer.OpenZipSegment(u"information.turtle/directives") as directives_segment:
directives_txt = utils.SmartUnicode(streams.ReadAll(directives_segment))
(current_directives_txt, current_triples_txt) = turtle.toDirectivesAndTripes(utils.SmartUnicode(self._DumpToTurtle(zipcontainer.urn)))
directives_difference = turtle.difference(directives_txt, current_directives_txt)
if len(directives_difference) > 0:
directives_txt = directives_txt + u"\r\n" + u"\r\n".join(directives_difference)
with zipcontainer.CreateZipSegment(u"information.turtle/directives") as directives_segment:
directives_segment.compression_method = ZIP_DEFLATE
directives_segment.write(utils.SmartStr(directives_txt))
directives_segment.Flush()
directives_segment.Close()
with zipcontainer.CreateZipSegment(u"information.turtle/%08d" % turtleContainerIndex) as turtle_segment:
turtle_segment.compression_method = ZIP_DEFLATE
turtle_segment.write(utils.SmartStr(current_triples_txt))
turtle_segment.Flush()
turtle_segment.Close()
with zipcontainer.CreateZipSegment(u"information.turtle") as turtle_segment:
turtle_segment.compression_method = ZIP_DEFLATE
turtle_segment.write(utils.SmartStr(directives_txt + u"\r\n\r\n"))
turtleContainerIndex = 0
while True:
turtleARN = escaping.urn_from_member_name(u"information.turtle/%08d" % turtleContainerIndex,
zipcontainer.urn, zipcontainer.version)
if zipcontainer.ContainsMember(turtleARN):
with zipcontainer.OpenZipSegment(
u"information.turtle/%08d" % turtleContainerIndex) as turtle_chunk_segment:
turtle_chunk_txt = utils.SmartUnicode(streams.ReadAll(turtle_chunk_segment))
turtle_segment.write(utils.SmartStr(turtle_chunk_txt + u"\r\n"))
turtleContainerIndex += 1
else:
break
turtle_segment.Flush()
turtle_segment.Close()