def publish_metadata(self, new_zips, exluded_zip=None):
"""
(Re)publish metadata with addition of new_zips. An excluded zip will be removed from previously published
metadata.
:param new_zips: a resourcelist with newly created zip resources
:param exluded_zip: local path to zip file that will be removed from previously published metadata.
"""
rs_dump_url = self.publish_url + RS_RESOURCE_DUMP_XML
rs_dump_path = os.path.join(self.publish_dir, RS_RESOURCE_DUMP_XML)
capa_list_url = self.publish_url + RS_CAPABILITY_LIST_XML
capa_list_path = os.path.join(self.publish_dir, RS_CAPABILITY_LIST_XML)
rs_dump = ResourceDump()
# Load existing resource-dump, if any. Else set start time.
if os.path.isfile(rs_dump_path):
with open(rs_dump_path, "r") as rs_dump_file:
sm = Sitemap()
sm.parse_xml(rs_dump_file, resources=rs_dump)
else:
rs_dump.md_at = w3cdt.datetime_to_str(no_fractions=True)
rs_dump.link_set(rel="up", href=capa_list_url)
# Remove excluded zip, if any
if exluded_zip:
loc = self.publish_url + os.path.basename(exluded_zip)
if loc in rs_dump.resources:
del rs_dump.resources[loc]
else:
raise RuntimeError("Could not find %s in %s" %
(loc, rs_dump_path))
# Add new zips
for resource in new_zips:
rs_dump.add(resource)
# Write resource-dump.xml
rs_dump.md_completed = w3cdt.datetime_to_str(no_fractions=True)
with open(rs_dump_path, "w") as rs_dump_file:
rs_dump_file.write(rs_dump.as_xml())
# There are several ways to decode base64, among them
# iri = base64.b64decode(os.path.basename(self.publish_dir)).rstrip('\n')
# iri = base64.b64decode(os.path.basename(self.publish_dir), '-_').rstrip('\n')
iri = base64.urlsafe_b64decode(os.path.basename(
self.publish_dir)).rstrip('\n')
print "New %s for graph %s" % (RS_RESOURCE_DUMP_XML, iri)
print "See %s" % rs_dump_url
# Write capability-list.xml
if not os.path.isfile(capa_list_path):
capa_list = CapabilityList()
capa_list.link_set(rel="up", href=self.src_desc_url)
capa_list.add_capability(rs_dump, rs_dump_url)
with open(capa_list_path, "w") as capa_list_file:
capa_list_file.write(capa_list.as_xml())
print "New %s. See %s" % (RS_CAPABILITY_LIST_XML, capa_list_url)
def test02_resourcelist(self):
rl = ResourceList()
caps = CapabilityList()
caps.add_capability( rl, "http://example.org/resourcelist.xml" )
caps.md['from'] = "2013-02-07T22:39:00"
self.assertEqual( len(caps), 1 )
self.assertEqual( caps.as_xml(), '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/"><rs:md capability="capabilitylist" from="2013-02-07T22:39:00" /><url><loc>http://example.org/resourcelist.xml</loc><rs:md capability="resourcelist" /></url></urlset>' )
def test01_resourcelist(self):
rl = ResourceList()
caps = CapabilityList()
caps.add_capability( rl, "http://example.org/resourcelist.xml" )
caps.md['modified'] = "2013-02-07T22:39:00"
self.assertEqual( len(caps), 1 )
self.assertEqual( caps.as_xml(), '<?xml version=\'1.0\' encoding=\'UTF-8\'?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:rs="http://www.openarchives.org/rs/terms/"><rs:md capability="capabilitylist" modified="2013-02-07T22:39:00" /><url><loc>http://example.org/resourcelist.xml</loc><rs:md capability="resourcelist" /></url></urlset>' )
def publish_metadata(self, new_zips, exluded_zip=None):
"""
(Re)publish metadata with addition of new_zips. An excluded zip will be removed from previously published
metadata.
:param new_zips: a resourcelist with newly created zip resources
:param exluded_zip: local path to zip file that will be removed from previously published metadata.
"""
rs_dump_url = self.publish_url + RS_RESOURCE_DUMP_XML
rs_dump_path = os.path.join(self.publish_dir, RS_RESOURCE_DUMP_XML)
capa_list_url = self.publish_url + RS_CAPABILITY_LIST_XML
capa_list_path = os.path.join(self.publish_dir, RS_CAPABILITY_LIST_XML)
rs_dump = ResourceDump()
# Load existing resource-dump, if any. Else set start time.
if os.path.isfile(rs_dump_path):
with open(rs_dump_path, "r") as rs_dump_file:
sm = Sitemap()
sm.parse_xml(rs_dump_file, resources=rs_dump)
else:
rs_dump.md_at = w3cdt.datetime_to_str(no_fractions=True)
rs_dump.link_set(rel="up", href=capa_list_url)
# Remove excluded zip, if any
if exluded_zip:
loc = self.publish_url + os.path.basename(exluded_zip)
if loc in rs_dump.resources:
del rs_dump.resources[loc]
else:
raise RuntimeError("Could not find %s in %s" % (loc, rs_dump_path))
# Add new zips
for resource in new_zips:
rs_dump.add(resource)
# Write resource-dump.xml
rs_dump.md_completed = w3cdt.datetime_to_str(no_fractions=True)
with open(rs_dump_path, "w") as rs_dump_file:
rs_dump_file.write(rs_dump.as_xml())
# There are several ways to decode base64, among them
# iri = base64.b64decode(os.path.basename(self.publish_dir)).rstrip('\n')
# iri = base64.b64decode(os.path.basename(self.publish_dir), '-_').rstrip('\n')
iri = base64.urlsafe_b64decode(os.path.basename(self.publish_dir)).rstrip("\n")
print "New %s for graph %s" % (RS_RESOURCE_DUMP_XML, iri)
print "See %s" % rs_dump_url
# Write capability-list.xml
if not os.path.isfile(capa_list_path):
capa_list = CapabilityList()
capa_list.link_set(rel="up", href=self.src_desc_url)
capa_list.add_capability(rs_dump, rs_dump_url)
with open(capa_list_path, "w") as capa_list_file:
capa_list_file.write(capa_list.as_xml())
print "New %s. See %s" % (RS_CAPABILITY_LIST_XML, capa_list_url)
def test_build_ex_06(self):
"""Simple Capability List document """
cl = CapabilityList()
cl.describedby = 'http://example.com/info_about_set1_of_resources.xml'
cl.up = 'http://example.com/resourcesync_description.xml'
cl.add_capability( uri='http://example.com/dataset1/resourcelist.xml', name='resourcelist' )
cl.add_capability( uri='http://example.com/dataset1/resourcedump.xml', name='resourcedump' )
cl.add_capability( uri='http://example.com/dataset1/changelist.xml', name='changelist' )
ex_xml = self._open_ex('resourcesync_ex_6').read()
self._assert_xml_equal( cl.as_xml(), ex_xml )
def test02_multiple(self):
caps = CapabilityList()
rl = ResourceList()
caps.add_capability( rl, "rl.xml" )
cl = ChangeList()
caps.add_capability( cl, "cl.xml" )
self.assertEqual( len(caps), 2 )
xml = caps.as_xml()
self.assertTrue( re.search( r'<loc>rl.xml</loc><rs:md capability="resourcelist" />', xml ) )
self.assertTrue( re.search( r'<loc>cl.xml</loc><rs:md capability="changelist" />', xml) )
def get(self):
capability_list = CapabilityList()
capability_list.describedby = self.source.describedby_uri
capability_list.add_capability(uri=self.source.resource_list_builder.uri,
name='resourcelist')
if self.source.has_changememory:
capability_list.add_capability(uri=self.source.changememory.base_uri,
name='changelist')
self.set_header("Content-Type", "application/xml")
self.write(capability_list.as_xml())
def test03_multiple(self):
caps = CapabilityList()
rl = ResourceList()
caps.add_capability( rl, "rl.xml" )
cl = ChangeList()
caps.add_capability( cl, "cl.xml" )
self.assertEqual( len(caps), 2 )
xml = caps.as_xml()
self.assertTrue( re.search( r'<loc>rl.xml</loc><rs:md capability="resourcelist" />', xml ) )
self.assertTrue( re.search( r'<loc>cl.xml</loc><rs:md capability="changelist" />', xml) )
def test_build_ex_13(self):
"""Capability List document with 4 entries"""
cl = CapabilityList()
cl.describedby = 'http://example.com/info_about_set1_of_resources.xml'
cl.up = 'http://example.com/resourcesync_description.xml'
cl.add_capability( capability=ResourceList( uri='http://example.com/dataset1/resourcelist.xml' ) )
cl.add_capability( capability=ResourceDump( uri='http://example.com/dataset1/resourcedump.xml' ) )
cl.add_capability( capability=ChangeList( uri='http://example.com/dataset1/changelist.xml' ) )
cl.add_capability( capability=ChangeDump( uri='http://example.com/dataset1/changedump.xml' ) )
ex_xml = self._open_ex('resourcesync_ex_13').read()
self._assert_xml_equal( cl.as_xml(), ex_xml )
def write_capability_list(self,capabilities=None,outfile=None,links=None):
"""Write a Capability List to outfile or STDOUT"""
capl = CapabilityList(ln=links)
if (capabilities is not None):
for name in capabilities.keys():
capl.add_capability(name=name, uri=capabilities[name])
kwargs = { 'pretty_xml': True }
if (outfile is None):
print capl.as_xml(**kwargs)
else:
capl.write(basename=outfile,**kwargs)
def get(self):
"""Implement GET for Capability List."""
capability_list = CapabilityList()
capability_list.describedby = self.source.describedby_uri
capability_list.up = self.source.source_description_uri
capability_list.add_capability(
uri=self.source.resource_list_builder.uri, name='resourcelist')
if self.source.has_changememory:
capability_list.add_capability(
uri=self.source.changememory.base_uri, name='changelist')
self.set_header("Content-Type", "application/xml")
self.write(capability_list.as_xml())
def write_capability_list(self,
capabilities=None,
outfile=None,
links=None):
"""Write a Capability List to outfile or STDOUT"""
capl = CapabilityList(ln=links)
capl.pretty_xml = self.pretty_xml
if (capabilities is not None):
for name in capabilities.keys():
capl.add_capability(name=name, uri=capabilities[name])
if (outfile is None):
print capl.as_xml()
else:
capl.write(basename=outfile)
raw_ts[10:12] + ":" + raw_ts[12:14] + "Z" ) timestamps.append(ts) rl.add(Resource(args.resource_url + filename, lastmod=ts)) # Print to file at args.resource_dir + "/resource-list.xml" resource_list_file = open(args.resource_dir + "/resource-list.xml", "w") resource_list_file.write(rl.as_xml()) resource_list_file.close() print "Wrote resource list to: " + args.resource_dir + "/resource-list.xml" timestamps.sort() caps = CapabilityList() caps.add_capability(rl, args.resource_url + "resource-list.xml") if len(timestamps) > 0: caps.md['from'] = timestamps[0] # Print to file at args.resource_dir + "/capability-list.xml" capability_list_file = open(args.resource_dir + "/capability-list.xml", "w") capability_list_file.write(caps.as_xml()) capability_list_file.close() print "Wrote capability list to: " + args.resource_dir + "/capability-list.xml" rsd = SourceDescription() rsd.md_at = None rsd.add_capability_list(args.resource_url + "capability-list.xml") # Print to file at args.resource_dir + "/resourcesync"
_, raw_ts = filename.split("-")
ts = (raw_ts[:4] + "-" + raw_ts[4:6] + "-" + raw_ts[6:8] + "T" +
raw_ts[8:10] + ":" + raw_ts[10:12] + ":" + raw_ts[12:14] + "Z")
timestamps.append(ts)
rl.add(Resource(args.resource_url + filename, lastmod=ts))
# Print to file at args.resource_dir + "/resource-list.xml"
resource_list_file = open(args.resource_dir + "/resource-list.xml", "w")
resource_list_file.write(rl.as_xml())
resource_list_file.close()
print "Wrote resource list to: " + args.resource_dir + "/resource-list.xml"
timestamps.sort()
caps = CapabilityList()
caps.add_capability(rl, args.resource_url + "resource-list.xml")
if len(timestamps) > 0:
caps.md['from'] = timestamps[0]
# Print to file at args.resource_dir + "/capability-list.xml"
capability_list_file = open(args.resource_dir + "/capability-list.xml", "w")
capability_list_file.write(caps.as_xml())
capability_list_file.close()
print "Wrote capability list to: " + args.resource_dir + "/capability-list.xml"
rsd = SourceDescription()
rsd.md_at = None
rsd.add_capability_list(args.resource_url + "capability-list.xml")
# Print to file at args.resource_dir + "/resourcesync"