Example #1
0
 def test_build_ex_24(self):
     cl = ChangeList()
     cl.up = "http://example.com/dataset1/capabilitylist.xml"
     cl.md_from = "2013-01-03T00:00:00Z"
     c1 = Resource(uri="http://example.com/res1",
                   lastmod="2013-01-03T18:00:00Z",
                   change="updated",
                   md5="1584abdf8ebdc9802ac0c6a7402c03b6",
                   length=8876,
                   mime_type="text/html")
     # Resource.link_set with add or change link depending on one with 
     # the particular rel exists unless allow_duplicates=True. 
     # Resource.link_add will always add. Test both here...
     c1.link_set(rel="duplicate",
                 href="http://mirror1.example.com/res1",
                 pri="1",
                 modified="2013-01-03T18:00:00Z")
     c1.link_set(rel="duplicate",
                 href="http://mirror2.example.com/res1",
                 pri="2",
                 modified="2013-01-03T18:00:00Z",
                 allow_duplicates=True)
     c1.link_add(rel="duplicate",
                 href="gsiftp://gridftp.example.com/res1",
                 pri="3",
                 modified="2013-01-03T18:00:00Z")
     cl.add( c1 )
     self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_24' )
Example #2
0
 def test_build_ex_27(self):
     cl = ChangeList()
     cl.up = "http://example.com/dataset1/capabilitylist.xml"
     cl.md_from = "2013-01-03T00:00:00Z"
     c1 = Resource(uri="http://example.com/res4",
                   lastmod="2013-01-03T17:00:00Z",
                   change="updated",
                   sha256="f4OxZX_x_DFGFDgghgdfb6rtSx-iosjf6735432nklj",
                   length=56778,
                   mime_type="application/json" )
     c1.link_set(rel="http://www.openarchives.org/rs/terms/patch",
                 href="http://example.com/res4-json-patch",
                 modified="2013-01-03T17:00:00Z",
                 hash="sha-256:y66dER_t_HWEIKpesdkeb7rtSc-ippjf9823742opld", #FIXME - inconsistent
                 length=73,
                 type="application/json-patch")
     c2 = Resource(uri="http://example.com/res5-full.tiff",
                   lastmod="2013-01-03T18:00:00Z",
                   change="updated",
                   sha256="f4OxZX_x_FO5LcGBSKHWXfwtSx-j1ncoSt3SABJtkGk",
                   length="9788456778",
                   mime_type="image/tiff")
     c2.link_set(rel="http://www.openarchives.org/rs/terms/patch",
                 href="http://example.com/res5-diff",
                 modified="2013-01-03T18:00:00Z",
                 hash="sha-256:h986gT_t_87HTkjHYE76G558hY-jdfgy76t55sadJUYT",
                 length=4533,
                 type="application/x-tiff-diff" )
     cl.add( [c1,c2] )
     self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_27' )
Example #3
0
 def test_build_ex_07(self):
     """A Source Description document """
     sd = SourceDescription()
     sd.describedby = 'http://example.com/info-about-source.xml'
     r = Resource( uri='http://example.com/dataset1/capabilitylist.xml',
                   capability='capabilitylist' )
     r.link_set( rel='describedby',
                 href='http://example.com/info_about_set1_of_resources.xml' )
     sd.add( r )
     ex_xml = self._open_ex('resourcesync_ex_7').read()
     self._assert_xml_equal( sd.as_xml(), ex_xml )
Example #4
0
 def test_build_ex_02(self):
     """Slightly more complex Resource List document """
     rl = ResourceList()
     rl.md_at = '2013-01-03T09:00:00Z'
     rl.add( Resource(uri='http://example.com/res1',
                      lastmod='2013-01-02T13:00:00Z',
                      md5='1584abdf8ebdc9802ac0c6a7402c03b6') )
     r2 =  Resource(uri='http://example.com/res2',
                    lastmod='2013-01-02T14:00:00Z',
                    md5='1e0d5cb8ef6ba40c99b14c0237be735e')
     r2.link_set(rel="duplicate",href="http://mirror.example.com/res2")
     rl.add( r2 )
     ex_xml = self._open_ex('resourcesync_ex_2').read()
     self._assert_xml_equal( rl.as_xml(), ex_xml )
Example #5
0
    def add_capability_list(self, capability_list=None):
        """Add a capability list.

        Adds either a CapabiltyList object specified in capability_list
        or else creates a Resource with the URI given in capability_list
        and adds that to the Source Description
        """
        if (hasattr(capability_list, 'uri')):
            r = Resource(uri=capability_list.uri,
                         capability=capability_list.capability_name)
            if (capability_list.describedby is not None):
                r.link_set(rel='describedby', href=capability_list.describedby)
        else:
            r = Resource(uri=capability_list,
                         capability='capabilitylist')
        self.add(r)
Example #6
0
 def test_build_ex_28(self):
     cl = ChangeList()
     cl.up = "http://example.com/dataset1/capabilitylist.xml"
     cl.md_from = "2013-01-03T00:00:00Z"
     c1 = Resource(uri="http://example.com/res2.pdf",
                   lastmod="2013-01-03T18:00:00Z",
                   change="updated",
                   md5="1584abdf8ebdc9802ac0c6a7402c03b6",
                   length=8876,
                   mime_type="application/pdf" )
     c1.link_set(rel="describedby",
                 href="http://example.com/res2_dublin-core_metadata.xml",
                 modified="2013-01-01T12:00:00Z",
                 type="application/xml")
     c2 = Resource(uri="http://example.com/res2_dublin-core_metadata.xml",
                   lastmod="2013-01-03T19:00:00Z",
                   change="updated",
                   mime_type="application/xml")
     c2.link_set(rel="describes",
                 href="http://example.com/res2.pdf",
                 modified="2013-01-03T18:00:00Z",
                 hash="md5:1584abdf8ebdc9802ac0c6a7402c03b6",
                 length="8876",
                 type="application/pdf")
     c2.link_set(rel="profile",
                 href="http://purl.org/dc/elements/1.1/")
     cl.add( [c1,c2] )
     self._assert_xml_equal_ex( cl.as_xml(), 'resourcesync_ex_28' )
Example #7
0
 def test_build_ex_17(self):
     """Resource Dump with 3 entries and some metadata"""
     rd = ResourceDump()
     rd.up='http://example.com/dataset1/capabilitylist.xml'
     rd.md_at="2013-01-03T09:00:00Z"
     rd.md_completed="2013-01-03T09:04:00Z"
     z1 = Resource( uri='http://example.com/resourcedump-part1.zip',
                    mime_type="application/zip",
                    length=4765,
                    md_at="2013-01-03T09:00:00Z",
                    md_completed="2013-01-03T09:02:00Z" )
     z1.link_set( rel="contents",
                  href="http://example.com/resourcedump_manifest-part1.xml",
                  mime_type="application/xml" )
     rd.add( z1 )
     z2 = Resource( uri='http://example.com/resourcedump-part2.zip',
                    mime_type="application/zip",
                    length=9875,
                    md_at="2013-01-03T09:01:00Z",
                    md_completed="2013-01-03T09:03:00Z" )
     z2.link_set( rel="contents",
                  href="http://example.com/resourcedump_manifest-part2.xml",
                  mime_type="application/xml" )
     rd.add( z2 )
     z3 = Resource( uri='http://example.com/resourcedump-part3.zip',
                    mime_type="application/zip",
                    length=2298,
                    md_at="2013-01-03T09:03:00Z",
                    md_completed="2013-01-03T09:04:00Z" )
     z3.link_set( rel="contents",
                  href="http://example.com/resourcedump_manifest-part3.xml",
                  mime_type="application/xml" )
     rd.add( z3 )
     ex_xml = self._open_ex('resourcesync_ex_17').read()
     self._assert_xml_equal( rd.as_xml(), ex_xml )
    def create_zip(self,
                   resourcelist,
                   prefix,
                   write_list=False,
                   write_manifest=True):
        """
        Dump local resources in resourcelist to a zip file with the specified prefix. The index in the zip file name
        will be 1 higher than the last zip file index with the same prefix. A manifest.xml will be included in the
        zip.
        --  The resync.Dump.write_zip method used in this method has the side effect of changing local paths in
            resourcelist into paths relative in zip.
        :param resourcelist: resources to zip
        :param prefix: prefix of the zip file
        :param write_list: True if resourcelist should be written to local disc. Default: False
        :param write_manifest: True if a separate manifest file should be written to disc, False otherwise. Default: True
        :return: the created zip as a resync.Resource.
        """

        md_at = None  # w3cdt.datetime_to_str(no_fractions=True) # attribute gets lost in read > write cycle with resync library.
        index = -1
        zipfiles = sorted(
            glob(os.path.join(self.publish_dir, prefix + "*.zip")))
        if len(zipfiles) > 0:
            last_zip_file = zipfiles[len(zipfiles) - 1]
            basename = os.path.basename(last_zip_file)
            index = int(re.findall('\d+', basename)[0])

        zip_name = "%s%05d" % (prefix, index + 1)
        if (write_list):
            # this is the given resourcelist with local paths. As such it is *not* the resourcedump_manifest.
            rl_file = open(os.path.join(self.publish_dir, zip_name + ".xml"),
                           "w")
            rl_file.write(resourcelist.as_xml())
            rl_file.close()

        zip_path = os.path.join(self.publish_dir, zip_name + ".zip")
        dump = Dump()
        dump.path_prefix = self.resource_dir
        dump.write_zip(resourcelist,
                       zip_path)  # paths in resourcelist will be stripped.
        md_completed = None  # w3cdt.datetime_to_str(no_fractions=True) # attribute gets lost in read > write cycle with resync library.
        #print "Zipped %d resources in %s" % (len(resourcelist), zip_path)

        loc = self.publish_url + zip_name + ".zip"  # mandatory
        lastmod = self.last_modified(resourcelist)  # optional
        md_type = "application/zip"  # recommended
        md_length = os.stat(zip_path).st_size
        md5 = compute_md5_for_file(zip_path)

        zip_resource = Resource(uri=loc,
                                lastmod=lastmod,
                                length=md_length,
                                md5=md5,
                                mime_type=md_type,
                                md_at=md_at,
                                md_completed=md_completed)
        if write_manifest:
            rdm = ResourceDumpManifest(resources=resourcelist.resources)
            rdm_file = open(
                os.path.join(self.publish_dir,
                             PREFIX_MANIFEST + zip_name + ".xml"), "w")
            rdm_url = self.publish_url + PREFIX_MANIFEST + zip_name + ".xml"
            rdm_file.write(rdm.as_xml())
            rdm_file.close()
            zip_resource.link_set(rel="content", href=rdm_url)

        return zip_resource
    def create_zip(self, resourcelist, prefix, write_list=False, write_manifest=True):
        """
        Dump local resources in resourcelist to a zip file with the specified prefix. The index in the zip file name
        will be 1 higher than the last zip file index with the same prefix. A manifest.xml will be included in the
        zip.
        --  The resync.Dump.write_zip method used in this method has the side effect of changing local paths in
            resourcelist into paths relative in zip.
        :param resourcelist: resources to zip
        :param prefix: prefix of the zip file
        :param write_list: True if resourcelist should be written to local disc. Default: False
        :param write_manifest: True if a separate manifest file should be written to disc, False otherwise. Default: True
        :return: the created zip as a resync.Resource.
        """

        md_at = (
            None
        )  # w3cdt.datetime_to_str(no_fractions=True) # attribute gets lost in read > write cycle with resync library.
        index = -1
        zipfiles = sorted(glob(os.path.join(self.publish_dir, prefix + "*.zip")))
        if len(zipfiles) > 0:
            last_zip_file = zipfiles[len(zipfiles) - 1]
            basename = os.path.basename(last_zip_file)
            index = int(re.findall("\d+", basename)[0])

        zip_name = "%s%05d" % (prefix, index + 1)
        if write_list:
            # this is the given resourcelist with local paths. As such it is *not* the resourcedump_manifest.
            rl_file = open(os.path.join(self.publish_dir, zip_name + ".xml"), "w")
            rl_file.write(resourcelist.as_xml())
            rl_file.close()

        zip_path = os.path.join(self.publish_dir, zip_name + ".zip")
        dump = Dump()
        dump.path_prefix = self.resource_dir
        dump.write_zip(resourcelist, zip_path)  # paths in resourcelist will be stripped.
        md_completed = (
            None
        )  # w3cdt.datetime_to_str(no_fractions=True) # attribute gets lost in read > write cycle with resync library.
        # print "Zipped %d resources in %s" % (len(resourcelist), zip_path)

        loc = self.publish_url + zip_name + ".zip"  # mandatory
        lastmod = self.last_modified(resourcelist)  # optional
        md_type = "application/zip"  # recommended
        md_length = os.stat(zip_path).st_size
        md5 = compute_md5_for_file(zip_path)

        zip_resource = Resource(
            uri=loc,
            lastmod=lastmod,
            length=md_length,
            md5=md5,
            mime_type=md_type,
            md_at=md_at,
            md_completed=md_completed,
        )
        if write_manifest:
            rdm = ResourceDumpManifest(resources=resourcelist.resources)
            rdm_file = open(os.path.join(self.publish_dir, PREFIX_MANIFEST + zip_name + ".xml"), "w")
            rdm_url = self.publish_url + PREFIX_MANIFEST + zip_name + ".xml"
            rdm_file.write(rdm.as_xml())
            rdm_file.close()
            zip_resource.link_set(rel="content", href=rdm_url)

        return zip_resource