def write_resource_list(self,
paths=None,
outfile=None,
links=None,
dump=None):
"""Write a Resource List or a Resource Dump for files on local disk
Set of resources included is based on paths setting or else the mappings.
Optionally links can be added. Output will be to stdout unless outfile
is specified.
If dump is true then a Resource Dump is written instead of a Resource
List. If outfile is not set then self.default_resource_dump will be used.
"""
rl = self.build_resource_list(paths=paths, set_path=dump)
if (links is not None):
rl.ln = links
if (dump):
if (outfile is None):
outfile = self.default_resource_dump
self.logger.info("Writing resource dump to %s..." % (dump))
d = Dump(format=self.dump_format)
d.write(resource_list=rl, dumpfile=outfile)
else:
if (outfile is None):
try:
print rl.as_xml()
except ListBaseIndexError as e:
raise ClientFatalError(
"%s. Use --output option to specify base name for output files."
% str(e))
else:
rl.write(basename=outfile)
def write_dump_if_requested(self,inventory,dump):
if (dump is None):
return
if (self.verbose):
print "Writing dump to %s..." % (dump)
d = Dump(format=self.dump_format)
d.write(inventory=inventory,dumpfile=dump)
def write_resource_list(self,paths=None,outfile=None,links=None,dump=None):
"""Write a Resource List or a Resource Dump for files on local disk
Set of resources included is based on paths setting or else the mappings.
Optionally links can be added. Output will be to stdout unless outfile
is specified.
If dump is true then a Resource Dump is written instead of a Resource
List. If outfile is not set then self.default_resource_dump will be used.
"""
rl = self.build_resource_list(paths=paths,set_path=dump)
if (links is not None):
rl.ln = links
if (dump):
if (outfile is None):
outfile = self.default_resource_dump
self.logger.info("Writing resource dump to %s..." % (dump))
d = Dump(format=self.dump_format)
d.write(resource_list=rl,dumpfile=outfile)
else:
if (outfile is None):
try:
print rl.as_xml()
except ListBaseIndexError as e:
raise ClientFatalError("%s. Use --output option to specify base name for output files." % str(e))
else:
rl.write(basename=outfile)
def write_dump_if_requested(self, inventory, dump):
if (dump is None):
return
if (self.verbose):
print "Writing dump to %s..." % (dump)
d = Dump(format=self.dump_format)
d.write(inventory=inventory, dumpfile=dump)
def test04_dump_multi_file_max_size(self):
rl = ResourceList()
for letter in map(chr, range(ord('a'), ord('l') + 1)):
uri = 'http://ex.org/%s' % (letter)
fname = 'tests/testdata/a_to_z/%s' % (letter)
rl.add(Resource(uri, path=fname))
self.assertEqual(len(rl), 12)
d2 = Dump(rl)
tmpbase = os.path.join(self.tmpdir, 'test0f_')
d2.max_size = 2000
n = d2.write(tmpbase)
self.assertEqual(n, 2, 'expect to write 2 dump files')
self.assertTrue(os.path.isfile(tmpbase + '00000.zip'))
self.assertTrue(os.path.isfile(tmpbase + '00001.zip'))
# Look at the first file in detail
zipf = tmpbase + '00000.zip'
zo = zipfile.ZipFile(zipf, 'r')
self.assertEqual(zo.namelist(),
['manifest.xml', 'a', 'b', 'c', 'd', 'e', 'f'])
#self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 )
self.assertEqual(zo.getinfo('a').file_size, 9)
self.assertEqual(zo.getinfo('b').file_size, 1116)
self.assertEqual(zo.getinfo('c').file_size, 32)
self.assertEqual(zo.getinfo('d').file_size, 13)
self.assertEqual(zo.getinfo('e').file_size, 20)
self.assertEqual(zo.getinfo('f').file_size, 1625)
zo.close()
os.unlink(zipf)
# Check second and third files have expected contents
zipf = tmpbase + '00001.zip'
zo = zipfile.ZipFile(zipf, 'r')
self.assertEqual(zo.namelist(),
['manifest.xml', 'g', 'h', 'i', 'j', 'k', 'l'])
zo.close()
os.unlink(zipf)
def test04_dump_multi_file_max_size(self):
rl=ResourceList()
for letter in map(chr,xrange(ord('a'),ord('l')+1)):
uri='http://ex.org/%s' % (letter)
fname='resync/test/testdata/a_to_z/%s' % (letter)
rl.add( Resource(uri, path=fname) )
self.assertEqual( len(rl), 12 )
d2=Dump(rl)
tmpbase=os.path.join(self.tmpdir,'test0f_')
d2.max_size=2000
n=d2.write(tmpbase)
self.assertEqual( n, 2, 'expect to write 2 dump files' )
self.assertTrue( os.path.isfile(tmpbase+'00000.zip') )
self.assertTrue( os.path.isfile(tmpbase+'00001.zip') )
# Look at the first file in detail
zipf=tmpbase+'00000.zip'
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( zo.namelist(), ['manifest.xml','a','b','c','d','e','f'] )
#self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 )
self.assertEqual( zo.getinfo('a').file_size, 9 )
self.assertEqual( zo.getinfo('b').file_size, 1116 )
self.assertEqual( zo.getinfo('c').file_size, 32 )
self.assertEqual( zo.getinfo('d').file_size, 13 )
self.assertEqual( zo.getinfo('e').file_size, 20 )
self.assertEqual( zo.getinfo('f').file_size, 1625 )
zo.close()
os.unlink(zipf)
# Check second and third files have expected contents
zipf=tmpbase+'00001.zip'
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( zo.namelist(), ['manifest.xml','g','h','i','j','k','l'] )
zo.close()
os.unlink(zipf)
def test03_dump_multi_file_max_size(self):
rl = ResourceList()
for letter in map(chr, range(ord('a'), ord('l') + 1)):
uri = 'http://ex.org/%s' % (letter)
fname = 'tests/testdata/a_to_z/%s' % (letter)
rl.add(Resource(uri, path=fname))
self.assertEqual(len(rl), 12)
#d=Dump(rl)
#tmpdir=tempfile.mkdtemp()
#tmpbase=os.path.join(tmpdir,'base')
#d.max_size=2000 # start new zip after size exceeds 2000 bytes
#n=d.write(tmpbase)
#self.assertEqual( n, 2, 'expect to write 2 dump files' )
#
# Now repeat with large size limit but small number of files limit
d2 = Dump(rl)
tmpbase = os.path.join(self.tmpdir, 'test03_')
d2.max_files = 4
n = d2.write(tmpbase)
self.assertEqual(n, 3, 'expect to write 3 dump files')
self.assertTrue(os.path.isfile(tmpbase + '00000.zip'))
self.assertTrue(os.path.isfile(tmpbase + '00001.zip'))
self.assertTrue(os.path.isfile(tmpbase + '00002.zip'))
# Look at the first file in detail
zipf = tmpbase + '00000.zip'
zo = zipfile.ZipFile(zipf, 'r')
self.assertEqual(zo.namelist(), ['manifest.xml', 'a', 'b', 'c', 'd'])
#self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 )
self.assertEqual(zo.getinfo('a').file_size, 9)
self.assertEqual(zo.getinfo('b').file_size, 1116)
self.assertEqual(zo.getinfo('c').file_size, 32)
self.assertEqual(zo.getinfo('d').file_size, 13)
zo.close()
os.unlink(zipf)
# Check second and third files have expected contents
zipf = tmpbase + '00001.zip'
zo = zipfile.ZipFile(zipf, 'r')
self.assertEqual(zo.namelist(), ['manifest.xml', 'e', 'f', 'g', 'h'])
zo.close()
os.unlink(zipf)
zipf = tmpbase + '00002.zip'
zo = zipfile.ZipFile(zipf, 'r')
self.assertEqual(zo.namelist(), ['manifest.xml', 'i', 'j', 'k', 'l'])
zo.close()
os.unlink(zipf)
def test03_dump_multi_file_max_size(self):
rl=ResourceList()
for letter in map(chr,xrange(ord('a'),ord('l')+1)):
uri='http://ex.org/%s' % (letter)
fname='resync/test/testdata/a_to_z/%s' % (letter)
rl.add( Resource(uri, path=fname) )
self.assertEqual( len(rl), 12 )
#d=Dump(rl)
#tmpdir=tempfile.mkdtemp()
#tmpbase=os.path.join(tmpdir,'base')
#d.max_size=2000 # start new zip after size exceeds 2000 bytes
#n=d.write(tmpbase)
#self.assertEqual( n, 2, 'expect to write 2 dump files' )
#
# Now repeat with large size limit but small number of files limit
d2=Dump(rl)
tmpbase=os.path.join(self.tmpdir,'test03_')
d2.max_files=4
n=d2.write(tmpbase)
self.assertEqual( n, 3, 'expect to write 3 dump files' )
self.assertTrue( os.path.isfile(tmpbase+'00000.zip') )
self.assertTrue( os.path.isfile(tmpbase+'00001.zip') )
self.assertTrue( os.path.isfile(tmpbase+'00002.zip') )
# Look at the first file in detail
zipf=tmpbase+'00000.zip'
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( zo.namelist(), ['manifest.xml','a','b','c','d'] )
#self.assertEqual( zo.getinfo('manifest.xml').file_size, 470 )
self.assertEqual( zo.getinfo('a').file_size, 9 )
self.assertEqual( zo.getinfo('b').file_size, 1116 )
self.assertEqual( zo.getinfo('c').file_size, 32 )
self.assertEqual( zo.getinfo('d').file_size, 13 )
zo.close()
os.unlink(zipf)
# Check second and third files have expected contents
zipf=tmpbase+'00001.zip'
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( zo.namelist(), ['manifest.xml','e','f','g','h'] )
zo.close()
os.unlink(zipf)
zipf=tmpbase+'00002.zip'
zo=zipfile.ZipFile(zipf,'r')
self.assertEqual( zo.namelist(), ['manifest.xml','i','j','k','l'] )
zo.close()
os.unlink(zipf)
def write_dump_if_requested(self, inventory, dump):
if dump is None:
return
self.logger.info("Writing dump to %s..." % (dump))
d = Dump(format=self.dump_format)
d.write(inventory=inventory, dumpfile=dump)
def __write_dump__():
rl = __create_resourcelist__("redump")
d = Dump(resources=rl)
d.write(basename="rs/source/redump/rd_")
def write_dump_if_requested(self,resourcelist,dump):
if (dump is None):
return
self.logger.info("Writing dump to %s..." % (dump))
d = Dump(format=self.dump_format)
d.write(resourcelist=resourcelist,dumpfile=dump)
def __write_dump__():
rl = __create_resourcelist__("redump")
d = Dump(resources=rl)
d.write(basename="rs/source/redump/rd_")
def write_dump_if_requested(self, inventory, dump):
if (dump is None):
return
self.logger.info("Writing dump to %s..." % (dump))
d = Dump(format=self.dump_format)
d.write(inventory=inventory, dumpfile=dump)