def files (self):
for chkfile in self.checksums:
try:
# we *need* a httpStat :-[
data= self.perlToPython ("%s/%s/%s" % (self.tempDir, self.repoDir, chkfile))
except IOError, e:
if e.errno!=2:
raise e
else:
logger.warn ('[Ign] %s (%s)' % (chkfile, str (e)))
else:
for filename in data.keys ():
isdir= data[filename].get ('isdir')
if isdir is None or not isdir:
yield ("%s/%s" % (dirname (chkfile), filename), int (data[filename]['size']))
else:
chksum= "%s/%s/CHECKSUMS" % (dirname (chkfile), filename)
# download the CHECKSUM
# BUG: this makes CHECKSUMS to be downloaded twice
filename= ("%(tempDir)s/%(repoDir)s/" % self)+chksum
url= "%s/%s" % (self.repoUrl, chksum)
found= grab (filename, url, cont=False, progress=self.progress)
if found!=0:
raise ProtocolError (proto=url[:url.index (':')].upper (), code=found, url=url)
# add it to checksums
# authors/id comes in chkfile path
# we're sure this must be there
self.checksums.append ( chksum )
def moduleFunc (self):
self.baseDir= self.baseDirTemplate % self
filename= '%(tempDir)s/%(repoDir)s/%(baseDir)s/repodata/repomd.xml' % self
url= '%(repoUrl)s/%(baseDir)s/repodata/repomd.xml' % self
if download:
# download repomd.xml and take from there
code= grab (filename, url,
cont=False, progress=self.progress, verbose=False)
if code!=0:
raise ProtocolError (proto=url[:url.index (':')].upper (), code=code, url=url)
if includeRepomd:
ans.append (('%(baseDir)s/repodata/repomd.xml' % self, True))
logger.debug ('opening '+filename)
repomd= libxml2.parseFile(filename).getRootElement()
repomdChild= repomd.children
while repomdChild:
if repomdChild.name=='data':
dataChild= repomdChild.children
while dataChild:
if dataChild.name=='location':
# only primary.xml is important
filename= dataChild.prop ('href')
filepath= ("%(baseDir)s/" % self)+dataChild.prop ('href')
if filename.endswith ('primary.xml.gz'):
self.primaries[self.arch]= filename
logger.debug ("found primary "+filename)
ans.append ( (filepath, True) )
else:
logger.debug ("found other "+filename)
ans.append ( (filepath, False) )
dataChild= dataChild.next
repomdChild= repomdChild.next
def releaseDatabases (self, download=True):
yield ('authors/00whois.html', False)
# yield ('authors/00whois.xml', True)
yield ('authors/01mailrc.txt.gz', False)
yield ('authors/02STAMP', False)
# CHECKSUMS' *are* databases :-|
# download 00whois.xml and get CHECKSUMS'
filename= '%(tempDir)s/%(repoDir)s/authors/00whois.xml' % self
url= '%(repoUrl)s/authors/00whois.xml' % self
if download:
found= grab (filename, url, cont=False, progress=self.progress)
if found!=0:
raise ProtocolError (proto=url[:url.index (':')].upper (), code=found, url=url)
whois= libxml2.parseFile(filename).getRootElement()
whoisChild= whois.children
while whoisChild:
if whoisChild.name=='cpanid':
dataChild= whoisChild.children
while dataChild:
if dataChild.name=='id':
cpanId= dataChild.content.strip ()
if dataChild.name=='type':
cpanType= dataChild.content.strip ()
dataChild= dataChild.next
if cpanType=='author':
# authors/id/A/AB/ABIGAIL will be the real dir
initial= cpanId[:1]
initials= cpanId[:2]
chksum= "authors/id/%s/%s/%s/CHECKSUMS" % (initial, initials, cpanId)
self.checksums.append (chksum)
yield (chksum, False)
makedirs ("%s/authors/id/%s/%s/%s" % (self.repoDir, initial, initials, cpanId))
# authors/id/A/ABIGAIL and
# authors/id/ABIGAIL will be symlinks to
# authors/id/A/AB/ABIGAIL
# note they're relative ones.
symlink ("%s/%s" % (initials, cpanId),
"%s/authors/id/%s/%s" % (self.repoDir, initial, cpanId), self.verbose)
symlink ("%s/%s/%s" % (initial, initials, cpanId),
"%s/authors/id/%s" % (self.repoDir, cpanId), self.verbose)
whoisChild= whoisChild.next
else:
for chksum in self.checksums:
yield (chksum, False)
def getReleaseDatabases (self):
"""
Downloads the database for the whole release
"""
# get the databases relative paths
databases= self.releaseDatabases ()
logger.debug (databases)
logger.info ('------ getting databases for %(repo)s/%(distro)s/%(release)s' % self)
for (database, critic) in databases:
# yes: per design, we don't follow the dry_run option here,
# but the databases won't be swapped at the end either.
dababaseFilename= ("%(tempDir)s/%(repoDir)s/" % self)+database
databaseUrl= ("%(repoUrl)s/" % self)+database
found= grab (dababaseFilename, databaseUrl,
limit=self.limit, progress=self.progress, cont=False, verbose=False)
logger.debug ("%s grabbed" % dababaseFilename)
if found!=0 and critic:
raise ProtocolError (proto=databaseUrl[:databaseUrl.index (':')].upper (), code=found, url=databaseUrl)
if s.st_size<size:
logger.warn ("%s: wrong size %d; should be %d" % (_file, s.st_size, size))
get= True
elif s.st_size>size:
# bigger? it cannot be bigger! reget!
logger.warn ("%s: wrong size %d; should be %d." % (_file, s.st_size, size))
logger.warn ("bigger means something went wrong. deleting and downloading.")
os.unlink (_file)
get= True
logger.debug ("getPackage(): %s, %d, get: %s, reget: %s" % (_file, size, get, reget))
if get:
logger.debug ("getting %s" % _file)
if not self.dry_run:
if not self.experiment:
ans= grab (_file, url, limit=self.limit, progress=self.progress, reget=reget)
else:
touch (_file, size)
ans= 0
if size is None:
try:
s= os.stat (_file)
except OSError, e:
if e.errno==errno.ENOENT:
size= 0
else:
size= s.st_size
logger.debug ("size: %d" % size)
self.downloadedSize+= size