def print_project_info(self, statsd):
""" Print project information """
nlinks = statsd['links']
nservers = statsd['extservers'] + 1
nfiles = statsd['files']
ndirs = statsd['extdirs'] + 1
numfailed = statsd['failed']
nretried = statsd['retries']
fatal = statsd['fatal']
fetchtime = statsd['fetchtime']
nfilesincache = statsd['filesincache']
nfilesinrepos = statsd['filesinrepos']
nbroken = statsd['broken']
# Bug fix, download time to be calculated
# precisely...
dnldtime = fetchtime
strings = [('link', nlinks), ('server', nservers),
('file', nfiles), ('file', nfilesinrepos),
('directory', ndirs), ('link', numfailed), ('link', fatal),
('link', nretried), ('file', nfilesincache), ('link', nbroken) ]
fns = map(plural, strings)
info(' ')
bytes = self.bytes
savedbytes = self.savedbytes
ratespec='KB/sec'
if bytes and dnldtime:
bps = float(bytes/dnldtime)/1024.0
if bps<1.0:
bps *= 1000.0
ratespec='bytes/sec'
bps = '%.2f' % bps
else:
bps = '0.0'
fetchtime = float((math.modf(fetchtime*100.0)[1])/100.0)
if self._cfg.simulate:
info("HarvestMan crawl simulation of",self._cfg.project,"completed in",fetchtime,"seconds.")
else:
info('HarvestMan mirror',self._cfg.project,'completed in',fetchtime,'seconds.')
if nlinks: info(nlinks,fns[0],'scanned in',nservers,fns[1],'.')
else: info('No links parsed.')
if nfiles: info(nfiles,fns[2],'written.')
else:info('No file written.')
if nfilesinrepos:
info(nfilesinrepos,fns[3],wasOrWere(nfilesinrepos),'already uptodate in the repository for this project and',wasOrWere(nfilesinrepos),'not updated.')
if nfilesincache:
info(nfilesincache,fns[8],wasOrWere(nfilesincache),'updated from the project cache.')
if nbroken: info(nbroken,fns[9],wasOrWere(nbroken),'were broken.')
if fatal: info(fatal,fns[5],'had fatal errors and failed to download.')
if bytes: info(bytes,' bytes received at the rate of',bps,ratespec,'.')
if savedbytes: info(savedbytes,' bytes were written to disk.\n')
info('*** Log Completed ***\n')
# get current time stamp
s=time.localtime()
tz=(time.tzname)[0]
format='%b %d %Y '+tz+' %H:%M:%S'
tstamp=time.strftime(format, s)
if not self._cfg.simulate:
# Write statistics to the crawl database
HarvestManDbManager.add_stats_record(statsd)
logconsole('Done.')
def print_project_info(self, statsd):
""" Print project information """
nlinks = statsd['links']
nservers = statsd['extservers'] + 1
nfiles = statsd['files']
ndirs = statsd['extdirs'] + 1
numfailed = statsd['failed']
nretried = statsd['retries']
fatal = statsd['fatal']
fetchtime = statsd['fetchtime']
nfilesincache = statsd['filesincache']
nfilesinrepos = statsd['filesinrepos']
nbroken = statsd['broken']
# Bug fix, download time to be calculated
# precisely...
dnldtime = fetchtime
strings = [('link', nlinks), ('server', nservers), ('file', nfiles),
('file', nfilesinrepos), ('directory', ndirs),
('link', numfailed), ('link', fatal), ('link', nretried),
('file', nfilesincache), ('link', nbroken)]
fns = map(plural, strings)
info(' ')
bytes = self.bytes
savedbytes = self.savedbytes
ratespec = 'KB/sec'
if bytes and dnldtime:
bps = float(bytes / dnldtime) / 1024.0
if bps < 1.0:
bps *= 1000.0
ratespec = 'bytes/sec'
bps = '%.2f' % bps
else:
bps = '0.0'
fetchtime = float((math.modf(fetchtime * 100.0)[1]) / 100.0)
if self._cfg.simulate:
info("HarvestMan crawl simulation of", self._cfg.project,
"completed in", fetchtime, "seconds.")
else:
info('HarvestMan mirror', self._cfg.project, 'completed in',
fetchtime, 'seconds.')
if nlinks: info(nlinks, fns[0], 'scanned in', nservers, fns[1], '.')
else: info('No links parsed.')
if nfiles: info(nfiles, fns[2], 'written.')
else: info('No file written.')
if nfilesinrepos:
info(nfilesinrepos, fns[3], wasOrWere(nfilesinrepos),
'already uptodate in the repository for this project and',
wasOrWere(nfilesinrepos), 'not updated.')
if nfilesincache:
info(nfilesincache, fns[8], wasOrWere(nfilesincache),
'updated from the project cache.')
if nbroken: info(nbroken, fns[9], wasOrWere(nbroken), 'were broken.')
if fatal:
info(fatal, fns[5], 'had fatal errors and failed to download.')
if bytes:
info(bytes, ' bytes received at the rate of', bps, ratespec, '.')
if savedbytes: info(savedbytes, ' bytes were written to disk.\n')
info('*** Log Completed ***\n')
# get current time stamp
s = time.localtime()
tz = (time.tzname)[0]
format = '%b %d %Y ' + tz + ' %H:%M:%S'
tstamp = time.strftime(format, s)
if not self._cfg.simulate:
# Write statistics to the crawl database
HarvestManDbManager.add_stats_record(statsd)
logconsole('Done.')
