def open_file(filename, istext=True, makebackup=False):
"""This returns an open file object which can be used for writing. This
file is created in the output directory. The output directory (stored in
config.OUTPUT_DIR is created if it does not yet exist. If the second
parameter is True (default) the file is opened as an UTF-8 text file."""
import os
# check if output directory exists and create it if needed
if not os.path.isdir(config.OUTPUT_DIR):
try:
os.mkdir(config.OUTPUT_DIR)
except OSError, (errno, strerror):
debugio.error('error creating directory %(dir)s: %(strerror)s' %
{ 'dir': config.OUTPUT_DIR,
'strerror': strerror })
sys.exit(1)
def open_file(filename, istext=True, makebackup=False):
"""This returns an open file object which can be used for writing. This
file is created in the output directory. The output directory (stored in
config.OUTPUT_DIR is created if it does not yet exist. If the second
parameter is True (default) the file is opened as an UTF-8 text file."""
import os
# check if output directory exists and create it if needed
if not os.path.isdir(config.OUTPUT_DIR):
try:
os.mkdir(config.OUTPUT_DIR)
except OSError, (errno, strerror):
debugio.error('error creating directory %(dir)s: %(strerror)s' % {
'dir': config.OUTPUT_DIR,
'strerror': strerror
})
sys.exit(1)
def install_file(source, text=False):
"""Install the given file in the output directory.
If the text flag is set to true it is assumed the file is text,
translating line endings."""
import shutil
import urlparse
# figure out mode to open the file with
mode = 'r'
if text:
mode += 'U'
# check with what kind of argument we are called
scheme = urlparse.urlsplit(source)[0]
if scheme == 'file':
# this is a file:/// url, translate to normal path and open
import urllib
source = urllib.url2pathname(urlparse.urlsplit(source)[2])
elif scheme == '' and os.path.isabs(source):
# this is an absolute path, just open it as is
pass
elif scheme == '':
# this is a relavite path, try to fetch it from the python path
for directory in sys.path:
tst = os.path.join(directory, source)
if os.path.isfile(tst):
source = tst
break
# TODO: support more schemes here
# figure out the destination name
target = os.path.join(config.OUTPUT_DIR, os.path.basename(source))
# test if source and target are the same
source = os.path.realpath(source)
if source == os.path.realpath(target):
debugio.warn('attempt to overwrite %(fname)s with itself' %
{'fname': source})
return
# open the input file
sfp = None
try:
sfp = open(source, mode)
except IOError, (errno, strerror):
debugio.error('%(fname)s: %(strerror)s' % {
'fname': source,
'strerror': strerror
})
sys.exit(1)
def main():
"""Main program."""
site = crawler.Site()
# parse command-line arguments
parse_args(site)
# read serialized file
if config.CONTINUE:
fname = os.path.join(config.OUTPUT_DIR, 'webcheck.dat')
debugio.info('reading stored crawler data....')
try:
fp = open(fname, 'r')
site = serialize.deserialize(fp)
fp.close()
except IOError, (errno, strerror):
debugio.error('%(fname)s: %(strerror)s' % {
'fname': fname,
'strerror': strerror
})
sys.exit(1)
debugio.info('done.')
res = raw_input('webcheck: overwrite %s? [y]es, [a]ll, [q]uit: ' % fname)
res = res.lower() + ' '
if res[0] == 'a':
config.OVERWRITE_FILES = True
elif res[0] != 'y':
print 'Aborted.'
sys.exit(1)
# open the file for writing
try:
if istext:
return open(fname, 'w')
else:
return open(fname, 'wb')
except IOError, (errno, strerror):
debugio.error('error creating output file %(fname)s: %(strerror)s' %
{ 'fname': fname,
'strerror': strerror })
sys.exit(1)
def _print_navbar(fp, plugin):
"""Return an html fragement representing the navigation bar for a page."""
fp.write(' <ul class="navbar">\n')
for p in config.PLUGINS:
# import the plugin
report = __import__('plugins.' + p, globals(), locals(), [p])
# skip if no outputfile
if not hasattr(report, '__outputfile__'):
continue
# generate a link to the plugin page
selected = ''
if report == plugin:
fname)
res = res.lower() + ' '
if res[0] == 'a':
config.OVERWRITE_FILES = True
elif res[0] != 'y':
print 'Aborted.'
sys.exit(1)
# open the file for writing
try:
if istext:
return open(fname, 'w')
else:
return open(fname, 'wb')
except IOError, (errno, strerror):
debugio.error('error creating output file %(fname)s: %(strerror)s' % {
'fname': fname,
'strerror': strerror
})
sys.exit(1)
def _print_navbar(fp, plugin):
"""Return an html fragement representing the navigation bar for a page."""
fp.write(' <ul class="navbar">\n')
for p in config.PLUGINS:
# import the plugin
report = __import__('plugins.' + p, globals(), locals(), [p])
# skip if no outputfile
if not hasattr(report, '__outputfile__'):
continue
# generate a link to the plugin page
selected = ''