コード例 #1
0
def open_file(filename, istext=True, makebackup=False):
    """This returns an open file object which can be used for writing. This
    file is created in the output directory. The output directory (stored in
    config.OUTPUT_DIR is created if it does not yet exist. If the second
    parameter is True (default) the file is opened as an UTF-8 text file."""
    import os
    # check if output directory exists and create it if needed
    if not os.path.isdir(config.OUTPUT_DIR):
        try:
            os.mkdir(config.OUTPUT_DIR)
        except OSError, (errno, strerror):
            debugio.error('error creating directory %(dir)s: %(strerror)s' %
                          { 'dir': config.OUTPUT_DIR,
                            'strerror': strerror })
            sys.exit(1)
コード例 #2
0
def open_file(filename, istext=True, makebackup=False):
    """This returns an open file object which can be used for writing. This
    file is created in the output directory. The output directory (stored in
    config.OUTPUT_DIR is created if it does not yet exist. If the second
    parameter is True (default) the file is opened as an UTF-8 text file."""
    import os
    # check if output directory exists and create it if needed
    if not os.path.isdir(config.OUTPUT_DIR):
        try:
            os.mkdir(config.OUTPUT_DIR)
        except OSError, (errno, strerror):
            debugio.error('error creating directory %(dir)s: %(strerror)s' % {
                'dir': config.OUTPUT_DIR,
                'strerror': strerror
            })
            sys.exit(1)
コード例 #3
0
def install_file(source, text=False):
    """Install the given file in the output directory.
    If the text flag is set to true it is assumed the file is text,
    translating line endings."""
    import shutil
    import urlparse
    # figure out mode to open the file with
    mode = 'r'
    if text:
        mode += 'U'
    # check with what kind of argument we are called
    scheme = urlparse.urlsplit(source)[0]
    if scheme == 'file':
        # this is a file:/// url, translate to normal path and open
        import urllib
        source = urllib.url2pathname(urlparse.urlsplit(source)[2])
    elif scheme == '' and os.path.isabs(source):
        # this is an absolute path, just open it as is
        pass
    elif scheme == '':
        # this is a relavite path, try to fetch it from the python path
        for directory in sys.path:
            tst = os.path.join(directory, source)
            if os.path.isfile(tst):
                source = tst
                break
    # TODO: support more schemes here
    # figure out the destination name
    target = os.path.join(config.OUTPUT_DIR, os.path.basename(source))
    # test if source and target are the same
    source = os.path.realpath(source)
    if source == os.path.realpath(target):
        debugio.warn('attempt to overwrite %(fname)s with itself' %
                     {'fname': source})
        return
    # open the input file
    sfp = None
    try:
        sfp = open(source, mode)
    except IOError, (errno, strerror):
        debugio.error('%(fname)s: %(strerror)s' % {
            'fname': source,
            'strerror': strerror
        })
        sys.exit(1)
コード例 #4
0
def main():
    """Main program."""
    site = crawler.Site()
    # parse command-line arguments
    parse_args(site)
    # read serialized file
    if config.CONTINUE:
        fname = os.path.join(config.OUTPUT_DIR, 'webcheck.dat')
        debugio.info('reading stored crawler data....')
        try:
            fp = open(fname, 'r')
            site = serialize.deserialize(fp)
            fp.close()
        except IOError, (errno, strerror):
            debugio.error('%(fname)s: %(strerror)s' % {
                'fname': fname,
                'strerror': strerror
            })
            sys.exit(1)
        debugio.info('done.')
コード例 #5
0
            res = raw_input('webcheck: overwrite %s? [y]es, [a]ll, [q]uit: ' % fname)
            res = res.lower() + ' '
            if res[0] == 'a':
                config.OVERWRITE_FILES = True
            elif res[0] != 'y':
                print 'Aborted.'
                sys.exit(1)
    # open the file for writing
    try:
        if istext:
            return open(fname, 'w')
        else:
            return open(fname, 'wb')
    except IOError, (errno, strerror):
        debugio.error('error creating output file %(fname)s: %(strerror)s' %
                      { 'fname': fname,
                        'strerror': strerror })
        sys.exit(1)

def _print_navbar(fp, plugin):
    """Return an html fragement representing the navigation bar for a page."""
    fp.write('  <ul class="navbar">\n')
    for p in config.PLUGINS:
        # import the plugin
        report = __import__('plugins.' + p, globals(), locals(), [p])
        # skip if no outputfile
        if not hasattr(report, '__outputfile__'):
            continue
        # generate a link to the plugin page
        selected = ''
        if report == plugin:
コード例 #6
0
                            fname)
            res = res.lower() + ' '
            if res[0] == 'a':
                config.OVERWRITE_FILES = True
            elif res[0] != 'y':
                print 'Aborted.'
                sys.exit(1)
    # open the file for writing
    try:
        if istext:
            return open(fname, 'w')
        else:
            return open(fname, 'wb')
    except IOError, (errno, strerror):
        debugio.error('error creating output file %(fname)s: %(strerror)s' % {
            'fname': fname,
            'strerror': strerror
        })
        sys.exit(1)


def _print_navbar(fp, plugin):
    """Return an html fragement representing the navigation bar for a page."""
    fp.write('  <ul class="navbar">\n')
    for p in config.PLUGINS:
        # import the plugin
        report = __import__('plugins.' + p, globals(), locals(), [p])
        # skip if no outputfile
        if not hasattr(report, '__outputfile__'):
            continue
        # generate a link to the plugin page
        selected = ''