Exemple #1
0
def insert_files(host, port, names):
    socket = PolledSocket(host, port)
    connection = FCPConnection(socket, True)

    inserts = []
    for name in names:
        client = FCPClient(connection)
        client.in_params.default_fcp_params['MaxRetries'] = 3
        client.in_params.default_fcp_params['PriorityClass'] = 1

        inserts.append(client)
        client.in_params.async = True
        parts = name.split('/')

        client.put_file('CHK@' + '/' + parts[-1], name)
         # Hmmmm... Ugly. Add FCPConnection.wait_until_upload_finishes() ?
        while connection.is_uploading():
            socket.poll()
            time.sleep(.25)

    while min([insert.is_finished() for insert in inserts]) == False:
        if not socket.poll():
            break
        time.sleep(.25)

    uris = []
    for insert in inserts:
        if insert.response == None or len(insert.response) < 2 or insert.response[0] <> 'PutSuccessful':
            uris.append(None)
            continue

        uris.append(insert.response[1]['URI'])

    return uris
Exemple #2
0
def do_freenet_insert(ui_, repo, params, insert_uri, progress_func):
    """ INTERNAL: Helper does the actual insert.

                  Caller must delete TMP_DUMP_DIR!
    """
    default_mime_type = "text/plain" # put_complex_dir() default. Hmmmm.
    if not params['ISWIKI']:
        site_root = os.path.join(repo.root, params['SITE_DIR'])
    else:
        # Because wiki html files have no extension to guess from.
        default_mime_type = 'text/html'

        ui_.status("Dumping wiki as HTML...\n")
        site_root = os.path.join(params['TMP_DIR'], TMP_DUMP_DIR)
        dump_wiki_html(os.path.join(repo.root, params['WIKI_ROOT']),
                       site_root,
                       params['OVERLAYED'])

    ui_.status('Default file: %s\n' % params['SITE_DEFAULT_FILE'])
    ui_.status('Reading files from:\n%s\n' % site_root)

    infos = get_file_infos(site_root)

    try:
        set_index_file(infos, params['SITE_DEFAULT_FILE'])
    except ValueError:
        raise util.Abort("Couldn't read %s" % params['SITE_DEFAULT_FILE'])

    ui_.status('--- files ---\n')

    for info in infos:
        ui_.status('%s %s\n' % (info[0], info[1]))
    ui_.status('---\n')

    if params['DRYRUN']:
        ui_.status('Would have inserted to:\n%s\n' % insert_uri)
        ui_.status('But --dryrun was set.\n')
        return

    client = FCPClient.connect(params['FCP_HOST'],
                               params['FCP_PORT'])
    client.in_params.default_fcp_params['DontCompress'] = False
    client.message_callback = progress_func
    try:
        ui_.status('Inserting to:\n%s\n' % insert_uri)
        try:
            request_uri = client.put_complex_dir(insert_uri, infos,
                                                 default_mime_type)[1]['URI']
            show_request_uri(ui_, params, request_uri)
        except FCPError, err:
            if err.is_code(9): # magick number for collision
                ui_.warn('An update was already inserted on that index.\n'
                         + 'Set a later index with --index and try again.\n')
                raise util.Abort("Key collision.")
            else:
                ui_.warn(str(err) + '\n')
                raise util.Abort("FCP Error")
    finally:
        client.close()
Exemple #3
0
def do_freenet_insert(ui_, repo, params, insert_uri, progress_func):
    """ INTERNAL: Helper does the actual insert.

                  Caller must delete TMP_DUMP_DIR!
    """
    default_mime_type = "text/plain"  # put_complex_dir() default. Hmmmm.
    if not params['ISWIKI']:
        site_root = os.path.join(repo.root, params['SITE_DIR'])
    else:
        # Because wiki html files have no extension to guess from.
        default_mime_type = 'text/html'

        ui_.status("Dumping wiki as HTML...\n")
        site_root = os.path.join(params['TMP_DIR'], TMP_DUMP_DIR)
        dump_wiki_html(os.path.join(repo.root, params['WIKI_ROOT']), site_root,
                       params['OVERLAYED'])

    ui_.status('Default file: %s\n' % params['SITE_DEFAULT_FILE'])
    ui_.status('Reading files from:\n%s\n' % site_root)

    infos = get_file_infos(site_root)

    try:
        set_index_file(infos, params['SITE_DEFAULT_FILE'])
    except ValueError:
        raise util.Abort("Couldn't read %s" % params['SITE_DEFAULT_FILE'])

    ui_.status('--- files ---\n')

    for info in infos:
        ui_.status('%s %s\n' % (info[0], info[1]))
    ui_.status('---\n')

    if params['DRYRUN']:
        ui_.status('Would have inserted to:\n%s\n' % insert_uri)
        ui_.status('But --dryrun was set.\n')
        return

    client = FCPClient.connect(params['FCP_HOST'], params['FCP_PORT'])
    client.in_params.default_fcp_params['DontCompress'] = False
    client.message_callback = progress_func
    try:
        ui_.status('Inserting to:\n%s\n' % insert_uri)
        try:
            request_uri = client.put_complex_dir(insert_uri, infos,
                                                 default_mime_type)[1]['URI']
            show_request_uri(ui_, params, request_uri)
        except FCPError, err:
            if err.is_code(9):  # magick number for collision
                ui_.warn('An update was already inserted on that index.\n' +
                         'Set a later index with --index and try again.\n')
                raise util.Abort("Key collision.")
            else:
                ui_.warn(str(err) + '\n')
                raise util.Abort("FCP Error")
    finally:
        client.close()
Exemple #4
0
def execute_genkey(ui_, params):
    """ Run the genkey command. """
    client = FCPClient.connect(params['FCP_HOST'], params['FCP_PORT'])

    client.message_callback = lambda x, y: None  # silence.
    resp = client.generate_ssk()
    ui_.status(MSG_FMT % (resp[1]['InsertURI'], resp[1]['RequestURI'],
                          resp[1]['InsertURI'].split('/')[0] + '/', "U" +
                          resp[1]['InsertURI'].split('/')[0][1:] + '/NAME/0'))
Exemple #5
0
def genkeypair(fcphost, fcpport):
    """ Generate a keypair 

    :returns: inserturi, requesturi (strings)
    """
    client = FCPClient.connect(fcphost, fcpport)
    client.message_callback = lambda x, y: None  # silence.
    resp = client.generate_ssk()
    return resp[1]['InsertURI'], resp[1]['RequestURI']
Exemple #6
0
def genkeypair(fcphost, fcpport):
    """ Generate a keypair 

    :returns: inserturi, requesturi (strings)
    """
    client = FCPClient.connect(fcphost, fcpport)
    client.message_callback = lambda x, y : None # silence.
    resp = client.generate_ssk()
    return resp[1]['InsertURI'], resp[1]['RequestURI']
Exemple #7
0
def execute_genkey(ui_, params):
    """ Run the genkey command. """
    client = FCPClient.connect(params["FCP_HOST"], params["FCP_PORT"])

    client.message_callback = lambda x, y: None  # silence.
    resp = client.generate_ssk()
    ui_.status(
        MSG_FMT
        % (
            resp[1]["InsertURI"],
            resp[1]["RequestURI"],
            resp[1]["InsertURI"].split("/")[0] + "/",
            "U" + resp[1]["InsertURI"].split("/")[0][1:] + "/NAME/0",
        )
    )
 def checkCHK(self, chk, logical_len, length, data=None):
     print "---"
     print "Checking: ", chk
     # Something is closing the connection?
     resp = FCPClient.connect(FCP_HOST, FCP_PORT).get(chk)
     self.assertTrue(resp[0] == 'AllData')
     print "Length: ", len(resp[2])
     print "Mime_Type: ", resp[1]['Metadata.ContentType']
     if len(resp[2]) != length:
         print "Expected len: %i, got: %i!" % (length, len(resp[2]))
         self.assertTrue(False)
     if not data is None and resp[2][:logical_len] != data:
         print "Data doesn't match! (only showing first 16 bytes below)"
         print "got: ", repr(resp[2][:logical_len][:16])
         print "expected: " , repr(data[:16])
         self.assertTrue(False)
Exemple #9
0
 def checkCHK(self, chk, logical_len, length, data=None):
     print "---"
     print "Checking: ", chk
     # Something is closing the connection?
     resp = FCPClient.connect(FCP_HOST, FCP_PORT).get(chk)
     self.assertTrue(resp[0] == 'AllData')
     print "Length: ", len(resp[2])
     print "Mime_Type: ", resp[1]['Metadata.ContentType']
     if len(resp[2]) != length:
         print "Expected len: %i, got: %i!" % (length, len(resp[2]))
         self.assertTrue(False)
     if not data is None and resp[2][:logical_len] != data:
         print "Data doesn't match! (only showing first 16 bytes below)"
         print "got: ", repr(resp[2][:logical_len][:16])
         print "expected: ", repr(data[:16])
         self.assertTrue(False)
Exemple #10
0
def execute_setup(ui_, host, port, tmp, cfg_file=None):
    """ Run the setup command. """
    def connection_failure(msg):
        """ INTERNAL: Display a warning string. """
        ui_.warn(msg)
        ui_.warn("It looks like your FCP host or port might be wrong.\n")
        ui_.warn("Set them with --fcphost and/or --fcpport and try again.\n")
        raise util.Abort("Connection to FCP server failed.")

    # Fix defaults.
    if host == '':
        host = '127.0.0.1'
    if port == 0:
        port = 9481

    if cfg_file is None:
        cfg_file = os.path.expanduser(DEFAULT_CFG_PATH)

    existing_name = ui_.config('infocalypse', 'cfg_file', None)
    if not existing_name is None:
        existing_name = os.path.expanduser(existing_name)
        ui_.status(MSG_HGRC_SET % existing_name)
        cfg_file = existing_name

    if os.path.exists(cfg_file):
        ui_.status(MSG_CFG_EXISTS % cfg_file)
        raise util.Abort("Refusing to modify existing configuration.")

    tmp = setup_tmp_dir(ui_, tmp)

    if not is_writable(tmp):
        raise util.Abort("Can't write to temp dir: %s\n" % tmp)

    # Test FCP connection.
    timeout_secs = 20
    connection = None
    default_private_key = None
    try:
        ui_.status("Testing FCP connection [%s:%i]...\n" % (host, port))

        connection = FCPConnection(PolledSocket(host, port))

        started = time.time()
        while (not connection.is_connected()
               and time.time() - started < timeout_secs):
            connection.socket.poll()
            time.sleep(.25)

        if not connection.is_connected():
            connection_failure(("\nGave up after waiting %i secs for an " +
                                "FCP NodeHello.\n") % timeout_secs)

        ui_.status("Looks good.\nGenerating a default private key...\n")

        # Hmmm... this waits on a socket. Will an ioerror cause an abort?
        # Lazy, but I've never seen this call fail except for IO reasons.
        client = FCPClient(connection)
        client.message_callback = lambda x, y: None  # Disable chatty default.
        default_private_key = client.generate_ssk()[1]['InsertURI']

    except FCPError:
        # Protocol error.
        connection_failure("\nMaybe that's not an FCP server?\n")

    except socket.error:  # Not an IOError until 2.6.
        # Horked.
        connection_failure("\nSocket level error.\n")

    except IOError:
        # Horked.
        connection_failure("\nSocket level error.\n")

    cfg = Config()
    cfg.defaults['HOST'] = host
    cfg.defaults['PORT'] = port
    cfg.defaults['TMP_DIR'] = tmp
    cfg.defaults['DEFAULT_PRIVATE_KEY'] = default_private_key
    Config.to_file(cfg, cfg_file)

    ui_.status("""\nFinished setting configuration.
FCP host: %s
FCP port: %i
Temp dir: %s
cfg file: %s

Default private key:
%s

The config file was successfully written!

""" % (host, port, tmp, cfg_file, default_private_key))
Exemple #11
0
def execute_setup(ui_, host, port, tmp, cfg_file = None):
    """ Run the setup command. """
    def connection_failure(msg):
        """ INTERNAL: Display a warning string. """
        ui_.warn(msg)
        ui_.warn("It looks like your FCP host or port might be wrong.\n")
        ui_.warn("Set them with --fcphost and/or --fcpport and try again.\n")
        raise util.Abort("Connection to FCP server failed.")

    # Fix defaults.
    if host == '':
        host = '127.0.0.1'
    if port == 0:
        port = 9481

    if cfg_file is None:
        cfg_file = os.path.expanduser(DEFAULT_CFG_PATH)

    existing_name = ui_.config('infocalypse', 'cfg_file', None)
    if not existing_name is None:
        existing_name = os.path.expanduser(existing_name)
        ui_.status(MSG_HGRC_SET % existing_name)
        cfg_file = existing_name

    if os.path.exists(cfg_file):
        ui_.status(MSG_CFG_EXISTS % cfg_file)
        raise util.Abort("Refusing to modify existing configuration.")

    tmp = setup_tmp_dir(ui_, tmp)

    if not is_writable(tmp):
        raise util.Abort("Can't write to temp dir: %s\n" % tmp)

    # Test FCP connection.
    timeout_secs = 20
    connection = None
    default_private_key = None
    try:
        ui_.status("Testing FCP connection [%s:%i]...\n" % (host, port))

        connection = FCPConnection(PolledSocket(host, port))

        started = time.time()
        while (not connection.is_connected() and
               time.time() - started < timeout_secs):
            connection.socket.poll()
            time.sleep(.25)

        if not connection.is_connected():
            connection_failure(("\nGave up after waiting %i secs for an "
                               + "FCP NodeHello.\n") % timeout_secs)

        ui_.status("Looks good.\nGenerating a default private key...\n")

        # Hmmm... this waits on a socket. Will an ioerror cause an abort?
        # Lazy, but I've never seen this call fail except for IO reasons.
        client = FCPClient(connection)
        client.message_callback = lambda x, y:None # Disable chatty default.
        default_private_key = client.generate_ssk()[1]['InsertURI']

    except FCPError:
        # Protocol error.
        connection_failure("\nMaybe that's not an FCP server?\n")

    except socket.error: # Not an IOError until 2.6.
        # Horked.
        connection_failure("\nSocket level error.\n")

    except IOError:
        # Horked.
        connection_failure("\nSocket level error.\n")

    cfg = Config()
    cfg.defaults['HOST'] = host
    cfg.defaults['PORT'] = port
    cfg.defaults['TMP_DIR'] = tmp
    cfg.defaults['DEFAULT_PRIVATE_KEY'] = default_private_key
    Config.to_file(cfg, cfg_file)

    ui_.status("""\nFinished setting configuration.
FCP host: %s
FCP port: %i
Temp dir: %s
cfg file: %s

Default private key:
%s

The config file was successfully written!

""" % (host, port, tmp, cfg_file, default_private_key))
Exemple #12
0
def get_params(base_dir):
    """ Return the parameters to run a WikiBot. """

    # Get working directories.
    (tmp_dir,         # MUST exist
     repo_dir,        # MUST exist and contain wikitext hg repo.
     bot_storage_dir, # MUST exist
     )  = get_dirs(base_dir)

    params = read_fnwiki_cfg(os.path.join(repo_dir, 'fnwiki.cfg'))

    # MUST contain SSK private key
    key_file = KEY_FILE_FMT % get_usk_hash(params['WIKI_REPO_USK'])
    print "Read insert key from: %s" % key_file

    # Load private key for the repo from a file..
    insert_ssk = open(os.path.expanduser(key_file), 'rb').read().strip()
    assert insert_ssk.startswith('SSK@')
    # Raw SSK insert key.
    insert_ssk = insert_ssk.split('/')[0].strip()

    # Make insert URI from request URI in config file.
    human = '/'.join(params['WIKI_REPO_USK'].split('/')[1:])
    insert_uri = 'U' + insert_ssk[1:] + '/' + human

    # Then invert the request_uri from it.
    print "Inverting public key from private one..."
    request_uri = FCPClient.connect(FCP_HOST, FCP_PORT). \
                  get_request_uri(insert_uri)
    print request_uri
    if get_usk_hash(request_uri) != get_usk_hash(params['WIKI_REPO_USK']):
        print "The insert SSK doesn't match WIKI_REPO_USK in fnwiki.cfg!"
        assert False

    # LATER: Name convention.
    # USK@/foo.wikitext.R1/0 -- wiki source
    # USK@/foo/0 -- freesite

    #print "Reading latest index from Freenet... This can take minutes."
    #index = prefetch_usk(FCPClient.connect(fcp_host, fcp_port),
    #                     request_uri)
    #insert_uri = get_usk_for_usk_version(insert_uri, index)
    #request_uri = get_usk_for_usk_version(request_uri, index) # needed?


    # Hmmmm... freesite index is read from 'I_<n>' tags in
    # repo. There is no way to set it.
    params.update({
        # FCP 2.0
        'MaxRetries':3,
        'PriorityClass':1,
        #'DontCompress':True,
        'Verbosity':1023, # MUST set this to get progress messages.

        # FCPConnection / RequestRunner
        'FCP_HOST':FCP_HOST,
        'FCP_PORT':FCP_PORT,
        'FCP_POLL_SECS':0.25,
        'N_CONCURRENT':4,
        'CANCEL_TIME_SECS': 15 * 60,

        # FMSBotRunner
        'FMS_HOST':FMS_HOST,
        'FMS_PORT':FMS_PORT,
        'FMS_POLL_SECS': 3 * 60,
        'BOT_STORAGE_DIR':bot_storage_dir,

        # WikiBot
        'FMS_NOTIFY_GROUP': ('infocalypse.notify' if POST_TO_INFOCALYPSE_NOTIFY
                             else ''),  # extra group to notify.
        'LATEST_INDEX':INDEX_HINT, # Just a hint, it is also stored in shelve db
        'SITE_KEY':insert_ssk,
        'INSERT_URI':insert_uri,
        'REQUEST_URI':request_uri,
        'VERBOSITY':VERBOSITY,
        'TMP_DIR':tmp_dir,
        'NO_SEARCH':False, # REQUIRED
        'USK_HASH':get_usk_hash(request_uri),
        'FNPUSH_COALESCE_SECS':60, # Time to wait before pushing
        'SITE_COALESCE_SECS':60, # Time to wait before inserting.
        'NOTIFY_COALESCE_SECS':60, # Time 2w8b4 sending fms repo update msg
        'COMMIT_COALESCE_SECS':-1, # Hack to force immediate commit
        'FMS_TRUST_CACHE_SECS': 1 * 60 * 60,
        'FMS_MIN_TRUST':55, # peer message trust
        'NONE_TRUST':49, # i.e. disable posting for 'None' peer msg trust
        'REPO_DIR':repo_dir,

        # Only uncomment for testing.
        #'MSG_SPOOL_DIR':'/tmp/fake_msgs',
        })

    return params