def insert_files(host, port, names):
socket = PolledSocket(host, port)
connection = FCPConnection(socket, True)
inserts = []
for name in names:
client = FCPClient(connection)
client.in_params.default_fcp_params['MaxRetries'] = 3
client.in_params.default_fcp_params['PriorityClass'] = 1
inserts.append(client)
client.in_params.async = True
parts = name.split('/')
client.put_file('CHK@' + '/' + parts[-1], name)
# Hmmmm... Ugly. Add FCPConnection.wait_until_upload_finishes() ?
while connection.is_uploading():
socket.poll()
time.sleep(.25)
while min([insert.is_finished() for insert in inserts]) == False:
if not socket.poll():
break
time.sleep(.25)
uris = []
for insert in inserts:
if insert.response == None or len(insert.response) < 2 or insert.response[0] <> 'PutSuccessful':
uris.append(None)
continue
uris.append(insert.response[1]['URI'])
return uris
def do_freenet_insert(ui_, repo, params, insert_uri, progress_func):
""" INTERNAL: Helper does the actual insert.
Caller must delete TMP_DUMP_DIR!
"""
default_mime_type = "text/plain" # put_complex_dir() default. Hmmmm.
if not params['ISWIKI']:
site_root = os.path.join(repo.root, params['SITE_DIR'])
else:
# Because wiki html files have no extension to guess from.
default_mime_type = 'text/html'
ui_.status("Dumping wiki as HTML...\n")
site_root = os.path.join(params['TMP_DIR'], TMP_DUMP_DIR)
dump_wiki_html(os.path.join(repo.root, params['WIKI_ROOT']),
site_root,
params['OVERLAYED'])
ui_.status('Default file: %s\n' % params['SITE_DEFAULT_FILE'])
ui_.status('Reading files from:\n%s\n' % site_root)
infos = get_file_infos(site_root)
try:
set_index_file(infos, params['SITE_DEFAULT_FILE'])
except ValueError:
raise util.Abort("Couldn't read %s" % params['SITE_DEFAULT_FILE'])
ui_.status('--- files ---\n')
for info in infos:
ui_.status('%s %s\n' % (info[0], info[1]))
ui_.status('---\n')
if params['DRYRUN']:
ui_.status('Would have inserted to:\n%s\n' % insert_uri)
ui_.status('But --dryrun was set.\n')
return
client = FCPClient.connect(params['FCP_HOST'],
params['FCP_PORT'])
client.in_params.default_fcp_params['DontCompress'] = False
client.message_callback = progress_func
try:
ui_.status('Inserting to:\n%s\n' % insert_uri)
try:
request_uri = client.put_complex_dir(insert_uri, infos,
default_mime_type)[1]['URI']
show_request_uri(ui_, params, request_uri)
except FCPError, err:
if err.is_code(9): # magick number for collision
ui_.warn('An update was already inserted on that index.\n'
+ 'Set a later index with --index and try again.\n')
raise util.Abort("Key collision.")
else:
ui_.warn(str(err) + '\n')
raise util.Abort("FCP Error")
finally:
client.close()
def do_freenet_insert(ui_, repo, params, insert_uri, progress_func):
""" INTERNAL: Helper does the actual insert.
Caller must delete TMP_DUMP_DIR!
"""
default_mime_type = "text/plain" # put_complex_dir() default. Hmmmm.
if not params['ISWIKI']:
site_root = os.path.join(repo.root, params['SITE_DIR'])
else:
# Because wiki html files have no extension to guess from.
default_mime_type = 'text/html'
ui_.status("Dumping wiki as HTML...\n")
site_root = os.path.join(params['TMP_DIR'], TMP_DUMP_DIR)
dump_wiki_html(os.path.join(repo.root, params['WIKI_ROOT']), site_root,
params['OVERLAYED'])
ui_.status('Default file: %s\n' % params['SITE_DEFAULT_FILE'])
ui_.status('Reading files from:\n%s\n' % site_root)
infos = get_file_infos(site_root)
try:
set_index_file(infos, params['SITE_DEFAULT_FILE'])
except ValueError:
raise util.Abort("Couldn't read %s" % params['SITE_DEFAULT_FILE'])
ui_.status('--- files ---\n')
for info in infos:
ui_.status('%s %s\n' % (info[0], info[1]))
ui_.status('---\n')
if params['DRYRUN']:
ui_.status('Would have inserted to:\n%s\n' % insert_uri)
ui_.status('But --dryrun was set.\n')
return
client = FCPClient.connect(params['FCP_HOST'], params['FCP_PORT'])
client.in_params.default_fcp_params['DontCompress'] = False
client.message_callback = progress_func
try:
ui_.status('Inserting to:\n%s\n' % insert_uri)
try:
request_uri = client.put_complex_dir(insert_uri, infos,
default_mime_type)[1]['URI']
show_request_uri(ui_, params, request_uri)
except FCPError, err:
if err.is_code(9): # magick number for collision
ui_.warn('An update was already inserted on that index.\n' +
'Set a later index with --index and try again.\n')
raise util.Abort("Key collision.")
else:
ui_.warn(str(err) + '\n')
raise util.Abort("FCP Error")
finally:
client.close()
def execute_genkey(ui_, params):
""" Run the genkey command. """
client = FCPClient.connect(params['FCP_HOST'], params['FCP_PORT'])
client.message_callback = lambda x, y: None # silence.
resp = client.generate_ssk()
ui_.status(MSG_FMT % (resp[1]['InsertURI'], resp[1]['RequestURI'],
resp[1]['InsertURI'].split('/')[0] + '/', "U" +
resp[1]['InsertURI'].split('/')[0][1:] + '/NAME/0'))
def genkeypair(fcphost, fcpport):
""" Generate a keypair
:returns: inserturi, requesturi (strings)
"""
client = FCPClient.connect(fcphost, fcpport)
client.message_callback = lambda x, y: None # silence.
resp = client.generate_ssk()
return resp[1]['InsertURI'], resp[1]['RequestURI']
def genkeypair(fcphost, fcpport):
""" Generate a keypair
:returns: inserturi, requesturi (strings)
"""
client = FCPClient.connect(fcphost, fcpport)
client.message_callback = lambda x, y : None # silence.
resp = client.generate_ssk()
return resp[1]['InsertURI'], resp[1]['RequestURI']
def execute_genkey(ui_, params):
""" Run the genkey command. """
client = FCPClient.connect(params["FCP_HOST"], params["FCP_PORT"])
client.message_callback = lambda x, y: None # silence.
resp = client.generate_ssk()
ui_.status(
MSG_FMT
% (
resp[1]["InsertURI"],
resp[1]["RequestURI"],
resp[1]["InsertURI"].split("/")[0] + "/",
"U" + resp[1]["InsertURI"].split("/")[0][1:] + "/NAME/0",
)
)
def checkCHK(self, chk, logical_len, length, data=None):
print "---"
print "Checking: ", chk
# Something is closing the connection?
resp = FCPClient.connect(FCP_HOST, FCP_PORT).get(chk)
self.assertTrue(resp[0] == 'AllData')
print "Length: ", len(resp[2])
print "Mime_Type: ", resp[1]['Metadata.ContentType']
if len(resp[2]) != length:
print "Expected len: %i, got: %i!" % (length, len(resp[2]))
self.assertTrue(False)
if not data is None and resp[2][:logical_len] != data:
print "Data doesn't match! (only showing first 16 bytes below)"
print "got: ", repr(resp[2][:logical_len][:16])
print "expected: " , repr(data[:16])
self.assertTrue(False)
def checkCHK(self, chk, logical_len, length, data=None):
print "---"
print "Checking: ", chk
# Something is closing the connection?
resp = FCPClient.connect(FCP_HOST, FCP_PORT).get(chk)
self.assertTrue(resp[0] == 'AllData')
print "Length: ", len(resp[2])
print "Mime_Type: ", resp[1]['Metadata.ContentType']
if len(resp[2]) != length:
print "Expected len: %i, got: %i!" % (length, len(resp[2]))
self.assertTrue(False)
if not data is None and resp[2][:logical_len] != data:
print "Data doesn't match! (only showing first 16 bytes below)"
print "got: ", repr(resp[2][:logical_len][:16])
print "expected: ", repr(data[:16])
self.assertTrue(False)
def execute_setup(ui_, host, port, tmp, cfg_file=None):
""" Run the setup command. """
def connection_failure(msg):
""" INTERNAL: Display a warning string. """
ui_.warn(msg)
ui_.warn("It looks like your FCP host or port might be wrong.\n")
ui_.warn("Set them with --fcphost and/or --fcpport and try again.\n")
raise util.Abort("Connection to FCP server failed.")
# Fix defaults.
if host == '':
host = '127.0.0.1'
if port == 0:
port = 9481
if cfg_file is None:
cfg_file = os.path.expanduser(DEFAULT_CFG_PATH)
existing_name = ui_.config('infocalypse', 'cfg_file', None)
if not existing_name is None:
existing_name = os.path.expanduser(existing_name)
ui_.status(MSG_HGRC_SET % existing_name)
cfg_file = existing_name
if os.path.exists(cfg_file):
ui_.status(MSG_CFG_EXISTS % cfg_file)
raise util.Abort("Refusing to modify existing configuration.")
tmp = setup_tmp_dir(ui_, tmp)
if not is_writable(tmp):
raise util.Abort("Can't write to temp dir: %s\n" % tmp)
# Test FCP connection.
timeout_secs = 20
connection = None
default_private_key = None
try:
ui_.status("Testing FCP connection [%s:%i]...\n" % (host, port))
connection = FCPConnection(PolledSocket(host, port))
started = time.time()
while (not connection.is_connected()
and time.time() - started < timeout_secs):
connection.socket.poll()
time.sleep(.25)
if not connection.is_connected():
connection_failure(("\nGave up after waiting %i secs for an " +
"FCP NodeHello.\n") % timeout_secs)
ui_.status("Looks good.\nGenerating a default private key...\n")
# Hmmm... this waits on a socket. Will an ioerror cause an abort?
# Lazy, but I've never seen this call fail except for IO reasons.
client = FCPClient(connection)
client.message_callback = lambda x, y: None # Disable chatty default.
default_private_key = client.generate_ssk()[1]['InsertURI']
except FCPError:
# Protocol error.
connection_failure("\nMaybe that's not an FCP server?\n")
except socket.error: # Not an IOError until 2.6.
# Horked.
connection_failure("\nSocket level error.\n")
except IOError:
# Horked.
connection_failure("\nSocket level error.\n")
cfg = Config()
cfg.defaults['HOST'] = host
cfg.defaults['PORT'] = port
cfg.defaults['TMP_DIR'] = tmp
cfg.defaults['DEFAULT_PRIVATE_KEY'] = default_private_key
Config.to_file(cfg, cfg_file)
ui_.status("""\nFinished setting configuration.
FCP host: %s
FCP port: %i
Temp dir: %s
cfg file: %s
Default private key:
%s
The config file was successfully written!
""" % (host, port, tmp, cfg_file, default_private_key))
def execute_setup(ui_, host, port, tmp, cfg_file = None):
""" Run the setup command. """
def connection_failure(msg):
""" INTERNAL: Display a warning string. """
ui_.warn(msg)
ui_.warn("It looks like your FCP host or port might be wrong.\n")
ui_.warn("Set them with --fcphost and/or --fcpport and try again.\n")
raise util.Abort("Connection to FCP server failed.")
# Fix defaults.
if host == '':
host = '127.0.0.1'
if port == 0:
port = 9481
if cfg_file is None:
cfg_file = os.path.expanduser(DEFAULT_CFG_PATH)
existing_name = ui_.config('infocalypse', 'cfg_file', None)
if not existing_name is None:
existing_name = os.path.expanduser(existing_name)
ui_.status(MSG_HGRC_SET % existing_name)
cfg_file = existing_name
if os.path.exists(cfg_file):
ui_.status(MSG_CFG_EXISTS % cfg_file)
raise util.Abort("Refusing to modify existing configuration.")
tmp = setup_tmp_dir(ui_, tmp)
if not is_writable(tmp):
raise util.Abort("Can't write to temp dir: %s\n" % tmp)
# Test FCP connection.
timeout_secs = 20
connection = None
default_private_key = None
try:
ui_.status("Testing FCP connection [%s:%i]...\n" % (host, port))
connection = FCPConnection(PolledSocket(host, port))
started = time.time()
while (not connection.is_connected() and
time.time() - started < timeout_secs):
connection.socket.poll()
time.sleep(.25)
if not connection.is_connected():
connection_failure(("\nGave up after waiting %i secs for an "
+ "FCP NodeHello.\n") % timeout_secs)
ui_.status("Looks good.\nGenerating a default private key...\n")
# Hmmm... this waits on a socket. Will an ioerror cause an abort?
# Lazy, but I've never seen this call fail except for IO reasons.
client = FCPClient(connection)
client.message_callback = lambda x, y:None # Disable chatty default.
default_private_key = client.generate_ssk()[1]['InsertURI']
except FCPError:
# Protocol error.
connection_failure("\nMaybe that's not an FCP server?\n")
except socket.error: # Not an IOError until 2.6.
# Horked.
connection_failure("\nSocket level error.\n")
except IOError:
# Horked.
connection_failure("\nSocket level error.\n")
cfg = Config()
cfg.defaults['HOST'] = host
cfg.defaults['PORT'] = port
cfg.defaults['TMP_DIR'] = tmp
cfg.defaults['DEFAULT_PRIVATE_KEY'] = default_private_key
Config.to_file(cfg, cfg_file)
ui_.status("""\nFinished setting configuration.
FCP host: %s
FCP port: %i
Temp dir: %s
cfg file: %s
Default private key:
%s
The config file was successfully written!
""" % (host, port, tmp, cfg_file, default_private_key))
def get_params(base_dir):
""" Return the parameters to run a WikiBot. """
# Get working directories.
(tmp_dir, # MUST exist
repo_dir, # MUST exist and contain wikitext hg repo.
bot_storage_dir, # MUST exist
) = get_dirs(base_dir)
params = read_fnwiki_cfg(os.path.join(repo_dir, 'fnwiki.cfg'))
# MUST contain SSK private key
key_file = KEY_FILE_FMT % get_usk_hash(params['WIKI_REPO_USK'])
print "Read insert key from: %s" % key_file
# Load private key for the repo from a file..
insert_ssk = open(os.path.expanduser(key_file), 'rb').read().strip()
assert insert_ssk.startswith('SSK@')
# Raw SSK insert key.
insert_ssk = insert_ssk.split('/')[0].strip()
# Make insert URI from request URI in config file.
human = '/'.join(params['WIKI_REPO_USK'].split('/')[1:])
insert_uri = 'U' + insert_ssk[1:] + '/' + human
# Then invert the request_uri from it.
print "Inverting public key from private one..."
request_uri = FCPClient.connect(FCP_HOST, FCP_PORT). \
get_request_uri(insert_uri)
print request_uri
if get_usk_hash(request_uri) != get_usk_hash(params['WIKI_REPO_USK']):
print "The insert SSK doesn't match WIKI_REPO_USK in fnwiki.cfg!"
assert False
# LATER: Name convention.
# USK@/foo.wikitext.R1/0 -- wiki source
# USK@/foo/0 -- freesite
#print "Reading latest index from Freenet... This can take minutes."
#index = prefetch_usk(FCPClient.connect(fcp_host, fcp_port),
# request_uri)
#insert_uri = get_usk_for_usk_version(insert_uri, index)
#request_uri = get_usk_for_usk_version(request_uri, index) # needed?
# Hmmmm... freesite index is read from 'I_<n>' tags in
# repo. There is no way to set it.
params.update({
# FCP 2.0
'MaxRetries':3,
'PriorityClass':1,
#'DontCompress':True,
'Verbosity':1023, # MUST set this to get progress messages.
# FCPConnection / RequestRunner
'FCP_HOST':FCP_HOST,
'FCP_PORT':FCP_PORT,
'FCP_POLL_SECS':0.25,
'N_CONCURRENT':4,
'CANCEL_TIME_SECS': 15 * 60,
# FMSBotRunner
'FMS_HOST':FMS_HOST,
'FMS_PORT':FMS_PORT,
'FMS_POLL_SECS': 3 * 60,
'BOT_STORAGE_DIR':bot_storage_dir,
# WikiBot
'FMS_NOTIFY_GROUP': ('infocalypse.notify' if POST_TO_INFOCALYPSE_NOTIFY
else ''), # extra group to notify.
'LATEST_INDEX':INDEX_HINT, # Just a hint, it is also stored in shelve db
'SITE_KEY':insert_ssk,
'INSERT_URI':insert_uri,
'REQUEST_URI':request_uri,
'VERBOSITY':VERBOSITY,
'TMP_DIR':tmp_dir,
'NO_SEARCH':False, # REQUIRED
'USK_HASH':get_usk_hash(request_uri),
'FNPUSH_COALESCE_SECS':60, # Time to wait before pushing
'SITE_COALESCE_SECS':60, # Time to wait before inserting.
'NOTIFY_COALESCE_SECS':60, # Time 2w8b4 sending fms repo update msg
'COMMIT_COALESCE_SECS':-1, # Hack to force immediate commit
'FMS_TRUST_CACHE_SECS': 1 * 60 * 60,
'FMS_MIN_TRUST':55, # peer message trust
'NONE_TRUST':49, # i.e. disable posting for 'None' peer msg trust
'REPO_DIR':repo_dir,
# Only uncomment for testing.
#'MSG_SPOOL_DIR':'/tmp/fake_msgs',
})
return params
