def do_freenet_insert(ui_, repo, params, insert_uri, progress_func):
""" INTERNAL: Helper does the actual insert.
Caller must delete TMP_DUMP_DIR!
"""
default_mime_type = "text/plain" # put_complex_dir() default. Hmmmm.
if not params['ISWIKI']:
site_root = os.path.join(repo.root, params['SITE_DIR'])
else:
# Because wiki html files have no extension to guess from.
default_mime_type = 'text/html'
ui_.status("Dumping wiki as HTML...\n")
site_root = os.path.join(params['TMP_DIR'], TMP_DUMP_DIR)
dump_wiki_html(os.path.join(repo.root, params['WIKI_ROOT']),
site_root,
params['OVERLAYED'])
ui_.status('Default file: %s\n' % params['SITE_DEFAULT_FILE'])
ui_.status('Reading files from:\n%s\n' % site_root)
infos = get_file_infos(site_root)
try:
set_index_file(infos, params['SITE_DEFAULT_FILE'])
except ValueError:
raise util.Abort("Couldn't read %s" % params['SITE_DEFAULT_FILE'])
ui_.status('--- files ---\n')
for info in infos:
ui_.status('%s %s\n' % (info[0], info[1]))
ui_.status('---\n')
if params['DRYRUN']:
ui_.status('Would have inserted to:\n%s\n' % insert_uri)
ui_.status('But --dryrun was set.\n')
return
client = FCPClient.connect(params['FCP_HOST'],
params['FCP_PORT'])
client.in_params.default_fcp_params['DontCompress'] = False
client.message_callback = progress_func
try:
ui_.status('Inserting to:\n%s\n' % insert_uri)
try:
request_uri = client.put_complex_dir(insert_uri, infos,
default_mime_type)[1]['URI']
show_request_uri(ui_, params, request_uri)
except FCPError, err:
if err.is_code(9): # magick number for collision
ui_.warn('An update was already inserted on that index.\n'
+ 'Set a later index with --index and try again.\n')
raise util.Abort("Key collision.")
else:
ui_.warn(str(err) + '\n')
raise util.Abort("FCP Error")
finally:
client.close()
def do_freenet_insert(ui_, repo, params, insert_uri, progress_func):
""" INTERNAL: Helper does the actual insert.
Caller must delete TMP_DUMP_DIR!
"""
default_mime_type = "text/plain" # put_complex_dir() default. Hmmmm.
if not params['ISWIKI']:
site_root = os.path.join(repo.root, params['SITE_DIR'])
else:
# Because wiki html files have no extension to guess from.
default_mime_type = 'text/html'
ui_.status("Dumping wiki as HTML...\n")
site_root = os.path.join(params['TMP_DIR'], TMP_DUMP_DIR)
dump_wiki_html(os.path.join(repo.root, params['WIKI_ROOT']), site_root,
params['OVERLAYED'])
ui_.status('Default file: %s\n' % params['SITE_DEFAULT_FILE'])
ui_.status('Reading files from:\n%s\n' % site_root)
infos = get_file_infos(site_root)
try:
set_index_file(infos, params['SITE_DEFAULT_FILE'])
except ValueError:
raise util.Abort("Couldn't read %s" % params['SITE_DEFAULT_FILE'])
ui_.status('--- files ---\n')
for info in infos:
ui_.status('%s %s\n' % (info[0], info[1]))
ui_.status('---\n')
if params['DRYRUN']:
ui_.status('Would have inserted to:\n%s\n' % insert_uri)
ui_.status('But --dryrun was set.\n')
return
client = FCPClient.connect(params['FCP_HOST'], params['FCP_PORT'])
client.in_params.default_fcp_params['DontCompress'] = False
client.message_callback = progress_func
try:
ui_.status('Inserting to:\n%s\n' % insert_uri)
try:
request_uri = client.put_complex_dir(insert_uri, infos,
default_mime_type)[1]['URI']
show_request_uri(ui_, params, request_uri)
except FCPError, err:
if err.is_code(9): # magick number for collision
ui_.warn('An update was already inserted on that index.\n' +
'Set a later index with --index and try again.\n')
raise util.Abort("Key collision.")
else:
ui_.warn(str(err) + '\n')
raise util.Abort("FCP Error")
finally:
client.close()
def execute_genkey(ui_, params):
""" Run the genkey command. """
client = FCPClient.connect(params['FCP_HOST'], params['FCP_PORT'])
client.message_callback = lambda x, y: None # silence.
resp = client.generate_ssk()
ui_.status(MSG_FMT % (resp[1]['InsertURI'], resp[1]['RequestURI'],
resp[1]['InsertURI'].split('/')[0] + '/', "U" +
resp[1]['InsertURI'].split('/')[0][1:] + '/NAME/0'))
def genkeypair(fcphost, fcpport):
""" Generate a keypair
:returns: inserturi, requesturi (strings)
"""
client = FCPClient.connect(fcphost, fcpport)
client.message_callback = lambda x, y: None # silence.
resp = client.generate_ssk()
return resp[1]['InsertURI'], resp[1]['RequestURI']
def genkeypair(fcphost, fcpport):
""" Generate a keypair
:returns: inserturi, requesturi (strings)
"""
client = FCPClient.connect(fcphost, fcpport)
client.message_callback = lambda x, y : None # silence.
resp = client.generate_ssk()
return resp[1]['InsertURI'], resp[1]['RequestURI']
def execute_genkey(ui_, params):
""" Run the genkey command. """
client = FCPClient.connect(params["FCP_HOST"], params["FCP_PORT"])
client.message_callback = lambda x, y: None # silence.
resp = client.generate_ssk()
ui_.status(
MSG_FMT
% (
resp[1]["InsertURI"],
resp[1]["RequestURI"],
resp[1]["InsertURI"].split("/")[0] + "/",
"U" + resp[1]["InsertURI"].split("/")[0][1:] + "/NAME/0",
)
)
def checkCHK(self, chk, logical_len, length, data=None):
print "---"
print "Checking: ", chk
# Something is closing the connection?
resp = FCPClient.connect(FCP_HOST, FCP_PORT).get(chk)
self.assertTrue(resp[0] == 'AllData')
print "Length: ", len(resp[2])
print "Mime_Type: ", resp[1]['Metadata.ContentType']
if len(resp[2]) != length:
print "Expected len: %i, got: %i!" % (length, len(resp[2]))
self.assertTrue(False)
if not data is None and resp[2][:logical_len] != data:
print "Data doesn't match! (only showing first 16 bytes below)"
print "got: ", repr(resp[2][:logical_len][:16])
print "expected: " , repr(data[:16])
self.assertTrue(False)
def checkCHK(self, chk, logical_len, length, data=None):
print "---"
print "Checking: ", chk
# Something is closing the connection?
resp = FCPClient.connect(FCP_HOST, FCP_PORT).get(chk)
self.assertTrue(resp[0] == 'AllData')
print "Length: ", len(resp[2])
print "Mime_Type: ", resp[1]['Metadata.ContentType']
if len(resp[2]) != length:
print "Expected len: %i, got: %i!" % (length, len(resp[2]))
self.assertTrue(False)
if not data is None and resp[2][:logical_len] != data:
print "Data doesn't match! (only showing first 16 bytes below)"
print "got: ", repr(resp[2][:logical_len][:16])
print "expected: ", repr(data[:16])
self.assertTrue(False)
def get_params(base_dir):
""" Return the parameters to run a WikiBot. """
# Get working directories.
(tmp_dir, # MUST exist
repo_dir, # MUST exist and contain wikitext hg repo.
bot_storage_dir, # MUST exist
) = get_dirs(base_dir)
params = read_fnwiki_cfg(os.path.join(repo_dir, 'fnwiki.cfg'))
# MUST contain SSK private key
key_file = KEY_FILE_FMT % get_usk_hash(params['WIKI_REPO_USK'])
print "Read insert key from: %s" % key_file
# Load private key for the repo from a file..
insert_ssk = open(os.path.expanduser(key_file), 'rb').read().strip()
assert insert_ssk.startswith('SSK@')
# Raw SSK insert key.
insert_ssk = insert_ssk.split('/')[0].strip()
# Make insert URI from request URI in config file.
human = '/'.join(params['WIKI_REPO_USK'].split('/')[1:])
insert_uri = 'U' + insert_ssk[1:] + '/' + human
# Then invert the request_uri from it.
print "Inverting public key from private one..."
request_uri = FCPClient.connect(FCP_HOST, FCP_PORT). \
get_request_uri(insert_uri)
print request_uri
if get_usk_hash(request_uri) != get_usk_hash(params['WIKI_REPO_USK']):
print "The insert SSK doesn't match WIKI_REPO_USK in fnwiki.cfg!"
assert False
# LATER: Name convention.
# USK@/foo.wikitext.R1/0 -- wiki source
# USK@/foo/0 -- freesite
#print "Reading latest index from Freenet... This can take minutes."
#index = prefetch_usk(FCPClient.connect(fcp_host, fcp_port),
# request_uri)
#insert_uri = get_usk_for_usk_version(insert_uri, index)
#request_uri = get_usk_for_usk_version(request_uri, index) # needed?
# Hmmmm... freesite index is read from 'I_<n>' tags in
# repo. There is no way to set it.
params.update({
# FCP 2.0
'MaxRetries':3,
'PriorityClass':1,
#'DontCompress':True,
'Verbosity':1023, # MUST set this to get progress messages.
# FCPConnection / RequestRunner
'FCP_HOST':FCP_HOST,
'FCP_PORT':FCP_PORT,
'FCP_POLL_SECS':0.25,
'N_CONCURRENT':4,
'CANCEL_TIME_SECS': 15 * 60,
# FMSBotRunner
'FMS_HOST':FMS_HOST,
'FMS_PORT':FMS_PORT,
'FMS_POLL_SECS': 3 * 60,
'BOT_STORAGE_DIR':bot_storage_dir,
# WikiBot
'FMS_NOTIFY_GROUP': ('infocalypse.notify' if POST_TO_INFOCALYPSE_NOTIFY
else ''), # extra group to notify.
'LATEST_INDEX':INDEX_HINT, # Just a hint, it is also stored in shelve db
'SITE_KEY':insert_ssk,
'INSERT_URI':insert_uri,
'REQUEST_URI':request_uri,
'VERBOSITY':VERBOSITY,
'TMP_DIR':tmp_dir,
'NO_SEARCH':False, # REQUIRED
'USK_HASH':get_usk_hash(request_uri),
'FNPUSH_COALESCE_SECS':60, # Time to wait before pushing
'SITE_COALESCE_SECS':60, # Time to wait before inserting.
'NOTIFY_COALESCE_SECS':60, # Time 2w8b4 sending fms repo update msg
'COMMIT_COALESCE_SECS':-1, # Hack to force immediate commit
'FMS_TRUST_CACHE_SECS': 1 * 60 * 60,
'FMS_MIN_TRUST':55, # peer message trust
'NONE_TRUST':49, # i.e. disable posting for 'None' peer msg trust
'REPO_DIR':repo_dir,
# Only uncomment for testing.
#'MSG_SPOOL_DIR':'/tmp/fake_msgs',
})
return params