class LaunchMany:
def __init__(self, config, Output):
try:
self.config = config
self.Output = Output
self.torrent_dir = config["torrent_dir"]
self.torrent_cache = {}
self.file_cache = {}
self.blocked_files = {}
self.scan_period = config["parse_dir_interval"]
self.stats_period = config["display_interval"]
self.torrent_list = []
self.downloads = {}
self.counter = 0
self.doneflag = Event()
self.hashcheck_queue = []
self.hashcheck_current = None
self.rawserver = RawServer(
self.doneflag,
config["timeout_check_interval"],
config["timeout"],
ipv6_enable=config["ipv6_enabled"],
failfunc=self.failed,
errorfunc=self.exchandler,
)
upnp_type = UPnP_test(config["upnp_nat_access"])
while 1:
try:
self.listen_port = self.rawserver.find_and_bind(
config["minport"],
config["maxport"],
config["bind"],
ipv6_socket_style=config["ipv6_binds_v4"],
upnp=upnp_type,
randomizer=config["random_port"],
)
break
except socketerror, e:
if upnp_type and e == UPnP_ERROR:
self.Output.message("WARNING: COULD NOT FORWARD VIA UPnP")
upnp_type = 0
continue
self.failed("Couldn't listen - " + str(e))
return
self.ratelimiter = RateLimiter(self.rawserver.add_task, config["upload_unit_size"])
self.ratelimiter.set_upload_rate(config["max_upload_rate"])
self.handler = MultiHandler(self.rawserver, self.doneflag)
seed(createPeerID())
self.rawserver.add_task(self.scan, 0)
self.rawserver.add_task(self.stats, 0)
self.start()
except:
def __init__(self, config, Output):
try:
self.config = config
self.Output = Output
self.torrent_dir = config['torrent_dir']
self.torrent_cache = {}
self.file_cache = {}
self.blocked_files = {}
self.scan_period = config['parse_dir_interval']
self.stats_period = config['display_interval']
self.torrent_list = []
self.downloads = {}
self.counter = 0
self.doneflag = Event()
self.hashcheck_queue = []
self.hashcheck_current = None
self.rawserver = RawServer(self.doneflag, config['timeout_check_interval'],
config['timeout'], ipv6_enable = config['ipv6_enabled'],
failfunc = self.failed, errorfunc = self.exchandler)
upnp_type = UPnP_test(config['upnp_nat_access'])
while True:
try:
self.listen_port = self.rawserver.find_and_bind(
config['minport'], config['maxport'], config['bind'],
ipv6_socket_style = config['ipv6_binds_v4'],
upnp = upnp_type, randomizer = config['random_port'])
break
except socketerror, e:
if upnp_type and e == UPnP_ERROR:
self.Output.message('WARNING: COULD NOT FORWARD VIA UPnP')
upnp_type = 0
continue
self.failed("Couldn't listen - " + str(e))
return
self.ratelimiter = RateLimiter(self.rawserver.add_task,
config['upload_unit_size'])
self.ratelimiter.set_upload_rate(config['max_upload_rate'])
self.handler = MultiHandler(self.rawserver, self.doneflag)
seed(createPeerID())
self.rawserver.add_task(self.scan, 0)
self.rawserver.add_task(self.stats, 0)
self.handler.listen_forever()
self.Output.message('shutting down')
self.hashcheck_queue = []
for hash in self.torrent_list:
self.Output.message('dropped "'+self.torrent_cache[hash]['path']+'"')
self.downloads[hash].shutdown()
self.rawserver.shutdown()
class LaunchMany:
def __init__(self, config, Output):
try:
self.config = config
self.Output = Output
self.torrent_dir = config['torrent_dir']
self.torrent_cache = {}
self.file_cache = {}
self.blocked_files = {}
self.scan_period = config['parse_dir_interval']
self.stats_period = config['display_interval']
self.torrent_list = []
self.downloads = {}
self.counter = 0
self.doneflag = Event()
self.hashcheck_queue = []
self.hashcheck_current = None
self.rawserver = RawServer(self.doneflag, config['timeout_check_interval'],
config['timeout'], ipv6_enable = config['ipv6_enabled'],
failfunc = self.failed, errorfunc = self.exchandler)
upnp_type = UPnP_test(config['upnp_nat_access'])
while True:
try:
self.listen_port = self.rawserver.find_and_bind(
config['minport'], config['maxport'], config['bind'],
ipv6_socket_style = config['ipv6_binds_v4'],
upnp = upnp_type, randomizer = config['random_port'])
break
except socketerror, e:
if upnp_type and e == UPnP_ERROR:
self.Output.message('WARNING: COULD NOT FORWARD VIA UPnP')
upnp_type = 0
continue
self.failed("Couldn't listen - " + str(e))
return
self.ratelimiter = RateLimiter(self.rawserver.add_task,
config['upload_unit_size'])
self.ratelimiter.set_upload_rate(config['max_upload_rate'])
self.handler = MultiHandler(self.rawserver, self.doneflag, config)
seed(createPeerID())
self.rawserver.add_task(self.scan, 0)
self.rawserver.add_task(self.stats, 0)
self.handler.listen_forever()
self.Output.message('shutting down')
self.hashcheck_queue = []
for hash in self.torrent_list:
self.Output.message('dropped "'+self.torrent_cache[hash]['path']+'"')
self.downloads[hash].shutdown()
self.rawserver.shutdown()
except:
def __init__(self, host, port, dbDir, ipv6_enable = False, upnp = 0, natpmp = False):
self.host = host
self.port = port
self.dbDir = dbDir
self.store = None
self.rawserver = RawServer(self, host, port, ipv6_enable, upnp, natpmp)
self.tokensHandler = TokensHandler(self)
self.krpc = KRPC(self)
self.contacts = [] # [ip,] - contacts added by bittorrent
self.announce = {} # {ip:[hash,],} - nodes that announced us
def __init__(self, config):
self.socket = {}
self.rs = RawServer(Event(), 100, 1000)
self.nh = NetworkHandler(self)
self.shutdown = Event()
self.config = config
self.post_commit = []
for pattern, action in self.config.items('post-commit'):
try:
self.post_commit.append((re.compile(pattern, re.I), action))
except re.error, msg:
raise ServerError, 'Bad post-commit pattern \"%s\": %s' % \
(pattern, msg)
class LaunchMany:
def __init__(self, config, Output):
try:
self.config = config
self.Output = Output
self.torrent_dir = config['torrent_dir']
self.torrent_cache = {}
self.file_cache = {}
self.blocked_files = {}
self.scan_period = config['parse_dir_interval']
self.stats_period = config['display_interval']
self.torrent_list = []
self.downloads = {}
self.counter = 0
self.doneflag = Event()
self.hashcheck_queue = []
self.hashcheck_current = None
self.rawserver = RawServer(
self.doneflag, config['timeout_check_interval'],
config['timeout'], ipv6_enable=config['ipv6_enabled'],
failfunc=self.failed, errorfunc=self.exchandler)
upnp_type = UPnP_test(config['upnp_nat_access'])
while True:
try:
self.listen_port = self.rawserver.find_and_bind(
config['minport'], config['maxport'], config['bind'],
ipv6_socket_style=config['ipv6_binds_v4'],
upnp=upnp_type, randomizer=config['random_port'])
break
except socket.error as e:
if upnp_type and e == UPnP_ERROR:
self.Output.message(
'WARNING: COULD NOT FORWARD VIA UPnP')
upnp_type = 0
continue
self.failed("Couldn't listen - " + str(e))
return
self.ratelimiter = RateLimiter(self.rawserver.add_task,
config['upload_unit_size'])
self.ratelimiter.set_upload_rate(config['max_upload_rate'])
self.handler = MultiHandler(self.rawserver, self.doneflag, config)
seed(createPeerID())
self.rawserver.add_task(self.scan, 0)
self.rawserver.add_task(self.stats, 0)
self.handler.listen_forever()
self.Output.message('shutting down')
self.hashcheck_queue = []
for hash in self.torrent_list:
self.Output.message('dropped "{}"'.format(
self.torrent_cache[hash]['path']))
self.downloads[hash].shutdown()
self.rawserver.shutdown()
except:
data = StringIO()
print_exc(file=data)
Output.exception(data.getvalue())
def scan(self):
self.rawserver.add_task(self.scan, self.scan_period)
r = parsedir(self.torrent_dir, self.torrent_cache, self.file_cache,
self.blocked_files, return_metainfo=True,
errfunc=self.Output.message)
(self.torrent_cache, self.file_cache, self.blocked_files, added,
removed) = r
for hash, data in removed.iteritems():
self.Output.message('dropped "{}"'.format(data['path']))
self.remove(hash)
for hash, data in added.iteritems():
self.Output.message('added "{}"'.format(data['path']))
self.add(hash, data)
def stats(self):
self.rawserver.add_task(self.stats, self.stats_period)
data = []
for hash in self.torrent_list:
cache = self.torrent_cache[hash]
if self.config['display_path']:
name = cache['path']
else:
name = cache['name']
size = cache['length']
d = self.downloads[hash]
progress = '0.0%'
peers = 0
seeds = 0
seedsmsg = "S"
dist = 0.0
uprate = 0.0
dnrate = 0.0
upamt = 0
dnamt = 0
t = 0
if d.is_dead():
status = 'stopped'
elif d.waiting:
status = 'waiting for hash check'
elif d.checking:
status = d.status_msg
progress = '{:.1%}'.format(d.status_done)
else:
stats = d.statsfunc()
s = stats['stats']
if d.seed:
status = 'seeding'
progress = '100.0%'
seeds = s.numOldSeeds
seedsmsg = "s"
dist = s.numCopies
else:
if s.numSeeds + s.numPeers:
t = stats['time']
if t == 0: # unlikely
t = 0.01
status = fmttime(t)
else:
t = -1
status = 'connecting to peers'
progress = '{:.1%}'.format(stats['frac'])
seeds = s.numSeeds
dist = s.numCopies2
dnrate = stats['down']
peers = s.numPeers
uprate = stats['up']
upamt = s.upTotal
dnamt = s.downTotal
if d.is_dead() or d.status_errtime + 300 > clock():
msg = d.status_err[-1]
else:
msg = ''
data.append((name, status, progress, peers, seeds, seedsmsg, dist,
uprate, dnrate, upamt, dnamt, size, t, msg))
stop = self.Output.display(data)
if stop:
self.doneflag.set()
def remove(self, hash):
self.torrent_list.remove(hash)
self.downloads[hash].shutdown()
del self.downloads[hash]
def add(self, hash, data):
c = self.counter
self.counter += 1
x = ''
for i in xrange(3):
x = mapbase64[c & 0x3F] + x
c >>= 6
peer_id = createPeerID(x)
d = SingleDownload(self, hash, data['metainfo'], self.config, peer_id)
self.torrent_list.append(hash)
self.downloads[hash] = d
d.start()
def saveAs(self, hash, name, saveas, isdir):
x = self.torrent_cache[hash]
style = self.config['saveas_style']
if style == 1 or style == 3:
if saveas:
saveas = os.path.join(saveas, x['file'][:-1 - len(x['type'])])
else:
saveas = x['path'][:-1 - len(x['type'])]
if style == 3:
if not os.path.isdir(saveas):
try:
os.mkdir(saveas)
except:
raise OSError("couldn't create directory for {} ({})"
''.format(x['path'], saveas))
if not isdir:
saveas = os.path.join(saveas, name)
else:
if saveas:
saveas = os.path.join(saveas, name)
else:
saveas = os.path.join(os.path.split(x['path'])[0], name)
if isdir and not os.path.isdir(saveas):
try:
os.mkdir(saveas)
except:
raise OSError("couldn't create directory for {} ({})".format(
x['path'], saveas))
return saveas
def hashchecksched(self, hash=None):
if hash:
self.hashcheck_queue.append(hash)
if not self.hashcheck_current:
self._hashcheck_start()
def _hashcheck_start(self):
self.hashcheck_current = self.hashcheck_queue.pop(0)
self.downloads[self.hashcheck_current].hashcheck_start(
self.hashcheck_callback)
def hashcheck_callback(self):
self.downloads[self.hashcheck_current].hashcheck_callback()
if self.hashcheck_queue:
self._hashcheck_start()
else:
self.hashcheck_current = None
def died(self, hash):
if hash in self.torrent_cache:
self.Output.message('DIED: "{}"'.format(
self.torrent_cache[hash]['path']))
def was_stopped(self, hash):
try:
self.hashcheck_queue.remove(hash)
except:
pass
if self.hashcheck_current == hash:
self.hashcheck_current = None
if self.hashcheck_queue:
self._hashcheck_start()
def failed(self, s):
self.Output.message('FAILURE: ' + s)
def exchandler(self, s):
self.Output.exception(s)
if v[0] == 172 and v[1] >= 16 and v[1] <= 31:
return 1
except ValueError:
return 0
def track(args):
if len(args) == 0:
print formatDefinitions(defaults, 80)
return
try:
config, files = parseargs(args, defaults, 0, 0)
except ValueError, e:
print 'error: ' + str(e)
print 'run with no arguments for parameter explanations'
return
r = RawServer(Event(), config['timeout_check_interval'], config['socket_timeout'])
t = Tracker(config, r)
r.bind(config['port'], config['bind'], True)
r.listen_forever(HTTPHandler(t.get, config['min_time_between_log_flushes']))
t.save_dfile()
print '# Shutting down: ' + isotime()
def size_format(s):
if (s < 1024):
r = str(s) + 'B'
elif (s < 1048576):
r = str(int(s/1024)) + 'KiB'
elif (s < 1073741824L):
r = str(int(s/1048576)) + 'MiB'
elif (s < 1099511627776L):
r = str(int((s/1073741824.0)*100.0)/100.0) + 'GiB'
def __init__(self, co):
self.co = co
self.socket = {}
self.rs = RawServer(Event(), 100, 1000)
self.nh = NetworkHandler(self)
class ClientHandler:
def __init__(self, co):
self.co = co
self.socket = {}
self.rs = RawServer(Event(), 100, 1000)
self.nh = NetworkHandler(self)
# Functions called from lower level code
def message_came_in(self, s, data):
try:
msg = bdecode(data)
except ValueError:
self.close(s)
raise NetworkError, 'garbage data'
if msg.has_key('error'):
raise ServerError, msg['error']
socket = self.socket[s]
srp = socket['srp']
if socket['state'] == 1:
K, m = self.auth.client_key(msg['s'], msg['B'], msg['u'],
srp['keys'])
socket['key'], socket['m_out'] = K, m
self._send_msg(s, {'m': socket['m_out'].digest()})
socket['state'] = 2
elif socket['state'] == 2:
socket['m_in'] = SRP.host_authenticator(socket['key'], srp['keys'][0], socket['m_out'].digest())
if socket['m_in'].digest() != msg['auth']:
raise ServerError, 'Bad host authentication'
return
self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
self.rs.doneflag.set()
elif socket['state'] == 3:
self.socket[s]['hash'] = msg['hash']
self.rs.doneflag.set()
elif socket['state'] == 4:
self.close(s)
secret = crypt(msg['secret'], socket['key'])[0]
self.auth.save_secret(secret)
self.rs.doneflag.set()
elif socket['state'] == 5:
self.close(s)
self.rs.doneflag.set()
elif socket['state'] == 6:
if len(msg['salt']) < 20:
self._send_error(s, None, 'Bad salt length')
self.close(s)
raise NetworkError, 'Bad salt from server'
salt = random_string(20)
key = self.auth.session_key(salt, msg['salt'])
socket['m_in'] = hmac.new(key, '', sha)
key = self.auth.session_key(msg['salt'], salt)
socket['m_out'] = hmac.new(key, '', sha)
self._send_msg(s, {'auth': socket['m_in'].digest(),
'salt': salt})
socket['state'] = 7
elif socket['state'] == 7:
if msg['auth'] != socket['m_out'].digest():
self._send_error(s, None, 'Bad auth')
self.close(s)
raise NetworkError, 'Bad server auth'
self._req_mode(s, 1)
self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
self.socket[s] = [{}, {}, {}, [], 1]
self.rs.doneflag.set()
else:
self.close(s)
def connection_flushed(self, s):
queue = self.socket[s][Queue]
socket = self.socket[s]
socket[Flushed] = 1
while len(queue) and socket[Flushed] == 1:
mid, msg = queue.pop(0)
diff = read_diff(self.co, msg['handle'], msg['changenum'], None)
socket[Flushed] = self._send_response(s, mid, {'diff': diff})
def connection_lost(self, s, msg):
del self.socket[s]
raise NetworkError, msg
def request_came_in(self, s, mid, data):
try:
msg = bdecode(data)
except ValueError:
self.close(s)
raise NetworkError, 'garbage request'
self.request_handlers[msg['request']](self, s, mid, msg)
def response_came_in(self, s, mid, data):
try:
msg = bdecode(data)
except ValueError:
self.close(s)
raise NetworkError, 'bad data from server'
rstate = self.socket[s][Request][mid]
if msg.has_key('error'):
# XXX: grody hack, diffs which don't belong in history
if rstate['request'] == 'get diff':
msg['diff'] = zlib.compress(bencode(''))
else:
raise ServerError, msg['error']
if self.response_handlers[rstate['request']](self, s, mid, msg, rstate):
del self.socket[s][Request][mid]
# request handlers
def _request_get_change(self, s, mid, msg):
resp = {'changeset': self.co.lcrepo.get(msg['changenum'], txn=self.txn)}
self._send_response(s, mid, resp)
def _request_get_diff(self, s, mid, msg):
if self.socket[s][Flushed] == 1:
diff = read_diff(self.co, msg['handle'], msg['changenum'], self.txn)
self.socket[s][Flushed] = self._send_response(s, mid, {'diff': diff})
else:
self.socket[s][Queue].append((mid, msg))
request_handlers = {'get change': _request_get_change,
'get diff': _request_get_diff}
# response handlers
def _response_get_head(self, s, mid, msg, rstate):
return self._merge_change(s, mid, msg['head'])
def _merge_change(self, s, mid, head):
lstate = self.socket[s][Request][mid]
lstate['head'] = head
self.socket[s][UpdateInfo]['head'] = head
if self.co.lcrepo.has_key(head):
named, modified, added, deleted = \
handles_in_branch(self.co, lstate['heads'], [head], None)
self._update_checks(s, mid, named, modified)
self._update_handle_list(s, lstate, named, modified, added, deleted)
self._update_finish(s, lstate)
self.socket[s][UpdateInfo]['head'] = head
self.rs.doneflag.set()
return 1
rid = self._get_change(s, head)
self.socket[s][Request][rid] = {'request': 'get change',
'changenum': head,
'ref': mid}
lstate['requests'][head] = 1
lstate['count'] = 1
return 0
def _response_get_change(self, s, mid, msg, rstate):
lstate = self.socket[s][Request][rstate['ref']]
if sha.new(msg['changeset']).digest() != rstate['changenum']:
raise ServerError, 'bad changeset'
# write it out, decode and eject from memory
write_changeset(self.co, rstate['changenum'], msg['changeset'],
lstate['txn'])
changeset = bdecode(msg['changeset'])
lstate['changes'][rstate['changenum']] = changeset
del msg['changeset']
# get any precursors we don't have and haven't yet requested
for change in changeset['precursors']:
if self.co.lcrepo.has_key(change):
continue
if lstate['changes'].has_key(change):
continue
if lstate['requests'].has_key(change):
continue
rid = self._get_change(s, change)
self.socket[s][Request][rid] = {'request': 'get change',
'changenum': change,
'ref': rstate['ref']}
lstate['requests'][change] = 1
lstate['count'] += 1
# record all the diffs we'll need to request
diffs = lstate['diffs']
for handle, hinfo in changeset['handles'].items():
if not hinfo.has_key('hash'):
continue
if not diffs.has_key(handle):
diffs[handle] = {}
lstate['counts'][handle] = 0
diffs[handle][rstate['changenum']] = 1
lstate['counts'][handle] += 1
# clean up state
del lstate['requests'][rstate['changenum']]
lstate['count'] -= 1
if lstate['count'] == 0:
sync_history(self.co, lstate['head'], lstate['txn'],
cache=lstate['changes'])
named, modified, added, deleted = \
handles_in_branch(self.co,
lstate['heads'], [lstate['head']],
lstate['txn'], cache=lstate['changes'])
del lstate['changes']
self._update_checks(s, rstate['ref'], named, modified)
self._update_handle_list(s, lstate, named, modified, added, deleted)
handle_list = lstate['handle list']
# get all the related file diffs
for i in xrange(len(handle_list)-1, -1, -1):
handle = handle_list[i][1]
if not diffs.has_key(handle):
continue
changes = diffs[handle]
requested = 0
for change in changes.keys():
requested = 1
self._queue_diff(s, change, handle, rstate['ref'])
lstate['count'] += requested
self._get_diff(s, rstate['ref'])
if lstate['count'] == 0:
self._update_finish(s, lstate)
del self.socket[s][Request][rstate['ref']]
self.rs.doneflag.set()
return 1
def _response_get_diff(self, s, mid, msg, rstate):
lstate = self.socket[s][Request][rstate['ref']]
# send out the next one
lstate['req-outstanding'] -= 1
self._get_diff(s, rstate['ref'])
diffs = lstate['diffs']
diffs[rstate['handle']][rstate['change']] = msg['diff']
lstate['counts'][rstate['handle']] -= 1
if lstate['counts'][rstate['handle']] == 0:
lstate['count'] -= 1
# XXX: do better ordering
WD = WriteDiff(self.co, rstate['handle'], lstate['txn'])
for change, diff in diffs[rstate['handle']].items():
WD.write(diff, change)
WD.close()
if lstate['modified'].has_key(rstate['handle']):
updates(self.co, self.socket[s][UpdateInfo], lstate,
rstate['handle'])
#del diffs[rstate['handle']]
if lstate['count'] == 0:
self._update_finish(s, lstate)
self.socket[s][UpdateInfo]['head'] = lstate['head']
del self.socket[s][Request][rstate['ref']]
self.rs.doneflag.set()
return 1
def _response_return(self, s, mid, msg, rstate):
self.rs.doneflag.set()
return 1
def _response_list_repositories(self, s, mid, msg, rstate):
self.socket[s][UpdateInfo] = msg['list']
self.rs.doneflag.set()
return 1
response_handlers = {'get head': _response_get_head,
'get change': _response_get_change,
'get diff': _response_get_diff,
'commit': _response_return,
'create repository': _response_return,
'destroy repository': _response_return,
'list repositories': _response_list_repositories}
# Internal helper functions
def _update_checks(self, s, mid, named, modified):
lstate = self.socket[s][Request][mid]
check_modified(self.co, modified)
lstate['modified'] = {}
for handle in modified:
lstate['modified'][handle] = 1
self.socket[s][UpdateInfo]['modified'] = []
self.socket[s][UpdateInfo]['modified conflicts'] = []
def _update_handle_list(self, s, lstate, named, modified, added, deleted):
uinfo = self.socket[s][UpdateInfo]
func = lstate['update function']
handles, nconflicts = func(self.co, uinfo,
named, modified, added, deleted,
lstate['txn'])
handle_list = lstate['handle list']
for handle, show in handles:
letters = (' ', ' ')
if uinfo['newfiles'].has_key(handle):
letters = ('A', 'A')
elif uinfo['deletes'].has_key(handle):
letters = ('D', 'D')
elif uinfo['names'].has_key(handle):
letters = (' ', 'N')
if nconflicts.has_key(handle):
letters = (letters[0], 'C')
handle_list.append((handle_to_filename(self.co, handle, lstate['txn']), handle, show, letters))
handle_list.sort()
handle_list.reverse()
uinfo['name conflicts'] = nconflicts.keys()
def _update_finish(self, s, lstate):
updates(self.co, self.socket[s][UpdateInfo], lstate, None)
def _req_mode(self, s, mode):
self.nh.req_mode(s, mode)
def _commit(self, s, repo, head):
rid = self._send_request(s, {'request': 'commit',
'repository': repo,
'changenum': head})
self.socket[s][Request][rid] = {'request': 'commit'}
def _update(self, s, repo, func, txn):
heads = bdecode(self.co.linforepo.get('heads'))
lstate = {'request': 'get head',
'txn': txn,
'changes': {},
'requests': {},
'diffs': {},
'count': 0,
'counts': {},
'handle list': [],
'heads': heads,
'reqq': [],
'req-outstanding': 0,
'update function': func}
if repo.startswith('{') and repo.endswith('}'):
head = binascii.unhexlify(repo[1:-1])
rid = self.nh.next_id(s)
self.socket[s][Request][rid] = lstate
if self._merge_change(s, rid, head):
del self.socket[s][Request][rid]
else:
rid = self._send_request(s, {'request': 'get head',
'repository': repo})
self.socket[s][Request][rid] = lstate
return
def _create_repo(self, s, repo):
rid = self._send_request(s, {'request': 'create repository',
'repository': repo})
self.socket[s][Request][rid] = {'request': 'create repository',
'repository': repo}
def _remove_repo(self, s, repo):
rid = self._send_request(s, {'request': 'destroy repository',
'repository': repo})
self.socket[s][Request][rid] = {'request': 'destroy repository',
'repository': repo}
def _list_repos(self, s):
rid = self._send_request(s, {'request': 'list repositories'})
self.socket[s][Request][rid] = {'request': 'list repositories'}
def _get_change(self, s, change):
req = {'request': 'get change', 'changenum': change}
return self._send_request(s, req)
def _queue_diff(self, s, change, handle, mid):
rstate = self.socket[s][Request][mid]
rstate['reqq'].append((change, handle))
def _get_diff(self, s, mid):
rstate = self.socket[s][Request][mid]
while len(rstate['reqq']) and rstate['req-outstanding'] <= 40:
change, handle = rstate['reqq'].pop(0)
req = {'request': 'get diff', 'changenum': change,
'handle': handle}
state = {'request': 'get diff', 'ref': mid, 'change': change,
'handle': handle}
rid = self._send_request(s, req)
self.socket[s][Request][rid] = state
rstate['req-outstanding'] += 1
def _send_msg(self, s, data):
self.nh.send_msg(s, bencode(data))
def _send_error(self, s, mid, msg):
self._send_response(s, mid, {'error': msg})
def _send_request(self, s, data):
return self.nh.send_request(s, bencode(data))
def _send_response(self, s, mid, data):
return self.nh.send_response(s, mid, bencode(data))
# Functions to be called from higher level code
def start_connection(self, dns):
s = self.nh.start_connection(dns)
self.socket[s] = {'state': 0, 'srp': {}}
return s
def close(self, s):
self.nh.close(s)
del self.socket[s]
def get_hash(self, s):
self._send_msg(s, {'op': 'get hash', 'user': self.co.user})
self.socket[s]['state'] = 3
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
secret_hash = self.socket[s]['hash']
del self.socket[s]['hash']
return secret_hash
def srp_auth(self, s):
socket = self.socket[s]
assert socket['state'] == 0 or socket['state'] == 3
srp = socket['srp'] = {}
srp['keys'] = SRP.client_begin()
self._send_msg(s, {'op': 'srp auth', 'user': self.co.user, 'A': srp['keys'][0]})
socket['state'] = 1
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def get_secret(self, s):
self._send_msg(s, {'op': 'get secret'})
self.socket[s]['state'] = 4
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
return
def set_password(self, s, password):
socket = self.socket[s]
salt, v = SRP.new_passwd(self.co.user, password)
cypherv = crypt(long_to_string(v), socket['key'])[0]
self._send_msg(s, {'op': 'set password', 's': salt, 'v': cypherv})
socket['state'] = 5
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def secret_auth(self, s):
socket = self.socket[s]
self._send_msg(s, {'op': 'secret auth', 'user': self.co.user})
socket['state'] = 6
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def commit(self, s, repo, changenum, txn):
self.txn = txn
self._commit(s, repo, changenum)
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def update(self, s, repo, func, txn):
self._update(s, repo, func, txn)
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
updateinfo = self.socket[s][UpdateInfo]
del self.socket[s][UpdateInfo]
return updateinfo
def create_repo(self, s, repo):
self._create_repo(s, repo)
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def remove_repo(self, s, repo):
self._remove_repo(s, repo)
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def list_repos(self, s):
self._list_repos(s)
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
rlist = self.socket[s][UpdateInfo]
del self.socket[s][UpdateInfo]
return rlist
def __init__(self, co):
self.co = co
self.socket = {}
self.rs = RawServer(Event(), 100, 1000)
self.nh = NetworkHandler(self)
# To connect, run: nc localhost 8080
from threading import Event
from RawServer import RawServer
done_flag = Event()
class Handler:
def external_connection_made(self, connection):
connection.write("hello there stranger!\n")
def connection_flushed(self, connection):
# Wrote string, so close all connections and quit.
done_flag.set()
timeout_check_interval = 60 * 60
timeout = timeout_check_interval
server = RawServer(done_flag, timeout_check_interval, timeout)
server.bind(8080)
server.listen_forever(Handler())
class Factory:
"""
Background functionality
"""
def __init__(self, host, port, dbDir, ipv6_enable = False, upnp = 0, natpmp = False):
self.host = host
self.port = port
self.dbDir = dbDir
self.store = None
self.rawserver = RawServer(self, host, port, ipv6_enable, upnp, natpmp)
self.tokensHandler = TokensHandler(self)
self.krpc = KRPC(self)
self.contacts = [] # [ip,] - contacts added by bittorrent
self.announce = {} # {ip:[hash,],} - nodes that announced us
def Node(self):
"""Create a new node"""
return KNode(self)
def start(self):
"""Start Factory"""
self.rawserver.add_task(self._init)
self.rawserver.add_task(self._checkpoint, 60)
self.rawserver.start()
self.rawserver.add_task(self._cleanDataBase, KEINITIAL_DELAY)
self.rawserver.add_task(self.refreshTable, 5, [True])
def _init(self):
"""Initialize Factory"""
self._loadDB()
self._loadSelfNode()
self._loadRoutingTable()
def _close(self):
"""Close Factory"""
self._updateDB()
self.rawserver.shutdown()
####################
# Database Handler
####################
def _loadDB(self):
"""Load the database"""
if DEBUG:
print("Debug: DHT - _loadDB")
# connect
self.store = sqlite3.connect(os.path.join(self.dbDir.encode("UTF-8"), "dht.db"))
self.store.text_factory = str
# create if neccacery
c = self.store.cursor()
statements = ["create table kv (key binary, value binary, age timestamp, primary key (key, value))",
"create table nodes (id binary primary key, host text, port number)",
"create table self (num number primary key, id binary, age timestamp)"]
try:
[c.execute(s) for s in statements]
except sqlite3.OperationalError:
pass
else:
self.store.commit()
c.close()
def _closeDB(self):
"""Close the database"""
self.store.close()
def _loadSelfNode(self):
"""Load the root node"""
if DEBUG:
print("Debug: DHT - loadSelfNode")
# Get ID
c = self.store.cursor()
c.execute('select id, age from self where num = 0')
data = c.fetchone()
# Clean if too old
if not data or time() - data[1] > 86400*5: # more than 5 days old
id = newID()
c.execute('delete from self')
c.execute("insert into self values (0, ?, ?)", (sqlite3.Binary(id), time()))
c.execute('delete from kv')
c.execute('delete from nodes')
self.store.commit()
else:
id = str(data[0])
c.close()
# Load self node
self.node = self.Node().init(id, self.host, self.port)
def _saveSelfNode(self):
"""Save the root node"""
if DEBUG:
print("Debug: DHT - saveSelfNode")
c = self.store.cursor()
c.execute('delete from self')
c.execute("insert into self values (0, ?, ?)", (sqlite3.Binary(self.node.id), time()))
self.store.commit()
c.close()
def _loadRoutingTable(self):
"""Load routing table from the database"""
self.table = KTable(self.node)
c = self.store.cursor()
c.execute("select id, host, port from nodes")
for row in c:
n = self.Node().init(str(row[0]), row[1], row[2])
self.table.insertNode(n, contacted = False)
c.close()
if DEBUG:
print("Debug: DHT - nodes loaded:", self.stats())
def _saveRoutingTable(self):
"""Save routing table nodes to the database"""
if DEBUG:
print("Debug: DHT - saveRoutingTable")
c = self.store.cursor()
c.execute("delete from nodes")
for bucket in self.table.buckets:
for node in bucket.l:
c.execute("insert into nodes values (?, ?, ?)", (sqlite3.Binary(node.id), node.host, node.port))
self.store.commit()
c.close()
def _updateDB(self):
"""Save info to database"""
if DEBUG:
print("Debug: DHT - updateDB")
self._saveSelfNode()
self._saveRoutingTable()
if DEBUG:
print("Debug: DHT - updateDB completed")
def _flushExpired(self):
"""Clean old values from database"""
c = self.store.cursor()
c.execute("delete from kv where age > ?", (KE_AGE,))
self.store.commit()
c.close()
def _retrieveValue(self, key):
"""Returns the value found for key in local table"""
values = []
c = self.store.cursor()
c.execute("select value from kv where key = ?", (sqlite3.Binary(key),))
for row in c:
values.append(str(row[0]))
c.close()
return values[:20]
def _storeValue(self, key, value):
"""Stores <key:value> pair in the database"""
c = self.store.cursor()
try:
c.execute("insert into kv values (?, ?, ?);", (sqlite3.Binary(key), sqlite3.Binary(value), time()))
except sqlite3.IntegrityError:
c.execute("update kv set age = ? where key = ? and value = ?", (time(), sqlite3.Binary(key), sqlite3.Binary(value)))
self.store.commit()
c.close()
####################
# Automatic updates
####################
def _checkpoint(self):
"""Make some saving and refreshing once in a while"""
if DEBUG:
print("Debug: DHT - checkpoint")
# Save DB to disk
self._updateDB()
# Find close nodes
self.findCloseNodes()
# Refresh Table
self.refreshTable()
self.rawserver.add_task(self._checkpoint, randrange(int(CHECKPOINT_INTERVAL * .9), int(CHECKPOINT_INTERVAL * 1.1)))
def _cleanDataBase(self):
self._flushExpired()
self.rawserver.add_task(self._cleanDataBase, KE_DELAY)
####################
# Interface
####################
def addContact(self, host, port):
"""
Ping this node and add the contact info to the table on pong!
"""
# Validation
if not isinstance(port, int):
port = int(port)
if not isinstance(host, str):
host = str(host)
if host in self.contacts:
return False
self.contacts.append(host)
# Add Contact
if is_valid_ip(host):
n = self.Node().init(NULL_ID, host, port)
self.sendPing(n)
else:
Thread(target = self.addRouterContact, args = [host, port]).start()
def addRouterContact(self, host, port):
try:
host = socket.gethostbyname(host)
except socket.error:
return False
n = self.Node().init(NULL_ID, host, port)
self.sendPing(n)
return True
def insertNode(self, n, contacted = True):
"""
Insert a node in our local table, pinging oldest contact in bucket, if necessary
If all you have is a host/port, then use addContact, which calls this method after
receiving the PONG from the remote node. The reason for the seperation is we can't insert
a node into the table without it's peer-ID. That means of course the node passed into this
method needs to be a properly formed Node object with a valid ID.
"""
old = self.table.insertNode(n, contacted = contacted)
if old and (clock() - old.lastSeen) > MIN_PING_INTERVAL and old.id != self.node.id:
# the bucket is full, check to see if old node is still around and if so, replace it
## these are the callbacks used when we ping the oldest node in a bucket
def _staleNodeHandler(oldnode=old, newnode = n):
""" called if the pinged node never responds """
self.table.replaceStaleNode(old, newnode)
def _notStaleNodeHandler(dict, old=old):
""" called when we get a pong from the old node """
dict = dict['rsp']
if dict['id'] == old.id:
self.table.justSeenNode(old.id)
try:
df = old.ping(self.node.id)
except KrpcGenericError:
_staleNodeHandler()
else:
df.addCallbacks(_notStaleNodeHandler, _staleNodeHandler)
def findCloseNodes(self, callback=lambda a: None):
"""
This does a findNode on the ID one away from our own.
This will allow us to populate our table with nodes on our network closest to our own.
This is called as soon as we start up with an empty table
"""
if DEBUG:
print("Debug: DHT - findCloseNodes")
id = self.node.id[:-1] + chr((ord(self.node.id[-1]) + 1) % 256)
self.findNode(id, callback)
def refreshTable(self, force = False, callback=lambda a: None):
"""
Refresh the table
force=True will refresh table regardless of last bucket access time
"""
if DEBUG:
print("Debug: DHT - refreshTable")
for bucket in self.table.buckets:
if force or (clock() - bucket.lastAccessed >= BUCKET_STALENESS):
id = newIDInRange(bucket.min, bucket.max)
self.findNode(id, callback)
def stats(self):
"""
Returns the number of contacts in our routing table
"""
return reduce(lambda a, b: a + len(b.l), self.table.buckets, 0)
####################
# RPC Handler
####################
def krpc_ping(self, id, _krpc_sender, **kwargs):
"""Incoming RPC: got ping"""
if len(id) != 20:
raise KrpcProtocolError("invalid id length: %d" % len(id))
n = self.Node().init(id, *_krpc_sender)
self.insertNode(n, contacted = False)
return {"id" : self.node.id}
def krpc_find_node(self, target, id, _krpc_sender, **kwargs):
"""Incoming RPC: got find_node"""
if len(id) != 20:
raise KrpcProtocolError("invalid id length: %d" % len(id))
if len(target) != 20:
raise KrpcProtocolError("invalid target id length: %d" % len(target))
nodes = self.table.findNodes(target)
nodes = map(lambda node: node.senderDict(), nodes)
n = self.Node().init(id, *_krpc_sender)
self.insertNode(n, contacted = False)
return {"nodes" : encodeNodes(nodes), "id" : self.node.id}
def krpc_get_peers(self, id, info_hash, _krpc_sender, **kwargs):
"""Incoming RPC: got get_peers"""
if len(id) != 20:
raise KrpcProtocolError("invalid id length: %d" % len(id))
if len(info_hash) != 20:
raise KrpcProtocolError("invalid info_hash length: %d" % len(info_hash))
if id == NULL_ID:
raise KrpcProtocolError("invalid id (NULL ID)")
n = self.Node().init(id, *_krpc_sender)
self.insertNode(n, contacted = False)
l = self._retrieveValue(info_hash)
if len(l) > 0:
return {'values' : l, "id": self.node.id,
"token" : self.tokensHandler.tokenToSend(info_hash, *_krpc_sender)}
else:
nodes = self.table.findNodes(info_hash)
nodes = map(lambda node: node.senderDict(), nodes)
return {'nodes' : encodeNodes(nodes), "id": self.node.id,
"token" : self.tokensHandler.tokenToSend(info_hash, *_krpc_sender)}
def krpc_announce_peer(self, id, info_hash, port, token, _krpc_sender, **kwargs):
"""Incoming RPC: got announce_peer"""
if len(id) != 20:
raise KrpcProtocolError("invalid id length: %d" % len(id))
if len(info_hash) != 20:
raise KrpcProtocolError("invalid info_hash length: %d" % len(info_hash))
if not isinstance(port, int):
try:
port = int(port)
except:
raise KrpcProtocolError("invalid port")
if not self.tokensHandler.checkToken(token, info_hash, *_krpc_sender):
raise KrpcProtocolError("Got invalid token")
# TODO: add a limit: maximum 3 info_hash announces per peer
ip = _krpc_sender[0]
if ip not in self.announce:
self.announce[ip] = []
if info_hash not in self.announce[ip]:
self.announce[ip].append(info_hash)
if len(self.announce[ip]) > 3:
raise KrpcGenericError("I only allow 3 infohash announces per peer!")
self._storeValue(info_hash, encodePeer((_krpc_sender[0], port)))
n = self.Node().init(id, *_krpc_sender)
self.insertNode(n, contacted = False)
return {"id" : self.node.id}
def sendPing(self, node, callback=None):
"""
Ping a node
"""
def _pongHandler(dict, node=node, table=self.table, callback=callback):
_krpc_sender = dict['_krpc_sender']
id = dict['rsp']['id']
if len(id) == 20:
n = self.Node().init(dict['rsp']['id'], *_krpc_sender)
table.insertNode(n)
if callback:
callback()
def _defaultPong(err, node=node, table=self.table, callback=callback):
table.nodeFailed(node)
if callback:
callback()
try:
df = node.ping(self.node.id)
except KrpcGenericError:
_defaultPong()
else:
df.addCallbacks(_pongHandler,_defaultPong)
def findNode(self, id, callback, errback = None):
"""
Returns the the k closest nodes to that ID from the global table
"""
# get K nodes out of local table/cache
nodes = self.table.findNodes(id)
d = Deferred()
if errback:
d.addCallbacks(callback, errback)
else:
d.addCallback(callback)
# create our search state
state = FindNode(self, id, d.callback)
self.rawserver.add_task(state.goWithNodes, 0, [nodes])
def getPeers(self, key, callback, searchlocal = True, donecallback = None):
"""
Get Value from global table
"""
if not hasattr(self, "store"):
self.rawserver.add_task(self.getPeers, 3, [key, callback, searchlocal, donecallback])
return
# get locals
if searchlocal:
l = self._retrieveValue(key)
if len(l) > 0:
l = decodePeers(l)
self.rawserver.add_task(callback, 0, [l])
else:
l = []
# create our search state
nodes = self.table.findNodes(key)
state = GetValue(self, key, callback, "getPeers", donecallback)
self.rawserver.add_task(state.goWithNodes, 0, [nodes, l])
def announcePeer(self, key, value, callback=None):
"""
Store Value in global table
"""
def _storeValueForKey(nodes, key=key, value=value, response=callback , table=self.table):
if not response:
# default callback
def _storedValueHandler(sender):
pass
response=_storedValueHandler
action = StoreValue(self, key, value, response, "announcePeer")
self.rawserver.add_task(action.goWithNodes, 0, [nodes])
# this call is asynch
self.findNode(key, _storeValueForKey)
def getPeersAndAnnounce(self, key, value, callback, searchlocal = True):
"""
Get value and store it
"""
def doneCallback(nodes, key = key, value = value):
action = StoreValue(self, key, value, None, "announcePeer")
self.rawserver.add_task(action.goWithNodes, 0, [nodes])
self.getPeers(key, callback, searchlocal, doneCallback)
return 1
except ValueError:
return 0
def track(args):
if len(args) == 0:
print formatDefinitions(defaults, 80)
return
try:
config, files = parseargs(args, defaults, 0, 0)
except ValueError, e:
print 'error: ' + str(e)
print 'run with no arguments for parameter explanations'
return
r = RawServer(Event(), config['timeout_check_interval'],
config['socket_timeout'])
t = Tracker(config, r)
r.bind(config['port'], config['bind'], True)
r.listen_forever(HTTPHandler(t.get,
config['min_time_between_log_flushes']))
t.save_dfile()
print '# Shutting down: ' + isotime()
def size_format(s):
if (s < 1024):
r = str(s) + 'B'
elif (s < 1048576):
r = str(int(s / 1024)) + 'KiB'
elif (s < 1073741824l):
r = str(int(s / 1048576)) + 'MiB'
class LaunchMany:
def __init__(self, config, Output):
try:
self.config = config
self.Output = Output
self.torrent_dir = config['torrent_dir']
self.torrent_cache = {}
self.file_cache = {}
self.blocked_files = {}
self.scan_period = config['parse_dir_interval']
self.stats_period = config['display_interval']
self.torrent_list = []
self.downloads = {}
self.counter = 0
self.doneflag = Event()
self.hashcheck_queue = []
self.hashcheck_current = None
self.rawserver = RawServer(self.doneflag,
config['timeout_check_interval'],
config['timeout'],
ipv6_enable=config['ipv6_enabled'],
failfunc=self.failed,
errorfunc=self.exchandler)
upnp_type = UPnP_test(config['upnp_nat_access'])
while True:
try:
self.listen_port = self.rawserver.find_and_bind(
config['minport'],
config['maxport'],
config['bind'],
ipv6_socket_style=config['ipv6_binds_v4'],
upnp=upnp_type,
randomizer=config['random_port'])
break
except socket.error as e:
if upnp_type and e == UPnP_ERROR:
self.Output.message(
'WARNING: COULD NOT FORWARD VIA UPnP')
upnp_type = 0
continue
self.failed("Couldn't listen - " + str(e))
return
self.ratelimiter = RateLimiter(self.rawserver.add_task,
config['upload_unit_size'])
self.ratelimiter.set_upload_rate(config['max_upload_rate'])
self.handler = MultiHandler(self.rawserver, self.doneflag, config)
seed(createPeerID())
self.rawserver.add_task(self.scan, 0)
self.rawserver.add_task(self.stats, 0)
self.handler.listen_forever()
self.Output.message('shutting down')
self.hashcheck_queue = []
for hash in self.torrent_list:
self.Output.message('dropped "{}"'.format(
self.torrent_cache[hash]['path']))
self.downloads[hash].shutdown()
self.rawserver.shutdown()
except:
data = StringIO()
print_exc(file=data)
Output.exception(data.getvalue())
def scan(self):
self.rawserver.add_task(self.scan, self.scan_period)
r = parsedir(self.torrent_dir,
self.torrent_cache,
self.file_cache,
self.blocked_files,
return_metainfo=True,
errfunc=self.Output.message)
(self.torrent_cache, self.file_cache, self.blocked_files, added,
removed) = r
for hash, data in removed.iteritems():
self.Output.message('dropped "{}"'.format(data['path']))
self.remove(hash)
for hash, data in added.iteritems():
self.Output.message('added "{}"'.format(data['path']))
self.add(hash, data)
def stats(self):
self.rawserver.add_task(self.stats, self.stats_period)
data = []
for hash in self.torrent_list:
cache = self.torrent_cache[hash]
if self.config['display_path']:
name = cache['path']
else:
name = cache['name']
size = cache['length']
d = self.downloads[hash]
progress = '0.0%'
peers = 0
seeds = 0
seedsmsg = "S"
dist = 0.0
uprate = 0.0
dnrate = 0.0
upamt = 0
dnamt = 0
t = 0
if d.is_dead():
status = 'stopped'
elif d.waiting:
status = 'waiting for hash check'
elif d.checking:
status = d.status_msg
progress = '{:.1%}'.format(d.status_done)
else:
stats = d.statsfunc()
s = stats['stats']
if d.seed:
status = 'seeding'
progress = '100.0%'
seeds = s.numOldSeeds
seedsmsg = "s"
dist = s.numCopies
else:
if s.numSeeds + s.numPeers:
t = stats['time']
if t == 0: # unlikely
t = 0.01
status = fmttime(t)
else:
t = -1
status = 'connecting to peers'
progress = '{:.1%}'.format(stats['frac'])
seeds = s.numSeeds
dist = s.numCopies2
dnrate = stats['down']
peers = s.numPeers
uprate = stats['up']
upamt = s.upTotal
dnamt = s.downTotal
if d.is_dead() or d.status_errtime + 300 > clock():
msg = d.status_err[-1]
else:
msg = ''
data.append((name, status, progress, peers, seeds, seedsmsg, dist,
uprate, dnrate, upamt, dnamt, size, t, msg))
stop = self.Output.display(data)
if stop:
self.doneflag.set()
def remove(self, hash):
self.torrent_list.remove(hash)
self.downloads[hash].shutdown()
del self.downloads[hash]
def add(self, hash, data):
c = self.counter
self.counter += 1
x = ''
for i in xrange(3):
x = mapbase64[c & 0x3F] + x
c >>= 6
peer_id = createPeerID(x)
d = SingleDownload(self, hash, data['metainfo'], self.config, peer_id)
self.torrent_list.append(hash)
self.downloads[hash] = d
d.start()
def saveAs(self, hash, name, saveas, isdir):
x = self.torrent_cache[hash]
style = self.config['saveas_style']
if style == 1 or style == 3:
if saveas:
saveas = os.path.join(saveas, x['file'][:-1 - len(x['type'])])
else:
saveas = x['path'][:-1 - len(x['type'])]
if style == 3:
if not os.path.isdir(saveas):
try:
os.mkdir(saveas)
except:
raise OSError("couldn't create directory for {} ({})"
''.format(x['path'], saveas))
if not isdir:
saveas = os.path.join(saveas, name)
else:
if saveas:
saveas = os.path.join(saveas, name)
else:
saveas = os.path.join(os.path.split(x['path'])[0], name)
if isdir and not os.path.isdir(saveas):
try:
os.mkdir(saveas)
except:
raise OSError("couldn't create directory for {} ({})".format(
x['path'], saveas))
return saveas
def hashchecksched(self, hash=None):
if hash:
self.hashcheck_queue.append(hash)
if not self.hashcheck_current:
self._hashcheck_start()
def _hashcheck_start(self):
self.hashcheck_current = self.hashcheck_queue.pop(0)
self.downloads[self.hashcheck_current].hashcheck_start(
self.hashcheck_callback)
def hashcheck_callback(self):
self.downloads[self.hashcheck_current].hashcheck_callback()
if self.hashcheck_queue:
self._hashcheck_start()
else:
self.hashcheck_current = None
def died(self, hash):
if hash in self.torrent_cache:
self.Output.message('DIED: "{}"'.format(
self.torrent_cache[hash]['path']))
def was_stopped(self, hash):
try:
self.hashcheck_queue.remove(hash)
except:
pass
if self.hashcheck_current == hash:
self.hashcheck_current = None
if self.hashcheck_queue:
self._hashcheck_start()
def failed(self, s):
self.Output.message('FAILURE: ' + s)
def exchandler(self, s):
self.Output.exception(s)
except OSError, e:
errorfunc("Couldn't allocate dir - " + str(e))
return
finflag = Event()
ann = [None]
myid = 'M' + version.replace('.', '-')
myid = myid + ('-' * (8 - len(myid))) + b2a_hex(sha(repr(time()) + ' ' + str(getpid())).digest()[-6:])
seed(myid)
pieces = [info['pieces'][x:x+20] for x in xrange(0,
len(info['pieces']), 20)]
def failed(reason, errorfunc = errorfunc, doneflag = doneflag):
doneflag.set()
if reason is not None:
errorfunc(reason)
rawserver = RawServer(doneflag, config['timeout_check_interval'], config['timeout'], errorfunc = errorfunc, maxconnects = config['max_allow_in'])
try:
try:
storage = Storage(files, open, path.exists, path.getsize)
except IOError, e:
errorfunc('trouble accessing files - ' + str(e))
return
def finished(finfunc = finfunc, finflag = finflag,
ann = ann, storage = storage, errorfunc = errorfunc):
finflag.set()
try:
storage.set_readonly()
except (IOError, OSError), e:
errorfunc('trouble setting readonly at end - ' + str(e))
if ann[0] is not None:
ann[0](1)
def download(params,
filefunc,
statusfunc,
finfunc,
errorfunc,
doneflag,
cols,
pathFunc=None,
presets={},
exchandler=None,
failed=_failfunc,
paramfunc=None):
try:
config = parse_params(params, presets)
except ValueError as e:
failed(
'error: {}\nrun with no args for parameter explanations'.format(e))
return
if not config:
errorfunc(get_usage())
return
myid = createPeerID()
random.seed(myid)
rawserver = RawServer(doneflag,
config['timeout_check_interval'],
config['timeout'],
ipv6_enable=config['ipv6_enabled'],
failfunc=failed,
errorfunc=exchandler)
upnp_type = UPnP_test(config['upnp_nat_access'])
try:
listen_port = rawserver.find_and_bind(
config['minport'],
config['maxport'],
config['bind'],
ipv6_socket_style=config['ipv6_binds_v4'],
upnp=upnp_type,
randomizer=config['random_port'])
except socket.error as e:
failed("Couldn't listen - " + str(e))
return
response = get_response(config['responsefile'], config['url'], failed)
if not response:
return
infohash = hashlib.sha1(bencode(response['info'])).digest()
d = BT1Download(statusfunc, finfunc, errorfunc, exchandler, doneflag,
config, response, infohash, myid, rawserver, listen_port)
if not d.saveAs(filefunc):
return
if pathFunc:
pathFunc(d.getFilename())
hashcheck = d.initFiles(old_style=True)
if not hashcheck:
return
if not hashcheck():
return
if not d.startEngine():
return
d.startRerequester()
d.autoStats()
statusfunc(activity='connecting to peers')
if paramfunc:
paramfunc({
# change_max_upload_rate(<int KiB/sec>)
'max_upload_rate': d.setUploadRate,
# change_max_uploads(<int max uploads>)
'max_uploads': d.setConns,
'listen_port': listen_port, # int
'peer_id': myid, # string
'info_hash': infohash, # string
# start_connection((<string ip>, <int port>), <peer id>)
'start_connection': d._startConnection,
})
rawserver.listen_forever(d.getPortHandler())
d.shutdown()
def download(params, filefunc, statusfunc, finfunc, errorfunc, doneflag, cols,
pathFunc=None, presets={}, exchandler=None, failed=_failfunc,
paramfunc=None):
try:
config = parse_params(params, presets)
except ValueError as e:
failed('error: {}\nrun with no args for parameter explanations'.format(
e))
return
if not config:
errorfunc(get_usage())
return
myid = createPeerID()
random.seed(myid)
rawserver = RawServer(doneflag, config['timeout_check_interval'],
config['timeout'],
ipv6_enable=config['ipv6_enabled'],
failfunc=failed, errorfunc=exchandler)
upnp_type = UPnP_test(config['upnp_nat_access'])
try:
listen_port = rawserver.find_and_bind(
config['minport'], config['maxport'], config['bind'],
ipv6_socket_style=config['ipv6_binds_v4'], upnp=upnp_type,
randomizer=config['random_port'])
except socket.error as e:
failed("Couldn't listen - " + str(e))
return
response = get_response(config['responsefile'], config['url'], failed)
if not response:
return
infohash = hashlib.sha1(bencode(response['info'])).digest()
d = BT1Download(statusfunc, finfunc, errorfunc, exchandler, doneflag,
config, response, infohash, myid, rawserver, listen_port)
if not d.saveAs(filefunc):
return
if pathFunc:
pathFunc(d.getFilename())
hashcheck = d.initFiles(old_style=True)
if not hashcheck:
return
if not hashcheck():
return
if not d.startEngine():
return
d.startRerequester()
d.autoStats()
statusfunc(activity='connecting to peers')
if paramfunc:
paramfunc({
# change_max_upload_rate(<int KiB/sec>)
'max_upload_rate': d.setUploadRate,
# change_max_uploads(<int max uploads>)
'max_uploads': d.setConns,
'listen_port': listen_port, # int
'peer_id': myid, # string
'info_hash': infohash, # string
# start_connection((<string ip>, <int port>), <peer id>)
'start_connection': d._startConnection,
})
rawserver.listen_forever(d.getPortHandler())
d.shutdown()
class ClientHandler:
def __init__(self, co):
self.co = co
self.socket = {}
self.rs = RawServer(Event(), 100, 1000)
self.nh = NetworkHandler(self)
# Functions called from lower level code
def message_came_in(self, s, data):
try:
msg = bdecode(data)
except ValueError:
self.close(s)
raise NetworkError, 'garbage data'
if msg.has_key('error'):
raise ServerError, msg['error']
socket = self.socket[s]
srp = socket['srp']
if socket['state'] == 1:
K, m = self.auth.client_key(msg['s'], msg['B'], msg['u'],
srp['keys'])
socket['key'], socket['m_out'] = K, m
self._send_msg(s, {'m': socket['m_out'].digest()})
socket['state'] = 2
elif socket['state'] == 2:
socket['m_in'] = SRP.host_authenticator(socket['key'],
srp['keys'][0],
socket['m_out'].digest())
if socket['m_in'].digest() != msg['auth']:
raise ServerError, 'Bad host authentication'
return
self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
self.rs.doneflag.set()
elif socket['state'] == 3:
self.socket[s]['hash'] = msg['hash']
self.rs.doneflag.set()
elif socket['state'] == 4:
self.close(s)
secret = crypt(msg['secret'], socket['key'])[0]
self.auth.save_secret(secret)
self.rs.doneflag.set()
elif socket['state'] == 5:
self.close(s)
self.rs.doneflag.set()
elif socket['state'] == 6:
if len(msg['salt']) < 20:
self._send_error(s, None, 'Bad salt length')
self.close(s)
raise NetworkError, 'Bad salt from server'
salt = random_string(20)
key = self.auth.session_key(salt, msg['salt'])
socket['m_in'] = hmac.new(key, '', sha)
key = self.auth.session_key(msg['salt'], salt)
socket['m_out'] = hmac.new(key, '', sha)
self._send_msg(s, {'auth': socket['m_in'].digest(), 'salt': salt})
socket['state'] = 7
elif socket['state'] == 7:
if msg['auth'] != socket['m_out'].digest():
self._send_error(s, None, 'Bad auth')
self.close(s)
raise NetworkError, 'Bad server auth'
self._req_mode(s, 1)
self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
self.socket[s] = [{}, {}, {}, [], 1]
self.rs.doneflag.set()
else:
self.close(s)
def connection_flushed(self, s):
queue = self.socket[s][Queue]
socket = self.socket[s]
socket[Flushed] = 1
while len(queue) and socket[Flushed] == 1:
mid, msg = queue.pop(0)
diff = read_diff(self.co, msg['handle'], msg['changenum'], None)
socket[Flushed] = self._send_response(s, mid, {'diff': diff})
def connection_lost(self, s, msg):
del self.socket[s]
raise NetworkError, msg
def request_came_in(self, s, mid, data):
try:
msg = bdecode(data)
except ValueError:
self.close(s)
raise NetworkError, 'garbage request'
self.request_handlers[msg['request']](self, s, mid, msg)
def response_came_in(self, s, mid, data):
try:
msg = bdecode(data)
except ValueError:
self.close(s)
raise NetworkError, 'bad data from server'
rstate = self.socket[s][Request][mid]
if msg.has_key('error'):
# XXX: grody hack, diffs which don't belong in history
if rstate['request'] == 'get diff':
msg['diff'] = zlib.compress(bencode(''))
else:
raise ServerError, msg['error']
if self.response_handlers[rstate['request']](self, s, mid, msg,
rstate):
del self.socket[s][Request][mid]
# request handlers
def _request_get_change(self, s, mid, msg):
resp = {
'changeset': self.co.lcrepo.get(msg['changenum'], txn=self.txn)
}
self._send_response(s, mid, resp)
def _request_get_diff(self, s, mid, msg):
if self.socket[s][Flushed] == 1:
diff = read_diff(self.co, msg['handle'], msg['changenum'],
self.txn)
self.socket[s][Flushed] = self._send_response(
s, mid, {'diff': diff})
else:
self.socket[s][Queue].append((mid, msg))
request_handlers = {
'get change': _request_get_change,
'get diff': _request_get_diff
}
# response handlers
def _response_get_head(self, s, mid, msg, rstate):
return self._merge_change(s, mid, msg['head'])
def _merge_change(self, s, mid, head):
lstate = self.socket[s][Request][mid]
lstate['head'] = head
self.socket[s][UpdateInfo]['head'] = head
if self.co.lcrepo.has_key(head):
named, modified, added, deleted = \
handles_in_branch(self.co, lstate['heads'], [head], None)
self._update_checks(s, mid, named, modified)
self._update_handle_list(s, lstate, named, modified, added,
deleted)
self._update_finish(s, lstate)
self.socket[s][UpdateInfo]['head'] = head
self.rs.doneflag.set()
return 1
rid = self._get_change(s, head)
self.socket[s][Request][rid] = {
'request': 'get change',
'changenum': head,
'ref': mid
}
lstate['requests'][head] = 1
lstate['count'] = 1
return 0
def _response_get_change(self, s, mid, msg, rstate):
lstate = self.socket[s][Request][rstate['ref']]
if sha.new(msg['changeset']).digest() != rstate['changenum']:
raise ServerError, 'bad changeset'
# write it out, decode and eject from memory
write_changeset(self.co, rstate['changenum'], msg['changeset'],
lstate['txn'])
changeset = bdecode(msg['changeset'])
lstate['changes'][rstate['changenum']] = changeset
del msg['changeset']
# get any precursors we don't have and haven't yet requested
for change in changeset['precursors']:
if self.co.lcrepo.has_key(change):
continue
if lstate['changes'].has_key(change):
continue
if lstate['requests'].has_key(change):
continue
rid = self._get_change(s, change)
self.socket[s][Request][rid] = {
'request': 'get change',
'changenum': change,
'ref': rstate['ref']
}
lstate['requests'][change] = 1
lstate['count'] += 1
# record all the diffs we'll need to request
diffs = lstate['diffs']
for handle, hinfo in changeset['handles'].items():
if not hinfo.has_key('hash'):
continue
if not diffs.has_key(handle):
diffs[handle] = {}
lstate['counts'][handle] = 0
diffs[handle][rstate['changenum']] = 1
lstate['counts'][handle] += 1
# clean up state
del lstate['requests'][rstate['changenum']]
lstate['count'] -= 1
if lstate['count'] == 0:
sync_history(self.co,
lstate['head'],
lstate['txn'],
cache=lstate['changes'])
named, modified, added, deleted = \
handles_in_branch(self.co,
lstate['heads'], [lstate['head']],
lstate['txn'], cache=lstate['changes'])
del lstate['changes']
self._update_checks(s, rstate['ref'], named, modified)
self._update_handle_list(s, lstate, named, modified, added,
deleted)
handle_list = lstate['handle list']
# get all the related file diffs
for i in xrange(len(handle_list) - 1, -1, -1):
handle = handle_list[i][1]
if not diffs.has_key(handle):
continue
changes = diffs[handle]
requested = 0
for change in changes.keys():
requested = 1
self._queue_diff(s, change, handle, rstate['ref'])
lstate['count'] += requested
self._get_diff(s, rstate['ref'])
if lstate['count'] == 0:
self._update_finish(s, lstate)
del self.socket[s][Request][rstate['ref']]
self.rs.doneflag.set()
return 1
def _response_get_diff(self, s, mid, msg, rstate):
lstate = self.socket[s][Request][rstate['ref']]
# send out the next one
lstate['req-outstanding'] -= 1
self._get_diff(s, rstate['ref'])
diffs = lstate['diffs']
diffs[rstate['handle']][rstate['change']] = msg['diff']
lstate['counts'][rstate['handle']] -= 1
if lstate['counts'][rstate['handle']] == 0:
lstate['count'] -= 1
# XXX: do better ordering
WD = WriteDiff(self.co, rstate['handle'], lstate['txn'])
for change, diff in diffs[rstate['handle']].items():
WD.write(diff, change)
WD.close()
if lstate['modified'].has_key(rstate['handle']):
updates(self.co, self.socket[s][UpdateInfo], lstate,
rstate['handle'])
#del diffs[rstate['handle']]
if lstate['count'] == 0:
self._update_finish(s, lstate)
self.socket[s][UpdateInfo]['head'] = lstate['head']
del self.socket[s][Request][rstate['ref']]
self.rs.doneflag.set()
return 1
def _response_return(self, s, mid, msg, rstate):
self.rs.doneflag.set()
return 1
def _response_list_repositories(self, s, mid, msg, rstate):
self.socket[s][UpdateInfo] = msg['list']
self.rs.doneflag.set()
return 1
response_handlers = {
'get head': _response_get_head,
'get change': _response_get_change,
'get diff': _response_get_diff,
'commit': _response_return,
'create repository': _response_return,
'destroy repository': _response_return,
'list repositories': _response_list_repositories
}
# Internal helper functions
def _update_checks(self, s, mid, named, modified):
lstate = self.socket[s][Request][mid]
check_modified(self.co, modified)
lstate['modified'] = {}
for handle in modified:
lstate['modified'][handle] = 1
self.socket[s][UpdateInfo]['modified'] = []
self.socket[s][UpdateInfo]['modified conflicts'] = []
def _update_handle_list(self, s, lstate, named, modified, added, deleted):
uinfo = self.socket[s][UpdateInfo]
func = lstate['update function']
handles, nconflicts = func(self.co, uinfo, named, modified, added,
deleted, lstate['txn'])
handle_list = lstate['handle list']
for handle, show in handles:
letters = (' ', ' ')
if uinfo['newfiles'].has_key(handle):
letters = ('A', 'A')
elif uinfo['deletes'].has_key(handle):
letters = ('D', 'D')
elif uinfo['names'].has_key(handle):
letters = (' ', 'N')
if nconflicts.has_key(handle):
letters = (letters[0], 'C')
handle_list.append(
(handle_to_filename(self.co, handle,
lstate['txn']), handle, show, letters))
handle_list.sort()
handle_list.reverse()
uinfo['name conflicts'] = nconflicts.keys()
def _update_finish(self, s, lstate):
updates(self.co, self.socket[s][UpdateInfo], lstate, None)
def _req_mode(self, s, mode):
self.nh.req_mode(s, mode)
def _commit(self, s, repo, head):
rid = self._send_request(s, {
'request': 'commit',
'repository': repo,
'changenum': head
})
self.socket[s][Request][rid] = {'request': 'commit'}
def _update(self, s, repo, func, txn):
heads = bdecode(self.co.linforepo.get('heads'))
lstate = {
'request': 'get head',
'txn': txn,
'changes': {},
'requests': {},
'diffs': {},
'count': 0,
'counts': {},
'handle list': [],
'heads': heads,
'reqq': [],
'req-outstanding': 0,
'update function': func
}
if repo.startswith('{') and repo.endswith('}'):
head = binascii.unhexlify(repo[1:-1])
rid = self.nh.next_id(s)
self.socket[s][Request][rid] = lstate
if self._merge_change(s, rid, head):
del self.socket[s][Request][rid]
else:
rid = self._send_request(s, {
'request': 'get head',
'repository': repo
})
self.socket[s][Request][rid] = lstate
return
def _create_repo(self, s, repo):
rid = self._send_request(s, {
'request': 'create repository',
'repository': repo
})
self.socket[s][Request][rid] = {
'request': 'create repository',
'repository': repo
}
def _remove_repo(self, s, repo):
rid = self._send_request(s, {
'request': 'destroy repository',
'repository': repo
})
self.socket[s][Request][rid] = {
'request': 'destroy repository',
'repository': repo
}
def _list_repos(self, s):
rid = self._send_request(s, {'request': 'list repositories'})
self.socket[s][Request][rid] = {'request': 'list repositories'}
def _get_change(self, s, change):
req = {'request': 'get change', 'changenum': change}
return self._send_request(s, req)
def _queue_diff(self, s, change, handle, mid):
rstate = self.socket[s][Request][mid]
rstate['reqq'].append((change, handle))
def _get_diff(self, s, mid):
rstate = self.socket[s][Request][mid]
while len(rstate['reqq']) and rstate['req-outstanding'] <= 40:
change, handle = rstate['reqq'].pop(0)
req = {
'request': 'get diff',
'changenum': change,
'handle': handle
}
state = {
'request': 'get diff',
'ref': mid,
'change': change,
'handle': handle
}
rid = self._send_request(s, req)
self.socket[s][Request][rid] = state
rstate['req-outstanding'] += 1
def _send_msg(self, s, data):
self.nh.send_msg(s, bencode(data))
def _send_error(self, s, mid, msg):
self._send_response(s, mid, {'error': msg})
def _send_request(self, s, data):
return self.nh.send_request(s, bencode(data))
def _send_response(self, s, mid, data):
return self.nh.send_response(s, mid, bencode(data))
# Functions to be called from higher level code
def start_connection(self, dns):
s = self.nh.start_connection(dns)
self.socket[s] = {'state': 0, 'srp': {}}
return s
def close(self, s):
self.nh.close(s)
del self.socket[s]
def get_hash(self, s):
self._send_msg(s, {'op': 'get hash', 'user': self.co.user})
self.socket[s]['state'] = 3
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
secret_hash = self.socket[s]['hash']
del self.socket[s]['hash']
return secret_hash
def srp_auth(self, s):
socket = self.socket[s]
assert socket['state'] == 0 or socket['state'] == 3
srp = socket['srp'] = {}
srp['keys'] = SRP.client_begin()
self._send_msg(s, {
'op': 'srp auth',
'user': self.co.user,
'A': srp['keys'][0]
})
socket['state'] = 1
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def get_secret(self, s):
self._send_msg(s, {'op': 'get secret'})
self.socket[s]['state'] = 4
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
return
def set_password(self, s, password):
socket = self.socket[s]
salt, v = SRP.new_passwd(self.co.user, password)
cypherv = crypt(long_to_string(v), socket['key'])[0]
self._send_msg(s, {'op': 'set password', 's': salt, 'v': cypherv})
socket['state'] = 5
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def secret_auth(self, s):
socket = self.socket[s]
self._send_msg(s, {'op': 'secret auth', 'user': self.co.user})
socket['state'] = 6
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def commit(self, s, repo, changenum, txn):
self.txn = txn
self._commit(s, repo, changenum)
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def update(self, s, repo, func, txn):
self._update(s, repo, func, txn)
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
updateinfo = self.socket[s][UpdateInfo]
del self.socket[s][UpdateInfo]
return updateinfo
def create_repo(self, s, repo):
self._create_repo(s, repo)
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def remove_repo(self, s, repo):
self._remove_repo(s, repo)
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def list_repos(self, s):
self._list_repos(s)
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
rlist = self.socket[s][UpdateInfo]
del self.socket[s][UpdateInfo]
return rlist
