def __init__(self, name, root, datacenter, bt_start_port, webip, webport, webdir, loglevel, lazy, create=False):
self.name = name # cluster name
self.bj_name = self.name+'__'+uuid.uuid4().hex # mDNS name
if webport is not None:
self.httpd = WebUIServer((webip, webport), WebUIHandler)
print 'WebUIServer listening on: http://%s:%d/' % (webip, webport)
else:
self.httpd = type('blank', (object,), {})()
self.httpd.api = {'webdir':webdir,
'webip':webip,
'webport':str(webport),
'lazy':str(lazy),
'name':name,
'btport':str(bt_start_port),
'root':root,
'servicename':self.bj_name,
'hostname':socket.gethostname(),
'datacenter':datacenter,
'mount':'-',
'freespace': 0,
'cluster_freespace': 0,
'version':'v%s' % APP_VERSION
}
self.httpd.bt_handles = {}
self.httpd.peers = {}
self.httpd.nametoaddr = {}
self.bootstrapping = True
self.LOGLEVEL = loglevel
self.lazy = lazy
self.root = os.path.realpath(root)
self.indexdir = os.path.join(self.root, 'meta', 'index').encode(FS_ENCODE)
self.metadir = os.path.join(self.root, 'meta').encode(FS_ENCODE)
self.chunksdir = os.path.join(self.root, 'chunks').encode(FS_ENCODE)
self.tmp = os.path.join(self.root, 'tmp').encode(FS_ENCODE)
if not os.path.isdir(self.root):
os.mkdir(self.root)
self.next_time_to_check_for_undermirrored_files = datetime.datetime.now() + datetime.timedelta(0,10+random.randint(0,30))
self.last_read_file = {}
self.bt_in_progress = set()
self.bt_session = libtorrent.session()
self.bt_session.listen_on(bt_start_port, bt_start_port+10)
pe_settings = libtorrent.pe_settings()
pe_enc_policy = {0:libtorrent.enc_policy.forced, 1:libtorrent.enc_policy.enabled, 2:libtorrent.enc_policy.disabled}
pe_settings.out_enc_policy = libtorrent.enc_policy(pe_enc_policy[0])
pe_settings.in_enc_policy = libtorrent.enc_policy(pe_enc_policy[0])
pe_enc_level = {0:libtorrent.enc_level.plaintext, 1:libtorrent.enc_level.rc4, 2:libtorrent.enc_level.both}
pe_settings.allowed_enc_level = libtorrent.enc_level(pe_enc_level[1])
self.bt_session.set_pe_settings(pe_settings)
self.bt_port = self.bt_session.listen_port()
self.httpd.api['btport'] = self.bt_port
# no libtorrent lsd for private if we use h.connect_peer
self.bt_session.start_lsd()
# self.bt_session.stop_lsd()
# no libtorrent dht for private if we use h.connect_peer
# self.bt_session.start_dht()
self.bt_session.stop_dht()
print 'libtorrent listening on:', self.bt_port
# self.bt_session.add_dht_router('localhost', 10670)
print '...dht_state()', self.bt_session.dht_state()
thread = Thread(target=self.__start_webui)
thread.daemon = True
thread.start()
thread = Thread(target=self.__bonjour_start_listening)
thread.daemon = True
thread.start()
print 'give me a sec to look for other peers...'
time.sleep(2)
cnfn = os.path.join(self.metadir, '__delugefs__', 'cluster_name').encode(FS_ENCODE)
if create:
if os.listdir(self.root):
files = [x for x in os.listdir(self.root) if x!=".git.meta" and x!="chunks" and x!="meta" and x!="tmp" ]
if files:
raise Exception('--create specified, but %s is not empty' % self.root)
if self.httpd.peers:
raise Exception('--create specified, but i found %i peer%s using --id "%s" already' % (len(self.httpd.peers), 's' if len(self.httpd.peers)>1 else '', self.name))
if not os.path.isdir(self.metadir): os.makedirs(self.metadir)
if not os.path.isdir(os.path.dirname(cnfn)): os.mkdir(os.path.dirname(cnfn))
with open(cnfn, 'w') as f:
f.write(self.name)
if not os.path.isdir(self.indexdir): os.makedirs(self.indexdir)
with open(os.path.join(self.indexdir, '.__delugefs_dir__').encode(FS_ENCODE),'w') as f:
f.write("git doesn't track empty dirs, so we add this file.")
else:
if os.path.isfile(cnfn):
with open(cnfn, 'r') as f:
existing_cluster_name = f.read().strip()
if existing_cluster_name != self.name:
raise Exception('a cluster root exists at %s, but its name is "%s", not "%s"' % (self.root, existing_cluster_name, self.name))
else:
if os.listdir(self.root):
raise Exception('root %s is not empty, but no cluster was found' % self.root)
if not self.httpd.peers:
raise Exception('--create not specified, no repo exists at %s and no peers of cluster "%s" found' % (self.root, self.name))
if not os.path.isdir(self.metadir):
raise Exception('no repo exists at %s' % self.metadir)
prune_empty_dirs(self.metadir)
print '='*80
if not os.path.isdir(self.tmp): os.makedirs(self.tmp)
for fn in os.listdir(self.tmp): os.remove(os.path.join(self.tmp,fn).encode(FS_ENCODE))
if not os.path.isdir(self.chunksdir): os.makedirs(self.chunksdir)
self.rwlock = Lock()
self.open_files = {} # used to track opened files except READONLY
self.bootstrapping = False
thread = Thread(target=self.__bonjour_register, args=())
thread.daemon = True
thread.start()
thread = Thread(target=self.__load_local_torrents)
thread.daemon = True
thread.start()
thread = Thread(target=self.__monitor)
thread.daemon = True
thread.start()
class DelugeFS(LoggingMixIn, Operations):
def __init__(self, name, root, datacenter, bt_start_port, webip, webport, webdir, loglevel, lazy, create=False):
self.name = name # cluster name
self.bj_name = self.name+'__'+uuid.uuid4().hex # mDNS name
if webport is not None:
self.httpd = WebUIServer((webip, webport), WebUIHandler)
print 'WebUIServer listening on: http://%s:%d/' % (webip, webport)
else:
self.httpd = type('blank', (object,), {})()
self.httpd.api = {'webdir':webdir,
'webip':webip,
'webport':str(webport),
'lazy':str(lazy),
'name':name,
'btport':str(bt_start_port),
'root':root,
'servicename':self.bj_name,
'hostname':socket.gethostname(),
'datacenter':datacenter,
'mount':'-',
'freespace': 0,
'cluster_freespace': 0,
'version':'v%s' % APP_VERSION
}
self.httpd.bt_handles = {}
self.httpd.peers = {}
self.httpd.nametoaddr = {}
self.bootstrapping = True
self.LOGLEVEL = loglevel
self.lazy = lazy
self.root = os.path.realpath(root)
self.indexdir = os.path.join(self.root, 'meta', 'index').encode(FS_ENCODE)
self.metadir = os.path.join(self.root, 'meta').encode(FS_ENCODE)
self.chunksdir = os.path.join(self.root, 'chunks').encode(FS_ENCODE)
self.tmp = os.path.join(self.root, 'tmp').encode(FS_ENCODE)
if not os.path.isdir(self.root):
os.mkdir(self.root)
self.next_time_to_check_for_undermirrored_files = datetime.datetime.now() + datetime.timedelta(0,10+random.randint(0,30))
self.last_read_file = {}
self.bt_in_progress = set()
self.bt_session = libtorrent.session()
self.bt_session.listen_on(bt_start_port, bt_start_port+10)
pe_settings = libtorrent.pe_settings()
pe_enc_policy = {0:libtorrent.enc_policy.forced, 1:libtorrent.enc_policy.enabled, 2:libtorrent.enc_policy.disabled}
pe_settings.out_enc_policy = libtorrent.enc_policy(pe_enc_policy[0])
pe_settings.in_enc_policy = libtorrent.enc_policy(pe_enc_policy[0])
pe_enc_level = {0:libtorrent.enc_level.plaintext, 1:libtorrent.enc_level.rc4, 2:libtorrent.enc_level.both}
pe_settings.allowed_enc_level = libtorrent.enc_level(pe_enc_level[1])
self.bt_session.set_pe_settings(pe_settings)
self.bt_port = self.bt_session.listen_port()
self.httpd.api['btport'] = self.bt_port
# no libtorrent lsd for private if we use h.connect_peer
self.bt_session.start_lsd()
# self.bt_session.stop_lsd()
# no libtorrent dht for private if we use h.connect_peer
# self.bt_session.start_dht()
self.bt_session.stop_dht()
print 'libtorrent listening on:', self.bt_port
# self.bt_session.add_dht_router('localhost', 10670)
print '...dht_state()', self.bt_session.dht_state()
thread = Thread(target=self.__start_webui)
thread.daemon = True
thread.start()
thread = Thread(target=self.__bonjour_start_listening)
thread.daemon = True
thread.start()
print 'give me a sec to look for other peers...'
time.sleep(2)
cnfn = os.path.join(self.metadir, '__delugefs__', 'cluster_name').encode(FS_ENCODE)
if create:
if os.listdir(self.root):
files = [x for x in os.listdir(self.root) if x!=".git.meta" and x!="chunks" and x!="meta" and x!="tmp" ]
if files:
raise Exception('--create specified, but %s is not empty' % self.root)
if self.httpd.peers:
raise Exception('--create specified, but i found %i peer%s using --id "%s" already' % (len(self.httpd.peers), 's' if len(self.httpd.peers)>1 else '', self.name))
if not os.path.isdir(self.metadir): os.makedirs(self.metadir)
if not os.path.isdir(os.path.dirname(cnfn)): os.mkdir(os.path.dirname(cnfn))
with open(cnfn, 'w') as f:
f.write(self.name)
if not os.path.isdir(self.indexdir): os.makedirs(self.indexdir)
with open(os.path.join(self.indexdir, '.__delugefs_dir__').encode(FS_ENCODE),'w') as f:
f.write("git doesn't track empty dirs, so we add this file.")
else:
if os.path.isfile(cnfn):
with open(cnfn, 'r') as f:
existing_cluster_name = f.read().strip()
if existing_cluster_name != self.name:
raise Exception('a cluster root exists at %s, but its name is "%s", not "%s"' % (self.root, existing_cluster_name, self.name))
else:
if os.listdir(self.root):
raise Exception('root %s is not empty, but no cluster was found' % self.root)
if not self.httpd.peers:
raise Exception('--create not specified, no repo exists at %s and no peers of cluster "%s" found' % (self.root, self.name))
if not os.path.isdir(self.metadir):
raise Exception('no repo exists at %s' % self.metadir)
prune_empty_dirs(self.metadir)
print '='*80
if not os.path.isdir(self.tmp): os.makedirs(self.tmp)
for fn in os.listdir(self.tmp): os.remove(os.path.join(self.tmp,fn).encode(FS_ENCODE))
if not os.path.isdir(self.chunksdir): os.makedirs(self.chunksdir)
self.rwlock = Lock()
self.open_files = {} # used to track opened files except READONLY
self.bootstrapping = False
thread = Thread(target=self.__bonjour_register, args=())
thread.daemon = True
thread.start()
thread = Thread(target=self.__load_local_torrents)
thread.daemon = True
thread.start()
thread = Thread(target=self.__monitor)
thread.daemon = True
thread.start()
'''
Internal functions, alphabetical
'''
def __add_peer(self, service_name, host, addr, bt_port):
print 'adding peer'
if not servicename in self.httpd.peers:
self.httpd.peers[servicename] = Peer(servicename, host, addr, bt_port)
def __add_torrent(self, torrent, path):
uid = torrent['info']['name']
info = libtorrent.torrent_info(torrent)
dat_file = os.path.join(self.chunksdir, uid[:2], uid).encode(FS_ENCODE)
if not os.path.isdir(os.path.dirname(dat_file)): os.mkdir(os.path.dirname(dat_file))
if not os.path.isfile(dat_file):
with open(dat_file,'wb') as f:
pass
h = self.bt_session.add_torrent({'ti':info, 'save_path':os.path.dirname(dat_file)})
for peer in self.httpd.peers.values():
if (peer.bt_port is not None) and (self.httpd.nametoaddr[peer.host] is not None):
if self.LOGLEVEL > 3: print 'adding peer:', (self.httpd.nametoaddr[peer.host], peer.bt_port)
h.connect_peer((self.httpd.nametoaddr[peer.host], peer.bt_port), 0)
if self.LOGLEVEL > 3: print 'added', uid
self.httpd.bt_handles[path] = h
self.bt_in_progress.add(path)
def __bonjour_browse_callback(self, sdRef, flags, interfaceIndex, errorCode, serviceName, regtype, replyDomain):
if errorCode != pybonjour.kDNSServiceErr_NoError:
return
if not (flags & pybonjour.kDNSServiceFlagsAdd):
if serviceName in self.httpd.peers:
del self.httpd.peers[serviceName]
print 'self.httpd.peers', self.httpd.peers
return
resolve_sdRef = pybonjour.DNSServiceResolve(0, interfaceIndex, serviceName, regtype, replyDomain, self.__bonjour_resolve_callback)
try:
while not resolved:
ready = select.select([resolve_sdRef], [], [], 5)
if resolve_sdRef not in ready[0]:
#print 'Resolve timed out'
break
pybonjour.DNSServiceProcessResult(resolve_sdRef)
else:
resolved.pop()
finally:
resolve_sdRef.close()
def __bonjour_register_callback(self, sdRef, flags, errorCode, name, regtype, domain):
if errorCode == pybonjour.kDNSServiceErr_NoError:
print '...bonjour listener', name+'.'+regtype+domain, 'now listening on', self.bt_port
def __bonjour_resolve_callback(self, sdRef, flags, interfaceIndex, errorCode, fullname, hosttarget, port, txtRecord):
if errorCode == pybonjour.kDNSServiceErr_NoError:
if fullname.startswith(self.bj_name):
#print 'ignoring my own service'
return
if not (fullname.startswith(self.name+'__') and ('._delugefs._tcp.' in fullname )):
#print 'ignoring unrelated service', fullname
return
servicename = fullname[:fullname.index('.')]
print 'bonjour resolve found peer', servicename
if not servicename in self.httpd.peers:
self.httpd.peers[servicename] = Peer(servicename, hosttarget)
if '._delugefs._tcp.' in fullname:
self.httpd.peers[servicename].bt_port = port
print 'self.httpd.peers', self.httpd.peers
query_sdRef = pybonjour.DNSServiceQueryRecord(interfaceIndex = interfaceIndex,
fullname = hosttarget,
rrtype = pybonjour.kDNSServiceType_A,
callBack = self.__bonjour_query_record_callback)
try:
while not queried:
ready = select.select([query_sdRef], [], [], 5)
if query_sdRef not in ready[0]:
#print 'Query timed out'
break
pybonjour.DNSServiceProcessResult(query_sdRef)
else:
queried.pop()
finally:
query_sdRef.close()
resolved.append(True)
def __bonjour_query_record_callback(self, sdRef, flags, interfaceIndex, errorCode, fullname, rrtype, rrclass, rdata, ttl):
if errorCode == pybonjour.kDNSServiceErr_NoError:
if self.LOGLEVEL > 3: print ' IP =', socket.inet_ntoa(rdata)
if self.LOGLEVEL > 3: print 'fullname ', fullname
self.httpd.nametoaddr[fullname] = socket.inet_ntoa(rdata)
if fullname in self.httpd.nametoaddr:
self.httpd.nametoaddr[fullname] = socket.inet_ntoa(rdata)
queried.append(True)
return 'pulling from new peer', fullname
def __bonjour_register(self):
print 'registering bonjour listener...'
bjservice = pybonjour.DNSServiceRegister(name=self.bj_name, regtype="_delugefs._tcp",
port=self.bt_port, callBack=self.__bonjour_register_callback)
try:
while True:
ready = select.select([bjservice], [], [])
if bjservice in ready[0]:
pybonjour.DNSServiceProcessResult(bjservice)
except KeyboardInterrupt:
pass
def __bonjour_start_listening(self):
browse_sdRef = pybonjour.DNSServiceBrowse(regtype="_delugefs._tcp", callBack=self.__bonjour_browse_callback)
try:
try:
while True:
ready = select.select([browse_sdRef], [], [])
if browse_sdRef in ready[0]:
pybonjour.DNSServiceProcessResult(browse_sdRef)
except KeyboardInterrupt:
pass
finally:
browse_sdRef.close()
def __check_for_undermirrored_files(self):
if self.lazy: return
if self.next_time_to_check_for_undermirrored_files > datetime.datetime.now(): return
try:
if self.LOGLEVEL > 2: print 'check_for_undermirrored_files @', datetime.datetime.now()
my_uids = set(self.__get_active_info_hashes())
counter = collections.Counter(my_uids)
peer_freespace = {'__self__': self.__get_freespace()}
uid_peers = collections.defaultdict(set)
for uid in my_uids:
uid_peers[uid].add('__self__')
for peer_id, peer in self.httpd.peers.items():
for s in self.__get_peer_active_info_hashes(peer_id, peer.host, peer.datacenter):
counter[s] += 1
uid_peers[s].add(peer_id)
# read from meta/info/servicename/freespace
peer_freespace[peer_id] = self.__get_peer_freespace(peer_id)
if self.LOGLEVEL > 2: print 'counter', counter
if self.LOGLEVEL > 3: print 'peer_freespace', peer_freespace
if len(self.httpd.peers.items()) < 1:
if self.LOGLEVEL > 3: print "can't do anything, since i'm the only peer!"
return
self.httpd.api['cluster_freespace'] = '%0.2fGB' % (sum(peer_freespace.values()) / math.pow(2,30))
if self.LOGLEVEL > 3: print 'cluster_freespace: %s' % self.httpd.api['cluster_freespace']
for root, dirs, files in os.walk(self.indexdir):
#print 'root, dirs, files', root, dirs, files
if root.startswith(os.path.join(self.indexdir, '__delugefs__').encode(FS_ENCODE)): continue
for fn in files:
if fn=='.__delugefs_dir__': continue
fn = os.path.join(root, fn).encode(FS_ENCODE)
e = get_torrent_dict(fn)
if not e:
if self.LOGLEVEL > 2: print 'not a torrent?', fn
continue
uid = e['info']['name']
size = e['info']['length']
path = fn[len(self.indexdir):]
if counter[uid] < 2:
peer_freespace_list = sorted([x for x in peer_freespace.items() if x[0] not in uid_peers[uid]], lambda x,y: x[1]<y[1])
if self.LOGLEVEL > 2: print 'peer_freespace_list', peer_freespace_list
for best_peer_id, freespace in peer_freespace_list:
if uid in my_uids and best_peer_id=='__self__':
best_peer_id = peer_freespace_list[1][0]
peer_freespace[best_peer_id] -= size
if self.LOGLEVEL > 2: print 'need to rep', path #, 'to', best_peer_id
if '__self__'==best_peer_id:
self.__please_mirror(path)
break
elif counter[uid] > 3:
if self.LOGLEVEL > 2: print 'uid_peers', uid_peers
peer_freespace_list = sorted([x for x in peer_freespace.items() if x[0] in uid_peers[uid]], lambda x,y: x[1]>y[1])
if self.LOGLEVEL > 2: print 'peer_freespace_list2', peer_freespace_list
for best_peer_id, freespace in peer_freespace_list:
if '__self__'==best_peer_id:
if self.__please_stop_mirroring(path): break
if self.LOGLEVEL > 2: print '__please_stop_mirroring', path, 'sent to'#, best_peer_id
except Exception as e:
traceback.print_exc()
self.next_time_to_check_for_undermirrored_files = datetime.datetime.now() + datetime.timedelta(0,SECONDS_TO_NEXT_CHECK+random.randint(0,10*(1+len(self.httpd.peers))))
def __finalize(self, path, olduid):
if self.LOGLEVEL > 2: print 'finalize', path, olduid
try:
# try sha256 before making torrent
old_tmp_fn = os.path.join(self.tmp, olduid).encode(FS_ENCODE)
uid = sha256sum(old_tmp_fn, 4096)
tmp_fn = os.path.join(self.tmp, uid).encode(FS_ENCODE)
path = None
if self.LOGLEVEL > 3: print 'olduid',olduid
for k,v in self.open_files.items():
if self.LOGLEVEL > 3: print 'k',k
if self.LOGLEVEL > 3: print 'v',v
if olduid.strip() == v:
if self.LOGLEVEL > 3: print 'found k',k
path = k
if self.LOGLEVEL > 3: print 'path',path
if self.LOGLEVEL > 3: print 'old_tmp',old_tmp_fn
if self.LOGLEVEL > 3: print 'sha_tmp_fn',tmp_fn
os.rename(old_tmp_fn, tmp_fn)
fs = libtorrent.file_storage()
#print tmp_fn
libtorrent.add_files(fs, tmp_fn)
t = libtorrent.create_torrent(fs)
t.set_creator("DelugeFS");
libtorrent.set_piece_hashes(t, self.tmp)
tdata = t.generate()
#print tdata
fn = os.path.join(self.indexdir, path).encode(FS_ENCODE)
with open(fn, 'wb') as f:
f.write(libtorrent.bencode(tdata))
#print 'wrote', fn
dat_dir = os.path.join(self.chunksdir, uid[:2]).encode(FS_ENCODE)
if not os.path.isdir(dat_dir):
try: os.mkdir(dat_dir)
except: pass
shutil.copyfile(tmp_fn, os.path.join(dat_dir, uid).encode(FS_ENCODE))
os.remove(tmp_fn)
#print 'committing', fn
self.__add_torrent(tdata, path)
del self.open_files[path]
except Exception as e:
traceback.print_exc()
raise e
def __get_active_info_hashes(self):
self.next_time_to_check_for_undermirrored_files = datetime.datetime.now() + datetime.timedelta(0,SECONDS_TO_NEXT_CHECK+random.randint(0,10*(1+len(self.httpd.peers))))
active_info_hashes = []
for k,h in self.httpd.bt_handles.items():
if not h: continue
try:
active_info_hashes.append(str(h.get_torrent_info().name()))
except:
traceback.print_exc()
del self.httpd.bt_handles[k]
if self.LOGLEVEL > 2: print 'active_info_hashes', active_info_hashes
return active_info_hashes
def __get_peer_active_info_hashes(self, servicename, host, datacenter):
# TODO read from metadir/datacenter/host/servicename/active
active_info_hashes = []
return active_info_hashes
def __get_freespace(self):
f = os.statvfs(self.root)
bsize = f[statvfs.F_BSIZE]
if bsize > 4096: bsize = 512
freebytes = (bsize * f[statvfs.F_BFREE])
# read, then write to only if different
fn = os.path.join(self.metadir, 'info', self.bj_name, 'freespace').encode(FS_ENCODE)
if not os.path.isdir(os.path.dirname(fn)): os.makedirs(os.path.dirname(fn))
old_freebytes = 0
if os.path.isfile(fn):
with open(fn, 'r') as f:
old_freebytes = f.read().strip()
if old_freebytes != freebytes:
with open(fn,'w') as f:
f.write("%i" % freebytes)
self.httpd.api['freespace'] = '%0.2fGB' % (freebytes / math.pow(2,30))
return freebytes
def __get_peer_freespace(self, servicename):
# very simply read from the info
fn = os.path.join(self.metadir, 'info', servicename, 'freespace').encode(FS_ENCODE)
if not os.path.isdir(os.path.dirname(fn)): os.makedirs(os.path.dirname(fn))
old_freebytes = 0
if os.path.isfile(fn):
with open(fn, 'r') as f:
try:
old_freebytes = int(f.read().strip())
except:
pass
return old_freebytes
def __load_local_torrents(self):
#print 'self.indexdir', self.indexdir
for root, dirs, files in os.walk(self.indexdir):
#print 'root, dirs, files', root, dirs, files
if root.startswith(os.path.join(self.indexdir, '__delugefs__').encode(FS_ENCODE)): continue
for fn in files:
if fn=='.__delugefs_dir__': continue
fn = os.path.join(root, fn).encode(FS_ENCODE)
if self.LOGLEVEL > 3: print 'loading torrent', fn
e = get_torrent_dict(fn)
if not e:
print 'not able to read torrent', fn
else:
uid = e['info']['name']
info = libtorrent.torrent_info(e)
dat_file = os.path.join(self.chunksdir, uid[:2], uid).encode(FS_ENCODE)
#print 'dat_file', dat_file
if os.path.isfile(dat_file):
if not os.path.isdir(os.path.dirname(dat_file)): os.mkdir(os.path.dirname(dat_file))
h = self.bt_session.add_torrent({'ti':info, 'save_path':os.path.dirname(dat_file)})
if self.LOGLEVEL > 3: print 'added ', fn, '(%s)'%uid
self.httpd.bt_handles[fn[len(self.indexdir):]] = h
if self.LOGLEVEL > 3: print 'self.httpd.bt_handles', self.httpd.bt_handles
def __monitor(self):
while True:
time.sleep(random.randint(3,7))
#print '='*80
#TODO remove self.__write_active_torrents()
self.__check_for_undermirrored_files()
def __please_mirror(self, path):
try:
if self.LOGLEVEL > 3: print '__please_mirror', path
fn = os.path.join(self.indexdir, path).encode(FS_ENCODE)
torrent = get_torrent_dict(fn)
if torrent:
self.__add_torrent(torrent, path)
return True
else:
return False
except:
traceback.print_exc()
return False
def __please_stop_mirroring(self, path):
try:
print 'got __please_stop_mirroring', path
if path in self.last_read_file:
if (datetime.datetime.now()-self.last_read_file[path]).seconds < 60*60*6:
print 'reject - too soon since we last used it', path
return False
print 'i would have stopped', path
return False
# THIS DELETES THE CHUNK
h = self.httpd.bt_handles[path]
if h:
uid = h.get_torrent_info().name()
self.bt_session.remove_torrent(h)
fn = os.path.join(self.chunksdir, uid[:2], uid).encode(FS_ENCODE)
if os.path.isfile(fn): os.remove(fn)
print 'stopped mirroring', path
return True
return False
except:
traceback.print_exc()
return False
def __start_webui(self):
try:
self.httpd.serve_forever()
except:
# not bound to a port because webui is disabled
pass
def __write_active_torrents(self):
try:
with open(os.path.join(self.metadir, '__delugefs__', 'active_torrents').encode(FS_ENCODE), 'w') as f:
for path, h in self.httpd.bt_handles.items():
s = h.status()
state_str = ['queued', 'checking', 'downloading metadata', 'downloading', 'finished', 'seeding', 'allocating', 'checking resume data']
f.write('%s %s is %.2f%% complete (down: %.1f kb/s up: %.1f kB/s peers: %d) %s\n' % \
(path, h.get_torrent_info().name(), s.progress * 100, s.download_rate / 1000, s.upload_rate / 1000, \
s.num_peers, state_str[s.state]))
except Exception as e:
traceback.print_exc()
def __call__(self, op, path, *args):
cid = random.randint(10000, 20000)
if self.LOGLEVEL > 4: print op, path, ('...data...' if op=='write' else args), cid
if path.startswith('/.Trash'): raise FuseOSError(errno.EACCES)
if path.endswith('/.__delugefs_dir__'): raise FuseOSError(errno.EACCES)
ret = super(DelugeFS, self).__call__(op, path, *args)
if self.LOGLEVEL > 4: print '...', cid
return ret
'''
Fuse FS calls in alphabetical order
'''
def access(self, path, mode):
fn = os.path.join(self.indexdir, path).encode(FS_ENCODE)
if not os.access(fn, mode):
raise FuseOSError(errno.EACCES)
#return os.access(fn, mode)
chmod = None
chown = None
def create(self, path, mode):
with self.rwlock:
tmp = uuid.uuid4().hex
self.open_files[path] = tmp
fn = os.path.join(self.indexdir, path).encode(FS_ENCODE)
print 'readdir', fn
print 'index', self.indexdir
print 'path', path
with open(fn,'wb') as f:
pass
return os.open(os.path.join(self.tmp, tmp).encode(FS_ENCODE), os.O_WRONLY | os.O_CREAT, mode)
def flush(self, path, fh):
with self.rwlock:
return os.fsync(fh)
def fsync(self, path, datasync, fh):
with self.rwlock:
return self.flush(path, fh)
def getattr(self, path, fh=None):
st_size = None
if path in self.open_files:
fn = os.path.join(self.tmp, self.open_files[path]).encode(FS_ENCODE)
else:
fn = os.path.join(self.indexdir, path).encode(FS_ENCODE)
torrent = get_torrent_dict(fn)
if torrent:
torrent_info = torrent['info'] if torrent else None
st_size = torrent_info['length'] if torrent_info else 0
st = os.lstat(fn)
ret = dict((key, getattr(st, key)) for key in ('st_atime', 'st_ctime',
'st_gid', 'st_mode', 'st_mtime', 'st_nlink', 'st_size', 'st_uid'))
if st_size is not None:
ret['st_size'] = st_size
if self.LOGLEVEL > 4: print ret
return ret
getxattr = None
link = None
listxattr = None
def mkdir(self, path, flags):
with self.rwlock:
fn = os.path.join(self.indexdir, path).encode(FS_ENCODE)
os.mkdir(fn, flags)
with open(os.path.join(fn, '.__delugefs_dir__').encode(FS_ENCODE),'w') as f:
f.write("git doesn't track empty dirs, so we add this file.")
return 0
mknod = None
def open(self, path, flags):
with self.rwlock:
fn = os.path.join(self.indexdir, path).encode(FS_ENCODE)
if not (flags & (os.O_WRONLY | os.O_RDWR | os.O_APPEND | os.O_CREAT | os.O_EXCL | os.O_TRUNC)):
if self.LOGLEVEL > 3: print '\treadonly'
t = get_torrent_dict(fn)
if t:
name = t['info']['name']
dat_fn = os.path.join(self.chunksdir, name[:2], name).encode(FS_ENCODE)
if not os.path.isfile(dat_fn):
self.__add_torrent(t, path)
self.last_read_file[path] = datetime.datetime.now()
return os.open(dat_fn, flags)
else:
return os.open(fn, flags)
else:
# read and write below
if path in self.open_files:
if self.LOGLEVEL > 3: print '%s in self.open_files' % (path)
tmp = self.open_files[path]
else:
if self.LOGLEVEL > 3: print '%s NOT in self.open_files' % (path)
tmp = uuid.uuid4().hex
if os.path.isfile(fn):
with open(fn, 'rb') as f:
prev = get_torrent_dict['info']['name']
prev_fn = os.path.join(self.chunksdir, prev[:2], prev).encode(FS_ENCODE)
if os.path.isfile(prev_fn):
if self.LOGLEVEL > 3: print 'shutil.copy to tmp'
shutil.copyfile(prev_fn, os.path.join(self.tmp, tmp).encode(FS_ENCODE))
self.open_files[path] = tmp
return os.open(os.path.join(self.tmp, tmp).encode(FS_ENCODE), flags)
def read(self, path, size, offset, fh):
with self.rwlock:
if self.LOGLEVEL > 3: print 'path in self.bt_in_progress', path in self.bt_in_progress
if path in self.bt_in_progress:
if self.LOGLEVEL > 3: print '%s in progress' % (path)
h = self.httpd.bt_handles[path]
if not h.is_seed():
torrent_info = h.get_torrent_info()
piece_length = torrent_info.piece_length()
num_pieces = torrent_info.num_pieces()
start_index = offset // piece_length
end_index = (offset+size) // piece_length
if self.LOGLEVEL > 3: print 'pieces', start_index, end_index
priorities = h.piece_priorities()
#print 'priorities', priorities
for i in range(start_index, min(end_index+1,num_pieces)):
priorities[i] = 7
h.prioritize_pieces(priorities)
if self.LOGLEVEL > 3: print 'priorities', priorities
# h.piece_priority(i, 8)
#print 'piece_priorities set'
for i in range(start_index, min(end_index+1,num_pieces)):
if self.LOGLEVEL > 3: print 'waiting for', i
for i in range(10):
if h.have_piece(i): break
time.sleep(1)
if self.LOGLEVEL > 3: print 'we have', i
os.lseek(fh, offset, 0)
ret = os.read(fh, size)
#print 'ret', ret
return ret
def readdir(self, path, fh):
with self.rwlock:
fn = os.path.join(self.indexdir, path).encode(FS_ENCODE)
return ['.', '..'] + [x for x in os.listdir(fn) if x!=".__delugefs_dir__" ]
readlink = None
def release(self, path, fh):
with self.rwlock:
ret = os.close(fh)
if self.LOGLEVEL > 3: print 'ret', ret, path
if path in self.open_files:
self.__finalize(path, self.open_files[path])
# moved to __finalize_callback
# del self.open_files[path]
if path in self.bt_in_progress:
h = self.httpd.bt_handles[path]
priorities = h.piece_priorities()
#h.prioritize_pieces([0 for x in priorities])
return 0
def rename(self, old, new):
with self.rwlock:
fn = os.path.join(self.indexdir,new).encode(FS_ENCODE)
if os.path.isfile(fn):
os.remove(fn)
os.rename(self.indexdir+old, self.indexdir+new)
return 0
def rmdir(self, path):
with self.rwlock:
fn = os.path.join(self.indexdir, path, '.__delugefs_dir__').encode(FS_ENCODE)
if os.path.isfile(fn):
os.remove(fn)
dn = os.path.dirname(dn)
if os.path.isdir(dn):
os.rmdir(dn)
return 0
def statfs(self, path):
fn = os.path.join(self.indexdir, path).encode(FS_ENCODE)
if self.LOGLEVEL > 3: print 'statfs %s' % (path)
stv = os.statvfs(fn)
return dict((key, getattr(stv, key)) for key in ('f_bavail', 'f_bfree',
'f_blocks', 'f_bsize', 'f_favail', 'f_ffree', 'f_files', 'f_flag',
'f_frsize', 'f_namemax'))
symlink = None
def truncate(self, path, length, fh=None):
with self.rwlock:
if self.LOGLEVEL > 3: print 'truncate fh is ', fh
fn = os.path.join(self.indexdir, path).encode(FS_ENCODE)
if path in self.open_files: # file was opened in READWRITE
with open(os.path.join(self.tmp, self.open_files[path]).encode(FS_ENCODE), 'r+') as f:
f.truncate(length)
return 0
else: # file was opened in READONLY
if self.LOGLEVEL > 3: print '%s not in open_files' % (path)
# open from dat_fn
t = get_torrent_dict(fn)
if t:
name = t['info']['name']
if self.LOGLEVEL > 3: print '%s is %s' % (path, name)
dat_fn = os.path.join(self.chunksdir, name[:2], name).encode(FS_ENCODE)
if not os.path.isfile(dat_fn):
if self.LOGLEVEL > 3: print 'truncate __add_torrent'
self.__add_torrent(t, path)
self.last_read_file[path] = datetime.datetime.now()
with open(dat_fn, 'r+') as f:
if self.LOGLEVEL > 3: print 'truncate f.truncate'
f.truncate(length)
return 0
utimens = None
def unlink(self, path):
with self.rwlock:
fn = os.path.join(self.indexdir, path).encode(FS_ENCODE)
torrent = get_torrent_dict(fn)
torrent_info = torrent.get('info') if torrent else None
name = torrent_info.get('name') if torrent_info else ''
dfn = os.path.join(self.chunksdir, name[:2], name).encode(FS_ENCODE)
if os.path.isfile(dfn):
os.remove(dfn)
print 'deleted', dfn
os.remove(fn)
return 0
def write(self, path, data, offset, fh):
with self.rwlock:
os.lseek(fh, offset, 0)
return os.write(fh, data)
