def donkey_retrieve_task(pipe, donkey):
donkey.results = []
donkey.waiting_results = 1
donkey.reading_results = 0
time_0 = time.time()
while donkey.waiting_results:
time.sleep(0.3)
if time.time() - time_0 > 60.0:
donkey.waiting_results = 0
donkey.lock.acquire()
donkey.reading_results = 1
donkey.lock.release()
donkey.tell("vr\n")
time.sleep(3)
donkey.lock.acquire()
donkey.reading_results = 0
donkey.lock.release()
for item in donkey.results:
ss2 = item.split("\t") # all fields
ss = ss2[0].split(" ", 1) # name and number, first field
if len(ss2) > 1:
pipe.write(
(
("overnet", donkey),
{
"keywords": [donkey.key],
"length": int(ss2[1]),
"type": "file",
"mime": utility.force_string(ss2[2].lower().replace("\n", "")),
"name": utility.force_string("file:" + ss[0][1:-1]),
"index": ss[0][1:-1],
"filename": ss[1],
},
)
)
def set_roots(self, config):
roots = [ ]
self.private_directory = ''
if config.get('public_dir','') != '':
list = map(string.strip,string.split(config['public_dir'],','))
if list:
roots.extend(list)
if config.get('download_dir','') != '':
list = map(string.strip,string.split(config['download_dir'],','))
if list:
#config['download_dir'] = list[0]
roots.extend(list)
if config.get('private_dir','') != '':
list = map(string.strip,string.split(config['private_dir'],','))
if list:
self.private_directory = list[0]
roots.extend(list)
for i in range(len(roots)):
roots[i] = (utility.force_string(roots[i]), flags_full)
if config.get('publish_apt',''):
roots.append(('/var/cache/apt/archives',flags_deb))
# avoid redundancy (prefixes) in the roots list
np_roots = [ ]
for root in roots:
ok = 1
for np_root in np_roots:
if utility.is_subdir(root[0],np_root[0]):
ok = 0
elif utility.is_subdir(np_root[0],root[0]):
np_roots.remove(np_root)
if ok:
np_roots.append(root)
self.lock.acquire()
#self.roots.append(os.path.abspath(path))
#self.roots.append((path,flags))
self.roots = np_roots
self.fresh = 0
self.lock.release()
def __init__(self, data, directory):
utility.Task_manager.__init__(self)
self.data = data
self.directory = directory
self.bytes_downloaded = 0L
self.comment = ''
self.remaining_time = 0.0
self.success=0
self.fields = [ ] #list of chat fields to update
if not directory:
raise Error('No download directory configured yet.')
#here we get rid of unicode for filename
self.filename = os.path.join(
self.directory,
os.path.basename(utility.force_string(self.data['filename'])))
self.basename = os.path.basename(self.filename)
def handle(self, request, address, call_id):
check.check_matches(request, (types.StringType,))
check.check_is_af_inet_address(address)
if request[0] == 'download chunk':
path, mtime = self.paths.get(request[1],(None,None))
if not path:
path, mtime = self.private_paths.get(request[1],(None,None))
try:
if not path or \
os.path.getmtime(path) != mtime:
return Error("no such file")
file = open(path,'rb')
except IOError:
return Error("no such file")
if address[0] !='127.0.0.1'\
and address not in self.node.trusted_addresses \
and self.private_directory != ''\
and utility.is_subdir(path,self.private_directory):
return Error("access denied")
try:
file.seek(request[2])
return file.read(request[3])
finally:
file.close()
elif request[0] == 'files available':
if len(request) == 3:
# this returns the keys of
# all published files from all directories
# regardless of the directory structure
list = self.paths.keys()
else:
list = [ ]
access_denied = 0
directory_found = 0
string_request = []
for str in request[3]:
string_request.append(utility.force_string(str))
if not self.roots:
return Error('No public directory')
request_dir = apply(
os.path.join,
[os.path.abspath(self.roots[0][0])] + string_request)
if address[0]=='127.0.0.1':
flags = flags_local
else:
flags = flags_fast
if os.path.exists(request_dir):
directory_found = 1
if address[0]!='127.0.0.1' \
and address not in self.node.trusted_addresses\
and self.private_directory != ''\
and utility.is_subdir(request_dir,self.private_directory):
access_denied = 1
if not directory_found:
return Error("no such directory: %s"%request_dir)
elif access_denied:
return Error("access denied")
entry = build_entry(request_dir,None,flags,self)
if entry:
if not entry.is_dir:
return Error("not a directory: "+request_dir)
for file in entry.files:
entry_2 = build_entry(file,None,flags,self)
if entry_2:
info = entry_2.info.copy()
info['path'] = request[3] + [ info['filename'] ]
list.append(info)
return list[request[1]:request[2]]
def build_directory(path, mtime, flags, server):
dir = Directory()
dir.mtime = mtime
dir.is_dir = 1
dir.names = [ ]
basename = os.path.basename(path)
dir.info = {
'type' : 'directory',
'filename' : basename,
'length' : 0L,
'path' : string.split(path,'/')
}
#print dir.info['path']
names = os.listdir(path)
#for i in range(len(names)):
# names[i] = utility.force_unicode(names[i])
# Option to limit number of files published
# TODO: make it work with subdirectories
if flags['max'] != None:
names = names[:flags['max']]
if flags.get('name'):
if server.cache.has_key((path,mtime)):
dir.hash, dir.length = server.cache[(path,mtime)]
else:
dir.length = 0
dir.hash = hash.hash_of('basename')
server.cache[(path,mtime)] = (dir.hash, dir.length)
dir.info['name'] = dir.hash
dir.names.append(dir.hash)
if not flags.get('name'):
dir.info['local_path'] = path
str = utility.remove_accents(string.lower(basename))
keywords = [ ]
if flags.get('filename'):
keywords.append(str)
if flags.get('keywords'):
for char in '+-_.,?!()[]':
str = string.replace(str,char," ")
keywords.extend(string.split(str))
dir.info['keywords'] = [ ]
dir.files = [ ]
for item in names:
if item[0] != '.':
dir.files.append(os.path.join(path,item))
#for the moment do not publish directories
return dir
for word in keywords:
word=utility.force_string(word)
if len(word) >= min_search_keyword_len and word not in dir.info['keywords']:
dir.info['keywords'].append(word)
if flags.get('name'):
dir.names.append(hash.hash_of(word))
# publish directory...
# todo: publish after all files have been hashed,
# generate name from their hash
if flags.get('name'):
if not server.entries.has_key(path):
for name in dir.names:
server.node.publish(name, dir.info)
elif server.entries[path].mtime != mtime:
#first unpublish outdated info
#print "unpublishing outdated dir"
server.node.unpublish(dir.info)
for name in dir.names:
server.node.publish(name, dir.info)
server.entries[path] = dir
server.paths[dir.hash] = (path, dir.mtime)
server.names[path] = dir.hash
return dir
def build_file(path, mtime,flags,server):
file = File()
file.is_dir = 0
file.mtime = mtime
# basename = utility.force_unicode(os.path.basename(path))
# do not convert to unicode, because published data should not
# depend on the terminal encoding of the client
basename = os.path.basename(path)
file.length = os.stat(path)[6]
file.names = [ ]
file.info = {
'type' : 'file',
'filename' : basename,
'length' : file.length,
}
if flags.get('name'):
if server.cache.has_key((path,mtime)):
file.hash, file.length = server.cache[(path,mtime)]
else:
try:
f = open(path,'rb')
m = md5.new()
file.length = 0L
while 1:
str = f.read(1<<20)
if str == '': break
m.update(str)
file.length = file.length + len(str)
f.close()
file.hash = m.digest()
except IOError:
raise Error('bad file')
server.cache[(path,mtime)] = (file.hash, file.length)
file.info['name'] = file.hash
file.names.append(file.hash)
if flags.get('local'):
file.info['local_path'] = path
str = utility.remove_accents(string.lower(basename))
keywords = [ ]
if flags.get('filename'):
keywords.append(str)
if flags.get('keywords'):
for char in '+-_.,?!()[]':
str = string.replace(str,char," ")
keywords.extend(string.split(str))
if flags.get('mime'):
list = {}
if string.lower(path[-4:]) =='.mp3':
list = mp3.mp3_info(path)
elif string.lower(path[-4:]) =='.ogg':
list = mp3.ogg_info(path)
if list:
for (k,v) in list.items():
file.info[k] = v
if file.info.get('music_title'):
keywords.extend(string.split(
utility.remove_accents(string.lower(file.info['music_title']))))
if file.info.get('music_artist'):
keywords.extend(string.split(
utility.remove_accents(string.lower(file.info['music_artist']))))
file.info['keywords'] = [ ]
if flags.get('mime'):
import classify
try:
information = classify.classifier.information(path)
for key in information.keys():
if information[key] == None:
#print "[Harmless warning] Can not classify : ", path
continue
if len(information[key]) >= min_search_keyword_len:
file.info[key] = information[key]
except:
sys.stderr.write("Exception caught while classifying file.\n")
for word in keywords:
word=utility.force_string(word)
if len(word) >= min_search_keyword_len and word not in file.info['keywords']:
file.info['keywords'].append(word)
if flags.get('name'):
file.names.append(hash.hash_of(word))
# publish immediately...
if flags.get('name'):
if not server.entries.has_key(path):
for name in file.names:
server.node.publish(name, file.info)
elif server.entries[path].mtime != mtime:
#first unpublish outdated info
print "unpublishing outdated:",path
server.node.unpublish(file.info)
for name in file.names:
server.node.publish(name, file.info)
server.entries[path] = file
server.paths[file.hash] = (path, file.mtime)
server.names[path] = file.hash
return file
def find_task(daemon, query, input, output, connection):
import safe_pickle
for char in "+-_.,?()![]":
query = query.replace(char, " ")
query = query.lower()
list = query.split()
if list:
key = list[0]
pipe = daemon.node.retrieve(hash.hash_of("identity-name " + key), settings.identity_redundancy)
else:
pipe = daemon.node.retrieve(hash.hash_of("service identity"), settings.identity_redundancy)
results = []
while not pipe.finished():
list = pipe.read_all()
prev_pair = None
for pair in list:
if pair == prev_pair:
continue
link, item = prev_pair = pair
try:
item = utility.check_and_demangle_item(item)
except:
continue
if item["key"] not in results:
results.append(item["key"])
name = hash.hash_of(safe_pickle.dumps(item["key"]))
check.check_is_name(name)
str = hash.hash_to_person(name)
output.write(str + " " + item["name"] + " (" + utility.force_string(item["human-name"]) + ")\n")
time.sleep(0.5)
try:
output.flush()
except:
return
if not results:
try:
output.write('No user matching "' + key + '"')
except:
pass
else:
if results.__len__() == 1:
msg = "1 user found."
else:
msg = "%d users found." % results.__len__()
output.write(msg)
pipe.stop()
try:
input.close()
output.close()
connection.close()
except:
# connection reset by peer...
pass
def search_task(daemon, query, input, output, connection):
for char in "+-_.,?()![]":
query = query.replace(char, " ")
query = query.lower()
list = query.split()
if list:
key = list[0]
else:
key = ""
if key.__len__() < 3:
output.write("Keyword %s too short: must be at least 3 characters" % key)
input.close()
output.close()
connection.close()
return
pipe = daemon.node.retrieve(hash.hash_of(key))
results = []
restricted = 0
while not pipe.finished() and not restricted:
for item in pipe.read_all():
if results.__len__() == 100:
restricted = 1
break
if item[1]["name"] not in results:
results.append(item[1]["name"])
filename = utility.force_string(item[1]["filename"])
extension = string.split(string.split(filename, ".")[-1], "-")[0]
lext = string.lower(extension)
if lext in ["mp3", "ogg"]:
music = 1
else:
music = 0
if item[1].has_key("music_title"):
ref = utility.force_string(item[1]["music_title"])
if ref.strip() == "":
ref = filename
else:
ref = utility.force_string(item[1]["filename"])
length = item[1].get("length")
if not length:
sl = ""
else:
sl = utility.human_size(length)
output.write(hash.hash_to_url(item[1]["name"]) + " \t" + sl + " \t" + filename + "\n")
time.sleep(0.5)
try:
output.flush()
except:
return
if not results:
try:
output.write('No document matching "' + key + '"')
except:
pass
else:
if results.__len__() == 1:
msg = "1 file found."
else:
msg = "%d files found." % results.__len__()
output.write(msg)
pipe.stop()
try:
input.close()
output.close()
connection.close()
except:
pass
