Esempio n. 1
0
def handle_search(bot, event):
    if not event.options: event.makeoptions()
    all = event.options.all
    res = []
    target = event.args
    if not target: event.missing("<search words seperated by space>") ; return
    coll = PersistCollection(getdatadir() + os.sep + 'spider' + os.sep + "data")
    files = coll.filenames(target)
    if files:
        for f in files:
            try: res.append(Persist(f).data.url)
            except AttributeError as ex: continue
    objs = coll.search('txt', event.rest)
    if not objs: objs = list(coll.objects().values())
    stats = makestats(objs, target, res)
    urls = stats_response(stats, target)
    res.extend(urls)
    if res:
        if len(res) < 4 or all: event.reply("found %s urls: " % len(res), res, dot=" -or- ")
        else: event.reply("found %s urls, use --all for more: " % len(res), res[:3], dot=" -or- ")
    else: event.reply("no urls found")
Esempio n. 2
0
def markovlearnspider(target):
    logging.warn("starting spider learn on %s" % target)
    coll = PersistCollection(getdatadir() + os.sep + "spider" + os.sep + "data")
    if target.startswith("spider://"):
        target = target[9:]
    objs = coll.search("url", target)
    for obj in objs:
        if not obj.data and obj.data.url:
            continue
        time.sleep(0.001)
        if target not in obj.data.url:
            continue
        logging.warn("url is %s" % obj.data.url)
        try:
            if obj.data and obj.data.txt:
                for line in obj.data.txt.split("\n"):
                    if line.count(";") > 1:
                        continue
                    markovtalk_learn(striphtml(line))
        except:
            handle_exception()
Esempio n. 3
0
 def __init__(self, *args, **kwargs):
     self.path = getdatadir() + os.sep + "spider" + os.sep + "data" + os.sep
     PersistCollection.__init__(self, self.path, *args, **kwargs)