Ejemplo n.º 1
0
 def CheckScantype(self, scantype = None):
     if scantype is not None:
         self.scantype = scantype
     if self.scantype is None:
         for scantype in scantypes[self.proto].discovery:
             try:
                 if self.CheckScantype(scantype) is not None:
                     return self.scantype
             except StopIteration:
                 break
         log("Cann't discover scantype for %s:%s", (self.host, self.ProtoOrPort()))
     else:
         result = run_scanner(scantypes[self.proto][self.scantype],
                              self.nscache(self.host), self.proto, self.port, "-l").wait()
         if result == 0:
             return self.scantype
         elif result == 2:
             raise StopIteration
     self.scantype = None
     return None
Ejemplo n.º 2
0
def scan_share(db, share_id, proto, host, port, tree_id, command):
    db.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED)
    cursor = db.cursor()
    hoststr = sharestr(proto, host, port)
    try:
        # asquire lock on the column from trees table
        cursor.execute("SELECT hash FROM ONLY trees WHERE tree_id=%(t)s FOR UPDATE NOWAIT", {'t': tree_id})
        oldhash = cursor.fetchone()['hash']
    except:
        # if other spider instance didn't complete scanning, do nothing
        # side effect is backing-off the next scan
        log("Scanning %s is running too long in another spider instance or database error.", (hoststr,))
        db.rollback()
        return
    savepath = share_save_path(proto, host, port)
    patchmode = oldhash != None and os.path.isfile(savepath)
    try:
        address = socket.gethostbyname(host)
    except:
        log("Name resolution failed for %s.", (hoststr,))
        db.rollback()
        return
    log("Scanning %s (%s) ...", (hoststr, address))
    start = datetime.datetime.now()
    if patchmode:
        data = run_scanner(command, address, proto, port, "-u " + quote_for_shell(savepath))
    else:
        data = run_scanner(command, address, proto, port)
    save = tempfile.TemporaryFile(bufsize = -1)
    line_count = 0
    line_count_patch = 0
    hash = hashlib.md5()
    for line in data.stdout:
        line_count += 1
        if line[0] in ('+', '-', '*'):
            line_count_patch += 1
        if line_count > max_lines_from_scanner:
            kill_process(data)
            data.stdout.close()
            data.wait()
            log("Scanning %s failed. Too many lines from scanner (elapsed time %s).", (hoststr, datetime.datetime.now() - start))
            db.rollback()
            return
        hash.update(line)
        save.write(line)
    if data.wait() != 0:
        cursor.execute("""
            UPDATE shares SET next_scan = now() + %(w)s
            WHERE share_id = %(s)s;
            """, {'s': share_id, 'w': wait_until_next_scan_failed})
        log("Scanning %s failed with return code %s (elapsed time %s).", (hoststr, data.returncode, datetime.datetime.now() - start))
        db.commit()
        return
    if patchmode and (line_count_patch > (line_count - line_count_patch) / patch_fallback):
        log("Patch is too long for %s (patch %s, non-patch %s). Fallback to non-patching mode", (hoststr, line_count_patch, line_count - line_count_patch))
        patchmode = False
    scan_time = datetime.datetime.now() - start
    start = datetime.datetime.now()
    qcache = PsycoCache(cursor)
    paths_buffer = dict()
    save.seek(0)
    if patchmode and (oldhash == None or save.readline() != "* " + oldhash + "\n"):
        save.seek(0)
        patchmode = False
        log("MD5 digest from scanner doesn't match the one from the database. Fallback to non-patching mode.")
    if patchmode:
        cursor.execute("""
            CREATE TEMPORARY TABLE newfiles (
                LIKE files INCLUDING DEFAULTS
                ) ON COMMIT DROP;
            CREATE INDEX newfiles_path ON newfiles(treepath_id);
            """)
        for line in save:
            if line[0] not in ('+', '-', '*'):
                break
            scan_line_patch(cursor, tree_id, line.strip('\n'), qcache, paths_buffer)
        for (dirid, pinfo) in paths_buffer.iteritems():
            if pinfo.modify:
                qcache.append("SELECT push_path_files(%(t)s, %(d)s)", {'t': tree_id, 'd': dirid})
    else:
        cursor.execute("DELETE FROM paths WHERE tree_id = %(t)s", {'t':tree_id})
        for line in save:
            if line[0] in ('+', '-', '*'):
                continue
            scan_line_patch(cursor, tree_id, "+ " + line.strip('\n'), qcache, paths_buffer)
    qcache.allcommit()
    try:
        if os.path.isfile(savepath):
            shutil.move(savepath, savepath + ".old")
        save.seek(0)
        file = open(savepath, 'wb')
        shutil.copyfileobj(save, file)
        file.close()
    except:
        log("Failed to save contents of %s to file %s.", (hoststr, savepath))
        traceback.print_exc()
    save.close()
    cursor.execute("""
        UPDATE shares SET last_scan = now(), next_scan = now() + %(w)s WHERE share_id = %(s)s;
        UPDATE trees SET hash = %(h)s WHERE tree_id = %(t)s;
        """, {'s': share_id, 't': tree_id, 'h': hash.hexdigest(), 'w': wait_until_next_scan})
    if qcache.totalsize >= 0:
        cursor.execute("""
            UPDATE shares SET size = %(sz)s WHERE share_id = %(s)s;
            """, {'s':share_id, 'sz': qcache.totalsize})
    db.commit()
    if patchmode:
        deleted = qcache.stat_pdelete + qcache.stat_fdelete
        added = qcache.stat_padd + qcache.stat_fadd
        modified = qcache.stat_fmodify
        log("Scanning %s succeded. Database updated in patching mode: delete %s, add %s, modify %s (scan time %s, update time %s).",
            (hoststr, str(deleted), str(added), str(modified), scan_time, datetime.datetime.now() - start))
    else:
        log("Scanning %s succeded. Database updated in non-patching mode (scan time %s, update time %s).",
            (hoststr, scan_time, datetime.datetime.now() - start))