def migration_v2_to_v3(self): logger.info( "Updating chunks column in filenames table (v2 to v3 migration)") with AutoDB(self.config.database_path) as db: sql = "SELECT id, filename, localpart FROM filenames WHERE chunks IS NULL" cursor = None l = [] try: cursor = db.connection.cursor() cursor.execute(sql) l = list(cursor.fetchall()) logger.info("Updating %s filename entries", len(l)) data = ((patterns.split_chunks(filename, localpart), _id) for _id, filename, localpart in l) cursor.executemany("UPDATE filenames SET chunks=? WHERE id=?", [(len(pattern), _id) for pattern, _id in data if pattern]) db.connection.commit() finally: db.connection.commit() cursor.close() self.clean()
def add_resource(self, resource, vtresult=None, localpart=None, domain=None): logger.debug("Adding resource %s with result %s and to (%s, %s) to database", resource, vtresult, localpart, domain) insert_sql = 'INSERT INTO filenames (filename, pattern, infected, "timestamp", sha256, localpart, domain, chunks) VALUES (?, ?, ?, ?, ?, ?, ?, ?)' update_sql = 'UPDATE filenames SET pattern = ?, timestamp = ? WHERE filename=?' infected = vtresult.infected if vtresult else False pattern = patterns.calculate(resource.filename, self.get_filename_localparts(), localpart=localpart) no_chunks = len(patterns.split_chunks(pattern, localpart)) if pattern else 0 values = ( resource.filename, pattern, infected, datetime.datetime.utcnow(), resource.sha256, localpart, domain, no_chunks ) with AutoDB(self.config.database_path) as db: cursor = None try: cursor = db.connection.cursor() cursor.execute(insert_sql, values) except sqlite3.IntegrityError: cursor.execute(update_sql, (pattern, datetime.datetime.utcnow(), resource.filename)) finally: db.connection.commit() cursor.close() cursor = db.connection.cursor() cursor.execute("UPDATE filenames SET infected=? WHERE sha256=? AND infected=0", (int(infected), resource.sha256)) db.connection.commit() cursor.close()
def migration_v2_to_v3(self): logger.info("Updating chunks column in filenames table (v2 to v3 migration)") with AutoDB(self.config.database_path) as db: sql = "SELECT id, filename, localpart FROM filenames WHERE chunks IS NULL" cursor = None l = [] try: cursor = db.connection.cursor() cursor.execute(sql) l = list(cursor.fetchall()) logger.info("Updating %s filename entries", len(l)) data = ( (patterns.split_chunks(filename, localpart), _id) for _id, filename, localpart in l ) cursor.executemany("UPDATE filenames SET chunks=? WHERE id=?", [ (len(pattern), _id) for pattern, _id in data if pattern ]) db.connection.commit() finally: db.connection.commit() cursor.close() self.clean()
def add_resource(self, resource, vtresult=None, localpart=None, domain=None): logger.debug( "Adding resource %s with result %s and to (%s, %s) to database", resource, vtresult, localpart, domain) insert_sql = 'INSERT INTO filenames (filename, pattern, infected, "timestamp", sha256, localpart, domain, chunks) VALUES (?, ?, ?, ?, ?, ?, ?, ?)' update_sql = 'UPDATE filenames SET pattern = ?, timestamp = ? WHERE filename=?' infected = vtresult.infected if vtresult else False pattern = patterns.calculate(resource.filename, self.get_filename_localparts(), localpart=localpart) no_chunks = len(patterns.split_chunks(pattern, localpart)) if pattern else 0 values = (resource.filename, pattern, infected, datetime.datetime.utcnow(), resource.sha256, localpart, domain, no_chunks) with AutoDB(self.config.database_path) as db: cursor = None try: cursor = db.connection.cursor() cursor.execute(insert_sql, values) except sqlite3.IntegrityError: cursor.execute( update_sql, (pattern, datetime.datetime.utcnow(), resource.filename)) finally: db.connection.commit() cursor.close() cursor = db.connection.cursor() cursor.execute( "UPDATE filenames SET infected=? WHERE sha256=? AND infected=0", (int(infected), resource.sha256)) db.connection.commit() cursor.close()
def test_string_split_null_or_empty(self): assert patterns.split_chunks(None, None) == () assert patterns.split_chunks('', None) == ()