Example #1
0
 def test_hash_file(self):
     filename = self.new_temp_file('hash_test.txt')
     random_str = ut.rand_str(1000)
     fu.write_to_file(filename, random_str)
     self.assertEqual(fu.hash_file(filename, 'sha1'), 
                      ut.hash_text(random_str, 'sha1'), 
                      'SHA1 hashes don\'t match')
     self.assertEqual(fu.hash_file(filename), ut.hash_text(random_str), 
                      'Hashes with default algo don\'t match')
Example #2
0
 def test_create_user(self):
     user = self._create_user()
     self.assertEqual(user.get('username'), self.test_user_username)
     self.assertEqual(user.get('email'), self.test_user_email)
     self.assertNotEqual(user.get('password'), self.test_user_password)
     self.assertEqual(user.get('password'),
                      utils.hash_text(self.test_user_password))
     self.assertEqual(user.get('is_admin'), self.test_user_is_admin)
Example #3
0
 def test_create_user(self):
     user = self._create_user()
     self.assertEqual(user.get('username'), self.test_user_username)
     self.assertEqual(user.get('email'), self.test_user_email)
     self.assertNotEqual(user.get('password'), self.test_user_password)
     self.assertEqual(user.get('password'),
         utils.hash_text(self.test_user_password))
     self.assertEqual(user.get('is_admin'), self.test_user_is_admin)
Example #4
0
def store_swfs(msg, crawl_id, dir_path='/tmp', prefix='?'):
    
    referer = msg.request.headers['Referer'][0] if msg.request.headers['Referer'] else ""
    
    if msg.response and msg.response.content:
        print msg.request.get_url()
        if (msg.response.content[:3] in SWF_MAGIC_NUMBERS): # to wide, but decompiler will discard them
            
            swf_hash = ut.hash_text(msg.response.content)
            swf_url = msg.request.get_url()
            
            db_conn = dbu.mysql_init_db()
            db_cursor = db_conn.cursor(dbu.mdb.cursors.DictCursor)
            rows = swu.get_swf_obj_from_db('hash', swf_hash, db_cursor)
            
            if not rows:
                swf_filename = os.path.join(dir_path, "%s-%s" % (prefix, msg.request.path.split('/')[-1]))
                swf_filename = swf_filename[:MAX_FILENAME_LEN]
                if not swf_filename.endswith('.swf'):
                    swf_filename += '.swf'
                    
                wl_log.info("SWF saved %s referrer: %s" % (os.path.basename(swf_filename), referer))
                
                fu.write_to_file(swf_filename, msg.response.content)
                vector = swu.get_occurence_vector_from_swf(swf_filename, os.path.join(dir_path, prefix))
                duplicate_swf = 0
            else:
                wl_log.info("A swf with same hash exists in DB: %s %s" % (swf_hash, swf_url))
                vector = swu.str_to_vector(rows[0]['occ_vector'])
                swf_filename = rows[0]['local_path']
                duplicate_swf = 1
            
            rank, domain = prefix.rsplit('/')[-1].split('-', 1)
            swf_info = swu.SwfInfo()
            
            swf_info.rank = rank # this might be fake
            swf_info.domain = domain
            swf_info.local_path = swf_filename
            swf_info.occ_vector = vector
            swf_info.hash = swf_hash
            swf_info.url = swf_url
            swf_info.referer = referer        
            swf_info.duplicate = duplicate_swf # !!! Y for repeated swfs(that we know before) 
            swf_info.feat_vector = []
            swf_info.page_url = ''
            swf_info.occ_string = ' '.join(swu.human_readable_occ_vector(vector))
            swf_info.crawl_id = crawl_id
            
            swu.add_swf_to_db(swf_info, db_conn)
            db_conn.commit()
            db_cursor.close()
            db_conn.close()
            
            
        elif '.swf' in msg.request.path:
            wl_log.warning(".swf in path but content seems non-swf %s %s" % (msg.request.path, msg.response.content[:100]))
        else:
            pass
Example #5
0
def change_password():
    if request.method == 'POST':
        form = request.form
        if form.get('password') != form.get('password_confirm'):
            flash(messages.PASSWORDS_NOT_MATCH, 'error')
            return redirect(url_for('accounts.change_password'))
        db.update_user(session.get('user').get('username'), \
            {'password': utils.hash_text(form.get('password'))})
        flash(messages.PASSWORD_UPDATED, 'success')
        return redirect(url_for('admin.index'))
    ctx = {}
    return render_template('accounts/change_password.html', **ctx)
Example #6
0
def change_password():
    if request.method == 'POST':
        form = request.form
        if form.get('password') != form.get('password_confirm'):
            flash(messages.PASSWORDS_NOT_MATCH, 'error')
            return redirect(url_for('accounts.change_password'))
        db.update_user(session.get('user').get('username'), \
            {'password': utils.hash_text(form.get('password'))})
        flash(messages.PASSWORD_UPDATED, 'success')
        return redirect(url_for('admin.index'))
    ctx = {}
    return render_template('accounts/change_password.html', **ctx)
Example #7
0
def login():
    username = ''
    if request.method == 'POST':
        form = request.form
        username = form.get('username')
        u = db.get_user(username)
        if u:
            if hash_text(form.get('password')) == u.get('password'):
                # login
                session['user'] = u
                return redirect(url_for('admin.index'))
        flash(messages.INVALID_USERNAME_PASSWORD, 'error')
    ctx = {'username': username}
    return render_template('accounts/login.html', **ctx)
Example #8
0
def login():
    username = ''
    if request.method == 'POST':
        form = request.form
        username = form.get('username')
        u = db.get_user(username)
        if u:
            if hash_text(form.get('password')) == u.get('password'):
                # login
                session['user'] = u
                return redirect(url_for('admin.index'))
        flash(messages.INVALID_USERNAME_PASSWORD, 'error')
    ctx = {'username': username}
    return render_template('accounts/login.html', **ctx)
Example #9
0
def login():
    if request.method == 'POST':
        form = request.form
        u = db.get_user({'username': form.get('username')})
        next_url = utils.get_redirect_target()
        if not next_url:
            next_url = url_for('admin.index')
        if u:
            if hash_text(form.get('password')) == u.get('password'):
                # login
                session['user'] = u
                return redirect(next_url)
            print(u)
        flash(messages.INVALID_USERNAME_PASSWORD, 'error')
        return redirect(url_for('accounts.login'))
    ctx = {}
    return render_template('accounts/login.html', **ctx)
Example #10
0
def login():
    if request.method == 'POST':
        form = request.form
        u = db.get_user({'username': form.get('username')})
        next_url = utils.get_redirect_target()
        if not next_url:
            next_url = url_for('admin.index')
        if u:
            if hash_text(form.get('password')) == u.get('password'):
                # login
                session['user'] = u
                return redirect(next_url)
            print(u)
        flash(messages.INVALID_USERNAME_PASSWORD, 'error')
        return redirect(url_for('accounts.login'))
    ctx = {}
    return render_template('accounts/login.html', **ctx)
Example #11
0
def create_user(username=None, password='', email=None, is_admin=False):
    """
    Creates a new user

    :param username: Username of user
    :param password: User password
    :param email: User email
    :param is_admin: Admin user

    """
    rds = get_redis_connection()
    data = {
        'username': username,
        'password': utils.hash_text(password),
        'email': email,
        'is_admin': is_admin,
    }
    return rds.set(USER_KEY.format(username), json.dumps(data))
Example #12
0
def create_user(username=None, password='', email=None, is_admin=False):
    """
    Creates a new user

    :param username: Username of user
    :param password: User password
    :param email: User email
    :param is_admin: Admin user

    """
    rds = get_redis_connection()
    data = {
        'username': username,
        'password': utils.hash_text(password),
        'email': email,
        'is_admin': is_admin,
    }
    return rds.set(USER_KEY.format(username), json.dumps(data))
Example #13
0
def update_user(username=None, data={}):
    """
    Updates a user with the specified data

    :param username: Username to update
    :param data: Data to update as a dict

    """
    rds = get_redis_connection()
    user_data = rds.get(USER_KEY.format(username))
    ret = None
    if user_data:
        user = json.loads(user_data)
        for k,v in data.iteritems():
            # hash password if present
            if k == 'password':
                v = utils.hash_text(v)
            user[k] = v
        ret = rds.set(USER_KEY.format(username), json.dumps(user))
    return ret
Example #14
0
def update_user(username=None, data={}):
    """
    Updates a user with the specified data

    :param username: Username to update
    :param data: Data to update as a dict

    """
    rds = get_redis_connection()
    user_data = rds.get(USER_KEY.format(username))
    ret = None
    if user_data:
        user = json.loads(user_data)
        for k, v in data.iteritems():
            # hash password if present
            if k == 'password':
                v = utils.hash_text(v)
            user[k] = v
        ret = rds.set(USER_KEY.format(username), json.dumps(user))
    return ret
Example #15
0
def create_user(username=None, password=None, first_name=None, last_name=None, \
    is_admin=False):
    """
    Creates a new user

    :param username: Username of user
    :param password: User password
    :param first_name: First name of user
    :param last_name: Last name of user
    :param is_admin: Admin user

    """
    mongo = get_mongo_connection()
    obj_id = mongo.db.accounts.save(
        {
            'username': username,
            'first_name': first_name,
            'last_name': last_name,
            'password': hash_text(password),
            'is_admin': True,
        },
        safe=True)
    return mongo.db.accounts.find_one(obj_id)
Example #16
0
def create_user(username=None, password=None, first_name=None, last_name=None, is_admin=False):
    """
    Creates a new user

    :param username: Username of user
    :param password: User password
    :param first_name: First name of user
    :param last_name: Last name of user
    :param is_admin: Admin user

    """
    mongo = get_mongo_connection()
    obj_id = mongo.db.accounts.save(
        {
            "username": username,
            "first_name": first_name,
            "last_name": last_name,
            "password": hash_text(password),
            "is_admin": True,
        },
        safe=True,
    )
    return mongo.db.accounts.find_one(obj_id)
Example #17
0
def store_swfs(msg, crawl_id, dir_path='/tmp', prefix='?'):

    referer = msg.request.headers['Referer'][0] if msg.request.headers[
        'Referer'] else ""

    if msg.response and msg.response.content:
        print msg.request.get_url()
        if (msg.response.content[:3] in SWF_MAGIC_NUMBERS
            ):  # to wide, but decompiler will discard them

            swf_hash = ut.hash_text(msg.response.content)
            swf_url = msg.request.get_url()

            db_conn = dbu.mysql_init_db()
            db_cursor = db_conn.cursor(dbu.mdb.cursors.DictCursor)
            rows = swu.get_swf_obj_from_db('hash', swf_hash, db_cursor)

            if not rows:
                swf_filename = os.path.join(
                    dir_path,
                    "%s-%s" % (prefix, msg.request.path.split('/')[-1]))
                swf_filename = swf_filename[:MAX_FILENAME_LEN]
                if not swf_filename.endswith('.swf'):
                    swf_filename += '.swf'

                wl_log.info("SWF saved %s referrer: %s" %
                            (os.path.basename(swf_filename), referer))

                fu.write_to_file(swf_filename, msg.response.content)
                vector = swu.get_occurence_vector_from_swf(
                    swf_filename, os.path.join(dir_path, prefix))
                duplicate_swf = 0
            else:
                wl_log.info("A swf with same hash exists in DB: %s %s" %
                            (swf_hash, swf_url))
                vector = swu.str_to_vector(rows[0]['occ_vector'])
                swf_filename = rows[0]['local_path']
                duplicate_swf = 1

            rank, domain = prefix.rsplit('/')[-1].split('-', 1)
            swf_info = swu.SwfInfo()

            swf_info.rank = rank  # this might be fake
            swf_info.domain = domain
            swf_info.local_path = swf_filename
            swf_info.occ_vector = vector
            swf_info.hash = swf_hash
            swf_info.url = swf_url
            swf_info.referer = referer
            swf_info.duplicate = duplicate_swf  # !!! Y for repeated swfs(that we know before)
            swf_info.feat_vector = []
            swf_info.page_url = ''
            swf_info.occ_string = ' '.join(
                swu.human_readable_occ_vector(vector))
            swf_info.crawl_id = crawl_id

            swu.add_swf_to_db(swf_info, db_conn)
            db_conn.commit()
            db_cursor.close()
            db_conn.close()

        elif '.swf' in msg.request.path:
            wl_log.warning(".swf in path but content seems non-swf %s %s" %
                           (msg.request.path, msg.response.content[:100]))
        else:
            pass
Example #18
0
def hash_file(filepath, algo='sha1'):
    """Return the hash value for the file content."""
    import utils as ut
    return ut.hash_text(read_file(filepath), algo)
Example #19
0
def hash_file(filepath, algo='sha1'):
    """Return the hash value for the file content."""
    import utils as ut
    return ut.hash_text(read_file(filepath), algo)