def test_hash_file(self):
filename = self.new_temp_file('hash_test.txt')
random_str = ut.rand_str(1000)
fu.write_to_file(filename, random_str)
self.assertEqual(fu.hash_file(filename, 'sha1'),
ut.hash_text(random_str, 'sha1'),
'SHA1 hashes don\'t match')
self.assertEqual(fu.hash_file(filename), ut.hash_text(random_str),
'Hashes with default algo don\'t match')
def test_create_user(self):
user = self._create_user()
self.assertEqual(user.get('username'), self.test_user_username)
self.assertEqual(user.get('email'), self.test_user_email)
self.assertNotEqual(user.get('password'), self.test_user_password)
self.assertEqual(user.get('password'),
utils.hash_text(self.test_user_password))
self.assertEqual(user.get('is_admin'), self.test_user_is_admin)
def test_create_user(self):
user = self._create_user()
self.assertEqual(user.get('username'), self.test_user_username)
self.assertEqual(user.get('email'), self.test_user_email)
self.assertNotEqual(user.get('password'), self.test_user_password)
self.assertEqual(user.get('password'),
utils.hash_text(self.test_user_password))
self.assertEqual(user.get('is_admin'), self.test_user_is_admin)
def store_swfs(msg, crawl_id, dir_path='/tmp', prefix='?'):
referer = msg.request.headers['Referer'][0] if msg.request.headers['Referer'] else ""
if msg.response and msg.response.content:
print msg.request.get_url()
if (msg.response.content[:3] in SWF_MAGIC_NUMBERS): # to wide, but decompiler will discard them
swf_hash = ut.hash_text(msg.response.content)
swf_url = msg.request.get_url()
db_conn = dbu.mysql_init_db()
db_cursor = db_conn.cursor(dbu.mdb.cursors.DictCursor)
rows = swu.get_swf_obj_from_db('hash', swf_hash, db_cursor)
if not rows:
swf_filename = os.path.join(dir_path, "%s-%s" % (prefix, msg.request.path.split('/')[-1]))
swf_filename = swf_filename[:MAX_FILENAME_LEN]
if not swf_filename.endswith('.swf'):
swf_filename += '.swf'
wl_log.info("SWF saved %s referrer: %s" % (os.path.basename(swf_filename), referer))
fu.write_to_file(swf_filename, msg.response.content)
vector = swu.get_occurence_vector_from_swf(swf_filename, os.path.join(dir_path, prefix))
duplicate_swf = 0
else:
wl_log.info("A swf with same hash exists in DB: %s %s" % (swf_hash, swf_url))
vector = swu.str_to_vector(rows[0]['occ_vector'])
swf_filename = rows[0]['local_path']
duplicate_swf = 1
rank, domain = prefix.rsplit('/')[-1].split('-', 1)
swf_info = swu.SwfInfo()
swf_info.rank = rank # this might be fake
swf_info.domain = domain
swf_info.local_path = swf_filename
swf_info.occ_vector = vector
swf_info.hash = swf_hash
swf_info.url = swf_url
swf_info.referer = referer
swf_info.duplicate = duplicate_swf # !!! Y for repeated swfs(that we know before)
swf_info.feat_vector = []
swf_info.page_url = ''
swf_info.occ_string = ' '.join(swu.human_readable_occ_vector(vector))
swf_info.crawl_id = crawl_id
swu.add_swf_to_db(swf_info, db_conn)
db_conn.commit()
db_cursor.close()
db_conn.close()
elif '.swf' in msg.request.path:
wl_log.warning(".swf in path but content seems non-swf %s %s" % (msg.request.path, msg.response.content[:100]))
else:
pass
def change_password():
if request.method == 'POST':
form = request.form
if form.get('password') != form.get('password_confirm'):
flash(messages.PASSWORDS_NOT_MATCH, 'error')
return redirect(url_for('accounts.change_password'))
db.update_user(session.get('user').get('username'), \
{'password': utils.hash_text(form.get('password'))})
flash(messages.PASSWORD_UPDATED, 'success')
return redirect(url_for('admin.index'))
ctx = {}
return render_template('accounts/change_password.html', **ctx)
def change_password():
if request.method == 'POST':
form = request.form
if form.get('password') != form.get('password_confirm'):
flash(messages.PASSWORDS_NOT_MATCH, 'error')
return redirect(url_for('accounts.change_password'))
db.update_user(session.get('user').get('username'), \
{'password': utils.hash_text(form.get('password'))})
flash(messages.PASSWORD_UPDATED, 'success')
return redirect(url_for('admin.index'))
ctx = {}
return render_template('accounts/change_password.html', **ctx)
def login():
username = ''
if request.method == 'POST':
form = request.form
username = form.get('username')
u = db.get_user(username)
if u:
if hash_text(form.get('password')) == u.get('password'):
# login
session['user'] = u
return redirect(url_for('admin.index'))
flash(messages.INVALID_USERNAME_PASSWORD, 'error')
ctx = {'username': username}
return render_template('accounts/login.html', **ctx)
def login():
username = ''
if request.method == 'POST':
form = request.form
username = form.get('username')
u = db.get_user(username)
if u:
if hash_text(form.get('password')) == u.get('password'):
# login
session['user'] = u
return redirect(url_for('admin.index'))
flash(messages.INVALID_USERNAME_PASSWORD, 'error')
ctx = {'username': username}
return render_template('accounts/login.html', **ctx)
def login():
if request.method == 'POST':
form = request.form
u = db.get_user({'username': form.get('username')})
next_url = utils.get_redirect_target()
if not next_url:
next_url = url_for('admin.index')
if u:
if hash_text(form.get('password')) == u.get('password'):
# login
session['user'] = u
return redirect(next_url)
print(u)
flash(messages.INVALID_USERNAME_PASSWORD, 'error')
return redirect(url_for('accounts.login'))
ctx = {}
return render_template('accounts/login.html', **ctx)
def login():
if request.method == 'POST':
form = request.form
u = db.get_user({'username': form.get('username')})
next_url = utils.get_redirect_target()
if not next_url:
next_url = url_for('admin.index')
if u:
if hash_text(form.get('password')) == u.get('password'):
# login
session['user'] = u
return redirect(next_url)
print(u)
flash(messages.INVALID_USERNAME_PASSWORD, 'error')
return redirect(url_for('accounts.login'))
ctx = {}
return render_template('accounts/login.html', **ctx)
def create_user(username=None, password='', email=None, is_admin=False):
"""
Creates a new user
:param username: Username of user
:param password: User password
:param email: User email
:param is_admin: Admin user
"""
rds = get_redis_connection()
data = {
'username': username,
'password': utils.hash_text(password),
'email': email,
'is_admin': is_admin,
}
return rds.set(USER_KEY.format(username), json.dumps(data))
def create_user(username=None, password='', email=None, is_admin=False):
"""
Creates a new user
:param username: Username of user
:param password: User password
:param email: User email
:param is_admin: Admin user
"""
rds = get_redis_connection()
data = {
'username': username,
'password': utils.hash_text(password),
'email': email,
'is_admin': is_admin,
}
return rds.set(USER_KEY.format(username), json.dumps(data))
def update_user(username=None, data={}):
"""
Updates a user with the specified data
:param username: Username to update
:param data: Data to update as a dict
"""
rds = get_redis_connection()
user_data = rds.get(USER_KEY.format(username))
ret = None
if user_data:
user = json.loads(user_data)
for k,v in data.iteritems():
# hash password if present
if k == 'password':
v = utils.hash_text(v)
user[k] = v
ret = rds.set(USER_KEY.format(username), json.dumps(user))
return ret
def update_user(username=None, data={}):
"""
Updates a user with the specified data
:param username: Username to update
:param data: Data to update as a dict
"""
rds = get_redis_connection()
user_data = rds.get(USER_KEY.format(username))
ret = None
if user_data:
user = json.loads(user_data)
for k, v in data.iteritems():
# hash password if present
if k == 'password':
v = utils.hash_text(v)
user[k] = v
ret = rds.set(USER_KEY.format(username), json.dumps(user))
return ret
def create_user(username=None, password=None, first_name=None, last_name=None, \
is_admin=False):
"""
Creates a new user
:param username: Username of user
:param password: User password
:param first_name: First name of user
:param last_name: Last name of user
:param is_admin: Admin user
"""
mongo = get_mongo_connection()
obj_id = mongo.db.accounts.save(
{
'username': username,
'first_name': first_name,
'last_name': last_name,
'password': hash_text(password),
'is_admin': True,
},
safe=True)
return mongo.db.accounts.find_one(obj_id)
def create_user(username=None, password=None, first_name=None, last_name=None, is_admin=False):
"""
Creates a new user
:param username: Username of user
:param password: User password
:param first_name: First name of user
:param last_name: Last name of user
:param is_admin: Admin user
"""
mongo = get_mongo_connection()
obj_id = mongo.db.accounts.save(
{
"username": username,
"first_name": first_name,
"last_name": last_name,
"password": hash_text(password),
"is_admin": True,
},
safe=True,
)
return mongo.db.accounts.find_one(obj_id)
def store_swfs(msg, crawl_id, dir_path='/tmp', prefix='?'):
referer = msg.request.headers['Referer'][0] if msg.request.headers[
'Referer'] else ""
if msg.response and msg.response.content:
print msg.request.get_url()
if (msg.response.content[:3] in SWF_MAGIC_NUMBERS
): # to wide, but decompiler will discard them
swf_hash = ut.hash_text(msg.response.content)
swf_url = msg.request.get_url()
db_conn = dbu.mysql_init_db()
db_cursor = db_conn.cursor(dbu.mdb.cursors.DictCursor)
rows = swu.get_swf_obj_from_db('hash', swf_hash, db_cursor)
if not rows:
swf_filename = os.path.join(
dir_path,
"%s-%s" % (prefix, msg.request.path.split('/')[-1]))
swf_filename = swf_filename[:MAX_FILENAME_LEN]
if not swf_filename.endswith('.swf'):
swf_filename += '.swf'
wl_log.info("SWF saved %s referrer: %s" %
(os.path.basename(swf_filename), referer))
fu.write_to_file(swf_filename, msg.response.content)
vector = swu.get_occurence_vector_from_swf(
swf_filename, os.path.join(dir_path, prefix))
duplicate_swf = 0
else:
wl_log.info("A swf with same hash exists in DB: %s %s" %
(swf_hash, swf_url))
vector = swu.str_to_vector(rows[0]['occ_vector'])
swf_filename = rows[0]['local_path']
duplicate_swf = 1
rank, domain = prefix.rsplit('/')[-1].split('-', 1)
swf_info = swu.SwfInfo()
swf_info.rank = rank # this might be fake
swf_info.domain = domain
swf_info.local_path = swf_filename
swf_info.occ_vector = vector
swf_info.hash = swf_hash
swf_info.url = swf_url
swf_info.referer = referer
swf_info.duplicate = duplicate_swf # !!! Y for repeated swfs(that we know before)
swf_info.feat_vector = []
swf_info.page_url = ''
swf_info.occ_string = ' '.join(
swu.human_readable_occ_vector(vector))
swf_info.crawl_id = crawl_id
swu.add_swf_to_db(swf_info, db_conn)
db_conn.commit()
db_cursor.close()
db_conn.close()
elif '.swf' in msg.request.path:
wl_log.warning(".swf in path but content seems non-swf %s %s" %
(msg.request.path, msg.response.content[:100]))
else:
pass
def hash_file(filepath, algo='sha1'):
"""Return the hash value for the file content."""
import utils as ut
return ut.hash_text(read_file(filepath), algo)
def hash_file(filepath, algo='sha1'):
"""Return the hash value for the file content."""
import utils as ut
return ut.hash_text(read_file(filepath), algo)