class AppIdsImporter(object):
def __init__(self):
self.market = None
self.category_ids = None
self.database_service = DatabaseService()
self.redis_service = RedisService()
def imported(self, date_str, **kargs):
print 'Started to import ids'
logger.info('Started to import ids')
for category_id in self.category_ids:
import_ids_set = set()
for letter in string.uppercase:
for content in self._load(date_str, category_id, letter):
import_ids_set.update(self._parser(content))
print len(import_ids_set)
self._save(import_ids_set)
garbage_number = gc.collect()
print 'Garbage number:', garbage_number
self.database_service.close()
def _load(self, date_str, category_id, letter):
category_page_key = CATEGORY_PAGE_KEY.format(date=date_str, category_id=category_id, market=self.market, letter=letter)
print category_page_key
category_pages = self.redis_service.members_set(category_page_key)
for category_page in category_pages:
yield zlib.decompress(category_page)
@abstractmethod
def _parser(self, content):
"""
:param content:
:return: ids set
"""
raise NotImplementedError()
def _save(self, ids_set):
if not ids_set:
return
try:
self.database_service.import_ids(self.market, ids_set)
print 'Succeed import ids: {}'.format(len(ids_set))
logger.info('Succeed import ids: {}'.format(len(ids_set)))
except Exception as ex:
logger.exception(ex)
print ex
logger.error('Failed import ids {}'.format(len(ids_set)))
print 'Failed import ids.'
def test_jwt_token_contains_subject_and_user_type(self):
email = "*****@*****.**"
password = "******"
db = DatabaseService()
api_config = APIConfig()
db.save_user(email=email, password=password)
data = json.dumps({
"email": "*****@*****.**",
"password": "******"
})
result = self.app.post("/api/v1/login", data=data)
try:
data = json.loads(result.data)
token = data["token"]
decoded = jwt.decode(token,
api_config.SECRET_KEY,
algorithms=["HS256"])
subject = decoded["subject"]
userType = decoded["userType"]
except:
decoded = None
self.assertIsNotNone(decoded)
def get_profile():
auth_header = request.headers.get("Authorization")
if auth_header is None:
return make_response("Valid token required", 401)
try:
encoded = auth_header.split(" ")[1]
decoded = jwt.decode(encoded,
app.config["SECRET_KEY"],
algorithms=["HS256"])
email = decoded["subject"]
except:
return make_response("Invalid token", 401)
db = DatabaseService()
user = db.get_user(email)
formattedUser = {
"email": user["email"],
"firstName": user["firstName"],
"lastName": user["lastName"],
"userType": user["userType"],
}
return jsonify(formattedUser)
def login():
try:
data = get_request_data(request, ["email", "password"])
except (ValueError, TypeError) as e:
return handle_error(
message="Invalid parameters: %s" % str(e),
logger=logger,
status_code=422
)
db = DatabaseService()
userType = db.authenticate_user(
email=data["email"],
password=data["password"],
)
if not userType:
return make_response("Error: invalid credentials", 401)
token = jwt.encode(
{"subject": data["email"], "userType": userType},
app.config["SECRET_KEY"],
algorithm="HS256",
)
return jsonify({"token": token})
def create_validation_token():
try:
data = get_request_data(
request,
required_params=["email"],
)
except (ValueError, TypeError) as e:
return handle_error(
message="%s: %s" % (request.url, str(e)),
logger=logger,
status_code=422,
)
token = DatabaseService().create_validation_token(email=data["email"])
if token is None:
return make_response("Could not generate unique token", 500)
return jsonify({"token": token})
def test_reset_password_updates_password(self):
email = "*****@*****.**"
token = "testtoken"
password1 = "testpassword1"
password2 = "testpassword2"
db = DatabaseService()
db.save_user(email=email, password=password1)
db.save_reset_token(email=email, token=token)
data = json.dumps({
"token": token,
"password": password2,
"passwordCheck": password2,
})
self.app.post("/api/v1/reset-password", data=data)
result = db.authenticate_user(email=email, password=password2)
self.assertIsNotNone(result)
def confirm_validation_token():
try:
data = get_request_data(
request,
required_params=["token"],
)
except (ValueError, TypeError) as e:
return handle_error(
message="%s: %s" % (request.url, str(e)),
logger=logger,
status_code=422,
)
try:
email = DatabaseService().confirm_validation_token(data["token"])
if email is not None:
return jsonify({"email": email})
except TypeError:
logger.error("Could not get email for token %s" % data["token"])
return make_response("Invalid token", 401)
def test_user_can_login_with_new_password_after_reset_password(self):
email = "*****@*****.**"
token = "testtoken"
password1 = "testpassword1"
password2 = "testpassword2"
db = DatabaseService()
db.save_user(email=email, password=password1)
db.save_reset_token(email=email, token=token)
data = json.dumps({
"token": token,
"password": password2,
"passwordCheck": password2,
})
self.app.post("/api/v1/reset-password", data=data)
login_data = json.dumps({"email": email, "password": password2})
result = self.app.post("/api/v1/login", data=login_data)
self.assertEqual(result.status_code, 200)
def __init__(self):
self.market = None
self.category_ids = None
self.database_service = DatabaseService()
self.redis_service = RedisService()
def setUp(self):
super(TestDatabaseService, self).setUp()
self.db = DatabaseService()
class TestDatabaseService(BaseTest):
def setUp(self):
super(TestDatabaseService, self).setUp()
self.db = DatabaseService()
def tearDown(self):
super(TestDatabaseService, self).tearDown()
def test_database_service_can_connect(self):
self.assertIsNotNone(self.db)
def test_database_service_can_save_user(self):
email = '*****@*****.**'
password = '******'
self.db.save_user(
email=email,
password=password,
)
query = "SELECT COUNT(*) FROM users"
curr = self.conn.cursor()
curr.execute(query)
results = curr.fetchall()
self.assertEqual(1, results[0][0])
curr.close()
def test_database_service_saves_user_with_correct_email(self):
email = '*****@*****.**'
password = '******'
self.db.save_user(
email=email,
password=password,
)
query = "SELECT email FROM users"
curr = self.conn.cursor()
curr.execute(query)
results = curr.fetchall()
self.assertEqual(email, results[0][0])
curr.close()
def test_database_service_saves_email_as_lowercase(self):
email = '*****@*****.**'
lowercaseEmail = '*****@*****.**'
password = '******'
self.db.save_user(
email=email,
password=password,
)
query = "SELECT email FROM users"
curr = self.conn.cursor()
curr.execute(query)
results = curr.fetchall()
self.assertEqual(lowercaseEmail, results[0][0])
curr.close()
def test_database_service_saves_hashed_password(self):
email = '*****@*****.**'
password = '******'
self.db.save_user(
email=email,
password=password,
)
query = "SELECT password FROM users"
curr = self.conn.cursor()
curr.execute(query)
results = curr.fetchall()
self.assertNotEqual(password, results[0][0])
curr.close()
def test_database_services_saves_correct_password(self):
email = '*****@*****.**'
password = '******'
self.db.save_user(
email=email,
password=password,
)
query = "SELECT password FROM users"
curr = self.conn.cursor()
curr.execute(query)
results = curr.fetchall()
self.assertTrue(check_password_hash(results[0][0], password))
curr.close()
def test_database_service_saves_first_and_last_name(self):
email = '*****@*****.**'
password = '******'
firstName = 'First'
lastName = 'Last'
self.db.save_user(
email=email,
password=password,
firstName=firstName,
lastName=lastName,
)
query = "SELECT first_name, last_name FROM users"
curr = self.conn.cursor()
curr.execute(query)
results = curr.fetchall()
self.assertEqual(firstName, results[0][0])
self.assertEqual(lastName, results[0][1])
curr.close()
def test_database_service_handles_default_values_for_first_and_last_name(
self):
email = '*****@*****.**'
password = '******'
self.db.save_user(
email=email,
password=password,
)
query = "SELECT first_name, last_name FROM users"
curr = self.conn.cursor()
curr.execute(query)
results = curr.fetchall()
self.assertEqual(None, results[0][0])
self.assertEqual(None, results[0][1])
curr.close()
def test_database_service_can_authorize_valid_user_credentials(self):
email = '*****@*****.**'
password = '******'
self.db.save_user(
email=email,
password=password,
)
result = self.db.authenticate_user(email=email, password=password)
self.assertIsNotNone(result)
def test_database_service_returns_user_type_after_authorizing(self):
email = '*****@*****.**'
password = '******'
userType = 'testtype'
self.db.save_user(
email=email,
password=password,
userType=userType,
)
result = self.db.authenticate_user(email=email, password=password)
self.assertEqual(userType, result)
def test_database_service_handles_nonexistent_user(self):
email = '*****@*****.**'
password = '******'
result = self.db.authenticate_user(email=email, password=password)
self.assertEqual(result, None)
def test_database_service_creates_user_with_correct_user_type(self):
email = '*****@*****.**'
password = '******'
userType = 'testtype'
self.db.save_user(
email=email,
password=password,
userType=userType,
)
query = "SELECT user_type FROM users"
curr = self.conn.cursor()
curr.execute(query)
results = curr.fetchall()
self.assertEqual(userType, results[0][0])
curr.close()
def test_database_service_can_return_user_data(self):
email = '*****@*****.**'
password = '******'
userType = 'testtype'
self.db.save_user(
email=email,
password=password,
userType=userType,
)
user = self.db.get_user(email=email)
self.assertIsNotNone(user)
def test_database_service_returns_correct_user_data_on_get_user(self):
email = '*****@*****.**'
password = '******'
userType = 'testtype'
firstName = 'First'
lastName = 'Last'
self.db.save_user(
email=email,
password=password,
userType=userType,
firstName=firstName,
lastName=lastName,
)
user = self.db.get_user(email=email)
self.assertEqual(user['email'], email)
self.assertEqual(user['firstName'], firstName)
self.assertEqual(user['lastName'], lastName)
self.assertEqual(user['userType'], userType)
def test_database_service_can_save_reset_token(self):
self.db.save_reset_token(
email='*****@*****.**',
token='thisisatesttoken',
)
query = "SELECT * FROM reset_tokens"
curr = self.conn.cursor()
curr.execute(query)
results = curr.fetchall()
curr.close()
self.assertIsNotNone(results[0])
def test_database_service_can_validate_saved_reset_token(self):
email = '*****@*****.**'
token = 'testtoken'
curr = self.conn.cursor()
data = (email, token)
query = "INSERT INTO reset_tokens (email, token) VALUES (%s, %s)"
curr.execute(query, data)
self.conn.commit()
user_email = self.db.validate_reset_token(token=token)
self.assertIsNotNone(user_email)
def test_database_service_can_update_password(self):
email = '*****@*****.**'
password1 = 'testpass1'
password2 = 'testpass2'
self.db.save_user(email=email, password=password1)
query = "SELECT password FROM users"
curr = self.conn.cursor()
curr.execute(query)
results = curr.fetchall()
curr.close()
first_password = results[0][0]
self.db.update_password(email=email, password=password2)
query = "SELECT password FROM users"
curr = self.conn.cursor()
curr.execute(query)
results = curr.fetchall()
curr.close()
second_password = results[0][0]
self.assertNotEqual(first_password, second_password)
class AppDetailImporter(object):
def __init__(self):
self.market = None
self.database_service = DatabaseService()
self.redis_service = RedisService()
def imported(self, date_str, **kwargs):
start_id = kwargs.get('start', 1)
end_id = kwargs.get('end', -1)
ids = self.database_service.load_ids(self.market, start_id, end_id)
for batch_app_ids in chunks(ids, DEFAULT_BATCH_SIZE):
print 'Started to import batch:', len(batch_app_ids)
logger.info('Started to import batch: {}'.format(len(batch_app_ids)))
for app_id in batch_app_ids:
content = self._load(date_str, app_id)
detail_dict = self._parser(content)
self._save(app_id, detail_dict)
garbage_number = gc.collect()
print 'Garbage number:', garbage_number
self.database_service.close()
def _load(self, date_str, app_id):
app_detail_key = DETAIL_SOURCE_KEY.format(date=date_str, market=self.market, app_id=app_id)
detail = self.redis_service.get(app_detail_key)
if detail:
return zlib.decompress(detail)
@abstractmethod
def _parser(self, content):
"""
:param content:
:return: detail dict
"""
raise NotImplementedError()
def _save(self, app_id, current_detail):
if not current_detail:
return
try:
last_detail = self.database_service.get_app_detail(self.market, app_id)
if self._need_to_update(last_detail, current_detail):
self.database_service.update_app_detail(self.market, app_id, current_detail)
logger.info('Succeed save detail for {} in {}'.format(app_id, self.market))
print 'Succeed save detail for {} in {}'.format(app_id, self.market)
if last_detail['name'] != 'NULL':
self.database_service.save_event(app_id, last_detail, current_detail)
logger.info('Succeed save event for {} in {}'.format(app_id, self.market))
print 'Succeed save event for {} in {}'.format(app_id, self.market)
# elif self._need_to_update_icon(last_detail, current_detail):
# self.database_service.save_icon_event(app_id, last_detail, current_detail)
# logger.info('Succeed save icon event for {} in {}'.format(app_id, self.market))
# print 'Succeed save icon event for {} in {}'.format(app_id, self.market)
else:
logger.info('Not need to update for {} in {}'.format(app_id, self.market))
print 'Not need to update for {} in {}'.format(app_id, self.market)
except Exception as ex:
print ex
logger.exception(ex)
logger.info('Failed update detail and event for {} in {}'.format(app_id, self.market))
print 'Failed save detail and event for {} in {}'.format(app_id, self.market)
@staticmethod
def _need_to_update(last_detail_dict, current_detail_dict):
if last_detail_dict['name'] != current_detail_dict['name']:
return True
@staticmethod
def _need_to_update_icon(last_detail, current_detail_dict):
if last_detail['icon_url'] != current_detail_dict['icon_url']:
return True