def _set_package_consumed(self, package_name):
query = 'UPDATE similar_app SET status=%s WHERE package_name=%s'
try:
MySQLDBUtil.update(query, (CONSUMED, package_name), self._db_conn)
except Exception:
self.logger.exception('Set package name %s as consumed error' %
package_name)
def _set_developer_consumed(self, developer):
query = 'UPDATE developer SET status=%s WHERE name=%s'
try:
MySQLDBUtil.update(query, (CONSUMED, developer), self._db_conn)
except Exception:
self.logger.exception('Set devloper %s as consumed error' %
developer)
def _set_category_consumed(self, category):
query = 'UPDATE category SET status=%s WHERE category=%s'
try:
MySQLDBUtil.update(query, (CONSUMED, category), self._db_conn)
except Exception:
self.logger.exception('Set category %s as consumed error' %
category)
def _store_key_words(self, key_words):
values = []
for key_word in key_words:
values.append('("%s")' % key_word)
if len(values) > 0:
query = 'INSERT IGNORE INTO key_word (key_word) VALUES ' + ','.join(
values)
try:
MySQLDBUtil.insert(query, None, self._db_conn)
except Exception:
self.logger.exception('Store key words into database fail')
def _store_package_names(self, package_names):
values = []
for package_name in package_names:
values.append('("%s")' % package_name)
if len(values) > 0:
query = 'INSERT IGNORE INTO package_name (package_name) VALUES ' + ','.join(
values)
try:
MySQLDBUtil.insert(query, None, self._db_conn)
except Exception:
self.logger.exception('Store package names into database fail')
def _update_status(self, status, developer=None):
if not developer:
query = 'UPDATE developer SET status=%s'
else:
query = 'UPDATE developer SET status=%s WHERE name=%s'
try:
if not developer:
MySQLDBUtil.update(query, (status, ), self._db_conn)
else:
MySQLDBUtil.update(query, (status, developer), self._db_conn)
except Exception:
self.logger.exception('Update record status')
def _store_app_description(self, app_detail):
if not app_detail:
return
description = ' '.join(app_detail.description)
description = description.replace('"', '').replace('\'', '')
if not description:
return
query = 'INSERT INTO raw_text (text) VALUES ("%s")' % description
try:
MySQLDBUtil.insert(query, None, self._mysql_db_conn)
except Exception:
self.logger.exception('Store app description error')
def _update_status(self, status, package_name=None):
if not package_name:
query = 'UPDATE package_name SET status=%s'
else:
query = 'UPDATE package_name SET status=%s WHERE package_name=%s'
try:
if not package_name:
MySQLDBUtil.update(query, (status,), self._db_conn)
else:
MySQLDBUtil.update(query, (status, package_name), self._db_conn)
except Exception:
self.logger.exception('Update record status')
def _update_status(self, status, category=None):
if not category:
query = 'UPDATE category SET status=%s'
else:
query = 'UPDATE category SET status=%s WHERE category=%s'
try:
if not category:
MySQLDBUtil.update(query, (status,), self._db_conn)
else:
MySQLDBUtil.update(query, (status, category), self._db_conn)
except Exception:
self.logger.exception('Update record status')
def _update_status(self, status, key_word=None):
if not key_word:
query = 'UPDATE key_word SET status=%s'
else:
query = 'UPDATE key_word SET status=%s WHERE key_word=%s'
try:
if not key_word:
MySQLDBUtil.update(query, (status, ), self._db_conn)
else:
MySQLDBUtil.update(query, (status, key_word), self._db_conn)
except Exception:
self.logger.exception('Update record status')
def run(self):
self.logger.info('Connect to the database...')
self._conn_db()
if not self._db_conn:
self.logger.debug(
'No database connection, cannot perform keyword extraction')
return
self.logger.info('Construct unnecessary words...')
self._construct_unnecessary_words()
while 1:
query = 'SELECT id,text FROM raw_text WHERE status=%s'
results = MySQLDBUtil.fetch_multiple_rows(query, (UN_PUBLISHED, ),
self._db_conn)
if not results:
time.sleep(SLEEP_TIME)
else:
for result in results:
(record_id, text) = result
key_words = self._extrack_key_words(text)
try:
self._store_key_words(key_words)
except Exception:
self.logger.exception('Store key words error')
try:
self._update_status(record_id, CONSUMED)
except Exception:
self.logger.exception('Update raw_text status error')
def _store_app_developer(self, app_detail):
if not app_detail:
return
developer_link = app_detail.developer_link
if not developer_link:
return
items = developer_link.split('id=')
if len(items) == 2:
developer_name = items[-1]
query = 'INSERT IGNORE INTO developer (name) VALUES ("%s")' % developer_name
try:
MySQLDBUtil.insert(query, None, self._mysql_db_conn)
self.logger.info('Stored app developer %s' % developer_name)
except Exception:
self.logger.exception('Store app developer error')
else:
return
def _is_no_more_records(self):
query = 'SELECT COUNT(*) FROM category WHERE status=%s'
try:
result = MySQLDBUtil.fetch_single_row(query, (UN_PUBLISHED,), self._db_conn)
if result:
(count,) = result
if count == 0:
return True
except Exception:
self.logger.exception('Check available records error')
return False
def _fetch_category_list(self):
category_list = []
self.logger.info('Get un-published category list...')
query = 'SELECT category FROM category WHERE status=%s LIMIT %s'
try:
results = MySQLDBUtil.fetch_multiple_rows(query, (UN_PUBLISHED, QUEUE_LIMIT), self._db_conn)
for result in results:
(category,) = result
category_list.append(category)
except Exception:
self.logger.exception('Query un-published category error')
return category_list
def _fetch_package_list(self):
package_list = []
self.logger.info('Get un-published package list...')
query = 'SELECT package_name FROM package_name WHERE status=%s LIMIT %s'
try:
results = MySQLDBUtil.fetch_multiple_rows(query, (UN_PUBLISHED, QUEUE_LIMIT), self._db_conn)
for result in results:
(package_name,) = result
package_list.append(package_name)
except Exception:
self.logger.exception('Query un-used package name error')
return package_list
def _fetch_developer_list(self):
developer_list = []
self.logger.info('Get un-published developer list...')
query = 'SELECT name FROM developer WHERE status=%s LIMIT %s'
try:
results = MySQLDBUtil.fetch_multiple_rows(
query, (UN_PUBLISHED, QUEUE_LIMIT), self._db_conn)
for result in results:
(developer, ) = result
developer_list.append(developer)
except Exception:
self.logger.exception('Query un-published developer error')
return developer_list
def _fetch_key_words(self):
key_words = []
self.logger.info('Get un-published keywords...')
query = 'SELECT key_word FROM key_word WHERE status=%s LIMIT %s'
try:
results = MySQLDBUtil.fetch_multiple_rows(
query, (UN_PUBLISHED, QUEUE_LIMIT), self._db_conn)
for result in results:
(key_word, ) = result
key_words.append(key_word)
except Exception:
self.logger.exception('Query un-published keywords error')
return key_words
def _set_key_word_consumed(self, key_word):
query = 'UPDATE key_word SET status=%s WHERE key_word=%s'
try:
MySQLDBUtil.update(query, (CONSUMED, key_word), self._db_conn)
except Exception:
self.logger.exception('Set key word %s as consumed error' % key_word)
def _store_package_name_similar(self, package_name):
query = 'INSERT IGNORE INTO similar_app (package_name) VALUES ("%s")' % package_name
try:
MySQLDBUtil.insert(query, None, self._mysql_db_conn)
except Exception:
self.logger.exception('Store package name into similar app database fail')
def _update_status(self, record_id, status):
query = 'UPDATE raw_text SET status=%s WHERE id=%s'
try:
MySQLDBUtil.update(query, (status, record_id), self._db_conn)
except Exception:
self.logger.exception('Update raw_text table status error')