def test_get_cursor(self, mc):
conn = MySQLConnector()
conn.connection.is_connected = Mock(return_value=True)
conn.connection.cursor.return_value = 42
self.assertEqual(conn.get_cursor(), 42)
conn.connection.is_connected.return_value = False
self.assertIsNone(conn.get_cursor())
def fill_submits(sqlite_cursor, base_dir, origin, mysql_config):
if '' in mysql_config.values():
logging.error('MySQL parameters are not specified')
exit()
logging.info('Now connecting to MySQL')
from mysql_connector import MySQLConnector
mysql_connector = MySQLConnector()
mysql_connector.create_connection(mysql_config)
logging.info('Connected to MySQL database')
ej_cursor = mysql_connector.get_cursor()
logging.info("Filling database from XML's and MySQL database")
fill_from_xml(sqlite_cursor, ej_cursor, base_dir, origin)
mysql_connector.close()
def fill_recommendations_table(self, limit=False):
processing = 0
log = 0
if limit:
self.our_db_cursor.execute('SELECT id FROM users WHERE id > ? AND id < ?', (limit[0], limit[1]))
else:
self.our_db_cursor.execute('SELECT id FROM users')
logging.info('Loading users ids..')
database_users_ids = self.our_db_cursor.fetchall()
users_num = len(database_users_ids)
logging.info('Loaded {} users ids'.format(users_num))
# the number of such sequences is the problem_ref-problem_ref
number_of_sequences = dict()
logging.info('Users pocessing..')
for user_id_row in database_users_ids:
processing += 1
if processing >= 100:
log += 1
processing = 0
logging.info('Processed {} from {}'.format(log * 100, users_num))
self.our_db_cursor.execute('SELECT problem_ref '
'FROM submits '
'WHERE user_ref=? AND outcome=? '
'ORDER BY timestamp', (user_id_row['id'], 'OK'))
sorted_problems_refs = self.our_db_cursor.fetchall()
for list_id, problem_ref_row in enumerate(sorted_problems_refs[:-1]):
problem_ref = problem_ref_row['problem_ref']
next_ref = sorted_problems_refs[list_id + 1]['problem_ref']
start_problem_id = self._get_problem_id_by_problem_ref(problem_ref)
next_problem_id = self._get_problem_id_by_problem_ref(next_ref)
if start_problem_id in number_of_sequences:
if next_problem_id not in number_of_sequences[start_problem_id]:
number_of_sequences[start_problem_id][next_problem_id] = 0
number_of_sequences[start_problem_id][next_problem_id] += 1
else:
number_of_sequences[start_problem_id] = {next_problem_id: 1}
logging.info('Processed {} from {}'.format(log * 100 + processing, users_num))
result = dict()
logging.info('Data processing')
for problem_id_start in number_of_sequences:
for problem_id_next in number_of_sequences[problem_id_start]:
node = (number_of_sequences[problem_id_start][problem_id_next], problem_id_next)
if problem_id_start in result:
result[problem_id_start].append(node)
else:
result[problem_id_start] = [node]
logging.info('Output database connection to recording')
mysql_connector = MySQLConnector()
mysql_connector.create_connection(self.output_db_config)
logging.info('Connected to MySQL database')
self.stats_db_cursor = mysql_connector.get_cursor()
logging.info('Writing in database')
for key in result:
result[key].sort()
for some in result[key][:min(len(result[key]), NUMBER_OF_RECOMMENDATIONS_TO_THE_PROBLEM)]:
self._write_to_db(key, some[1])
logging.info('Committing changes')
mysql_connector.connection.commit()
mysql_connector.close()
logging.info('Output mysql database closed')
