def get_serialized_legacy_workflow_logs(connection): """ Get the serialized log records that correspond to legacy workflows """ query = connection.execute( text(""" SELECT db_dblog.id, db_dblog.time, db_dblog.loggername, db_dblog.levelname, db_dblog.objpk, db_dblog.objname, db_dblog.message, db_dblog.metadata FROM db_dblog WHERE (db_dblog.objname LIKE 'aiida.workflows.user.%') """)) res = list() for row in query: res.append(dict(list(zip(row.keys(), row)))) return dumps_json(res)
def get_serialized_logs_with_no_nodes(connection): """ Get the serialized log records that correspond to nodes that were deleted """ query = connection.execute( text(""" SELECT db_dblog.id, db_dblog.time, db_dblog.loggername, db_dblog.levelname, db_dblog.objpk, db_dblog.objname, db_dblog.message, db_dblog.metadata FROM db_dblog WHERE (db_dblog.objname LIKE 'node.%') AND NOT EXISTS (SELECT 1 FROM db_dbnode WHERE db_dbnode.id = db_dblog.objpk LIMIT 1) """)) res = list() for row in query: res.append(dict(list(zip(row.keys(), row)))) return dumps_json(res)
def get_serialized_legacy_workflow_logs(schema_editor): """ Get the serialized log records that correspond to legacy workflows """ with schema_editor.connection.cursor() as cursor: cursor.execute((""" SELECT db_dblog.id, db_dblog.time, db_dblog.loggername, db_dblog.levelname, db_dblog.objpk, db_dblog.objname, db_dblog.message, db_dblog.metadata FROM db_dblog WHERE (db_dblog.objname LIKE 'aiida.workflows.user.%') """)) keys = [ 'id', 'time', 'loggername', 'levelname', 'objpk', 'objname', 'message', 'metadata' ] res = list() for row in cursor.fetchall(): res.append(dict(list(zip(keys, row)))) return dumps_json(res)
def get_serialized_logs_with_no_nodes(schema_editor): """ Get the serialized log records that don't correspond to a node """ with schema_editor.connection.cursor() as cursor: cursor.execute((""" SELECT db_dblog.id, db_dblog.time, db_dblog.loggername, db_dblog.levelname, db_dblog.objpk, db_dblog.objname, db_dblog.message, db_dblog.metadata FROM db_dblog WHERE (db_dblog.objname LIKE 'node.%') AND NOT EXISTS (SELECT 1 FROM db_dbnode WHERE db_dbnode.id = db_dblog.objpk LIMIT 1) """)) keys = [ 'id', 'time', 'loggername', 'levelname', 'objpk', 'objname', 'message', 'metadata' ] res = list() for row in cursor.fetchall(): res.append(dict(list(zip(keys, row)))) return dumps_json(res)
def setUpBeforeMigration(self): # pylint: disable=too-many-locals import json import importlib from aiida.backends.general.migrations.utils import dumps_json update_024 = importlib.import_module( 'aiida.backends.djsite.db.migrations.0024_dblog_update') DbNode = self.apps.get_model('db', 'DbNode') DbWorkflow = self.apps.get_model('db', 'DbWorkflow') DbLog = self.apps.get_model('db', 'DbLog') # Creating the needed nodes & workflows calc_1 = DbNode(type='node.process.calculation.CalculationNode.', user_id=self.default_user.id) param = DbNode(type='data.dict.Dict.', user_id=self.default_user.id) leg_workf = DbWorkflow(label='Legacy WorkflowNode', user_id=self.default_user.id) calc_2 = DbNode(type='node.process.calculation.CalculationNode.', user_id=self.default_user.id) # Storing them calc_1.save() param.save() leg_workf.save() calc_2.save() # Creating the corresponding log records and storing them log_1 = DbLog(loggername='CalculationNode logger', objpk=calc_1.pk, objname='node.calculation.job.quantumespresso.pw.', message='calculation node 1', metadata=json.dumps({ 'msecs': 719.0849781036377, 'objpk': calc_1.pk, 'lineno': 350, 'thread': 140011612940032, 'asctime': '10/21/2018 12:39:51 PM', 'created': 1540118391.719085, 'levelno': 23, 'message': 'calculation node 1', 'objname': 'node.calculation.job.quantumespresso.pw.', })) log_2 = DbLog(loggername='something.else logger', objpk=param.pk, objname='something.else.', message='parameter data with log message') log_3 = DbLog( loggername='TopologicalWorkflow logger', objpk=leg_workf.pk, objname= 'aiida.workflows.user.topologicalworkflows.topo.TopologicalWorkflow', message='parameter data with log message') log_4 = DbLog(loggername='CalculationNode logger', objpk=calc_2.pk, objname='node.calculation.job.quantumespresso.pw.', message='calculation node 2', metadata=json.dumps({ 'msecs': 719.0849781036377, 'objpk': calc_2.pk, 'lineno': 360, 'levelno': 23, 'message': 'calculation node 1', 'objname': 'node.calculation.job.quantumespresso.pw.', })) # Creating two more log records that don't correspond to a node log_5 = DbLog(loggername='CalculationNode logger', objpk=(calc_2.pk + 1000), objname='node.calculation.job.quantumespresso.pw.', message='calculation node 1000', metadata=json.dumps({ 'msecs': 718, 'objpk': (calc_2.pk + 1000), 'lineno': 361, 'levelno': 25, 'message': 'calculation node 1000', 'objname': 'node.calculation.job.quantumespresso.pw.', })) log_6 = DbLog(loggername='CalculationNode logger', objpk=(calc_2.pk + 1001), objname='node.calculation.job.quantumespresso.pw.', message='calculation node 10001', metadata=json.dumps({ 'msecs': 722, 'objpk': (calc_2.pk + 1001), 'lineno': 362, 'levelno': 24, 'message': 'calculation node 1001', 'objname': 'node.calculation.job.quantumespresso.pw.', })) # Storing the log records log_1.save() log_2.save() log_3.save() log_4.save() log_5.save() log_6.save() # Storing temporarily information needed for the check at the test self.to_check = dict() # Keeping calculation & calculation log ids self.to_check['CalculationNode'] = ( calc_1.pk, log_1.pk, calc_2.pk, log_4.pk, ) # Getting the serialized Dict logs param_data = DbLog.objects.filter(objpk=param.pk).filter( objname='something.else.').values(*update_024.values_to_export)[:1] serialized_param_data = dumps_json(list(param_data)) # Getting the serialized logs for the unknown entity logs (as the export migration fuction # provides them) - this should coincide to the above serialized_unknown_exp_logs = update_024.get_serialized_unknown_entity_logs( self.schema_editor) # Getting their number unknown_exp_logs_number = update_024.get_unknown_entity_log_number( self.schema_editor) self.to_check['Dict'] = (serialized_param_data, serialized_unknown_exp_logs, unknown_exp_logs_number) # Getting the serialized legacy workflow logs leg_wf = DbLog.objects.filter(objpk=leg_workf.pk).filter( objname= 'aiida.workflows.user.topologicalworkflows.topo.TopologicalWorkflow' ).values(*update_024.values_to_export)[:1] serialized_leg_wf_logs = dumps_json(list(leg_wf)) # Getting the serialized logs for the legacy workflow logs (as the export migration function # provides them) - this should coincide to the above serialized_leg_wf_exp_logs = update_024.get_serialized_legacy_workflow_logs( self.schema_editor) eg_wf_exp_logs_number = update_024.get_legacy_workflow_log_number( self.schema_editor) self.to_check['WorkflowNode'] = (serialized_leg_wf_logs, serialized_leg_wf_exp_logs, eg_wf_exp_logs_number) # Getting the serialized logs that don't correspond to a DbNode record logs_no_node = DbLog.objects.filter( id__in=[log_5.id, log_6.id]).values(*update_024.values_to_export) serialized_logs_no_node = dumps_json(list(logs_no_node)) # Getting the serialized logs that don't correspond to a node (as the export migration function # provides them) - this should coincide to the above serialized_logs_exp_no_node = update_024.get_serialized_logs_with_no_nodes( self.schema_editor) logs_no_node_number = update_024.get_logs_with_no_nodes_number( self.schema_editor) self.to_check['NoNode'] = (serialized_logs_no_node, serialized_logs_exp_no_node, logs_no_node_number)