def _setUpReaders(self):
logging.debug('start setup readers')
if not hasattr(self, 'reader1'):
if re.match('^' + settings.SOURCE_TYPE_DATABASE_PREFIX, self._rds1,
re.I):
# left config is database source
self.reader1 = DatabaseReader(
json.loads(self._rds1[len(settings.
SOURCE_TYPE_DATABASE_PREFIX):]),
self._taskModel.left_query_sql)
elif re.match('^' + settings.SOURCE_TYPE_CSV_PREFIX, self._rds1,
re.I):
self.reader1 = CsvReader(
self._rds1[len(settings.SOURCE_TYPE_CSV_PREFIX):])
else:
raise InvalidDataSourceException(self._rds1 +
' is not a valid source type')
if not hasattr(self, 'reader2'):
if re.match('^' + settings.SOURCE_TYPE_DATABASE_PREFIX, self._rds2,
re.I):
# right config is database source
self.reader2 = DatabaseReader(
json.loads(self._rds2[len(settings.
SOURCE_TYPE_DATABASE_PREFIX):]),
self._taskModel.right_query_sql)
elif re.match('^' + settings.SOURCE_TYPE_CSV_PREFIX, self._rds2,
re.I):
self.reader2 = CsvReader(
self._rds2[len(settings.SOURCE_TYPE_CSV_PREFIX):])
else:
raise InvalidDataSourceException(self._rds2 +
' is not a valid source type')
return (self.reader1, self.reader2)
def testGetRowsListWithSqlite(self):
newDBConfig = {}
newDBConfig['id'] = 'default'
newDBConfig['ENGINE'] = 'django.db.backends.sqlite3'
newDBConfig['NAME'] = ':memory:'
query_sql = 'SELECT * FROM temp;'
r = DatabaseReader(newDBConfig, query_sql)
rows = r.getRowsList()
self.assertEquals(10, len(rows))
r.close()
def testGetColumns(self):
newDBConfig = {}
newDBConfig['id'] = 'default'
newDBConfig['ENGINE'] = 'django.db.backends.sqlite3'
newDBConfig['NAME'] = ':memory:'
query_sql = 'SELECT * FROM temp;'
r = DatabaseReader(newDBConfig, query_sql)
columns = r.getColumns()
r.close()
self.assertEqual(columns, ['col1', 'col2'])
class ConflictRecordReader(object):
def __init__(self, tableName):
defaultConfigs = settings.DATABASES['default'].copy()
self.dr = DatabaseReader(defaultConfigs,
'SELECT * FROM %s;' % (tableName))
def getConflictRecords(self):
return self.dr.getRowsList()
def getColumns(self):
return self.dr.getColumns()
def testDataType4(self):
DbConfig = {}
DbConfig['id'] = 'default'
DbConfig['ENGINE'] = 'django.db.backends.sqlite3'
DbConfig['NAME'] = ':memory:'
query_sql1 = 'SELECT * FROM r1;'
r1 = DatabaseReader(DbConfig, query_sql1)
r2 = TestReader([[
u'col1', u'igfield2', u'col2', ], [
'1', 'ignore&', '1', ], [
'2', 'ignore&', '2', ], [
'3', 'ignore&', '3', ], [
'4', 'ignore&', '4', ], [
'5', 'ignore&', '5', ], [
'6', 'ignore&', '6',
]])
model = Task.objects.create(
summary='test_task',
left_source='database:dummy',
left_query_sql='SELECT * FROM r1;',
right_source='database:dummy',
right_query_sql='SELECT * FROM ds2;',
)
comparator = FieldComparator(r1, r2, [], ['igfield2'], taskModel=model)
self.assertTrue(comparator.isSame())
def testDataType3(self):
DbConfig = {}
DbConfig['id'] = 'default'
DbConfig['ENGINE'] = 'django.db.backends.sqlite3'
DbConfig['NAME'] = ':memory:'
query_sql1 = 'SELECT * FROM r1;'
r1 = DatabaseReader(DbConfig, query_sql1)
r2 = TestReader([[
'col1', 'igfield2', 'datacol', ], [
'row1', 'ignore&', 'data1', ], [
'row2', 'ignore&', 'data2', ], [
'row3', 'ignore&', 'data3', ], [
'row4', 'ignore&', 'data4', ], [
'row5', 'ignore&', 'data5', ], [
'row6', 'ignore&', 'data6',
]])
model = Task.objects.create(
summary='test_task',
left_source='database:dummy',
left_query_sql='SELECT * FROM r1;',
right_source='database:dummy',
right_query_sql='SELECT * FROM ds2;',
)
comparator = FieldComparator(r1, r2, taskModel=model)
self.assertTrue(not comparator.isSame())
self.assertEqual(model.result, 'Fields are inconsistent!')
def testGetSchema(self):
newDBConfig = {}
newDBConfig['id'] = 'default'
newDBConfig['ENGINE'] = 'django.db.backends.sqlite3'
newDBConfig['NAME'] = ':memory:'
query_sql = 'SELECT * FROM temp;'
r = DatabaseReader(newDBConfig, query_sql)
schema = r.getSchema()
self.assertEqual(
schema, {
'fields': [{
'name': 'col1',
'type': 'integer',
'format': 'default'
}, {
'name': 'col2',
'type': 'integer',
'format': 'default'
}],
'missingValues':
settings.SCHEMA_DATABASE_MISSING_VALUES
})
def testSameDataComparisonWithDatabase(self):
DbConfig = {}
DbConfig['id'] = 'default'
DbConfig['ENGINE'] = 'django.db.backends.sqlite3'
DbConfig['NAME'] = ':memory:'
query_sql1 = 'SELECT * FROM r1;'
r1 = DatabaseReader(DbConfig, query_sql1)
query_sql2 = 'SELECT * FROM r2;'
r2 = DatabaseReader(DbConfig, query_sql2)
w1 = DatabaseWriter(DbConfig, 'w1')
w2 = DatabaseWriter(DbConfig, 'w2')
comparator = ValueComparator(r1, r2, w1, w2, [])
isSame = comparator.isSame()
with connection.cursor() as cursor:
cursor.execute('SELECT * FROM w1;')
records1 = list(cursor.fetchall())
cursor.execute('SELECT * FROM w2;')
records2 = list(cursor.fetchall())
self.assertEqual(len(records1), 0)
self.assertEqual(len(records2), 1)
self.assertEqual(isSame, False)
def post(self, request, format=None):
'''
create configuration file for download and return the file name.
it also encrypt the file.
'''
errors = []
configsList = listInPostDataToList(request.POST)
configDict = {config[0]: config[1] for config in configsList}
# DatabaseReader to test if it can show table
try:
dr = DatabaseReader(configDict, 'show tables')
if dr.getCursor() is None:
errors.append('Cannot access the database')
except Exception as e:
errors.append('Invalid database configuration')
errors.append(str(e))
if len(errors) > 0:
return Response({'errors': errors},
status=status.HTTP_400_BAD_REQUEST)
fc = FernetCipher()
configJsonStr = json.dumps(configDict)
code = fc.encode(configJsonStr)
filename = (str(configDict.get('NAME', None)) + '_' +
str(configDict.get('HOST', None)) + '_')
filename += sha256(configJsonStr.encode()).hexdigest()
if not os.path.exists(settings.TEMP_FOLDER):
os.makedirs(settings.TEMP_FOLDER)
with open(os.path.join(settings.TEMP_FOLDER, filename + '.csv'),
'w+') as f:
f.write(code)
f.flush()
f.seek(0)
return Response({'key': filename})
def __init__(self, tableName):
defaultConfigs = settings.DATABASES['default'].copy()
self.dr = DatabaseReader(defaultConfigs,
'SELECT * FROM %s;' % (tableName))