def setUp(self):
jsonFileName = os.path.join(TestModulestoreImporter.dataDir,
'modulestore_latest.json')
self.importer = ModulestoreImporter(jsonFileName)
self.pickleCachePath = os.path.join(TestModulestoreImporter.dataDir,
'tmpTestHashLookup.pkl')
self.uuidRegex = '[a-f0-9]{8}_[a-f0-9]{4}_[a-f0-9]{4}_[a-f0-9]{4}_[a-f0-9]{12}'
self.pattern = re.compile(self.uuidRegex)
self.timestampRegex = r'[1-2][0-9]{3}-[0-1][1-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9]\.[0-9]{0,6}Z{0,1}'
self.timestampPattern = re.compile(self.timestampRegex)
# Pattern that recognizes our tmp files. They start with
# 'oolala', followed by random junk, followed by a period and
# 0 to 3 file extension letters (always 3, I believe):
self.tmpFileNamePattern = re.compile('/tmp/oolala[^.]*\..{0,3}')
def setUp(self):
jsonFileName = os.path.join(TestModulestoreImporter.dataDir, 'modulestore_latest.json')
self.importer = ModulestoreImporter(jsonFileName)
self.pickleCachePath = os.path.join(TestModulestoreImporter.dataDir, 'tmpTestHashLookup.pkl')
self.uuidRegex = '[a-f0-9]{8}_[a-f0-9]{4}_[a-f0-9]{4}_[a-f0-9]{4}_[a-f0-9]{12}'
self.pattern = re.compile(self.uuidRegex)
self.timestampRegex = r'[1-2][0-9]{3}-[0-1][1-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9]\.[0-9]{0,6}Z{0,1}'
self.timestampPattern = re.compile(self.timestampRegex)
# Pattern that recognizes our tmp files. They start with
# 'oolala', followed by random junk, followed by a period and
# 0 to 3 file extension letters (always 3, I believe):
self.tmpFileNamePattern = re.compile('/tmp/oolala[^.]*\..{0,3}')
class TestModulestoreImporter(unittest.TestCase):
# Setup method called once:
@classmethod
def setUpClass(cls):
# Move the existing, full-sized modulestore_latest.json
# aside, and replace it with a smaller test version:
# Dir of currently running test script:
currDir = os.path.dirname(__file__)
# Where the modulestore_latest.json is stored:
TestModulestoreImporter.dataDir = os.path.join(currDir, '..', 'data')
TestModulestoreImporter.currModStoreLatest = os.path.join(TestModulestoreImporter.dataDir, 'modulestore_latest.json')
# Save current, full-size modulestore_latest.json
# (preserving its symlink):
TestModulestoreImporter.savedModStoreLatest = TestModulestoreImporter.currModStoreLatest + '.SAVED'
try:
shutil.move(TestModulestoreImporter.currModStoreLatest, TestModulestoreImporter.savedModStoreLatest)
except IOError:
pass
# Do the replacement with the test version:
testModStoreLatest = os.path.join(currDir, 'data', 'mini_modulestore_latest.json')
shutil.copy(testModStoreLatest, TestModulestoreImporter.currModStoreLatest)
# The hashLookup.pkl, if it exists, is moved to a save-copy:
TestModulestoreImporter.hashLookupPicklePath = os.path.join(TestModulestoreImporter.dataDir, 'hashLookup.pkl')
TestModulestoreImporter.savedHashLookupPicklePath = TestModulestoreImporter.hashLookupPicklePath + '.SAVED'
if (os.path.exists(TestModulestoreImporter.hashLookupPicklePath)):
shutil.move(TestModulestoreImporter.hashLookupPicklePath, TestModulestoreImporter.savedHashLookupPicklePath)
TestModulestoreImporter.csvTestOutPath = os.path.join(TestModulestoreImporter.dataDir, 'tmpCacheCSVFile.csv')
# Teardown method called once:
@classmethod
def tearDownClass(cls):
# Put the true, full size modulestore_latest.json back
# into its place (preserving its symlink):
try:
shutil.move(TestModulestoreImporter.savedModStoreLatest, TestModulestoreImporter.currModStoreLatest)
except IOError:
print('NOTE: no modulestore_latest.json found in <projRoot>/json_to_relation/data; run scripts/cronRefreshModuleStore.sh.')
# If hashLookup.pkl existed at the the outset, and was saved,
# restore it:
try:
shutil.move(TestModulestoreImporter.savedHashLookupPicklePath, TestModulestoreImporter.hashLookupPicklePath)
except:
pass
# Remove the csv test output:
try:
os.remove(TestModulestoreImporter.csvTestOutPath)
except:
pass
def setUp(self):
jsonFileName = os.path.join(TestModulestoreImporter.dataDir, 'modulestore_latest.json')
self.importer = ModulestoreImporter(jsonFileName)
self.pickleCachePath = os.path.join(TestModulestoreImporter.dataDir, 'tmpTestHashLookup.pkl')
self.uuidRegex = '[a-f0-9]{8}_[a-f0-9]{4}_[a-f0-9]{4}_[a-f0-9]{4}_[a-f0-9]{12}'
self.pattern = re.compile(self.uuidRegex)
self.timestampRegex = r'[1-2][0-9]{3}-[0-1][1-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9]\.[0-9]{0,6}Z{0,1}'
self.timestampPattern = re.compile(self.timestampRegex)
# Pattern that recognizes our tmp files. They start with
# 'oolala', followed by random junk, followed by a period and
# 0 to 3 file extension letters (always 3, I believe):
self.tmpFileNamePattern = re.compile('/tmp/oolala[^.]*\..{0,3}')
def tearDown(self):
# Some of the tests create a temporary
# pickle file; remove it:
try:
os.remove(TestModulestoreImporter.hashLookupPicklePath)
except:
pass
@unittest.skipIf(not TEST_ALL, "Temporarily disabled")
def testModuleStoreImporter(self):
self.useTheDict()
@unittest.skipIf(not TEST_ALL, "Temporarily disabled")
def testExportHashInfoToCSV(self):
csvOutFile = TestModulestoreImporter.csvTestOutPath
# Write the current cache out to csv:
self.importer.exportHashInfo(csvOutFile)
csvOutExpected = os.path.join(os.path.dirname(__file__), 'data/modulestoreImporterCsvTruth.csv')
self.assertFileContentEquals(csvOutExpected, csvOutFile)
# Check whether dict ops still work:
self.useTheDict()
def useTheDict(self):
self.assertEqual('Module One video', self.importer.getDisplayName('c89417444f4443f9a34039be3054962e'))
self.assertEqual('Medicine', self.importer.getOrg('c89417444f4443f9a34039be3054962e'))
self.assertEqual('HRP258', self.importer.getCourseShortName('c89417444f4443f9a34039be3054962e'))
self.assertEqual('video', self.importer.getCategory('c89417444f4443f9a34039be3054962e'))
self.assertIsNone(self.importer.getRevision('c89417444f4443f9a34039be3054962e'))
#--------------------------------------------------------------------------------------------------
def assertFileContentEquals(self, expected, filePathOrStrToCompareTo):
'''
Compares two file or string contents. First arg is either an open file or a string.
That is the ground truth to compare to. Second argument is the same: file or string.
@param expected: the file that contains the ground truth
@type expected: {File | String }
@param filePathOrStrToCompareTo: the actual file as constructed by the module being tested
@type filePathOrStrToCompareTo: { File | String }
'''
# Ensure that 'expected' is a File-like object:
if isinstance(expected, file):
# It's a file, just use it:
strFile = expected
else:
# Turn into a string 'file':
with open(expected, 'r') as fd:
strFile = StringIO.StringIO(fd.read())
# Check whether filePathOrStrToCompareTo is a file path or a string:
try:
open(filePathOrStrToCompareTo, 'r').close()
filePath = filePathOrStrToCompareTo
except IOError:
# We are to compare against a string. Just
# write it to a tmp file that deletes itself:
tmpFile = tempfile.NamedTemporaryFile()
tmpFile.write(filePathOrStrToCompareTo)
tmpFile.flush()
filePath = tmpFile.name
with open(filePath, 'r') as fd:
for fileLine in fd:
expectedLine = strFile.readline()
# Some events have one or more UUIDs, which are different with each run.
# Cut those out of both expected and what we got:
uuidsRemoved = False
while not uuidsRemoved:
uuidMatch = self.pattern.search(fileLine)
if uuidMatch is None:
uuidsRemoved = True
continue
expectedLine = expectedLine[0:uuidMatch.start()] + expectedLine[uuidMatch.end():]
fileLine = fileLine[0:uuidMatch.start()] + fileLine[uuidMatch.end():]
# The timestamp in the LoadInfo table changes on each run,
# similar to UUIDs:
timestampMatch = self.timestampPattern.search(fileLine)
if timestampMatch is not None:
expectedLine = expectedLine[0:timestampMatch.start()] + expectedLine[timestampMatch.end():]
fileLine = fileLine[0:timestampMatch.start()] + fileLine[timestampMatch.end():]
# Temp file names also change from run to run.
# Replace the variable parts (ollalaxxxxx.yyy) with "foo":
fileLine = self.tmpFileNamePattern.sub("foo", fileLine)
expectedLine = self.tmpFileNamePattern.sub("foo", expectedLine)
try:
self.assertEqual(expectedLine.strip(), fileLine.strip())
except:
#print(expectedLine + '\n' + fileLine)
raise
if strFile.readline() != "":
# expected is longer than what's in the file:
self.fail("Expected string is longer than content of output file %s" % filePath)
class TestModulestoreImporter(unittest.TestCase):
# Setup method called once:
@classmethod
def setUpClass(cls):
# Move the existing, full-sized modulestore_latest.json
# aside, and replace it with a smaller test version:
# Dir of currently running test script:
currDir = os.path.dirname(__file__)
# Where the modulestore_latest.json is stored:
TestModulestoreImporter.dataDir = os.path.join(currDir, '..', 'data')
TestModulestoreImporter.currModStoreLatest = os.path.join(
TestModulestoreImporter.dataDir, 'modulestore_latest.json')
# Save current, full-size modulestore_latest.json
# (preserving its symlink):
TestModulestoreImporter.savedModStoreLatest = TestModulestoreImporter.currModStoreLatest + '.SAVED'
try:
shutil.move(TestModulestoreImporter.currModStoreLatest,
TestModulestoreImporter.savedModStoreLatest)
except IOError:
pass
# Do the replacement with the test version:
testModStoreLatest = os.path.join(currDir, 'data',
'mini_modulestore_latest.json')
shutil.copy(testModStoreLatest,
TestModulestoreImporter.currModStoreLatest)
# The hashLookup.pkl, if it exists, is moved to a save-copy:
TestModulestoreImporter.hashLookupPicklePath = os.path.join(
TestModulestoreImporter.dataDir, 'hashLookup.pkl')
TestModulestoreImporter.savedHashLookupPicklePath = TestModulestoreImporter.hashLookupPicklePath + '.SAVED'
if (os.path.exists(TestModulestoreImporter.hashLookupPicklePath)):
shutil.move(TestModulestoreImporter.hashLookupPicklePath,
TestModulestoreImporter.savedHashLookupPicklePath)
TestModulestoreImporter.csvTestOutPath = os.path.join(
TestModulestoreImporter.dataDir, 'tmpCacheCSVFile.csv')
# Teardown method called once:
@classmethod
def tearDownClass(cls):
# Put the true, full size modulestore_latest.json back
# into its place (preserving its symlink):
try:
shutil.move(TestModulestoreImporter.savedModStoreLatest,
TestModulestoreImporter.currModStoreLatest)
except IOError:
print(
'NOTE: no modulestore_latest.json found in <projRoot>/json_to_relation/data; run scripts/cronRefreshModuleStore.sh.'
)
# If hashLookup.pkl existed at the the outset, and was saved,
# restore it:
try:
shutil.move(TestModulestoreImporter.savedHashLookupPicklePath,
TestModulestoreImporter.hashLookupPicklePath)
except:
pass
# Remove the csv test output:
try:
os.remove(TestModulestoreImporter.csvTestOutPath)
except:
pass
def setUp(self):
jsonFileName = os.path.join(TestModulestoreImporter.dataDir,
'modulestore_latest.json')
self.importer = ModulestoreImporter(jsonFileName)
self.pickleCachePath = os.path.join(TestModulestoreImporter.dataDir,
'tmpTestHashLookup.pkl')
self.uuidRegex = '[a-f0-9]{8}_[a-f0-9]{4}_[a-f0-9]{4}_[a-f0-9]{4}_[a-f0-9]{12}'
self.pattern = re.compile(self.uuidRegex)
self.timestampRegex = r'[1-2][0-9]{3}-[0-1][1-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9]\.[0-9]{0,6}Z{0,1}'
self.timestampPattern = re.compile(self.timestampRegex)
# Pattern that recognizes our tmp files. They start with
# 'oolala', followed by random junk, followed by a period and
# 0 to 3 file extension letters (always 3, I believe):
self.tmpFileNamePattern = re.compile('/tmp/oolala[^.]*\..{0,3}')
def tearDown(self):
# Some of the tests create a temporary
# pickle file; remove it:
try:
os.remove(TestModulestoreImporter.hashLookupPicklePath)
except:
pass
@unittest.skipIf(not TEST_ALL, "Temporarily disabled")
def testModuleStoreImporter(self):
self.useTheDict()
@unittest.skipIf(not TEST_ALL, "Temporarily disabled")
def testExportHashInfoToCSV(self):
csvOutFile = TestModulestoreImporter.csvTestOutPath
# Write the current cache out to csv:
self.importer.exportHashInfo(csvOutFile)
csvOutExpected = os.path.join(os.path.dirname(__file__),
'data/modulestoreImporterCsvTruth.csv')
self.assertFileContentEquals(csvOutExpected, csvOutFile)
# Check whether dict ops still work:
self.useTheDict()
def useTheDict(self):
self.assertEqual(
'Module One video',
self.importer.getDisplayName('c89417444f4443f9a34039be3054962e'))
self.assertEqual(
'Medicine',
self.importer.getOrg('c89417444f4443f9a34039be3054962e'))
self.assertEqual(
'HRP258',
self.importer.getCourseShortName(
'c89417444f4443f9a34039be3054962e'))
self.assertEqual(
'video',
self.importer.getCategory('c89417444f4443f9a34039be3054962e'))
self.assertIsNone(
self.importer.getRevision('c89417444f4443f9a34039be3054962e'))
#--------------------------------------------------------------------------------------------------
def assertFileContentEquals(self, expected, filePathOrStrToCompareTo):
'''
Compares two file or string contents. First arg is either an open file or a string.
That is the ground truth to compare to. Second argument is the same: file or string.
@param expected: the file that contains the ground truth
@type expected: {File | String }
@param filePathOrStrToCompareTo: the actual file as constructed by the module being tested
@type filePathOrStrToCompareTo: { File | String }
'''
# Ensure that 'expected' is a File-like object:
if isinstance(expected, file):
# It's a file, just use it:
strFile = expected
else:
# Turn into a string 'file':
with open(expected, 'r') as fd:
strFile = StringIO.StringIO(fd.read())
# Check whether filePathOrStrToCompareTo is a file path or a string:
try:
open(filePathOrStrToCompareTo, 'r').close()
filePath = filePathOrStrToCompareTo
except IOError:
# We are to compare against a string. Just
# write it to a tmp file that deletes itself:
tmpFile = tempfile.NamedTemporaryFile()
tmpFile.write(filePathOrStrToCompareTo)
tmpFile.flush()
filePath = tmpFile.name
with open(filePath, 'r') as fd:
for fileLine in fd:
expectedLine = strFile.readline()
# Some events have one or more UUIDs, which are different with each run.
# Cut those out of both expected and what we got:
uuidsRemoved = False
while not uuidsRemoved:
uuidMatch = self.pattern.search(fileLine)
if uuidMatch is None:
uuidsRemoved = True
continue
expectedLine = expectedLine[0:uuidMatch.start(
)] + expectedLine[uuidMatch.end():]
fileLine = fileLine[0:uuidMatch.start(
)] + fileLine[uuidMatch.end():]
# The timestamp in the LoadInfo table changes on each run,
# similar to UUIDs:
timestampMatch = self.timestampPattern.search(fileLine)
if timestampMatch is not None:
expectedLine = expectedLine[0:timestampMatch.start(
)] + expectedLine[timestampMatch.end():]
fileLine = fileLine[0:timestampMatch.start(
)] + fileLine[timestampMatch.end():]
# Temp file names also change from run to run.
# Replace the variable parts (ollalaxxxxx.yyy) with "foo":
fileLine = self.tmpFileNamePattern.sub("foo", fileLine)
expectedLine = self.tmpFileNamePattern.sub("foo", expectedLine)
try:
self.assertEqual(expectedLine.strip(), fileLine.strip())
except:
#print(expectedLine + '\n' + fileLine)
raise
if strFile.readline() != "":
# expected is longer than what's in the file:
self.fail(
"Expected string is longer than content of output file %s"
% filePath)