def test_gitlog_pagination_out_of_range_invalid(self):
"""
Make sure the pagination behaves properly when the requested page is out
of range.
"""
self._setstaff_login()
mongoengine.connect(TEST_MONGODB_LOG['db'])
for _ in range(15):
CourseImportLog(
course_id=CourseLocator.from_string("test/test/test"),
location="location",
import_log="import_log",
git_log="git_log",
repo_dir="repo_dir",
created=datetime.now()).save()
for page, expected in [(-1, 1), (1, 1), (2, 2), (30, 2), ('abc', 1)]:
response = self.client.get('{}?page={}'.format(
reverse('gitlogs'), page))
self.assertContains(response, u'Page {} of 2'.format(expected))
CourseImportLog.objects.delete()
def add_repo(repo, rdir_in, branch=None):
"""
This will add a git repo into the mongo modulestore.
If branch is left as None, it will fetch the most recent
version of the current branch.
"""
# pylint: disable=too-many-statements
git_repo_dir = getattr(settings, 'GIT_REPO_DIR', DEFAULT_GIT_REPO_DIR)
git_import_static = getattr(settings, 'GIT_IMPORT_STATIC', True)
git_import_python_lib = getattr(settings, 'GIT_IMPORT_PYTHON_LIB', True)
python_lib_filename = getattr(settings, 'PYTHON_LIB_FILENAME',
DEFAULT_PYTHON_LIB_FILENAME)
# Set defaults even if it isn't defined in settings
mongo_db = {
'host': 'localhost',
'port': 27017,
'user': '',
'password': '',
'db': 'xlog',
}
# Allow overrides
if hasattr(settings, 'MONGODB_LOG'):
for config_item in ['host', 'user', 'password', 'db', 'port']:
mongo_db[config_item] = settings.MONGODB_LOG.get(
config_item, mongo_db[config_item])
if not os.path.isdir(git_repo_dir):
raise GitImportErrorNoDir(git_repo_dir)
# pull from git
if not (repo.endswith('.git') or repo.startswith(
('http:', 'https:', 'git:', 'file:'))):
raise GitImportErrorUrlBad()
if rdir_in:
rdir = os.path.basename(rdir_in)
else:
rdir = repo.rsplit('/', 1)[-1].rsplit('.git', 1)[0]
log.debug('rdir = %s', rdir)
rdirp = f'{git_repo_dir}/{rdir}'
if os.path.exists(rdirp):
log.info('directory already exists, doing a git pull instead '
'of git clone')
cmd = [
'git',
'pull',
]
cwd = rdirp
else:
cmd = [
'git',
'clone',
repo,
]
cwd = git_repo_dir
cwd = os.path.abspath(cwd)
try:
ret_git = cmd_log(cmd, cwd=cwd)
except subprocess.CalledProcessError as ex:
log.exception('Error running git pull: %r', ex.output)
raise GitImportErrorCannotPull() # lint-amnesty, pylint: disable=raise-missing-from
if branch:
switch_branch(branch, rdirp)
# get commit id
cmd = [
'git',
'log',
'-1',
'--format=%H',
]
try:
commit_id = cmd_log(cmd, cwd=rdirp)
except subprocess.CalledProcessError as ex:
log.exception('Unable to get git log: %r', ex.output)
raise GitImportErrorBadRepo() # lint-amnesty, pylint: disable=raise-missing-from
ret_git += f'\nCommit ID: {commit_id}'
# get branch
cmd = [
'git',
'symbolic-ref',
'--short',
'HEAD',
]
try:
branch = cmd_log(cmd, cwd=rdirp)
except subprocess.CalledProcessError as ex:
# I can't discover a way to excercise this, but git is complex
# so still logging and raising here in case.
log.exception('Unable to determine branch: %r', ex.output)
raise GitImportErrorBadRepo() # lint-amnesty, pylint: disable=raise-missing-from
ret_git += '{}Branch: {}'.format(' \n', branch)
# Get XML logging logger and capture debug to parse results
output = StringIO()
import_log_handler = logging.StreamHandler(output)
import_log_handler.setLevel(logging.DEBUG)
logger_names = [
'xmodule.modulestore.xml_importer',
'git_add_course',
'xmodule.modulestore.xml',
'xmodule.seq_module',
]
loggers = []
for logger_name in logger_names:
logger = logging.getLogger(logger_name)
logger.setLevel(logging.DEBUG)
logger.addHandler(import_log_handler)
loggers.append(logger)
try:
management.call_command('import',
git_repo_dir,
rdir,
nostatic=not git_import_static,
nopythonlib=not git_import_python_lib,
python_lib_filename=python_lib_filename)
except CommandError:
raise GitImportErrorXmlImportFailed() # lint-amnesty, pylint: disable=raise-missing-from
except NotImplementedError:
raise GitImportErrorUnsupportedStore() # lint-amnesty, pylint: disable=raise-missing-from
ret_import = output.getvalue()
# Remove handler hijacks
for logger in loggers:
logger.setLevel(logging.NOTSET)
logger.removeHandler(import_log_handler)
course_key = None
location = 'unknown'
# extract course ID from output of import-command-run and make symlink
# this is needed in order for custom course scripts to work
match = re.search(r'(?ms)===> IMPORTING courselike (\S+)', ret_import)
if match:
course_id = match.group(1).split('/')
# we need to transform course key extracted from logs into CourseLocator instance, because
# we are using split module store and course keys store as instance of CourseLocator.
# please see common.lib.xmodule.xmodule.modulestore.split_mongo.split.SplitMongoModuleStore#make_course_key
# We want set course id in CourseImportLog as CourseLocator. So that in split module
# environment course id remain consistent as CourseLocator instance.
course_key = CourseLocator(*course_id)
cdir = f'{git_repo_dir}/{course_key.course}'
log.debug('Studio course dir = %s', cdir)
if os.path.exists(cdir) and not os.path.islink(cdir):
log.debug(' -> exists, but is not symlink')
log.debug(
subprocess.check_output([
'ls',
'-l',
],
cwd=os.path.abspath(cdir)))
try:
os.rmdir(os.path.abspath(cdir))
except OSError:
log.exception('Failed to remove course directory')
if not os.path.exists(cdir):
log.debug(' -> creating symlink between %s and %s', rdirp, cdir)
try:
os.symlink(os.path.abspath(rdirp), os.path.abspath(cdir))
except OSError:
log.exception('Unable to create course symlink')
log.debug(
subprocess.check_output([
'ls',
'-l',
],
cwd=os.path.abspath(cdir)))
# store import-command-run output in mongo
mongouri = 'mongodb://{user}:{password}@{host}:{port}/{db}'.format(
**mongo_db)
try:
if mongo_db['user'] and mongo_db['password']:
mdb = mongoengine.connect(mongo_db['db'], host=mongouri)
else:
mdb = mongoengine.connect(mongo_db['db'],
host=mongo_db['host'],
port=mongo_db['port'])
except mongoengine.connection.ConnectionFailure:
log.exception('Unable to connect to mongodb to save log, please '
'check MONGODB_LOG settings')
cil = CourseImportLog(
course_id=course_key,
location=location,
repo_dir=rdir,
created=timezone.now(),
import_log=ret_import,
git_log=ret_git,
)
cil.save()
log.debug('saved CourseImportLog for %s', cil.course_id)
mdb.close()