def test_regenerate_inplace(self):
# take a generated example where a base layer has changed
# regenerate in place
# make some assertions
bu = build.Builder()
bu.log_level = "WARNING"
bu.output_dir = "out"
bu.series = "trusty"
bu.name = "foo"
bu.charm = "trusty/b"
bu.hide_metrics = True
bu()
base = path("out/trusty/foo")
self.assertTrue(base.exists())
# verify the 1st gen worked
self.assertTrue((base / "a").exists())
self.assertTrue((base / "README.md").exists())
# now regenerate from the target
with utils.cd("out/trusty/foo"):
bu = build.Builder()
bu.log_level = "WARNING"
bu.output_dir = path(os.getcwd())
bu.series = "trusty"
# The generate target and source are now the same
bu.name = "foo"
bu.charm = "."
bu.hide_metrics = True
bu()
base = bu.output_dir
self.assertTrue(base.exists())
# Check that the generated layer.yaml makes sense
cy = base / "layer.yaml"
config = yaml.load(cy.open())
self.assertEquals(config["includes"], ["trusty/a", "interface:mysql"])
self.assertEquals(config["is"], "foo")
# We can even run it more than once
bu()
cy = base / "layer.yaml"
config = yaml.load(cy.open())
self.assertEquals(config["includes"], ["trusty/a", "interface:mysql"])
self.assertEquals(config["is"], "foo")
# We included an interface, we should be able to assert things about it
# in its final form as well
provides = base / "hooks/relations/mysql/provides.py"
requires = base / "hooks/relations/mysql/requires.py"
self.assertTrue(provides.exists())
self.assertTrue(requires.exists())
# and that we generated the hooks themselves
for kind in ["joined", "changed", "broken", "departed"]:
self.assertTrue((base / "hooks" / "mysql-relation-{}".format(kind)).exists())
# and ensure we have an init file (the interface doesn't its added)
init = base / "hooks/relations/mysql/__init__.py"
self.assertTrue(init.exists())
def test_merge(self):
testdir = path(l2emod.__file__).parent / 'testtex'
with make_temp_directory() as tmdir:
fn = testdir / 'example1.tex'
print "file %s" % fn
nfn = '%s/%s' % (tmdir, fn.basename())
os.system('cp %s/* %s' % (testdir, tmdir))
os.chdir(tmdir)
l2e = latex2edx(nfn, output_dir=tmdir)
l2e.convert()
fn = testdir / 'example2.tex'
print "file %s" % fn
nfn = '%s/%s' % (tmdir, fn.basename())
l2e = latex2edx(nfn, output_dir=tmdir, do_merge=True)
l2e.convert()
cfn = path(tmdir) / 'course/2013_Fall.xml'
self.assertTrue(os.path.exists(cfn))
self.assertIn('<chapter url_name="Unit_1"', open(cfn).read())
self.assertIn('<chapter url_name="Unit_2"', open(cfn).read())
cfn = path(tmdir) / 'chapter/Unit_1.xml'
self.assertTrue(os.path.exists(cfn))
cfn = path(tmdir) / 'chapter/Unit_2.xml'
self.assertTrue(os.path.exists(cfn))
def get_problem(base_dir, problem_id, goproblems_url, save_html=False):
"""Get the given problem.
:param str base_dir: the base dir where the problem is to be saved
:param int problem_id: the id of the problem to be downloaded
:param str goproblems_url: the url where the problems can be found
:param boolean save_html: save the downloaded HTML
"""
if save_html:
html_dir = check_directory(path(base_dir) / 'html')
html_file = html_dir / ('%d.html' % problem_id)
if html_file.exists():
print "already downloaded %d" % problem_id
return False
response = requests.get(goproblems_url % problem_id)
if save_html:
with open(html_file, 'w') as f:
f.write(response.text)
if not response.ok:
raise requests.HTTPError(response.text)
problem, difficulty, sgf, rating = parse_problem(response.text)
problem_dir = check_directory(path(base_dir) / difficulty.replace(' ', '_'))
problem_name = '%s_%s.sgf' % (difficulty, (problem or str(problem_id)))
if rating:
problem_name = '[%s]' % rating + problem_name
with open(problem_dir / problem_name, 'w') as f:
f.write(sgf)
return True
def __init__(self, *location_parts, **k):
self.httpopen = k.pop('httpopen', asynchttp.httpopen)
location_parts = list(location_parts)
first = location_parts.pop(0)
while len(location_parts) > 0:
next = location_parts.pop(0)
if next.startswith("http:"):
first = next
elif first.startswith("http:"):
first = net.httpjoin(first, next)
else:
# path will handle relative vs. absolute
first = path.path(first)
if first.ext:
first = first.parent
first = first / next
first = first.normpath()
self.web = first.startswith("http:")
self.content_location = path.path(first)
self.content = None
self._load_callback = None
self.meta = None
def test_example1(self):
testdir = path(l2emod.__file__).parent / 'testtex'
fn = testdir / 'example1.tex'
print "file %s" % fn
with make_temp_directory() as tmdir:
nfn = '%s/%s' % (tmdir, fn.basename())
os.system('cp %s/* %s' % (testdir, tmdir))
os.chdir(tmdir)
l2e = latex2edx(nfn, output_dir=tmdir)
l2e.convert()
xbfn = nfn[:-4] + '.xbundle'
self.assertTrue(os.path.exists(xbfn))
# xb = open(xbfn).read()
# self.assertIn('<chapter display_name="Unit 1" start="2013-11-22" url_name="Unit_1">', xb)
xml = etree.parse(xbfn).getroot()
chapter = xml.find('.//chapter')
self.assertTrue(chapter.get('display_name') == 'Unit 1')
self.assertTrue(chapter.get('start') == '2013-11-22')
self.assertTrue(chapter.get('url_name') == 'Unit_1')
cfn = path(tmdir) / 'course/2013_Fall.xml'
self.assertTrue(os.path.exists(cfn))
cfn = path(tmdir) / 'chapter/Unit_1.xml'
self.assertTrue(os.path.exists(cfn))
# self.assertIn('<sequential display_name="Introduction" due="2013-11-22" url_name="Introduction"', open(cfn).read())
xml = etree.parse(cfn).getroot()
seq = xml.find('.//sequential')
self.assertTrue(seq.get('display_name') == 'Introduction')
self.assertTrue(seq.get('due') == '2013-11-22')
self.assertTrue(seq.get('url_name') == 'Introduction')
self.assertIn('<problem url_name="p1"/>', open(cfn).read())
def get_course_syllabus_section(course, section_key):
"""
This returns the snippet of html to be rendered on the syllabus page,
given the key for the section.
Valid keys:
- syllabus
- guest_syllabus
"""
# Many of these are stored as html files instead of some semantic
# markup. This can change without effecting this interface when we find a
# good format for defining so many snippets of text/html.
if section_key in ['syllabus', 'guest_syllabus']:
try:
filesys = course.system.resources_fs
# first look for a run-specific version
dirs = [path("syllabus") / course.url_name, path("syllabus")]
filepath = find_file(filesys, dirs, section_key + ".html")
with filesys.open(filepath) as html_file:
return replace_static_urls(
html_file.read().decode('utf-8'),
getattr(course, 'data_dir', None),
course_id=course.id,
static_asset_path=course.static_asset_path,
)
except ResourceNotFoundError:
log.exception(
u"Missing syllabus section %s in course %s",
section_key, course.location.to_deprecated_string()
)
return "! Syllabus missing !"
raise KeyError("Invalid about key " + str(section_key))
def setUp(self):
super(ImportTestCase, self).setUp()
self.url = reverse_course_url('import_handler', self.course.id)
self.content_dir = path(tempfile.mkdtemp())
def touch(name):
""" Equivalent to shell's 'touch'"""
with file(name, 'a'):
os.utime(name, None)
# Create tar test files -----------------------------------------------
# OK course:
good_dir = tempfile.mkdtemp(dir=self.content_dir)
os.makedirs(os.path.join(good_dir, "course"))
with open(os.path.join(good_dir, "course.xml"), "w+") as f:
f.write('<course url_name="2013_Spring" org="EDx" course="0.00x"/>')
with open(os.path.join(good_dir, "course", "2013_Spring.xml"), "w+") as f:
f.write('<course></course>')
self.good_tar = os.path.join(self.content_dir, "good.tar.gz")
with tarfile.open(self.good_tar, "w:gz") as gtar:
gtar.add(good_dir)
# Bad course (no 'course.xml' file):
bad_dir = tempfile.mkdtemp(dir=self.content_dir)
touch(os.path.join(bad_dir, "bad.xml"))
self.bad_tar = os.path.join(self.content_dir, "bad.tar.gz")
with tarfile.open(self.bad_tar, "w:gz") as btar:
btar.add(bad_dir)
self.unsafe_common_dir = path(tempfile.mkdtemp(dir=self.content_dir))
def fetch(self):
try:
fetcher = get_fetcher(self.url)
except FetchError:
# We might be passing a local dir path directly
# which fetchers don't currently support
self.directory = path(self.url)
else:
if hasattr(fetcher, "path") and fetcher.path.exists():
self.directory = path(fetcher.path)
else:
if not self.target_repo.exists():
self.target_repo.makedirs_p()
self.directory = path(fetcher.fetch(self.target_repo))
if not self.directory.exists():
raise OSError(
"Unable to locate {}. "
"Do you need to set {}?".format(
self.url, self.ENVIRON))
self.config_file = self.directory / self.CONFIG_FILE
if not self.config_file.exists():
if self.OLD_CONFIG and (self.directory / self.OLD_CONFIG).exists():
self.config_file = (self.directory / self.OLD_CONFIG)
self._name = self.config.name
return self
def render_webadmin_config(service_name,
source=path("/opt/admin-ui/config/default.yml"),
dest=path('/etc/cf-webadmin.yml')):
secretctx = contexts.StoredContext(utils.secrets_file, {})
orchctx = contexts.OrchestratorRelation()
dbctx = contexts.MysqlRelation()
ccdbctx = contexts.CloudControllerDBRelation()
natsctx = contexts.NatsRelation()
uaadbctx = None # XXX
template = yaml.safe_load(source.text())
domain = orchctx[orchctx.name][0]['domain']
ui_secret = secretctx['ui_secret']
ccdb_uri = ccdbctx[ccdbctx.name][0]['dsn'] # ccdbctx.get('dsn', unit=0) would be nice
db_uri = dbctx[dbctx.name][0]['dsn']
nats_uri = 'nats://{user}:{password}@{address}:{port}'.format(**natsctx[natsctx.name][0])
uaadb_uri = '' # XXX
template['uaa_client']['secret'] = ui_secret
template['cloud_controller_uri'] = 'http://api.%s' % domain
template['ccdb_uri'] = ccdb_uri
template['db_uri'] = db_uri
template['uaadb_uri'] = uaadb_uri
template['mbus'] = nats_uri
dest.write_text(yaml.safe_dump(template))
def _copy_manager_blueprint(self):
inputs_path, mb_path = util.generate_unique_configurations(
workdir=self.workdir,
original_inputs_path=self.env.cloudify_config_path,
original_manager_blueprint_path=self.env._manager_blueprint_path)
self.test_manager_blueprint_path = path(mb_path)
self.test_inputs_path = path(inputs_path)
def __init__(self, dir, cfg=CFG, mem=MEM, sym=SYM, skipweb=SKIPWEB,
registry=REGISTRY, tcp=TCP, ssl=SSL,
savevars=SAVEVARS, savevarsfile=SAVEVARSFILE):
print "%s: Upgrading %s (%s)..." % (self.__class__.__name__, dir, sym)
self.mem = mem
self.sym = sym
self.skipweb = skipweb
self.registry = registry
self.tcp = tcp
self.ssl = ssl
# setup_script_environment() may cause the creation of a default
# config.xml, so we must check for it here
noconfigure = self.has_config(dir)
cli = self.setup_script_environment(dir)
bin = self.setup_previous_omero_env(sym, savevarsfile)
# Need lib/python set above
import path
self.cfg = path.path(cfg)
self.dir = path.path(dir)
self.stop(bin)
self.configure(cli, noconfigure)
self.directories(cli)
self.save_env_vars(savevarsfile, savevars.split())
self.start(cli)
def on_step_run(self):
# `get_step_options` is provided by the `StepOptionsController` mixin.
options = self.get_step_options()
if options.run_pstrace and options.script:
app_values = self.get_app_values()
exe_path = path(app_values['pstrace_exe'])
script = path(options.script)
if not exe_path.isfile():
logger.error('[PSTraceLauncher] invalid exe-path: %s' %
exe_path.abspath())
elif not script.isfile():
logger.error('[PSTraceLauncher] invalid script-path: %s' %
script.abspath())
elif os.name != 'nt':
logger.error('[PSTraceLauncher] This plugin is only supported '
'on Windows')
else:
pstrace_processes = [p for p in psutil.process_iter() if
safe_psutil_attr(p, 'exe') ==
exe_path.abspath()]
if not pstrace_processes:
if options.delay_ms > 0:
logger.info('[PSTraceLauncher] delay: %s ms',
options.delay_ms)
gtk.timeout_add(options.delay_ms, self._execute,
exe_path, script)
else:
self._execute(exe_path, script)
else:
logger.info('[PSTraceLauncher] skipping, since PSTrace is '
'already running as process %s',
[p.pid for p in pstrace_processes])
self.complete_step()
def person_detail(person_id):
ctx = dal.get_person(person_id, missing=NotFound)
ctx.update(dal.get_mandate2012_details(person_id))
if 'picture_filename' in ctx:
picture_rel_path = path('mandate-pictures') / ctx['picture_filename']
if (path(flask.current_app.static_folder) / picture_rel_path).isfile():
ctx['picture_url'] = flask.url_for(
'static',
filename=picture_rel_path,
)
for item in ctx['recent_activity']:
if item['type'] == 'proposal':
item['url'] = flask.url_for(
'.policy_proposal',
proposal_id=item['proposal_id'],
)
elif item['type'] == 'question':
item['url'] = flask.url_for(
'.person_question',
question_id=item['question_id'],
)
return flask.render_template('person_detail.html', **ctx)
def from_upload(cls, upload, addon, platforms, send_signal=True):
data = utils.parse_addon(upload, addon)
try:
license = addon.versions.latest().license_id
except Version.DoesNotExist:
license = None
v = cls.objects.create(addon=addon, version=data['version'],
license_id=license)
log.info('New version: %r (%s) from %r' % (v, v.id, upload))
# appversions
AV = ApplicationsVersions
for app in data.get('apps', []):
AV(version=v, min=app.min, max=app.max,
application_id=app.id).save()
if addon.type == amo.ADDON_SEARCH:
# Search extensions are always for all platforms.
platforms = [Platform.objects.get(id=amo.PLATFORM_ALL.id)]
else:
platforms = cls._make_safe_platform_files(platforms)
for platform in platforms:
File.from_upload(upload, v, platform, parse_data=data)
v.disable_old_files()
# After the upload has been copied to all
# platforms, remove the upload.
path.path(upload.path).unlink()
if send_signal:
version_uploaded.send(sender=v)
return v
def __init__(self, filename):
"""
Constructor for the File class. A File is an abstraction of the file itself along with all of the version
control metadata that it uses.
:param filename: The name of the file on disk in the directory that the command was executed.
"""
# Set up the member variables
self.filename = filename
self.current_file = path(filename)
self.datafile = path('.vcs/' + filename)
self.branches = {'main': []}
self.current_branch = 'main' # Default branch name
if not self.current_file.exists():
#File doesn't exist.
exit_with_message("File does <%s> not exist." % self.filename)
# Some operations are allowed if not in VCS (add).
self.in_vcs = self.datafile.exists()
if self.in_vcs:
#Load file history and branch information.
data = self.datafile.bytes()
parsed_data = json.loads(data)
self.branches = parsed_data['branches']
self.current_branch = parsed_data['current_branch']
if self.branches.get(self.current_branch) is None:
# "Current Branch" not in Branches list.
exit_with_message("There was a problem with your datafile.")
def test_spool_no_time_no_user(self):
"""Ensure `spool` works when no `time` attribute is supplied, and no
`user` attribute exists.
"""
c = CallFile(self.call, self.action, spool_dir=self.spool_dir)
c.spool()
ok_((path(c.spool_dir) / path(c.filename)).abspath().exists())
def _getFileStats(self, pathString):
"""
Private method to get stats for a file
:Parameters:
fileId : string
A string uniquely identifying a file on this Monitor.
:return: stats
:rtype: monitors.FileStats
"""
stats = monitors.FileStats()
stats.baseName = pathModule.path(pathString).name
stats.owner = pathModule.path(pathString).owner
stats.size = pathModule.path(pathString).size
stats.mTime = pathModule.path(pathString).mtime
stats.cTime = pathModule.path(pathString).ctime
stats.aTime = pathModule.path(pathString).atime
if pathModule.path(pathString).isfile():
stats.type = monitors.FileType.File
elif pathModule.path(pathString).isdir():
stats.type = monitors.FileType.Dir
elif pathModule.path(pathString).islink():
stats.type = monitors.FileType.Link
elif pathModule.path(pathString).ismount():
stats.type = monitors.FileType.Mount
else:
stats.type = monitors.FileType.Unknown
return stats
def test_crop_view(self):
"""avatar gets cropped"""
with open('%s/%s' % (settings.MEDIA_ROOT, 'misago.png')) as avatar:
handler_link = reverse('misago:usercp_upload_avatar_handler')
ajax_header = {'HTTP_X_REQUESTED_WITH': 'XMLHttpRequest'}
response = self.client.post(handler_link,
data={'new-avatar': avatar},
**ajax_header)
self.assertEqual(response.status_code, 200)
crop_link = reverse('misago:usercp_crop_new_avatar')
response = self.client.post(crop_link, data={'crop': '1245'})
self.assertEqual(response.status_code, 200)
test_crop = '619,201,150,150,0,0,150,150'
response = self.client.post(crop_link, data={'crop': test_crop})
self.assertEqual(response.status_code, 302)
avatar_dir = store.get_existing_avatars_dir(self.user)
avatar = path('%s/%s_tmp.png' % (avatar_dir, self.user.pk))
self.assertFalse(avatar.exists())
avatar = path('%s/%s_org.png' % (avatar_dir, self.user.pk))
self.assertTrue(avatar.exists())
self.assertTrue(avatar.isfile())
def test_edXmath1(self):
testdir = path(l2emod.__file__).parent / 'testtex'
fn = testdir / 'example6.tex'
print "file %s" % fn
with make_temp_directory() as tmdir:
nfn = '%s/%s' % (tmdir, fn.basename())
os.system('cp %s/* %s' % (testdir, tmdir))
os.mkdir('%s/dnd' % tmdir)
os.system('cp %s/quadratic.tex %s/dnd/' % (testdir, tmdir))
os.chdir(tmdir)
l2e = latex2edx(nfn, output_dir=tmdir)
l2e.convert()
xbfn = nfn[:-4]+'.xbundle'
self.assertTrue(os.path.exists(xbfn))
xb = open(xbfn).read()
cfn = path(tmdir) / 'problem/p1.xml'
assert(os.path.exists(cfn))
os.system('ls -sFC %s' % tmdir)
os.system('ls -sFC %s/problem' % tmdir)
self.assertTrue(os.path.exists(cfn))
data = open(cfn).read()
expect = r"""[mathjax]\begin{eqnarray}
S(\rho) &=& -\lambda_{1} \log \lambda_{1} -\lambda_{2} \log \lambda_{2} \\
&=& H((1+r)/2)
\end{eqnarray}[/mathjax]"""
assert(expect in data)
def test_theme_outside_repo(self):
# Need to create a temporary theme, and defer decorating the function
# until it is done, which leads to this strange nested-function style
# of test.
# Make a temp directory as a theme.
themes_dir = path(mkdtemp_clean())
tmp_theme = "temp_theme"
template_dir = themes_dir / tmp_theme / "lms/templates"
template_dir.makedirs()
with open(template_dir / "footer.html", "w") as footer:
footer.write("<footer>TEMPORARY THEME</footer>")
dest_path = path(settings.COMPREHENSIVE_THEME_DIR) / tmp_theme
create_symlink(themes_dir / tmp_theme, dest_path)
@with_comprehensive_theme(tmp_theme)
def do_the_test(self):
"""A function to do the work so we can use the decorator."""
resp = self.client.get('/')
self.assertEqual(resp.status_code, 200)
self.assertContains(resp, "TEMPORARY THEME")
do_the_test(self)
# remove symlinks before running subsequent tests
delete_symlink(dest_path)
def on_step_swapped(self, original_step_number, new_step_number):
if not self.get_step_value('grab_frame'):
print '[VideoFrameGrabber] on_step_swapped():'\
'frame grab disabled for step %d' % (new_step_number)
return
if self.get_app_value('frame_output_dir'):
output_dir = path(self.get_app_value('frame_output_dir'))
else:
output_dir = path('.')
filename = '%d-%d.jpg' % (original_step_number, new_step_number)
filename = output_dir.joinpath(filename)
print '[VideoFrameGrabber] on_step_swapped():'\
'save image of %d -> %d to %s' % (original_step_number,
new_step_number, filename)
# Grab frame from dmf_device_controller plugin
service = get_service_instance_by_name(
'microdrop.gui.dmf_device_controller', env='microdrop')
frame = service.grab_frame()
if frame:
# Save frame to file
cv.SaveImage(filename, frame)
def edit_path(path_or_obj, start_text):
f = path.path(path_or_obj)
editor = os.getenv("VISUAL") or os.getenv("EDITOR")
if not editor:
if platform.system() == "Windows":
editor = "Notepad.exe"
else:
editor = "vi"
f.write_text(start_text)
# If absolute, then use the path
# as is (ticket:4246). Otherwise,
# use which.py to find it.
editor_obj = path.path(editor)
if editor_obj.isabs():
editor_path = editor
else:
from omero_ext.which import which
editor_path = which(editor)
pid = os.spawnl(os.P_WAIT, editor_path, editor_path, f)
if pid:
re = RuntimeError("Couldn't spawn editor: %s" % editor)
re.pid = pid
raise re
def read_file(asg, spelling, decl):
ast = decl.get_ast_context()
sm = ast.get_source_manager()
loc = decl.get_location()
fid = sm.get_file_id(loc)
if not fid.is_invalid():
filename = sm.get_filename(loc).str()
if filename:
filename = str(path(filename).abspath())
filenode = asg.add_file(filename, proxy=HeaderProxy)
filenode.language = asg._language
asg._nodes[spelling]['_header'] = filenode.globalname
while True:
loc = sm.get_include_loc(fid)
fid = sm.get_file_id(loc)
if not fid.is_invalid():
includename = sm.get_filename(loc).str()
if includename:
includename = str(path(includename).abspath())
filenode = asg.add_file(includename, proxy=HeaderProxy)
asg._include_edges[filename] = filenode._node
filename = includename
else:
break
else:
break
def load(find_functions, search_path=None):
"""
Load the modules in the search_path.
If search_path is None, then load modules in the same folder as the function looking for them.
"""
caller_module = inspect.getmodule(inspect.stack()[1][0])
system_files = [caller_module.__file__]
module_path = path(caller_module.__file__).abspath().dirname()
sys_files = set()
for f in system_files:
if f.endswith(".pyo") or f.endswith(".pyc"):
f = f[:-3] + "py"
sys_files.add(path(f).abspath())
if search_path is None:
search_path = module_path
else:
search_path = path(search_path).abspath()
fcts = {}
# Search for python, cython and modules
for f in (search_path.files("*.py") +
search_path.files("*.pyx") +
search_path.files(sys_modules)):
if f not in sys_files:
module_name = f.namebase
pack_name = '%s.%s_%s' % (caller_module.__name__,
bad_chars.sub('_', module_path),
module_name)
try:
mod_desc = imp.find_module(module_name, [search_path])
mod = imp.load_module(pack_name, *mod_desc)
fcts.update(find_functions(mod))
except ImportError as ex:
print("Warning, cannot import module '{0}' from {1}: {2}"
.format(module_name, caller_module.__name__, ex), file=sys.stderr)
return fcts
def factory(reserve_only=False):
temp_dir = mkdtemp()
if reserve_only:
# It'll still be cleaned at the end
path(temp_dir).rmtree(ignore_errors=True)
created.add(temp_dir)
return path(temp_dir)
def default_avatar(self, db_settings):
if db_settings['default_avatar'] == 'gallery':
try:
avatars_list = []
try:
# First try, _default path
galleries = path(settings.STATICFILES_DIRS[0]).joinpath('avatars').joinpath('_default')
avatars_list += galleries.files('*.gif')
avatars_list += galleries.files('*.jpg')
avatars_list += galleries.files('*.jpeg')
avatars_list += galleries.files('*.png')
except Exception as e:
pass
# Second try, all paths
if not avatars_list:
avatars_list = []
for directory in path(settings.STATICFILES_DIRS[0]).joinpath('avatars').dirs():
if not directory[-7:] == '_locked' and not directory[-7:] == '_thumbs':
avatars_list += directory.files('*.gif')
avatars_list += directory.files('*.jpg')
avatars_list += directory.files('*.jpeg')
avatars_list += directory.files('*.png')
if avatars_list:
# Pick random avatar from list
self.avatar_type = 'gallery'
self.avatar_image = '/'.join(path(choice(avatars_list)).splitall()[-2:])
return True
except Exception as e:
pass
self.avatar_type = 'gravatar'
self.avatar_image = None
return True
def setup_method(self, method):
# Non-temp directories
build_dir = path() / "build"
top_dir = path() / ".." / ".." / ".."
etc_dir = top_dir / "etc"
# Necessary fiels
prefs_file = build_dir / "prefs.class"
internal_cfg = etc_dir / "internal.cfg"
master_cfg = etc_dir / "master.cfg"
# Temp directories
tmp_dir = create_path(folder=True)
tmp_etc_dir = tmp_dir / "etc"
tmp_grid_dir = tmp_etc_dir / "grid"
tmp_lib_dir = tmp_dir / "lib"
tmp_var_dir = tmp_dir / "var"
# Setup tmp dir
[x.makedirs() for x in (tmp_grid_dir, tmp_lib_dir, tmp_var_dir)]
prefs_file.copy(tmp_lib_dir)
master_cfg.copy(tmp_etc_dir)
internal_cfg.copy(tmp_etc_dir)
# Other setup
self.cli = MockCLI()
self.cli.dir = tmp_dir
self.cli.register("admin", AdminControl, "TEST")
self.cli.register("config", PrefsControl, "TEST")
def pypi():
module_name, dry_run, _ = config.common_arguments()
tmp_dir = make_virtualenv()
install_cmd = '%s/bin/pip install %s' % (tmp_dir, module_name)
package_index = 'pypi'
pypi = config.arguments['--pypi']
if pypi:
install_cmd += '-i %s' % pypi
package_index = pypi
try:
result = shell.execute(install_cmd, dry_run=dry_run)
if result:
log.info('Successfully installed %s from %s',
module_name, package_index)
else:
log.error('Failed to install %s from %s',
module_name, package_index)
verification.run_test_command()
except:
log.exception(
'error installing %s from %s', module_name, package_index
)
raise Exception(
'Error installing %s from %s', module_name, package_index
)
path(tmp_dir).rmtree(path(tmp_dir))
def testOldTemplates(self):
old_templates = path(__file__).dirname() / ".." / "old_templates.xml"
old_templates.copy(
path(self.cli.dir) / "etc" / "templates" / "grid" /
"templates.xml")
with pytest.raises(NonZeroReturnCode):
self.cli.invoke(self.args, strict=True)
def __init__(self, call, action, archive=None, filename=None, tempdir=None, user=None, spool_dir=None): """Create a new `CallFile` obeject. :param obj call: A `pycall.Call` instance. :param obj action: Either a `pycall.actions.Application` instance or a `pycall.actions.Context` instance. :param bool archive: Should Asterisk archive the call file? :param str filename: Filename of the call file. :param str tempdir: Temporary directory to store the call file before spooling. :param str user: Username to spool the call file as. :param str spool_dir: Directory to spool the call file to. :rtype: `CallFile` object. """ self.call = call self.action = action self.archive = archive self.user = user self.spool_dir = spool_dir or self.DEFAULT_SPOOL_DIR if filename and tempdir: self.filename = path(filename) self.tempdir = path(tempdir) else: f = path(mkstemp(suffix='.call')[1]) self.filename = f.name self.tempdir = f.parent
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
_system = 'lms'
_report_dir = REPO_ROOT / 'reports' / _system
_report_dir.makedirs_p()
NOSE_ARGS = [
'--id-file',
REPO_ROOT / '.testids' / _system / 'noseids',
'--xunit-file',
_report_dir / 'nosetests.xml',
]
# Local Directories
TEST_ROOT = path("test_root")
# Want static files in the same dir for running on jenkins.
STATIC_ROOT = TEST_ROOT / "staticfiles"
STATUS_MESSAGE_PATH = TEST_ROOT / "status_message.json"
COURSES_ROOT = TEST_ROOT / "data"
DATA_DIR = COURSES_ROOT
COMMON_TEST_DATA_ROOT = COMMON_ROOT / "test" / "data"
# Where the content data is checked out. This may not exist on jenkins.
GITHUB_REPO_ROOT = ENV_ROOT / "data"
USE_I18N = True
LANGUAGE_CODE = 'en' # tests assume they will get English.
import shutil
import os
#import util # ~maxwl/util/lib/python/util.py
#import measure_prop # ~maxwl/mp/lib/python/measure_prop.py
parser = argparse.ArgumentParser()
parser.add_argument("workdir", type=path)
args = parser.parse_args()
workdir = args.workdir
if not workdir.exists():
workdir.makedirs()
shutil.copy(__file__, workdir / "run.py")
experiment_dir = path(os.getcwd())
project_dir = experiment_dir / ".." / ".."
os.chdir(workdir)
workdir = path(".")
import logging
logging.basicConfig(format="%(asctime)s %(levelname)s:%(message)s")
logger = logging.getLogger('log')
logger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
fh = logging.FileHandler("stdout.txt")
fh.setLevel(logging.DEBUG) # >> this determines the file level
# create console handler with a higher log level
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR) # >> this determines the output level
# create formatter and add it to the handlers
def _write_chunk(request, courselike_key):
"""
Write the OLX file data chunk from the given request to the local filesystem.
"""
# Upload .tar.gz to local filesystem for one-server installations not using S3 or Swift
data_root = path(settings.GITHUB_REPO_ROOT)
subdir = base64.urlsafe_b64encode(repr(courselike_key))
course_dir = data_root / subdir
filename = request.FILES['course-data'].name
courselike_string = text_type(courselike_key) + filename
# Do everything in a try-except block to make sure everything is properly cleaned up.
try:
# Use sessions to keep info about import progress
_save_request_status(request, courselike_string, 0)
if not filename.endswith('.tar.gz'):
_save_request_status(request, courselike_string, -1)
return JsonResponse(
{
'ErrMsg': _('We only support uploading a .tar.gz file.'),
'Stage': -1
},
status=415)
temp_filepath = course_dir / filename
if not course_dir.isdir():
os.mkdir(course_dir)
logging.debug(u'importing course to {0}'.format(temp_filepath))
# Get upload chunks byte ranges
try:
matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
content_range = matches.groupdict()
except KeyError: # Single chunk
# no Content-Range header, so make one that will work
content_range = {'start': 0, 'stop': 1, 'end': 2}
# stream out the uploaded files in chunks to disk
if int(content_range['start']) == 0:
mode = "wb+"
else:
mode = "ab+"
size = os.path.getsize(temp_filepath)
# Check to make sure we haven't missed a chunk
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier.
if size < int(content_range['start']):
_save_request_status(request, courselike_string, -1)
log.warning(
"Reported range %s does not match size downloaded so far %s",
content_range['start'], size)
return JsonResponse(
{
'ErrMsg': _('File upload corrupted. Please try again'),
'Stage': -1
},
status=409)
# The last request sometimes comes twice. This happens because
# nginx sends a 499 error code when the response takes too long.
elif size > int(content_range['stop']) and size == int(
content_range['end']):
return JsonResponse({'ImportStatus': 1})
with open(temp_filepath, mode) as temp_file:
for chunk in request.FILES['course-data'].chunks():
temp_file.write(chunk)
size = os.path.getsize(temp_filepath)
if int(content_range['stop']) != int(content_range['end']) - 1:
# More chunks coming
return JsonResponse({
"files": [{
"name":
filename,
"size":
size,
"deleteUrl":
"",
"deleteType":
"",
"url":
reverse_course_url('import_handler', courselike_key),
"thumbnailUrl":
""
}]
})
log.info(u"Course import %s: Upload complete", courselike_key)
with open(temp_filepath, 'rb') as local_file:
django_file = File(local_file)
storage_path = course_import_export_storage.save(
u'olx_import/' + filename, django_file)
import_olx.delay(request.user.id, text_type(courselike_key),
storage_path, filename, request.LANGUAGE_CODE)
# Send errors to client with stage at which error occurred.
except Exception as exception: # pylint: disable=broad-except
_save_request_status(request, courselike_string, -1)
if course_dir.isdir():
shutil.rmtree(course_dir)
log.info(u"Course import %s: Temp data cleared", courselike_key)
log.exception("error importing course")
return JsonResponse({
'ErrMsg': str(exception),
'Stage': -1
},
status=400)
return JsonResponse({'ImportStatus': 1})
"""
Settings for bok choy tests
"""
import os
from path import path
########################## Prod-like settings ###################################
# These should be as close as possible to the settings we use in production.
# As in prod, we read in environment and auth variables from JSON files.
# Unlike in prod, we use the JSON files stored in this repo.
# This is a convenience for ensuring (a) that we can consistently find the files
# and (b) that the files are the same in Jenkins as in local dev.
os.environ['SERVICE_VARIANT'] = 'bok_choy'
os.environ['CONFIG_ROOT'] = path(__file__).abspath().dirname() #pylint: disable=E1120
from .aws import * # pylint: disable=W0401, W0614
from xmodule.x_module import prefer_xmodules
######################### Testing overrides ####################################
# Needed for the `reset_db` management command
INSTALLED_APPS += ('django_extensions', )
# Redirect to the test_root folder within the repo
TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root" #pylint: disable=E1120
GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath()
LOG_DIR = (TEST_ROOT / "log").abspath()
# Configure Mongo modulestore to use the test folder within the repo
for store in ["default", "direct"]:
def test_mongo_proc(mongo_proc, mongo_proc2, mongo_proc3):
for m in (mongo_proc, mongo_proc2, mongo_proc3):
assert path('/tmp/mongo.{port}.log'.format(port=m.port)).isfile()
import re
import os
import pandas as pd
from path import path
exp_name = 'exp9'
# define directories:
home_dir = '/share/ScratchGeneral/jamtor/'
project_dir = home_dir + '/projects/hgsoc_repeats/RNA-seq/'
results_dir = project_dir + '/results/'
report_dir = project_dir + '/reports/' + exp_name + '/'
ribo_dir = results_dir + '/star/ribo/' + exp_name + '/'
gc_dir = results_dir + '/star/GC/' + exp_name + '/'
htseq_dir = path(results_dir + '/htseq/' + exp_name + '/')
rkey_dir = project_dir + '/raw_files/fullsamples/bowtell_primary/'
record_dir = project_dir + '/record/'
if not os.path.exists(record_dir):
os.makedirs(record_dir)
print('The report_dir is ' + report_dir)
print('The ribo_dir is ' + ribo_dir)
print('The gc_dir is ' + gc_dir)
print('The htseq_dir is ' + htseq_dir)
print('The record_dir is ' + record_dir)
### 1. Fetch directory names in reports directory ###
rep = []
# This is a minimal settings file allowing us to run "update_assets"
# in the Dockerfile for the production image of the edxapp CMS
from openedx.core.lib.derived import derive_settings
from path import Path as path
from ..common import *
DATABASES = {"default": {}}
XQUEUE_INTERFACE = {"url": None, "django_auth": None}
# We need to override STATIC_ROOT because for CMS, edX appends the value of
# "EDX_PLATFORM_REVISION" to it by default and we don't want to use this.
# We should use Django's ManifestStaticFilesStorage for this purpose.
STATIC_URL = "/static/studio/"
STATIC_ROOT = path("/edx/app/edxapp/staticfiles/studio")
########################## Derive Any Derived Settings #######################
derive_settings(__name__)
def test_report_dir_without_files(self):
os.remove(self.test_file.name)
pavelib.quality._prepare_report_dir(path(self.test_dir)) # pylint: disable=protected-access
assert os.listdir(path(self.test_dir)) == []
def setUp(self):
super().setUp()
self.report_dir = path(tempfile.mkdtemp())
self.addCleanup(shutil.rmtree, self.report_dir)
def __init__(self, filepath):
self.filepath = path(filepath)
self.uncommitted = {}
if self.filepath.exists():
with open(self.filepath) as stream:
self.uncommitted = json.load(stream)
'ALLOW_PUBLIC_ACCOUNT_CREATION': True,
# Whether or not the dynamic EnrollmentTrackUserPartition should be registered.
'ENABLE_ENROLLMENT_TRACK_USER_PARTITION': True,
}
ENABLE_JASMINE = False
############################# SOCIAL MEDIA SHARING #############################
SOCIAL_SHARING_SETTINGS = {
# Note: Ensure 'CUSTOM_COURSE_URLS' has a matching value in lms/envs/common.py
'CUSTOM_COURSE_URLS': False
}
############################# SET PATH INFORMATION #############################
PROJECT_ROOT = path(__file__).abspath().dirname().dirname() # /edx-platform/cms
REPO_ROOT = PROJECT_ROOT.dirname()
COMMON_ROOT = REPO_ROOT / "common"
OPENEDX_ROOT = REPO_ROOT / "openedx"
CMS_ROOT = REPO_ROOT / "cms"
LMS_ROOT = REPO_ROOT / "lms"
ENV_ROOT = REPO_ROOT.dirname() # virtualenv dir /edx-platform is in
GITHUB_REPO_ROOT = ENV_ROOT / "data"
sys.path.append(REPO_ROOT)
sys.path.append(PROJECT_ROOT / 'djangoapps')
sys.path.append(COMMON_ROOT / 'djangoapps')
# For geolocation ip database
GEOIP_PATH = REPO_ROOT / "common/static/data/geoip/GeoIP.dat"
def test_report_dir_with_files(self):
assert os.path.exists(self.test_file.name)
pavelib.quality._prepare_report_dir(path(self.test_dir)) # pylint: disable=protected-access
assert not os.path.exists(self.test_file.name)
def import_olx(self, user_id, course_key_string, archive_path, archive_name,
language):
"""
Import a course or library from a provided OLX .tar.gz archive.
"""
courselike_key = CourseKey.from_string(course_key_string)
try:
user = User.objects.get(pk=user_id)
except User.DoesNotExist:
with respect_language(language):
self.status.fail(_(u'Unknown User ID: {0}').format(user_id))
return
if not has_course_author_access(user, courselike_key):
with respect_language(language):
self.status.fail(_(u'Permission denied'))
return
is_library = isinstance(courselike_key, LibraryLocator)
is_course = not is_library
if is_library:
root_name = LIBRARY_ROOT
courselike_module = modulestore().get_library(courselike_key)
import_func = import_library_from_xml
else:
root_name = COURSE_ROOT
courselike_module = modulestore().get_course(courselike_key)
import_func = import_course_from_xml
# Locate the uploaded OLX archive (and download it from S3 if necessary)
# Do everything in a try-except block to make sure everything is properly cleaned up.
data_root = path(settings.GITHUB_REPO_ROOT)
subdir = base64.urlsafe_b64encode(
repr(courselike_key).encode('utf-8')).decode('utf-8')
course_dir = data_root / subdir
try:
self.status.set_state(u'Unpacking')
if not archive_name.endswith(u'.tar.gz'):
with respect_language(language):
self.status.fail(
_(u'We only support uploading a .tar.gz file.'))
return
temp_filepath = course_dir / get_valid_filename(archive_name)
if not course_dir.isdir():
os.mkdir(course_dir)
LOGGER.debug(u'importing course to {0}'.format(temp_filepath))
# Copy the OLX archive from where it was uploaded to (S3, Swift, file system, etc.)
if not course_import_export_storage.exists(archive_path):
LOGGER.info(u'Course import %s: Uploaded file %s not found',
courselike_key, archive_path)
with respect_language(language):
self.status.fail(_(u'Tar file not found'))
return
with course_import_export_storage.open(archive_path, 'rb') as source:
with open(temp_filepath, 'wb') as destination:
def read_chunk():
"""
Read and return a sequence of bytes from the source file.
"""
return source.read(FILE_READ_CHUNK)
for chunk in iter(read_chunk, b''):
destination.write(chunk)
LOGGER.info(u'Course import %s: Download from storage complete',
courselike_key)
# Delete from source location
course_import_export_storage.delete(archive_path)
# If the course has an entrance exam then remove it and its corresponding milestone.
# current course state before import.
if is_course:
if courselike_module.entrance_exam_enabled:
fake_request = RequestFactory().get(u'/')
fake_request.user = user
from contentstore.views.entrance_exam import remove_entrance_exam_milestone_reference
# TODO: Is this really ok? Seems dangerous for a live course
remove_entrance_exam_milestone_reference(
fake_request, courselike_key)
LOGGER.info(
u'entrance exam milestone content reference for course %s has been removed',
courselike_module.id)
# Send errors to client with stage at which error occurred.
except Exception as exception: # pylint: disable=broad-except
if course_dir.isdir():
shutil.rmtree(course_dir)
LOGGER.info(u'Course import %s: Temp data cleared', courselike_key)
LOGGER.exception(u'Error importing course %s',
courselike_key,
exc_info=True)
self.status.fail(text_type(exception))
return
# try-finally block for proper clean up after receiving file.
try:
tar_file = tarfile.open(temp_filepath)
try:
safetar_extractall(tar_file, (course_dir + u'/'))
except SuspiciousOperation as exc:
LOGGER.info(u'Course import %s: Unsafe tar file - %s',
courselike_key, exc.args[0])
with respect_language(language):
self.status.fail(_(u'Unsafe tar file. Aborting import.'))
return
finally:
tar_file.close()
LOGGER.info(u'Course import %s: Uploaded file extracted',
courselike_key)
self.status.set_state(u'Verifying')
self.status.increment_completed_steps()
# find the 'course.xml' file
def get_all_files(directory):
"""
For each file in the directory, yield a 2-tuple of (file-name,
directory-path)
"""
for directory_path, _dirnames, filenames in os.walk(directory):
for filename in filenames:
yield (filename, directory_path)
def get_dir_for_filename(directory, filename):
"""
Returns the directory path for the first file found in the directory
with the given name. If there is no file in the directory with
the specified name, return None.
"""
for name, directory_path in get_all_files(directory):
if name == filename:
return directory_path
return None
dirpath = get_dir_for_filename(course_dir, root_name)
if not dirpath:
with respect_language(language):
self.status.fail(
_(u'Could not find the {0} file in the package.').format(
root_name))
return
dirpath = os.path.relpath(dirpath, data_root)
LOGGER.debug(u'found %s at %s', root_name, dirpath)
LOGGER.info(u'Course import %s: Extracted file verified',
courselike_key)
self.status.set_state(u'Updating')
self.status.increment_completed_steps()
courselike_items = import_func(modulestore(),
user.id,
settings.GITHUB_REPO_ROOT, [dirpath],
load_error_modules=False,
static_content_store=contentstore(),
target_id=courselike_key)
new_location = courselike_items[0].location
LOGGER.debug(u'new course at %s', new_location)
LOGGER.info(u'Course import %s: Course import successful',
courselike_key)
except Exception as exception: # pylint: disable=broad-except
LOGGER.exception(u'error importing course', exc_info=True)
self.status.fail(text_type(exception))
finally:
if course_dir.isdir():
shutil.rmtree(course_dir)
LOGGER.info(u'Course import %s: Temp data cleared', courselike_key)
if self.status.state == u'Updating' and is_course:
# Reload the course so we have the latest state
course = modulestore().get_course(courselike_key)
if course.entrance_exam_enabled:
entrance_exam_chapter = modulestore().get_items(
course.id,
qualifiers={u'category': u'chapter'},
settings={u'is_entrance_exam': True})[0]
metadata = {
u'entrance_exam_id':
text_type(entrance_exam_chapter.location)
}
CourseMetadata.update_from_dict(metadata, course, user)
from contentstore.views.entrance_exam import add_entrance_exam_milestone
add_entrance_exam_milestone(course.id, entrance_exam_chapter)
LOGGER.info(u'Course %s Entrance exam imported', course.id)
import os
from werkzeug.wsgi import DispatcherMiddleware
from paste.cgiapp import CGIApplication
from path import path
from webob.dec import wsgify
VIEWER_HOME = path(__file__).abspath().parent / 'maps'
def create_mapserver_app():
mapserv_cgi = CGIApplication({}, os.environ.get('MAPSERV_BIN', 'mapserv'))
@wsgify
def mapserv_wrapper(request):
request.GET['map'] = VIEWER_HOME / 'money.map'
request.GET['SRS'] = 'EPSG:3857'
return request.get_response(mapserv_cgi)
return mapserv_wrapper
def initialize(app):
app.wsgi_app = DispatcherMiddleware(app.wsgi_app, {
'/mapserv': create_mapserver_app(),
})
class RequirementDownloader(object):
build_dir = path(build_prefix).abspath()
pkginfo_from_file = IndexManager.pkginfo_from_file
parse_requirements = staticmethod(parse_requirements)
def __init__(self, req_set, finder=None, upgrade=False, seen=None):
#@@ start with req_set??
self.req_set = req_set
self.upgrade = False
download_dir = req_set.download_dir
self.download_dir = path(download_dir)
self.finder = finder
self.seen = seen
self.skip = []
if self.seen is None:
self.seen = set()
self.errors = []
# toupe for pip
options = type('Options', (),
dict(skip_requirements_regex='', default_vcs=''))
@classmethod
def req_set_from_file(cls, filename, download_dir, deplinks=None):
src_dir = path(src_prefix).abspath()
finder = cls.package_finder(deplinks)
requirement_set = RequirementSet(build_dir=cls.build_dir,
src_dir=src_dir,
download_dir=download_dir,
download_cache=None,
upgrade=False,
ignore_installed=True,
ignore_dependencies=False)
options = cls.options()
names = []
for req in cls.parse_requirements(filename,
finder=finder,
options=options):
requirement_set.add_requirement(req)
names.append(req.req)
yield requirement_set
yield finder
@staticmethod
def readzip(archive, name):
return archive.read(name)
@staticmethod
def readtar(archive, name):
return archive.extractfile(name).read()
@staticmethod
def find_file(names, tail):
try:
return next(n for n in names if n.endswith(tail))
except StopIteration:
return None
@classmethod
def depinfo_for_file(cls, filename):
if filename.endswith('.zip'):
archive = zipfile.ZipFile(filename)
names = archive.namelist()
read = partial(cls.readzip, archive)
elif filename.endswith('gz') or filename.endswith('bz2'):
archive = tarfile.TarFile.open(filename)
names = archive.getnames()
read = partial(cls.readtar, archive)
dl_file = cls.find_file(names, '.egg-info/dependency_links.txt')
reqs_file = cls.find_file(names, '.egg-info/requires.txt')
deplinks = dl_file and read(dl_file) or ''
requires = reqs_file and read(reqs_file) or ''
return [x.strip() for x in deplinks.split('\n') if x],\
[x.strip() for x in requires.split('\n') if x and not x.startswith('[')]
def download_url(self, link):
target_url = link.url.split('#', 1)[0]
resp = requests.get(target_url)
logger.info('Downloading: %s' % target_url)
outfile = self.download_dir / link.filename
outfile.write_bytes(resp.content)
# requests.iter_content
pkginfo = self.pkginfo_from_file(outfile)
return pkginfo, outfile
def temp_req(self, name, content=None):
fp = path(tempfile.gettempdir()) / ('temp-req-%s.txt' % name)
if content:
logger.debug("Reqs for %s:\n%s", name, content)
fp.write_text(content)
return fp
def handle_requirement(self, req, finder):
"""
Download requirement, return a new requirement set of
requirements dependencies.
"""
if req.editable:
msg = "Editables not supported: %s" % req
logger.warn(msg)
self.errors.append("%s: %s" % (req, msg))
return
try:
url = finder.find_requirement(req, self.upgrade)
except DistributionNotFound:
msg = "No distribution found for %s" % req.name
logger.warn(msg)
self.errors.append(msg)
return
if url.hash in self.seen:
logger.debug('Seen: %s', url)
self.skip.append(url)
return
try:
pkginfo, outfile = self.download_url(url)
except HTTPError, e:
msg = "Issue with download: %s" % e
logger.error(msg)
self.errors.append("%s: %s" % (req, msg))
return
except TypeError:
raise
# Include a non-ascii character in STUDIO_NAME and STUDIO_SHORT_NAME to uncover possible
# UnicodeEncodeErrors in tests. Also use lazy text to reveal possible json dumps errors
STUDIO_NAME = ugettext_lazy(u"Your Platform 𝓢𝓽𝓾𝓭𝓲𝓸")
STUDIO_SHORT_NAME = ugettext_lazy(u"𝓢𝓽𝓾𝓭𝓲𝓸")
# Allow all hosts during tests, we use a lot of different ones all over the codebase.
ALLOWED_HOSTS = ['*']
# mongo connection settings
MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
THIS_UUID = uuid4().hex[:5]
TEST_ROOT = path('test_root')
# Want static files in the same dir for running on jenkins.
STATIC_ROOT = TEST_ROOT / "staticfiles"
WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"
GITHUB_REPO_ROOT = TEST_ROOT / "data"
DATA_DIR = TEST_ROOT / "data"
COMMON_TEST_DATA_ROOT = COMMON_ROOT / "test" / "data"
# For testing "push to lms"
FEATURES['ENABLE_EXPORT_GIT'] = True
GIT_REPO_EXPORT_DIR = TEST_ROOT / "export_course_repos"
# TODO (cpennington): We need to figure out how envs/test.py can inject things into common.py so that we don't have to repeat this sort of thing # lint-amnesty, pylint: disable=line-too-long
STATICFILES_DIRS = [
from .common import *
from openedx.core.lib.logsettings import get_logger_config
import os
from path import Path as path
from xmodule.modulestore.modulestore_settings import convert_module_store_setting_if_needed
# SERVICE_VARIANT specifies name of the variant used, which decides what JSON
# configuration files are read during startup.
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)
# CONFIG_ROOT specifies the directory where the JSON configuration
# files are expected to be found. If not specified, use the project
# directory.
CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT))
# CONFIG_PREFIX specifies the prefix of the JSON configuration files,
# based on the service variant. If no variant is use, don't use a
# prefix.
CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""
############### ALWAYS THE SAME ################################
DEBUG = False
EMAIL_BACKEND = 'django_ses.SESBackend'
SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
# IMPORTANT: With this enabled, the server must always be behind a proxy that
# strips the header HTTP_X_FORWARDED_PROTO from client requests. Otherwise,
def create_export_tarball(course_module, course_key, context, status=None):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key,
root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(),
course_module.id, root_dir, name)
if status:
status.set_state(u'Compressing')
status.increment_completed_steps()
LOGGER.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
LOGGER.exception(u'There was an error exporting %s',
course_key,
exc_info=True)
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(
failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err':
True,
'raw_err_msg':
str(exc),
'edit_unit_url':
reverse_usage_url("container_handler", parent.location)
if parent else "",
})
if status:
status.fail(
json.dumps({
'raw_error_msg': context['raw_err_msg'],
'edit_unit_url': context['edit_unit_url']
}))
raise
except Exception as exc:
LOGGER.exception(u'There was an error exporting %s',
course_key,
exc_info=True)
context.update({
'in_err': True,
'edit_unit_url': None,
'raw_err_msg': str(exc)
})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']}))
raise
finally:
if os.path.exists(root_dir / name):
shutil.rmtree(root_dir / name)
return export_file
def export_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None):
"""
The restful handler for exporting a course.
GET
html: return html page for import page
application/x-tgz: return tar.gz file containing exported course
json: not supported
Note that there are 2 ways to request the tar.gz file. The request header can specify
application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz).
If the tar.gz file has been requested but the export operation fails, an HTML page will be returned
which describes the error.
"""
location = BlockUsageLocator(package_id=package_id, branch=branch, version_guid=version_guid, block_id=block)
if not has_course_access(request.user, location):
raise PermissionDenied()
old_location = loc_mapper().translate_locator_to_location(location)
course_module = modulestore().get_item(old_location)
# an _accept URL parameter will be preferred over HTTP_ACCEPT in the header.
requested_format = request.REQUEST.get('_accept', request.META.get('HTTP_ACCEPT', 'text/html'))
export_url = location.url_reverse('export') + '?_accept=application/x-tgz'
if 'application/x-tgz' in requested_format:
name = old_location.name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
export_to_xml(modulestore('direct'), contentstore(), old_location, root_dir, name, modulestore())
logging.debug('tar file being generated at {0}'.format(export_file.name))
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError, e:
logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e)))
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_instance(course_module.location.course_id, e.location)
parent_locs = modulestore().get_parent_locations(failed_item.location, course_module.location.course_id)
if len(parent_locs) > 0:
parent = modulestore().get_item(parent_locs[0])
if parent.location.category == 'vertical':
unit = parent
except:
# if we have a nested exception, then we'll show the more generic error message
pass
unit_locator = loc_mapper().translate_location(old_location.course_id, parent.location, False, True)
return render_to_response('export.html', {
'context_course': course_module,
'in_err': True,
'raw_err_msg': str(e),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': unit_locator.url_reverse("unit") if parent else "",
'course_home_url': location.url_reverse("course"),
'export_url': export_url
})
except Exception, e:
logging.exception('There was an error exporting course {0}. {1}'.format(course_module.location, unicode(e)))
return render_to_response('export.html', {
'context_course': course_module,
'in_err': True,
'unit': None,
'raw_err_msg': str(e),
'course_home_url': location.url_reverse("course"),
'export_url': export_url
})
def temp_req(self, name, content=None):
fp = path(tempfile.gettempdir()) / ('temp-req-%s.txt' % name)
if content:
logger.debug("Reqs for %s:\n%s", name, content)
fp.write_text(content)
return fp
def decompresser_base(base, date_fond, dates_majo, cache='cache'):
# Vérification des paramètres
base = base.upper()
if base not in bases:
raise NomBaseException()
if not isinstance(date_fond, datetime) or not isinstance(dates_majo, list):
raise ValueError()
for date in dates_majo:
if not isinstance(date, datetime):
raise ValueError()
# Créer le répertoire rattaché à ce dump complet
rep = os.path.join(cache, 'bases-xml', date_fond.strftime('%Y%m%d-%H%M%S'))
path(rep).mkdir_p()
# Décompresser le dump complet
date = date_fond.strftime('%Y%m%d-%H%M%S')
if not os.path.exists(os.path.join(rep, 'fond-' + date)) or \
os.path.exists(os.path.join(rep, 'fond-' + date, 'erreur-tar')):
if os.path.exists(os.path.join(rep, 'fond-' + date, 'erreur-tar')):
shutil.rmtree(os.path.join(rep, 'fond-' + date))
path(os.path.join(rep, 'fond-' + date)).mkdir_p()
open(os.path.join(rep, 'fond-' + date, 'erreur-tar'), 'w').close()
subprocess.call([
'tar', 'xzf',
os.path.join(cache, 'tar', base + '-fond-' + date + '.tar.gz'),
'-C',
os.path.join(rep, 'fond-' + date)
])
os.remove(os.path.join(rep, 'fond-' + date, 'erreur-tar'))
# Inscrire cette livraison dans la base de données
try:
entree_livraison = Livraison.get(Livraison.date == date_fond)
except Livraison.DoesNotExist:
entree_livraison = Livraison.create(date=date_fond,
type='fondation',
base=base,
precedent=None,
fondation=None)
entree_livraison_fondation = entree_livraison
# Décompresser les dumps incrémentaux
for date_majo in dates_majo:
date = date_majo.strftime('%Y%m%d-%H%M%S')
if not os.path.exists(os.path.join(rep, 'majo-' + date)) or \
os.path.exists(os.path.join(rep, date)) or \
os.path.exists(os.path.join(rep, 'majo-' + date, 'erreur-tar')):
if os.path.exists(os.path.join(rep, date)):
shutil.rmtree(os.path.join(rep, date), True)
shutil.rmtree(os.path.join(rep, 'majo-' + date), True)
if os.path.exists(os.path.join(rep, 'majo-' + date, 'erreur-tar')):
shutil.rmtree(os.path.join(rep, 'majo-' + date), True)
path(os.path.join(rep, date)).mkdir_p()
open(os.path.join(rep, date, 'erreur-tar'), 'w').close()
subprocess.call([
'tar', 'xzf',
os.path.join(cache, 'tar', base + '-majo-' + date + '.tar.gz'),
'-C', rep
])
os.rename(os.path.join(rep, date),
os.path.join(rep, 'majo-' + date))
os.remove(os.path.join(rep, 'majo-' + date, 'erreur-tar'))
# Inscrire cette livraison dans la base de données
try:
entree_livraison = Livraison.get(Livraison.date == date_majo)
except Livraison.DoesNotExist:
entree_livraison = Livraison.create(
date=date_majo,
type='miseajour',
base=base,
precedent=entree_livraison,
fondation=entree_livraison_fondation)
def import_handler(request, tag=None, package_id=None, branch=None, version_guid=None, block=None):
"""
The restful handler for importing a course.
GET
html: return html page for import page
json: not supported
POST or PUT
json: import a course via the .tar.gz file specified in request.FILES
"""
location = BlockUsageLocator(package_id=package_id, branch=branch, version_guid=version_guid, block_id=block)
if not has_course_access(request.user, location):
raise PermissionDenied()
old_location = loc_mapper().translate_locator_to_location(location)
if 'application/json' in request.META.get('HTTP_ACCEPT', 'application/json'):
if request.method == 'GET':
raise NotImplementedError('coming soon')
else:
data_root = path(settings.GITHUB_REPO_ROOT)
course_subdir = "{0}-{1}-{2}".format(old_location.org, old_location.course, old_location.name)
course_dir = data_root / course_subdir
filename = request.FILES['course-data'].name
if not filename.endswith('.tar.gz'):
return JsonResponse(
{
'ErrMsg': _('We only support uploading a .tar.gz file.'),
'Stage': 1
},
status=415
)
temp_filepath = course_dir / filename
if not course_dir.isdir():
os.mkdir(course_dir)
logging.debug('importing course to {0}'.format(temp_filepath))
# Get upload chunks byte ranges
try:
matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"])
content_range = matches.groupdict()
except KeyError: # Single chunk
# no Content-Range header, so make one that will work
content_range = {'start': 0, 'stop': 1, 'end': 2}
# stream out the uploaded files in chunks to disk
if int(content_range['start']) == 0:
mode = "wb+"
else:
mode = "ab+"
size = os.path.getsize(temp_filepath)
# Check to make sure we haven't missed a chunk
# This shouldn't happen, even if different instances are handling
# the same session, but it's always better to catch errors earlier.
if size < int(content_range['start']):
log.warning(
"Reported range %s does not match size downloaded so far %s",
content_range['start'],
size
)
return JsonResponse(
{
'ErrMsg': _('File upload corrupted. Please try again'),
'Stage': 1
},
status=409
)
# The last request sometimes comes twice. This happens because
# nginx sends a 499 error code when the response takes too long.
elif size > int(content_range['stop']) and size == int(content_range['end']):
return JsonResponse({'ImportStatus': 1})
with open(temp_filepath, mode) as temp_file:
for chunk in request.FILES['course-data'].chunks():
temp_file.write(chunk)
size = os.path.getsize(temp_filepath)
if int(content_range['stop']) != int(content_range['end']) - 1:
# More chunks coming
return JsonResponse({
"files": [{
"name": filename,
"size": size,
"deleteUrl": "",
"deleteType": "",
"url": location.url_reverse('import'),
"thumbnailUrl": ""
}]
})
else: # This was the last chunk.
# Use sessions to keep info about import progress
session_status = request.session.setdefault("import_status", {})
key = location.package_id + filename
session_status[key] = 1
request.session.modified = True
# Do everything from now on in a try-finally block to make sure
# everything is properly cleaned up.
try:
tar_file = tarfile.open(temp_filepath)
try:
safetar_extractall(tar_file, (course_dir + '/').encode('utf-8'))
except SuspiciousOperation as exc:
return JsonResponse(
{
'ErrMsg': 'Unsafe tar file. Aborting import.',
'SuspiciousFileOperationMsg': exc.args[0],
'Stage': 1
},
status=400
)
finally:
tar_file.close()
session_status[key] = 2
request.session.modified = True
# find the 'course.xml' file
def get_all_files(directory):
"""
For each file in the directory, yield a 2-tuple of (file-name,
directory-path)
"""
for dirpath, _dirnames, filenames in os.walk(directory):
for filename in filenames:
yield (filename, dirpath)
def get_dir_for_fname(directory, filename):
"""
Returns the dirpath for the first file found in the directory
with the given name. If there is no file in the directory with
the specified name, return None.
"""
for fname, dirpath in get_all_files(directory):
if fname == filename:
return dirpath
return None
fname = "course.xml"
dirpath = get_dir_for_fname(course_dir, fname)
if not dirpath:
return JsonResponse(
{
'ErrMsg': _('Could not find the course.xml file in the package.'),
'Stage': 2
},
status=415
)
logging.debug('found course.xml at {0}'.format(dirpath))
if dirpath != course_dir:
for fname in os.listdir(dirpath):
shutil.move(dirpath / fname, course_dir)
_module_store, course_items = import_from_xml(
modulestore('direct'),
settings.GITHUB_REPO_ROOT,
[course_subdir],
load_error_modules=False,
static_content_store=contentstore(),
target_location_namespace=old_location,
draft_store=modulestore()
)
new_location = course_items[0].location
logging.debug('new course at {0}'.format(new_location))
session_status[key] = 3
request.session.modified = True
auth.add_users(request.user, CourseInstructorRole(new_location), request.user)
auth.add_users(request.user, CourseStaffRole(new_location), request.user)
logging.debug('created all course groups at {0}'.format(new_location))
# Send errors to client with stage at which error occurred.
except Exception as exception: # pylint: disable=W0703
log.exception(
"error importing course"
)
return JsonResponse(
{
'ErrMsg': str(exception),
'Stage': session_status[key]
},
status=400
)
finally:
shutil.rmtree(course_dir)
return JsonResponse({'Status': 'OK'})
elif request.method == 'GET': # assume html
course_module = modulestore().get_item(old_location)
return render_to_response('import.html', {
'context_course': course_module,
'successful_import_redirect_url': location.url_reverse("course"),
'import_status_url': location.url_reverse("import_status", "fillerName"),
})
else:
return HttpResponseNotFound()
def test_tester_layer(self):
bu = build.Builder()
bu.log_level = "WARNING"
bu.output_dir = "out"
bu.series = "trusty"
bu.name = "foo"
bu.charm = "trusty/tester"
bu.hide_metrics = True
bu.report = False
remove_layer_file = self.dirname / 'trusty/tester/to_remove'
remove_layer_file.touch()
self.addCleanup(remove_layer_file.remove_p)
with mock.patch.object(build.builder, 'log') as log:
bu()
log.warn.assert_called_with(
'Please add a `repo` key to your layer.yaml, '
'with a url from which your layer can be cloned.')
base = path('out/trusty/foo')
self.assertTrue(base.exists())
# Verify ignore rules applied
self.assertFalse((base / ".bzr").exists())
self.assertEqual((base / "ignore").text(), "mysql\n")
self.assertEqual((base / "exclude").text(), "test-base\n")
self.assertEqual((base / "override-ignore").text(), "tester\n")
self.assertEqual((base / "override-exclude").text(), "tester\n")
self.assertFalse((base / "tests/00-setup").exists())
self.assertFalse((base / "tests/15-configs").exists())
self.assertTrue((base / "tests/20-deploy").exists())
actions = yaml.load((base / "actions.yaml").text())
resources = yaml.load((base / "resources.yaml").text())
self.assertNotIn("test-base", actions)
self.assertIn("mysql", actions)
self.assertIn("tester", actions)
self.assertIn("test-base", resources)
self.assertNotIn("mysql", resources)
self.assertIn("tester", resources)
# Metadata should have combined provides fields
metadata = base / "metadata.yaml"
self.assertTrue(metadata.exists())
metadata_data = yaml.load(metadata.open())
self.assertIn("shared-db", metadata_data['provides'])
self.assertIn("storage", metadata_data['provides'])
# The maintainer, maintainers values should only be from the top layer.
self.assertIn("maintainer", metadata_data)
self.assertEqual(metadata_data['maintainer'], "Tester <*****@*****.**>")
self.assertNotIn("maintainers", metadata_data)
# The tags list must be de-duplicated.
self.assertEqual(metadata_data['tags'], ["databases"])
# Config should have keys but not the ones in deletes
config = base / "config.yaml"
self.assertTrue(config.exists())
config_data = yaml.load(config.open())['options']
self.assertIn("bind-address", config_data)
self.assertNotIn("vip", config_data)
self.assertIn("key", config_data)
self.assertEqual(config_data["key"]["default"], None)
# Issue #99 where strings lose their quotes in a charm build.
self.assertIn("numeric-string", config_data)
default_value = config_data['numeric-string']['default']
self.assertEqual(default_value, "0123456789", "value must be a string")
# Issue 218, ensure proper order of layer application
self.assertEqual(config_data['backup_retention_count']['default'], 7,
'Config from layers was merged in wrong order')
cyaml = base / "layer.yaml"
self.assertTrue(cyaml.exists())
cyaml_data = yaml.load(cyaml.open())
self.assertEquals(cyaml_data['includes'],
['trusty/test-base', 'trusty/mysql'])
self.assertEquals(cyaml_data['is'], 'foo')
self.assertEquals(cyaml_data['options']['mysql']['qux'], 'one')
self.assertTrue((base / "hooks/config-changed").exists())
# Files from the top layer as overrides
start = base / "hooks/start"
self.assertTrue(start.exists())
self.assertIn("Overridden", start.text())
self.assertTrue((base / "README.md").exists())
self.assertEqual("dynamic tactics", (base / "README.md").text())
sigs = base / ".build.manifest"
self.assertTrue(sigs.exists())
data = json.load(sigs.open())
self.assertEquals(data['signatures']["README.md"], [
u'foo', "static", u'cfac20374288c097975e9f25a0d7c81783acdbc81'
'24302ff4a731a4aea10de99'
])
self.assertEquals(data["signatures"]['metadata.yaml'], [
u'foo', "dynamic",
u'5691b0c0aaf43d0f27d8eca6afbd1145aa3f4307456757e592827f002cf603f2'
])
storage_attached = base / "hooks/data-storage-attached"
storage_detaching = base / "hooks/data-storage-detaching"
self.assertTrue(storage_attached.exists())
self.assertTrue(storage_detaching.exists())
self.assertIn("Hook: data", storage_attached.text())
self.assertIn("Hook: data", storage_detaching.text())
# confirm that files removed from a base layer get cleaned up
self.assertTrue((base / 'to_remove').exists())
remove_layer_file.remove()
bu()
self.assertFalse((base / 'to_remove').exists())
def get_id(self, dump_file):
""" Returns restore point ID for a given filepath
"""
return md5.new(path(dump_file).basename()).hexdigest()
def test_regenerate_inplace(self):
# take a generated example where a base layer has changed
# regenerate in place
# make some assertions
bu = build.Builder()
bu.log_level = "WARNING"
bu.output_dir = "out"
bu.series = "trusty"
bu.name = "foo"
bu.charm = "trusty/b"
bu.hide_metrics = True
bu.report = False
bu()
base = path('out/trusty/foo')
self.assertTrue(base.exists())
# verify the 1st gen worked
self.assertTrue((base / "a").exists())
self.assertTrue((base / "README.md").exists())
# now regenerate from the target
with utils.cd("out/trusty/foo"):
bu = build.Builder()
bu.log_level = "WARNING"
bu.output_dir = path(os.getcwd())
bu.series = "trusty"
# The generate target and source are now the same
bu.name = "foo"
bu.charm = "."
bu.hide_metrics = True
bu.report = False
bu()
base = bu.output_dir
self.assertTrue(base.exists())
# Check that the generated layer.yaml makes sense
cy = base / "layer.yaml"
config = yaml.load(cy.open())
self.assertEquals(config["includes"],
["trusty/a", "interface:mysql"])
self.assertEquals(config["is"], "foo")
# We can even run it more than once
bu()
cy = base / "layer.yaml"
config = yaml.load(cy.open())
self.assertEquals(config["includes"],
["trusty/a", "interface:mysql"])
self.assertEquals(config["is"], "foo")
# We included an interface, we should be able to assert things about it
# in its final form as well
provides = base / "hooks/relations/mysql/provides.py"
requires = base / "hooks/relations/mysql/requires.py"
self.assertTrue(provides.exists())
self.assertTrue(requires.exists())
# and that we generated the hooks themselves
for kind in ["joined", "changed", "broken", "departed"]:
self.assertTrue((base / "hooks" /
"mysql-relation-{}".format(kind)).exists())
# and ensure we have an init file (the interface doesn't its added)
init = base / "hooks/relations/mysql/__init__.py"
self.assertTrue(init.exists())
def telecharger_base(base, livraison='tout', cache='cache'):
# Vérification des paramètres
base = base.upper()
if base not in bases:
raise NomBaseException()
if livraison not in ['fondation','tout'] and \
not isinstance(livraison, datetime):
livraison = datetime.strptime(livraison, '%Y%m%d-%H%M%S')
if serveurs[base][0] != 'ftp':
raise NonImplementeException()
# Créer le dossier de cache des fichiers téléchargés
path(os.path.join(cache, 'tar')).mkdir_p()
# Connexion FTP
serveur = serveurs[base][0] + ':' + \
'//' + serveurs[base][2] + ':' + serveurs[base][3] + \
'@' + serveurs[base][1] + serveurs[base][4]
connexion_ftp = ftplib.FTP(serveurs[base][1], \
serveurs[base][2], \
serveurs[base][3])
# Reconnaître les dates des fichiers
connexion_ftp.cwd(serveurs[base][4])
fichiers = connexion_ftp.nlst()
date_fond = None
dates_majo = []
for fichier in fichiers:
# Si c’est un fichier de dump complet
try:
datetime.strptime(fichier, fichiers_fond[base])
if date_fond: raise FondationNonUniqueException()
date_fond = datetime.strptime(fichier, fichiers_fond[base])
except ValueError:
pass
# Si c’est un fichier de dump incrémental
try:
dates_majo.append(datetime.strptime(fichier, fichiers_majo[base]))
except ValueError:
pass
# Normaliser les paramètres relatifs aux dates
dates_majo.sort()
if not date_fond:
raise FondationNonTrouveeException()
if livraison == 'fondation':
livraison = date_fond
if livraison == 'tout':
livraison = dates_majo[-1]
dates_majo = [date for date in dates_majo if date <= livraison]
# Téléchargement du dump complet
telecharger_cache(
serveur + date_fond.strftime(fichiers_fond[base]),
os.path.join(cache, 'tar',
base + date_fond.strftime('-fond-%Y%m%d-%H%M%S.tar.gz')))
# Téléchargement des dumps incrémentaux
for date_majo in dates_majo:
telecharger_cache(
serveur + date_majo.strftime(fichiers_majo[base]),
os.path.join(
cache, 'tar',
base + date_majo.strftime('-majo-%Y%m%d-%H%M%S.tar.gz')))
# Clôturer la connexion
connexion_ftp.close()
return date_fond, dates_majo
# Mongo modulestore beneath mixed.
# Returns the entire collection with *all* courses' asset metadata.
return store.asset_collection
else:
# Split modulestore beneath mixed.
# Split stores all asset metadata in the structure collection.
return store.db_connection.structures
THIS_UUID = uuid4().hex
COMMON_DOCSTORE_CONFIG = {
'host': MONGO_HOST,
'port': MONGO_PORT_NUM,
}
DATA_DIR = path(__file__).dirname().parent.parent / "tests" / "data" / "xml-course-root"
TEST_DATA_DIR = 'common/test/data/'
XBLOCK_MIXINS = (InheritanceMixin, XModuleMixin)
MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([
('draft', MongoModulestoreBuilder()),
('split', VersioningModulestoreBuilder())
])
DRAFT_MODULESTORE_SETUP = MixedModulestoreBuilder([('draft', MongoModulestoreBuilder())])
SPLIT_MODULESTORE_SETUP = MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())])
MIXED_MODULESTORE_SETUPS = (
DRAFT_MODULESTORE_SETUP,
SPLIT_MODULESTORE_SETUP,
)
def tearDown(self):
path("out").rmtree_p()
def on_task_filter(self, task, config):
if not task.accepted:
log.debug('Scanning not needed')
return
config = self.prepare_config(config)
accepted_series = {}
paths = set()
for entry in task.accepted:
if 'series_parser' in entry:
if entry['series_parser'].valid:
accepted_series.setdefault(entry['series_parser'].name,
[]).append(entry)
for folder in config['path']:
try:
paths.add(entry.render(folder))
except RenderError as e:
log.error('Error rendering path `%s`: %s', folder,
e)
else:
log.debug('entry %s series_parser invalid', entry['title'])
if not accepted_series:
log.warning(
'No accepted entries have series information. exists_series cannot filter them'
)
return
# scan through
# For speed, only test accepted entries since our priority should be after everything is accepted.
for series in accepted_series:
# make new parser from parser in entry
series_parser = accepted_series[series][0]['series_parser']
for folder in paths:
folder = path(folder).expanduser()
if not folder.isdir():
log.warn('Directory %s does not exist' % folder, log)
continue
for filename in folder.walk(errors='warn'):
# run parser on filename data
try:
disk_parser = get_plugin_by_name(
'parsing').instance.parse_series(
data=filename.name, name=series_parser.name)
except ParseWarning as pw:
disk_parser = pw.parsed
log_once(pw.value, logger=log)
if disk_parser.valid:
log.debug('name %s is same series as %s',
filename.name, series)
log.debug('disk_parser.identifier = %s',
disk_parser.identifier)
log.debug('disk_parser.quality = %s',
disk_parser.quality)
log.debug('disk_parser.proper_count = %s',
disk_parser.proper_count)
for entry in accepted_series[series]:
log.debug('series_parser.identifier = %s',
entry['series_parser'].identifier)
if disk_parser.identifier != entry[
'series_parser'].identifier:
log.trace('wrong identifier')
continue
log.debug('series_parser.quality = %s',
entry['series_parser'].quality)
if config.get(
'allow_different_qualities') == 'better':
if entry[
'series_parser'].quality > disk_parser.quality:
log.trace('better quality')
continue
elif config.get('allow_different_qualities'):
if disk_parser.quality != entry[
'series_parser'].quality:
log.trace('wrong quality')
continue
log.debug('entry parser.proper_count = %s',
entry['series_parser'].proper_count)
if disk_parser.proper_count >= entry[
'series_parser'].proper_count:
entry.reject('proper already exists')
continue
else:
log.trace('new one is better proper, allowing')
continue
def setUp(self):
self.dirname = path(pkg_resources.resource_filename(__name__, ""))
os.environ["LAYER_PATH"] = self.dirname
os.environ["INTERFACE_PATH"] = self.dirname / "interfaces"
path("out").rmtree_p()
