def test_get_filter_for_alias():
d = Document()
for k, v in d.__class__.filter_list.iteritems():
assert isinstance(k, str)
assert issubclass(v, DexyFilter)
assert d.get_filter_for_alias("cp") == dexy.filters.python_filters.CopyFilter
assert d.get_filter_for_alias("-") == DexyFilter
assert d.get_filter_for_alias("-xxxalias") == DexyFilter
def test_init():
"""document: filters should be processed correctly"""
doc = Document(FileSystemJsonArtifact, "data/test.py|abc")
assert doc.name == "data/test.py"
assert doc.filters == ['abc']
doc.filters += ['def', 'xyz']
assert doc.filters == ['abc', 'def', 'xyz']
assert doc.key() == "data/test.py|abc|def|xyz"
def test_get_filter_for_alias():
Document.filter_list = dexy.introspect.filters()
d = Document()
for k, v in d.__class__.filter_list.iteritems():
assert isinstance(k, str)
assert issubclass(v, DexyFilter)
assert d.get_filter_for_alias("pyg") == dexy.filters.pygments_filters.PygmentsFilter
assert d.get_filter_for_alias("-") == DexyFilter
assert d.get_filter_for_alias("-xxxalias") == DexyFilter
def test_document_set_name_and_filters():
doc = Document()
doc.set_name_and_filters("data/test.py|abc")
assert doc.name == "data/test.py"
assert doc.filters == ['abc']
doc.filters += ['def', 'xyz']
assert doc.filters == ['abc', 'def', 'xyz']
assert doc.key() == "data/test.py|abc|def|xyz"
def test_document_set_name_and_filters():
doc = Document()
doc.set_name_and_filters("data/test.py|abc")
assert doc.name == "data/test.py"
assert doc.filters == ["abc"]
doc.filters += ["def", "xyz"]
assert doc.filters == ["abc", "def", "xyz"]
assert doc.key() == "data/test.py|abc|def|xyz"
def parse_doc(path, input_directive, args = {}):
# If a specification is nested in a dependency, then input_directive
# may be a dict. If so, split it into parts before continuing.
try:
a, b = input_directive.popitem()
input_directive = a
args = b
except AttributeError:
pass
tokens = input_directive.split("|")
if "/" in tokens[0]:
raise Exception("paths not allowed in tokens: %s" % tokens[0])
if path == '.':
glob_string = tokens[0]
else:
glob_string = os.path.join(re.sub("^\./", "", path), tokens[0])
filters = tokens[1:]
docs = []
# virtual document
if re.search("@", glob_string):
# TODO some virtual files are local, not remote. test on
# presence of 'url' or something more appropriate.
virtual = True
if not self.allow_remote:
raise Exception("""You are attempting to access a remote file.
You must enable --dangerous mode to do this.
Please check Dexy help and call the dexy
command again.""")
glob_string = glob_string.replace("@", "")
else:
virtual = False
regex = fnmatch.translate(glob_string).replace(".*", "(.*)")
matcher = re.compile(regex)
files = glob.glob(glob_string)
nofiles = len(files) == 0
if nofiles and virtual:
files = [glob_string]
for f in files:
create = True
if not virtual:
if os.path.isdir(f):
create = False
if args.has_key('disabled'):
if args['disabled']:
create = False
print "document %s|%s disabled" % (f, "|".join(filters))
inputs = []
if args.has_key('inputs'):
if isinstance(args['inputs'], str):
raise Exception("""this input should be an array,
not a string: %s""" % args['inputs'])
for i in args['inputs']:
for doc in parse_doc(path, i):
inputs.append(doc.key())
m = matcher.match(f)
if m and len(m.groups()) > 0:
rootname = matcher.match(f).group(1)
# The 'ifinput' directive says that if an input exists matching
# the specified pattern, we should create this document and it
# will depend on the specified input.
if args.has_key('ifinput'):
self.log.debug(f)
if isinstance(args['ifinput'], str) or isinstance(args['ifinput'], unicode):
ifinputs = [args['ifinput']]
else:
self.log.debug("treating input %s as iterable. class: %s" % (
args['ifinput'], args['ifinput'].__class__.__name__))
ifinputs = args['ifinput']
for s in ifinputs:
self.log.debug("evaluating ifinput %s" % s)
ifinput = s.replace("%", rootname)
self.log.debug("evaluating ifinput %s" % ifinput)
input_docs = parse_doc(path, ifinput, {})
for input_doc in input_docs:
self.log.debug(input_doc.key())
inputs.append(input_doc.key())
if len(input_docs) == 0:
create = False
if args.has_key('ifnoinput'):
ifinput = args['ifnoinput'].replace("%", rootname)
input_docs = parse_doc(path, ifinput, {})
if len(input_docs) > 0:
create = False
if args.has_key('except'):
try:
except_re = re.compile(args['except'])
except sre_constants.error as e:
raise Exception("""You passed 'except' value of %s.
Please pass a valid Python-style regular expression for
'except', NOT a glob-style matcher. Error message from
re.compile: %s""" % (args['except'], e))
if re.match(except_re, f):
print "skipping %s as it matches except pattern %s" % (f, args['except'])
create = False
if create:
# Filters can either be included in the name...
doc = Document(self.artifact_class, f, filters)
# ...or they may be listed explicitly.
if args.has_key('filters'):
doc.filters += args['filters']
# Here we are assuming that if we get a key with blank args
# this should not override a previous key. A key which does
# have args should override any previous key.
key = doc.key()
self.log.debug("creating doc %s for glob %s" % (key, glob_string))
if self.members.has_key(key):
self.log.debug("found existing key %s" % key)
doc = self.members[key]
else:
self.log.debug("no existing key %s" % key)
if len(args) > 0:
self.log.debug("args: %s" % args)
doc.args = args
doc.use_all_inputs = args.has_key('allinputs')
for i in inputs:
doc.add_input_key(i)
if not hasattr(doc, 'args'):
doc.args = args
self.members[key] = doc
docs.append(doc) # just a local list
return docs
