def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
global ArchiveFileIOError
if ArchiveFileIOError is None:
from arelle.FileSource import ArchiveFileIOError
unNamedCounter = 1
txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1",
"http://xbrl.org/PWD/2014-01-15/taxonomy-package",
"http://xbrl.org/PWD/2015-01-14/taxonomy-package",
"http://xbrl.org/PR/2015-12-09/taxonomy-package",
"http://xbrl.org/2016/taxonomy-package",
"http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package")
catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog", )
pkg = {}
currentLang = Locale.getLanguageCode()
_file = filesource.file(metadataFile)[
0] # URL in zip, plain file in file system or web
parser = lxmlResolvingParser(cntlr)
try:
tree = etree.parse(_file, parser=parser)
# schema validate tp xml
xsdTree = etree.parse(TP_XSD, parser=parser)
etree.XMLSchema(xsdTree).assertValid(tree)
except (etree.XMLSyntaxError, etree.DocumentInvalid) as err:
cntlr.addToLog(_("Taxonomy package file syntax error %(error)s"),
messageArgs={"error": str(err)},
messageCode="tpe:invalidMetaDataFile",
file=os.path.basename(metadataFile),
level=logging.ERROR)
errors.append("tpe:invalidMetaDataFile")
return pkg
root = tree.getroot()
ns = root.tag.partition("}")[0][1:]
nsPrefix = "{{{}}}".format(ns)
if ns in txmyPkgNSes: # package file
for eltName in ("identifier", "version", "license", "publisher",
"publisherURL", "publisherCountry", "publicationDate"):
pkg[eltName] = ''
for m in root.iterchildren(tag=nsPrefix + eltName):
if eltName == "license":
pkg[eltName] = m.get("name")
else:
pkg[eltName] = (m.text or "").strip()
break # take first entry if several
for eltName in ("name", "description"):
closest = ''
closestLen = 0
for m in root.iterchildren(tag=nsPrefix + eltName):
s = (m.text or "").strip()
eltLang = xmlLang(m)
l = langCloseness(eltLang, currentLang)
if l > closestLen:
closestLen = l
closest = s
elif closestLen == 0 and eltLang.startswith("en"):
closest = s # pick english if nothing better
if not closest and eltName == "name": # assign default name when none in taxonomy package
closest = os.path.splitext(os.path.basename(
filesource.baseurl))[0]
pkg[eltName] = closest
for eltName in ("supersededTaxonomyPackages", "versioningReports"):
pkg[eltName] = []
for m in root.iterchildren(tag=nsPrefix +
"supersededTaxonomyPackages"):
pkg['supersededTaxonomyPackages'] = [
r.text.strip()
for r in m.iterchildren(tag=nsPrefix + "taxonomyPackageRef")
]
for m in root.iterchildren(tag=nsPrefix + "versioningReports"):
pkg['versioningReports'] = [
r.get("href")
for r in m.iterchildren(tag=nsPrefix + "versioningReport")
]
# check for duplicate multi-lingual elements (among children of nodes)
langElts = defaultdict(list)
for n in root.iter(tag=nsPrefix + "*"):
for eltName in ("name", "description", "publisher"):
langElts.clear()
for m in n.iterchildren(tag=nsPrefix + eltName):
langElts[xmlLang(m)].append(m)
for lang, elts in langElts.items():
if not lang:
cntlr.addToLog(
_("Multi-lingual element %(element)s has no in-scope xml:lang attribute"
),
messageArgs={"element": eltName},
messageCode="tpe:missingLanguageAttribute",
refs=[{
"href": os.path.basename(metadataFile),
"sourceLine": m.sourceline
} for m in elts],
level=logging.ERROR)
errors.append("tpe:missingLanguageAttribute")
elif len(elts) > 1:
cntlr.addToLog(
_("Multi-lingual element %(element)s has multiple (%(count)s) in-scope xml:lang %(lang)s elements"
),
messageArgs={
"element": eltName,
"lang": lang,
"count": len(elts)
},
messageCode="tpe:duplicateLanguagesForElement",
refs=[{
"href": os.path.basename(metadataFile),
"sourceLine": m.sourceline
} for m in elts],
level=logging.ERROR)
errors.append("tpe:duplicateLanguagesForElement")
del langElts # dereference
else: # oasis catalog, use dirname as the package name
# metadataFile may be a File object (with name) or string filename
fileName = getattr(
metadataFile,
'fileName', # for FileSource named objects
getattr(
metadataFile,
'name', # for io.file named objects
metadataFile)) # for string
pkg["name"] = os.path.basename(os.path.dirname(fileName))
pkg["description"] = "oasis catalog"
pkg["version"] = "(none)"
remappings = {}
rewriteTree = tree
catalogFile = metadataFile
if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package",
"http://xbrl.org/PR/2015-12-09/taxonomy-package",
"http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package",
"http://xbrl.org/2016/taxonomy-package",
"http://xbrl.org/REC/2016-04-19/taxonomy-package"):
catalogFile = metadataFile.replace('taxonomyPackage.xml',
'catalog.xml')
try:
rewriteTree = etree.parse(filesource.file(catalogFile)[0],
parser=parser)
# schema validate tp xml
xsdTree = etree.parse(CAT_XSD, parser=parser)
etree.XMLSchema(xsdTree).assertValid(rewriteTree)
except (etree.XMLSyntaxError, etree.DocumentInvalid) as err:
cntlr.addToLog(_("Catalog file syntax error %(error)s"),
messageArgs={"error": str(err)},
messageCode="tpe:invalidCatalogFile",
file=os.path.basename(metadataFile),
level=logging.ERROR)
errors.append("tpe:invalidCatalogFile")
except ArchiveFileIOError:
pass
for tag, prefixAttr, replaceAttr in (
(nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem",
"systemIdStartString", "rewritePrefix"),
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI",
"uriStartString", "rewritePrefix")): # oasis catalog
for m in rewriteTree.iter(tag=tag):
prefixValue = m.get(prefixAttr)
replaceValue = m.get(replaceAttr)
if prefixValue and replaceValue is not None:
if prefixValue not in remappings:
base = baseForElement(m)
if base:
replaceValue = os.path.join(base, replaceValue)
if replaceValue: # neither None nor ''
if not isAbsolute(replaceValue):
if not os.path.isabs(replaceValue):
replaceValue = fileBase + replaceValue
replaceValue = replaceValue.replace("/", os.sep)
_normedValue = cntlr.webCache.normalizeUrl(replaceValue)
if replaceValue.endswith(
os.sep) and not _normedValue.endswith(os.sep):
_normedValue += os.sep
remappings[prefixValue] = _normedValue
else:
cntlr.addToLog(
_("Package catalog duplicate rewrite start string %(rewriteStartString)s"
),
messageArgs={"rewriteStartString": prefixValue},
messageCode="tpe:multipleRewriteURIsForStartString",
file=os.path.basename(catalogFile),
level=logging.ERROR)
errors.append("tpe:multipleRewriteURIsForStartString")
pkg["remappings"] = remappings
entryPoints = defaultdict(list)
pkg["entryPoints"] = entryPoints
for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"):
name = None
closestLen = 0
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"):
s = (nameNode.text or "").strip()
nameLang = xmlLang(nameNode)
l = langCloseness(nameLang, currentLang)
if l > closestLen:
closestLen = l
name = s
elif closestLen == 0 and nameLang.startswith("en"):
name = s # pick english if nothing better
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(nsPrefix +
"entryPointDocument"):
epUrl = epDoc.get('href')
base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base'
) # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
epDocCount += 1
#perform prefix remappings
remappedUrl = resolvedUrl
longestPrefix = 0
for mapFrom, mapTo in remappings.items():
if remappedUrl.startswith(mapFrom):
prefixLength = len(mapFrom)
if prefixLength > longestPrefix:
_remappedUrl = remappedUrl[prefixLength:]
if not (_remappedUrl[0] in (os.sep, '/')
or mapTo[-1] in (os.sep, '/')):
_remappedUrl = mapTo + os.sep + _remappedUrl
else:
_remappedUrl = mapTo + _remappedUrl
longestPrefix = prefixLength
if longestPrefix:
remappedUrl = _remappedUrl.replace(
os.sep, "/") # always used as FileSource select
# find closest language description
closest = ''
closestLen = 0
for m in entryPointSpec.iterchildren(tag=nsPrefix + "description"):
s = (m.text or "").strip()
eltLang = xmlLang(m)
l = langCloseness(eltLang, currentLang)
if l > closestLen:
closestLen = l
closest = s
elif closestLen == 0 and eltLang.startswith("en"):
closest = s # pick english if nothing better
if not closest and name: # assign default name when none in taxonomy package
closest = name
entryPoints[name].append((remappedUrl, resolvedUrl, closest))
return pkg
def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]):
global ArchiveFileIOError
if ArchiveFileIOError is None:
from arelle.FileSource import ArchiveFileIOError
unNamedCounter = 1
txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1",
"http://xbrl.org/PWD/2014-01-15/taxonomy-package",
"http://xbrl.org/PWD/2015-01-14/taxonomy-package",
"http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package")
catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog",)
pkg = {}
currentLang = Locale.getLanguageCode()
_file = filesource.file(metadataFile)[0] # URL in zip, plain file in file system or web
try:
tree = etree.parse(_file)
except etree.XMLSyntaxError as err:
cntlr.addToLog(_("Package catalog syntax error %(error)s"),
messageArgs={"error": str(err)},
messageCode="tpe:invalidMetaDataFile",
file=os.path.basename(metadataFile),
level=logging.ERROR)
errors.append("tpe:invalidMetaDataFile")
raise # reraise error
root = tree.getroot()
ns = root.tag.partition("}")[0][1:]
nsPrefix = "{{{}}}".format(ns)
if ns in txmyPkgNSes: # package file
for eltName in ("identifier", "version", "license", "publisher", "publisherURL", "publisherCountry", "publicationDate"):
pkg[eltName] = ''
for m in root.iterchildren(tag=nsPrefix + eltName):
if eltName == "license":
pkg[eltName] = m.get("name")
else:
pkg[eltName] = (m.text or "").strip()
break # take first entry if several
for eltName in ("name", "description"):
closest = ''
closestLen = 0
for m in root.iterchildren(tag=nsPrefix + eltName):
s = (m.text or "").strip()
l = langCloseness(xmlLang(m), currentLang)
if l > closestLen:
closestLen = l
closest = s
pkg[eltName] = closest
for eltName in ("supersededTaxonomyPackages", "versioningReports"):
pkg[eltName] = []
for m in root.iterchildren(tag=nsPrefix + "supersededTaxonomyPackages"):
pkg['supersededTaxonomyPackages'] = [
r.text.strip()
for r in m.iterchildren(tag=nsPrefix + "taxonomyPackageRef")]
for m in root.iterchildren(tag=nsPrefix + "versioningReports"):
pkg['versioningReports'] = [
r.get("href")
for r in m.iterchildren(tag=nsPrefix + "versioningReport")]
# check for duplicate multi-lingual elements (among children of nodes)
langElts = defaultdict(list)
for n in root.iter(tag=nsPrefix + "*"):
for eltName in ("name", "description"):
langElts.clear()
for m in n.iterchildren(tag=nsPrefix + eltName):
langElts[xmlLang(m)].append(m)
for lang, elts in langElts.items():
if not lang:
cntlr.addToLog(_("Multi-lingual element %(element)s has no in-scope xml:lang attribute"),
messageArgs={"element": eltName},
messageCode="tpe:missingLanguageAttribute",
refs=[{"href":os.path.basename(metadataFile), "sourceLine":m.sourceline} for m in elts],
level=logging.ERROR)
errors.append("tpe:missingLanguageAttribute")
elif len(elts) > 1:
cntlr.addToLog(_("Multi-lingual element %(element)s has multiple (%(count)s) in-scope xml:lang %(lang)s elements"),
messageArgs={"element": eltName, "lang": lang, "count": len(elts)},
messageCode="tpe:duplicateLanguagesForElement",
refs=[{"href":os.path.basename(metadataFile), "sourceLine":m.sourceline} for m in elts],
level=logging.ERROR)
errors.append("tpe:duplicateLanguagesForElement")
del langElts # dereference
else: # oasis catalog, use dirname as the package name
# metadataFile may be a File object (with name) or string filename
fileName = getattr(metadataFile, 'fileName', # for FileSource named objects
getattr(metadataFile, 'name', # for io.file named objects
metadataFile)) # for string
pkg["name"] = os.path.basename(os.path.dirname(fileName))
pkg["description"] = "oasis catalog"
pkg["version"] = "(none)"
remappings = {}
rewriteTree = tree
catalogFile = metadataFile
if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package",
"http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package"):
catalogFile = metadataFile.replace('taxonomyPackage.xml','catalog.xml')
try:
rewriteTree = etree.parse(filesource.file(catalogFile)[0])
except ArchiveFileIOError:
pass
for tag, prefixAttr, replaceAttr in (
(nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"),
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog
for m in rewriteTree.iter(tag=tag):
prefixValue = m.get(prefixAttr)
replaceValue = m.get(replaceAttr)
if prefixValue and replaceValue is not None:
if prefixValue not in remappings:
base = baseForElement(m)
if base:
replaceValue = os.path.join(base, replaceValue)
if replaceValue: # neither None nor ''
if not isAbsolute(replaceValue):
if not os.path.isabs(replaceValue):
replaceValue = fileBase + replaceValue
replaceValue = replaceValue.replace("/", os.sep)
_normedValue = os.path.normpath(replaceValue)
if replaceValue.endswith(os.sep) and not _normedValue.endswith(os.sep):
_normedValue += os.sep
remappings[prefixValue] = _normedValue
else:
cntlr.addToLog(_("Package catalog duplicate rewrite start string %(rewriteStartString)s"),
messageArgs={"rewriteStartString": prefixValue},
messageCode="tpe:multipleRewriteURIsForStartString",
file=os.path.basename(catalogFile),
level=logging.ERROR)
errors.append("tpe:multipleRewriteURIsForStartString")
pkg["remappings"] = remappings
nameToUrls = {}
pkg["nameToUrls"] = nameToUrls
for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"):
name = None
closestLen = 0
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"):
s = (nameNode.text or "").strip()
l = langCloseness(xmlLang(nameNode), currentLang)
if l > closestLen:
closestLen = l
name = s
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"):
epUrl = epDoc.get('href')
base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base') # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
if epDocCount:
cntlr.addToLog(_("Skipping multiple-document entry point (not supported) %(href)s"),
messageArgs={"href": epUrl},
messageCode="arelle.packageMultipleDocumentEntryPoints",
file=os.path.basename(metadataFile),
level=logging.WARNING)
errors.append("arelle.packageMultipleDocumentEntryPoints")
continue
epDocCount += 1
#perform prefix remappings
remappedUrl = resolvedUrl
longestPrefix = 0
for mapFrom, mapTo in remappings.items():
if remappedUrl.startswith(mapFrom):
prefixLength = len(mapFrom)
if prefixLength > longestPrefix:
_remappedUrl = remappedUrl[prefixLength:]
if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')):
_remappedUrl = mapTo + os.sep + _remappedUrl
else:
_remappedUrl = mapTo + _remappedUrl
longestPrefix = prefixLength
if longestPrefix:
remappedUrl = _remappedUrl.replace(os.sep, "/") # always used as FileSource select
nameToUrls[name] = (remappedUrl, resolvedUrl)
return pkg
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False):
#TODO several directories, eg User Application Data
moduleFilename = _cntlr.webCache.getfilename(moduleURL,
reload=reload,
normalize=True,
base=_pluginBase)
if moduleFilename:
f = None
try:
# if moduleFilename is a directory containing an __ini__.py file, open that instead
if os.path.isdir(moduleFilename):
if os.path.isfile(os.path.join(moduleFilename, "__init__.py")):
moduleFilename = os.path.join(moduleFilename,
"__init__.py")
else: # impossible to get a moduleinfo from a directory without an __init__.py
return None
elif not moduleFilename.endswith(".py") and not os.path.exists(
moduleFilename) and os.path.exists(moduleFilename + ".py"):
moduleFilename += ".py" # extension module without .py suffix
moduleDir, moduleName = os.path.split(moduleFilename)
if PLUGIN_TRACE_FILE:
with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh:
fh.write("Scanning module for plug-in info: {}\n".format(
moduleFilename))
f = openFileStream(_cntlr, moduleFilename)
tree = ast.parse(f.read(), filename=moduleFilename)
constantStrings = {}
functionDefNames = set()
moduleImports = []
for item in tree.body:
if isinstance(item, ast.Assign):
attr = item.targets[0].id
if attr == "__pluginInfo__":
f.close()
moduleInfo = {"name": None}
classMethods = []
importURLs = []
for i, key in enumerate(item.value.keys):
_key = key.s
_value = item.value.values[i]
_valueType = _value.__class__.__name__
if _key == "import":
if _valueType == 'Str':
importURLs.append(_value.s)
elif _valueType in ("List", "Tuple"):
for elt in _value.elts:
importURLs.append(elt.s)
elif _valueType == 'Str':
moduleInfo[_key] = _value.s
elif _valueType == 'Name':
if _value.id in constantStrings:
moduleInfo[_key] = constantStrings[
_value.id]
elif _value.id in functionDefNames:
classMethods.append(_key)
elif _key == "imports" and _valueType in ("List",
"Tuple"):
importURLs = [elt.s for elt in _value.elts]
moduleInfo['classMethods'] = classMethods
moduleInfo["moduleURL"] = moduleURL
moduleInfo["status"] = 'enabled'
moduleInfo["fileDate"] = time.strftime(
'%Y-%m-%dT%H:%M:%S UTC',
time.gmtime(os.path.getmtime(moduleFilename)))
mergedImportURLs = []
_moduleImportsSubtree = False
for _url in importURLs:
if _url.startswith("module_import"):
for moduleImport in moduleImports:
mergedImportURLs.append(moduleImport +
".py")
if _url == "module_import_subtree":
_moduleImportsSubtree = True
elif _url == "module_subtree":
for _dir in os.listdir(moduleDir):
_subtreeModule = os.path.join(
moduleDir, _dir)
if os.path.isdir(
_subtreeModule
) and _dir != "__pycache__":
mergedImportURLs.append(_subtreeModule)
else:
mergedImportURLs.append(_url)
if parentImportsSubtree and not _moduleImportsSubtree:
_moduleImportsSubtree = True
for moduleImport in moduleImports:
mergedImportURLs.append(moduleImport + ".py")
imports = []
for _url in mergedImportURLs:
if isAbsolute(_url) or os.path.isabs(_url):
_importURL = _url # URL is absolute http or local file system
else: # check if exists relative to this module's directory
_importURL = os.path.join(
os.path.dirname(moduleURL),
os.path.normpath(_url))
if not os.path.exists(
_importURL
): # not relative to this plugin, assume standard plugin base
_importURL = _url # moduleModuleInfo adjusts relative URL to plugin base
_importModuleInfo = moduleModuleInfo(
_importURL, reload, _moduleImportsSubtree)
if _importModuleInfo:
_importModuleInfo["isImported"] = True
imports.append(_importModuleInfo)
moduleInfo["imports"] = imports
return moduleInfo
elif isinstance(
item.value, ast.Str
): # possible constant used in plugininfo, such as VERSION
for assignmentName in item.targets:
constantStrings[assignmentName.id] = item.value.s
elif isinstance(item, ast.ImportFrom):
if item.level == 1: # starts with .
if item.module is None: # from . import module1, module2, ...
for importee in item.names:
if importee.name == '*': #import all submodules
for _file in os.listdir(moduleDir):
if _file != moduleFile and os.path.isfile(
_file) and _file.endswith(
".py"):
moduleImports.append(_file)
elif (os.path.isfile(
os.path.join(moduleDir,
importee.name + ".py"))
and importee.name not in moduleImports):
moduleImports.append(importee.name)
else:
modulePkgs = item.module.split('.')
modulePath = os.path.join(*modulePkgs)
if (os.path.isfile(
os.path.join(moduleDir, modulePath) +
".py")
and modulePath not in moduleImports):
moduleImports.append(modulePath)
for importee in item.names:
_importeePfxName = os.path.join(
modulePath, importee.name)
if (os.path.isfile(
os.path.join(moduleDir,
_importeePfxName) + ".py")
and _importeePfxName
not in moduleImports):
moduleImports.append(_importeePfxName)
elif isinstance(item, ast.FunctionDef
): # possible functionDef used in plugininfo
functionDefNames.add(item.name)
if PLUGIN_TRACE_FILE:
with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh:
fh.write("Successful module plug-in info: " +
moduleFilename + '\n')
except Exception as err:
_msg = _(
"Exception obtaining plug-in module info: {moduleFilename}\n{error}\n{traceback}"
).format(error=err,
moduleFilename=moduleFilename,
traceback=traceback.format_tb(sys.exc_info()[2]))
if PLUGIN_TRACE_FILE:
with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh:
fh.write(_msg + '\n')
else:
print(_msg, file=sys.stderr)
if f:
f.close()
return None
def parsePackage(cntlr, filesource, metadataFile, fileBase):
global ArchiveFileIOError
if ArchiveFileIOError is None:
from arelle.FileSource import ArchiveFileIOError
unNamedCounter = 1
txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1",
"http://xbrl.org/PWD/2014-01-15/taxonomy-package",
"http://xbrl.org/PWD/2015-01-14/taxonomy-package")
catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog",)
pkg = {}
currentLang = Locale.getLanguageCode()
_file = filesource.file(metadataFile)[0] # URL in zip, plain file in file system or web
tree = etree.parse(_file)
root = tree.getroot()
ns = root.tag.partition("}")[0][1:]
nsPrefix = "{{{}}}".format(ns)
if ns in txmyPkgNSes: # package file
for eltName in ("name", "description", "version"):
pkg[eltName] = ''
for m in root.iterchildren(tag=nsPrefix + eltName):
pkg[eltName] = m.text.strip()
break # take first entry if several
else: # oasis catalog, use dirname as the package name
# metadataFile may be a File object (with name) or string filename
fileName = getattr(metadataFile, 'fileName', # for FileSource named objects
getattr(metadataFile, 'name', # for io.file named objects
metadataFile)) # for string
pkg["name"] = os.path.basename(os.path.dirname(fileName))
pkg["description"] = "oasis catalog"
pkg["version"] = "(none)"
remappings = {}
rewriteTree = tree
catalogFile = metadataFile
if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package",):
catalogFile = metadataFile.replace('taxonomyPackage.xml','catalog.xml')
try:
rewriteTree = etree.parse(filesource.file(catalogFile)[0])
except ArchiveFileIOError:
pass
for tag, prefixAttr, replaceAttr in (
(nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"),
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog
for m in rewriteTree.iter(tag=tag):
prefixValue = m.get(prefixAttr)
replaceValue = m.get(replaceAttr)
if prefixValue and replaceValue is not None:
if prefixValue not in remappings:
base = baseForElement(m)
if base:
replaceValue = os.path.join(base, replaceValue)
if replaceValue: # neither None nor ''
if not isAbsolute(replaceValue):
if not os.path.isabs(replaceValue):
replaceValue = fileBase + replaceValue
replaceValue = replaceValue.replace("/", os.sep)
_normedValue = os.path.normpath(replaceValue)
if replaceValue.endswith(os.sep) and not _normedValue.endswith(os.sep):
_normedValue += os.sep
remappings[prefixValue] = _normedValue
else:
cntlr.addToLog(_("Package catalog duplicate rewrite start string %(rewriteStartString)s"),
messageArgs={"rewriteStartString": prefixValue},
messageCode="arelle.catalogDuplicateRewrite",
file=os.path.basename(catalogFile),
level=logging.ERROR)
pkg["remappings"] = remappings
nameToUrls = {}
pkg["nameToUrls"] = nameToUrls
for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"):
name = None
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"):
xmlLang = nameNode.get('{http://www.w3.org/XML/1998/namespace}lang')
if name is None or not xmlLang or currentLang == xmlLang:
name = nameNode.text
if currentLang == xmlLang: # most prefer one with the current locale's language
break
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"):
epUrl = epDoc.get('href')
base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base') # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
if epDocCount:
cntlr.addToLog(_("Skipping multiple-document entry point (not supported) %(href)s"),
messageArgs={"href": epUrl},
messageCode="arelle.packageMultipleDocumentEntryPoints",
file=os.path.basename(metadataFile),
level=logging.WARNING)
continue
epDocCount += 1
#perform prefix remappings
remappedUrl = resolvedUrl
longestPrefix = 0
for mapFrom, mapTo in remappings.items():
if remappedUrl.startswith(mapFrom):
prefixLength = len(mapFrom)
if prefixLength > longestPrefix:
_remappedUrl = remappedUrl[prefixLength:]
if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')):
_remappedUrl = mapTo + os.sep + _remappedUrl
else:
_remappedUrl = mapTo + _remappedUrl
longestPrefix = prefixLength
if longestPrefix:
remappedUrl = _remappedUrl.replace(os.sep, "/") # always used as FileSource select
nameToUrls[name] = (remappedUrl, resolvedUrl)
return pkg
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False):
#TODO several directories, eg User Application Data
moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase)
if moduleFilename:
f = None
try:
# if moduleFilename is a directory containing an __ini__.py file, open that instead
if os.path.isdir(moduleFilename) and os.path.isfile(os.path.join(moduleFilename, "__init__.py")):
moduleFilename = os.path.join(moduleFilename, "__init__.py")
moduleDir = os.path.dirname(moduleFilename)
if PLUGIN_TRACE_FILE:
with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh:
fh.write("Scanning module for plug-in info: {}\n".format(moduleFilename))
f = openFileStream(_cntlr, moduleFilename)
tree = ast.parse(f.read(), filename=moduleFilename)
moduleImports = []
for item in tree.body:
if isinstance(item, ast.Assign):
attr = item.targets[0].id
if attr == "__pluginInfo__":
f.close()
moduleInfo = {"name":None}
classMethods = []
importURLs = []
for i, key in enumerate(item.value.keys):
_key = key.s
_value = item.value.values[i]
_valueType = _value.__class__.__name__
if _key == "import":
if _valueType == 'Str':
importURLs.append(_value.s)
elif _valueType in ("List", "Tuple"):
for elt in _value.elts:
importURLs.append(elt.s)
elif _valueType == 'Str':
moduleInfo[_key] = _value.s
elif _valueType == 'Name':
classMethods.append(_key)
elif _key == "imports" and _valueType in ("List", "Tuple"):
importURLs = [elt.s for elt in _value.elts]
moduleInfo['classMethods'] = classMethods
moduleInfo["moduleURL"] = moduleURL
moduleInfo["status"] = 'enabled'
moduleInfo["fileDate"] = time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename)))
mergedImportURLs = []
_moduleImportsSubtree = False
for _url in importURLs:
if _url.startswith("module_import"):
for moduleImport in moduleImports:
mergedImportURLs.append(moduleImport + ".py")
if _url == "module_import_subtree":
_moduleImportsSubtree = True
else:
mergedImportURLs.append(_url)
if parentImportsSubtree and not _moduleImportsSubtree:
_moduleImportsSubtree = True
for moduleImport in moduleImports:
mergedImportURLs.append(moduleImport + ".py")
imports = []
for _url in mergedImportURLs:
if isAbsolute(_url) or os.path.isabs(_url):
_importURL = _url # URL is absolute http or local file system
else: # check if exists relative to this module's directory
_importURL = os.path.join(os.path.dirname(moduleURL), _url)
if not os.path.exists(_importURL): # not relative to this plugin, assume standard plugin base
_importURL = os.path.join(_pluginBase, _url)
_importModuleInfo = moduleModuleInfo(_importURL, reload, _moduleImportsSubtree)
if _importModuleInfo:
_importModuleInfo["isImported"] = True
imports.append(_importModuleInfo)
moduleInfo["imports"] = imports
return moduleInfo
elif isinstance(item, ast.ImportFrom):
if item.level == 1: # starts with .
if item.module is None: # from . import module1, module2, ...
for importee in item.names:
if (os.path.isfile(os.path.join(moduleDir, importee.name + ".py"))
and importee.name not in moduleImports):
moduleImports.append(importee.name)
else:
modulePkgs = item.module.split('.')
modulePath = os.path.join(*modulePkgs)
if (os.path.isfile(os.path.join(moduleDir, modulePath) + ".py")
and modulePath not in moduleImports):
moduleImports.append(modulePath)
for importee in item.names:
_importeePfxName = os.path.join(modulePath, importee.name)
if (os.path.isfile(os.path.join(moduleDir, _importeePfxName) + ".py")
and _importeePfxName not in moduleImports):
moduleImports.append(_importeePfxName)
if PLUGIN_TRACE_FILE:
with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh:
fh.write("Successful module plug-in info: " + moduleFilename + '\n')
except Exception as err:
_msg = _("Exception obtaining plug-in module info: {error}\n{traceback}").format(
error=err, traceback=traceback.format_tb(sys.exc_info()[2]))
if PLUGIN_TRACE_FILE:
with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh:
fh.write(_msg + '\n')
else:
print(_msg, file=sys.stderr)
if f:
f.close()
return None
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False):
#TODO several directories, eg User Application Data
moduleFilename = _cntlr.webCache.getfilename(moduleURL,
reload=reload,
normalize=True,
base=_pluginBase)
if moduleFilename:
f = None
try:
# if moduleFilename is a directory containing an __ini__.py file, open that instead
if os.path.isdir(moduleFilename) and os.path.isfile(
os.path.join(moduleFilename, "__init__.py")):
moduleFilename = os.path.join(moduleFilename, "__init__.py")
moduleDir = os.path.dirname(moduleFilename)
f = openFileStream(_cntlr, moduleFilename)
tree = ast.parse(f.read(), filename=moduleFilename)
moduleImports = []
for item in tree.body:
if isinstance(item, ast.Assign):
attr = item.targets[0].id
if attr == "__pluginInfo__":
f.close()
moduleInfo = {"name": None}
classMethods = []
importURLs = []
for i, key in enumerate(item.value.keys):
_key = key.s
_value = item.value.values[i]
_valueType = _value.__class__.__name__
if _key == "import":
if _valueType == 'Str':
importURLs.append(_value.s)
elif _valueType in ("List", "Tuple"):
for elt in _value.elts:
importURLs.append(elt.s)
elif _valueType == 'Str':
moduleInfo[_key] = _value.s
elif _valueType == 'Name':
classMethods.append(_key)
elif _key == "imports" and _valueType in ("List",
"Tuple"):
importURLs = [elt.s for elt in _value.elts]
moduleInfo['classMethods'] = classMethods
moduleInfo["moduleURL"] = moduleURL
moduleInfo["status"] = 'enabled'
moduleInfo["fileDate"] = time.strftime(
'%Y-%m-%dT%H:%M:%S UTC',
time.gmtime(os.path.getmtime(moduleFilename)))
mergedImportURLs = []
_moduleImportsSubtree = False
for _url in importURLs:
if _url.startswith("module_import"):
for moduleImport in moduleImports:
mergedImportURLs.append(moduleImport +
".py")
if _url == "module_import_subtree":
_moduleImportsSubtree = True
else:
mergedImportURLs.append(_url)
if parentImportsSubtree and not _moduleImportsSubtree:
_moduleImportsSubtree = True
for moduleImport in moduleImports:
mergedImportURLs.append(moduleImport + ".py")
imports = []
for _url in mergedImportURLs:
_importURL = (
_url if isAbsolute(_url)
or os.path.isabs(_url) else os.path.join(
os.path.dirname(moduleURL), _url))
_importModuleInfo = moduleModuleInfo(
_importURL, reload, _moduleImportsSubtree)
if _importModuleInfo:
_importModuleInfo["isImported"] = True
imports.append(_importModuleInfo)
moduleInfo["imports"] = imports
return moduleInfo
elif isinstance(item, ast.ImportFrom):
if item.level == 1: # starts with .
if item.module is None: # from . import module1, module2, ...
for importee in item.names:
if (os.path.isfile(
os.path.join(moduleDir,
importee.name + ".py"))
and importee.name
not in moduleImports):
moduleImports.append(importee.name)
else:
modulePkgs = item.module.split('.')
modulePath = os.path.join(*modulePkgs)
if (os.path.isfile(
os.path.join(moduleDir, modulePath) +
".py")
and modulePath not in moduleImports):
moduleImports.append(modulePath)
for importee in item.names:
_importeePfxName = os.path.join(
modulePath, importee.name)
if (os.path.isfile(
os.path.join(moduleDir,
_importeePfxName) + ".py")
and _importeePfxName
not in moduleImports):
moduleImports.append(_importeePfxName)
except EnvironmentError:
pass
if f:
f.close()
return None
def parsePackage(cntlr, filesource, metadataFile, fileBase):
global ArchiveFileIOError
if ArchiveFileIOError is None:
from arelle.FileSource import ArchiveFileIOError
unNamedCounter = 1
txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1",
"http://xbrl.org/PWD/2014-01-15/taxonomy-package",
"http://xbrl.org/PWD/2015-01-14/taxonomy-package")
catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog", )
pkg = {}
currentLang = Locale.getLanguageCode()
_file = filesource.file(metadataFile)[
0] # URL in zip, plain file in file system or web
tree = etree.parse(_file)
root = tree.getroot()
ns = root.tag.partition("}")[0][1:]
nsPrefix = "{{{}}}".format(ns)
if ns in txmyPkgNSes: # package file
for eltName in ("name", "description", "version"):
pkg[eltName] = ''
for m in root.iterchildren(tag=nsPrefix + eltName):
pkg[eltName] = m.text.strip()
break # take first entry if several
else: # oasis catalog, use dirname as the package name
# metadataFile may be a File object (with name) or string filename
fileName = getattr(
metadataFile,
'fileName', # for FileSource named objects
getattr(
metadataFile,
'name', # for io.file named objects
metadataFile)) # for string
pkg["name"] = os.path.basename(os.path.dirname(fileName))
pkg["description"] = "oasis catalog"
pkg["version"] = "(none)"
remappings = {}
rewriteTree = tree
catalogFile = metadataFile
if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package", ):
catalogFile = metadataFile.replace('taxonomyPackage.xml',
'catalog.xml')
try:
rewriteTree = etree.parse(filesource.file(catalogFile)[0])
except ArchiveFileIOError:
pass
for tag, prefixAttr, replaceAttr in (
(nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem",
"systemIdStartString", "rewritePrefix"),
("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI",
"uriStartString", "rewritePrefix")): # oasis catalog
for m in rewriteTree.iter(tag=tag):
prefixValue = m.get(prefixAttr)
replaceValue = m.get(replaceAttr)
if prefixValue and replaceValue is not None:
if prefixValue not in remappings:
base = baseForElement(m)
if base:
replaceValue = os.path.join(base, replaceValue)
if replaceValue: # neither None nor ''
if not isAbsolute(replaceValue):
if not os.path.isabs(replaceValue):
replaceValue = fileBase + replaceValue
replaceValue = replaceValue.replace("/", os.sep)
_normedValue = os.path.normpath(replaceValue)
if replaceValue.endswith(
os.sep) and not _normedValue.endswith(os.sep):
_normedValue += os.sep
remappings[prefixValue] = _normedValue
else:
cntlr.addToLog(_(
"Package catalog duplicate rewrite start string %(rewriteStartString)s"
),
messageArgs={
"rewriteStartString": prefixValue
},
messageCode="arelle.catalogDuplicateRewrite",
file=os.path.basename(catalogFile),
level=logging.ERROR)
pkg["remappings"] = remappings
nameToUrls = {}
pkg["nameToUrls"] = nameToUrls
for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"):
name = None
# find closest match name node given xml:lang match to current language or no xml:lang
for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"):
xmlLang = nameNode.get(
'{http://www.w3.org/XML/1998/namespace}lang')
if name is None or not xmlLang or currentLang == xmlLang:
name = nameNode.text
if currentLang == xmlLang: # most prefer one with the current locale's language
break
if not name:
name = _("<unnamed {0}>").format(unNamedCounter)
unNamedCounter += 1
epDocCount = 0
for epDoc in entryPointSpec.iterchildren(nsPrefix +
"entryPointDocument"):
epUrl = epDoc.get('href')
base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base'
) # cope with xml:base
if base:
resolvedUrl = urljoin(base, epUrl)
else:
resolvedUrl = epUrl
if epDocCount:
cntlr.addToLog(
_("Skipping multiple-document entry point (not supported) %(href)s"
),
messageArgs={"href": epUrl},
messageCode="arelle.packageMultipleDocumentEntryPoints",
file=os.path.basename(metadataFile),
level=logging.WARNING)
continue
epDocCount += 1
#perform prefix remappings
remappedUrl = resolvedUrl
longestPrefix = 0
for mapFrom, mapTo in remappings.items():
if remappedUrl.startswith(mapFrom):
prefixLength = len(mapFrom)
if prefixLength > longestPrefix:
_remappedUrl = remappedUrl[prefixLength:]
if not (_remappedUrl[0] in (os.sep, '/')
or mapTo[-1] in (os.sep, '/')):
_remappedUrl = mapTo + os.sep + _remappedUrl
else:
_remappedUrl = mapTo + _remappedUrl
longestPrefix = prefixLength
if longestPrefix:
remappedUrl = _remappedUrl.replace(
os.sep, "/") # always used as FileSource select
nameToUrls[name] = (remappedUrl, resolvedUrl)
return pkg
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False):
#TODO several directories, eg User Application Data
moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase)
if moduleFilename:
f = None
try:
# if moduleFilename is a directory containing an __ini__.py file, open that instead
if os.path.isdir(moduleFilename) and os.path.isfile(os.path.join(moduleFilename, "__init__.py")):
moduleFilename = os.path.join(moduleFilename, "__init__.py")
moduleDir = os.path.dirname(moduleFilename)
f = openFileStream(_cntlr, moduleFilename)
tree = ast.parse(f.read(), filename=moduleFilename)
moduleImports = []
for item in tree.body:
if isinstance(item, ast.Assign):
attr = item.targets[0].id
if attr == "__pluginInfo__":
f.close()
moduleInfo = {"name":None}
classMethods = []
importURLs = []
for i, key in enumerate(item.value.keys):
_key = key.s
_value = item.value.values[i]
_valueType = _value.__class__.__name__
if _key == "import":
if _valueType == 'Str':
importURLs.append(_value.s)
elif _valueType in ("List", "Tuple"):
for elt in _value.elts:
importURLs.append(elt.s)
elif _valueType == 'Str':
moduleInfo[_key] = _value.s
elif _valueType == 'Name':
classMethods.append(_key)
elif _key == "imports" and _valueType in ("List", "Tuple"):
importURLs = [elt.s for elt in _value.elts]
moduleInfo['classMethods'] = classMethods
moduleInfo["moduleURL"] = moduleURL
moduleInfo["status"] = 'enabled'
moduleInfo["fileDate"] = time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename)))
mergedImportURLs = []
_moduleImportsSubtree = False
for _url in importURLs:
if _url.startswith("module_import"):
for moduleImport in moduleImports:
mergedImportURLs.append(moduleImport + ".py")
if _url == "module_import_subtree":
_moduleImportsSubtree = True
else:
mergedImportURLs.append(_url)
if parentImportsSubtree and not _moduleImportsSubtree:
_moduleImportsSubtree = True
for moduleImport in moduleImports:
mergedImportURLs.append(moduleImport + ".py")
imports = []
for _url in mergedImportURLs:
_importURL = (_url if isAbsolute(_url) or os.path.isabs(_url)
else os.path.join(os.path.dirname(moduleURL), _url))
_importModuleInfo = moduleModuleInfo(_importURL, reload, _moduleImportsSubtree)
if _importModuleInfo:
_importModuleInfo["isImported"] = True
imports.append(_importModuleInfo)
moduleInfo["imports"] = imports
return moduleInfo
elif isinstance(item, ast.ImportFrom):
if item.level == 1: # starts with .
if item.module is None: # from . import module1, module2, ...
for importee in item.names:
if (os.path.isfile(os.path.join(moduleDir, importee.name + ".py"))
and importee.name not in moduleImports):
moduleImports.append(importee.name)
else:
modulePkgs = item.module.split('.')
modulePath = os.path.join(*modulePkgs)
if (os.path.isfile(os.path.join(moduleDir, modulePath) + ".py")
and modulePath not in moduleImports):
moduleImports.append(modulePath)
for importee in item.names:
_importeePfxName = os.path.join(modulePath, importee.name)
if (os.path.isfile(os.path.join(moduleDir, _importeePfxName) + ".py")
and _importeePfxName not in moduleImports):
moduleImports.append(_importeePfxName)
except EnvironmentError:
pass
if f:
f.close()
return None
