def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]): global ArchiveFileIOError if ArchiveFileIOError is None: from arelle.FileSource import ArchiveFileIOError unNamedCounter = 1 txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1", "http://xbrl.org/PWD/2014-01-15/taxonomy-package", "http://xbrl.org/PWD/2015-01-14/taxonomy-package", "http://xbrl.org/PR/2015-12-09/taxonomy-package", "http://xbrl.org/2016/taxonomy-package", "http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package") catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog", ) pkg = {} currentLang = Locale.getLanguageCode() _file = filesource.file(metadataFile)[ 0] # URL in zip, plain file in file system or web parser = lxmlResolvingParser(cntlr) try: tree = etree.parse(_file, parser=parser) # schema validate tp xml xsdTree = etree.parse(TP_XSD, parser=parser) etree.XMLSchema(xsdTree).assertValid(tree) except (etree.XMLSyntaxError, etree.DocumentInvalid) as err: cntlr.addToLog(_("Taxonomy package file syntax error %(error)s"), messageArgs={"error": str(err)}, messageCode="tpe:invalidMetaDataFile", file=os.path.basename(metadataFile), level=logging.ERROR) errors.append("tpe:invalidMetaDataFile") return pkg root = tree.getroot() ns = root.tag.partition("}")[0][1:] nsPrefix = "{{{}}}".format(ns) if ns in txmyPkgNSes: # package file for eltName in ("identifier", "version", "license", "publisher", "publisherURL", "publisherCountry", "publicationDate"): pkg[eltName] = '' for m in root.iterchildren(tag=nsPrefix + eltName): if eltName == "license": pkg[eltName] = m.get("name") else: pkg[eltName] = (m.text or "").strip() break # take first entry if several for eltName in ("name", "description"): closest = '' closestLen = 0 for m in root.iterchildren(tag=nsPrefix + eltName): s = (m.text or "").strip() eltLang = xmlLang(m) l = langCloseness(eltLang, currentLang) if l > closestLen: closestLen = l closest = s elif closestLen == 0 and eltLang.startswith("en"): closest = s # pick english if nothing better if not closest and eltName == "name": # assign default name when none in taxonomy package closest = os.path.splitext(os.path.basename( filesource.baseurl))[0] pkg[eltName] = closest for eltName in ("supersededTaxonomyPackages", "versioningReports"): pkg[eltName] = [] for m in root.iterchildren(tag=nsPrefix + "supersededTaxonomyPackages"): pkg['supersededTaxonomyPackages'] = [ r.text.strip() for r in m.iterchildren(tag=nsPrefix + "taxonomyPackageRef") ] for m in root.iterchildren(tag=nsPrefix + "versioningReports"): pkg['versioningReports'] = [ r.get("href") for r in m.iterchildren(tag=nsPrefix + "versioningReport") ] # check for duplicate multi-lingual elements (among children of nodes) langElts = defaultdict(list) for n in root.iter(tag=nsPrefix + "*"): for eltName in ("name", "description", "publisher"): langElts.clear() for m in n.iterchildren(tag=nsPrefix + eltName): langElts[xmlLang(m)].append(m) for lang, elts in langElts.items(): if not lang: cntlr.addToLog( _("Multi-lingual element %(element)s has no in-scope xml:lang attribute" ), messageArgs={"element": eltName}, messageCode="tpe:missingLanguageAttribute", refs=[{ "href": os.path.basename(metadataFile), "sourceLine": m.sourceline } for m in elts], level=logging.ERROR) errors.append("tpe:missingLanguageAttribute") elif len(elts) > 1: cntlr.addToLog( _("Multi-lingual element %(element)s has multiple (%(count)s) in-scope xml:lang %(lang)s elements" ), messageArgs={ "element": eltName, "lang": lang, "count": len(elts) }, messageCode="tpe:duplicateLanguagesForElement", refs=[{ "href": os.path.basename(metadataFile), "sourceLine": m.sourceline } for m in elts], level=logging.ERROR) errors.append("tpe:duplicateLanguagesForElement") del langElts # dereference else: # oasis catalog, use dirname as the package name # metadataFile may be a File object (with name) or string filename fileName = getattr( metadataFile, 'fileName', # for FileSource named objects getattr( metadataFile, 'name', # for io.file named objects metadataFile)) # for string pkg["name"] = os.path.basename(os.path.dirname(fileName)) pkg["description"] = "oasis catalog" pkg["version"] = "(none)" remappings = {} rewriteTree = tree catalogFile = metadataFile if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package", "http://xbrl.org/PR/2015-12-09/taxonomy-package", "http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package", "http://xbrl.org/2016/taxonomy-package", "http://xbrl.org/REC/2016-04-19/taxonomy-package"): catalogFile = metadataFile.replace('taxonomyPackage.xml', 'catalog.xml') try: rewriteTree = etree.parse(filesource.file(catalogFile)[0], parser=parser) # schema validate tp xml xsdTree = etree.parse(CAT_XSD, parser=parser) etree.XMLSchema(xsdTree).assertValid(rewriteTree) except (etree.XMLSyntaxError, etree.DocumentInvalid) as err: cntlr.addToLog(_("Catalog file syntax error %(error)s"), messageArgs={"error": str(err)}, messageCode="tpe:invalidCatalogFile", file=os.path.basename(metadataFile), level=logging.ERROR) errors.append("tpe:invalidCatalogFile") except ArchiveFileIOError: pass for tag, prefixAttr, replaceAttr in ( (nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"), ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog for m in rewriteTree.iter(tag=tag): prefixValue = m.get(prefixAttr) replaceValue = m.get(replaceAttr) if prefixValue and replaceValue is not None: if prefixValue not in remappings: base = baseForElement(m) if base: replaceValue = os.path.join(base, replaceValue) if replaceValue: # neither None nor '' if not isAbsolute(replaceValue): if not os.path.isabs(replaceValue): replaceValue = fileBase + replaceValue replaceValue = replaceValue.replace("/", os.sep) _normedValue = cntlr.webCache.normalizeUrl(replaceValue) if replaceValue.endswith( os.sep) and not _normedValue.endswith(os.sep): _normedValue += os.sep remappings[prefixValue] = _normedValue else: cntlr.addToLog( _("Package catalog duplicate rewrite start string %(rewriteStartString)s" ), messageArgs={"rewriteStartString": prefixValue}, messageCode="tpe:multipleRewriteURIsForStartString", file=os.path.basename(catalogFile), level=logging.ERROR) errors.append("tpe:multipleRewriteURIsForStartString") pkg["remappings"] = remappings entryPoints = defaultdict(list) pkg["entryPoints"] = entryPoints for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"): name = None closestLen = 0 # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"): s = (nameNode.text or "").strip() nameLang = xmlLang(nameNode) l = langCloseness(nameLang, currentLang) if l > closestLen: closestLen = l name = s elif closestLen == 0 and nameLang.startswith("en"): name = s # pick english if nothing better if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"): epUrl = epDoc.get('href') base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base' ) # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl epDocCount += 1 #perform prefix remappings remappedUrl = resolvedUrl longestPrefix = 0 for mapFrom, mapTo in remappings.items(): if remappedUrl.startswith(mapFrom): prefixLength = len(mapFrom) if prefixLength > longestPrefix: _remappedUrl = remappedUrl[prefixLength:] if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')): _remappedUrl = mapTo + os.sep + _remappedUrl else: _remappedUrl = mapTo + _remappedUrl longestPrefix = prefixLength if longestPrefix: remappedUrl = _remappedUrl.replace( os.sep, "/") # always used as FileSource select # find closest language description closest = '' closestLen = 0 for m in entryPointSpec.iterchildren(tag=nsPrefix + "description"): s = (m.text or "").strip() eltLang = xmlLang(m) l = langCloseness(eltLang, currentLang) if l > closestLen: closestLen = l closest = s elif closestLen == 0 and eltLang.startswith("en"): closest = s # pick english if nothing better if not closest and name: # assign default name when none in taxonomy package closest = name entryPoints[name].append((remappedUrl, resolvedUrl, closest)) return pkg
def parsePackage(cntlr, filesource, metadataFile, fileBase, errors=[]): global ArchiveFileIOError if ArchiveFileIOError is None: from arelle.FileSource import ArchiveFileIOError unNamedCounter = 1 txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1", "http://xbrl.org/PWD/2014-01-15/taxonomy-package", "http://xbrl.org/PWD/2015-01-14/taxonomy-package", "http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package") catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog",) pkg = {} currentLang = Locale.getLanguageCode() _file = filesource.file(metadataFile)[0] # URL in zip, plain file in file system or web try: tree = etree.parse(_file) except etree.XMLSyntaxError as err: cntlr.addToLog(_("Package catalog syntax error %(error)s"), messageArgs={"error": str(err)}, messageCode="tpe:invalidMetaDataFile", file=os.path.basename(metadataFile), level=logging.ERROR) errors.append("tpe:invalidMetaDataFile") raise # reraise error root = tree.getroot() ns = root.tag.partition("}")[0][1:] nsPrefix = "{{{}}}".format(ns) if ns in txmyPkgNSes: # package file for eltName in ("identifier", "version", "license", "publisher", "publisherURL", "publisherCountry", "publicationDate"): pkg[eltName] = '' for m in root.iterchildren(tag=nsPrefix + eltName): if eltName == "license": pkg[eltName] = m.get("name") else: pkg[eltName] = (m.text or "").strip() break # take first entry if several for eltName in ("name", "description"): closest = '' closestLen = 0 for m in root.iterchildren(tag=nsPrefix + eltName): s = (m.text or "").strip() l = langCloseness(xmlLang(m), currentLang) if l > closestLen: closestLen = l closest = s pkg[eltName] = closest for eltName in ("supersededTaxonomyPackages", "versioningReports"): pkg[eltName] = [] for m in root.iterchildren(tag=nsPrefix + "supersededTaxonomyPackages"): pkg['supersededTaxonomyPackages'] = [ r.text.strip() for r in m.iterchildren(tag=nsPrefix + "taxonomyPackageRef")] for m in root.iterchildren(tag=nsPrefix + "versioningReports"): pkg['versioningReports'] = [ r.get("href") for r in m.iterchildren(tag=nsPrefix + "versioningReport")] # check for duplicate multi-lingual elements (among children of nodes) langElts = defaultdict(list) for n in root.iter(tag=nsPrefix + "*"): for eltName in ("name", "description"): langElts.clear() for m in n.iterchildren(tag=nsPrefix + eltName): langElts[xmlLang(m)].append(m) for lang, elts in langElts.items(): if not lang: cntlr.addToLog(_("Multi-lingual element %(element)s has no in-scope xml:lang attribute"), messageArgs={"element": eltName}, messageCode="tpe:missingLanguageAttribute", refs=[{"href":os.path.basename(metadataFile), "sourceLine":m.sourceline} for m in elts], level=logging.ERROR) errors.append("tpe:missingLanguageAttribute") elif len(elts) > 1: cntlr.addToLog(_("Multi-lingual element %(element)s has multiple (%(count)s) in-scope xml:lang %(lang)s elements"), messageArgs={"element": eltName, "lang": lang, "count": len(elts)}, messageCode="tpe:duplicateLanguagesForElement", refs=[{"href":os.path.basename(metadataFile), "sourceLine":m.sourceline} for m in elts], level=logging.ERROR) errors.append("tpe:duplicateLanguagesForElement") del langElts # dereference else: # oasis catalog, use dirname as the package name # metadataFile may be a File object (with name) or string filename fileName = getattr(metadataFile, 'fileName', # for FileSource named objects getattr(metadataFile, 'name', # for io.file named objects metadataFile)) # for string pkg["name"] = os.path.basename(os.path.dirname(fileName)) pkg["description"] = "oasis catalog" pkg["version"] = "(none)" remappings = {} rewriteTree = tree catalogFile = metadataFile if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package", "http://xbrl.org/WGWD/YYYY-MM-DD/taxonomy-package"): catalogFile = metadataFile.replace('taxonomyPackage.xml','catalog.xml') try: rewriteTree = etree.parse(filesource.file(catalogFile)[0]) except ArchiveFileIOError: pass for tag, prefixAttr, replaceAttr in ( (nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"), ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog for m in rewriteTree.iter(tag=tag): prefixValue = m.get(prefixAttr) replaceValue = m.get(replaceAttr) if prefixValue and replaceValue is not None: if prefixValue not in remappings: base = baseForElement(m) if base: replaceValue = os.path.join(base, replaceValue) if replaceValue: # neither None nor '' if not isAbsolute(replaceValue): if not os.path.isabs(replaceValue): replaceValue = fileBase + replaceValue replaceValue = replaceValue.replace("/", os.sep) _normedValue = os.path.normpath(replaceValue) if replaceValue.endswith(os.sep) and not _normedValue.endswith(os.sep): _normedValue += os.sep remappings[prefixValue] = _normedValue else: cntlr.addToLog(_("Package catalog duplicate rewrite start string %(rewriteStartString)s"), messageArgs={"rewriteStartString": prefixValue}, messageCode="tpe:multipleRewriteURIsForStartString", file=os.path.basename(catalogFile), level=logging.ERROR) errors.append("tpe:multipleRewriteURIsForStartString") pkg["remappings"] = remappings nameToUrls = {} pkg["nameToUrls"] = nameToUrls for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"): name = None closestLen = 0 # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"): s = (nameNode.text or "").strip() l = langCloseness(xmlLang(nameNode), currentLang) if l > closestLen: closestLen = l name = s if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"): epUrl = epDoc.get('href') base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base') # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl if epDocCount: cntlr.addToLog(_("Skipping multiple-document entry point (not supported) %(href)s"), messageArgs={"href": epUrl}, messageCode="arelle.packageMultipleDocumentEntryPoints", file=os.path.basename(metadataFile), level=logging.WARNING) errors.append("arelle.packageMultipleDocumentEntryPoints") continue epDocCount += 1 #perform prefix remappings remappedUrl = resolvedUrl longestPrefix = 0 for mapFrom, mapTo in remappings.items(): if remappedUrl.startswith(mapFrom): prefixLength = len(mapFrom) if prefixLength > longestPrefix: _remappedUrl = remappedUrl[prefixLength:] if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')): _remappedUrl = mapTo + os.sep + _remappedUrl else: _remappedUrl = mapTo + _remappedUrl longestPrefix = prefixLength if longestPrefix: remappedUrl = _remappedUrl.replace(os.sep, "/") # always used as FileSource select nameToUrls[name] = (remappedUrl, resolvedUrl) return pkg
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False): #TODO several directories, eg User Application Data moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase) if moduleFilename: f = None try: # if moduleFilename is a directory containing an __ini__.py file, open that instead if os.path.isdir(moduleFilename): if os.path.isfile(os.path.join(moduleFilename, "__init__.py")): moduleFilename = os.path.join(moduleFilename, "__init__.py") else: # impossible to get a moduleinfo from a directory without an __init__.py return None elif not moduleFilename.endswith(".py") and not os.path.exists( moduleFilename) and os.path.exists(moduleFilename + ".py"): moduleFilename += ".py" # extension module without .py suffix moduleDir, moduleName = os.path.split(moduleFilename) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write("Scanning module for plug-in info: {}\n".format( moduleFilename)) f = openFileStream(_cntlr, moduleFilename) tree = ast.parse(f.read(), filename=moduleFilename) constantStrings = {} functionDefNames = set() moduleImports = [] for item in tree.body: if isinstance(item, ast.Assign): attr = item.targets[0].id if attr == "__pluginInfo__": f.close() moduleInfo = {"name": None} classMethods = [] importURLs = [] for i, key in enumerate(item.value.keys): _key = key.s _value = item.value.values[i] _valueType = _value.__class__.__name__ if _key == "import": if _valueType == 'Str': importURLs.append(_value.s) elif _valueType in ("List", "Tuple"): for elt in _value.elts: importURLs.append(elt.s) elif _valueType == 'Str': moduleInfo[_key] = _value.s elif _valueType == 'Name': if _value.id in constantStrings: moduleInfo[_key] = constantStrings[ _value.id] elif _value.id in functionDefNames: classMethods.append(_key) elif _key == "imports" and _valueType in ("List", "Tuple"): importURLs = [elt.s for elt in _value.elts] moduleInfo['classMethods'] = classMethods moduleInfo["moduleURL"] = moduleURL moduleInfo["status"] = 'enabled' moduleInfo["fileDate"] = time.strftime( '%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename))) mergedImportURLs = [] _moduleImportsSubtree = False for _url in importURLs: if _url.startswith("module_import"): for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") if _url == "module_import_subtree": _moduleImportsSubtree = True elif _url == "module_subtree": for _dir in os.listdir(moduleDir): _subtreeModule = os.path.join( moduleDir, _dir) if os.path.isdir( _subtreeModule ) and _dir != "__pycache__": mergedImportURLs.append(_subtreeModule) else: mergedImportURLs.append(_url) if parentImportsSubtree and not _moduleImportsSubtree: _moduleImportsSubtree = True for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") imports = [] for _url in mergedImportURLs: if isAbsolute(_url) or os.path.isabs(_url): _importURL = _url # URL is absolute http or local file system else: # check if exists relative to this module's directory _importURL = os.path.join( os.path.dirname(moduleURL), os.path.normpath(_url)) if not os.path.exists( _importURL ): # not relative to this plugin, assume standard plugin base _importURL = _url # moduleModuleInfo adjusts relative URL to plugin base _importModuleInfo = moduleModuleInfo( _importURL, reload, _moduleImportsSubtree) if _importModuleInfo: _importModuleInfo["isImported"] = True imports.append(_importModuleInfo) moduleInfo["imports"] = imports return moduleInfo elif isinstance( item.value, ast.Str ): # possible constant used in plugininfo, such as VERSION for assignmentName in item.targets: constantStrings[assignmentName.id] = item.value.s elif isinstance(item, ast.ImportFrom): if item.level == 1: # starts with . if item.module is None: # from . import module1, module2, ... for importee in item.names: if importee.name == '*': #import all submodules for _file in os.listdir(moduleDir): if _file != moduleFile and os.path.isfile( _file) and _file.endswith( ".py"): moduleImports.append(_file) elif (os.path.isfile( os.path.join(moduleDir, importee.name + ".py")) and importee.name not in moduleImports): moduleImports.append(importee.name) else: modulePkgs = item.module.split('.') modulePath = os.path.join(*modulePkgs) if (os.path.isfile( os.path.join(moduleDir, modulePath) + ".py") and modulePath not in moduleImports): moduleImports.append(modulePath) for importee in item.names: _importeePfxName = os.path.join( modulePath, importee.name) if (os.path.isfile( os.path.join(moduleDir, _importeePfxName) + ".py") and _importeePfxName not in moduleImports): moduleImports.append(_importeePfxName) elif isinstance(item, ast.FunctionDef ): # possible functionDef used in plugininfo functionDefNames.add(item.name) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write("Successful module plug-in info: " + moduleFilename + '\n') except Exception as err: _msg = _( "Exception obtaining plug-in module info: {moduleFilename}\n{error}\n{traceback}" ).format(error=err, moduleFilename=moduleFilename, traceback=traceback.format_tb(sys.exc_info()[2])) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write(_msg + '\n') else: print(_msg, file=sys.stderr) if f: f.close() return None
def parsePackage(cntlr, filesource, metadataFile, fileBase): global ArchiveFileIOError if ArchiveFileIOError is None: from arelle.FileSource import ArchiveFileIOError unNamedCounter = 1 txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1", "http://xbrl.org/PWD/2014-01-15/taxonomy-package", "http://xbrl.org/PWD/2015-01-14/taxonomy-package") catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog",) pkg = {} currentLang = Locale.getLanguageCode() _file = filesource.file(metadataFile)[0] # URL in zip, plain file in file system or web tree = etree.parse(_file) root = tree.getroot() ns = root.tag.partition("}")[0][1:] nsPrefix = "{{{}}}".format(ns) if ns in txmyPkgNSes: # package file for eltName in ("name", "description", "version"): pkg[eltName] = '' for m in root.iterchildren(tag=nsPrefix + eltName): pkg[eltName] = m.text.strip() break # take first entry if several else: # oasis catalog, use dirname as the package name # metadataFile may be a File object (with name) or string filename fileName = getattr(metadataFile, 'fileName', # for FileSource named objects getattr(metadataFile, 'name', # for io.file named objects metadataFile)) # for string pkg["name"] = os.path.basename(os.path.dirname(fileName)) pkg["description"] = "oasis catalog" pkg["version"] = "(none)" remappings = {} rewriteTree = tree catalogFile = metadataFile if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package",): catalogFile = metadataFile.replace('taxonomyPackage.xml','catalog.xml') try: rewriteTree = etree.parse(filesource.file(catalogFile)[0]) except ArchiveFileIOError: pass for tag, prefixAttr, replaceAttr in ( (nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"), ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog for m in rewriteTree.iter(tag=tag): prefixValue = m.get(prefixAttr) replaceValue = m.get(replaceAttr) if prefixValue and replaceValue is not None: if prefixValue not in remappings: base = baseForElement(m) if base: replaceValue = os.path.join(base, replaceValue) if replaceValue: # neither None nor '' if not isAbsolute(replaceValue): if not os.path.isabs(replaceValue): replaceValue = fileBase + replaceValue replaceValue = replaceValue.replace("/", os.sep) _normedValue = os.path.normpath(replaceValue) if replaceValue.endswith(os.sep) and not _normedValue.endswith(os.sep): _normedValue += os.sep remappings[prefixValue] = _normedValue else: cntlr.addToLog(_("Package catalog duplicate rewrite start string %(rewriteStartString)s"), messageArgs={"rewriteStartString": prefixValue}, messageCode="arelle.catalogDuplicateRewrite", file=os.path.basename(catalogFile), level=logging.ERROR) pkg["remappings"] = remappings nameToUrls = {} pkg["nameToUrls"] = nameToUrls for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"): name = None # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"): xmlLang = nameNode.get('{http://www.w3.org/XML/1998/namespace}lang') if name is None or not xmlLang or currentLang == xmlLang: name = nameNode.text if currentLang == xmlLang: # most prefer one with the current locale's language break if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"): epUrl = epDoc.get('href') base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base') # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl if epDocCount: cntlr.addToLog(_("Skipping multiple-document entry point (not supported) %(href)s"), messageArgs={"href": epUrl}, messageCode="arelle.packageMultipleDocumentEntryPoints", file=os.path.basename(metadataFile), level=logging.WARNING) continue epDocCount += 1 #perform prefix remappings remappedUrl = resolvedUrl longestPrefix = 0 for mapFrom, mapTo in remappings.items(): if remappedUrl.startswith(mapFrom): prefixLength = len(mapFrom) if prefixLength > longestPrefix: _remappedUrl = remappedUrl[prefixLength:] if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')): _remappedUrl = mapTo + os.sep + _remappedUrl else: _remappedUrl = mapTo + _remappedUrl longestPrefix = prefixLength if longestPrefix: remappedUrl = _remappedUrl.replace(os.sep, "/") # always used as FileSource select nameToUrls[name] = (remappedUrl, resolvedUrl) return pkg
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False): #TODO several directories, eg User Application Data moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase) if moduleFilename: f = None try: # if moduleFilename is a directory containing an __ini__.py file, open that instead if os.path.isdir(moduleFilename) and os.path.isfile(os.path.join(moduleFilename, "__init__.py")): moduleFilename = os.path.join(moduleFilename, "__init__.py") moduleDir = os.path.dirname(moduleFilename) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write("Scanning module for plug-in info: {}\n".format(moduleFilename)) f = openFileStream(_cntlr, moduleFilename) tree = ast.parse(f.read(), filename=moduleFilename) moduleImports = [] for item in tree.body: if isinstance(item, ast.Assign): attr = item.targets[0].id if attr == "__pluginInfo__": f.close() moduleInfo = {"name":None} classMethods = [] importURLs = [] for i, key in enumerate(item.value.keys): _key = key.s _value = item.value.values[i] _valueType = _value.__class__.__name__ if _key == "import": if _valueType == 'Str': importURLs.append(_value.s) elif _valueType in ("List", "Tuple"): for elt in _value.elts: importURLs.append(elt.s) elif _valueType == 'Str': moduleInfo[_key] = _value.s elif _valueType == 'Name': classMethods.append(_key) elif _key == "imports" and _valueType in ("List", "Tuple"): importURLs = [elt.s for elt in _value.elts] moduleInfo['classMethods'] = classMethods moduleInfo["moduleURL"] = moduleURL moduleInfo["status"] = 'enabled' moduleInfo["fileDate"] = time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename))) mergedImportURLs = [] _moduleImportsSubtree = False for _url in importURLs: if _url.startswith("module_import"): for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") if _url == "module_import_subtree": _moduleImportsSubtree = True else: mergedImportURLs.append(_url) if parentImportsSubtree and not _moduleImportsSubtree: _moduleImportsSubtree = True for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") imports = [] for _url in mergedImportURLs: if isAbsolute(_url) or os.path.isabs(_url): _importURL = _url # URL is absolute http or local file system else: # check if exists relative to this module's directory _importURL = os.path.join(os.path.dirname(moduleURL), _url) if not os.path.exists(_importURL): # not relative to this plugin, assume standard plugin base _importURL = os.path.join(_pluginBase, _url) _importModuleInfo = moduleModuleInfo(_importURL, reload, _moduleImportsSubtree) if _importModuleInfo: _importModuleInfo["isImported"] = True imports.append(_importModuleInfo) moduleInfo["imports"] = imports return moduleInfo elif isinstance(item, ast.ImportFrom): if item.level == 1: # starts with . if item.module is None: # from . import module1, module2, ... for importee in item.names: if (os.path.isfile(os.path.join(moduleDir, importee.name + ".py")) and importee.name not in moduleImports): moduleImports.append(importee.name) else: modulePkgs = item.module.split('.') modulePath = os.path.join(*modulePkgs) if (os.path.isfile(os.path.join(moduleDir, modulePath) + ".py") and modulePath not in moduleImports): moduleImports.append(modulePath) for importee in item.names: _importeePfxName = os.path.join(modulePath, importee.name) if (os.path.isfile(os.path.join(moduleDir, _importeePfxName) + ".py") and _importeePfxName not in moduleImports): moduleImports.append(_importeePfxName) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write("Successful module plug-in info: " + moduleFilename + '\n') except Exception as err: _msg = _("Exception obtaining plug-in module info: {error}\n{traceback}").format( error=err, traceback=traceback.format_tb(sys.exc_info()[2])) if PLUGIN_TRACE_FILE: with open(PLUGIN_TRACE_FILE, "at", encoding='utf-8') as fh: fh.write(_msg + '\n') else: print(_msg, file=sys.stderr) if f: f.close() return None
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False): #TODO several directories, eg User Application Data moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase) if moduleFilename: f = None try: # if moduleFilename is a directory containing an __ini__.py file, open that instead if os.path.isdir(moduleFilename) and os.path.isfile( os.path.join(moduleFilename, "__init__.py")): moduleFilename = os.path.join(moduleFilename, "__init__.py") moduleDir = os.path.dirname(moduleFilename) f = openFileStream(_cntlr, moduleFilename) tree = ast.parse(f.read(), filename=moduleFilename) moduleImports = [] for item in tree.body: if isinstance(item, ast.Assign): attr = item.targets[0].id if attr == "__pluginInfo__": f.close() moduleInfo = {"name": None} classMethods = [] importURLs = [] for i, key in enumerate(item.value.keys): _key = key.s _value = item.value.values[i] _valueType = _value.__class__.__name__ if _key == "import": if _valueType == 'Str': importURLs.append(_value.s) elif _valueType in ("List", "Tuple"): for elt in _value.elts: importURLs.append(elt.s) elif _valueType == 'Str': moduleInfo[_key] = _value.s elif _valueType == 'Name': classMethods.append(_key) elif _key == "imports" and _valueType in ("List", "Tuple"): importURLs = [elt.s for elt in _value.elts] moduleInfo['classMethods'] = classMethods moduleInfo["moduleURL"] = moduleURL moduleInfo["status"] = 'enabled' moduleInfo["fileDate"] = time.strftime( '%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename))) mergedImportURLs = [] _moduleImportsSubtree = False for _url in importURLs: if _url.startswith("module_import"): for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") if _url == "module_import_subtree": _moduleImportsSubtree = True else: mergedImportURLs.append(_url) if parentImportsSubtree and not _moduleImportsSubtree: _moduleImportsSubtree = True for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") imports = [] for _url in mergedImportURLs: _importURL = ( _url if isAbsolute(_url) or os.path.isabs(_url) else os.path.join( os.path.dirname(moduleURL), _url)) _importModuleInfo = moduleModuleInfo( _importURL, reload, _moduleImportsSubtree) if _importModuleInfo: _importModuleInfo["isImported"] = True imports.append(_importModuleInfo) moduleInfo["imports"] = imports return moduleInfo elif isinstance(item, ast.ImportFrom): if item.level == 1: # starts with . if item.module is None: # from . import module1, module2, ... for importee in item.names: if (os.path.isfile( os.path.join(moduleDir, importee.name + ".py")) and importee.name not in moduleImports): moduleImports.append(importee.name) else: modulePkgs = item.module.split('.') modulePath = os.path.join(*modulePkgs) if (os.path.isfile( os.path.join(moduleDir, modulePath) + ".py") and modulePath not in moduleImports): moduleImports.append(modulePath) for importee in item.names: _importeePfxName = os.path.join( modulePath, importee.name) if (os.path.isfile( os.path.join(moduleDir, _importeePfxName) + ".py") and _importeePfxName not in moduleImports): moduleImports.append(_importeePfxName) except EnvironmentError: pass if f: f.close() return None
def parsePackage(cntlr, filesource, metadataFile, fileBase): global ArchiveFileIOError if ArchiveFileIOError is None: from arelle.FileSource import ArchiveFileIOError unNamedCounter = 1 txmyPkgNSes = ("http://www.corefiling.com/xbrl/taxonomypackage/v1", "http://xbrl.org/PWD/2014-01-15/taxonomy-package", "http://xbrl.org/PWD/2015-01-14/taxonomy-package") catalogNSes = ("urn:oasis:names:tc:entity:xmlns:xml:catalog", ) pkg = {} currentLang = Locale.getLanguageCode() _file = filesource.file(metadataFile)[ 0] # URL in zip, plain file in file system or web tree = etree.parse(_file) root = tree.getroot() ns = root.tag.partition("}")[0][1:] nsPrefix = "{{{}}}".format(ns) if ns in txmyPkgNSes: # package file for eltName in ("name", "description", "version"): pkg[eltName] = '' for m in root.iterchildren(tag=nsPrefix + eltName): pkg[eltName] = m.text.strip() break # take first entry if several else: # oasis catalog, use dirname as the package name # metadataFile may be a File object (with name) or string filename fileName = getattr( metadataFile, 'fileName', # for FileSource named objects getattr( metadataFile, 'name', # for io.file named objects metadataFile)) # for string pkg["name"] = os.path.basename(os.path.dirname(fileName)) pkg["description"] = "oasis catalog" pkg["version"] = "(none)" remappings = {} rewriteTree = tree catalogFile = metadataFile if ns in ("http://xbrl.org/PWD/2015-01-14/taxonomy-package", ): catalogFile = metadataFile.replace('taxonomyPackage.xml', 'catalog.xml') try: rewriteTree = etree.parse(filesource.file(catalogFile)[0]) except ArchiveFileIOError: pass for tag, prefixAttr, replaceAttr in ( (nsPrefix + "remapping", "prefix", "replaceWith"), # taxonomy package ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteSystem", "systemIdStartString", "rewritePrefix"), ("{urn:oasis:names:tc:entity:xmlns:xml:catalog}rewriteURI", "uriStartString", "rewritePrefix")): # oasis catalog for m in rewriteTree.iter(tag=tag): prefixValue = m.get(prefixAttr) replaceValue = m.get(replaceAttr) if prefixValue and replaceValue is not None: if prefixValue not in remappings: base = baseForElement(m) if base: replaceValue = os.path.join(base, replaceValue) if replaceValue: # neither None nor '' if not isAbsolute(replaceValue): if not os.path.isabs(replaceValue): replaceValue = fileBase + replaceValue replaceValue = replaceValue.replace("/", os.sep) _normedValue = os.path.normpath(replaceValue) if replaceValue.endswith( os.sep) and not _normedValue.endswith(os.sep): _normedValue += os.sep remappings[prefixValue] = _normedValue else: cntlr.addToLog(_( "Package catalog duplicate rewrite start string %(rewriteStartString)s" ), messageArgs={ "rewriteStartString": prefixValue }, messageCode="arelle.catalogDuplicateRewrite", file=os.path.basename(catalogFile), level=logging.ERROR) pkg["remappings"] = remappings nameToUrls = {} pkg["nameToUrls"] = nameToUrls for entryPointSpec in tree.iter(tag=nsPrefix + "entryPoint"): name = None # find closest match name node given xml:lang match to current language or no xml:lang for nameNode in entryPointSpec.iter(tag=nsPrefix + "name"): xmlLang = nameNode.get( '{http://www.w3.org/XML/1998/namespace}lang') if name is None or not xmlLang or currentLang == xmlLang: name = nameNode.text if currentLang == xmlLang: # most prefer one with the current locale's language break if not name: name = _("<unnamed {0}>").format(unNamedCounter) unNamedCounter += 1 epDocCount = 0 for epDoc in entryPointSpec.iterchildren(nsPrefix + "entryPointDocument"): epUrl = epDoc.get('href') base = epDoc.get('{http://www.w3.org/XML/1998/namespace}base' ) # cope with xml:base if base: resolvedUrl = urljoin(base, epUrl) else: resolvedUrl = epUrl if epDocCount: cntlr.addToLog( _("Skipping multiple-document entry point (not supported) %(href)s" ), messageArgs={"href": epUrl}, messageCode="arelle.packageMultipleDocumentEntryPoints", file=os.path.basename(metadataFile), level=logging.WARNING) continue epDocCount += 1 #perform prefix remappings remappedUrl = resolvedUrl longestPrefix = 0 for mapFrom, mapTo in remappings.items(): if remappedUrl.startswith(mapFrom): prefixLength = len(mapFrom) if prefixLength > longestPrefix: _remappedUrl = remappedUrl[prefixLength:] if not (_remappedUrl[0] in (os.sep, '/') or mapTo[-1] in (os.sep, '/')): _remappedUrl = mapTo + os.sep + _remappedUrl else: _remappedUrl = mapTo + _remappedUrl longestPrefix = prefixLength if longestPrefix: remappedUrl = _remappedUrl.replace( os.sep, "/") # always used as FileSource select nameToUrls[name] = (remappedUrl, resolvedUrl) return pkg
def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False): #TODO several directories, eg User Application Data moduleFilename = _cntlr.webCache.getfilename(moduleURL, reload=reload, normalize=True, base=_pluginBase) if moduleFilename: f = None try: # if moduleFilename is a directory containing an __ini__.py file, open that instead if os.path.isdir(moduleFilename) and os.path.isfile(os.path.join(moduleFilename, "__init__.py")): moduleFilename = os.path.join(moduleFilename, "__init__.py") moduleDir = os.path.dirname(moduleFilename) f = openFileStream(_cntlr, moduleFilename) tree = ast.parse(f.read(), filename=moduleFilename) moduleImports = [] for item in tree.body: if isinstance(item, ast.Assign): attr = item.targets[0].id if attr == "__pluginInfo__": f.close() moduleInfo = {"name":None} classMethods = [] importURLs = [] for i, key in enumerate(item.value.keys): _key = key.s _value = item.value.values[i] _valueType = _value.__class__.__name__ if _key == "import": if _valueType == 'Str': importURLs.append(_value.s) elif _valueType in ("List", "Tuple"): for elt in _value.elts: importURLs.append(elt.s) elif _valueType == 'Str': moduleInfo[_key] = _value.s elif _valueType == 'Name': classMethods.append(_key) elif _key == "imports" and _valueType in ("List", "Tuple"): importURLs = [elt.s for elt in _value.elts] moduleInfo['classMethods'] = classMethods moduleInfo["moduleURL"] = moduleURL moduleInfo["status"] = 'enabled' moduleInfo["fileDate"] = time.strftime('%Y-%m-%dT%H:%M:%S UTC', time.gmtime(os.path.getmtime(moduleFilename))) mergedImportURLs = [] _moduleImportsSubtree = False for _url in importURLs: if _url.startswith("module_import"): for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") if _url == "module_import_subtree": _moduleImportsSubtree = True else: mergedImportURLs.append(_url) if parentImportsSubtree and not _moduleImportsSubtree: _moduleImportsSubtree = True for moduleImport in moduleImports: mergedImportURLs.append(moduleImport + ".py") imports = [] for _url in mergedImportURLs: _importURL = (_url if isAbsolute(_url) or os.path.isabs(_url) else os.path.join(os.path.dirname(moduleURL), _url)) _importModuleInfo = moduleModuleInfo(_importURL, reload, _moduleImportsSubtree) if _importModuleInfo: _importModuleInfo["isImported"] = True imports.append(_importModuleInfo) moduleInfo["imports"] = imports return moduleInfo elif isinstance(item, ast.ImportFrom): if item.level == 1: # starts with . if item.module is None: # from . import module1, module2, ... for importee in item.names: if (os.path.isfile(os.path.join(moduleDir, importee.name + ".py")) and importee.name not in moduleImports): moduleImports.append(importee.name) else: modulePkgs = item.module.split('.') modulePath = os.path.join(*modulePkgs) if (os.path.isfile(os.path.join(moduleDir, modulePath) + ".py") and modulePath not in moduleImports): moduleImports.append(modulePath) for importee in item.names: _importeePfxName = os.path.join(modulePath, importee.name) if (os.path.isfile(os.path.join(moduleDir, _importeePfxName) + ".py") and _importeePfxName not in moduleImports): moduleImports.append(_importeePfxName) except EnvironmentError: pass if f: f.close() return None