def insertDocuments(self): self.showStatus("insert documents") table = self.getTable('document', 'document_id', ('document_url', 'document_type', 'namespace'), ('document_url',), set((ensureUrl(docUrl), mdlDoc.type, mdlDoc.targetNamespace) for docUrl, mdlDoc in self.modelXbrl.urlDocs.items() if mdlDoc not in self.existingDocumentIds and self.isSemanticDocument(mdlDoc)), checkIfExisting=True) self.documentIds = dict((self.urlDocs[url], id) for id, url in table) self.documentIds.update(self.existingDocumentIds) referencedDocuments = set() # instance documents are filing references for mdlDoc in self.modelXbrl.urlDocs.values(): if mdlDoc.type in (Type.INSTANCE, Type.INLINEXBRL): referencedDocuments.add( (self.reportId, self.documentIds[mdlDoc] )) if mdlDoc in self.documentIds and mdlDoc not in self.existingDocumentIds: for refDoc, ref in mdlDoc.referencesDocument.items(): if refDoc.inDTS and ref.referenceType in ("href", "import", "include") \ and refDoc in self.documentIds: referencedDocuments.add( (self.documentIds[mdlDoc], self.documentIds[refDoc] )) table = self.getTable('referenced_documents', None, # no id column in this table ('object_id','document_id'), ('object_id','document_id'), referencedDocuments, checkIfExisting=True)
def extXdcFiling(xbrlOpenDb, now): md = xbrlOpenDb.metadata table = xbrlOpenDb.getTable('filing_xdc', None, ('filing_pk', 'filing_id' ), ('filing_pk', ), ((xbrlOpenDb.filingPk, md["filingId"] ),), checkIfExisting=True) # add supplemental document references supplementalDocumentsTable = xbrlOpenDb.getTable( 'document', 'document_pk', ('url', 'type'), ('url',), set((ensureUrl(docUrl), "attachment") for docUrl in md["supplementalUrls"]), checkIfExisting=True) table = xbrlOpenDb.getTable('referenced_documents', None, # no id column in this table ('object_fk','document_fk'), ('object_fk','document_fk'), tuple((xbrlOpenDb.filingPk, supplementalDocumentId) for supplementalDocumentId, _url in supplementalDocumentsTable), checkIfExisting=True)
def identifyPreexistingDocuments(self): self.existingDocumentIds = {} self.urlDocs = {} docUris = set() for modelDocument in self.modelXbrl.urlDocs.values(): url = ensureUrl(modelDocument.uri) self.urlDocs[url] = modelDocument if self.isSemanticDocument(modelDocument): docUris.add(self.dbStr(url)) if docUris: results = self.execute("SELECT document_id, document_url FROM {} WHERE document_url IN ({})" .format(self.dbTableName("document"), ', '.join(docUris))) self.existingDocumentIds = dict((self.urlDocs[docUrl],docId) for docId, docUrl in results)