def create_type(content, name, filename, buckets=[], dependencies=[], user=None): is_schema_valid('Type', content, name) hash_value = XSDhash.get_hash(content) # save the type type_versions = TypeVersion(nbVersions=1, isDeleted=False).save() new_type = Type(title=name, filename=filename, content=content, version=1, typeVersion=str(type_versions.id), hash=hash_value, user=user, dependencies=dependencies).save() # Add to the selected buckets for bucket_id in buckets: bucket = Bucket.objects.get(pk=bucket_id) bucket.types.append(str(type_versions.id)) bucket.save() type_versions.versions = [str(new_type.id)] type_versions.current = str(new_type.id) type_versions.save() new_type.save() return new_type
def create_template(content, name, filename, dependencies=[], user=None, validation=True): if validation: is_schema_valid('Template', content, name) flattener = XSDFlattenerDatabaseOrURL(content.encode('utf-8')) content_encoded = flattener.get_flat() hash_value = XSDhash.get_hash(content_encoded) # save the template template_versions = TemplateVersion(nbVersions=1, isDeleted=False).save() new_template = Template(title=name, filename=filename, content=content, version=1, templateVersion=str(template_versions.id), hash=hash_value, user=user, dependencies=dependencies).save() # Add default exporters try: exporters = Exporter.objects.filter(available_for_all=True) new_template.exporters = exporters except: pass template_versions.versions = [str(new_template.id)] template_versions.current = str(new_template.id) template_versions.save() new_template.save() return new_template
def test_wrong_enum(self): # makes sure that an XSD with different enumeration does not produce the same hash file = open(join(RESOURCES_PATH, 'wrong-enum.xsd'), 'r') content = file.read() hash = XSDhash.get_hash(content) print "self.hash: " + self.hash print "hash: " + hash self.assertNotEqual(self.hash, hash)
def test_different_namespace(self): # makes sure that an XSD with different comments produces the same hash file = open(join(RESOURCES_PATH, 'namespace.xsd'), 'r') content = file.read() hash = XSDhash.get_hash(content) print "self.hash: " + self.hash print "hash: " + hash self.assertNotEqual(self.hash, hash)
def test_comments(self): # makes sure that an XSD with documentation tags produces different hash file = open(join(RESOURCES_PATH, 'different-comments.xsd'), 'r') content = file.read() hash = XSDhash.get_hash(content) print "self.hash: " + self.hash print "hash: " + hash self.assertEqual(self.hash, hash)
def test_spaces2(self): # makes sure that an XSD with additional spaces, returns,tabs produces the same hash file = open(join(RESOURCES_PATH, 'spaces-return-tab.xsd'), 'r') content = file.read() hash = XSDhash.get_hash(content) print "self.hash: " + self.hash print "hash: " + hash self.assertEqual(self.hash, hash)
def test_order(self): # makes sure that an XSD with elements in a different order produces the same hash file = open(join(RESOURCES_PATH, 'order2.xsd'), 'r') content = file.read() hash = XSDhash.get_hash(content) print "self.hash: " + self.hash print "hash: " + hash self.assertEqual(self.hash, hash)
def test_same(self): # makes sure that the same XSD produces the same hash file = open(join(RESOURCES_PATH, 'composition.xsd'), 'r') content = file.read() hash = XSDhash.get_hash(content) print "self.hash: " + self.hash print "hash: " + hash self.assertEqual(self.hash, hash)
def createTemplateWithTemplateVersion(self, templateVersionId): hash = XSDhash.get_hash('<test>test xmldata</test>') return Template(title='test', filename='test', content='<test>test xmldata</test>', version=1, templateVersion=templateVersionId, hash=hash).save()
def test_type(self): # makes sure that an XSD with different type names does not produce the same hash file = open(join(RESOURCES_PATH, 'type-name.xsd'), 'r') content = file.read() hash = XSDhash.get_hash(content) print "self.hash: " + self.hash print "hash: " + hash self.assertNotEqual(self.hash, hash)
def save_version(request): print 'BEGIN def saveVersion(request, objectType)' versionFilename = None versionContent = None objectVersionID = None objectType = None if ('uploadVersionValid' in request.session and request.session['uploadVersionValid'] == True and 'uploadVersionID' in request.session and request.session['uploadVersionID'] is not None and 'uploadVersionType' in request.session and request.session['uploadVersionType'] is not None and 'uploadVersionFilename' in request.session and request.session['uploadVersionFilename'] is not None and 'uploadVersionContent' in request.session and request.session['uploadVersionContent'] is not None): versionFilename = request.session['uploadVersionFilename'] versionContent = request.session['uploadVersionContent'] objectVersionID = request.session['uploadVersionID'] objectType = request.session['uploadVersionType'] if 'uploadVersionFlat' in request.session and request.session['uploadVersionFlat'] is not None: versionFlat = request.session['uploadVersionFlat'] else: versionFlat = None if 'uploadVersionAPIurl' in request.session and request.session['uploadVersionAPIurl'] is not None: versionApiurl = request.session['uploadVersionAPIurl'] else: versionApiurl = None if 'uploadDependencies' in request.session and request.session['uploadDependencies'] is not None: dependencies = request.session['uploadDependencies'] else: dependencies = None hash = XSDhash.get_hash(versionContent) # save the object if objectType == "Template": objectVersions = TemplateVersion.objects.get(pk=objectVersionID) objectVersions.nbVersions += 1 object = Template.objects.get(pk=objectVersions.current) newObject = Template(title=object.title, filename=versionFilename, content=versionContent, version=objectVersions.nbVersions, templateVersion=objectVersionID, hash=hash).save() elif objectType == "Type": objectVersions = TypeVersion.objects.get(pk=objectVersionID) objectVersions.nbVersions += 1 object = Type.objects.get(pk=objectVersions.current) newObject = Type(title=object.title, filename=versionFilename, content=versionContent, version=objectVersions.nbVersions, typeVersion=objectVersionID, hash=hash).save() objectVersions.versions.append(str(newObject.id)) objectVersions.save() if versionFlat is not None and versionApiurl is not None and dependencies is not None: MetaSchema(schemaId=str(newObject.id), flat_content=versionFlat, api_content=versionApiurl).save() object.dependencies = dependencies object.save() clearVersion(request) else: response_dict = {'errors': 'True'} return HttpResponse(json.dumps(response_dict), content_type='application/javascript') return HttpResponse(json.dumps({}), content_type='application/javascript')
def createTemplate(self): hash = XSDhash.get_hash('<test>test xmldata</test>') objectVersions = self.createTemplateVersion() return Template(title='test', filename='test', content='<test>test xmldata</test>', version=1, templateVersion=str(objectVersions.id), hash=hash).save()
def createTemplateWithTemplateVersionValidContent(self, templateVersionId): hash = XSDhash.get_hash(TEMPLATE_VALID_CONTENT) template = Template(title='test', filename='test', content=TEMPLATE_VALID_CONTENT, version=1, templateVersion=templateVersionId, hash=hash).save() TemplateVersion.objects.get(pk=templateVersionId).update( set__current=template.id) return template
def create_type_version(content, filename, versions_id): hash_value = XSDhash.get_hash(content) type_versions = TypeVersion.objects.get(pk=versions_id) type_versions.nbVersions += 1 current_type = Type.objects.get(pk=type_versions.current) new_type = Type(title=current_type.title, filename=filename, content=content, version=type_versions.nbVersions, typeVersion=str(versions_id), hash=hash_value).save() type_versions.versions.append(str(new_type.id)) type_versions.save() return new_type
def createTemplate(self, title='test', filename='test', user=None): countTemplate = len(Template.objects()) hash = XSDhash.get_hash('<test>test xmldata</test>') objectVersions = self.createTemplateVersion() template = Template(title=title, filename=filename, content='<test>test xmldata</test>', version=1, templateVersion=str(objectVersions.id), hash=hash, user=str(user)).save() self.assertEquals(len(Template.objects()), countTemplate + 1) return template
def create_template_version(content, filename, versions_id, dependencies=[]): is_schema_valid('Template', content) hash_value = XSDhash.get_hash(content) template_versions = TemplateVersion.objects.get(pk=versions_id) template_versions.nbVersions += 1 current_template = Template.objects.get(pk=template_versions.current) new_template = Template(title=current_template.title, filename=filename, content=content, version=template_versions.nbVersions, templateVersion=str(versions_id), hash=hash_value, dependencies=dependencies).save() template_versions.versions.append(str(new_template.id)) template_versions.save() return new_template
def create_type(content, name, filename, buckets=[], dependencies=[], user=None): hash_value = XSDhash.get_hash(content) # save the type type_versions = TypeVersion(nbVersions=1, isDeleted=False).save() new_type = Type(title=name, filename=filename, content=content, version=1, typeVersion=str(type_versions.id), hash=hash_value, user=user).save() new_type.dependencies = dependencies # Add to the selected buckets for bucket_id in buckets: bucket = Bucket.objects.get(pk=bucket_id) bucket.types.append(str(type_versions.id)) bucket.save() type_versions.versions = [str(new_type.id)] type_versions.current = str(new_type.id) type_versions.save() new_type.save() return new_type
def create_template(content, name, filename, dependencies=[], user=None): hash_value = XSDhash.get_hash(content) # save the template template_versions = TemplateVersion(nbVersions=1, isDeleted=False).save() new_template = Template(title=name, filename=filename, content=content, version=1, templateVersion=str(template_versions.id), hash=hash_value, user=user).save() new_template.dependencies = dependencies # Add default exporters try: exporters = Exporter.objects.filter(available_for_all=True) new_template.exporters = exporters except: pass template_versions.versions = [str(new_template.id)] template_versions.current = str(new_template.id) template_versions.save() new_template.save() return new_template
def create_template_version(content, filename, versions_id, dependencies=[]): is_schema_valid('Template', content) flattener = XSDFlattenerDatabaseOrURL(content.encode('utf-8')) content_encoded = flattener.get_flat() hash_value = XSDhash.get_hash(content_encoded) template_versions = TemplateVersion.objects.get(pk=versions_id) template_versions.nbVersions += 1 current_template = Template.objects.get(pk=template_versions.current) new_template = Template(title=current_template.title, filename=filename, content=content, version=template_versions.nbVersions, templateVersion=str(versions_id), hash=hash_value, dependencies=dependencies).save() template_versions.versions.append(str(new_template.id)) template_versions.save() return new_template
def setUp(self): print "In method", self._testMethodName file = open(join(RESOURCES_PATH, 'demo.diffusion.xsd'), 'r') self.content = file.read() self.hash = XSDhash.get_hash(self.content)
return HttpResponse(json.dumps(response_dict), content_type='application/javascript') flattener = XSDFlattenerMDCS(etree.tostring(xmlTree)) flatStr = flattener.get_flat() flatTree = etree.fromstring(flatStr) try: # is it a valid XML schema ? xmlSchema = etree.XMLSchema(flatTree) except Exception, e: response_dict['errors'] = e.message.replace("'", "") return HttpResponse(json.dumps(response_dict), content_type='application/javascript') hash = XSDhash.get_hash(content) dependencies = [] for uri in request.session["includedTypesCompose"]: url = urlparse(uri) id = url.query.split("=")[1] dependencies.append(id) template = Template(title=template_name, filename=template_name, content=content, hash=hash, user=str(request.user.id), dependencies=dependencies) #We add default exporters try: exporters = Exporter.objects.filter(available_for_all=True) template.exporters = exporters
def setUp(self): print "In method", self._testMethodName file = open(join(RESOURCES_PATH, 'chemical-element.xsd'), 'r') self.content = file.read() self.hash = XSDhash.get_hash(self.content)
def save_version(request): print 'BEGIN def saveVersion(request, objectType)' versionFilename = None versionContent = None objectVersionID = None objectType = None if ('uploadVersionValid' in request.session and request.session['uploadVersionValid'] == True and 'uploadVersionID' in request.session and request.session['uploadVersionID'] is not None and 'uploadVersionType' in request.session and request.session['uploadVersionType'] is not None and 'uploadVersionFilename' in request.session and request.session['uploadVersionFilename'] is not None and 'uploadVersionContent' in request.session and request.session['uploadVersionContent'] is not None): versionFilename = request.session['uploadVersionFilename'] versionContent = request.session['uploadVersionContent'] objectVersionID = request.session['uploadVersionID'] objectType = request.session['uploadVersionType'] if 'uploadVersionFlat' in request.session and request.session[ 'uploadVersionFlat'] is not None: versionFlat = request.session['uploadVersionFlat'] else: versionFlat = None if 'uploadVersionAPIurl' in request.session and request.session[ 'uploadVersionAPIurl'] is not None: versionApiurl = request.session['uploadVersionAPIurl'] else: versionApiurl = None if 'uploadDependencies' in request.session and request.session[ 'uploadDependencies'] is not None: dependencies = request.session['uploadDependencies'] else: dependencies = None hash = XSDhash.get_hash(versionContent) # save the object if objectType == "Template": objectVersions = TemplateVersion.objects.get(pk=objectVersionID) objectVersions.nbVersions += 1 object = Template.objects.get(pk=objectVersions.current) newObject = Template(title=object.title, filename=versionFilename, content=versionContent, version=objectVersions.nbVersions, templateVersion=objectVersionID, hash=hash).save() elif objectType == "Type": objectVersions = TypeVersion.objects.get(pk=objectVersionID) objectVersions.nbVersions += 1 object = Type.objects.get(pk=objectVersions.current) newObject = Type(title=object.title, filename=versionFilename, content=versionContent, version=objectVersions.nbVersions, typeVersion=objectVersionID, hash=hash).save() objectVersions.versions.append(str(newObject.id)) objectVersions.save() if versionFlat is not None and versionApiurl is not None and dependencies is not None: MetaSchema(schemaId=str(newObject.id), flat_content=versionFlat, api_content=versionApiurl).save() object.dependencies = dependencies object.save() clearVersion(request) else: response_dict = {'errors': 'True'} return HttpResponse(json.dumps(response_dict), content_type='application/javascript') return HttpResponse(json.dumps({}), content_type='application/javascript')
def save_object(request): print 'BEGIN def save_object(request)' objectName = None objectFilename = None objectContent = None objectType = None objectFlat = None objectApiurl = None if ('uploadObjectValid' in request.session and request.session['uploadObjectValid'] == True and 'uploadObjectName' in request.session and request.session['uploadObjectName'] is not None and 'uploadObjectFilename' in request.session and request.session['uploadObjectFilename'] is not None and 'uploadObjectContent' in request.session and request.session['uploadObjectContent'] is not None and 'uploadObjectType' in request.session and request.session['uploadObjectType'] is not None): objectName = request.session['uploadObjectName'] objectFilename = request.session['uploadObjectFilename'] objectContent = request.session['uploadObjectContent'] objectType = request.session['uploadObjectType'] if 'uploadObjectFlat' in request.session and request.session['uploadObjectFlat'] is not None: objectFlat = request.session['uploadObjectFlat'] else: objectFlat = None if 'uploadObjectAPIurl' in request.session and request.session['uploadObjectAPIurl'] is not None: objectApiurl = request.session['uploadObjectAPIurl'] else: objectApiurl = None if 'uploadDependencies' in request.session and request.session['uploadDependencies'] is not None: dependencies = request.session['uploadDependencies'] else: dependencies = None hash = XSDhash.get_hash(objectContent) # save the object if objectType == "Template": objectVersions = TemplateVersion(nbVersions=1, isDeleted=False).save() object = Template(title=objectName, filename=objectFilename, content=objectContent, version=1, templateVersion=str(objectVersions.id), hash=hash).save() #We add default exporters try: exporters = Exporter.objects.filter(available_for_all=True) object.exporters = exporters except: pass elif objectType == "Type": objectVersions = TypeVersion(nbVersions=1, isDeleted=False).save() object = Type(title=objectName, filename=objectFilename, content=objectContent, version=1, typeVersion=str(objectVersions.id), hash=hash).save() buckets = request.POST.getlist('buckets[]') for bucket_id in buckets: bucket = Bucket.objects.get(pk=bucket_id) bucket.types.append(str(objectVersions.id)) bucket.save() objectVersions.versions = [str(object.id)] objectVersions.current = str(object.id) objectVersions.save() object.save() if objectFlat is not None and objectApiurl is not None and dependencies is not None: MetaSchema(schemaId=str(object.id), flat_content=objectFlat, api_content=objectApiurl).save() object.dependencies = dependencies object.save() clear_object(request) else: response_dict = {'errors': 'True'} return HttpResponse(json.dumps(response_dict), content_type='application/javascript') return HttpResponse(json.dumps({}), content_type='application/javascript')
def _strip(input_str): return XSDhash.get_hash(input_str)
def save_object(request): print 'BEGIN def save_object(request)' objectName = None objectFilename = None objectContent = None objectType = None objectFlat = None objectApiurl = None if ('uploadObjectValid' in request.session and request.session['uploadObjectValid'] == True and 'uploadObjectName' in request.session and request.session['uploadObjectName'] is not None and 'uploadObjectFilename' in request.session and request.session['uploadObjectFilename'] is not None and 'uploadObjectContent' in request.session and request.session['uploadObjectContent'] is not None and 'uploadObjectType' in request.session and request.session['uploadObjectType'] is not None): objectName = request.session['uploadObjectName'] objectFilename = request.session['uploadObjectFilename'] objectContent = request.session['uploadObjectContent'] objectType = request.session['uploadObjectType'] if 'uploadObjectFlat' in request.session and request.session[ 'uploadObjectFlat'] is not None: objectFlat = request.session['uploadObjectFlat'] else: objectFlat = None if 'uploadObjectAPIurl' in request.session and request.session[ 'uploadObjectAPIurl'] is not None: objectApiurl = request.session['uploadObjectAPIurl'] else: objectApiurl = None if 'uploadDependencies' in request.session and request.session[ 'uploadDependencies'] is not None: dependencies = request.session['uploadDependencies'] else: dependencies = None hash = XSDhash.get_hash(objectContent) # save the object if objectType == "Template": objectVersions = TemplateVersion(nbVersions=1, isDeleted=False).save() object = Template(title=objectName, filename=objectFilename, content=objectContent, version=1, templateVersion=str(objectVersions.id), hash=hash).save() #We add default exporters try: exporters = Exporter.objects.filter(available_for_all=True) object.exporters = exporters except: pass elif objectType == "Type": objectVersions = TypeVersion(nbVersions=1, isDeleted=False).save() object = Type(title=objectName, filename=objectFilename, content=objectContent, version=1, typeVersion=str(objectVersions.id), hash=hash).save() buckets = request.POST.getlist('buckets[]') for bucket_id in buckets: bucket = Bucket.objects.get(pk=bucket_id) bucket.types.append(str(objectVersions.id)) bucket.save() objectVersions.versions = [str(object.id)] objectVersions.current = str(object.id) objectVersions.save() object.save() if objectFlat is not None and objectApiurl is not None and dependencies is not None: MetaSchema(schemaId=str(object.id), flat_content=objectFlat, api_content=objectApiurl).save() object.dependencies = dependencies object.save() clear_object(request) else: response_dict = {'errors': 'True'} return HttpResponse(json.dumps(response_dict), content_type='application/javascript') return HttpResponse(json.dumps({}), content_type='application/javascript')
except Exception, e: response_dict['errors'] = e.message.replace("'","") return HttpResponse(json.dumps(response_dict), content_type='application/javascript') flattener = XSDFlattenerMDCS(etree.tostring(xmlTree)) flatStr = flattener.get_flat() flatTree = etree.fromstring(flatStr) try: # is it a valid XML schema ? xmlSchema = etree.XMLSchema(flatTree) except Exception, e: response_dict['errors'] = e.message.replace("'","") return HttpResponse(json.dumps(response_dict), content_type='application/javascript') hash = XSDhash.get_hash(content) dependencies = [] for uri in request.session["includedTypesCompose"]: url = urlparse(uri) id = url.query.split("=")[1] dependencies.append(id) template = Template(title=template_name, filename=template_name, content=content, hash=hash, user=str(request.user.id), dependencies=dependencies) #We add default exporters try: exporters = Exporter.objects.filter(available_for_all=True) template.exporters = exporters except: pass template.save()
def load_templates(): """ Loads templates/xslt for NMRR the first time """ # if templates are already present, initialization already happened existing_templates = Template.objects() if len(existing_templates) == 0: templates = { 'all':'AllResources.xsd', 'organization': 'Organization.xsd', 'datacollection': 'DataCollection.xsd', 'repository': 'Repository.xsd', 'projectarchive': 'ProjectArchive.xsd', 'database': 'Database.xsd', 'dataset': 'Dataset.xsd', 'service': 'Service.xsd', 'informational': 'Informational.xsd', 'software': 'Software.xsd', } template_ids = [] template_results = { 'full': 'nmrr-full.xsl', 'detail': 'nmrr-detail.xsl', } template_results_id = { 'full': None, 'detail': None, } # connect to mongo client = MongoClient(MONGODB_URI) # connect to the db 'mgi' db = client[MGI_DB] # Add the templates for template_name, template_path in templates.iteritems(): file = open(os.path.join(SITE_ROOT, 'static', 'resources', 'xsd', template_path),'r') templateContent = file.read() hash = XSDhash.get_hash(templateContent) #create template/ template version objectVersions = TemplateVersion(nbVersions=1, isDeleted=False).save() object = Template(title=template_name, filename=template_path, content=templateContent, version=1, templateVersion=str(objectVersions.id), hash=hash).save() objectVersions.versions = [str(object.id)] objectVersions.current = str(object.id) objectVersions.save() object.save() # save template id template_ids.append(str(object.id)) # Add xslt xsl_col = db['result_xslt'] for xsl_name, xsl_path in template_results.iteritems(): file = open(os.path.join(SITE_ROOT, 'static', 'resources', 'xsl', xsl_path),'r') fileContent = file.read() xsl = {} xsl['name'] = xsl_name xsl['filename'] = xsl_path xsl['content'] = fileContent xsl_id = xsl_col.insert(xsl) template_results_id[xsl_name] = str(xsl_id) templates = db['template'] results_xslt = {'ResultXsltList': template_results_id['full'], 'ResultXsltDetailed': template_results_id['detail']} templates.update({}, {"$set":results_xslt}, upsert=False, multi=True)
def load_templates(): """ Loads templates/xslt for NMRR the first time """ # if templates are already present, initialization already happened existing_templates = Template.objects() if len(existing_templates) == 0: templates = { 'all': 'AllResources.xsd', 'organization': 'Organization.xsd', 'datacollection': 'DataCollection.xsd', 'repository': 'Repository.xsd', 'projectarchive': 'ProjectArchive.xsd', 'database': 'Database.xsd', 'dataset': 'Dataset.xsd', 'document': 'Document.xsd', 'service': 'Service.xsd', 'informational': 'Informational.xsd', 'software': 'Software.xsd', } template_ids = [] template_results = { 'full': 'nmrr-full.xsl', 'detail': 'nmrr-detail.xsl', } template_results_id = { 'full': None, 'detail': None, } # connect to mongo client = MongoClient(MONGODB_URI) # connect to the db 'mgi' db = client[MGI_DB] # Add the templates for template_name, template_path in templates.iteritems(): file = open( os.path.join(SITE_ROOT, 'static', 'resources', 'xsd', template_path), 'r') templateContent = file.read() hash = XSDhash.get_hash(templateContent) #create template/ template version objectVersions = TemplateVersion(nbVersions=1, isDeleted=False).save() object = Template(title=template_name, filename=template_path, content=templateContent, version=1, templateVersion=str(objectVersions.id), hash=hash).save() objectVersions.versions = [str(object.id)] objectVersions.current = str(object.id) objectVersions.save() object.save() # save template id template_ids.append(str(object.id)) # Add xslt xsl_col = db['result_xslt'] for xsl_name, xsl_path in template_results.iteritems(): file = open( os.path.join(SITE_ROOT, 'static', 'resources', 'xsl', xsl_path), 'r') fileContent = file.read() xsl = {} xsl['name'] = xsl_name xsl['filename'] = xsl_path xsl['content'] = fileContent xsl_id = xsl_col.insert(xsl) template_results_id[xsl_name] = str(xsl_id) templates = db['template'] results_xslt = { 'ResultXsltList': template_results_id['full'], 'ResultXsltDetailed': template_results_id['detail'] } templates.update({}, {"$set": results_xslt}, upsert=False, multi=True)
def createTemplateWithTemplateVersionInvalidContent(self, templateVersionId): hash = XSDhash.get_hash(TEMPLATE_VALID_CONTENT) template = Template(title='test', filename='test', content=TEMPLATE_INVALID_CONTENT, version=1, templateVersion=templateVersionId, hash=hash).save() TemplateVersion.objects.get(pk=templateVersionId).update(set__current=template.id) return template