def main():
sourceId = os.environ.get('PUSH_SOURCE_ID') or '--Enter your source id--'
orgId = os.environ.get('PUSH_ORG_ID') or '--Enter your org id--'
apiKey = os.environ.get('PUSH_API_KEY') or '--Enter your API key--'
# Setup the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
myfile = os.path.join('testfiles', 'Example.pptx')
# Create a document
mydoc = Document('file:///' + myfile)
# Get the file contents and add it to the document
mydoc.GetFileAndCompress(myfile)
# Set the metadata
mydoc.AddMetadata("connectortype", "PPTX")
# rssauthors should be set as a multi-value field in your Coveo Cloud organization
mydoc.AddMetadata("rssauthors", ["Coveo", "R&D"])
# Set the title
mydoc.Title = "THIS IS A TEST"
# Set permissions
user_email = "*****@*****.**"
# Create a permission Identity
myperm = CoveoPermissions.PermissionIdentity(
CoveoConstants.Constants.PermissionIdentityType.User, "", user_email)
# Set the permissions on the document
allowAnonymous = True
mydoc.SetAllowedAndDeniedPermissions([myperm], [], allowAnonymous)
# Push the document
push.AddSingleDocument(mydoc)
def main():
# setup Push client
sourceId = os.environ.get('PUSH_SOURCE_ID') or '--Enter your source id--'
orgId = os.environ.get('PUSH_ORG_ID') or '--Enter your org id--'
apiKey = os.environ.get('PUSH_API_KEY') or '--Enter your API key--'
updateSourceStatus = True
deleteOlder = True
# Create the push client
push = CoveoPush.Push(sourceId,
orgId,
apiKey,
p_Mode=CoveoConstants.Constants.Mode.UpdateStream)
push.Start(updateSourceStatus, deleteOlder)
# Create a batch of documents
batch = []
myfile = os.path.join('testfiles', 'People.csv')
with open(myfile, 'r') as infile:
posts = csv.DictReader(infile, delimiter=';')
# loop through each post and add to Coveo
for row in posts:
myCoveoDocument = add_document(row)
push.Add(myCoveoDocument)
# End the Push
push.End(updateSourceStatus, deleteOlder)
def main():
sourceId = '--Enter your source id--'
orgId = '--Enter your org id--'
apiKey = '--Enter your API key--'
#Setup the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
myfile = 'testfiles\\Example.pptx'
# Create a document
mydoc = Document('file:///' + myfile)
# Get the file contents and add it to the document
mydoc.GetFileAndCompress(myfile)
# Set the metadata
mydoc.AddMetadata("connectortype", "CSV")
authors = []
authors.append("Coveo")
authors.append("R&D")
# rssauthors should be set as a multi-value field in your Coveo Cloud organization
mydoc.AddMetadata("rssauthors", authors)
# Set the title
mydoc.Title = "THIS IS A TEST"
# Set permissions
user_email = "*****@*****.**"
# Create a permission Identity
myperm = CoveoPermissions.PermissionIdentity(
CoveoConstants.Constants.PermissionIdentityType.User, "", user_email)
# Set the permissions on the document
allowAnonymous = True
mydoc.SetAllowedAndDeniedPermissions([myperm], [], allowAnonymous)
# Push the document
push.AddSingleDocument(mydoc)
def main():
sourceId = '--Enter your source id--'
orgId = '--Enter your org id--'
apiKey = '--Enter your API key--'
# Setup the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
# First add the document
mydoc = Document("https://myreference&id=TESTME")
# Set plain text
mydoc.SetData("ALL OF THESE WORDS ARE SEARCHABLE")
# Set FileExtension
mydoc.FileExtension = ".html"
# Add Metadata
mydoc.AddMetadata("connectortype", "CSV")
# Set the title
mydoc.Title = "THIS IS A TEST"
# Set permissions
user_email = "*****@*****.**"
# Create a permission identity
myperm = CoveoPermissions.PermissionIdentity(
CoveoConstants.Constants.PermissionIdentityType.User, "", user_email)
# Set the permissions on the document
allowAnonymous = True
mydoc.SetAllowedAndDeniedPermissions([myperm], [], allowAnonymous)
# Push the document
push.AddSingleDocument(mydoc)
time.sleep(100)
# Remove it
push.RemoveSingleDocument('https://myreference&id=TESTME')
def main():
sourceId = os.environ.get('PUSH_SOURCE_ID') or '--Enter your source id--'
orgId = os.environ.get('PUSH_ORG_ID') or '--Enter your org id--'
apiKey = os.environ.get('PUSH_API_KEY') or '--Enter your API key--'
updateSourceStatus = True
deleteOlder = True
# Setup the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
# Start the batch
push.Start(updateSourceStatus, deleteOlder)
# Set the maximum
push.SetSizeMaxRequest(150 * 1024 * 1024)
# Add the documents, if the buffer is full it will be pushed
push.Add(createDoc(os.path.join('testfiles', 'Large1.pptx'), '1'))
push.Add(createDoc(os.path.join('testfiles', 'Large2.pptx'), '1'))
push.Add(createDoc(os.path.join('testfiles', 'Large3.pptx'), '1'))
push.Add(createDoc(os.path.join('testfiles', 'Large4.pptx'), '1'))
push.Add(createDoc(os.path.join('testfiles', 'Large5.pptx'), '1'))
push.Add(createDoc(os.path.join('testfiles', 'Large1.pptx'), '2'))
push.Add(createDoc(os.path.join('testfiles', 'Large2.pptx'), '2'))
push.Add(createDoc(os.path.join('testfiles', 'Large3.pptx'), '2'))
push.Add(createDoc(os.path.join('testfiles', 'Large4.pptx'), '2'))
push.Add(createDoc(os.path.join('testfiles', 'Large5.pptx'), '2'))
# End the Push
push.End(updateSourceStatus, deleteOlder)
def get_push_client() -> CoveoPush.Push:
source_id = os.environ.get("PUSH_SOURCE_ID")
org_id = os.environ.get("PUSH_ORG_ID")
api_key = os.environ.get("PUSH_API_KEY")
if not source_id or not org_id or not api_key:
raise EnvironmentError("coveo environment variable are not set")
return CoveoPush.Push(source_id, org_id, api_key)
def main():
sourceId = '--Enter your source id--'
orgId = '--Enter your org id--'
apiKey = '--Enter your API key--'
updateSourceStatus = True
deleteOlder = True
# Setup the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
# Start the batch
push.Start(updateSourceStatus, deleteOlder)
# Set the maximum
push.SetSizeMaxRequest(150 * 1024 * 1024)
# Add the documents, if the buffer is full it will be pushed
push.Add(createDoc('testfiles\\Large1.pptx', '1'))
push.Add(createDoc('testfiles\\Large2.pptx', '1'))
push.Add(createDoc('testfiles\\Large3.pptx', '1'))
push.Add(createDoc('testfiles\\Large4.pptx', '1'))
push.Add(createDoc('testfiles\\Large5.pptx', '1'))
push.Add(createDoc('testfiles\\Large1.pptx', '2'))
push.Add(createDoc('testfiles\\Large2.pptx', '2'))
push.Add(createDoc('testfiles\\Large3.pptx', '2'))
push.Add(createDoc('testfiles\\Large4.pptx', '2'))
push.Add(createDoc('testfiles\\Large5.pptx', '2'))
# End the Push
push.End(updateSourceStatus, deleteOlder)
def main():
sourceId = '--Enter your source id--'
orgId = '--Enter your org id--'
apiKey = '--Enter your API key--'
updateSourceStatus = True
deleteOlder = True
# Setup the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
# Create a batch of documents
batch = []
batch.append(createDoc('testfiles\\BigExample.pdf'))
batch.append(createDoc('testfiles\\BigExample2.pptx'))
#Push the documents
push.AddDocuments(batch, [], updateSourceStatus, deleteOlder)
def main():
sourceId = os.environ.get('PUSH_SOURCE_ID') or '--Enter your source id--'
orgId = os.environ.get('PUSH_ORG_ID') or '--Enter your org id--'
apiKey = os.environ.get('PUSH_API_KEY') or '--Enter your API key--'
# Setup the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
# Create a document
mydoc = Document("https://myreference/doc2")
mydoc.SetData("This is document Two")
mydoc.FileExtension = ".html"
mydoc.AddMetadata("authors", "*****@*****.**")
mydoc.Title = "What's up Doc 2?"
# Push the document
push.AddSingleDocument(mydoc)
def main():
sourceId = os.environ.get('PUSH_SOURCE_ID') or '--Enter your source id--'
orgId = os.environ.get('PUSH_ORG_ID') or '--Enter your org id--'
apiKey = os.environ.get('PUSH_API_KEY') or '--Enter your API key--'
updateSourceStatus = True
deleteOlder = True
# Setup the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
# Create a batch of documents
batch = [
createDoc(os.path.join('testfiles', 'BigExample.pdf')),
createDoc(os.path.join('testfiles', 'BigExample2.pptx'))
]
# Push the documents
push.AddDocuments(batch, [], updateSourceStatus, deleteOlder)
def main():
# setup Push client
sourceId = '--Enter your source id--'
orgId = '--Enter your org id--'
apiKey = '--Enter your API key--'
push = CoveoPush.Push(sourceId, orgId, apiKey)
# Get first ordering id
startOrderingId = push.CreateOrderingId()
with open('testfiles\\people.csv', 'r') as infile:
posts = csv.DictReader(infile, delimiter=';')
# Loop through each post and add to Coveo
for row in posts:
myCoveoDocument = add_document(row)
# Push the document
push.AddSingleDocument(myCoveoDocument)
# Delete older documents
push.DeleteOlderThan(startOrderingId)
def main():
# setup Push client
sourceId = '--Enter your source id--'
orgId = '--Enter your org id--'
apiKey = '--Enter your API key--'
updateSourceStatus = True
deleteOlder = True
# Create the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
push.Start(updateSourceStatus, deleteOlder)
# Create the documents
path = '../Authors'
for filename in os.listdir(path):
with open(path + "/" + filename, encoding='utf8') as data_file:
people = json.load(data_file)
push.Add(create_document(people))
push.End(updateSourceStatus, deleteOlder)
def main():
# setup Push client
sourceId = os.environ.get('PUSH_SOURCE_ID') or '--Enter your source id--'
orgId = os.environ.get('PUSH_ORG_ID') or '--Enter your org id--'
apiKey = os.environ.get('PUSH_API_KEY') or '--Enter your API key--'
push = CoveoPush.Push(sourceId, orgId, apiKey)
# Get first ordering id
startOrderingId = push.CreateOrderingId()
myfile = os.path.join('testfiles', 'People.csv')
with open(myfile, 'r') as infile:
posts = csv.DictReader(infile, delimiter=';')
# Loop through each post and add to Coveo
for row in posts:
myCoveoDocument = add_document(row)
# Push the document
push.AddSingleDocument(myCoveoDocument)
# Delete older documents
push.DeleteOlderThan(startOrderingId)
def main():
#setup Push client
sourceId = '--Enter your source id--'
orgId = '--Enter your org id--'
apiKey = '--Enter your API key--'
updateSourceStatus = True
deleteOlder = True
#Create the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
#Create a batch of documents
batch = []
with open('testfiles\\people.csv', 'r') as infile:
posts = csv.DictReader(infile, delimiter=';')
# loop through each post and add to Coveo
for row in posts:
myCoveoDocument = add_document(row)
batch.append(myCoveoDocument)
#Push the documents
push.AddDocuments(batch, [], updateSourceStatus, deleteOlder)
def main():
sourceId = '--Enter your source id--'
orgId = '--Enter your org id--'
apiKey = '--Enter your API key--'
# Setup the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
# Get a first Ordering Id
startOrderingId = push.CreateOrderingId()
# Create a document
mydoc = Document("https://myreference&id=TESTME")
# Set plain text
mydoc.SetData("ALL OF THESE WORDS ARE SEARCHABLE")
# Set FileExtension
mydoc.FileExtension = ".html"
# Add Metadata
mydoc.AddMetadata("connectortype", "CSV")
authors = []
authors.append("Coveo")
authors.append("R&D")
# rssauthors should be set as a multi-value field in your Coveo Cloud organization
mydoc.AddMetadata("rssauthors", authors)
# Set the Title
mydoc.Title = "THIS IS A TEST"
# Set permissions
user_email = "*****@*****.**"
# Create a permission identity
myperm = CoveoPermissions.PermissionIdentity(
CoveoConstants.Constants.PermissionIdentityType.User, "", user_email)
# Set the permissions on the document
allowAnonymous = True
mydoc.SetAllowedAndDeniedPermissions([myperm], [], allowAnonymous)
# Push the document
push.AddSingleDocument(mydoc)
# Delete older documents
push.DeleteOlderThan(startOrderingId)
def main():
sourceId = '--Enter your source id--'
orgId = '--Enter your org id--'
apiKey = '--Enter your API key--'
# Setup the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
# Get a first Ordering Id
startOrderingId = push.CreateOrderingId()
# Create a document
mydoc = Document('https://myreference&id=TESTME')
# Set the content. This will also be available as the quickview.
content = "<meta charset='UTF-16'><meta http-equiv='Content-Type' content='text/html; charset=UTF-16'><html><head><title>My First Title</title><style>.datagrid table { border-collapse: collapse; text-align: left; } .datagrid {display:table !important;font: normal 12px/150% Arial, Helvetica, sans-serif; background: #fff; overflow: hidden; border: 1px solid #006699; -webkit-border-radius: 3px; -moz-border-radius: 3px; border-radius: 3px; }.datagrid table td, .datagrid table th { padding: 3px 10px; }.datagrid table thead th {background:-webkit-gradient( linear, left top, left bottom, color-stop(0.05, #006699), color-stop(1, #00557F) );background:-moz-linear-gradient( center top, #006699 5%, #00557F 100% );filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#006699', endColorstr='#00557F');background-color:#006699; color:#FFFFFF; font-size: 15px; font-weight: bold; border-left: 1px solid #0070A8; } .datagrid table thead th:first-child { border: none; }.datagrid table tbody td { color: #00496B; border-left: 1px solid #E1EEF4;font-size: 12px;font-weight: normal; }.datagrid table tbody tr:nth-child(even) td { background: #E1EEF4; color: #00496B; }.datagrid table tbody td:first-child { border-left: none; }.datagrid table tbody tr:last-child td { border-bottom: none; }</style></head><body style='Font-family:Arial'><div class='datagrid'><table><tbody><tr><td>FirstName</td><td>Willem</td></tr><tr><td>MiddleName</td><td>Van</td></tr><tr><td>LastName</td><td>Post</td></tr><tr><td>PositionDescription</td><td>VP Engineering</td></tr><tr><td>JobFunction</td><td>CTO</td></tr><tr><td>JobFamily</td><td>Management</td></tr></tbody></table></div></body></html>"
mydoc.SetContentAndZLibCompress(content)
# Set the metadata
mydoc.AddMetadata("connectortype", "CSV")
authors = []
authors.append("Coveo")
authors.append("R&D")
# rssauthors should be set as a multi-value field in your Coveo Cloud organization
mydoc.AddMetadata("rssauthors", authors)
# Set the title
mydoc.Title = "THIS IS A TEST"
# Add a user email to be used for identities
user_email = "*****@*****.**"
# Create a permission identity
myperm = CoveoPermissions.PermissionIdentity(
CoveoConstants.Constants.PermissionIdentityType.User, "", user_email)
# Set the permissions on the document
allowAnonymous = True
mydoc.SetAllowedAndDeniedPermissions([myperm], [], allowAnonymous)
# Push the document
push.AddSingleDocument(mydoc)
# Delete older documents
push.DeleteOlderThan(startOrderingId)
def main():
# setup Push client
sourceId = '--Enter your source id--'
orgId = '--Enter your org id--'
apiKey = '--Enter your API key--'
updateSourceStatus = True
deleteOlder = False
# Create the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
push.Start(updateSourceStatus, deleteOlder)
# Create the documents
path = '../Files'
for filename in os.listdir(path):
print(path + "/" + filename)
if os.path.isfile(path + "/" + filename):
push.Add(create_document(path + "/" + filename))
else:
checkDirectory(push, path + "/" + filename)
push.End(updateSourceStatus, deleteOlder)
def main():
sourceId = '--Enter your source id--'
orgId = '--Enter your org id--'
apiKey = '--Enter your API key--'
# Setup the push client
push = CoveoPush.Push(sourceId, orgId, apiKey)
# First set the securityprovidername
mysecprovidername = "MySecurityProviderTest"
# Define cascading security provider information
cascading = {
"Email Security Provider": {
"name": "Email Security Provider",
"type": "EMAIL"
}
}
# Create it
push.AddSecurityProvider(mysecprovidername, "EXPANDED", cascading)
startOrderingId = push.CreateOrderingId()
# Delete all old entries
push.DeletePermissionsOlderThan(mysecprovidername, startOrderingId)
print ("Old ids removed. Updating security cache")
input ("Press any key to continue...")
# Create a document
mydoc = Document('https://myreference&id=TESTMESECURITY')
# Set the content. This will also be available as quickview for that document.
content = "<meta charset='UTF-16'><meta http-equiv='Content-Type' content='text/html; charset=UTF-16'><html><head><title>My First Title</title><style>.datagrid table { border-collapse: collapse; text-align: left; } .datagrid {display:table !important;font: normal 12px/150% Arial, Helvetica, sans-serif; background: #fff; overflow: hidden; border: 1px solid #006699; -webkit-border-radius: 3px; -moz-border-radius: 3px; border-radius: 3px; }.datagrid table td, .datagrid table th { padding: 3px 10px; }.datagrid table thead th {background:-webkit-gradient( linear, left top, left bottom, color-stop(0.05, #006699), color-stop(1, #00557F) );background:-moz-linear-gradient( center top, #006699 5%, #00557F 100% );filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#006699', endColorstr='#00557F');background-color:#006699; color:#FFFFFF; font-size: 15px; font-weight: bold; border-left: 1px solid #0070A8; } .datagrid table thead th:first-child { border: none; }.datagrid table tbody td { color: #00496B; border-left: 1px solid #E1EEF4;font-size: 12px;font-weight: normal; }.datagrid table tbody tr:nth-child(even) td { background: #E1EEF4; color: #00496B; }.datagrid table tbody td:first-child { border-left: none; }.datagrid table tbody tr:last-child td { border-bottom: none; }</style></head><body style='Font-family:Arial'><div class='datagrid'><table><tbody><tr><td>FirstName</td><td>Willem</td></tr><tr><td>MiddleName</td><td>Van</td></tr><tr><td>LastName</td><td>Post</td></tr><tr><td>PositionDescription</td><td>VP Engineering</td></tr><tr><td>JobFunction</td><td>CTO</td></tr><tr><td>JobFamily</td><td>Management</td></tr></tbody></table></div></body></html>"
mydoc.SetContentAndZLibCompress(content)
# Set the metadata
mydoc.AddMetadata("connectortype","CSV")
authors = []
authors.append( "Coveo" )
authors.append( "R&D" )
# rssauthors should be set as a multi-value field in your Coveo Cloud organization
mydoc.AddMetadata("rssauthors", authors)
# Set the title
mydoc.Title = "THIS IS A TEST"
# Define a list of users that should have access to the document.
users = []
users.append("wim")
users.append("peter")
# Define a list of users that should not have access to the document.
deniedusers = []
deniedusers.append("alex")
deniedusers.append("anne")
# Define a list of groups that should have access to the document.
groups = []
groups.append("HR")
groups.append("RD")
groups.append("SALES")
# Create the permission Levels. Each level can include multiple sets.
permLevel1 = CoveoPermissions.DocumentPermissionLevel('First')
permLevel1Set1 = CoveoPermissions.DocumentPermissionSet('1Set1')
permLevel1Set2 = CoveoPermissions.DocumentPermissionSet('1Set2')
permLevel1Set1.AllowAnonymous = False
permLevel1Set2.AllowAnonymous = False
permLevel2 = CoveoPermissions.DocumentPermissionLevel('Second')
permLevel2Set = CoveoPermissions.DocumentPermissionSet('2Set1')
permLevel2Set.AllowAnonymous = False
# Set the allowed permissions for the first set of the first level
for user in users:
# Create the permission identity
permLevel1Set1.AddAllowedPermission(CoveoPermissions.PermissionIdentity(CoveoConstants.Constants.PermissionIdentityType.User, mysecprovidername, user))
#Set the denied permissions for the second set of the first level
for user in deniedusers:
# Create the permission identity
permLevel1Set2.AddDeniedPermission(CoveoPermissions.PermissionIdentity(CoveoConstants.Constants.PermissionIdentityType.User, mysecprovidername, user))
# Set the allowed permissions for the first set of the second level
for group in groups:
# Create the permission identity
permLevel2Set.AddAllowedPermission(CoveoPermissions.PermissionIdentity(CoveoConstants.Constants.PermissionIdentityType.Group, mysecprovidername, group))
# Set the permission sets to the appropriate level
permLevel1.AddPermissionSet(permLevel1Set1)
permLevel1.AddPermissionSet(permLevel1Set2)
permLevel2.AddPermissionSet(permLevel2Set)
# Set the permissions on the document
mydoc.Permissions.append(permLevel1)
mydoc.Permissions.append(permLevel2)
# Push the document
push.AddSingleDocument(mydoc)
# First do a single call to update an identity
# We now also need to add the expansion/memberships/mappings to the security cache
# The previouslt defined identities were: alex, anne, wim, peter
usersingroup = []
usersingroup.append("wimingroup")
usersingroup.append("peteringroup")
# Remove the last group, so we can add it later with a single call
groups.pop()
push.StartExpansion( mysecprovidername )
# group memberships for: HR, RD
for group in groups:
# for each group set the users
members = []
for user in usersingroup:
# Create a permission Identity
members.append(CoveoPermissions.PermissionIdentityExpansion( CoveoConstants.Constants.PermissionIdentityType.User, mysecprovidername, user ))
push.AddExpansionMember(CoveoPermissions.PermissionIdentityExpansion( CoveoConstants.Constants.PermissionIdentityType.Group, mysecprovidername, group ), members, [],[] )
# mappings for all users, from userid to email address
users.extend(deniedusers)
users.extend(usersingroup)
for user in users:
# Create a permission Identity
mappings=[]
mappings.append(CoveoPermissions.PermissionIdentityExpansion( CoveoConstants.Constants.PermissionIdentityType.User, "Email Security Provider", user+"@coveo.com" ))
wellknowns=[]
wellknowns.append(CoveoPermissions.PermissionIdentityExpansion( CoveoConstants.Constants.PermissionIdentityType.Group, mysecprovidername, "Everyone"))
push.AddExpansionMapping(CoveoPermissions.PermissionIdentityExpansion( CoveoConstants.Constants.PermissionIdentityType.User, mysecprovidername, user ), [], mappings, wellknowns )
# Remove deleted users
# Deleted Users
delusers = []
delusers.append("wimn")
delusers.append("petern")
for user in delusers:
# Add each identity to delete to the Deleted
push.AddExpansionDeleted(CoveoPermissions.PermissionIdentityExpansion( CoveoConstants.Constants.PermissionIdentityType.User, mysecprovidername, user ),[],[],[])
# End the expansion and write the last batch
push.EndExpansion( mysecprovidername )
print ("Now updating security cache.")
print ("Check:")
print (" HR/RD groups: members wimingroup, peteringroup")
print (" SALES: should not have any members")
print (" each user: wim, peter, anne, wimingroup should have also mappings to Email security providers")
input ("Press any key to continue...")
# Add a single call, add the Sales group
usersingroup = []
usersingroup.append("wiminsalesgroup")
usersingroup.append("peterinsalesgroup")
members = []
for user in usersingroup:
# Create a permission identity
mappings = []
mappings.append(CoveoPermissions.PermissionIdentityExpansion(CoveoConstants.Constants.PermissionIdentityType.User, "Email Security Provider", user + "@coveo.com"))
wellknowns = []
wellknowns.append(CoveoPermissions.PermissionIdentityExpansion(CoveoConstants.Constants.PermissionIdentityType.Group, mysecprovidername, "Everyone"))
members.append(CoveoPermissions.PermissionIdentityExpansion(CoveoConstants.Constants.PermissionIdentityType.User, mysecprovidername, user))
push.AddPermissionExpansion(mysecprovidername, CoveoPermissions.PermissionIdentityExpansion(CoveoConstants.Constants.PermissionIdentityType.User, mysecprovidername, user), [], mappings, wellknowns)
push.AddPermissionExpansion(mysecprovidername, CoveoPermissions.PermissionIdentityExpansion( CoveoConstants.Constants.PermissionIdentityType.Group, mysecprovidername, "Everyone"), members, [],[])
push.AddPermissionExpansion(mysecprovidername, CoveoPermissions.PermissionIdentityExpansion( CoveoConstants.Constants.PermissionIdentityType.Group, mysecprovidername, "SALES"), members, [],[])
print ("Now updating security cache.")
print ("Check:")
print (" HR/RD groups: members wimingroup, peteringroup")
print (" SALES: should have members wiminsalesgroup, peterinsalesgroup")
print (" each user: wim, peter, anne, wimingroup should also have mappings to Email security providers")
input ("Press any key to continue...")
# Remove a Identity
# Group SALES should be removed
push.RemovePermissionIdentity(mysecprovidername, CoveoPermissions.PermissionIdentityExpansion( CoveoConstants.Constants.PermissionIdentityType.Group, mysecprovidername, "SALES"))
print ("Now updating security cache.")
print ("Check:")
print (" HR/RD groups: members wimingroup,peteringroup")
print (" NO wiminsalesgroup,peterinsalesgroup")
print (" each user: wim, peter, anne, wimingroup should have also mappings to Email security providers")
def scrap():
pokemon_list_page = requests.get('https://pokemondb.net/pokedex/national')
soup_pokemon_list_page = BeautifulSoup(pokemon_list_page.content,
'html.parser')
results = soup_pokemon_list_page.find(id='main')
info_cards = results.find_all('div', class_='infocard')
coveo_source_id = os.environ.get("COVEO_SOURCE_ID")
coveo_api_key = os.environ.get("COVEO_API_KEY")
coveo_org_id = os.environ.get("COVEO_ORG_ID")
push = CoveoPush.Push(coveo_source_id, coveo_org_id, coveo_api_key)
push.Start(True, True)
push.SetSizeMaxRequest(150 * 1024 * 1024)
user_email = os.environ.get("USER_EMAIL")
my_permissions = CoveoPermissions.PermissionIdentity(
CoveoConstants.Constants.PermissionIdentityType.User, "", user_email)
for info_card in info_cards:
pokemon_name = info_card.find('a', class_='ent-name').text
pokemon_page_url = 'https://pokemondb.net' + info_card.find(
'a', class_='ent-name')['href']
document = Document(pokemon_page_url)
pokemon_picture_url = info_card.find('span',
class_='img-fixed img-sprite')
if pokemon_picture_url is None:
pokemon_picture_url = info_card.find(
'span',
class_='img-fixed img-sprite img-sprite-v18')['data-src']
else:
pokemon_picture_url = info_card.find(
'span', class_='img-fixed img-sprite')['data-src']
pokemon_number = info_card.find('small').text[1:]
pokemon_gen = find_gen(int(pokemon_number))
pokemon_types = []
pokemon_types_tags = info_card.find_all('small')[1].find_all('a')
print('scrapping pokemon: ' + pokemon_name + ' | index : ' +
pokemon_number)
for pokemon_type_tag in pokemon_types_tags:
pokemon_types.append(pokemon_type_tag.text)
pokemon_page = requests.get(pokemon_page_url)
soup_pokemon_page = BeautifulSoup(pokemon_page.content, 'html.parser')
results = soup_pokemon_page.find(id='main')
tables = results.find_all('table', class_='vitals-table')
pokemon_species = tables[0].find_all('tr')[2].find('td').text
pokemon_height = tables[0].find_all('tr')[3].find('td').text
pokemon_weight = tables[0].find_all('tr')[4].find('td').text
base_stats = {}
base_stats_tags = tables[3].find_all('tr')
for base_stat_tag in base_stats_tags:
base_stats[base_stat_tag.find('th').text] = base_stat_tag.find(
'td').text
defense = {}
defenses_tables = results.find_all(
'table', class_='type-table type-table-pokedex')
for defense_table in defenses_tables:
for x in range(
0, len(defense_table.find_all('tr')[0].find_all('th'))):
defense[defense_table.find_all('tr')[0].find_all('th')[x].find('a').text] = \
defense_table.find_all('tr')[1].find_all('td')[x].text
document.AddMetadata(
defense_table.find_all('tr')[0].find_all('th')[x].find(
'a').text,
defense_table.find_all('tr')[1].find_all('td')[x].text)
document.Title = pokemon_name
document.SetData(pokemon_page.text)
document.FileExtension = ".html"
document.AddMetadata('name', pokemon_name)
document.AddMetadata('url', pokemon_page_url)
document.AddMetadata('number', pokemon_number)
document.AddMetadata('generation', pokemon_gen)
document.AddMetadata('types', pokemon_types)
document.AddMetadata('specie', pokemon_species)
document.AddMetadata('weight', pokemon_weight)
document.AddMetadata('weight_int',
pokemon_weight[0:pokemon_weight.index('kg') - 1])
document.AddMetadata('height', pokemon_height)
document.AddMetadata('height_int',
pokemon_height[0:pokemon_height.index('m') - 1])
document.AddMetadata('hp', base_stats.get('HP'))
document.AddMetadata('hp_int', base_stats.get('HP'))
document.AddMetadata('attack', base_stats.get('Attack'))
document.AddMetadata('attack_int', base_stats.get('Attack'))
document.AddMetadata('defense', base_stats.get('Defense'))
document.AddMetadata('defense_int', base_stats.get('Defense'))
document.AddMetadata('sp_atk', base_stats.get('Sp.Atk'))
document.AddMetadata('sp_def', base_stats.get('Sp.Def'))
document.AddMetadata('speed', base_stats.get('Speed'))
document.AddMetadata('speed_int', base_stats.get('Speed'))
document.AddMetadata('picture_url', pokemon_picture_url)
document.SetAllowedAndDeniedPermissions([my_permissions], [], True)
print('Send: ' + pokemon_name + ' | index : ' + pokemon_number +
' to the PUSH API')
push.Add(document)
print('Sent: ' + pokemon_name + ' | index : ' + pokemon_number +
' to the PUSH API')
push.End(True, True)
print(str(line_count) + " from: " + filename)
line_count += 1
#Save the export file
filename = "output/" + filename + "_ForSiteCore.json"
os.makedirs(os.path.dirname(filename), exist_ok=True)
filerev = open(filename, "wb")
filerev.write(
json.dumps(currentExport, ensure_ascii=False).encode('utf-8'))
filerev.close()
return
#push = CoveoPush.Push(sourceId, orgId, apiKey, CoveoPush.Constants.PushApiEndpoint.DEV_PUSH_API_URL)
push = CoveoPush.Push(sourceId, orgId, apiKey)
push.SetSizeMaxRequest(50 * 1024 * 1024)
nopush = False
if not nopush:
push.Start(True, True)
seed(1)
noratings = True
parseFile('RefData\\Amsterdam.csv')
parseFile('RefData\\Montreal.csv')
parseFile('RefData\\Paris.csv')
parseFile('RefData\\Quebec.csv')
parseFile('RefData\\Sydney.csv')
parseFile('RefData\\Barcelona.csv')
parseFile('RefData\\SF.csv')
parseFile('RefData\\NY.csv')