def ProcessData(subjectAreaSearch, lst, title):
subjectArea = filemap.FindOntology(subjectAreaSearch)
event_fileList = filemap.GetFullFilename(filemap.FindType('event'),
subjectArea[0])
object_fileList = filemap.GetFullFilename(filemap.FindType('object'),
subjectArea[0])
#print('filemap.FindType(object) = ', filemap.FindType('object'))
#print('subjectAreaSearch = ', subjectAreaSearch)
#print('subjectArea = ', subjectArea)
print('Processing - ', title)
location_fileList = filemap.GetFullFilename(filemap.FindType('location'),
subjectArea[0])
eventsList = [[0, 'date', 'event_type', 'name']]
locationsList = [[0, 'folder']]
objectsList = [[0, 'resources']]
#print(dic.name)
for dictRecord in lst:
text = ''
#print(dictRecord)
for k, v in dictRecord.items(
): # this is the projects, tasks, contacts or goals
#print('key=',k,', val=',v)
#text = v['name']
if k == 'name':
text = v
for e in aikif_events:
if e['table'] == title:
#print('TESTING k=', k, ' == ', e['column'])
if e['column'] in k:
# if text != '':
print('mapping ', text, ' to ', k)
eventsList.append([0, v, k, text])
for dic in aikif_objects:
if dic['table'] == title:
if dic['column'] in k:
objectsList.append([0, text])
for dic in aikif_locations:
if dic['table'] == title:
if dic['column'] in k:
objectsList.append([0, text])
appendToExistingFiles = False
if len(locationsList) > 1:
print('LOCATION : ' + location_fileList + ' (' +
str(len(locationsList)) + ') rows')
ai.SaveFileDataToFile(locationsList, location_fileList,
appendToExistingFiles)
if len(eventsList) > 1:
print('EVENTS : ' + event_fileList + ' (' + str(len(eventsList)) +
') rows')
ai.SaveFileDataToFile(eventsList, event_fileList,
appendToExistingFiles)
if len(objectsList) > 1:
print('OBJECTS : ' + object_fileList + ' (' + str(len(objectsList)) +
') rows')
ai.SaveFileDataToFile(objectsList, object_fileList,
appendToExistingFiles)
import os
import sys
sys.path.append('..//..//aspytk')
import lib_data as dat
import lib_file as fle
import lib_net as net
import AIKIF_utils as aikif
import fileMapping as filemap
srcURL = filemap.GetDataPath() + '\\raw\\shoppingList.txt'
subjectArea = filemap.FindOntology(
'shopping') # should return 'INFO-PIM-SHOPPING'
#print ( 'subjectArea = ' + subjectArea[0])
shopping_fileList = filemap.GetFullFilename(filemap.FindType('thing'),
subjectArea[0])
# Standard functions for all loading scripts to allow querying from file mapper and checker
def GetSrcURL():
return srcURL
def GetTempFile():
return tmpFile
def GetOutputFile():
return shopping_fileList
if len(sys.argv) == 2:
viewStructure = sys.argv[1]
securityLevels = [
'NONE', # anyone
'NETWORK', # all users on any device in the network
'WORK', # your work group (customise this - have as many as desired)
'FAMILY', # your family
'PARTNER', # your partner
'SELF' # yourself
]
privateFiles = []
privateFiles.append(
filemap.GetFullFilename(filemap.FindType('thing'),
filemap.FindOntology('shopping')[0]))
privateFiles.append(
filemap.GetFullFilename(filemap.FindType('event'),
filemap.FindOntology('shopping')[0]))
workFiles = []
workFiles.append(
filemap.GetFullFilename(filemap.FindType('location'),
filemap.FindOntology('file')[0]))
securityMappings = []
def MapSecurityLevel(files, security):
# call this function when adding sensitive data
for file in files:
import lib_data as dat
import AIKIF_utils as aikif
import fileMapping as filemap
silent = 'N'
if len(sys.argv) == 2:
if sys.argv[1] == 'Q':
silent = 'Y'
rawData_fileList = r"T:\user\dev\src\python\AI\data\raw\sample-filelist-for-AIKIF.csv"
#rawData_fileList = r"T:\user\dev\src\python\AI\data\small-filelist-for-AIKIF.csv" # medium list creates 107,000 links - all combinations
subjectArea = filemap.FindOntology('file') # should return 'SYSTEM-PC-FILE'
#print ( 'subjectArea = ' + subjectArea[0])
event_fileList = filemap.GetFullFilename(filemap.FindType('event'),
subjectArea[0])
object_fileList = filemap.GetFullFilename(filemap.FindType('object'),
subjectArea[0])
location_fileList = filemap.GetFullFilename(filemap.FindType('location'),
subjectArea[0])
link_fileList = filemap.GetFullFilename(filemap.FindType('link'),
subjectArea[0])
# Standard functions for all loading scripts to allow querying from file mapper and checker
def GetSrcURL():
return ''
def GetTempFile():
return ''
import AIKIF_utils as aikif
import fileMapping as filemap
import index
import json
from pprint import pprint
silent = 'N'
if len(sys.argv) == 2:
if sys.argv[1] == 'Q':
silent = 'Y'
rootFolderLectures = r"P:\__Downloads\lectures"
subjectArea = filemap.FindOntology('Course') # should return 'INFO-COURSE'
object_fileList = filemap.GetFullFilename(filemap.FindType('object'), subjectArea[0])
location_fileList = filemap.GetFullFilename(filemap.FindType('location'), subjectArea[0])
course_fileList = filemap.GetFullFilename(filemap.FindType('location'), subjectArea[0])
fileSubjectArea = filemap.FindOntology('FILE-LECTURES') # should return 'SYSTEM-PC-FILE-LECTURES'
lectures_fileList = filemap.GetFullFilename(filemap.FindType('thing'), fileSubjectArea[0])
def main():
if silent == 'N':
print('loadInfoCourseLectures.py - process raw information to core tables in AIKIF')
courses, locations, objects = LoadData_Lectures(rootFolderLectures)
aikif.SaveFileDataToFile(locations, location_fileList)
aikif.SaveFileDataToFile(objects, lectures_fileList)
aikif.SaveFileDataToFile(courses, object_fileList)
if silent == 'N':
DisplayShortListOutput(objects, 'objects')
# idea is to document where the datasets are coming from and how they are loaded into Tables
# and or python structures (preferably SQL tables)
import os
import sys
sys.path.append('..//..//aspytk')
import lib_data as dat
import lib_file as fle
import lib_net as net
import AIKIF_utils as aikif
import fileMapping as filemap
srcURL = 'http://gdeltproject.org/data/lookups/CAMEO.country.txt'
tmpFile = filemap.GetDataPath() + '\\raw\\CAMEO.country.txt'
location_fileList = filemap.GetFullFilename(filemap.FindType('location'),
'WORLD')
# Standard functions for all loading scripts to allow querying from file mapper and checker
def GetSrcURL():
return srcURL
def GetTempFile():
return tmpFile
def GetOutputFile():
return location_filelist