def main(filename, schemaname): value = validateJson.validate(schemaname, filename) if value == 'False': sys.exit() json_file = open(filename, 'r') json_data = json.load(json_file) numlines = (len(json_data)) #print(numlines) for i in range(numlines): actionrequest = json_data[i]["actions"] for index, actions in enumerate(actionrequest): action = actionrequest[index]["action"] bin = actionrequest[index]["bin"] if not os.path.exists(os.getcwd( ) + '/data-analytics-pipeline/test/results/h6/output/' + action): os.makedirs( os.getcwd() + '/data-analytics-pipeline/test/results/h6/output/' + action) phases = actionrequest[index]["phases"] cdPlot(action, bin, len(phases), phases) print(" -- h6 --") print(" -- good termination --")
def main(): if (len(sys.argv) != 2): print (" Error. Incorrect usage.") print (" usage: exec infile outfile.") print (" Halt.") quit() configfile=sys.argv[1] json_file = open(configfile, 'r') json_data = json.load(json_file) numlines= (len(json_data)) #for i in range(numlines): schemainputdata=os.getcwd()+'/jsonInputPipeline/schemas/property-inference-pipeline.json' inputdata=os.getcwd()+'/jsonInputPipeline/input/property-inference-pipeline.json' value=validateJson.validate(schemainputdata,inputdata) if value=='False': sys.exit() functions=json_data["functions"] numlines= (len(functions)) for i in range(numlines): function=functions[i]["function"] if function=='h1': if os.path.exists(os.getcwd()+'/property-inference-pipeline/json-schema/inputTransformation/datasets/h1.json') and os.path.exists(os.getcwd()+'/property-inference-pipeline/json-schema/inputTransformation/schemas/h1.json'): filename = os.getcwd()+'/property-inference-pipeline/json-schema/inputTransformation/datasets/h1.json' schemaname = os.getcwd()+'/property-inference-pipeline/json-schema/inputTransformation/schemas/h1.json' h1Transformation.main(filename,schemaname) foldermodel=functions[i]["foldermodel"] inputpath=os.getcwd()+'/property-inference-pipeline/json-schema/h1/input/datasets/' filename="tsData.csv" #print(groupslist) h1.main(inputpath,filename,foldermodel) #else: # print("No input of schema for h1") if function=='h2': groupslist=[] groups=functions[i]["groups"] numlinesgroups= (len(groups)) for ii in range(numlinesgroups): groupslist.append([groups[ii]["n"],groups[ii]["d"]]) if os.path.exists(os.getcwd()+'/property-inference-pipeline/json-schema/inputTransformation/datasets/h1.json') and os.path.exists(os.getcwd()+'/property-inference-pipeline/json-schema/inputTransformation/schemas/h1.json'): filename = os.getcwd()+'/property-inference-pipeline/json-schema/inputTransformation/datasets/h1.json' schemaname = os.getcwd()+'/property-inference-pipeline/json-schema/inputTransformation/schemas/h1.json' h2Transformation.main(filename,schemaname,groupslist) inputpath=os.getcwd()+'/property-inference-pipeline/json-schema/h1/input/datasets/' filename="tsData.csv" h2.main(inputpath,filename,groupslist) else: print("No input of schema for h2") endTime=datetime.now() #print (" elapsed time (seconds): ",endTime-startTime) #print (" elapsed time (hours): ",(endTime-startTime)/3600.0) print (" -- good termination --") return
def main(): if (len(sys.argv) != 2): print(" Error. Incorrect usage.") print(" usage: exec infile outfile.") print(" Halt.") quit() configfile = sys.argv[1] json_file = open(configfile, 'r') json_data = json.load(json_file) numlines = (len(json_data)) #for i in range(numlines): schemainputdata = os.getcwd( ) + '/jsonInputPipeline/schemas/experiment-pipeline.json' inputdata = os.getcwd( ) + '/jsonInputPipeline/input/experiment-pipeline.json' value = validateJson.validate(schemainputdata, inputdata) if value == 'False': sys.exit() functions = json_data["functions"] numlines = (len(functions)) for i in range(numlines): function = functions[i]["function"] if function == 'h1': if "path" in functions[i]: inputpath = os.getcwd() + functions[i]["path"] anagrams = functions[i]["anagrams"] CompletedSessionSummary = functions[i][ "CompletedSessionSummary"] InstructionsAnagrams = functions[i]["InstructionsAnagrams"] LetterTransactions = functions[i]["LetterTransactions"] Neighbors = functions[i]["Neighbors"] PublicGoods = functions[i]["PublicGoods"] TeamWords = functions[i]["TeamWords"] TimeSpent = functions[i]["TimeSpent"] UserLetters = functions[i]["UserLetters"] h1Transformation.main(inputpath, anagrams, CompletedSessionSummary, InstructionsAnagrams, LetterTransactions, Neighbors, PublicGoods, TeamWords, TimeSpent, UserLetters) #if os.path.exists(os.getcwd()+'/experiment-pipeline/json-schema/h1/input/datasets/h6.json') and os.path.exists(os.getcwd()+'/data-analytics-pipeline/json-schema/h6/input/schemas/h6.json'): h1.main( os.getcwd() + '/experiment-pipeline/json-schema/h1/input/datasets', anagrams, CompletedSessionSummary, InstructionsAnagrams, LetterTransactions, Neighbors, PublicGoods, TeamWords, TimeSpent, UserLetters) endTime = datetime.now() #print (" elapsed time (seconds): ",endTime-startTime) #print (" elapsed time (hours): ",(endTime-startTime)/3600.0) print(" -- good termination --") return
def main(filename, schemaname): value = validateJson.validate(schemaname, filename) if value == 'False': sys.exit() json_file = open(filename, 'r') json_data = json.load(json_file) numlines = (len(json_data)) #print(numlines) if not os.path.exists( os.getcwd() + '/data-analytics-pipeline/test/results/h13/output/all'): os.makedirs(os.getcwd() + '/data-analytics-pipeline/test/results/h13/output/all') csvfileall = open( os.getcwd() + '/data-analytics-pipeline/test/results/h13/output/all/tsData.csv', 'w') csvfileall.write( 'session,player,n,d,letters,time,requestsSent,repliesReceived,requestsReceived,repliesSent,words,lastWord,LevenshteinDistance,minLevenshteinDistance,w2,wordlocalrank,wordrank,wordlocalrankp\n' ) #with open(os.getcwd()+'/data-analytics-pipeline/src/h13/all_words.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h13/3k.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h13/20k.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h13/google-10000-english-no-swears.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h13/google-10000-english-usa.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h13/5k1k.txt', 'r') as f: with open(os.getcwd() + '/data-analytics-pipeline/src/h13/20k.txt', 'r') as f: all_words_file = f.readlines() seq = 1 wordranklist = [] for rowword in all_words_file: wordfile = rowword.strip('\n') wordranklist.append([wordfile, seq]) seq = seq + 1 #print(wordranklist) print("num words:", len(all_words_file)) for i in range(numlines): actionrequest = json_data[i]["features"] for index, actions in enumerate(actionrequest): n = actionrequest[index]["n"] d = actionrequest[index]["d"] windowsize = actionrequest[index]["windowsize"] numseconds = actionrequest[index]["numseconds"] #if not os.path.exists(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action): # os.makedirs(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action) phases = actionrequest[index]["phases"] filescsv(n, d, numseconds, len(phases), phases, csvfileall, windowsize, all_words_file, wordranklist) print(" -- h13 --") print(" -- good termination --")
def main(filename, schemaname): value = validateJson.validate(schemaname, filename) if value == 'False': sys.exit() json_file = open(filename, 'r') json_data = json.load(json_file) numlines = (len(json_data)) #print(numlines) if not os.path.exists( os.getcwd() + '/data-analytics-pipeline/test/results/h14/output/all'): os.makedirs(os.getcwd() + '/data-analytics-pipeline/test/results/h14/output/all') csvfileall = open( os.getcwd() + '/data-analytics-pipeline/test/results/h14/output/all/tsData.csv', 'w') csvfileall.write( 'session,player,n,d,letters,neighborLetters,time,requestsSent,repliesReceived,requestsReceived,repliesSent,words,countBuffer,numAllReplies,secondsSpan\n' ) csvfileallP = open( os.getcwd() + '/data-analytics-pipeline/test/results/h14/output/all/tsDataParameters.csv', 'w') csvfileallP.write( 'session,player,n,d,numWords,initialLetters,iLScrabbleSco,numRequests,numReplies,fracReplies\n' ) #with open(os.getcwd()+'/data-analytics-pipeline/src/h14/all_words.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h14/3k.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h14/20k.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h14/google-10000-english-no-swears.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h14/google-10000-english-usa.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h14/5k1k.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h14/5k.txt', 'r') as f: # all_words_file = f.readlines() for i in range(numlines): actionrequest = json_data[i]["features"] for index, actions in enumerate(actionrequest): n = actionrequest[index]["n"] d = actionrequest[index]["d"] windowsize = actionrequest[index]["windowsize"] numseconds = actionrequest[index]["numseconds"] #if not os.path.exists(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action): # os.makedirs(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action) phases = actionrequest[index]["phases"] filescsv(n, d, numseconds, len(phases), phases, csvfileall, windowsize, csvfileallP) print(" -- h14 --") print(" -- good termination --")
def main(filename, schemaname): value = validateJson.validate(schemaname, filename) if value == 'False': sys.exit() json_file = open(filename, 'r') json_data = json.load(json_file) numlines = (len(json_data)) #print(numlines) if not os.path.exists( os.getcwd() + '/data-analytics-pipeline/test/results/h12/output/all'): os.makedirs(os.getcwd() + '/data-analytics-pipeline/test/results/h12/output/all') csvfileall = open( os.getcwd() + '/data-analytics-pipeline/test/results/h12/output/all/tsData.csv', 'w') csvfileall.write( 'session,player,n,d,letters,neighborLetters,time,requestsSent,repliesReceived,requestsReceived,repliesSent,words,lastWord,delta,rankingChosen\n' ) #with open(os.getcwd()+'/data-analytics-pipeline/src/h12/all_words.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h12/3k.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h12/20k.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h12/google-10000-english-no-swears.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h12/google-10000-english-usa.txt', 'r') as f: #with open(os.getcwd()+'/data-analytics-pipeline/src/h12/5k1k.txt', 'r') as f: with open(os.getcwd() + '/data-analytics-pipeline/src/h12/5k.txt', 'r') as f: all_words_file = f.readlines() print("num words:", len(all_words_file)) ranktable = getRank(all_words_file, len(all_words_file)) #print(ranktable) for i in range(numlines): actionrequest = json_data[i]["features"] for index, actions in enumerate(actionrequest): n = actionrequest[index]["n"] d = actionrequest[index]["d"] windowsize = actionrequest[index]["windowsize"] numseconds = actionrequest[index]["numseconds"] #if not os.path.exists(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action): # os.makedirs(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action) phases = actionrequest[index]["phases"] filescsv(n, d, numseconds, len(phases), phases, csvfileall, windowsize, all_words_file, ranktable) print(" -- h12 --") print(" -- good termination --")
def main(filename, schemaname): value = validateJson.validate(schemaname, filename) if value == 'False': sys.exit() json_file = open(filename, 'r') json_data = json.load(json_file) numlines = (len(json_data)) #print(numlines) if not os.path.exists( os.getcwd() + '/data-analytics-pipeline/test/results/h8/output/all'): os.makedirs(os.getcwd() + '/data-analytics-pipeline/test/results/h8/output/all') csvfileallP = open( os.getcwd() + '/data-analytics-pipeline/test/results/h8/output/all/tsDataParameters.csv', 'w') csvfileall = open( os.getcwd() + '/data-analytics-pipeline/test/results/h8/output/all/tsData.csv', 'w') csvfileall.write( 'session,player,type,letters,time,requestsSent,repliesReceived,requestsReceived,repliesSent,words,LevenshteinDistance\n' ) csvfileallP.write( 'session,player,neighbors,n,numWords,fracWordsOwnL,initialLetters,iLScrabbleSco,iLScrabbleSco1,iLScrabbleSco2,iLScrabbleSco3,iLScrabbleSco4,iLScrabbleSco5\n' ) with open(os.getcwd() + '/data-analytics-pipeline/src/h8/all_words.txt', 'r') as f: all_words_file = f.readlines() print("num words:", len(all_words_file)) for i in range(numlines): actionrequest = json_data[i]["features"] for index, actions in enumerate(actionrequest): n = actionrequest[index]["n"] d = actionrequest[index]["d"] windowsize = actionrequest[index]["windowsize"] numseconds = actionrequest[index]["numseconds"] #if not os.path.exists(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action): # os.makedirs(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action) phases = actionrequest[index]["phases"] filescsv(n, d, numseconds, len(phases), phases, csvfileall, windowsize, all_words_file, csvfileallP) print(" -- h8 --") print(" -- good termination --")
def main(filename, schemaname): value = validateJson.validate(schemaname, filename) if value == 'False': sys.exit() json_file = open(filename, 'r') json_data = json.load(json_file) numlines = (len(json_data)) #print(numlines) if not os.path.exists( os.getcwd() + '/data-analytics-pipeline/test/results/h9/output/all'): os.makedirs(os.getcwd() + '/data-analytics-pipeline/test/results/h9/output/all') csvfileall = open( os.getcwd() + '/data-analytics-pipeline/test/results/h9/output/all/tsData.csv', 'w') csvfileall.write( 'session,player,type,category,requestsSent,repliesReceived\n') for i in range(numlines): actionrequest = json_data[i]["features"] for index, actions in enumerate(actionrequest): n = actionrequest[index]["n"] d = actionrequest[index]["d"] cat1numletters = actionrequest[index]["cat1numletters"] cat1time = actionrequest[index]["cat1time"] cat2numwords = actionrequest[index]["cat2numwords"] cat2time = actionrequest[index]["cat2time"] numseconds = actionrequest[index]["numseconds"] #if not os.path.exists(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action): # os.makedirs(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action) phases = actionrequest[index]["phases"] filescsv(n, d, numseconds, len(phases), phases, csvfileall, cat1numletters, cat1time, cat2numwords, cat2time) csvfileall.close() rpath = os.getcwd() + '/data-analytics-pipeline/src/h9' + '/plot.R' subprocess.call([ 'Rscript', rpath, os.getcwd() + '/data-analytics-pipeline/test/results/h9/output/all/tsData.csv' ]) print(" -- h9 --") print(" -- good termination --")
def main(filename, schemaname, groupslist): value = validateJson.validate(schemaname, filename) if value == 'False': sys.exit() json_file = open(filename, 'r') json_data = json.load(json_file) numlines = (len(json_data)) #print(numlines) if not os.path.exists( os.getcwd() + '/property-inference-pipeline/json-schema/h2/input/datasets/all'): os.makedirs( os.getcwd() + '/property-inference-pipeline/json-schema/h2/input/datasets/all') csvfileall = open( os.getcwd() + '/property-inference-pipeline/json-schema/h2/input/datasets/all/tsData.csv', 'w') csvfileall.write('session,player,type,') for i in range(numlines): actionrequest = json_data[i]["features"] for index, actions in enumerate(actionrequest): n = actionrequest[index]["n"] d = actionrequest[index]["d"] windowsize = actionrequest[index]["windowsize"] numseconds = actionrequest[index]["numseconds"] for row in groupslist: ndvalue = [str(n), str(d)] if ndvalue == row: if i == 0 and index == 0: for x in range(0, numseconds): if x == numseconds - 1: csvfileall.write(str(x) + '\n') else: csvfileall.write(str(x) + ',') phases = actionrequest[index]["phases"] filescsv(n, d, numseconds, len(phases), phases, csvfileall) print(" -- h2 Transformation --") print(" -- good termination --")
def main(filename, schemaname): value = validateJson.validate(schemaname, filename) if value == 'False': sys.exit() json_file = open(filename, 'r') json_data = json.load(json_file) numlines = (len(json_data)) #print(numlines) for i in range(numlines): actionrequest = json_data[i]["actions"] for index, actions in enumerate(actionrequest): action = actionrequest[index]["action"] if not os.path.exists(os.getcwd( ) + '/data-analytics-pipeline/test/results/h3/output/' + action): os.makedirs( os.getcwd() + '/data-analytics-pipeline/test/results/h3/output/' + action) phases = actionrequest[index]["phases"] for index2, row in enumerate(phases): session = phases[index2]["phaseid"] #print(session) begin = phases[index2]["begin"] n = phases[index2]["n"] d = phases[index2]["d"] windowsize = phases[index]["windowsize"] if not os.path.exists( os.getcwd() + '/data-analytics-pipeline/test/results/h3/output/' + action + '/' + str(windowsize) + 'S'): os.makedirs( os.getcwd() + '/data-analytics-pipeline/test/results/h3/output/' + action + '/' + str(windowsize) + 'S') players = phases[index2]["players"] cdPlot(action, len(players), players, session, begin, n, d, windowsize) print(" -- h3 --") print(" -- good termination --")
def main(filename, schemaname): value = validateJson.validate(schemaname, filename) if value == 'False': sys.exit() json_file = open(filename, 'r') json_data = json.load(json_file) numlines = (len(json_data)) #print(numlines) if not os.path.exists( os.getcwd() + '/data-analytics-pipeline/test/results/h7/output/all'): os.makedirs(os.getcwd() + '/data-analytics-pipeline/test/results/h7/output/all') csvfileall = open( os.getcwd() + '/data-analytics-pipeline/test/results/h7/output/all/tsData.csv', 'w') csvfileall.write('session,player,type,') for i in range(numlines): actionrequest = json_data[i]["features"] for index, actions in enumerate(actionrequest): n = actionrequest[index]["n"] d = actionrequest[index]["d"] windowsize = actionrequest[index]["windowsize"] numseconds = actionrequest[index]["numseconds"] if i == 0 and index == 0: for x in range(0, numseconds): if x == numseconds - 1: csvfileall.write(str(x) + '\n') else: csvfileall.write(str(x) + ',') #if not os.path.exists(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action): # os.makedirs(os.getcwd()+'/data-analytics-pipeline/test/results/h7/output/'+action) phases = actionrequest[index]["phases"] filescsv(n, d, numseconds, len(phases), phases, csvfileall) print(" -- h7 --") print(" -- good termination --")
def main(filename,schemaname): if not os.path.exists(os.getcwd()+'/data-analytics-pipeline/test/results/h1/visualizationOutput/'): os.makedirs(os.getcwd()+'/data-analytics-pipeline/test/results/h1/visualizationOutput/') value=validateJson.validate(schemaname,filename) if value=='False': sys.exit() json_file = open(filename, 'r') json_data = json.load(json_file) numlines= (len(json_data)) players=[] for i in range(numlines): session=json_data[i]["phaseid"] begin=json_data[i]["begin"] n=json_data[i]["n"] d=json_data[i]["d"] numseconds=json_data[i]["duration"] players=json_data[i]["players"] interaction(len(players),players,session,begin,n,d,numseconds) print (" -- h1 --") print (" -- good termination --")
def main(): if (len(sys.argv) != 2): print(" Error. Incorrect usage.") print(" usage: exec infile outfile.") print(" Halt.") quit() configfile = sys.argv[1] json_file = open(configfile, 'r') json_data = json.load(json_file) numlines = (len(json_data)) #for i in range(numlines): schemainputdata = os.getcwd( ) + '/jsonInputPipeline/schemas/property-inference-pipeline.json' inputdata = os.getcwd( ) + '/jsonInputPipeline/input/property-inference-pipeline.json' value = validateJson.validate(schemainputdata, inputdata) if value == 'False': sys.exit() functions = json_data["functions"] numlines = (len(functions)) for i in range(numlines): function = functions[i]["function"] if function == 'h1': groupslist = [] if "path" in functions[i] and "filename" in functions[i]: inputpath = os.getcwd() + functions[i]["path"] filename = functions[i]["filename"] else: if os.path.exists( os.getcwd() + '/property-inference-pipeline/json-schema/inputTransformation/datasets/h1.json' ) and os.path.exists(os.getcwd( ) + '/data-analytics-pipeline/json-schema/h7/input/schemas/h7.json' ): filename = os.getcwd( ) + '/property-inference-pipeline/json-schema/inputTransformation/datasets/h1.json' schemaname = os.getcwd( ) + '/property-inference-pipeline/json-schema/inputTransformation/schemas/h1.json' h1Transformation.main(filename, schemaname) inputpath = os.getcwd( ) + '/property-inference-pipeline/json-schema/h1/input/datasets/' filename = "tsData.csv" groups = functions[i]["groups"] numlinesgroups = (len(groups)) for ii in range(numlinesgroups): groupslist.append([groups[ii]["n"], groups[ii]["d"]]) h1.main(inputpath, filename, groupslist) #else: # print("No input of schema for h1") if "actionId" in functions[i]: actionId = functions[i]["actionId"] if function == 'h2': #h2Transformation.main(experimentfile,phasedescfile,phasefile,actionfile,actionId) if os.path.exists( os.getcwd() + '/data-analytics-pipeline/json-schema/h2/input/datasets/h2.json' ) and os.path.exists(os.getcwd( ) + '/data-analytics-pipeline/json-schema/h2/input/schemas/h2.json' ): filename = os.getcwd( ) + '/data-analytics-pipeline/json-schema/h2/input/datasets/h2.json' schemaname = os.getcwd( ) + '/data-analytics-pipeline/json-schema/h2/input/schemas/h2.json' #h2.main(filename,schemaname) else: print("No input of schema for h2") if function == 'h3': if "windowSize" in functions[i]: windowSize = functions[i]["windowSize"] h3Transformation.main(experimentfile, phasedescfile, phasefile, actionfile, actionId, windowSize) if os.path.exists( os.getcwd() + '/data-analytics-pipeline/json-schema/h3/input/datasets/h3.json' ) and os.path.exists(os.getcwd( ) + '/data-analytics-pipeline/json-schema/h3/input/schemas/h3.json' ): filename = os.getcwd( ) + '/data-analytics-pipeline/json-schema/h3/input/datasets/h3.json' schemaname = os.getcwd( ) + '/data-analytics-pipeline/json-schema/h3/input/schemas/h3.json' h3.main(filename, schemaname) else: print("No input of schema for h3") endTime = datetime.now() #print (" elapsed time (seconds): ",endTime-startTime) #print (" elapsed time (hours): ",(endTime-startTime)/3600.0) print(" -- good termination --") return