def makePostProcessDownloads(): global WORKER_INSTANCES start = datetime.datetime.now() currTime = time.strftime("%d/%m/%Y_%H:%M:%S") importRDFfiles() end = datetime.datetime.now() downloadDocs.saveUpdateStatistics(currTime, start, end, action="download_RDFs") start = datetime.datetime.now() currTime = time.strftime("%d/%m/%Y_%H:%M:%S") importRemoteFiles.importExcelFiles(WORKER_INSTANCES) end = datetime.datetime.now() downloadDocs.saveUpdateStatistics(currTime, start, end, action="download_Excels")
span = end - start #save statistics! note = "importing-aggregating RDF files: \n" + "time spent (h:m:s.mm): " + str( span) + " \n\n" comm.saveStatistics(mylargefile, note) ### ### #Download web documents, which metadata are saved in json/object in cloud storage, save them to alivemaster #list of json objects start = datetime.datetime.now() ###CLIENT client = authenticate_gce.getMyAuthService('storage', 'v1') listOfJsonObjects(client) importRemoteFiles.importExcelFiles(WORKER_INSTANCES) #the time spent end = datetime.datetime.now() span = end - start #save statistics! note = "downloading files: \n" + "time spent (h:m:s.mm): " + str( span) + " \nNumber of downloads: " + str(nrOfDownloads) + "\n" comm.saveStatistics(mylargefile, note) #store info about state of current rdf graph sizes filedic = dict() filedic["statfile"] = mylargefile subprocess.Popen( ["python3", "amountOfTriples.py", json.dumps(filedic)])
end = datetime.datetime.now() span = end-start #save statistics! note = "importing-aggregating RDF files: \n"+"time spent (h:m:s.mm): " + str(span) + " \n\n" comm.saveStatistics(mylargefile, note) ### ### #Download web documents, which metadata are saved in json/object in cloud storage, save them to alivemaster #list of json objects start = datetime.datetime.now() ###CLIENT client = authenticate_gce.getMyAuthService('storage', 'v1') listOfJsonObjects(client) importRemoteFiles.importExcelFiles(WORKER_INSTANCES) #the time spent end = datetime.datetime.now() span = end-start #save statistics! note = "downloading files: \n"+"time spent (h:m:s.mm): " + str(span) + " \nNumber of downloads: " + str(nrOfDownloads) + "\n" comm.saveStatistics(mylargefile, note) #store info about state of current rdf graph sizes filedic = dict() filedic["statfile"] = mylargefile subprocess.Popen(["python3", "amountOfTriples.py", json.dumps(filedic)]) #end of statistics! comm.saveStatistics(mylargefile, "\n--------------------------- ")