def makePostProcessDownloads():
global WORKER_INSTANCES
start = datetime.datetime.now()
currTime = time.strftime("%d/%m/%Y_%H:%M:%S")
importRDFfiles()
end = datetime.datetime.now()
downloadDocs.saveUpdateStatistics(currTime, start, end, action="download_RDFs")
start = datetime.datetime.now()
currTime = time.strftime("%d/%m/%Y_%H:%M:%S")
importRemoteFiles.importExcelFiles(WORKER_INSTANCES)
end = datetime.datetime.now()
downloadDocs.saveUpdateStatistics(currTime, start, end, action="download_Excels")
def makePostProcessDownloads():
global WORKER_INSTANCES
start = datetime.datetime.now()
currTime = time.strftime("%d/%m/%Y_%H:%M:%S")
importRDFfiles()
end = datetime.datetime.now()
downloadDocs.saveUpdateStatistics(currTime,
start,
end,
action="download_RDFs")
start = datetime.datetime.now()
currTime = time.strftime("%d/%m/%Y_%H:%M:%S")
importRemoteFiles.importExcelFiles(WORKER_INSTANCES)
end = datetime.datetime.now()
downloadDocs.saveUpdateStatistics(currTime,
start,
end,
action="download_Excels")
span = end - start
#save statistics!
note = "importing-aggregating RDF files: \n" + "time spent (h:m:s.mm): " + str(
span) + " \n\n"
comm.saveStatistics(mylargefile, note)
###
###
#Download web documents, which metadata are saved in json/object in cloud storage, save them to alivemaster
#list of json objects
start = datetime.datetime.now()
###CLIENT
client = authenticate_gce.getMyAuthService('storage', 'v1')
listOfJsonObjects(client)
importRemoteFiles.importExcelFiles(WORKER_INSTANCES)
#the time spent
end = datetime.datetime.now()
span = end - start
#save statistics!
note = "downloading files: \n" + "time spent (h:m:s.mm): " + str(
span) + " \nNumber of downloads: " + str(nrOfDownloads) + "\n"
comm.saveStatistics(mylargefile, note)
#store info about state of current rdf graph sizes
filedic = dict()
filedic["statfile"] = mylargefile
subprocess.Popen(
["python3", "amountOfTriples.py",
json.dumps(filedic)])
end = datetime.datetime.now()
span = end-start
#save statistics!
note = "importing-aggregating RDF files: \n"+"time spent (h:m:s.mm): " + str(span) + " \n\n"
comm.saveStatistics(mylargefile, note)
###
###
#Download web documents, which metadata are saved in json/object in cloud storage, save them to alivemaster
#list of json objects
start = datetime.datetime.now()
###CLIENT
client = authenticate_gce.getMyAuthService('storage', 'v1')
listOfJsonObjects(client)
importRemoteFiles.importExcelFiles(WORKER_INSTANCES)
#the time spent
end = datetime.datetime.now()
span = end-start
#save statistics!
note = "downloading files: \n"+"time spent (h:m:s.mm): " + str(span) + " \nNumber of downloads: " + str(nrOfDownloads) + "\n"
comm.saveStatistics(mylargefile, note)
#store info about state of current rdf graph sizes
filedic = dict()
filedic["statfile"] = mylargefile
subprocess.Popen(["python3", "amountOfTriples.py", json.dumps(filedic)])
#end of statistics!
comm.saveStatistics(mylargefile, "\n--------------------------- ")
