def makePostProcessDownloads():
global WORKER_INSTANCES
start = datetime.datetime.now()
currTime = time.strftime("%d/%m/%Y_%H:%M:%S")
importRDFfiles()
end = datetime.datetime.now()
downloadDocs.saveUpdateStatistics(currTime, start, end, action="download_RDFs")
start = datetime.datetime.now()
currTime = time.strftime("%d/%m/%Y_%H:%M:%S")
importRemoteFiles.importExcelFiles(WORKER_INSTANCES)
end = datetime.datetime.now()
downloadDocs.saveUpdateStatistics(currTime, start, end, action="download_Excels")
def makePostProcessDownloads():
global WORKER_INSTANCES
start = datetime.datetime.now()
currTime = time.strftime("%d/%m/%Y_%H:%M:%S")
importRDFfiles()
end = datetime.datetime.now()
downloadDocs.saveUpdateStatistics(currTime,
start,
end,
action="download_RDFs")
start = datetime.datetime.now()
currTime = time.strftime("%d/%m/%Y_%H:%M:%S")
importRemoteFiles.importExcelFiles(WORKER_INSTANCES)
end = datetime.datetime.now()
downloadDocs.saveUpdateStatistics(currTime,
start,
end,
action="download_Excels")
#download updated RDFs and excel-files
makePostProcessDownloads()
#download new json-files and error-files
start = datetime.datetime.now()
currTime = time.strftime("%d/%m/%Y_%H:%M:%S")
try:
p = subprocess.Popen(["python2", comm.parentDir + "upload_logfile/post_update_downloads.py"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
except Exception as e:
comm.printException(comm.updateErrorsFilePath, errString="POSTUPDATEerror_" + str(e))
pass
end = datetime.datetime.now()
downloadDocs.saveUpdateStatistics(currTime, start, end, action="download_errors_and_jsons")
#download updated datasets
downloadDocs.downloadDocuments(timedir)
###
###
#delete RDF-files and download excel-files in each worker
postToWorker.deleteRDFsInWorkers(ipList)
else:
comm.printException(comm.updateErrorsFilePath, errString='No_instances_to_list.')
except:
comm.printException(comm.updateErrorsFilePath, errString="update_main")
pass
start = datetime.datetime.now()
currTime = time.strftime("%d/%m/%Y_%H:%M:%S")
try:
p = subprocess.Popen([
"python2",
comm.parentDir + "upload_logfile/post_update_downloads.py"
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = p.communicate()
except Exception as e:
comm.printException(comm.updateErrorsFilePath,
errString="POSTUPDATEerror_" + str(e))
pass
end = datetime.datetime.now()
downloadDocs.saveUpdateStatistics(
currTime, start, end, action="download_errors_and_jsons")
#download updated datasets
downloadDocs.downloadDocuments(timedir)
###
###
#delete RDF-files and download excel-files in each worker
postToWorker.deleteRDFsInWorkers(ipList)
else:
comm.printException(comm.updateErrorsFilePath,
errString='No_instances_to_list.')
except:
comm.printException(comm.updateErrorsFilePath, errString="update_main")
pass
