# "email" : "*****@*****.**",
# "custom_name" : "My Full test request",
# "status" : 1,
# "priority" : 0,
# "stage" : [
# {
# "name" : "submit",
# "time" : 1470299931
# },
# {
# "name" : "prep",
# "time" : 1470419931
# },
# {
# "name" : "process",
# "time" : 1470419931
# },
# {
# "name" : "complete",
# "time" : 1470419933
# }
# ]
# }
from documentation_tool import DocBuilder
doc = DocBuilder(client, request, '/sciclone/aiddata10/REU/test/doc_test.pdf', "geo.aiddata.org")
doc.build_doc()
def build_output(self, request, merge_list, branch):
"""build output
merge extracts, generate documentation, update status,
cleanup working directory, send final email
"""
request_id = str(request['_id'])
request['_id'] = request_id
results_dir = os.path.join(self.branch_info.data_root, "outputs",
self.branch, "det/results")
request_dir = os.path.join(results_dir, request_id)
shutil.rmtree(request_dir, ignore_errors=True)
merge_output = os.path.join(request_dir,
"{0}_results.csv".format(request_id))
# merge cached results if all are available
merge_status = self.merge_file_list(merge_list, merge_output)
if not merge_status:
raise Exception('\tWarning: no extracts merged for '
'request_id = {0}').format(request_id)
else:
print '\tMerge completed for {0}'.format(request_id)
# generate documentation
doc_output = os.path.join(request_dir,
"{0}_documentation.pdf".format(request_id))
doc = DocBuilder(self.client, request, doc_output, self.branch_info.det['download_server'])
bd_status = doc.build_doc()
# print bd_status
# output request doc as json
print "creating request json"
rdoc_path = os.path.join(request_dir, "request_details.json")
rdoc_file = open(rdoc_path, "w")
json.dump(request, rdoc_file, indent=4)
rdoc_file.close()
geo_pdf_src = self.dir_base + "/other/IntroducingtheAidDataGeoFramework.pdf"
geo_pdf_dst = os.path.join(request_dir, "IntroducingtheAidDataGeoFramework.pdf")
shutil.copyfile(geo_pdf_src, geo_pdf_dst)
# # make msr json folder in request_dir
# msr_jsons_dir = os.path.join(request_dir, 'msr_jsons')
# make_dir(msr_jsons_dir)
# # copy all msr jsons into msr json folder
# for i in request['release_data']:
# tmp_dataset = i['dataset']
# tmp_hash = i['hash']
# src = "{0}/outputs/{1}/msr/done/{2}/{3}/summary.json".format(
# self.branch_info.data_root, branch, tmp_dataset, tmp_hash)
# dst = os.path.join(msr_jsons_dir, "{0}_{1}.json".format(
# tmp_dataset, tmp_hash))
# shutil.copyfile(src, dst)
# make msr aid folder in request_dir
msr_aid_dir = os.path.join(request_dir, 'raw_aid_data')
make_dir(msr_aid_dir)
# copy all aid csv into msr aid folder
for i in request['release_data']:
tmp_dataset = i['dataset']
tmp_hash = i['hash']
src = "{0}/outputs/{1}/msr/done/{2}/{3}/project_locations.csv".format(
self.branch_info.data_root, branch, tmp_dataset, tmp_hash)
dst = os.path.join(msr_aid_dir, "{0}_{1}.csv".format(
tmp_dataset, tmp_hash))
try:
shutil.copyfile(src, dst)
except:
pass
# make zip of request dir
shutil.make_archive(request_dir, "zip", request_dir)
# move zip of request dir into request dir
shutil.move(request_dir + ".zip", request_dir)
# remove unzipped files which do not need direct access
os.remove(geo_pdf_dst)
def build_output(self, request, merge_list, branch):
"""build output
merge extracts, generate documentation, update status,
cleanup working directory, send final email
"""
request_id = str(request['_id'])
request['_id'] = request_id
results_dir = ("/sciclone/aiddata10/REU/outputs/" +
self.branch + "/det/results")
request_dir = os.path.join(results_dir, request_id)
shutil.rmtree(request_dir, ignore_errors=True)
merge_output = os.path.join(request_dir,
"{0}_results.csv".format(request_id))
# merge cached results if all are available
merge_status = self.merge_file_list(merge_list, merge_output)
if not merge_status:
raise Exception('\tWarning: no extracts merged for '
'request_id = {0}').format(request_id)
else:
print '\tMerge completed for {0}'.format(request_id)
# generate documentation
doc_output = os.path.join(request_dir,
"{0}_documentation.pdf".format(request_id))
doc = DocBuilder(self.client, request, doc_output, self.branch_info.det['download_server'])
bd_status = doc.build_doc()
# print bd_status
# output request doc as json
print "creating request json"
rdoc_path = os.path.join(request_dir, "request_details.json")
rdoc_file = open(rdoc_path, "w")
json.dump(request, rdoc_file, indent=4)
rdoc_file.close()
# make msr json folder in request_dir
msr_jsons_dir = os.path.join(request_dir, 'msr_jsons')
make_dir(msr_jsons_dir)
# copy all msr jsons into msr json folder
for i in request['release_data']:
tmp_dataset = i['dataset']
tmp_hash = i['hash']
src = "/sciclone/aiddata10/REU/outputs/{0}/msr/done/{1}/{2}/summary.json".format(
branch, tmp_dataset, tmp_hash)
dst = os.path.join(msr_jsons_dir, "{0}_{1}.json".format(
tmp_dataset, tmp_hash))
shutil.copyfile(src, dst)
# make zip of request dir
shutil.make_archive(request_dir, "zip", request_dir)
# move zip of request dir into request dir
shutil.move(request_dir + ".zip", request_dir)
# remove msr json folder from request dir
shutil.rmtree(msr_jsons_dir)