#     "email" : "*****@*****.**",
#     "custom_name" : "My Full test request",
#     "status" : 1,
#     "priority" : 0,
#     "stage" : [
#         {
#             "name" : "submit",
#             "time" : 1470299931
#         },
#         {
#             "name" : "prep",
#             "time" : 1470419931
#         },
#         {
#             "name" : "process",
#             "time" : 1470419931
#         },
#         {
#             "name" : "complete",
#             "time" : 1470419933
#         }
#     ]
# }



from documentation_tool import DocBuilder
doc = DocBuilder(client, request, '/sciclone/aiddata10/REU/test/doc_test.pdf', "geo.aiddata.org")
doc.build_doc()

Example #2
0
    def build_output(self, request, merge_list, branch):
        """build output

        merge extracts, generate documentation, update status,
            cleanup working directory, send final email
        """
        request_id = str(request['_id'])
        request['_id'] = request_id

        results_dir = os.path.join(self.branch_info.data_root, "outputs",
                       self.branch, "det/results")

        request_dir = os.path.join(results_dir, request_id)

        shutil.rmtree(request_dir, ignore_errors=True)

        merge_output = os.path.join(request_dir,
                                    "{0}_results.csv".format(request_id))

        # merge cached results if all are available
        merge_status = self.merge_file_list(merge_list, merge_output)

        if not merge_status:
            raise Exception('\tWarning: no extracts merged for '
                            'request_id = {0}').format(request_id)
        else:
            print '\tMerge completed for {0}'.format(request_id)


        # generate documentation
        doc_output =  os.path.join(request_dir,
                                   "{0}_documentation.pdf".format(request_id))
        doc = DocBuilder(self.client, request, doc_output, self.branch_info.det['download_server'])
        bd_status = doc.build_doc()
        # print bd_status


        # output request doc as json
        print "creating request json"
        rdoc_path = os.path.join(request_dir, "request_details.json")
        rdoc_file = open(rdoc_path, "w")
        json.dump(request, rdoc_file, indent=4)
        rdoc_file.close()


        geo_pdf_src = self.dir_base + "/other/IntroducingtheAidDataGeoFramework.pdf"
        geo_pdf_dst = os.path.join(request_dir, "IntroducingtheAidDataGeoFramework.pdf")
        shutil.copyfile(geo_pdf_src, geo_pdf_dst)


        # # make msr json folder in request_dir
        # msr_jsons_dir = os.path.join(request_dir, 'msr_jsons')
        # make_dir(msr_jsons_dir)

        # # copy all msr jsons into msr json folder
        # for i in request['release_data']:
        #     tmp_dataset = i['dataset']
        #     tmp_hash = i['hash']

        #     src = "{0}/outputs/{1}/msr/done/{2}/{3}/summary.json".format(
        #         self.branch_info.data_root, branch, tmp_dataset, tmp_hash)
        #     dst = os.path.join(msr_jsons_dir, "{0}_{1}.json".format(
        #         tmp_dataset, tmp_hash))

        #     shutil.copyfile(src, dst)


        # make msr aid folder in request_dir
        msr_aid_dir = os.path.join(request_dir, 'raw_aid_data')
        make_dir(msr_aid_dir)

        # copy all aid csv into msr aid folder
        for i in request['release_data']:
            tmp_dataset = i['dataset']
            tmp_hash = i['hash']

            src = "{0}/outputs/{1}/msr/done/{2}/{3}/project_locations.csv".format(
                self.branch_info.data_root, branch, tmp_dataset, tmp_hash)
            dst = os.path.join(msr_aid_dir, "{0}_{1}.csv".format(
                tmp_dataset, tmp_hash))
            try:
                shutil.copyfile(src, dst)
            except:
                pass


        # make zip of request dir
        shutil.make_archive(request_dir, "zip", request_dir)

        # move zip of request dir into request dir
        shutil.move(request_dir + ".zip", request_dir)


        # remove unzipped files which do not need direct access
        os.remove(geo_pdf_dst)
Example #3
0
    def build_output(self, request, merge_list, branch):
        """build output

        merge extracts, generate documentation, update status,
            cleanup working directory, send final email
        """
        request_id = str(request['_id'])
        request['_id'] = request_id

        results_dir = ("/sciclone/aiddata10/REU/outputs/" +
                       self.branch + "/det/results")

        request_dir = os.path.join(results_dir, request_id)

        shutil.rmtree(request_dir, ignore_errors=True)

        merge_output = os.path.join(request_dir,
                                    "{0}_results.csv".format(request_id))

        # merge cached results if all are available
        merge_status = self.merge_file_list(merge_list, merge_output)

        if not merge_status:
            raise Exception('\tWarning: no extracts merged for '
                            'request_id = {0}').format(request_id)
        else:
            print '\tMerge completed for {0}'.format(request_id)


        # generate documentation
        doc_output =  os.path.join(request_dir,
                                   "{0}_documentation.pdf".format(request_id))
        doc = DocBuilder(self.client, request, doc_output, self.branch_info.det['download_server'])
        bd_status = doc.build_doc()
        # print bd_status


        # output request doc as json
        print "creating request json"
        rdoc_path = os.path.join(request_dir, "request_details.json")
        rdoc_file = open(rdoc_path, "w")
        json.dump(request, rdoc_file, indent=4)
        rdoc_file.close()


        # make msr json folder in request_dir
        msr_jsons_dir = os.path.join(request_dir, 'msr_jsons')
        make_dir(msr_jsons_dir)

        # copy all msr jsons into msr json folder
        for i in request['release_data']:
            tmp_dataset = i['dataset']
            tmp_hash = i['hash']

            src = "/sciclone/aiddata10/REU/outputs/{0}/msr/done/{1}/{2}/summary.json".format(
                branch, tmp_dataset, tmp_hash)
            dst = os.path.join(msr_jsons_dir, "{0}_{1}.json".format(
                tmp_dataset, tmp_hash))

            shutil.copyfile(src, dst)


        # make zip of request dir
        shutil.make_archive(request_dir, "zip", request_dir)

        # move zip of request dir into request dir
        shutil.move(request_dir + ".zip", request_dir)

        # remove msr json folder from request dir
        shutil.rmtree(msr_jsons_dir)