Esempio n. 1
0
def main():
    '''
    This is what will be executed on the cluster
    '''
    logger.info("facttools executor has been started.")

    input_path = os.environ.get("INPUTFILE")
    jar = os.environ.get("JARFILE")
    xml = os.environ.get("XMLFILE")
    output_path = os.environ.get("OUTPUTFILE")
    db_path = os.environ.get("DBPATH")

    hdf_output_path = output_path+".hdf"

    with tempfile.TemporaryDirectory() as output_directory:
        name, _ = os.path.splitext(os.path.basename(output_path))
        json_output_path = os.path.join(output_directory, "{}.json".format(name))

        # logger.info("Writing {} entries to json file  {}".format(len(df), filename))
        call = [
                'java',
                '-XX:MaxHeapSize=1024m',
                '-XX:InitialHeapSize=512m',
                '-XX:CompressedClassSpaceSize=64m',
                '-XX:MaxMetaspaceSize=128m',
                '-XX:+UseConcMarkSweepGC',
                '-XX:+UseParNewGC',
                '-jar',
                jar,
                xml,
                '-Dinput=file:{}'.format(input_path),
                '-Doutput=file:{}'.format(json_output_path),
                '-Ddb=file:{}'.format(db_path),
        ]

        subprocess.check_call(['which', 'java'])
        subprocess.check_call(['free', '-m'])
        subprocess.check_call(['java', '-Xmx512m', '-version'])

        logger.info("Calling fact-tools with call: {}".format(call))
        try:
            subprocess.check_call(call)
        except subprocess.CalledProcessError:
            logger.exception("Fact tools returned an error:")
            if os.path.exists(json_output_path):
                logger.error("Trying to collect output files")
            else:
                logger.error("fact-tools error")

        try:
            df_out = ft_json_to_df(json_output_path)
            df_out["fact_tools"] = os.path.basename(jar)
            df_out["xml"] = os.path.basename(xml)
            df_out.to_hdf(os.path.abspath(hdf_output_path), 'data', mode='w')
            os.remove(input_path)
        except (KeyboardInterrupt, SystemExit):
            raise
        except:
            logger.exception("Fact-tools output could not be read.")