示例#1
0
    def test_complex_jython(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if "travis" in sys.executable:
            return

        download_java_standalone()
        assert is_java_installed()

        this = os.path.abspath(os.path.dirname(__file__))
        temp = os.path.join(this, "temp_jython")
        if not os.path.exists(temp):
            os.mkdir(temp)

        jyt = os.path.join(temp, "jy2.py")
        with open(jyt, "w", encoding="utf8") as f:
            f.write('''
                            if __name__ != '__lib__':
                                def outputSchema(dont_care):
                                    def wrapper(func):
                                        def inner(*args, **kwargs):
                                            return func(*args, **kwargs)
                                        return inner
                                    return wrapper

                            import datetime

                            #@outputSchema("brow: (available_bike_stands:chararray, available_bikes:chararray, lat:chararray, lng:chararray, name:chararray, status:chararray)")
                            @outputSchema("brow:chararray")
                            def extract_columns_from_js(row):
                                cols = ["available_bike_stands","available_bikes","lat","lng","name","status"]
                                js = eval(row)
                                res = [ ]
                                for station in js:
                                    vals = [ str(station[c]) for c in cols ]
                                    res.append( tuple(vals) )
                                return res
                            if __name__ != '__lib__':
                                import sys
                                for row in sys.stdin:
                                    row = row.strip()
                                    res = extract_columns_from_js(row)
                                    sys.stdout.write(str(res))
                                    sys.stdout.write("\\n")
                                    sys.stdout.flush()
                    '''.replace("                            ", ""))

        sin =     '''
                    [{'address': 'RUE DES CHAMPEAUX (PRES DE LA GARE ROUTIERE) - 93170 BAGNOLET', 'collect_date': datetime.datetime(2014, 11, 11, 22, 2, 18, 47270), 'lng': 2.416170724425901, 'contract_name': 'Paris', 'name': '31705 - CHAMPEAUX (BAGNOLET)', 'banking': 0, 'lat': 48.8645278209514, 'bonus': 0, 'status': 'OPEN', 'available_bikes': 1, 'last_update': datetime.datetime(2014, 11, 11, 21, 55, 22), 'number': 31705, 'available_bike_stands': 49, 'bike_stands': 50}]
                    [{'address': 'RUE DES CHAMPEAUX (PRES DE LA GARE ROUTIERE) - 93170 BAGNOLET', 'collect_date': datetime.datetime(2014, 11, 11, 22, 2, 18, 47270), 'lng': 2.416170724425901, 'contract_name': 'Paris', 'name': '31705 - CHAMPEAUX (BAGNOLET)', 'banking': 0, 'lat': 48.8645278209514, 'bonus': 0, 'status': 'OPEN', 'available_bikes': 1, 'last_update': datetime.datetime(2014, 11, 11, 21, 55, 22), 'number': 31705, 'available_bike_stands': 49, 'bike_stands': 50}]
                    '''.replace("                    ", "").strip("\r\n ")
        out, err = run_jython(jyt, sin=sin, fLOG=fLOG)
        fLOG("OUT:\n", out)
        fLOG("ERR:\n", err)
        exp = "[('49', '1', '48.864527821',"
        if exp not in out:
            raise Exception("OUT:\n{0}\nERR:\n{1}\n".format(out, err))
示例#2
0
    def test_complex_jython(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        if is_travis_or_appveyor():
            # to avoid timeout on appveyor
            return

        download_java_standalone()
        assert is_java_installed()

        this = os.path.abspath(os.path.dirname(__file__))
        temp = os.path.join(this, "temp_jython")
        if not os.path.exists(temp):
            os.mkdir(temp)

        jyt = os.path.join(temp, "jy2.py")
        with open(jyt, "w", encoding="utf8") as f:
            f.write('''
                            if __name__ != '__lib__':
                                def outputSchema(dont_care):
                                    def wrapper(func):
                                        def inner(*args, **kwargs):
                                            return func(*args, **kwargs)
                                        return inner
                                    return wrapper

                            import datetime

                            #@outputSchema("brow: (available_bike_stands:chararray, available_bikes:chararray, lat:chararray, lng:chararray, name:chararray, status:chararray)")
                            @outputSchema("brow:chararray")
                            def extract_columns_from_js(row):
                                cols = ["available_bike_stands","available_bikes","lat","lng","name","status"]
                                js = eval(row)
                                res = [ ]
                                for station in js:
                                    vals = [ str(station[c]) for c in cols ]
                                    res.append( tuple(vals) )
                                return res
                            if __name__ != '__lib__':
                                import sys
                                for row in sys.stdin:
                                    row = row.strip()
                                    res = extract_columns_from_js(row)
                                    sys.stdout.write(str(res))
                                    sys.stdout.write("\\n")
                                    sys.stdout.flush()
                    '''.replace("                            ", ""))

        sin = '''
                    [{'address': 'RUE DES CHAMPEAUX (PRES DE LA GARE ROUTIERE) - 93170 BAGNOLET', 'collect_date': datetime.datetime(2014, 11, 11, 22, 2, 18, 47270), 'lng': 2.416170724425901, 'contract_name': 'Paris', 'name': '31705 - CHAMPEAUX (BAGNOLET)', 'banking': 0, 'lat': 48.8645278209514, 'bonus': 0, 'status': 'OPEN', 'available_bikes': 1, 'last_update': datetime.datetime(2014, 11, 11, 21, 55, 22), 'number': 31705, 'available_bike_stands': 49, 'bike_stands': 50}]
                    [{'address': 'RUE DES CHAMPEAUX (PRES DE LA GARE ROUTIERE) - 93170 BAGNOLET', 'collect_date': datetime.datetime(2014, 11, 11, 22, 2, 18, 47270), 'lng': 2.416170724425901, 'contract_name': 'Paris', 'name': '31705 - CHAMPEAUX (BAGNOLET)', 'banking': 0, 'lat': 48.8645278209514, 'bonus': 0, 'status': 'OPEN', 'available_bikes': 1, 'last_update': datetime.datetime(2014, 11, 11, 21, 55, 22), 'number': 31705, 'available_bike_stands': 49, 'bike_stands': 50}]
                    '''.replace("                    ", "").strip("\r\n ")
        out, err = run_jython(jyt, sin=sin, fLOG=fLOG)
        fLOG("OUT:\n", out)
        fLOG("ERR:\n", err)
        exp = "[('49', '1', '48.864527821',"
        if exp not in out:
            raise Exception("OUT:\n{0}\nERR:\n{1}\n".format(out, err))
示例#3
0
    def test_magic_command_jython(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if "travis" in sys.executable:
            return

        temp = get_temp_folder(__file__, "temp_magic_command_jython")

        download_java_standalone()
        assert is_java_installed()

        script = """
                    import random

                    @schemaFunction("rsSchema")
                    def rsSchema(input):
                        return input

                    @outputSchemaFunction("rsSchema")
                    def reservoir_sampling(ensemble):
                        ensemble = eval(ensemble)
                        k = 10
                        N = len(ensemble)
                        echantillon = []
                        for i, e in enumerate(ensemble):
                            if len(echantillon) < k:
                                echantillon.append(e)
                            else:
                                j = random.randint(0, i)
                                if j < k:
                                    echantillon[j] = e
                        return echantillon
                    """.replace("                    ", "")

        dest = os.path.join(temp, "script.py")
        mg = MagicFile()
        cmd = dest
        fLOG("**", cmd)
        res = mg.PYTHON(cmd, cell=script)
        fLOG(res)
        assert os.path.exists(dest)

        mg = MagicAzure()
        cmd = dest + " reservoir_sampling"
        cell = '{(100001,"AAAAAA"),(99999,"D99999"),(99998,"C99998")}\n'
        res = mg.jython(cmd, cell=cell)
        fLOG(res)
        assert res
示例#4
0
    def test_magic_command_jython(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        if "travis" in sys.executable:
            return

        temp = get_temp_folder(__file__, "temp_magic_command_jython")

        download_java_standalone()
        assert is_java_installed()

        script = """
                    import random

                    @schemaFunction("rsSchema")
                    def rsSchema(input):
                        return input

                    @outputSchemaFunction("rsSchema")
                    def reservoir_sampling(ensemble):
                        ensemble = eval(ensemble)
                        k = 10
                        N = len(ensemble)
                        echantillon = []
                        for i, e in enumerate(ensemble):
                            if len(echantillon) < k:
                                echantillon.append(e)
                            else:
                                j = random.randint(0, i)
                                if j < k:
                                    echantillon[j] = e
                        return echantillon
                    """.replace("                    ", "")

        dest = os.path.join(temp, "script.py")
        mg = MagicFile()
        cmd = dest
        fLOG("**", cmd)
        res = mg.PYTHON(cmd, cell=script)
        fLOG(res)
        assert os.path.exists(dest)

        mg = MagicAzure()
        cmd = dest + " reservoir_sampling"
        cell = '{(100001,"AAAAAA"),(99999,"D99999"),(99998,"C99998")}\n'
        res = mg.jython(cmd, cell=cell)
        fLOG(res)
        assert res
示例#5
0
    def test_simple_pig(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if is_travis_or_appveyor():
            return

        download_java_standalone()
        assert is_java_installed()

        try:
            download_pig_standalone(fLOG=fLOG)
        except FileNotFoundError:
            # for some unknown reason, it requires to be done twice
            # due to FileNotFoundError: [Errno 2] No such file or directory:
            # 'pyensae\\src\\pyensae\\file_helper\\pigjar\\pig-0.15.0\\contrib\\piggybank\\java\\build\\classes\\org\\apache\\pig\\piggybank\\storage\\IndexedStorage$IndexedStorageInputFormat$IndexedStorageRecordReader$IndexedStorageRecordReaderComparator.class'
            download_pig_standalone(fLOG=fLOG)

        # it does not work for the time being
        return

        this = os.path.abspath(os.path.dirname(__file__))
        temp = os.path.join(this, "temp_pig")
        if not os.path.exists(temp):
            os.mkdir(temp)

        rnd = os.path.join(temp, "random.sample.txt")
        with open(rnd, "w") as f:
            for i in range(0, 1000):
                x = random.random()
                f.write(str(x) + "\n")

        pg = os.path.normpath(os.path.join(temp, "..", "p1.pig"))
        tf = "file:/" + rnd.replace("\\", "/")
        with open(pg, "w", encoding="utf8") as f:
            f.write('''
                        values = LOAD '%s' USING PigStorage('\t') AS (x:double);
                        values_h = FOREACH values GENERATE x, ((int)(x / 0.1)) * 0.1 AS h ;
                        hist_group = GROUP values_h BY h ;
                        hist = FOREACH hist_group GENERATE group, COUNT(values_h) AS nb ;
                        STORE hist INTO '%s' USING PigStorage('\t') ;
                        '''.replace("                        ", "") %
                    (tf, tf + ".out.txt"))

        out, err = run_pig(pg, fLOG=fLOG, logpath=temp)
        if "first try with pig" not in out:
            raise Exception("OUT:\n{0}\nERR:\n{1}".format(out, err))
示例#6
0
    def test_simple_pig(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if "travis" in sys.executable:
            return

        download_java_standalone()
        assert is_java_installed()

        download_pig_standalone(fLOG=fLOG)

        # it does not work for the time being
        return

        this = os.path.abspath(os.path.dirname(__file__))
        temp = os.path.join(this, "temp_pig")
        if not os.path.exists(temp):
            os.mkdir(temp)

        rnd = os.path.join(temp, "random.sample.txt")
        with open(rnd, "w") as f:
            for i in range(0, 1000):
                x = random.random()
                f.write(str(x) + "\n")

        pg = os.path.normpath(os.path.join(temp, "..", "p1.pig"))
        tf = "file:/" + rnd.replace("\\", "/")
        with open(pg, "w", encoding="utf8") as f:
            f.write('''
                        values = LOAD '%s' USING PigStorage('\t') AS (x:double);
                        values_h = FOREACH values GENERATE x, ((int)(x / 0.1)) * 0.1 AS h ;
                        hist_group = GROUP values_h BY h ;
                        hist = FOREACH hist_group GENERATE group, COUNT(values_h) AS nb ;
                        STORE hist INTO '%s' USING PigStorage('\t') ;
                        '''.replace("                        ", "") %
                    (tf, tf + ".out.txt"))

        out, err = run_pig(pg, fLOG=fLOG, logpath=temp)
        if "first try with pig" not in out:
            raise Exception("OUT:\n{0}\nERR:\n{1}".format(out, err))
示例#7
0
    def test_simple_jython(self):
        fLOG(__file__,
             self._testMethodName,
             OutputPrint=__name__ == "__main__")

        if is_travis_or_appveyor():
            return

        download_java_standalone()
        assert is_java_installed()

        this = os.path.abspath(os.path.dirname(__file__))
        temp = os.path.join(this, "temp_jython")
        if not os.path.exists(temp):
            os.mkdir(temp)

        jyt = os.path.join(temp, "jy1.py")
        with open(jyt, "w", encoding="utf8") as f:
            f.write('print("first try with jython")')

        out, err = run_jython(jyt, fLOG=fLOG)
        if "first try with jython" not in out:
            raise Exception("OUT:\n{0}\nERR:\n{1}\n".format(out, err))
示例#8
0
    def test_simple_jython(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        if is_travis_or_appveyor():
            return

        download_java_standalone()
        assert is_java_installed()

        this = os.path.abspath(os.path.dirname(__file__))
        temp = os.path.join(this, "temp_jython")
        if not os.path.exists(temp):
            os.mkdir(temp)

        jyt = os.path.join(temp, "jy1.py")
        with open(jyt, "w", encoding="utf8") as f:
            f.write('print("first try with jython")')

        out, err = run_jython(jyt, fLOG=fLOG)
        if "first try with jython" not in out:
            raise Exception("OUT:\n{0}\nERR:\n{1}\n".format(out, err))