def test_complex_jython(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if "travis" in sys.executable:
return
download_java_standalone()
assert is_java_installed()
this = os.path.abspath(os.path.dirname(__file__))
temp = os.path.join(this, "temp_jython")
if not os.path.exists(temp):
os.mkdir(temp)
jyt = os.path.join(temp, "jy2.py")
with open(jyt, "w", encoding="utf8") as f:
f.write('''
if __name__ != '__lib__':
def outputSchema(dont_care):
def wrapper(func):
def inner(*args, **kwargs):
return func(*args, **kwargs)
return inner
return wrapper
import datetime
#@outputSchema("brow: (available_bike_stands:chararray, available_bikes:chararray, lat:chararray, lng:chararray, name:chararray, status:chararray)")
@outputSchema("brow:chararray")
def extract_columns_from_js(row):
cols = ["available_bike_stands","available_bikes","lat","lng","name","status"]
js = eval(row)
res = [ ]
for station in js:
vals = [ str(station[c]) for c in cols ]
res.append( tuple(vals) )
return res
if __name__ != '__lib__':
import sys
for row in sys.stdin:
row = row.strip()
res = extract_columns_from_js(row)
sys.stdout.write(str(res))
sys.stdout.write("\\n")
sys.stdout.flush()
'''.replace(" ", ""))
sin = '''
[{'address': 'RUE DES CHAMPEAUX (PRES DE LA GARE ROUTIERE) - 93170 BAGNOLET', 'collect_date': datetime.datetime(2014, 11, 11, 22, 2, 18, 47270), 'lng': 2.416170724425901, 'contract_name': 'Paris', 'name': '31705 - CHAMPEAUX (BAGNOLET)', 'banking': 0, 'lat': 48.8645278209514, 'bonus': 0, 'status': 'OPEN', 'available_bikes': 1, 'last_update': datetime.datetime(2014, 11, 11, 21, 55, 22), 'number': 31705, 'available_bike_stands': 49, 'bike_stands': 50}]
[{'address': 'RUE DES CHAMPEAUX (PRES DE LA GARE ROUTIERE) - 93170 BAGNOLET', 'collect_date': datetime.datetime(2014, 11, 11, 22, 2, 18, 47270), 'lng': 2.416170724425901, 'contract_name': 'Paris', 'name': '31705 - CHAMPEAUX (BAGNOLET)', 'banking': 0, 'lat': 48.8645278209514, 'bonus': 0, 'status': 'OPEN', 'available_bikes': 1, 'last_update': datetime.datetime(2014, 11, 11, 21, 55, 22), 'number': 31705, 'available_bike_stands': 49, 'bike_stands': 50}]
'''.replace(" ", "").strip("\r\n ")
out, err = run_jython(jyt, sin=sin, fLOG=fLOG)
fLOG("OUT:\n", out)
fLOG("ERR:\n", err)
exp = "[('49', '1', '48.864527821',"
if exp not in out:
raise Exception("OUT:\n{0}\nERR:\n{1}\n".format(out, err))
def test_complex_jython(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
# to avoid timeout on appveyor
return
download_java_standalone()
assert is_java_installed()
this = os.path.abspath(os.path.dirname(__file__))
temp = os.path.join(this, "temp_jython")
if not os.path.exists(temp):
os.mkdir(temp)
jyt = os.path.join(temp, "jy2.py")
with open(jyt, "w", encoding="utf8") as f:
f.write('''
if __name__ != '__lib__':
def outputSchema(dont_care):
def wrapper(func):
def inner(*args, **kwargs):
return func(*args, **kwargs)
return inner
return wrapper
import datetime
#@outputSchema("brow: (available_bike_stands:chararray, available_bikes:chararray, lat:chararray, lng:chararray, name:chararray, status:chararray)")
@outputSchema("brow:chararray")
def extract_columns_from_js(row):
cols = ["available_bike_stands","available_bikes","lat","lng","name","status"]
js = eval(row)
res = [ ]
for station in js:
vals = [ str(station[c]) for c in cols ]
res.append( tuple(vals) )
return res
if __name__ != '__lib__':
import sys
for row in sys.stdin:
row = row.strip()
res = extract_columns_from_js(row)
sys.stdout.write(str(res))
sys.stdout.write("\\n")
sys.stdout.flush()
'''.replace(" ", ""))
sin = '''
[{'address': 'RUE DES CHAMPEAUX (PRES DE LA GARE ROUTIERE) - 93170 BAGNOLET', 'collect_date': datetime.datetime(2014, 11, 11, 22, 2, 18, 47270), 'lng': 2.416170724425901, 'contract_name': 'Paris', 'name': '31705 - CHAMPEAUX (BAGNOLET)', 'banking': 0, 'lat': 48.8645278209514, 'bonus': 0, 'status': 'OPEN', 'available_bikes': 1, 'last_update': datetime.datetime(2014, 11, 11, 21, 55, 22), 'number': 31705, 'available_bike_stands': 49, 'bike_stands': 50}]
[{'address': 'RUE DES CHAMPEAUX (PRES DE LA GARE ROUTIERE) - 93170 BAGNOLET', 'collect_date': datetime.datetime(2014, 11, 11, 22, 2, 18, 47270), 'lng': 2.416170724425901, 'contract_name': 'Paris', 'name': '31705 - CHAMPEAUX (BAGNOLET)', 'banking': 0, 'lat': 48.8645278209514, 'bonus': 0, 'status': 'OPEN', 'available_bikes': 1, 'last_update': datetime.datetime(2014, 11, 11, 21, 55, 22), 'number': 31705, 'available_bike_stands': 49, 'bike_stands': 50}]
'''.replace(" ", "").strip("\r\n ")
out, err = run_jython(jyt, sin=sin, fLOG=fLOG)
fLOG("OUT:\n", out)
fLOG("ERR:\n", err)
exp = "[('49', '1', '48.864527821',"
if exp not in out:
raise Exception("OUT:\n{0}\nERR:\n{1}\n".format(out, err))
def test_magic_command_jython(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if "travis" in sys.executable:
return
temp = get_temp_folder(__file__, "temp_magic_command_jython")
download_java_standalone()
assert is_java_installed()
script = """
import random
@schemaFunction("rsSchema")
def rsSchema(input):
return input
@outputSchemaFunction("rsSchema")
def reservoir_sampling(ensemble):
ensemble = eval(ensemble)
k = 10
N = len(ensemble)
echantillon = []
for i, e in enumerate(ensemble):
if len(echantillon) < k:
echantillon.append(e)
else:
j = random.randint(0, i)
if j < k:
echantillon[j] = e
return echantillon
""".replace(" ", "")
dest = os.path.join(temp, "script.py")
mg = MagicFile()
cmd = dest
fLOG("**", cmd)
res = mg.PYTHON(cmd, cell=script)
fLOG(res)
assert os.path.exists(dest)
mg = MagicAzure()
cmd = dest + " reservoir_sampling"
cell = '{(100001,"AAAAAA"),(99999,"D99999"),(99998,"C99998")}\n'
res = mg.jython(cmd, cell=cell)
fLOG(res)
assert res
def test_magic_command_jython(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if "travis" in sys.executable:
return
temp = get_temp_folder(__file__, "temp_magic_command_jython")
download_java_standalone()
assert is_java_installed()
script = """
import random
@schemaFunction("rsSchema")
def rsSchema(input):
return input
@outputSchemaFunction("rsSchema")
def reservoir_sampling(ensemble):
ensemble = eval(ensemble)
k = 10
N = len(ensemble)
echantillon = []
for i, e in enumerate(ensemble):
if len(echantillon) < k:
echantillon.append(e)
else:
j = random.randint(0, i)
if j < k:
echantillon[j] = e
return echantillon
""".replace(" ", "")
dest = os.path.join(temp, "script.py")
mg = MagicFile()
cmd = dest
fLOG("**", cmd)
res = mg.PYTHON(cmd, cell=script)
fLOG(res)
assert os.path.exists(dest)
mg = MagicAzure()
cmd = dest + " reservoir_sampling"
cell = '{(100001,"AAAAAA"),(99999,"D99999"),(99998,"C99998")}\n'
res = mg.jython(cmd, cell=cell)
fLOG(res)
assert res
def test_simple_pig(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
return
download_java_standalone()
assert is_java_installed()
try:
download_pig_standalone(fLOG=fLOG)
except FileNotFoundError:
# for some unknown reason, it requires to be done twice
# due to FileNotFoundError: [Errno 2] No such file or directory:
# 'pyensae\\src\\pyensae\\file_helper\\pigjar\\pig-0.15.0\\contrib\\piggybank\\java\\build\\classes\\org\\apache\\pig\\piggybank\\storage\\IndexedStorage$IndexedStorageInputFormat$IndexedStorageRecordReader$IndexedStorageRecordReaderComparator.class'
download_pig_standalone(fLOG=fLOG)
# it does not work for the time being
return
this = os.path.abspath(os.path.dirname(__file__))
temp = os.path.join(this, "temp_pig")
if not os.path.exists(temp):
os.mkdir(temp)
rnd = os.path.join(temp, "random.sample.txt")
with open(rnd, "w") as f:
for i in range(0, 1000):
x = random.random()
f.write(str(x) + "\n")
pg = os.path.normpath(os.path.join(temp, "..", "p1.pig"))
tf = "file:/" + rnd.replace("\\", "/")
with open(pg, "w", encoding="utf8") as f:
f.write('''
values = LOAD '%s' USING PigStorage('\t') AS (x:double);
values_h = FOREACH values GENERATE x, ((int)(x / 0.1)) * 0.1 AS h ;
hist_group = GROUP values_h BY h ;
hist = FOREACH hist_group GENERATE group, COUNT(values_h) AS nb ;
STORE hist INTO '%s' USING PigStorage('\t') ;
'''.replace(" ", "") %
(tf, tf + ".out.txt"))
out, err = run_pig(pg, fLOG=fLOG, logpath=temp)
if "first try with pig" not in out:
raise Exception("OUT:\n{0}\nERR:\n{1}".format(out, err))
def test_simple_pig(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if "travis" in sys.executable:
return
download_java_standalone()
assert is_java_installed()
download_pig_standalone(fLOG=fLOG)
# it does not work for the time being
return
this = os.path.abspath(os.path.dirname(__file__))
temp = os.path.join(this, "temp_pig")
if not os.path.exists(temp):
os.mkdir(temp)
rnd = os.path.join(temp, "random.sample.txt")
with open(rnd, "w") as f:
for i in range(0, 1000):
x = random.random()
f.write(str(x) + "\n")
pg = os.path.normpath(os.path.join(temp, "..", "p1.pig"))
tf = "file:/" + rnd.replace("\\", "/")
with open(pg, "w", encoding="utf8") as f:
f.write('''
values = LOAD '%s' USING PigStorage('\t') AS (x:double);
values_h = FOREACH values GENERATE x, ((int)(x / 0.1)) * 0.1 AS h ;
hist_group = GROUP values_h BY h ;
hist = FOREACH hist_group GENERATE group, COUNT(values_h) AS nb ;
STORE hist INTO '%s' USING PigStorage('\t') ;
'''.replace(" ", "") %
(tf, tf + ".out.txt"))
out, err = run_pig(pg, fLOG=fLOG, logpath=temp)
if "first try with pig" not in out:
raise Exception("OUT:\n{0}\nERR:\n{1}".format(out, err))
def test_simple_jython(self):
fLOG(__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
return
download_java_standalone()
assert is_java_installed()
this = os.path.abspath(os.path.dirname(__file__))
temp = os.path.join(this, "temp_jython")
if not os.path.exists(temp):
os.mkdir(temp)
jyt = os.path.join(temp, "jy1.py")
with open(jyt, "w", encoding="utf8") as f:
f.write('print("first try with jython")')
out, err = run_jython(jyt, fLOG=fLOG)
if "first try with jython" not in out:
raise Exception("OUT:\n{0}\nERR:\n{1}\n".format(out, err))
def test_simple_jython(self):
fLOG(
__file__,
self._testMethodName,
OutputPrint=__name__ == "__main__")
if is_travis_or_appveyor():
return
download_java_standalone()
assert is_java_installed()
this = os.path.abspath(os.path.dirname(__file__))
temp = os.path.join(this, "temp_jython")
if not os.path.exists(temp):
os.mkdir(temp)
jyt = os.path.join(temp, "jy1.py")
with open(jyt, "w", encoding="utf8") as f:
f.write('print("first try with jython")')
out, err = run_jython(jyt, fLOG=fLOG)
if "first try with jython" not in out:
raise Exception("OUT:\n{0}\nERR:\n{1}\n".format(out, err))
