def _generate_procedural_method_code(gateway, client, mapop, name, signatures, instance):
methods = _generate_methods(instance, signatures)
# print("working on " + name)
jvm = gateway.jvm
cls = JavaClass(mapop, gateway_client=client)
is_export = is_remote() and is_instance_of(gateway, cls, jvm.ExportMapOp)
if len(methods) == 0:
return None
signature, self_method = _generate_proc_signature(methods)
generator = CodeGenerator()
# Signature
generator.write("def " + name + "(" + signature + "):", post_indent=True)
# code += " print('" + name + "')\n"
_generate_imports(generator, mapop, is_export)
_generate_calls(generator, methods, is_export=is_export)
_generate_run(generator, instance, is_export)
# print(code)
code = generator.generate()
code = code.replace("self", self_method)
generator.begin()
for line in code.split("\n"):
generator.write(line)
return {name: generator}
def _generate_oo_method_code(gateway, client, mapop, name, signatures,
instance):
methods = _generate_methods(instance, signatures)
# print("working on " + name)
jvm = gateway.jvm
cls = JavaClass(mapop, gateway_client=client)
is_export = is_remote() and is_instance_of(gateway, cls, jvm.ExportMapOp)
if len(methods) == 0:
return None
signature = _generate_oo_signature(methods)
generator = CodeGenerator()
# Signature
generator.write("def " + name + "(" + signature + "):", post_indent=True)
# code += " print('" + name + "')\n"
_generate_imports(generator, mapop, is_export)
_generate_calls(generator, methods, is_export=is_export)
_generate_run(generator, instance, is_export)
# print(code)
return {name: generator}
def _generate_method_code(gateway, client, mapop, name, signatures, instance):
methods = _generate_methods(instance, signatures)
# print("working on " + name)
jvm = gateway.jvm
cls = JavaClass(mapop, gateway_client=client)
is_export = is_remote() and is_instance_of(gateway, cls, jvm.ExportMapOp)
if len(methods) == 0:
return None
signature = _generate_signature(methods)
code = ""
# Signature
code += "def " + name + "(" + signature + "):" + "\n"
# code += " print('" + name + "')\n"
code += _generate_imports(mapop, is_export)
code += _generate_calls(methods, is_export)
code += _generate_run(instance, is_export)
# print(code)
return {name: code}
def _generate_oo_method_code(gateway, client, mapop, name, signatures, instance):
methods = _generate_methods(instance, signatures)
# print("working on " + name)
jvm = gateway.jvm
cls = JavaClass(mapop, gateway_client=client)
is_export = is_remote() and is_instance_of(gateway, cls, jvm.ExportMapOp)
if len(methods) == 0:
return None
signature = _generate_oo_signature(methods)
generator = CodeGenerator()
# Signature
generator.write("def " + name + "(" + signature + "):", post_indent=True)
# code += " print('" + name + "')\n"
_generate_imports(generator, mapop, is_export)
_generate_calls(generator, methods, is_export=is_export)
_generate_run(generator, instance, is_export)
# print(code)
return {name: generator}
def comparevector(self, vector, testname):
if self.GENERATE_BASELINE_DATA:
self.savevector(vector, str(testname))
else:
jvm = self.mrgeo._get_jvm()
# test = raster.mapop.toDataset(False)
java_import(jvm, "org.mrgeo.hdfs.vector.DelimitedVectorReader")
testvector = str(self.outputhdfs + testname + ".tsv")
vector.ssave(testvector)
expectedvector = str(self.inputdir + testname + ".tsv")
vdp_expected = jvm.DataProviderFactory.getVectorDataProvider(
expectedvector, jvm.DataProviderFactory.AccessMode.READ,
jvm.HadoopUtils.createConfiguration())
expected_geom_reader = vdp_expected.getVectorReader().get()
vdp = jvm.DataProviderFactory.getVectorDataProvider(
testvector, jvm.DataProviderFactory.AccessMode.READ,
jvm.HadoopUtils.createConfiguration())
self.assertTrue(vdp is not None)
vector_reader = vdp.getVectorReader()
self.assertTrue(vector_reader is not None)
self.assertTrue(
is_instance_of(self.mrgeo.gateway, vector_reader,
jvm.DelimitedVectorReader))
self.assertEquals(vdp_expected.getVectorReader().count(),
vector_reader.count())
geom_reader = vector_reader.get()
self.assertTrue(geom_reader is not None)
while expected_geom_reader.hasNext():
expected_geom = expected_geom_reader.next()
geom = geom_reader.next()
self.assertTrue(geom is not None)
self.assertEquals(expected_geom.type(), geom.type())
self.assertAlmostEquals(float(
expected_geom.getAttribute("COST_S")),
float(geom.getAttribute("COST_S")),
delta=0.001)
self.assertAlmostEquals(float(
expected_geom.getAttribute("DISTANCE_M")),
float(geom.getAttribute("DISTANCE_M")),
delta=0.001)
self.assertAlmostEquals(
float(expected_geom.getAttribute("MINSPEED_MPS")),
float(geom.getAttribute("MINSPEED_MPS")),
delta=0.001)
self.assertAlmostEquals(
float(expected_geom.getAttribute("MAXSPEED_MPS")),
float(geom.getAttribute("MAXSPEED_MPS")),
delta=0.001)
self.assertAlmostEquals(
float(expected_geom.getAttribute("AVGSPEED_MPS")),
float(geom.getAttribute("AVGSPEED_MPS")),
delta=0.001)
# Should not be any more geometries in the actual output
self.assertFalse(geom_reader.hasNext())
jvm.HadoopFileUtils.delete(testvector)
def _get_instance_type(signatures, gateway, cls, mapop):
type_map = {'RasterMapOp': 0, 'VectorMapOp': 0, 'MapOp': 0}
for sig in signatures:
has_type = {'RasterMapOp': False, 'VectorMapOp': False, 'MapOp': False}
for variable in sig.split("|"):
# print("variable: " + variable)
names = re.split("[:=]+", variable)
new_type = names[1][names[1].rfind('.') + 1:]
if new_type.endswith("*"):
new_type = new_type[:-1]
if new_type == 'RasterMapOp' or new_type == 'VectorMapOp' or new_type == 'MapOp':
has_type[new_type] = True
for t, v in has_type.iteritems():
if v:
type_map[t] += 1
# Make sure that all of the signatures have an argument of one of the map op types.
# If the map op is either RasterMapOp or VectorMapOp, and all of the signatures have
# an argument of that type, then use that for the instance type.
if is_instance_of(gateway, cls, 'org.mrgeo.mapalgebra.raster.RasterMapOp'):
if type_map['RasterMapOp'] == len(signatures):
return 'RasterMapOp'
elif is_instance_of(gateway, cls,
'org.mrgeo.mapalgebra.vector.VectorMapOp'):
if type_map['VectorMapOp'] == len(signatures):
return 'VectorMapOp'
# There is at least one signature that does not include a parameter of the same type
# as the map op itself. Instead, we get a type that is represented in all the signatures.
for t, v in type_map.iteritems():
if v == len(signatures):
return t
msg = 'Cannot determine an instance type to use for ' + mapop
print(msg)
raise Exception(msg)
def _get_instance_type(signatures, gateway, cls, mapop):
type_map = {'RasterMapOp': 0, 'VectorMapOp': 0, 'MapOp': 0}
for sig in signatures:
has_type = {'RasterMapOp': False, 'VectorMapOp': False, 'MapOp': False}
for variable in sig.split("|"):
# print("variable: " + variable)
names = re.split("[:=]+", variable)
new_type = names[1][names[1].rfind('.')+1:]
if new_type.endswith("*"):
new_type = new_type[:-1]
if new_type == 'RasterMapOp' or new_type == 'VectorMapOp' or new_type == 'MapOp':
has_type[new_type] = True
for t,v in has_type.iteritems():
if v:
type_map[t] += 1
# Make sure that all of the signatures have an argument of one of the map op types.
# If the map op is either RasterMapOp or VectorMapOp, and all of the signatures have
# an argument of that type, then use that for the instance type.
if is_instance_of(gateway, cls, 'org.mrgeo.mapalgebra.raster.RasterMapOp'):
if type_map['RasterMapOp'] == len(signatures):
return 'RasterMapOp'
elif is_instance_of(gateway, cls, 'org.mrgeo.mapalgebra.vector.VectorMapOp'):
if type_map['VectorMapOp'] == len(signatures):
return 'VectorMapOp'
# There is at least one signature that does not include a parameter of the same type
# as the map op itself. Instead, we get a type that is represented in all the signatures.
for t,v in type_map.iteritems():
if v == len(signatures):
return t
msg = 'Cannot determine an instance type to use for ' + mapop
print(msg)
raise Exception(msg)
def comparevector(self, vector, testname):
if self.GENERATE_BASELINE_DATA:
self.savevector(vector, str(testname))
else:
jvm = self.mrgeo._get_jvm()
# test = raster.mapop.toDataset(False)
java_import(jvm, "org.mrgeo.hdfs.vector.DelimitedVectorReader")
testvector = str(self.outputhdfs + testname + ".tsv")
vector.ssave(testvector)
expectedvector = str(self.inputdir + testname + ".tsv")
vdp_expected = jvm.DataProviderFactory.getVectorDataProvider(
expectedvector,
jvm.DataProviderFactory.AccessMode.READ,
jvm.HadoopUtils.createConfiguration())
expected_geom_reader = vdp_expected.getVectorReader().get()
vdp = jvm.DataProviderFactory.getVectorDataProvider(
testvector,
jvm.DataProviderFactory.AccessMode.READ,
jvm.HadoopUtils.createConfiguration())
self.assertTrue(vdp is not None)
vector_reader = vdp.getVectorReader()
self.assertTrue(vector_reader is not None)
self.assertTrue(is_instance_of(self.mrgeo.gateway, vector_reader, jvm.DelimitedVectorReader))
self.assertEquals(vdp_expected.getVectorReader().count(), vector_reader.count())
geom_reader = vector_reader.get()
self.assertTrue(geom_reader is not None)
while expected_geom_reader.hasNext():
expected_geom = expected_geom_reader.next()
geom = geom_reader.next()
self.assertTrue(geom is not None)
self.assertEquals(expected_geom.type(), geom.type())
self.assertAlmostEquals(float(expected_geom.getAttribute("COST_S")),
float(geom.getAttribute("COST_S")), delta=0.001)
self.assertAlmostEquals(float(expected_geom.getAttribute("DISTANCE_M")),
float(geom.getAttribute("DISTANCE_M")), delta=0.001)
self.assertAlmostEquals(float(expected_geom.getAttribute("MINSPEED_MPS")),
float(geom.getAttribute("MINSPEED_MPS")), delta=0.001)
self.assertAlmostEquals(float(expected_geom.getAttribute("MAXSPEED_MPS")),
float(geom.getAttribute("MAXSPEED_MPS")), delta=0.001)
self.assertAlmostEquals(float(expected_geom.getAttribute("AVGSPEED_MPS")),
float(geom.getAttribute("AVGSPEED_MPS")), delta=0.001)
# Should not be any more geometries in the actual output
self.assertFalse(geom_reader.hasNext())
jvm.HadoopFileUtils.delete(testvector)
def _generate_procedural_method_code(gateway, client, mapop, name, signatures,
instance):
methods = _generate_methods(instance, signatures)
# print("working on " + name)
if (name == "add"):
pass
jvm = gateway.jvm
cls = JavaClass(mapop, gateway_client=client)
is_export = is_remote() and is_instance_of(gateway, cls, jvm.ExportMapOp)
if len(methods) == 0:
return None
signature, self_method = _generate_proc_signature(methods)
generator = CodeGenerator()
# Signature
generator.write("def " + name + "(" + signature + "):", post_indent=True)
# code += " print('" + name + "')\n"
_generate_imports(generator, mapop, is_export)
_generate_calls(generator, methods, is_export=is_export)
_generate_run(generator, instance, is_export)
# print(code)
code = generator.generate()
code = code.replace("self", self_method)
generator.begin()
for line in code.split("\n"):
generator.write(line)
return {name: generator}
def generate(mrgeo, gateway, gateway_client):
global _initialized
if _initialized:
# Make sure the object have the proper code in them. In case somewhere we've made a new mrgeo object
for method_name, code in _mapop_code.items():
if not hasattr(mrgeo, method_name):
setattr(mrgeo, method_name,
code.compile(method_name).get(method_name))
for method_name, code in _rastermapop_code.items():
if not hasattr(RasterMapOp, method_name):
setattr(RasterMapOp, method_name,
code.compile(method_name).get(method_name))
for method_name, code in _vectormapop_code.items():
if not hasattr(VectorMapOp, method_name):
setattr(VectorMapOp, method_name,
code.compile(method_name).get(method_name))
return
jvm = gateway.jvm
client = gateway_client
java_import(jvm, "org.mrgeo.job.*")
java_import(jvm, "org.mrgeo.mapalgebra.MapOpFactory")
java_import(jvm, "org.mrgeo.mapalgebra.raster.RasterMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.vector.VectorMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.raster.MrsPyramidMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.IngestImageMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.ExportMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.PointsMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.MapOp")
java_import(jvm, "org.mrgeo.utils.SparkUtils")
java_import(jvm, "org.mrgeo.hdfs.utils.HadoopFileUtils")
java_import(jvm, "org.mrgeo.data.*")
mapops = jvm.MapOpFactory.getMapOpClasses()
for rawmapop in mapops:
mapop = str(rawmapop.getCanonicalName().rstrip('$'))
# Skip IngestImageMapOp because there is an explicit method defined in
# MrGeo class for ingesting an image, and _get_instance_type will raise
# an exception when run against that map op.
if not mapop.endswith(".IngestImageMapOp") and not mapop.endswith(
".InlineCsvMapOp"):
java_import(jvm, mapop)
cls = JavaClass(mapop, gateway_client=client)
signatures = jvm.MapOpFactory.getSignatures(mapop)
instance = _get_instance_type(signatures, gateway, cls, mapop)
# for s in signatures:
# print("signature: " + s)
for method in cls.register():
ooCodes = None
procCodes = None
if method is not None:
name = method.strip().lower()
if len(name) > 0:
if name in _reserved:
# print("reserved: " + name)
continue
elif name in _operators:
# print("operator: " + name)
ooCodes = _generate_operator_code(
mapop, name, signatures, instance)
else:
# print("method: " + name)
ooCodes = _generate_oo_method_code(
gateway, client, mapop, name, signatures,
instance)
procCodes = _generate_procedural_method_code(
gateway, client, mapop, name, signatures,
instance)
if ooCodes is not None:
for method_name, code in ooCodes.items():
# if method_name == "export":
# print(code.generate(), file=sys.stderr)
if instance == 'RasterMapOp':
_rastermapop_code[method_name] = code
setattr(RasterMapOp, method_name,
code.compile(method_name).get(method_name))
elif instance == "VectorMapOp":
_vectormapop_code[method_name] = code
setattr(VectorMapOp, method_name,
code.compile(method_name).get(method_name))
elif is_instance_of(gateway, cls, jvm.MapOp):
# _mapop_code[method_name] = code
_rastermapop_code[method_name] = code
setattr(RasterMapOp, method_name,
code.compile(method_name).get(method_name))
_vectormapop_code[method_name] = code
setattr(VectorMapOp, method_name,
code.compile(method_name).get(method_name))
if procCodes is not None:
for method_name, code in procCodes.items():
print(method_name)
_mapop_code[method_name] = code
setattr(mrgeo, method_name,
code.compile(method_name).get(method_name))
_initialized = True
print("add: " + str(hasattr(mrgeo, "add")))
def generate(gateway, gateway_client):
global _initialized
if _initialized:
return
jvm = gateway.jvm
client = gateway_client
java_import(jvm, "org.mrgeo.job.*")
java_import(jvm, "org.mrgeo.mapalgebra.MapOpFactory")
java_import(jvm, "org.mrgeo.mapalgebra.raster.RasterMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.vector.VectorMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.raster.MrsPyramidMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.IngestImageMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.ExportMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.PointsMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.MapOp")
java_import(jvm, "org.mrgeo.utils.SparkUtils")
java_import(jvm, "org.mrgeo.hdfs.utils.HadoopFileUtils")
java_import(jvm, "org.mrgeo.data.*")
mapops = jvm.MapOpFactory.getMapOpClasses()
for rawmapop in mapops:
mapop = str(rawmapop.getCanonicalName().rstrip('$'))
# Skip IngestImageMapOp because there is an explicit method defined in
# MrGeo class for ingesting an image, and _get_instance_type will raise
# an exception when run against that map op.
if not mapop.endswith(".IngestImageMapOp") and not mapop.endswith(".InlineCsvMapOp"):
java_import(jvm, mapop)
cls = JavaClass(mapop, gateway_client=client)
signatures = jvm.MapOpFactory.getSignatures(mapop)
instance = _get_instance_type(signatures, gateway, cls, mapop)
# for s in signatures:
# print("signature: " + s)
for method in cls.register():
codes = None
if method is not None:
name = method.strip().lower()
if len(name) > 0:
if name in _reserved:
# print("reserved: " + name)
continue
elif name in _operators:
# print("operator: " + name)
codes = _generate_operator_code(mapop, name, signatures, instance)
else:
# print("method: " + name)
codes = _generate_method_code(gateway, client, mapop, name, signatures, instance)
if codes is not None:
for method_name, code in codes.items():
# print(code)
compiled = {}
exec code in compiled
if instance == 'RasterMapOp':
_rastermapop_code[method_name] = code
setattr(RasterMapOp, method_name, compiled.get(method_name))
elif instance == "VectorMapOp":
_vectormapop_code[method_name] = code
setattr(VectorMapOp, method_name, compiled.get(method_name))
elif is_instance_of(gateway, cls, jvm.MapOp):
_mapop_code[method_name] = code
setattr(RasterMapOp, method_name, compiled.get(method_name))
setattr(VectorMapOp, method_name, compiled.get(method_name))
_initialized = True
def generate(gateway, gateway_client):
global _initialized
if _initialized:
return
jvm = gateway.jvm
client = gateway_client
java_import(jvm, "org.mrgeo.job.*")
java_import(jvm, "org.mrgeo.mapalgebra.MapOpFactory")
java_import(jvm, "org.mrgeo.mapalgebra.raster.RasterMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.vector.VectorMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.raster.MrsPyramidMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.IngestImageMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.ExportMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.PointsMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.MapOp")
java_import(jvm, "org.mrgeo.utils.SparkUtils")
java_import(jvm, "org.mrgeo.hdfs.utils.HadoopFileUtils")
java_import(jvm, "org.mrgeo.data.*")
mapops = jvm.MapOpFactory.getMapOpClasses()
for rawmapop in mapops:
mapop = str(rawmapop.getCanonicalName().rstrip('$'))
java_import(jvm, mapop)
cls = JavaClass(mapop, gateway_client=client)
if is_instance_of(gateway, cls, jvm.RasterMapOp):
instance = 'RasterMapOp'
elif is_instance_of(gateway, cls, jvm.VectorMapOp):
instance = 'VectorMapOp'
elif is_instance_of(gateway, cls, jvm.MapOp):
instance = "MapOp"
else:
# raise Exception("mapop (" + mapop + ") is not a RasterMapOp, VectorMapOp, or MapOp")
print("mapop (" + mapop + ") is not a RasterMapOp, VectorMapOp, or MapOp")
continue
signatures = jvm.MapOpFactory.getSignatures(mapop)
# for s in signatures:
# print("signature: " + s)
for method in cls.register():
codes = None
if method is not None:
name = method.strip().lower()
if len(name) > 0:
if name in _reserved:
# print("reserved: " + name)
continue
elif name in _operators:
# print("operator: " + name)
codes = _generate_operator_code(mapop, name, signatures, instance)
else:
# print("method: " + name)
codes = _generate_method_code(gateway, client, mapop, name, signatures, instance)
if codes is not None:
for method_name, code in codes.items():
# print(code)
compiled = {}
exec code in compiled
if instance == 'RasterMapOp':
_rastermapop_code[method_name] = code
setattr(RasterMapOp, method_name, compiled.get(method_name))
elif instance == "VectorMapOp":
_vectormapop_code[method_name] = code
setattr(VectorMapOp, method_name, compiled.get(method_name))
elif is_instance_of(gateway, cls, jvm.MapOp):
_mapop_code[method_name] = code
setattr(RasterMapOp, method_name, compiled.get(method_name))
setattr(VectorMapOp, method_name, compiled.get(method_name))
_initialized = True
