def _generate_procedural_method_code(gateway, client, mapop, name, signatures, instance): methods = _generate_methods(instance, signatures) # print("working on " + name) jvm = gateway.jvm cls = JavaClass(mapop, gateway_client=client) is_export = is_remote() and is_instance_of(gateway, cls, jvm.ExportMapOp) if len(methods) == 0: return None signature, self_method = _generate_proc_signature(methods) generator = CodeGenerator() # Signature generator.write("def " + name + "(" + signature + "):", post_indent=True) # code += " print('" + name + "')\n" _generate_imports(generator, mapop, is_export) _generate_calls(generator, methods, is_export=is_export) _generate_run(generator, instance, is_export) # print(code) code = generator.generate() code = code.replace("self", self_method) generator.begin() for line in code.split("\n"): generator.write(line) return {name: generator}
def _generate_oo_method_code(gateway, client, mapop, name, signatures, instance): methods = _generate_methods(instance, signatures) # print("working on " + name) jvm = gateway.jvm cls = JavaClass(mapop, gateway_client=client) is_export = is_remote() and is_instance_of(gateway, cls, jvm.ExportMapOp) if len(methods) == 0: return None signature = _generate_oo_signature(methods) generator = CodeGenerator() # Signature generator.write("def " + name + "(" + signature + "):", post_indent=True) # code += " print('" + name + "')\n" _generate_imports(generator, mapop, is_export) _generate_calls(generator, methods, is_export=is_export) _generate_run(generator, instance, is_export) # print(code) return {name: generator}
def _generate_method_code(gateway, client, mapop, name, signatures, instance): methods = _generate_methods(instance, signatures) # print("working on " + name) jvm = gateway.jvm cls = JavaClass(mapop, gateway_client=client) is_export = is_remote() and is_instance_of(gateway, cls, jvm.ExportMapOp) if len(methods) == 0: return None signature = _generate_signature(methods) code = "" # Signature code += "def " + name + "(" + signature + "):" + "\n" # code += " print('" + name + "')\n" code += _generate_imports(mapop, is_export) code += _generate_calls(methods, is_export) code += _generate_run(instance, is_export) # print(code) return {name: code}
def comparevector(self, vector, testname): if self.GENERATE_BASELINE_DATA: self.savevector(vector, str(testname)) else: jvm = self.mrgeo._get_jvm() # test = raster.mapop.toDataset(False) java_import(jvm, "org.mrgeo.hdfs.vector.DelimitedVectorReader") testvector = str(self.outputhdfs + testname + ".tsv") vector.ssave(testvector) expectedvector = str(self.inputdir + testname + ".tsv") vdp_expected = jvm.DataProviderFactory.getVectorDataProvider( expectedvector, jvm.DataProviderFactory.AccessMode.READ, jvm.HadoopUtils.createConfiguration()) expected_geom_reader = vdp_expected.getVectorReader().get() vdp = jvm.DataProviderFactory.getVectorDataProvider( testvector, jvm.DataProviderFactory.AccessMode.READ, jvm.HadoopUtils.createConfiguration()) self.assertTrue(vdp is not None) vector_reader = vdp.getVectorReader() self.assertTrue(vector_reader is not None) self.assertTrue( is_instance_of(self.mrgeo.gateway, vector_reader, jvm.DelimitedVectorReader)) self.assertEquals(vdp_expected.getVectorReader().count(), vector_reader.count()) geom_reader = vector_reader.get() self.assertTrue(geom_reader is not None) while expected_geom_reader.hasNext(): expected_geom = expected_geom_reader.next() geom = geom_reader.next() self.assertTrue(geom is not None) self.assertEquals(expected_geom.type(), geom.type()) self.assertAlmostEquals(float( expected_geom.getAttribute("COST_S")), float(geom.getAttribute("COST_S")), delta=0.001) self.assertAlmostEquals(float( expected_geom.getAttribute("DISTANCE_M")), float(geom.getAttribute("DISTANCE_M")), delta=0.001) self.assertAlmostEquals( float(expected_geom.getAttribute("MINSPEED_MPS")), float(geom.getAttribute("MINSPEED_MPS")), delta=0.001) self.assertAlmostEquals( float(expected_geom.getAttribute("MAXSPEED_MPS")), float(geom.getAttribute("MAXSPEED_MPS")), delta=0.001) self.assertAlmostEquals( float(expected_geom.getAttribute("AVGSPEED_MPS")), float(geom.getAttribute("AVGSPEED_MPS")), delta=0.001) # Should not be any more geometries in the actual output self.assertFalse(geom_reader.hasNext()) jvm.HadoopFileUtils.delete(testvector)
def _get_instance_type(signatures, gateway, cls, mapop): type_map = {'RasterMapOp': 0, 'VectorMapOp': 0, 'MapOp': 0} for sig in signatures: has_type = {'RasterMapOp': False, 'VectorMapOp': False, 'MapOp': False} for variable in sig.split("|"): # print("variable: " + variable) names = re.split("[:=]+", variable) new_type = names[1][names[1].rfind('.') + 1:] if new_type.endswith("*"): new_type = new_type[:-1] if new_type == 'RasterMapOp' or new_type == 'VectorMapOp' or new_type == 'MapOp': has_type[new_type] = True for t, v in has_type.iteritems(): if v: type_map[t] += 1 # Make sure that all of the signatures have an argument of one of the map op types. # If the map op is either RasterMapOp or VectorMapOp, and all of the signatures have # an argument of that type, then use that for the instance type. if is_instance_of(gateway, cls, 'org.mrgeo.mapalgebra.raster.RasterMapOp'): if type_map['RasterMapOp'] == len(signatures): return 'RasterMapOp' elif is_instance_of(gateway, cls, 'org.mrgeo.mapalgebra.vector.VectorMapOp'): if type_map['VectorMapOp'] == len(signatures): return 'VectorMapOp' # There is at least one signature that does not include a parameter of the same type # as the map op itself. Instead, we get a type that is represented in all the signatures. for t, v in type_map.iteritems(): if v == len(signatures): return t msg = 'Cannot determine an instance type to use for ' + mapop print(msg) raise Exception(msg)
def _get_instance_type(signatures, gateway, cls, mapop): type_map = {'RasterMapOp': 0, 'VectorMapOp': 0, 'MapOp': 0} for sig in signatures: has_type = {'RasterMapOp': False, 'VectorMapOp': False, 'MapOp': False} for variable in sig.split("|"): # print("variable: " + variable) names = re.split("[:=]+", variable) new_type = names[1][names[1].rfind('.')+1:] if new_type.endswith("*"): new_type = new_type[:-1] if new_type == 'RasterMapOp' or new_type == 'VectorMapOp' or new_type == 'MapOp': has_type[new_type] = True for t,v in has_type.iteritems(): if v: type_map[t] += 1 # Make sure that all of the signatures have an argument of one of the map op types. # If the map op is either RasterMapOp or VectorMapOp, and all of the signatures have # an argument of that type, then use that for the instance type. if is_instance_of(gateway, cls, 'org.mrgeo.mapalgebra.raster.RasterMapOp'): if type_map['RasterMapOp'] == len(signatures): return 'RasterMapOp' elif is_instance_of(gateway, cls, 'org.mrgeo.mapalgebra.vector.VectorMapOp'): if type_map['VectorMapOp'] == len(signatures): return 'VectorMapOp' # There is at least one signature that does not include a parameter of the same type # as the map op itself. Instead, we get a type that is represented in all the signatures. for t,v in type_map.iteritems(): if v == len(signatures): return t msg = 'Cannot determine an instance type to use for ' + mapop print(msg) raise Exception(msg)
def comparevector(self, vector, testname): if self.GENERATE_BASELINE_DATA: self.savevector(vector, str(testname)) else: jvm = self.mrgeo._get_jvm() # test = raster.mapop.toDataset(False) java_import(jvm, "org.mrgeo.hdfs.vector.DelimitedVectorReader") testvector = str(self.outputhdfs + testname + ".tsv") vector.ssave(testvector) expectedvector = str(self.inputdir + testname + ".tsv") vdp_expected = jvm.DataProviderFactory.getVectorDataProvider( expectedvector, jvm.DataProviderFactory.AccessMode.READ, jvm.HadoopUtils.createConfiguration()) expected_geom_reader = vdp_expected.getVectorReader().get() vdp = jvm.DataProviderFactory.getVectorDataProvider( testvector, jvm.DataProviderFactory.AccessMode.READ, jvm.HadoopUtils.createConfiguration()) self.assertTrue(vdp is not None) vector_reader = vdp.getVectorReader() self.assertTrue(vector_reader is not None) self.assertTrue(is_instance_of(self.mrgeo.gateway, vector_reader, jvm.DelimitedVectorReader)) self.assertEquals(vdp_expected.getVectorReader().count(), vector_reader.count()) geom_reader = vector_reader.get() self.assertTrue(geom_reader is not None) while expected_geom_reader.hasNext(): expected_geom = expected_geom_reader.next() geom = geom_reader.next() self.assertTrue(geom is not None) self.assertEquals(expected_geom.type(), geom.type()) self.assertAlmostEquals(float(expected_geom.getAttribute("COST_S")), float(geom.getAttribute("COST_S")), delta=0.001) self.assertAlmostEquals(float(expected_geom.getAttribute("DISTANCE_M")), float(geom.getAttribute("DISTANCE_M")), delta=0.001) self.assertAlmostEquals(float(expected_geom.getAttribute("MINSPEED_MPS")), float(geom.getAttribute("MINSPEED_MPS")), delta=0.001) self.assertAlmostEquals(float(expected_geom.getAttribute("MAXSPEED_MPS")), float(geom.getAttribute("MAXSPEED_MPS")), delta=0.001) self.assertAlmostEquals(float(expected_geom.getAttribute("AVGSPEED_MPS")), float(geom.getAttribute("AVGSPEED_MPS")), delta=0.001) # Should not be any more geometries in the actual output self.assertFalse(geom_reader.hasNext()) jvm.HadoopFileUtils.delete(testvector)
def _generate_procedural_method_code(gateway, client, mapop, name, signatures, instance): methods = _generate_methods(instance, signatures) # print("working on " + name) if (name == "add"): pass jvm = gateway.jvm cls = JavaClass(mapop, gateway_client=client) is_export = is_remote() and is_instance_of(gateway, cls, jvm.ExportMapOp) if len(methods) == 0: return None signature, self_method = _generate_proc_signature(methods) generator = CodeGenerator() # Signature generator.write("def " + name + "(" + signature + "):", post_indent=True) # code += " print('" + name + "')\n" _generate_imports(generator, mapop, is_export) _generate_calls(generator, methods, is_export=is_export) _generate_run(generator, instance, is_export) # print(code) code = generator.generate() code = code.replace("self", self_method) generator.begin() for line in code.split("\n"): generator.write(line) return {name: generator}
def generate(mrgeo, gateway, gateway_client): global _initialized if _initialized: # Make sure the object have the proper code in them. In case somewhere we've made a new mrgeo object for method_name, code in _mapop_code.items(): if not hasattr(mrgeo, method_name): setattr(mrgeo, method_name, code.compile(method_name).get(method_name)) for method_name, code in _rastermapop_code.items(): if not hasattr(RasterMapOp, method_name): setattr(RasterMapOp, method_name, code.compile(method_name).get(method_name)) for method_name, code in _vectormapop_code.items(): if not hasattr(VectorMapOp, method_name): setattr(VectorMapOp, method_name, code.compile(method_name).get(method_name)) return jvm = gateway.jvm client = gateway_client java_import(jvm, "org.mrgeo.job.*") java_import(jvm, "org.mrgeo.mapalgebra.MapOpFactory") java_import(jvm, "org.mrgeo.mapalgebra.raster.RasterMapOp") java_import(jvm, "org.mrgeo.mapalgebra.vector.VectorMapOp") java_import(jvm, "org.mrgeo.mapalgebra.raster.MrsPyramidMapOp") java_import(jvm, "org.mrgeo.mapalgebra.IngestImageMapOp") java_import(jvm, "org.mrgeo.mapalgebra.ExportMapOp") java_import(jvm, "org.mrgeo.mapalgebra.PointsMapOp") java_import(jvm, "org.mrgeo.mapalgebra.MapOp") java_import(jvm, "org.mrgeo.utils.SparkUtils") java_import(jvm, "org.mrgeo.hdfs.utils.HadoopFileUtils") java_import(jvm, "org.mrgeo.data.*") mapops = jvm.MapOpFactory.getMapOpClasses() for rawmapop in mapops: mapop = str(rawmapop.getCanonicalName().rstrip('$')) # Skip IngestImageMapOp because there is an explicit method defined in # MrGeo class for ingesting an image, and _get_instance_type will raise # an exception when run against that map op. if not mapop.endswith(".IngestImageMapOp") and not mapop.endswith( ".InlineCsvMapOp"): java_import(jvm, mapop) cls = JavaClass(mapop, gateway_client=client) signatures = jvm.MapOpFactory.getSignatures(mapop) instance = _get_instance_type(signatures, gateway, cls, mapop) # for s in signatures: # print("signature: " + s) for method in cls.register(): ooCodes = None procCodes = None if method is not None: name = method.strip().lower() if len(name) > 0: if name in _reserved: # print("reserved: " + name) continue elif name in _operators: # print("operator: " + name) ooCodes = _generate_operator_code( mapop, name, signatures, instance) else: # print("method: " + name) ooCodes = _generate_oo_method_code( gateway, client, mapop, name, signatures, instance) procCodes = _generate_procedural_method_code( gateway, client, mapop, name, signatures, instance) if ooCodes is not None: for method_name, code in ooCodes.items(): # if method_name == "export": # print(code.generate(), file=sys.stderr) if instance == 'RasterMapOp': _rastermapop_code[method_name] = code setattr(RasterMapOp, method_name, code.compile(method_name).get(method_name)) elif instance == "VectorMapOp": _vectormapop_code[method_name] = code setattr(VectorMapOp, method_name, code.compile(method_name).get(method_name)) elif is_instance_of(gateway, cls, jvm.MapOp): # _mapop_code[method_name] = code _rastermapop_code[method_name] = code setattr(RasterMapOp, method_name, code.compile(method_name).get(method_name)) _vectormapop_code[method_name] = code setattr(VectorMapOp, method_name, code.compile(method_name).get(method_name)) if procCodes is not None: for method_name, code in procCodes.items(): print(method_name) _mapop_code[method_name] = code setattr(mrgeo, method_name, code.compile(method_name).get(method_name)) _initialized = True print("add: " + str(hasattr(mrgeo, "add")))
def generate(gateway, gateway_client): global _initialized if _initialized: return jvm = gateway.jvm client = gateway_client java_import(jvm, "org.mrgeo.job.*") java_import(jvm, "org.mrgeo.mapalgebra.MapOpFactory") java_import(jvm, "org.mrgeo.mapalgebra.raster.RasterMapOp") java_import(jvm, "org.mrgeo.mapalgebra.vector.VectorMapOp") java_import(jvm, "org.mrgeo.mapalgebra.raster.MrsPyramidMapOp") java_import(jvm, "org.mrgeo.mapalgebra.IngestImageMapOp") java_import(jvm, "org.mrgeo.mapalgebra.ExportMapOp") java_import(jvm, "org.mrgeo.mapalgebra.PointsMapOp") java_import(jvm, "org.mrgeo.mapalgebra.MapOp") java_import(jvm, "org.mrgeo.utils.SparkUtils") java_import(jvm, "org.mrgeo.hdfs.utils.HadoopFileUtils") java_import(jvm, "org.mrgeo.data.*") mapops = jvm.MapOpFactory.getMapOpClasses() for rawmapop in mapops: mapop = str(rawmapop.getCanonicalName().rstrip('$')) # Skip IngestImageMapOp because there is an explicit method defined in # MrGeo class for ingesting an image, and _get_instance_type will raise # an exception when run against that map op. if not mapop.endswith(".IngestImageMapOp") and not mapop.endswith(".InlineCsvMapOp"): java_import(jvm, mapop) cls = JavaClass(mapop, gateway_client=client) signatures = jvm.MapOpFactory.getSignatures(mapop) instance = _get_instance_type(signatures, gateway, cls, mapop) # for s in signatures: # print("signature: " + s) for method in cls.register(): codes = None if method is not None: name = method.strip().lower() if len(name) > 0: if name in _reserved: # print("reserved: " + name) continue elif name in _operators: # print("operator: " + name) codes = _generate_operator_code(mapop, name, signatures, instance) else: # print("method: " + name) codes = _generate_method_code(gateway, client, mapop, name, signatures, instance) if codes is not None: for method_name, code in codes.items(): # print(code) compiled = {} exec code in compiled if instance == 'RasterMapOp': _rastermapop_code[method_name] = code setattr(RasterMapOp, method_name, compiled.get(method_name)) elif instance == "VectorMapOp": _vectormapop_code[method_name] = code setattr(VectorMapOp, method_name, compiled.get(method_name)) elif is_instance_of(gateway, cls, jvm.MapOp): _mapop_code[method_name] = code setattr(RasterMapOp, method_name, compiled.get(method_name)) setattr(VectorMapOp, method_name, compiled.get(method_name)) _initialized = True
def generate(gateway, gateway_client): global _initialized if _initialized: return jvm = gateway.jvm client = gateway_client java_import(jvm, "org.mrgeo.job.*") java_import(jvm, "org.mrgeo.mapalgebra.MapOpFactory") java_import(jvm, "org.mrgeo.mapalgebra.raster.RasterMapOp") java_import(jvm, "org.mrgeo.mapalgebra.vector.VectorMapOp") java_import(jvm, "org.mrgeo.mapalgebra.raster.MrsPyramidMapOp") java_import(jvm, "org.mrgeo.mapalgebra.IngestImageMapOp") java_import(jvm, "org.mrgeo.mapalgebra.ExportMapOp") java_import(jvm, "org.mrgeo.mapalgebra.PointsMapOp") java_import(jvm, "org.mrgeo.mapalgebra.MapOp") java_import(jvm, "org.mrgeo.utils.SparkUtils") java_import(jvm, "org.mrgeo.hdfs.utils.HadoopFileUtils") java_import(jvm, "org.mrgeo.data.*") mapops = jvm.MapOpFactory.getMapOpClasses() for rawmapop in mapops: mapop = str(rawmapop.getCanonicalName().rstrip('$')) java_import(jvm, mapop) cls = JavaClass(mapop, gateway_client=client) if is_instance_of(gateway, cls, jvm.RasterMapOp): instance = 'RasterMapOp' elif is_instance_of(gateway, cls, jvm.VectorMapOp): instance = 'VectorMapOp' elif is_instance_of(gateway, cls, jvm.MapOp): instance = "MapOp" else: # raise Exception("mapop (" + mapop + ") is not a RasterMapOp, VectorMapOp, or MapOp") print("mapop (" + mapop + ") is not a RasterMapOp, VectorMapOp, or MapOp") continue signatures = jvm.MapOpFactory.getSignatures(mapop) # for s in signatures: # print("signature: " + s) for method in cls.register(): codes = None if method is not None: name = method.strip().lower() if len(name) > 0: if name in _reserved: # print("reserved: " + name) continue elif name in _operators: # print("operator: " + name) codes = _generate_operator_code(mapop, name, signatures, instance) else: # print("method: " + name) codes = _generate_method_code(gateway, client, mapop, name, signatures, instance) if codes is not None: for method_name, code in codes.items(): # print(code) compiled = {} exec code in compiled if instance == 'RasterMapOp': _rastermapop_code[method_name] = code setattr(RasterMapOp, method_name, compiled.get(method_name)) elif instance == "VectorMapOp": _vectormapop_code[method_name] = code setattr(VectorMapOp, method_name, compiled.get(method_name)) elif is_instance_of(gateway, cls, jvm.MapOp): _mapop_code[method_name] = code setattr(RasterMapOp, method_name, compiled.get(method_name)) setattr(VectorMapOp, method_name, compiled.get(method_name)) _initialized = True