def convert(self, obj, gateway_client):
name = obj.value
if name == 'NearestNeighbor':
sample = JavaClass("geotrellis.raster.resample.NearestNeighbor$", gateway_client)
elif name == 'Bilinear':
sample = JavaClass("geotrellis.raster.resample.Bilinear$", gateway_client)
elif name == 'CubicConvolution':
sample = JavaClass("geotrellis.raster.resample.CubicConvolution$", gateway_client)
elif name == 'CubicSpline':
sample = JavaClass("geotrellis.raster.resample.CubicSpline$", gateway_client)
elif name == 'Lanczos':
sample = JavaClass("geotrellis.raster.resample.Lanczos$", gateway_client)
elif name == 'Average':
sample = JavaClass("geotrellis.raster.resample.Average$", gateway_client)
elif name == 'Mode':
sample = JavaClass("geotrellis.raster.resample.Mode$", gateway_client)
elif name == 'Median':
sample = JavaClass("geotrellis.raster.resample.Median$", gateway_client)
elif name == 'Max':
sample = JavaClass("geotrellis.raster.resample.Max$", gateway_client)
elif name == 'Min':
sample = JavaClass("geotrellis.raster.resample.Min$", gateway_client)
else:
raise TypeError(name, "Could not be converted to a GeoTrellis ResampleMethod.")
return sample.__getattr__("MODULE$")
def convert(self, obj, gateway_client):
if isinstance(obj, GlobalLayout):
JavaGlobalLayout = JavaClass("geopyspark.geotrellis.GlobalLayout", gateway_client)
return JavaGlobalLayout(obj.tile_size, obj.zoom, float(obj.threshold))
elif isinstance(obj, LocalLayout):
JavaLocalLayout = JavaClass("geopyspark.geotrellis.LocalLayout", gateway_client)
return JavaLocalLayout(obj.tile_cols, obj.tile_rows)
else:
raise TypeError("Could not convert {} to geotrellis.raster.LayoutType".format(obj.sampleType))
def convert(self, object, gateway_client):
JavaRasterizerOptions = JavaClass("geotrellis.raster.rasterize.Rasterizer$Options$", gateway_client)
if (object.sampleType == 'PixelIsPoint'):
sample = JavaClass("geotrellis.raster.PixelIsPoint$", gateway_client)
elif (object.sampleType == 'PixelIsArea'):
sample = JavaClass("geotrellis.raster.PixelIsArea$", gateway_client)
else:
raise TypeError("Could not convert {} to geotrellis.raster.PixelSampleType".format(object.sampleType))
sample_instance = sample.__getattr__("MODULE$")
return JavaRasterizerOptions().apply(object.includePartial, sample_instance)
def convert(self, obj, gateway_client):
python_extent = obj.extent
python_tile_layout = obj.tileLayout
ScalaExtent = JavaClass("geotrellis.vector.Extent", gateway_client)
ScalaTileLayout = JavaClass("geotrellis.raster.TileLayout", gateway_client)
ScalaLayoutDefinition = JavaClass("geotrellis.spark.tiling.LayoutDefinition", gateway_client)
extent = ScalaExtent(python_extent.xmin, python_extent.ymin, python_extent.xmax, python_extent.ymax)
tile_layout = ScalaTileLayout(python_tile_layout.layoutCols,
python_tile_layout.layoutRows,
python_tile_layout.tileCols,
python_tile_layout.tileRows)
return ScalaLayoutDefinition(extent, tile_layout)
def _generate_method_code(self, mapop, name, signatures, instance):
methods = self._generate_methods(instance, signatures)
jvm = self.gateway.jvm
client = self.gateway._gateway_client
cls = JavaClass(mapop, gateway_client=client)
is_export = self.is_instance_of(cls, jvm.ExportMapOp)
if len(methods) == 0:
return None
signature = self._generate_signature(methods)
code = ""
# Signature
code += "def " + name + "(" + signature + "):" + "\n"
# code += " print('" + name + "')\n"
code += self._generate_imports(mapop, is_export)
code += self._generate_calls(methods, is_export)
code += self._generate_run(is_export)
# print(code)
return {name: code}
def jdbc(self, url, table, mode="error", properties={}):
"""
Saves the content of the :class:`DataFrame` to a external database table
via JDBC.
In the case the table already exists in the external database,
behavior of this function depends on the save mode, specified by the `mode`
function (default to throwing an exception). There are four modes:
* `append`: Append contents of this :class:`DataFrame` to existing data.
* `overwrite`: Overwrite existing data.
* `error`: Throw an exception if data already exists.
* `ignore`: Silently ignore this operation if data already exists.
:param url: a JDBC URL of the form `jdbc:subprotocol:subname`
:param table: Name of the table in the external database.
:param mode: one of `append`, `overwrite`, `error`, `ignore` (default: error)
:param properties: JDBC database connection arguments, a list of
arbitrary string tag/value. Normally at least a
"user" and "password" property should be included.
"""
jprop = JavaClass("java.util.Properties",
self._sqlContext._sc._gateway._gateway_client)()
for k in properties:
jprop.setProperty(k, properties[k])
self._jwrite.mode(mode).jdbc(url, table, jprop)
def jdbc(self, url, table, mode=None, properties=None):
"""Saves the content of the :class:`DataFrame` to an external database table via JDBC.
.. note:: Don't create too many partitions in parallel on a large cluster; \
otherwise Spark might crash your external database systems.
:param url: a JDBC URL of the form ``jdbc:subprotocol:subname``
:param table: Name of the table in the external database.
:param mode: specifies the behavior of the save operation when data already exists.
* ``append``: Append contents of this :class:`DataFrame` to existing data.
* ``overwrite``: Overwrite existing data.
* ``ignore``: Silently ignore this operation if data already exists.
* ``error`` (default case): Throw an exception if data already exists.
:param properties: a dictionary of JDBC database connection arguments. Normally at
least properties "user" and "password" with their corresponding values.
For example { 'user' : 'SYSTEM', 'password' : 'mypassword' }
"""
if properties is None:
properties = dict()
jprop = JavaClass("java.util.Properties",
self._spark._sc._gateway._gateway_client)()
for k in properties:
jprop.setProperty(k, properties[k])
self._jwrite.mode(mode).jdbc(url, table, jprop)
def _generate_oo_method_code(gateway, client, mapop, name, signatures,
instance):
methods = _generate_methods(instance, signatures)
# print("working on " + name)
jvm = gateway.jvm
cls = JavaClass(mapop, gateway_client=client)
is_export = is_remote() and is_instance_of(gateway, cls, jvm.ExportMapOp)
if len(methods) == 0:
return None
signature = _generate_oo_signature(methods)
generator = CodeGenerator()
# Signature
generator.write("def " + name + "(" + signature + "):", post_indent=True)
# code += " print('" + name + "')\n"
_generate_imports(generator, mapop, is_export)
_generate_calls(generator, methods, is_export=is_export)
_generate_run(generator, instance, is_export)
# print(code)
return {name: generator}
def java_array(l, java_type=None, fqn=None):
'''
Convert list l to a Java array of element_type.
By default Py4J creates java ArrayList out of a Python list
when calling Java. By wrapping a Python list in java_array()
it "casts" it to being a Java native array with the element
type as defined by java_type or fqn:
- if java_type and fqn are None, java.lang.Object is used.
- if java_type is not None, it should be a py4j.java_gateway.JavaClass
of the instance type desired
- if java_type is None, fqn is the Fully Qualified Name for the Java
class to use as the instance type. For primitives, use the
primitive name (e.g. "double")
'''
# # TODO type inference?
if java_type is None:
if fqn is None:
java_type = java.lang.Object
else:
java_type = JavaClass(fqn, gateway)
new_array = gateway.new_array(java_type, len(l))
for i, entry in enumerate(l):
new_array[i] = entry
return new_array
def convert(self, object, gateway_client):
current_converter = proto.OUTPUT_CONVERTER
proto.OUTPUT_CONVERTER = DEFAULT_OUTPUT_CONVERTER
cls = JavaClass("java.time.LocalDate", gateway_client)
result = cls.of(object.year, object.month, object.day)
proto.OUTPUT_CONVERTER = current_converter
return result
def convert(self, obj, gateway_client):
Timestamp = JavaClass("java.sql.Timestamp", gateway_client)
seconds = (calendar.timegm(obj.utctimetuple())
if obj.tzinfo else time.mktime(obj.timetuple()))
t = Timestamp(int(seconds) * 1000)
t.setNanos(obj.microsecond * 1000)
return t
def jdbc(self,
url,
table,
column=None,
lowerBound=None,
upperBound=None,
numPartitions=None,
predicates=None,
properties=None):
"""
Construct a :class:`DataFrame` representing the database table named ``table``
accessible via JDBC URL ``url`` and connection ``properties``.
Partitions of the table will be retrieved in parallel if either ``column`` or
``predicates`` is specified. ``lowerBound`, ``upperBound`` and ``numPartitions``
is needed when ``column`` is specified.
If both ``column`` and ``predicates`` are specified, ``column`` will be used.
.. note:: Don't create too many partitions in parallel on a large cluster; \
otherwise Spark might crash your external database systems.
:param url: a JDBC URL of the form ``jdbc:subprotocol:subname``
:param table: the name of the table
:param column: the name of an integer column that will be used for partitioning;
if this parameter is specified, then ``numPartitions``, ``lowerBound``
(inclusive), and ``upperBound`` (exclusive) will form partition strides
for generated WHERE clause expressions used to split the column
``column`` evenly
:param lowerBound: the minimum value of ``column`` used to decide partition stride
:param upperBound: the maximum value of ``column`` used to decide partition stride
:param numPartitions: the number of partitions
:param predicates: a list of expressions suitable for inclusion in WHERE clauses;
each one defines one partition of the :class:`DataFrame`
:param properties: a dictionary of JDBC database connection arguments. Normally at
least properties "user" and "password" with their corresponding values.
For example { 'user' : 'SYSTEM', 'password' : 'mypassword' }
:return: a DataFrame
"""
if properties is None:
properties = dict()
jprop = JavaClass("java.util.Properties",
self._spark._sc._gateway._gateway_client)()
for k in properties:
jprop.setProperty(k, properties[k])
if column is not None:
assert lowerBound is not None, "lowerBound can not be None when ``column`` is specified"
assert upperBound is not None, "upperBound can not be None when ``column`` is specified"
assert numPartitions is not None, \
"numPartitions can not be None when ``column`` is specified"
return self._df(
self._jreader.jdbc(url, table, column, int(lowerBound),
int(upperBound), int(numPartitions), jprop))
if predicates is not None:
gateway = self._spark._sc._gateway
jpredicates = utils.toJArray(gateway, gateway.jvm.java.lang.String,
predicates)
return self._df(self._jreader.jdbc(url, table, jpredicates, jprop))
return self._df(self._jreader.jdbc(url, table, jprop))
def jdbc(self,
url,
table,
column=None,
lowerBound=None,
upperBound=None,
numPartitions=None,
predicates=None,
properties=None):
"""
Construct a :class:`DataFrame` representing the database table accessible
via JDBC URL `url` named `table` and connection `properties`.
The `column` parameter could be used to partition the table, then it will
be retrieved in parallel based on the parameters passed to this function.
The `predicates` parameter gives a list expressions suitable for inclusion
in WHERE clauses; each one defines one partition of the :class:`DataFrame`.
::Note: Don't create too many partitions in parallel on a large cluster;
otherwise Spark might crash your external database systems.
:param url: a JDBC URL
:param table: name of table
:param column: the column used to partition
:param lowerBound: the lower bound of partition column
:param upperBound: the upper bound of the partition column
:param numPartitions: the number of partitions
:param predicates: a list of expressions
:param properties: JDBC database connection arguments, a list of arbitrary string
tag/value. Normally at least a "user" and "password" property
should be included.
:return: a DataFrame
"""
if properties is None:
properties = dict()
jprop = JavaClass("java.util.Properties",
self._sqlContext._sc._gateway._gateway_client)()
for k in properties:
jprop.setProperty(k, properties[k])
if column is not None:
if numPartitions is None:
numPartitions = self._sqlContext._sc.defaultParallelism
return self._df(
self._jreader.jdbc(url, table, column, int(lowerBound),
int(upperBound), int(numPartitions), jprop))
if predicates is not None:
gateway = self._sqlContext._sc._gateway
jpredicates = utils.toJArray(gateway, gateway.jvm.java.lang.String,
predicates)
return self._df(self._jreader.jdbc(url, table, jpredicates, jprop))
return self._df(self._jreader.jdbc(url, table, jprop))
def data_ids(self, data_ids):
"""
Constrain a query by data IDs.
Args:
data_ids (list of bytes): The data IDs to constrain by.
Returns:
A `pygw.query.query_constraints.QueryConstraints` with the given data ids.
"""
byte_array_class = JavaClass("[B", java_gateway._gateway_client)
j_data_ids = java_gateway.new_array(byte_array_class, len(data_ids))
for idx, data_id in enumerate(data_ids):
j_data_ids[idx] = _pbat.to_java(data_id)
j_qc = self._java_ref.dataIds(j_data_ids)
return QueryConstraints(j_qc)
def convert(self, obj, gateway_client):
ScalaTemporalStrategy = JavaClass(
"geopyspark.geotrellis.SpaceTimePartitionStrategy", gateway_client)
scala_time_unit = obj.time_unit.value
if obj.time_resolution:
scala_time_resolution = str(obj.time_resolution)
else:
scala_time_resolution = None
return ScalaTemporalStrategy.apply(obj.num_partitions, obj.bits,
scala_time_unit,
scala_time_resolution)
def ssave(self, name):
import copy
from py4j.java_gateway import JavaClass
cls = JavaClass('org.mrgeo.mapalgebra.save.SaveMapOp', gateway_client=self.gateway._gateway_client)
if hasattr(self, 'mapop') and self.is_instance_of(self.mapop, 'org.mrgeo.mapalgebra.MapOp') and type(
name) is str:
op = cls.create(self.mapop, str(name))
else:
raise Exception('input types differ (TODO: expand this message!)')
if (op.setup(self.job, self.context.getConf()) and
op.execute(self.context) and
op.teardown(self.job, self.context.getConf())):
new_resource = copy.copy(self)
new_resource.mapop = op
return new_resource
return
def jdbc(self, url, table, mode=None, properties=None):
"""Saves the content of the :class:`DataFrame` to an external database table via JDBC.
.. versionadded:: 1.4.0
Parameters
----------
table : str
Name of the table in the external database.
mode : str, optional
specifies the behavior of the save operation when data already exists.
* ``append``: Append contents of this :class:`DataFrame` to existing data.
* ``overwrite``: Overwrite existing data.
* ``ignore``: Silently ignore this operation if data already exists.
* ``error`` or ``errorifexists`` (default case): Throw an exception if data already \
exists.
properties : dict
a dictionary of JDBC database connection arguments. Normally at
least properties "user" and "password" with their corresponding values.
For example { 'user' : 'SYSTEM', 'password' : 'mypassword' }
Other Parameters
----------------
Extra options
For the extra options, refer to
`Data Source Option <https://spark.apache.org/docs/latest/sql-data-sources-jdbc.html#data-source-option>`_
in the version you use.
.. # noqa
Notes
-----
Don't create too many partitions in parallel on a large cluster;
otherwise Spark might crash your external database systems.
"""
if properties is None:
properties = dict()
jprop = JavaClass("java.util.Properties",
self._spark._sc._gateway._gateway_client)()
for k in properties:
jprop.setProperty(k, properties[k])
self.mode(mode)._jwrite.jdbc(url, table, jprop)
def jdbc(self, url, table, mode=None, properties={}):
"""Saves the content of the :class:`DataFrame` to a external database table via JDBC.
.. note:: Don't create too many partitions in parallel on a large cluster;\
otherwise Spark might crash your external database systems.
:param url: a JDBC URL of the form ``jdbc:subprotocol:subname``
:param table: Name of the table in the external database.
:param mode: specifies the behavior of the save operation when data already exists.
* ``append``: Append contents of this :class:`DataFrame` to existing data.
* ``overwrite``: Overwrite existing data.
* ``ignore``: Silently ignore this operation if data already exists.
* ``error`` (default case): Throw an exception if data already exists.
:param properties: JDBC database connection arguments, a list of
arbitrary string tag/value. Normally at least a
"user" and "password" property should be included.
"""
jprop = JavaClass("java.util.Properties", self._sqlContext._sc._gateway._gateway_client)()
for k in properties:
jprop.setProperty(k, properties[k])
self._jwrite.mode(mode).jdbc(url, table, jprop)
def _generate_procedural_method_code(gateway, client, mapop, name, signatures,
instance):
methods = _generate_methods(instance, signatures)
# print("working on " + name)
if (name == "add"):
pass
jvm = gateway.jvm
cls = JavaClass(mapop, gateway_client=client)
is_export = is_remote() and is_instance_of(gateway, cls, jvm.ExportMapOp)
if len(methods) == 0:
return None
signature, self_method = _generate_proc_signature(methods)
generator = CodeGenerator()
# Signature
generator.write("def " + name + "(" + signature + "):", post_indent=True)
# code += " print('" + name + "')\n"
_generate_imports(generator, mapop, is_export)
_generate_calls(generator, methods, is_export=is_export)
_generate_run(generator, instance, is_export)
# print(code)
code = generator.generate()
code = code.replace("self", self_method)
generator.begin()
for line in code.split("\n"):
generator.write(line)
return {name: generator}
def convert(self, obj, gateway_client):
JavaCellType = JavaClass("geotrellis.raster.CellType", gateway_client)
return JavaCellType.fromName(obj.value)
def jdbc(self,
url,
table,
column=None,
lowerBound=None,
upperBound=None,
numPartitions=None,
predicates=None,
properties=None):
"""
Construct a :class:`DataFrame` representing the database table named ``table``
accessible via JDBC URL ``url`` and connection ``properties``.
Partitions of the table will be retrieved in parallel if either ``column`` or
``predicates`` is specified. ``lowerBound``, ``upperBound`` and ``numPartitions``
is needed when ``column`` is specified.
If both ``column`` and ``predicates`` are specified, ``column`` will be used.
.. versionadded:: 1.4.0
Parameters
----------
table : str
the name of the table
column : str, optional
alias of ``partitionColumn`` option. Refer to ``partitionColumn`` in
`Data Source Option <https://spark.apache.org/docs/latest/sql-data-sources-jdbc.html#data-source-option>`_
in the version you use.
predicates : list, optional
a list of expressions suitable for inclusion in WHERE clauses;
each one defines one partition of the :class:`DataFrame`
properties : dict, optional
a dictionary of JDBC database connection arguments. Normally at
least properties "user" and "password" with their corresponding values.
For example { 'user' : 'SYSTEM', 'password' : 'mypassword' }
Other Parameters
----------------
Extra options
For the extra options, refer to
`Data Source Option <https://spark.apache.org/docs/latest/sql-data-sources-jdbc.html#data-source-option>`_
in the version you use.
.. # noqa
Notes
-----
Don't create too many partitions in parallel on a large cluster;
otherwise Spark might crash your external database systems.
Returns
-------
:class:`DataFrame`
"""
if properties is None:
properties = dict()
jprop = JavaClass("java.util.Properties",
self._spark._sc._gateway._gateway_client)()
for k in properties:
jprop.setProperty(k, properties[k])
if column is not None:
assert lowerBound is not None, "lowerBound can not be None when ``column`` is specified"
assert upperBound is not None, "upperBound can not be None when ``column`` is specified"
assert numPartitions is not None, \
"numPartitions can not be None when ``column`` is specified"
return self._df(
self._jreader.jdbc(url, table, column, int(lowerBound),
int(upperBound), int(numPartitions), jprop))
if predicates is not None:
gateway = self._spark._sc._gateway
jpredicates = utils.toJArray(gateway, gateway.jvm.java.lang.String,
predicates)
return self._df(self._jreader.jdbc(url, table, jpredicates, jprop))
return self._df(self._jreader.jdbc(url, table, jprop))
def convert(self, obj, gateway_client):
Timestamp = JavaClass("java.sql.Timestamp", gateway_client)
return Timestamp(
int(time.mktime(obj.timetuple())) * 1000 + obj.microsecond // 1000)
def generate(mrgeo, gateway, gateway_client):
global _initialized
if _initialized:
# Make sure the object have the proper code in them. In case somewhere we've made a new mrgeo object
for method_name, code in _mapop_code.items():
if not hasattr(mrgeo, method_name):
setattr(mrgeo, method_name,
code.compile(method_name).get(method_name))
for method_name, code in _rastermapop_code.items():
if not hasattr(RasterMapOp, method_name):
setattr(RasterMapOp, method_name,
code.compile(method_name).get(method_name))
for method_name, code in _vectormapop_code.items():
if not hasattr(VectorMapOp, method_name):
setattr(VectorMapOp, method_name,
code.compile(method_name).get(method_name))
return
jvm = gateway.jvm
client = gateway_client
java_import(jvm, "org.mrgeo.job.*")
java_import(jvm, "org.mrgeo.mapalgebra.MapOpFactory")
java_import(jvm, "org.mrgeo.mapalgebra.raster.RasterMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.vector.VectorMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.raster.MrsPyramidMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.IngestImageMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.ExportMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.PointsMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.MapOp")
java_import(jvm, "org.mrgeo.utils.SparkUtils")
java_import(jvm, "org.mrgeo.hdfs.utils.HadoopFileUtils")
java_import(jvm, "org.mrgeo.data.*")
mapops = jvm.MapOpFactory.getMapOpClasses()
for rawmapop in mapops:
mapop = str(rawmapop.getCanonicalName().rstrip('$'))
# Skip IngestImageMapOp because there is an explicit method defined in
# MrGeo class for ingesting an image, and _get_instance_type will raise
# an exception when run against that map op.
if not mapop.endswith(".IngestImageMapOp") and not mapop.endswith(
".InlineCsvMapOp"):
java_import(jvm, mapop)
cls = JavaClass(mapop, gateway_client=client)
signatures = jvm.MapOpFactory.getSignatures(mapop)
instance = _get_instance_type(signatures, gateway, cls, mapop)
# for s in signatures:
# print("signature: " + s)
for method in cls.register():
ooCodes = None
procCodes = None
if method is not None:
name = method.strip().lower()
if len(name) > 0:
if name in _reserved:
# print("reserved: " + name)
continue
elif name in _operators:
# print("operator: " + name)
ooCodes = _generate_operator_code(
mapop, name, signatures, instance)
else:
# print("method: " + name)
ooCodes = _generate_oo_method_code(
gateway, client, mapop, name, signatures,
instance)
procCodes = _generate_procedural_method_code(
gateway, client, mapop, name, signatures,
instance)
if ooCodes is not None:
for method_name, code in ooCodes.items():
# if method_name == "export":
# print(code.generate(), file=sys.stderr)
if instance == 'RasterMapOp':
_rastermapop_code[method_name] = code
setattr(RasterMapOp, method_name,
code.compile(method_name).get(method_name))
elif instance == "VectorMapOp":
_vectormapop_code[method_name] = code
setattr(VectorMapOp, method_name,
code.compile(method_name).get(method_name))
elif is_instance_of(gateway, cls, jvm.MapOp):
# _mapop_code[method_name] = code
_rastermapop_code[method_name] = code
setattr(RasterMapOp, method_name,
code.compile(method_name).get(method_name))
_vectormapop_code[method_name] = code
setattr(VectorMapOp, method_name,
code.compile(method_name).get(method_name))
if procCodes is not None:
for method_name, code in procCodes.items():
print(method_name)
_mapop_code[method_name] = code
setattr(mrgeo, method_name,
code.compile(method_name).get(method_name))
_initialized = True
print("add: " + str(hasattr(mrgeo, "add")))
def convert(self, obj, gateway_client):
ScalaSourceInfo = JavaClass("geopyspark.geotrellis.vlm.SourceInfo",
gateway_client)
return ScalaSourceInfo.apply(obj.source, obj.source_to_target_band)
def convert(self, obj, gateway_client):
Date = JavaClass("java.sql.Date", gateway_client)
return Date.valueOf(obj.strftime("%Y-%m-%d"))
def __init__(self, subtype, py_type=(list, tuple)):
self.subtype = subtype
self._j_class = JavaClass(self.subtype.binding,
java_gateway._gateway_client)
super().__init__("[L%s;" % self.subtype.binding, py_type)
def _load_mapops(self):
jvm = self.gateway.jvm
client = self.gateway._gateway_client
java_import(jvm, "org.mrgeo.job.*")
java_import(jvm, "org.mrgeo.mapalgebra.MapOpFactory")
java_import(jvm, "org.mrgeo.mapalgebra.raster.RasterMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.raster.MrsPyramidMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.ExportMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.vector.VectorMapOp")
java_import(jvm, "org.mrgeo.mapalgebra.MapOp")
java_import(jvm, "org.mrgeo.utils.SparkUtils")
java_import(jvm, "org.mrgeo.data.*")
mapops = jvm.MapOpFactory.getMapOpClasses()
for rawmapop in mapops:
mapop = str(rawmapop.getCanonicalName().rstrip('$'))
java_import(jvm, mapop)
cls = JavaClass(mapop, gateway_client=client)
if self.is_instance_of(cls, jvm.RasterMapOp):
instance = 'RasterMapOp'
elif self.is_instance_of(cls, jvm.VectorMapOp):
instance = 'VectorMapOp'
elif self.is_instance_of(cls, jvm.MapOp):
instance = "MapOp"
else:
# raise Exception("mapop (" + mapop + ") is not a RasterMapOp, VectorMapOp, or MapOp")
print("mapop (" + mapop +
") is not a RasterMapOp, VectorMapOp, or MapOp")
continue
signatures = jvm.MapOpFactory.getSignatures(mapop)
for method in cls.register():
codes = None
if method is not None:
name = method.strip().lower()
if len(name) > 0:
if name in self.reserved:
# print("reserved: " + name)
continue
elif name in self.operators:
# print("operator: " + name)
codes = self._generate_operator_code(
mapop, name, signatures, instance)
else:
# print("method: " + name)
codes = self._generate_method_code(
mapop, name, signatures, instance)
if codes is not None:
for method_name, code in codes.iteritems():
# print(code)
compiled = {}
exec code in compiled
if instance == 'RasterMapOp':
setattr(RasterMapOp, method_name,
compiled.get(method_name))
elif instance == "VectorMapOp":
# setattr(VectorMapOp, method_name, compiled.get(method_name))
pass
elif self.is_instance_of(cls, jvm.MapOp):
setattr(RasterMapOp, method_name,
compiled.get(method_name))
def eexport(self,name,singleFile=False,zoom=-1,numTiles=-1,mosaic=-1,format="tif",randomTiles=False,tms=False,colorscale="",tileids="",bounds="",allLevels=False,overridenodata=float('-inf')):
import copy
from numbers import Number
import base64
import numpy
from osgeo import gdal_array
import zlib
from osgeo import gdal
from py4j.java_gateway import JavaClass
cls = JavaClass('org.mrgeo.mapalgebra.ExportMapOp', gateway_client=self.gateway._gateway_client)
local_name = name
name = 'In-Memory'
if hasattr(self, 'mapop') and self.is_instance_of(self.mapop, 'org.mrgeo.mapalgebra.raster.RasterMapOp') and type(name) is str and isinstance(singleFile, (int, long, float, str)) and isinstance(zoom, (int, long, float)) and isinstance(numTiles, (int, long, float)) and isinstance(mosaic, (int, long, float)) and type(format) is str and isinstance(randomTiles, (int, long, float, str)) and isinstance(tms, (int, long, float, str)) and type(colorscale) is str and type(tileids) is str and type(bounds) is str and isinstance(allLevels, (int, long, float, str)) and isinstance(overridenodata, (int, long, float)):
op = cls.create(self.mapop, str(name), True if singleFile else False, int(zoom), int(numTiles), int(mosaic), str(format), True if randomTiles else False, True if tms else False, str(colorscale), str(tileids), str(bounds), True if allLevels else False, float(overridenodata))
else:
raise Exception('input types differ (TODO: expand this message!)')
if (op.setup(self.job, self.context.getConf()) and
op.execute(self.context) and
op.teardown(self.job, self.context.getConf())):
new_resource = copy.copy(self)
new_resource.mapop = op
gdalutils = JavaClass('org.mrgeo.utils.GDALUtils', gateway_client=self.gateway._gateway_client)
java_image = op.image()
width = java_image.getRasterXSize()
height = java_image.getRasterYSize()
options = []
if format == 'jpg' or format == 'jpeg':
driver_name = 'jpeg'
extension = 'jpg'
elif format == 'tif' or format == 'tiff' or format == 'geotif' or format == 'geotiff' or format == 'gtif' or format == 'gtiff':
driver_name = 'GTiff'
options.append('INTERLEAVE=BAND')
options.append('COMPRESS=DEFLATE')
options.append('PREDICTOR=1')
options.append('ZLEVEL=6')
options.append('TILES=YES')
if width < 2048:
options.append('BLOCKXSIZE=' + str(width))
else:
options.append('BLOCKXSIZE=2048')
if height < 2048:
options.append('BLOCKYSIZE=' + str(height))
else:
options.append('BLOCKYSIZE=2048')
extension = 'tif'
else:
driver_name = format
extension = format
datatype = java_image.GetRasterBand(1).getDataType()
if not local_name.endswith(extension):
local_name += "." + extension
driver = gdal.GetDriverByName(driver_name)
local_image = driver.Create(local_name, width, height, java_image.getRasterCount(), datatype, options)
local_image.SetProjection(str(java_image.GetProjection()))
local_image.SetGeoTransform(java_image.GetGeoTransform())
java_nodatas = gdalutils.getnodatas(java_image)
for i in xrange(1, local_image.RasterCount + 1):
start = time.time()
raw_data = gdalutils.getRasterDataAsCompressedBase64(java_image, i, 0, 0, width, height)
print('compressed/encoded data ' + str(len(raw_data)))
decoded_data = base64.b64decode(raw_data)
print('decoded data ' + str(len(decoded_data)))
decompressed_data = zlib.decompress(decoded_data, 16 + zlib.MAX_WBITS)
print('decompressed data ' + str(len(decompressed_data)))
byte_data = numpy.frombuffer(decompressed_data, dtype='b')
print('byte data ' + str(len(byte_data)))
image_data = byte_data.view(gdal_array.GDALTypeCodeToNumericTypeCode(datatype))
print('gdal-type data ' + str(len(image_data)))
image_data = image_data.reshape((-1, width))
#print('reshaped ' + str(len(byte_data)))
#print(byte_data)
#for j in xrange(0, 10):
# print(byte_data[j])
print('reshaped ' + str(len(image_data)) + " x " + str(len(image_data[0])))
band = local_image.GetRasterBand(i)
print('writing local image')
band.WriteArray(image_data)
print('done')
end = time.time()
print("elapsed time: " + str(end - start) + " sec.")
band.SetNoDataValue(java_nodatas[i - 1])
local_image.FlushCache()
print('flushed cache')
return new_resource
return None
def scala_companion(class_name, gateway_client=None):
"""Returns referece to Scala companion object"""
gateway_client = gateway_client or get_spark_context(
)._gateway._gateway_client
return JavaClass(class_name + "$", gateway_client).__getattr__("MODULE$")
def convert(self, x, gateway_client):
"""We assume that a class with the same name exists in jnegmas"""
class_name = f'j' + x.__class__.__module__ + '.' + x.__class__.__name__
java_object = JavaClass(class_name, gateway_client)()
java_object.fill(x.to_java())
return java_object
