def main(port: int):
initialize_logging()
register_input_converter(DateTimeConverter())
gateway = ClientServer(java_parameters=JavaParameters(eager_load=False,
auto_convert=True,
auto_field=True),
python_parameters=PythonParameters(port=port,
eager_load=True),
python_server_entry_point=Runner())
register_output_converter(
proto.REFERENCE_TYPE,
lambda target_id, gateway_client: convert_reference_type(
target_id, gateway_client, gateway))
class DatetimeConverter(object):
def can_convert(self, obj):
return isinstance(obj, datetime.datetime)
def convert(self, obj, gateway_client):
Timestamp = JavaClass("java.sql.Timestamp", gateway_client)
seconds = calendar.timegm(obj.utctimetuple()) if obj.tzinfo else time.mktime(obj.timetuple())
t = Timestamp(int(seconds) * 1000)
t.setNanos(obj.microsecond * 1000)
return t
# datetime is a subclass of date, we should register DatetimeConverter first
register_input_converter(DatetimeConverter())
register_input_converter(DateConverter())
def _test():
import doctest
from pyspark.context import SparkContext
from pyspark.sql import SQLContext
globs = globals()
sc = SparkContext("local[4]", "PythonTest")
globs["sc"] = sc
globs["sqlContext"] = SQLContext(sc)
(failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
globs["sc"].stop()
if failure_count:
Date = JavaClass("java.sql.Date", gateway_client)
return Date.valueOf(obj.strftime("%Y-%m-%d"))
class DatetimeConverter(object):
def can_convert(self, obj):
return isinstance(obj, datetime.datetime)
def convert(self, obj, gateway_client):
Timestamp = JavaClass("java.sql.Timestamp", gateway_client)
return Timestamp(
int(time.mktime(obj.timetuple())) * 1000 + obj.microsecond // 1000)
# datetime is a subclass of date, we should register DatetimeConverter first
register_input_converter(DatetimeConverter())
register_input_converter(DateConverter())
def _test():
import doctest
from pyspark.context import SparkContext
# let doctest run in pyspark.sql.types, so DataTypes can be picklable
import pyspark.sql.types
from pyspark.sql import Row, SQLContext
from pyspark.sql.tests import ExamplePoint, ExamplePointUDT
globs = pyspark.sql.types.__dict__.copy()
sc = SparkContext('local[4]', 'PythonTest')
globs['sc'] = sc
globs['sqlContext'] = SQLContext(sc)
globs['ExamplePoint'] = ExamplePoint
java_list.add(element)
return java_list
class MapConverter(object):
def can_convert(self, object):
return hasattr2(object, "keys") and hasattr2(object, "__getitem__")
def convert(self, object, gateway_client):
HashMap = JavaClass("java.util.HashMap", gateway_client)
java_map = HashMap()
for key in object.keys():
java_map[key] = object[key]
return java_map
register_input_converter(SetConverter())
register_input_converter(MapConverter())
register_input_converter(ListConverter())
register_output_converter(
proto.MAP_TYPE, lambda target_id, gateway_client:
JavaMap(target_id, gateway_client))
register_output_converter(
proto.LIST_TYPE, lambda target_id, gateway_client:
JavaList(target_id, gateway_client))
register_output_converter(
proto.ARRAY_TYPE, lambda target_id, gateway_client:
JavaArray(target_id, gateway_client))
register_output_converter(
proto.SET_TYPE, lambda target_id, gateway_client:
JavaSet(target_id, gateway_client))
if obj.time_resolution:
scala_time_resolution = str(obj.time_resolution)
else:
scala_time_resolution = None
return ScalaTemporalStrategy.apply(obj.num_partitions, obj.bits,
scala_time_unit,
scala_time_resolution)
class SourceInfoConverter(object):
def can_convert(self, object):
return isinstance(object, SourceInfo)
def convert(self, obj, gateway_client):
ScalaSourceInfo = JavaClass("geopyspark.geotrellis.vlm.SourceInfo",
gateway_client)
return ScalaSourceInfo.apply(obj.source, obj.source_to_target_band)
register_input_converter(CellTypeConverter(), prepend=True)
register_input_converter(RasterizerOptionsConverter(), prepend=True)
register_input_converter(LayoutTypeConverter(), prepend=True)
register_input_converter(ResampleMethodConverter(), prepend=True)
register_input_converter(LayoutDefinitionConverter(), prepend=True)
register_input_converter(HashPartitionStrategyConverter(), prepend=True)
register_input_converter(SpatialPartitionStrategyConverter(), prepend=True)
register_input_converter(SpaceTimePartitionStrategyConverter(), prepend=True)
register_input_converter(SourceInfoConverter(), prepend=True)
return java_list
class MapConverter(object):
def can_convert(self, object):
return hasattr2(object, "keys") and hasattr2(object, "__getitem__")
def convert(self, object, gateway_client):
HashMap = JavaClass("java.util.HashMap", gateway_client)
java_map = HashMap()
for key in object.keys():
java_map[key] = object[key]
return java_map
register_input_converter(SetConverter())
register_input_converter(MapConverter())
register_input_converter(ListConverter())
register_output_converter(
proto.MAP_TYPE,
lambda target_id, gateway_client: JavaMap(target_id, gateway_client))
register_output_converter(
proto.LIST_TYPE,
lambda target_id, gateway_client: JavaList(target_id, gateway_client))
register_output_converter(
proto.ARRAY_TYPE,
lambda target_id, gateway_client: JavaArray(target_id, gateway_client))
register_output_converter(
proto.SET_TYPE,
lambda target_id, gateway_client: JavaSet(target_id, gateway_client))
return java_list
class MapConverter(object):
def can_convert(self, object):
return hasattr2(object, "keys") and hasattr2(object, "__getitem__")
def convert(self, object, gateway_client):
HashMap = JavaClass("java.util.HashMap", gateway_client)
java_map = HashMap()
for key in object.keys():
java_map[key] = object[key]
return java_map
register_input_converter(ListConverter(), prepend=True)
register_input_converter(SetConverter())
register_input_converter(MapConverter())
register_output_converter(
proto.MAP_TYPE,
lambda target_id, gateway_client: JavaMap(target_id, gateway_client))
register_output_converter(
proto.LIST_TYPE,
lambda target_id, gateway_client: JavaList(target_id, gateway_client))
register_output_converter(
proto.ARRAY_TYPE,
lambda target_id, gateway_client: JavaArray(target_id, gateway_client))
register_output_converter(
proto.SET_TYPE,
lambda target_id, gateway_client: JavaSet(target_id, gateway_client))
return sample.__getattr__("MODULE$")
class LayoutDefinitionConverter:
def can_convert(self, object):
return isinstance(object, LayoutDefinition)
def convert(self, obj, gateway_client):
python_extent = obj.extent
python_tile_layout = obj.tileLayout
ScalaExtent = JavaClass("geotrellis.vector.Extent", gateway_client)
ScalaTileLayout = JavaClass("geotrellis.raster.TileLayout", gateway_client)
ScalaLayoutDefinition = JavaClass("geotrellis.spark.tiling.LayoutDefinition", gateway_client)
extent = ScalaExtent(python_extent.xmin, python_extent.ymin, python_extent.xmax, python_extent.ymax)
tile_layout = ScalaTileLayout(python_tile_layout.layoutCols,
python_tile_layout.layoutRows,
python_tile_layout.tileCols,
python_tile_layout.tileRows)
return ScalaLayoutDefinition(extent, tile_layout)
register_input_converter(CellTypeConverter(), prepend=True)
register_input_converter(RasterizerOptionsConverter(), prepend=True)
register_input_converter(LayoutTypeConverter(), prepend=True)
register_input_converter(ResampleMethodConverter(), prepend=True)
register_input_converter(LayoutDefinitionConverter(), prepend=True)
"""
Register SQL extensions and py4j converters for a Spark session.
Args:
session: Spark session
new_session: If ``True``, create a new Spark session using ``session.newSession()`` before registering
extensions. This may be necessary if you're using functions that register new
analysis rules. The new session has isolated UDFs, configurations, and temporary tables,
but shares the existing ``SparkContext`` and cached data.
Example:
>>> import glow
>>> spark = glow.register(spark)
"""
assert check_argument_types()
sc = session._sc
return SparkSession(
sc,
session._jvm.io.projectglow.Glow.register(session._jsparkSession,
new_session))
# Register input converters in idempotent fashion
glow_input_converters = [
OneDimensionalDoubleNumpyArrayConverter,
TwoDimensionalDoubleNumpyArrayConverter
]
for gic in glow_input_converters:
if not any(type(pic) is gic for pic in protocol.INPUT_CONVERTER):
register_input_converter(gic(), prepend=True)
