def check():
imm_df = spark.read.parquet(
's3a://<s3-bucket>/processed/immigration/i94_{0}_sub.parquet'.format(month_year) # noqa
)
result = imm_df.count()
if result == 0:
raise Py4JError('data count is 0!')
def check():
imm_city_df = spark.read.parquet(
's3a://<s3-bucket>/processed/imm_{0}_city_demographics.parquet'.format(
month_year) # noqa
)
result = imm_city_df.count()
if result == 0:
raise Py4JError('data count is 0!')
def shutdown_gateway(self):
"""Sends a shutdown command to the Java side.
This will close the ClientServer on the Java side: all active
connections will be closed. This may be useful if the lifecycle
of the Java program must be tied to the Python program.
"""
if not self.is_connected:
raise Py4JError("Gateway must be connected to send shutdown cmd.")
try:
quiet_close(self.stream)
self.socket.sendall(
proto.SHUTDOWN_GATEWAY_COMMAND_NAME.encode("utf-8"))
self.close()
except Exception:
# Do nothing! Exceptions might occur anyway.
logger.debug("Exception occurred while shutting down gateway",
exc_info=True)
def send_command(self, command):
# TODO At some point extract common code from wait_for_commands
logger.debug("Command to send: {0}".format(command))
try:
self.socket.sendall(command.encode("utf-8"))
while True:
answer = smart_decode(self.stream.readline()[:-1])
logger.debug("Answer received: {0}".format(answer))
# Happens when a the other end is dead. There might be an empty
# answer before the socket raises an error.
if answer.strip() == "":
self.close()
raise Py4JError("Answer from Java side is empty")
if answer.startswith(proto.RETURN_MESSAGE):
return answer[1:]
else:
command = answer
obj_id = smart_decode(self.stream.readline())[:-1]
if command == proto.CALL_PROXY_COMMAND_NAME:
return_message = self._call_proxy(obj_id, self.stream)
self.socket.sendall(return_message.encode("utf-8"))
elif command == proto.GARBAGE_COLLECT_PROXY_COMMAND_NAME:
self.stream.readline()
del (self.pool[obj_id])
self.socket.sendall(
proto.SUCCESS_RETURN_MESSAGE.encode("utf-8"))
else:
logger.error("Unknown command {0}".format(command))
# We're sending something to prevent blocking,
# but at this point, the protocol is broken.
self.socket.sendall(
proto.ERROR_RETURN_MESSAGE.encode("utf-8"))
except Exception as e:
logger.exception("Error while sending or receiving.")
raise Py4JNetworkError("Error while sending or receiving", e)