def check_revived_treads():
"""
Super restrictive mode. We control how many threads relive, and if we overcome a threshold,
daemon will be killed.
"""
if revived_threads > MAX_REVIVED_THREADS:
msg = "ERROR: Raise MAX_REVIVED_THREADS threads: revived_threads: %d" % revived_threads
log.error(msg)
g = GalotecniaSupport()
g.process_error(msg)
pf = file(options.pidFile, 'r')
pid = int(pf.read().strip())
pf.close()
try:
while 1:
os.kill(pid, signal.SIGTERM)
time.sleep(1)
except IOError, err:
err = str(err)
if err.find("No such process") > 0:
log.info("Leaving daemon...")
os.remove(options.pidFile)
sys.exit(0)
else:
log.error(str(err))
sys.exit(1)
def check_processing_msg_list(self):
connectors_dict = {}
now = datetime.datetime.now()
top = now - datetime.timedelta(settings.DEFAULT_CHECK_DAYS)
# Obtenemos la lista de mensajes que tienen estado temporal en el servidor
query = SMSHistory.objects.filter(server_status__in = [NONE_STATUS, PROCESSING_STATUS], local_status = SENT_STATUS,
sentDate__gte = top).order_by('sentDate')
log.info("MessageConsulter: Consulting a list of %d messages" % len(query))
i = 0
for msg in query:
status = NONE_STATUS
try:
# PANIC MODE!!!
log.debug("History %d: updating status" % msg.id)
acc_id = msg.message.account.id
# Checking if message account is on memory, otherwise we add it
if not connectors_dict.has_key(acc_id):
backend = msg.message.account.access.backend
GenericConnector = backends[backend]
connector = GenericConnector(msg.message.account.args())
connectors_dict[acc_id] = connector
con = connectors_dict[acc_id]
# Checking msg status, we only take first object
[status, info] = con.get_info(msg.remote_id)[0]
msg.server_status = status
msg.status_info = info
# If we got a valid status, we update send date field of msg
if status != NONE_STATUS and status != PROCESSING_STATUS:
msg.sentDate = datetime.datetime.now()
log.debug("History %s: setting server status to %s" %(msg.id, msg.server_status))
msg.save()
except Exception, e:
# PANIC MODE!!!
# If checking status proccess fails (because connector error, status
# consult error, or while saving the msg), we continue with next task
log.error("Error al consultar el estado del mensaje %s", msg.id)
get_traceback(e)
g = GalotecniaSupport()
g.process_exception('Actualizacion del estado de mensajes', e)
# Not a valid status
if status == PROCESSING_STATUS:
i += 1
# Every 25 not valid consults, we wait a while
if i % 25 == 0:
time.sleep(30)
def update_incoming_list(self):
# PANIC MODE: This function cannot be blocked
try:
# Process unprocessed messages (because a daemon fail happended)
self.process_unprocessed_msgs()
log.debug("Checking if there are new incoming messages in %s", self.access)
# Updating incoming message list and start to process new messages
self.conector.decode(self.access)
except Exception, e:
# An unhandled exception in process message happened
log.error("Access thread %s has failed trying to update incoming messages because %s", self.access, e)
get_traceback(e)
g = GalotecniaSupport()
g.process_exception('Updating incoming messages', e)
def process_unprocessed_msgs(self):
"""
Processes associated messages that were not processed
Useful in cases where, after devil fails, we want to retrieve the state in which
stayed (in the event that access is required to process the incoming messages)
"""
if 'process' in self.access.args():
for no_proc_imsg in IncomingMessage.objects.filter(account__in = self.access.account_set.all(), processed = False):
try:
self.conector.encode(no_proc_imsg)
except Exception, e:
log.error("Error: trying to process %s message in %s access, %s", no_proc_imsg, self.access, e)
get_traceback(e)
g = GalotecniaSupport()
g.process_exception('Error while processing a incoming message', e)
# If there is no fault to be processed by the system load is not generated more
# and is marked as processed without processing date, so you can
# retrieve the messages that failed because they have no processing time
if not no_proc_imsg.processed:
no_proc_imsg.processed = True
no_proc_imsg.save()
def run(self):
"""
Send sms method. It sends SMS over a given connector (by SMS's account)
"""
log.info("starting thread in AccountThread")
n = 0
while self.parent.running:
log.info("Waiting for tasks")
try:
sq = self.parent.work.get(True, 10)
n += 1
except Queue.Empty:
log.info("No more tasks, empty queue")
break
log.info("id %s: start sending %d" % (sq.get_id(), n))
try:
# KIND OF MESSAGE: SMS | MMS
credit = SMS_CREDIT
if sq.message.body.mms:
credit = MMS_CREDIT
local_status = NONE_STATUS
server_status = NONE_STATUS
status_info = ''
batch_id = ''
today = datetime.datetime.now()
available_credit = self.account.get_available_credit()
if not available_credit:
# Run out of credit, delay sending process
sq.nextProcDate = today + datetime.timedelta(0,settings.DEFAULT_DEACTIVATION_TIME)
sq.save()
status_info = "Run out of credit, delayed %d day(s)" % NEXT_SEND
local_status = PROCESSING_STATUS
else:
if sq.message.activationDate > today:
# This message is not active yet
sq.nextProcDate = sq.message.activationDate
sq.save()
status_info = "Sending delayed until activation date"
local_status = PROCESSING_STATUS
else:
# Active message
if sq.message.deactivationDate and today > sq.message.deactivationDate:
# Expired
status_info = "Sending date expired"
local_status = FAIL_STATUS
else:
# Not deactivation date or not sent, we create an instance
# of backend (connector) related to this message.
GenericConnector = backends[self.account.access.backend]
connector = GenericConnector(self.account.args())
log.debug("Account connector = %s (%d) is %s", self.account, self.account.id, connector)
# Send message
server_status, status_info, batch_id = self._send_message(connector, credit, sq)
if server_status != NONE_STATUS:
local_status = SENT_STATUS
else:
local_status = PROCESSING_STATUS
sq.nextProcDate = today + datetime.timedelta(0,settings.DEFAULT_RETRY_TIME_CONNECTOR_TEMP_FAIL)
log.warn("Cannot send the message over his defined connector, it will be retried in a few minutes.")
sq.save()
log.debug("Status of message %d = (local = %d, server = %d)" % (sq.message.id, local_status, server_status))
self._update_credit(sq, local_status)
# Status message updated
if not sq.message.firstSentDate:
sq.message.firstSentDate = datetime.datetime.now()
sq.message.lastSentDate = datetime.datetime.now()
sq.message.save()
except Exception, e:
#
# !!PANIC MODE!!
#
# Unhandled error. Activate a panic mode. It will mark the local error message
# and will be removed from the queue. For this to work the code below should return no exception.
# If you enter this code strange things can happen, for example, may have failed to
# update the credit and yet having sent the message to the operator. To try to prevent
# something worse going to delete the problem message. Also possible that the error is
# software that will be marked with all error messages.
log.error('Fatal error in sending on AccountThread: %s' % e)
# Forzamos el borrado del mensaje problemático
local_status = FAIL_STATUS
if not status_info:
status_info = "A falta error has happened when daemon tried to send %s over this AccountThread" % sq
g = GalotecniaSupport()
g.process_exception('Sending process over singularms daemon', e)
get_traceback(e)
# MessageID_smsqueueID saved before object will be deleted
id = sq.get_id()
# Delete smsqueue (if it doesn't have to be fowarded again)
if local_status != PROCESSING_STATUS:
log.info("Deleting SMSQueue object id: %d" % sq.id)
sq.delete()
# Temporal SMSHistory data
message = sq.message
mobile = sq.mobile
priority = sq.priority
# SMSHistory object creation. Write errors, sent time, sq.id and local_id about it
sh = SMSHistory(message = message, mobile = mobile, priority = priority,
sentDate = datetime.datetime.now(), local_id = id, remote_id = batch_id,
local_status = local_status, status_info = status_info[:255], server_status = server_status)
sh.save()
# Task done. If this thread die before now, when parent thread calls join() method it will
# be blocked
self.parent.work.task_done()
# Main loop can continue
log.debug("id %s: end sending" % id)