예제 #1
0
def real_doout(dest_path, element):
    
    if dest_path is None:
        stdout(element)
        stdoutf()
    else:
        dfile=os.path.join(dest_path, element)
        code, msg=touch(dfile)
        if code!="ok":
            logging.warning("Can't touch file '%s'  (%s)" % (dfile, msg))
예제 #2
0
def do_write_done(src_filename):
    dfile = "%s.done" % src_filename
    code, _ = touch(dfile)
    if code != "ok":
        raise Exception("Can't write '.done' file: %s" % dfile)
예제 #3
0
def run_aws(node_id, proc, polling_interval, queue_name, topic_name, dst_path, delete_queue):

    if topic_name is None:
        raise Exception("Need a topic_name")

    auto_queue = False
    if queue_name is None:
        auto_queue = True
        queue_name = gen_queue_name()

    def setup_private_queue():
        conn = boto.connect_sqs()
        q = conn.create_queue(queue_name)
        q.set_message_class(RawMessage)
        return (conn, q)

    # SETUP PRIVATE QUEUE
    logging.info("Creating queue '%s'" % queue_name)
    sqs_conn, q = retry(setup_private_queue, logmsg="Having trouble creating queue...")

    topic_arn = build_topic_arn(sqs_conn, topic_name)

    logging.info("topic_arn: %s" % topic_arn)

    ### create topic
    def create_topic():
        """
        {'CreateTopicResponse': 
            {'ResponseMetadata': 
                {'RequestId': '5e2c6700-4dd0-11e1-b421-41716ce69b95'}, 
            'CreateTopicResult': {'TopicArn': 'arn:aws:sns:us-east-1:674707187858:election'}}}
        """
        snsconn = boto.connect_sns()
        snsconn.create_topic(topic_name)

    retry(create_topic, logmsg="Having trouble creating topic...")

    # SUBSCRIBE TO TOPIC
    def sub_topic():
        snsconn = boto.connect_sns()
        snsconn.subscribe_sqs_queue(topic_arn, q)
        return snsconn

    snsconn = retry(sub_topic, logmsg="Having trouble subscribing queue to topic...")

    logging.info("Subscribed to topic '%s'" % topic_name)

    current_state = "NL"

    MSGS = {"NL": "Leadership lost", "L": "Leadership gained", "ML": "Leadership momentum"}

    poll_count = 0

    def cleanup():
        rm(dst_path)
        if auto_queue or delete_queue:
            logging.info("... deleting queue")
            try:
                sqs_conn.delete_queue(q)
            except:
                pass

    ppid = os.getppid()
    logging.info("Process pid: %s" % os.getpid())
    logging.info("Parent pid: %s" % ppid)
    logging.info("Starting loop...")
    while True:
        if os.getppid() != ppid:
            logging.warning("Parent terminated... exiting")
            cleanup()
            break

        try:
            ## do a bit of garbage collection :)
            global sigtermReceived
            if sigtermReceived:
                cleanup()
                raise SignalTerminate()
            #########################################

            try:
                ### BATCH PROCESS - required!!!
                while True:
                    rawmsg = q.read()
                    if rawmsg is not None:
                        jsonmsg = json.loads(rawmsg.get_body())
                        q.delete_message(rawmsg)

                        ## SNS encapsulates the original message...
                        nodeid = str(jsonmsg["Message"])

                        transition, current_state = proc.send((poll_count, nodeid))
                        jstdout({"state": current_state})

                        if transition:
                            logging.info(MSGS[current_state])
                            if current_state == "L":
                                code, _ = touch(dst_path)
                                logging.info("Created '%s': %s" % (dst_path, code))
                            else:
                                code, _ = rm(dst_path)
                                logging.info("Deleted '%s': %s" % (dst_path, code))
                    else:
                        break

            except SQSDecodeError:
                logging.warning("Message decoding error")

            except Exception, e:
                logging.error(str(e))
                continue

            msg = str(node_id)

            logging.debug("Publishing our 'node id': %s" % node_id)
            try:
                snsconn.publish(topic_arn, msg)
            except:
                try:
                    snsconn.publish(topic_arn, msg)
                except:
                    logging.warning("Can't publish to topic '%s'" % topic_name)

            logging.debug("... sleeping for %s seconds" % polling_interval)
            sleep(polling_interval)
            poll_count = poll_count + 1

        except KeyboardInterrupt:
            cleanup()
            raise