Пример #1
0
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
"""
Helper for multiprocessing
"""
import multiprocessing
from helpers.logging_helper import get_logger

LOG = get_logger(__name__)


class Consumer(multiprocessing.Process):
    def __init__(self, task_queue):
        multiprocessing.Process.__init__(self)
        self.task_queue = task_queue

    def run(self):
        proc_name = self.name
        while True:
            next_task = self.task_queue.get()
            if next_task is None:
                # Poison pill means shutdown
                LOG.info('{0}: Exiting'.format(proc_name))
                self.task_queue.task_done()
#    MA 02111-1307  USA
#
"""
Add ids to queue - filtered
"""
import argparse
import sys
from boto.sqs.message import Message
from sqlalchemy import create_engine, select, func, and_
from configuration import DB_LOGIN
from database.tables import GALAXY
from helpers.logging_helper import get_logger
from helpers.sqs_helper import SqsHelper


LOG = get_logger(__name__)
LOG.info('PYTHONPATH = {0}'.format(sys.path))


def add_ids_to_queue(args, and_expression):
    engine = create_engine(DB_LOGIN)
    connection = engine.connect()

    # noinspection PyBroadException
    try:
        queue = None
        if not args.test:
            sqs_helper = SqsHelper('us-east-1')
            queue = sqs_helper.get_queue(args.queue_name)

        select_galaxy = select([GALAXY.c.galaxy_id]).where(and_expression).order_by(func.RAND())
Пример #3
0
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
"""
Common code
"""
from _socket import error
import os
import socket
from sqlalchemy import select, and_
from constants import PARAMETERS, STATISTICS
from database.tables import DATA_TABLES_SED, DATA_TABLES_ORIGINAL, STEP_DONE_NEW
from helpers import logging_helper

LOG = logging_helper.get_logger(__name__)


def get_galaxy_file_name(galaxy_name, run_id, galaxy_id):
    """
    Return the fully formatted galaxy name

    :param galaxy_name:
    :param galaxy_id:
    :param run_id:
    :return: the galaxy name
    """
    return '{0:06d}__{1}__{2}'.format(galaxy_id, run_id, galaxy_name)


def get_galaxy_file_name_pogs(galaxy_name, run_id, galaxy_id):