# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307 USA
#
"""
Helper for multiprocessing
"""
import multiprocessing
from helpers.logging_helper import get_logger
LOG = get_logger(__name__)
class Consumer(multiprocessing.Process):
def __init__(self, task_queue):
multiprocessing.Process.__init__(self)
self.task_queue = task_queue
def run(self):
proc_name = self.name
while True:
next_task = self.task_queue.get()
if next_task is None:
# Poison pill means shutdown
LOG.info('{0}: Exiting'.format(proc_name))
self.task_queue.task_done()
# MA 02111-1307 USA
#
"""
Add ids to queue - filtered
"""
import argparse
import sys
from boto.sqs.message import Message
from sqlalchemy import create_engine, select, func, and_
from configuration import DB_LOGIN
from database.tables import GALAXY
from helpers.logging_helper import get_logger
from helpers.sqs_helper import SqsHelper
LOG = get_logger(__name__)
LOG.info('PYTHONPATH = {0}'.format(sys.path))
def add_ids_to_queue(args, and_expression):
engine = create_engine(DB_LOGIN)
connection = engine.connect()
# noinspection PyBroadException
try:
queue = None
if not args.test:
sqs_helper = SqsHelper('us-east-1')
queue = sqs_helper.get_queue(args.queue_name)
select_galaxy = select([GALAXY.c.galaxy_id]).where(and_expression).order_by(func.RAND())
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
# MA 02111-1307 USA
#
"""
Common code
"""
from _socket import error
import os
import socket
from sqlalchemy import select, and_
from constants import PARAMETERS, STATISTICS
from database.tables import DATA_TABLES_SED, DATA_TABLES_ORIGINAL, STEP_DONE_NEW
from helpers import logging_helper
LOG = logging_helper.get_logger(__name__)
def get_galaxy_file_name(galaxy_name, run_id, galaxy_id):
"""
Return the fully formatted galaxy name
:param galaxy_name:
:param galaxy_id:
:param run_id:
:return: the galaxy name
"""
return '{0:06d}__{1}__{2}'.format(galaxy_id, run_id, galaxy_name)
def get_galaxy_file_name_pogs(galaxy_name, run_id, galaxy_id):