# Authors:
# - Wen Guan, <*****@*****.**>, 2019
"""
Main start entry point for iDDS service
"""
import logging
import signal
import time
import traceback
from idds.common.constants import Sections
from idds.common.config import config_has_section, config_has_option, config_list_options, config_get
from idds.common.utils import setup_logging
setup_logging('idds.log')
AGENTS = {
'baseagent': ['idds.agents.common.baseagent.BaseAgent', Sections.Common],
'clerk': ['idds.agents.clerk.clerk.Clerk', Sections.Clerk],
'marshaller':
['idds.agents.marshaller.marshaller.Marshaller', Sections.Marshaller],
'transformer':
['idds.agents.transformer.transformer.Transformer', Sections.Transformer],
'transporter':
['idds.agents.transporter.transporter.Transporter', Sections.Transporter],
'carrier': ['idds.agents.carrier.carrier.Carrier', Sections.Carrier],
'conductor':
['idds.agents.conductor.conductor.Conductor', Sections.Conductor]
}
RUNNING_AGENTS = []
# python 3
from queue import Queue
except ImportError:
# Python 2
from Queue import Queue
from idds.common.constants import (Sections, TransformType, ProcessingStatus,
ProcessingLocking, MessageType,
MessageStatus, MessageSource)
from idds.common.exceptions import (AgentPluginError)
from idds.common.utils import setup_logging
from idds.core import (catalog as core_catalog, transforms as core_transforms,
processings as core_processings)
from idds.agents.common.baseagent import BaseAgent
setup_logging(__name__)
class Carrier(BaseAgent):
"""
Carrier works to submit and monitor tasks to WFMS.
"""
def __init__(self,
num_threads=1,
poll_time_period=1800,
retrieve_bulk_size=None,
message_bulk_size=1000,
**kwargs):
super(Carrier, self).__init__(num_threads=num_threads, **kwargs)
self.config_section = Sections.Carrier
self.poll_time_period = int(poll_time_period)
import logging
import sys
import datetime
from idds.common.utils import setup_logging, json_dumps # noqa F401
setup_logging(__name__, stream=sys.stdout, loglevel=logging.DEBUG)
from idds.common.constants import ContentStatus, ContentType, ContentRelationType, ContentLocking # noqa F401
from idds.core.requests import get_requests # noqa F401
from idds.core.messages import retrieve_messages # noqa F401
from idds.core.transforms import get_transforms # noqa F401
from idds.core.workprogress import get_workprogresses # noqa F401
from idds.core.processings import get_processings # noqa F401
from idds.core import transforms as core_transforms # noqa F401
from idds.orm.contents import get_input_contents # noqa F401
from idds.core.transforms import release_inputs_by_collection, release_inputs_by_collection_old # noqa F401
def show_works(req):
workflow = req['processing_metadata']['workflow']
print(workflow.independent_works)
print(len(workflow.independent_works))
print(workflow.works_template.keys())
print(len(workflow.works_template.keys()))
print(workflow.work_sequence.keys())
print(len(workflow.work_sequence.keys()))
print(workflow.works.keys())
print(len(workflow.works.keys()))
work_ids = []
for i_id in workflow.works:
