Beispiel #1
0
    def start(self,
              settings,
              article_id=None,
              version=None,
              run=None,
              publication_data=None):

        # TODO : much of this is common to many starters and could probably be streamlined

        # Log
        logger = helper.get_starter_logger(
            settings.setLevel, helper.get_starter_identity(self.const_name))

        if article_id is None or version is None or publication_data is None:
            raise NullRequiredDataException(
                "Did not get an article id, version or publication data")

        info = {
            'article_id': article_id,
            'version': str(version),
            'run': run,
            'publication_data': publication_data
        }

        workflow_id, \
        workflow_name, \
        workflow_version, \
        child_policy, \
        execution_start_to_close_timeout, \
        workflow_input = helper.set_workflow_information(self.const_name, "1", None, info,
                                                         article_id + "." + str(version))

        # Simple connect
        conn = boto.swf.layer1.Layer1(settings.aws_access_key_id,
                                      settings.aws_secret_access_key)

        try:
            response = conn.start_workflow_execution(
                settings.domain, workflow_id, workflow_name, workflow_version,
                settings.default_task_list, child_policy,
                execution_start_to_close_timeout, workflow_input)

            logger.info('got response: \n%s' %
                        json.dumps(response, sort_keys=True, indent=4))

        except NullRequiredDataException as e:
            logger.exception(e.message)
            raise

        except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
            # There is already a running workflow with that ID, cannot start another
            message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
            logger.info(message)
    def start(self, settings, article_id, version, requested_action, result, expanded_folder, status, eif_location, run, update_date, message=None):

        # Log
        logger = helper.get_starter_logger(settings.setLevel, helper.get_starter_identity(self.const_name))

        if article_id is None or run is None or version is None:
            raise NullRequiredDataException("article id or version or run is Null. "
                                            "Possible error: Lax did not send back valid data.")

        input = {
            "run": run,
            "article_id": article_id,
            "result": result,
            "status": status,
            "version": version,
            "expanded_folder": expanded_folder,
            "eif_location": eif_location,
            "requested_action": requested_action,
            "message": message,
            "update_date": update_date,
            "force": True
        }

        workflow_id, \
        workflow_name, \
        workflow_version, \
        child_policy, \
        execution_start_to_close_timeout, \
        workflow_input = helper.set_workflow_information(self.const_name, "1", None, input, article_id)

        # Simple connect
        conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)

        try:
            response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version,
                                                     settings.default_task_list, child_policy,
                                                     execution_start_to_close_timeout, workflow_input)

            logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))

        except NullRequiredDataException as e:
            logger.exception(e.message)
            raise

        except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
            # There is already a running workflow with that ID, cannot start another
            message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
            logger.info(message)
    def start(self,eif_location, eif_bucket, article_id, version, run, article_path ,expanded_folder, status, update_date, published, settings, info=None):

        # Log
        logger = helper.get_starter_logger(settings.setLevel, helper.get_starter_identity(self.const_name))

        if article_id is None or run is None or version is None:
            raise NullRequiredDataException("article id or version or run is Null. "
                                            "Possible error: Lax did not send back valid data from ingest.")

        input = {
            'eif_location': eif_location,
            'eif_bucket':  eif_bucket,
            'article_id': article_id,
            'version': version,
            'run': run,
            'article_path': article_path,
            'expanded_folder': expanded_folder,
            'status': status,
            'update_date': update_date,
            'published': published
        }

        workflow_id, \
        workflow_name, \
        workflow_version, \
        child_policy, \
        execution_start_to_close_timeout, \
        workflow_input = helper.set_workflow_information(self.const_name, "1", None, input, article_id, os.getpid())

        # Simple connect
        conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)

        try:
            response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version,
                                                     settings.default_task_list, child_policy,
                                                     execution_start_to_close_timeout, workflow_input)

            logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))

        except NullRequiredDataException as e:
            logger.exception(e.message)
            raise

        except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
            # There is already a running workflow with that ID, cannot start another
            message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
            logger.info(message)
    def start(self, settings, article_id, version, requested_action, result, expanded_folder, status, eif_location, run, update_date, message=None):

        logger = helper.get_starter_logger(settings.setLevel, helper.get_starter_identity(self.const_name))

        if article_id is None or run is None or version is None:
            raise NullRequiredDataException("article id or version or run is Null. "
                                            "Possible error: Lax did not send back valid data from ingest.")
        input = {
            "run": run,
            "article_id": article_id,
            "result": result,
            "status": status,
            "version": version,
            "expanded_folder": expanded_folder,
            "eif_location": eif_location,
            "requested_action": requested_action,
            "message": message,
            "update_date": update_date
        }

        workflow_id, \
        workflow_name, \
        workflow_version, \
        child_policy, \
        execution_start_to_close_timeout, \
        workflow_input = helper.set_workflow_information(self.const_name, "1", None, input, article_id)

        # Simple connect
        conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)

        try:
            response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version,
                                                     settings.default_task_list, child_policy,
                                                     execution_start_to_close_timeout, workflow_input)

            logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))

        except NullRequiredDataException as e:
            logger.exception(e.message)
            raise

        except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
            # There is already a running workflow with that ID, cannot start another
            message = 'SWFWorkflowExecutionAlreadyStartedError: ' \
                      'There is already a running workflow with ID %s' % workflow_id
            logger.info(message)
Beispiel #5
0
    def start(self, settings, info=None, run=None):

        # Log
        logger = helper.get_starter_logger(
            settings.setLevel, helper.get_starter_identity(self.const_name))

        if hasattr(info, 'file_name') == False or info.file_name is None:
            raise NullRequiredDataException(
                "filename is Null / Did not get a filename.")

        input = S3NotificationInfo.to_dict(info)
        input['run'] = run
        input['version_lookup_function'] = "article_highest_version"
        input['force'] = True

        workflow_id, \
        workflow_name, \
        workflow_version, \
        child_policy, \
        execution_start_to_close_timeout, \
        workflow_input = helper.set_workflow_information(self.const_name, "1", None, input,
                                                         info.file_name.replace('/', '_'))

        # Simple connect
        conn = boto.swf.layer1.Layer1(settings.aws_access_key_id,
                                      settings.aws_secret_access_key)

        try:
            response = conn.start_workflow_execution(
                settings.domain, workflow_id, workflow_name, workflow_version,
                settings.default_task_list, child_policy,
                execution_start_to_close_timeout, workflow_input)

            logger.info('got response: \n%s' %
                        json.dumps(response, sort_keys=True, indent=4))

        except NullRequiredDataException as e:
            logger.exception(e.message)
            raise

        except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
            # There is already a running workflow with that ID, cannot start another
            message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
            logger.info(message)
    def start(self, settings, article_id=None, version=None, run=None, publication_data=None):

        # TODO : much of this is common to many starters and could probably be streamlined

        # Log
        logger = helper.get_starter_logger(settings.setLevel, helper.get_starter_identity(self.const_name))

        if article_id is None or version is None or publication_data is None:
            raise NullRequiredDataException("Did not get an article id, version or publication data")

        info = {
            'article_id': article_id,
            'version': str(version),
            'run': run,
            'publication_data': publication_data
        }

        workflow_id, \
        workflow_name, \
        workflow_version, \
        child_policy, \
        execution_start_to_close_timeout, \
        workflow_input = helper.set_workflow_information(self.const_name, "1", None, info,
                                                         article_id + "." + str(version),
                                                         os.getpid())

        # Simple connect
        conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)

        try:
            response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version,
                                                     settings.default_task_list, child_policy,
                                                     execution_start_to_close_timeout, workflow_input)

            logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))

        except NullRequiredDataException as e:
            logger.exception(e.message)
            raise

        except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
            # There is already a running workflow with that ID, cannot start another
            message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
            logger.info(message)
    def start(self, info, settings):

        logger = helper.get_starter_logger(
            settings.setLevel, helper.get_starter_identity(self.const_name))

        if info['article_id'] is None:
            raise NullRequiredDataException(
                "article id is Null. Possible error: "
                "Lax did not send back valid data from ingest.")

        publication_from = "lax" if 'requested_action' in info else 'website'

        workflow_id, \
        workflow_name, \
        workflow_version, \
        child_policy, \
        execution_start_to_close_timeout, \
        workflow_input = helper.set_workflow_information(self.const_name, "1", None, info, info['article_id'],
                                                         publication_from)

        # Simple connect
        conn = boto.swf.layer1.Layer1(settings.aws_access_key_id,
                                      settings.aws_secret_access_key)

        try:
            response = conn.start_workflow_execution(
                settings.domain, workflow_id, workflow_name, workflow_version,
                settings.default_task_list, child_policy,
                execution_start_to_close_timeout, workflow_input)

            logger.info('got response: \n%s' %
                        json.dumps(response, sort_keys=True, indent=4))

        except NullRequiredDataException as e:
            logger.exception(e.message)
            raise

        except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
            # There is already a running workflow with that ID, cannot start another
            message = 'SWFWorkflowExecutionAlreadyStartedError: ' \
                      'There is already a running workflow with ID %s' % workflow_id
            logger.error(message)
Beispiel #8
0
    def start(self, settings, article_id=None, version=None, run=None, standalone=False, standalone_is_poa=False):

        # Log
        logger = helper.get_starter_logger(settings.setLevel, helper.get_starter_identity(self.const_name))

        if article_id is None:
            raise NullRequiredDataException("Did not get an article id. Required.")

        info = {
            'run': run,
            'article_id': article_id,
            'version': version,
            'standalone': standalone,
            'standalone_is_poa': standalone_is_poa
        }

        workflow_id, \
        workflow_name, \
        workflow_version, \
        child_policy, \
        execution_start_to_close_timeout, \
        workflow_input = helper.set_workflow_information(self.const_name, "1", None, info, article_id)

        # Simple connect
        conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)

        try:
            response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version,
                                                     settings.default_task_list, child_policy,
                                                     execution_start_to_close_timeout, workflow_input)

            logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))

        except NullRequiredDataException as e:
            logger.exception(e.message)
            raise

        except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
            # There is already a running workflow with that ID, cannot start another
            message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
            logger.exception(message)
    def start(self, settings, run, info):

        # Log
        logger = helper.get_starter_logger(settings.setLevel, helper.get_starter_identity(self.const_name))

        if hasattr(info, 'file_name') == False or info.file_name is None:
            raise NullRequiredDataException("filename is Null. Did not get a filename.")

        input = S3NotificationInfo.to_dict(info)
        input['run'] = run
        input['version_lookup_function'] = "article_next_version"

        workflow_id, \
        workflow_name, \
        workflow_version, \
        child_policy, \
        execution_start_to_close_timeout, \
        workflow_input = helper.set_workflow_information(self.const_name, "1", None, input,
                                                         info.file_name.replace('/', '_'),
                                                         start_to_close_timeout=str(60 * 60 * 5))

        # Simple connect
        conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)

        try:
            response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version,
                                                     settings.default_task_list, child_policy,
                                                     execution_start_to_close_timeout, workflow_input)

            logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))

        except NullRequiredDataException as e:
            logger.exception(e.message)
            raise

        except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
            # There is already a running workflow with that ID, cannot start another
            message = 'SWFWorkflowExecutionAlreadyStartedError: ' \
                      'There is already a running workflow with ID %s' % workflow_id
            logger.info(message)
    def start(self, info, settings):

        logger = helper.get_starter_logger(settings.setLevel, helper.get_starter_identity(self.const_name))

        if info['article_id'] is None:
            raise NullRequiredDataException("article id is Null. Possible error: "
                                            "Lax did not send back valid data from ingest.")

        publication_from = "lax" if 'requested_action' in info else 'website'

        workflow_id, \
        workflow_name, \
        workflow_version, \
        child_policy, \
        execution_start_to_close_timeout, \
        workflow_input = helper.set_workflow_information(self.const_name, "1", None, info, info['article_id'],
                                                         publication_from)

        # Simple connect
        conn = boto.swf.layer1.Layer1(settings.aws_access_key_id, settings.aws_secret_access_key)

        try:
            response = conn.start_workflow_execution(settings.domain, workflow_id, workflow_name, workflow_version,
                                                     settings.default_task_list, child_policy,
                                                     execution_start_to_close_timeout, workflow_input)

            logger.info('got response: \n%s' % json.dumps(response, sort_keys=True, indent=4))

        except NullRequiredDataException as e:
            logger.exception(e.message)
            raise

        except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
            # There is already a running workflow with that ID, cannot start another
            message = 'SWFWorkflowExecutionAlreadyStartedError: ' \
                      'There is already a running workflow with ID %s' % workflow_id
            logger.info(message)
Beispiel #11
0
    def start(self,
              eif_location,
              eif_bucket,
              article_id,
              version,
              run,
              article_path,
              expanded_folder,
              status,
              update_date,
              published,
              settings,
              info=None):

        # Log
        logger = helper.get_starter_logger(
            settings.setLevel, helper.get_starter_identity(self.const_name))

        if article_id is None or run is None or version is None:
            raise NullRequiredDataException(
                "article id or version or run is Null. "
                "Possible error: Lax did not send back valid data from ingest."
            )

        input = {
            'eif_location': eif_location,
            'eif_bucket': eif_bucket,
            'article_id': article_id,
            'version': version,
            'run': run,
            'article_path': article_path,
            'expanded_folder': expanded_folder,
            'status': status,
            'update_date': update_date,
            'published': published
        }

        workflow_id, \
        workflow_name, \
        workflow_version, \
        child_policy, \
        execution_start_to_close_timeout, \
        workflow_input = helper.set_workflow_information(self.const_name, "1", None, input, article_id)

        # Simple connect
        conn = boto.swf.layer1.Layer1(settings.aws_access_key_id,
                                      settings.aws_secret_access_key)

        try:
            response = conn.start_workflow_execution(
                settings.domain, workflow_id, workflow_name, workflow_version,
                settings.default_task_list, child_policy,
                execution_start_to_close_timeout, workflow_input)

            logger.info('got response: \n%s' %
                        json.dumps(response, sort_keys=True, indent=4))

        except NullRequiredDataException as e:
            logger.exception(e.message)
            raise

        except boto.swf.exceptions.SWFWorkflowExecutionAlreadyStartedError:
            # There is already a running workflow with that ID, cannot start another
            message = 'SWFWorkflowExecutionAlreadyStartedError: There is already a running workflow with ID %s' % workflow_id
            logger.info(message)