Beispiel #1
0
    def configure(self, config):
        logging.info("Configure.")

        self.mappings = {}
        if config.has_section("Mappings"):
            self.mappings = {k: v for k, v in config.items("Mappings")}
        logging.log("Mappings", self.mappings, 'pp')
Beispiel #2
0
 def read_status(self, status_line):
     try:
         logging.log('Status line', status_line, 'info')
         if '%' in status_line:
             with_percent, rest = status_line.split('%', 1)
             status, value = with_percent.split(' ')
             self.update_progress(value)
     except e:
         logging.error("Error parsing status line from arftp: " + str(e))
Beispiel #3
0
    def start(self, data):
        # Parse the PluginData
        self.use(data)

        # Verfify that we have metadata (otherwise this plugin makes little sense)
        if self.asset.describedby_link.metadata is None:
            self.fail("Asset has no metadata")

        # Set the progress to 0 (this will show an empty progress bar in Studio)
        self.update_progress(0)

        logging.info("Media source:      %s", self.source)
        logging.info("Media destination: %s", self.destination)
        logging.info("Setting up FTP...")

        # Connect to the source FTP server
        ftp, source_path = connect(self.source, debug=True)
        source_filename = os.path.basename(source_path)
        source_directory = os.path.dirname(source_path)

        # Here you can also run other SITE commands before starting the transfer

        # Set up ArdFTP for copying the media file (you might have to use other
        # settings)
        ftp.voidcmd('SITE ARDENDO FORMAT MXF')
        ftp.voidcmd('SITE ARDENDO USE MXF %s' % source_filename)
        ftp.voidcmd('SITE ARDENDO STOR %s' % self.destination)

        # This "downloads" a status file from the source FTP server, containing
        # information about percentage done of the transfer. This is the main
        # advantage compared to using FXP
        ftp.retrlines('RETR status.log', callback=self.read_status)

        # When that transfer is finished, so is the media transfer and we may
        # close the connection to the source FTP server.
        ftp.quit()
        logging.log(u'FTP done', None, 'ok')

        # Now let's adjust the progress, we're not quite done yet
        self.update_progress(99)

        # Construct a xml destiniation path
        xml_destination = self.destination + ".xml"
        logging.info("XML Destination:   %s", xml_destination)
        logging.info("Writing metadata...")

        # Write the Asset Metadata to the xml destination path
        ftp_write(xml_destination, asset.describedby_link.metadata.generate())
        logging.log("Metadata writing done", None, 'ok')

        # Set the progress to 100% and we're done
        self.update_progress(100)
Beispiel #4
0
    def start(self, asset, conflict=False):
        # Fetch metadata with a fresh etag
        try:
            old_metadata = Payload(self.client.GET(asset.describedby_link))

        # If no metadata is found, we cannot do any metadata mapping
        except HTTPClientError as e:
            logging.error(u'Asset Metadata fetching failed %s', str(e))
            return

        # If there was an internal server error (5xx), wait and retry
        except HTTPServerError as e:
            logging.error(u'Asset Metadata fetching failed %s', str(e))
            logging.warn(u'Retrying in 10 seconds...')
            time.sleep(10)
            raise Retry

        # Copy metadata and operate on the copy, so we can compare them later
        new_metadata = Payload(old_metadata.generate())
        scope = {
            'asset': asset,
            'metadata': new_metadata,
        }
        for field, expr in self.mappings.items():
            logging.info('%s = %s', field, expr)
            new_metadata.set(field, eval(expr, scope))

        logging.log(u'Resulting payload', new_metadata.generate(), 'xml')
        
        # Only update if we produced any differences, else we will have a circle of updates
        if new_metadata != old_metadata:
            logging.info(u'Updating metadata...')
            self.client.PUT(
                asset.describedby_link,
                new_metadata,
                etag=old_metadata.response_etag
            )
            logging.info(u'Updated metadata.')
        else:
            logging.info(u'No changes to metadata.')
Beispiel #5
0
def get_stomp(username, password, app_name, heartbeat_interval, heartbeat_timeout, heartbeats):
    if hasattr(get_stomp, 'cached'):
        return get_stomp.cached

    get_stomp.cached = ConnectionManager(
        "flow_%s" % app_name,
        username=username,
        password=password,
        heartbeat_interval=heartbeat_interval
            if heartbeats else None,
        heartbeat_timeout=heartbeat_timeout
            if heartbeats else None,
    )

    logging.log("Stomp settings", {
        '(from Flow section) app name': app_name,
        'heartbeats': heartbeats,
        'heartbeat interval': heartbeat_interval,
        'heartbeat timeout': heartbeat_timeout,
    }, 'pp')

    return get_stomp.cached
Beispiel #6
0
    def start(self, data):
        # Parse the PluginData
        self.use(data)

        # Verfify that we have metadata (otherwise this plugin makes little sense)
        if self.asset.describedby_link.metadata is None:
            self.fail("Asset has no metadata")

        # Set the progress to 0 (this will show an empty progress bar in Studio)
        self.update_progress(0)

        logging.info("Media source:      %s", self.source)
        logging.info("Media destination: %s", self.destination)
        logging.info("Setting up FXP...")

        # Set up an FXP transfer (what is FXP? see
        # https://en.wikipedia.org/wiki/File_eXchange_Protocol)
        fxp = FXP(self.source, self.destination, debug=True)
        logging.info("FXP copying...")
        fxp.run()
        fxp.quit()
        logging.log("FXP done", None, 'ok')

        # Update the progress to 99%, we're not there yet
        self.update_progress(99)

        # Construct a xml destiniation path
        xml_destination = self.destination + ".xml"
        logging.info("XML Destination:   %s", xml_destination)
        logging.info("Writing metadata...")

        # Write the Asset Metadata to the xml destination path
        ftp_write(xml_destination, asset.describedby_link.metadata.generate())
        logging.log("Metadata done", None, 'ok')

        # Set the progress to 100% and we're done
        self.update_progress(100)
Beispiel #7
0
    def start(self, data):
        # Parse the PluginData
        self.use(data)

        # Verfify that we have metadata (otherwise this plugin makes little sense)
        if self.asset.describedby_link.metadata is None:
            self.fail("Asset has no metadata")

        # Set the progress to 0 (this will show an empty progress bar in Studio)
        self.update_progress(0)

        logging.info("Media source:      %s", self.source)
        logging.info("Media destination: %s", self.destination)
        logging.info("Setting up FXP...")

        # Set up an FXP transfer (what is FXP? see
        # https://en.wikipedia.org/wiki/File_eXchange_Protocol)
        fxp = FXP(self.source, self.destination, debug=True)
        logging.info("FXP copying...")
        fxp.run()
        fxp.quit()
        logging.log("FXP done", None, 'ok')

        # Update the progress to 99%, we're not there yet
        self.update_progress(99)

        # Construct a xml destiniation path
        xml_destination = self.destination + ".xml"
        logging.info("XML Destination:   %s", xml_destination)
        logging.info("Writing metadata...")

        # Write the Asset Metadata to the xml destination path
        ftp_write(xml_destination, asset.describedby_link.metadata.generate())
        logging.log("Metadata done", None, 'ok')

        # Set the progress to 100% and we're done
        self.update_progress(100)
Beispiel #8
0
        logger = HtmlLogger()
        logging.set_logger(logger)

    elif args.logger == 'file':
        from vizone.logging.file import FileLogger
        logger = FileLogger(logging_file
                            or os.path.join('/tmp', app_name + '.log'))
        logging.set_logger(logger)

    logging.get_default_logger().debug = args.debug or logging_debug

    logging.log(
        "Flow settings", {
            'app name': app_name,
            'class': Flow.__name__,
            'workers': workers,
            '(from ini path) working directory': working_dir,
            '(from main class SOURCE) source class': Flow.SOURCE.__name__,
            '(from $PYTHON_CONFIG_ROOT) base config dir': base_config_dir,
        }, 'pp')

    # Set up Viz One Client Instance
    viz_one_enabled = config.getboolean('Viz One', 'enabled')
    viz_one_hostname = os.path.expandvars(config.get('Viz One', 'hostname'))
    viz_one_username = os.path.expandvars(config.get('Viz One', 'username'))
    viz_one_password = os.path.expandvars(config.get('Viz One', 'password'))
    viz_one_use_https = config.getboolean('Viz One', 'use https')
    viz_one_check_certificates = config.getboolean('Viz One',
                                                   'check certificates')
    viz_one_pem_file = os.path.expanduser(
        os.path.expandvars(config.get('Viz One', 'pem file'))) or None
Beispiel #9
0
    elif args.logger == 'html':
        from vizone.logging.html import HtmlLogger
        logger = HtmlLogger()
        logging.set_logger(logger)

    elif args.logger == 'file':
        from vizone.logging.file import FileLogger
        logger = FileLogger(logging_file or os.path.join('/tmp', app_name + '.log'))
        logging.set_logger(logger)

    logging.get_default_logger().debug = args.debug or logging_debug

    logging.log("Flow settings", {
        'app name': app_name,
        'class': Flow.__name__,
        'workers': workers,
        '(from ini path) working directory': working_dir,
        '(from main class SOURCE) source class': Flow.SOURCE.__name__,
        '(from $PYTHON_CONFIG_ROOT) base config dir': base_config_dir,
    }, 'pp')

    # Set up Viz One Client Instance
    viz_one_enabled = config.getboolean('Viz One', 'enabled')
    viz_one_hostname = os.path.expandvars(config.get('Viz One', 'hostname'))
    viz_one_username = os.path.expandvars(config.get('Viz One', 'username'))
    viz_one_password = os.path.expandvars(config.get('Viz One', 'password'))
    viz_one_use_https = config.getboolean('Viz One', 'use https')
    viz_one_check_certificates = config.getboolean('Viz One', 'check certificates')
    viz_one_pem_file = os.path.expanduser(os.path.expandvars(config.get('Viz One', 'pem file'))) or None
    viz_one_time_out = config.getfloat('Viz One', 'time out')

    logging.log("Viz One Settings", {