def configure(self, config): logging.info("Configure.") self.mappings = {} if config.has_section("Mappings"): self.mappings = {k: v for k, v in config.items("Mappings")} logging.log("Mappings", self.mappings, 'pp')
def work(obj): current_log_id = log_id.next() pool.spawn(flow.start, obj, logger=logger, log_id=current_log_id) logging.info("Spawned (%s)." % (str(current_log_id)))
def delete_unmanaged_file(unmanaged_file, client=None): """ Delete an Unmanaged File from Viz One Args: unmanaged_file (vizone.payload.media.UnmanagedFile): The Unmanaged File to delete client (Optional[vizone.client.Instance]): A Viz One client to use (None means the default) """ logging.info(u'Deleting file %s.', unmanaged_file.title) (client or get_default_instance()).DELETE(unmanaged_file.delete_link)
def __exit__(self, type, value, traceback): logging.debug("Exit pool.") if self.join: for thread in self.threads.values(): thread.join(self.timeout) logging.debug("Exit joined %s. (%s)", thread.name, "timed out" if thread.is_alive() else "ok") total_time = time.time() - self.start_time logging.info("Ran %i tasks in %f seconds (avg %f seconds per task).", self.counter, total_time, self.avg_time)
def process_file_event(self, event): unmanaged_files = UnmanagedFileCollection(event) for unmanaged_file in FeedIterator(unmanaged_files, self.client): logging.info('Event for unmanaged file "%s"', unmanaged_file.title) if self.skip_empty_files and unmanaged_file.media.filesize == 0: logging.info('Skipping empty file %s.', unmanaged_file.title) return if callable(self.callback): self.callback(unmanaged_file)
def start(self, data): # Parse the PluginData self.use(data) # Verfify that we have metadata (otherwise this plugin makes little sense) if self.asset.describedby_link.metadata is None: self.fail("Asset has no metadata") # Set the progress to 0 (this will show an empty progress bar in Studio) self.update_progress(0) logging.info("Media source: %s", self.source) logging.info("Media destination: %s", self.destination) logging.info("Setting up FTP...") # Connect to the source FTP server ftp, source_path = connect(self.source, debug=True) source_filename = os.path.basename(source_path) source_directory = os.path.dirname(source_path) # Here you can also run other SITE commands before starting the transfer # Set up ArdFTP for copying the media file (you might have to use other # settings) ftp.voidcmd('SITE ARDENDO FORMAT MXF') ftp.voidcmd('SITE ARDENDO USE MXF %s' % source_filename) ftp.voidcmd('SITE ARDENDO STOR %s' % self.destination) # This "downloads" a status file from the source FTP server, containing # information about percentage done of the transfer. This is the main # advantage compared to using FXP ftp.retrlines('RETR status.log', callback=self.read_status) # When that transfer is finished, so is the media transfer and we may # close the connection to the source FTP server. ftp.quit() logging.log(u'FTP done', None, 'ok') # Now let's adjust the progress, we're not quite done yet self.update_progress(99) # Construct a xml destiniation path xml_destination = self.destination + ".xml" logging.info("XML Destination: %s", xml_destination) logging.info("Writing metadata...") # Write the Asset Metadata to the xml destination path ftp_write(xml_destination, self.asset.describedby_link.metadata.generate()) logging.log("Metadata writing done", None, 'ok') # Set the progress to 100% and we're done self.update_progress(100)
def start(self, data): # Parse the PluginData self.use(data) # Verfify that we have metadata (otherwise this plugin makes little sense) if self.asset.describedby_link.metadata is None: self.fail("Asset has no metadata") # Set the progress to 0 (this will show an empty progress bar in Studio) self.update_progress(0) logging.info("Media source: %s", self.source) logging.info("Media destination: %s", self.destination) logging.info("Setting up FTP...") # Connect to the source FTP server ftp, source_path = connect(self.source, debug=True) source_filename = os.path.basename(source_path) source_directory = os.path.dirname(source_path) # Here you can also run other SITE commands before starting the transfer # Set up ArdFTP for copying the media file (you might have to use other # settings) ftp.voidcmd('SITE ARDENDO FORMAT MXF') ftp.voidcmd('SITE ARDENDO USE MXF %s' % source_filename) ftp.voidcmd('SITE ARDENDO STOR %s' % self.destination) # This "downloads" a status file from the source FTP server, containing # information about percentage done of the transfer. This is the main # advantage compared to using FXP ftp.retrlines('RETR status.log', callback=self.read_status) # When that transfer is finished, so is the media transfer and we may # close the connection to the source FTP server. ftp.quit() logging.log(u'FTP done', None, 'ok') # Now let's adjust the progress, we're not quite done yet self.update_progress(99) # Construct a xml destiniation path xml_destination = self.destination + ".xml" logging.info("XML Destination: %s", xml_destination) logging.info("Writing metadata...") # Write the Asset Metadata to the xml destination path ftp_write(xml_destination, asset.describedby_link.metadata.generate()) logging.log("Metadata writing done", None, 'ok') # Set the progress to 100% and we're done self.update_progress(100)
def __init__(self, workers=1, join=True, timeout=60): logging.info("Create pool with %i workers (%s, timeout=%i).", workers, 'join' if join else 'no join', timeout) self.start_time = time.time() self.worker_count = workers self.join = join self.timeout = timeout self.resource = threading.BoundedSemaphore(workers) self.threads_lock = threading.Lock() self.threads = {} self.timing_lock = threading.Lock() self.counter = 0 self.avg_time = 0
def import_unmanaged_file(asset, uri_list, client=None): """ Start an Unmanaged File import to a given Asset Entry Args: asset (vizone.payload.asset.Item): The Asset Entry to import to uri_list (vizone.urilist.UriList): A URI List containing the link to the media client (Optional[vizone.client.Instance]): A Viz One client to use (None means the default) Returns: bool: True if successful, False on error """ logging.info(u'Importing file %s to %s.', uri_list.generate().strip(), asset.id) try: client.POST(asset.import_unmanaged_link, uri_list) return True except HTTPClientError: logging.error('Unable to import unmanaged file "%s" to asset %s', uri_list.generate().strip(), asset.id) return False
def __init__(self, interval=60, window_start=None, window_end=None): self.interval = int(interval) self.window_start = datetime.time(*[int(d) for d in window_start.split(':')]) \ if window_start is not None else None self.window_end = datetime.time(*[int(d) for d in window_end.split(':')]) \ if window_end is not None else None self.callback = None logging.info('Interval: %s seconds', self.interval) if self.window_start is not None: logging.info('Time window start: %s', self.window_start.isoformat()) if self.window_end is not None: logging.info('Time window end: %s', self.window_end.isoformat())
def start(self, asset, conflict=False): # Fetch metadata with a fresh etag try: old_metadata = Payload(self.client.GET(asset.describedby_link)) # If no metadata is found, we cannot do any metadata mapping except HTTPClientError as e: logging.error(u'Asset Metadata fetching failed %s', str(e)) return # If there was an internal server error (5xx), wait and retry except HTTPServerError as e: logging.error(u'Asset Metadata fetching failed %s', str(e)) logging.warn(u'Retrying in 10 seconds...') time.sleep(10) raise Retry # Copy metadata and operate on the copy, so we can compare them later new_metadata = Payload(old_metadata.generate()) scope = { 'asset': asset, 'metadata': new_metadata, } for field, expr in self.mappings.items(): logging.info('%s = %s', field, expr) new_metadata.set(field, eval(expr, scope)) logging.log(u'Resulting payload', new_metadata.generate(), 'xml') # Only update if we produced any differences, else we will have a circle of updates if new_metadata != old_metadata: logging.info(u'Updating metadata...') self.client.PUT( asset.describedby_link, new_metadata, etag=old_metadata.response_etag ) logging.info(u'Updated metadata.') else: logging.info(u'No changes to metadata.')
def start(self, asset, conflict=False): # Fetch metadata with a fresh etag try: old_metadata = Payload(self.client.GET(asset.describedby_link)) # If no metadata is found, we cannot do any metadata mapping except HTTPClientError as e: logging.error(u'Asset Metadata fetching failed %s', str(e)) return # If there was an internal server error (5xx), wait and retry except HTTPServerError as e: logging.error(u'Asset Metadata fetching failed %s', str(e)) logging.warn(u'Retrying in 10 seconds...') time.sleep(10) raise Retry # Copy metadata and operate on the copy, so we can compare them later new_metadata = Payload(old_metadata.generate()) scope = { 'asset': asset, 'metadata': new_metadata, } for field, expr in self.mappings.items(): logging.info('%s = %s', field, expr) new_metadata.set(field, eval(expr, scope)) logging.log(u'Resulting payload', new_metadata.generate(), 'xml') # Only update if we produced any differences, else we will have a circle of updates if new_metadata != old_metadata: logging.info(u'Updating metadata...') self.client.PUT(asset.describedby_link, new_metadata, etag=old_metadata.response_etag) logging.info(u'Updated metadata.') else: logging.info(u'No changes to metadata.')
obj = source.run() flow.start(obj) # Run source.start multiple times as long as there are free workers in the # pool. Stop when source.next() raises StopIteration. elif issubclass(Flow.SOURCE, Iterable): with Pool(workers=workers, join=True, timeout=worker_timeout) as pool: log_id = LogId() for obj in source: current_log_id = log_id.next() pool.spawn(flow.start, obj, logger=logger, log_id=current_log_id) logging.info("Spawned worker (%s).", str(current_log_id)) logging.info("Source is out of data.") # Run source.start once and go to an idle loop. Source is typically an # event listener of some kind and will call the callback upon # external triggers. elif issubclass(Flow.SOURCE, EventBased): with Pool(workers=workers, join=True, timeout=worker_timeout) as pool: log_id = LogId() def work(obj): current_log_id = log_id.next() pool.spawn(flow.start, obj, logger=logger,
def create_or_update_asset( id=None, metadata=None, acl=None, mediaacl=None, tags=None, materialtype=None, category=None, rightscode=None, client=None): """ Creates or Updates an Item with a given ``id``. Metadata updates will be retried three times if there are conflicts. Args: id (Optional[unicode]): An asset id or "site identity" metadata (Optional[vizone.vdf.Payload]): The metadata to update to. Can be a dict with a 'form' field too. acl (Optional[vizone.vizone.payload.user_group.Acl]): Specify a ACL when creating the Asset mediaacl (Optional[vizone.vizone.payload.user_group.Acl]): Specify a Media ACL when creating the Asset tags (Optional[dict]): scheme => term dictionary for custom tags when creating the Asset materialtype (Optional[unicode]): Set the Material Type to this when creating the Asset category (Optional[unicode]): Set the Category to this when creating the Asset rightscode (Optional[unicode]): Set the Rights Code to this when creating the Asset client (Optional[vizone.client.Instance]): A Viz One client to use (None means the default) Returns: vizone.payload.asset.Item: The updated or created Asset Entry """ old_payload = None client = client or get_default_instance() # Create or Update Placeholder try: asset = get_asset_by_id(id, headers={'X-Inline': 'describedby'}, client=client) old_payload = asset.describedby_link.metadata except HTTPClientError: try: logging.info(u'Item %s does not exist, creating.', id) asset = Item(id=id) if acl: asset.acl = acl if mediaacl: asset.mediaacl = mediaacl if materialtype: asset.materialtype = materialtype if category: asset.category = category if rightscode: asset.rightscode = rightscode if tags: for scheme, term in sorted(tags.items()): asset.keywords.append( AtomCategory(scheme=scheme, term=term) ) asset = create_asset(asset, client=client) except (HTTPClientError, HTTPServerError): logging.error(u'Could not create asset %s, skipping.', id) return # Create payload if metadata is a dict if type(metadata) is dict: form = metadata.pop('form') if old_payload is None: models = MetadataFormCollection(client.GET(asset.models_link)) model_link = [model.self_link for model in models.entries if model.name == form][0] model = Model(client.GET(model_link)) payload = model.to_payload() else: payload = old_payload for name, value in metadata.items(): payload.set(name, value) else: payload = metadata # Update Metadata if needed if payload is not None and payload != old_payload: logging.info(u'Updating metadata for asset %s.', asset.id) for _ in range(3): try: asset.describedby_link.metadata = Payload(client.PUT(asset.describedby_link, payload)) break except HTTPServerError: logging.error(u'Could not update metadata for asset %s.', asset.id) return except HTTPClientError as e: logging.info(u'Asset Metadata update failed %s', str(e)) if e.responstatus_code == 412: logging.warn(u'Retrying...') asset.parse(client.GET(asset.self_link)) else: break else: logging.info(u'Updating metadata for asset %s not needed.', asset.id) return asset
def start(self, data): # Parse the PluginData self.use(data) # Verfify that we have metadata (otherwise this plugin makes little sense) if self.asset.describedby_link.metadata is None: self.fail("Asset has no metadata") # Set the progress to 0 (this will show an empty progress bar in Studio) self.update_progress(0) logging.info("Media source: %s", self.source) logging.info("Media destination: %s", self.destination) logging.info("Setting up FXP...") # Set up an FXP transfer (what is FXP? see # https://en.wikipedia.org/wiki/File_eXchange_Protocol) fxp = FXP(self.source, self.destination, debug=True) logging.info("FXP copying...") fxp.run() fxp.quit() logging.log("FXP done", None, 'ok') # Update the progress to 99%, we're not there yet self.update_progress(99) # Construct a xml destiniation path xml_destination = self.destination + ".xml" logging.info("XML Destination: %s", xml_destination) logging.info("Writing metadata...") # Write the Asset Metadata to the xml destination path ftp_write(xml_destination, asset.describedby_link.metadata.generate()) logging.log("Metadata done", None, 'ok') # Set the progress to 100% and we're done self.update_progress(100)
def start(self, data): # Parse the PluginData self.use(data) # Verfify that we have metadata (otherwise this plugin makes little sense) if self.asset.describedby_link.metadata is None: self.fail("Asset has no metadata") # Set the progress to 0 (this will show an empty progress bar in Studio) self.update_progress(0) logging.info("Media source: %s", self.source) logging.info("Media destination: %s", self.destination) logging.info("Setting up FXP...") # Set up an FXP transfer (what is FXP? see # https://en.wikipedia.org/wiki/File_eXchange_Protocol) fxp = FXP(self.source, self.destination, debug=True) logging.info("FXP copying...") fxp.run() fxp.quit() logging.log("FXP done", None, 'ok') # Update the progress to 99%, we're not there yet self.update_progress(99) # Construct a xml destiniation path xml_destination = self.destination + ".xml" logging.info("XML Destination: %s", xml_destination) logging.info("Writing metadata...") # Write the Asset Metadata to the xml destination path ftp_write(xml_destination, self.asset.describedby_link.metadata.generate()) logging.log("Metadata done", None, 'ok') # Set the progress to 100% and we're done self.update_progress(100)
def start(self, message): logging.info("Got callback.")
def process_asset_event(self, event): asset = Item(event) logging.info('Event for asset %s "%s"', asset.id, asset.title) if callable(self.callback): self.callback(asset)
# Run source.run once, which should call the workers' start method once or # more. No parallelisation is done here if issubclass(Flow.SOURCE, Once): obj = source.run() flow.start(obj) # Run source.start multiple times as long as there are free workers in the # pool. Stop when source.next() raises StopIteration. elif issubclass(Flow.SOURCE, Iterable): with Pool(workers=workers, join=True) as pool: log_id = LogId() for obj in source: current_log_id = log_id.next() pool.spawn(flow.start, obj, logger=logger, log_id=current_log_id) logging.info("Spawned worker (%s).", str(current_log_id)) logging.info("Source is out of data.") # Run source.start once and go to an idle loop. Source is typically an # event listener of some kind and will call the callback upon # external triggers. elif issubclass(Flow.SOURCE, EventBased): with Pool(workers=workers, join=True) as pool: log_id = LogId() def work(obj): current_log_id = log_id.next() pool.spawn(flow.start, obj, logger=logger, log_id=current_log_id) logging.info("Spawned (%s)." % (str(current_log_id)))
def start(self, f): # Queue up multiple events for the same file with Locked(f.title): logging.info('Processing XML file %s.', f.title) if is_xml(f): # XML file if f.media.filesize == 0: logging.info('Skipping empty XML file %s.', f.title) return # Extract and translate the metadata from the XML media_filename = None if self.xml_format == 'default': xml = ImportExport(self.client.GET(f.media.url)) metadata= (xml.describedby_link.metadata if xml.describedby_link is not None else None) if xml.content and xml.content.src: media_filename = xml.content.src asset_id = xml.id # Custom Xml mode requires some further settings in the INI elif self.xml_format == 'custom': r = self.client.GET(f.media.url) asset_id, media_filename, metadata = self.custom_parse(f.title, r.content) # Create or Update a placeholder, including Metadata update from XML asset = create_or_update_asset( id=asset_id, metadata=metadata, # may be a vizone.vdf.Payload or a dict client=self.client, ) # Remove the XML file, we're done with it delete_unmanaged_file( unmanaged_file=f, client=self.client, ) # Jump out here if this is not a placeholder if asset.assetmediatype != 'placeholder': logging.info('(%i) Asset is not a placeholder, skip import.') return # Check if a media file waas mentioned in atom:content/@src if media_filename: logging.info('Wants media file %s.', media_filename) # Check if we have a MIN for it already stored_info = self.store.get(media_filename) if stored_info is not None and stored_info.get('type') == 'media': # Start the import import_unmanaged_file( asset, UriList([stored_info.get('link')]), client=self.client, ) self.store.delete(media_filename) else: logging.info('Remember media file %s -> asset %s.', media_filename, asset.id) self.store.put(media_filename, {'type': 'asset', 'link': asset.self_link.href}) else: # Media file # Check if there is an XML that mentioned this media file stored_info = self.store.get(f.title) if stored_info is not None and stored_info.get('type') == 'asset': # Fetch the asset and import to it asset = Item(self.client.GET(stored_info.get('link'))) import_unmanaged_file( asset, UriList([f.self_link.href]), client=self.client, ) self.store.delete(f.title) elif stored_info is None: logging.info('Remember media file %s -> unmanaged file %s.', f.title, f.self_link.href) self.store.put(f.title, {'type': 'media', 'link': f.self_link.href})
def start(self, message): logging.info('Got callback.')