def configure(self, config):
logging.info("Configure.")
self.mappings = {}
if config.has_section("Mappings"):
self.mappings = {k: v for k, v in config.items("Mappings")}
logging.log("Mappings", self.mappings, 'pp')
def configure(self, config):
logging.info("Configure.")
self.mappings = {}
if config.has_section("Mappings"):
self.mappings = {k: v for k, v in config.items("Mappings")}
logging.log("Mappings", self.mappings, 'pp')
def work(obj):
current_log_id = log_id.next()
pool.spawn(flow.start,
obj,
logger=logger,
log_id=current_log_id)
logging.info("Spawned (%s)." % (str(current_log_id)))
def delete_unmanaged_file(unmanaged_file, client=None):
"""
Delete an Unmanaged File from Viz One
Args:
unmanaged_file (vizone.payload.media.UnmanagedFile): The Unmanaged File to delete
client (Optional[vizone.client.Instance]): A Viz One client to use (None means the default)
"""
logging.info(u'Deleting file %s.', unmanaged_file.title)
(client or get_default_instance()).DELETE(unmanaged_file.delete_link)
def __exit__(self, type, value, traceback):
logging.debug("Exit pool.")
if self.join:
for thread in self.threads.values():
thread.join(self.timeout)
logging.debug("Exit joined %s. (%s)", thread.name,
"timed out" if thread.is_alive() else "ok")
total_time = time.time() - self.start_time
logging.info("Ran %i tasks in %f seconds (avg %f seconds per task).",
self.counter, total_time, self.avg_time)
def __exit__(self, type, value, traceback):
logging.debug("Exit pool.")
if self.join:
for thread in self.threads.values():
thread.join(self.timeout)
logging.debug("Exit joined %s. (%s)", thread.name,
"timed out" if thread.is_alive() else "ok")
total_time = time.time() - self.start_time
logging.info("Ran %i tasks in %f seconds (avg %f seconds per task).",
self.counter, total_time, self.avg_time)
def process_file_event(self, event):
unmanaged_files = UnmanagedFileCollection(event)
for unmanaged_file in FeedIterator(unmanaged_files, self.client):
logging.info('Event for unmanaged file "%s"', unmanaged_file.title)
if self.skip_empty_files and unmanaged_file.media.filesize == 0:
logging.info('Skipping empty file %s.', unmanaged_file.title)
return
if callable(self.callback):
self.callback(unmanaged_file)
def process_file_event(self, event):
unmanaged_files = UnmanagedFileCollection(event)
for unmanaged_file in FeedIterator(unmanaged_files, self.client):
logging.info('Event for unmanaged file "%s"', unmanaged_file.title)
if self.skip_empty_files and unmanaged_file.media.filesize == 0:
logging.info('Skipping empty file %s.', unmanaged_file.title)
return
if callable(self.callback):
self.callback(unmanaged_file)
def start(self, data):
# Parse the PluginData
self.use(data)
# Verfify that we have metadata (otherwise this plugin makes little sense)
if self.asset.describedby_link.metadata is None:
self.fail("Asset has no metadata")
# Set the progress to 0 (this will show an empty progress bar in Studio)
self.update_progress(0)
logging.info("Media source: %s", self.source)
logging.info("Media destination: %s", self.destination)
logging.info("Setting up FTP...")
# Connect to the source FTP server
ftp, source_path = connect(self.source, debug=True)
source_filename = os.path.basename(source_path)
source_directory = os.path.dirname(source_path)
# Here you can also run other SITE commands before starting the transfer
# Set up ArdFTP for copying the media file (you might have to use other
# settings)
ftp.voidcmd('SITE ARDENDO FORMAT MXF')
ftp.voidcmd('SITE ARDENDO USE MXF %s' % source_filename)
ftp.voidcmd('SITE ARDENDO STOR %s' % self.destination)
# This "downloads" a status file from the source FTP server, containing
# information about percentage done of the transfer. This is the main
# advantage compared to using FXP
ftp.retrlines('RETR status.log', callback=self.read_status)
# When that transfer is finished, so is the media transfer and we may
# close the connection to the source FTP server.
ftp.quit()
logging.log(u'FTP done', None, 'ok')
# Now let's adjust the progress, we're not quite done yet
self.update_progress(99)
# Construct a xml destiniation path
xml_destination = self.destination + ".xml"
logging.info("XML Destination: %s", xml_destination)
logging.info("Writing metadata...")
# Write the Asset Metadata to the xml destination path
ftp_write(xml_destination, self.asset.describedby_link.metadata.generate())
logging.log("Metadata writing done", None, 'ok')
# Set the progress to 100% and we're done
self.update_progress(100)
def start(self, data):
# Parse the PluginData
self.use(data)
# Verfify that we have metadata (otherwise this plugin makes little sense)
if self.asset.describedby_link.metadata is None:
self.fail("Asset has no metadata")
# Set the progress to 0 (this will show an empty progress bar in Studio)
self.update_progress(0)
logging.info("Media source: %s", self.source)
logging.info("Media destination: %s", self.destination)
logging.info("Setting up FTP...")
# Connect to the source FTP server
ftp, source_path = connect(self.source, debug=True)
source_filename = os.path.basename(source_path)
source_directory = os.path.dirname(source_path)
# Here you can also run other SITE commands before starting the transfer
# Set up ArdFTP for copying the media file (you might have to use other
# settings)
ftp.voidcmd('SITE ARDENDO FORMAT MXF')
ftp.voidcmd('SITE ARDENDO USE MXF %s' % source_filename)
ftp.voidcmd('SITE ARDENDO STOR %s' % self.destination)
# This "downloads" a status file from the source FTP server, containing
# information about percentage done of the transfer. This is the main
# advantage compared to using FXP
ftp.retrlines('RETR status.log', callback=self.read_status)
# When that transfer is finished, so is the media transfer and we may
# close the connection to the source FTP server.
ftp.quit()
logging.log(u'FTP done', None, 'ok')
# Now let's adjust the progress, we're not quite done yet
self.update_progress(99)
# Construct a xml destiniation path
xml_destination = self.destination + ".xml"
logging.info("XML Destination: %s", xml_destination)
logging.info("Writing metadata...")
# Write the Asset Metadata to the xml destination path
ftp_write(xml_destination, asset.describedby_link.metadata.generate())
logging.log("Metadata writing done", None, 'ok')
# Set the progress to 100% and we're done
self.update_progress(100)
def __init__(self, workers=1, join=True, timeout=60):
logging.info("Create pool with %i workers (%s, timeout=%i).",
workers, 'join' if join else 'no join', timeout)
self.start_time = time.time()
self.worker_count = workers
self.join = join
self.timeout = timeout
self.resource = threading.BoundedSemaphore(workers)
self.threads_lock = threading.Lock()
self.threads = {}
self.timing_lock = threading.Lock()
self.counter = 0
self.avg_time = 0
def __init__(self, workers=1, join=True, timeout=60):
logging.info("Create pool with %i workers (%s, timeout=%i).", workers,
'join' if join else 'no join', timeout)
self.start_time = time.time()
self.worker_count = workers
self.join = join
self.timeout = timeout
self.resource = threading.BoundedSemaphore(workers)
self.threads_lock = threading.Lock()
self.threads = {}
self.timing_lock = threading.Lock()
self.counter = 0
self.avg_time = 0
def import_unmanaged_file(asset, uri_list, client=None):
"""
Start an Unmanaged File import to a given Asset Entry
Args:
asset (vizone.payload.asset.Item): The Asset Entry to import to
uri_list (vizone.urilist.UriList): A URI List containing the link to the media
client (Optional[vizone.client.Instance]): A Viz One client to use (None means the default)
Returns:
bool: True if successful, False on error
"""
logging.info(u'Importing file %s to %s.',
uri_list.generate().strip(), asset.id)
try:
client.POST(asset.import_unmanaged_link, uri_list)
return True
except HTTPClientError:
logging.error('Unable to import unmanaged file "%s" to asset %s',
uri_list.generate().strip(), asset.id)
return False
def __init__(self, interval=60, window_start=None, window_end=None):
self.interval = int(interval)
self.window_start = datetime.time(*[int(d) for d in window_start.split(':')]) \
if window_start is not None else None
self.window_end = datetime.time(*[int(d) for d in window_end.split(':')]) \
if window_end is not None else None
self.callback = None
logging.info('Interval: %s seconds', self.interval)
if self.window_start is not None:
logging.info('Time window start: %s', self.window_start.isoformat())
if self.window_end is not None:
logging.info('Time window end: %s', self.window_end.isoformat())
def start(self, asset, conflict=False):
# Fetch metadata with a fresh etag
try:
old_metadata = Payload(self.client.GET(asset.describedby_link))
# If no metadata is found, we cannot do any metadata mapping
except HTTPClientError as e:
logging.error(u'Asset Metadata fetching failed %s', str(e))
return
# If there was an internal server error (5xx), wait and retry
except HTTPServerError as e:
logging.error(u'Asset Metadata fetching failed %s', str(e))
logging.warn(u'Retrying in 10 seconds...')
time.sleep(10)
raise Retry
# Copy metadata and operate on the copy, so we can compare them later
new_metadata = Payload(old_metadata.generate())
scope = {
'asset': asset,
'metadata': new_metadata,
}
for field, expr in self.mappings.items():
logging.info('%s = %s', field, expr)
new_metadata.set(field, eval(expr, scope))
logging.log(u'Resulting payload', new_metadata.generate(), 'xml')
# Only update if we produced any differences, else we will have a circle of updates
if new_metadata != old_metadata:
logging.info(u'Updating metadata...')
self.client.PUT(
asset.describedby_link,
new_metadata,
etag=old_metadata.response_etag
)
logging.info(u'Updated metadata.')
else:
logging.info(u'No changes to metadata.')
def __init__(self, interval=60, window_start=None, window_end=None):
self.interval = int(interval)
self.window_start = datetime.time(*[int(d) for d in window_start.split(':')]) \
if window_start is not None else None
self.window_end = datetime.time(*[int(d) for d in window_end.split(':')]) \
if window_end is not None else None
self.callback = None
logging.info('Interval: %s seconds', self.interval)
if self.window_start is not None:
logging.info('Time window start: %s',
self.window_start.isoformat())
if self.window_end is not None:
logging.info('Time window end: %s', self.window_end.isoformat())
def start(self, asset, conflict=False):
# Fetch metadata with a fresh etag
try:
old_metadata = Payload(self.client.GET(asset.describedby_link))
# If no metadata is found, we cannot do any metadata mapping
except HTTPClientError as e:
logging.error(u'Asset Metadata fetching failed %s', str(e))
return
# If there was an internal server error (5xx), wait and retry
except HTTPServerError as e:
logging.error(u'Asset Metadata fetching failed %s', str(e))
logging.warn(u'Retrying in 10 seconds...')
time.sleep(10)
raise Retry
# Copy metadata and operate on the copy, so we can compare them later
new_metadata = Payload(old_metadata.generate())
scope = {
'asset': asset,
'metadata': new_metadata,
}
for field, expr in self.mappings.items():
logging.info('%s = %s', field, expr)
new_metadata.set(field, eval(expr, scope))
logging.log(u'Resulting payload', new_metadata.generate(), 'xml')
# Only update if we produced any differences, else we will have a circle of updates
if new_metadata != old_metadata:
logging.info(u'Updating metadata...')
self.client.PUT(asset.describedby_link,
new_metadata,
etag=old_metadata.response_etag)
logging.info(u'Updated metadata.')
else:
logging.info(u'No changes to metadata.')
obj = source.run()
flow.start(obj)
# Run source.start multiple times as long as there are free workers in the
# pool. Stop when source.next() raises StopIteration.
elif issubclass(Flow.SOURCE, Iterable):
with Pool(workers=workers, join=True, timeout=worker_timeout) as pool:
log_id = LogId()
for obj in source:
current_log_id = log_id.next()
pool.spawn(flow.start,
obj,
logger=logger,
log_id=current_log_id)
logging.info("Spawned worker (%s).", str(current_log_id))
logging.info("Source is out of data.")
# Run source.start once and go to an idle loop. Source is typically an
# event listener of some kind and will call the callback upon
# external triggers.
elif issubclass(Flow.SOURCE, EventBased):
with Pool(workers=workers, join=True, timeout=worker_timeout) as pool:
log_id = LogId()
def work(obj):
current_log_id = log_id.next()
pool.spawn(flow.start,
obj,
logger=logger,
def create_or_update_asset(
id=None,
metadata=None,
acl=None,
mediaacl=None,
tags=None,
materialtype=None,
category=None,
rightscode=None,
client=None):
"""
Creates or Updates an Item with a given ``id``. Metadata updates will
be retried three times if there are conflicts.
Args:
id (Optional[unicode]): An asset id or "site identity"
metadata (Optional[vizone.vdf.Payload]): The metadata to update to. Can be a dict with a 'form' field too.
acl (Optional[vizone.vizone.payload.user_group.Acl]): Specify a ACL when creating the Asset
mediaacl (Optional[vizone.vizone.payload.user_group.Acl]): Specify a Media ACL when creating the Asset
tags (Optional[dict]): scheme => term dictionary for custom tags when creating the Asset
materialtype (Optional[unicode]): Set the Material Type to this when creating the Asset
category (Optional[unicode]): Set the Category to this when creating the Asset
rightscode (Optional[unicode]): Set the Rights Code to this when creating the Asset
client (Optional[vizone.client.Instance]): A Viz One client to use (None means the default)
Returns:
vizone.payload.asset.Item: The updated or created Asset Entry
"""
old_payload = None
client = client or get_default_instance()
# Create or Update Placeholder
try:
asset = get_asset_by_id(id, headers={'X-Inline': 'describedby'}, client=client)
old_payload = asset.describedby_link.metadata
except HTTPClientError:
try:
logging.info(u'Item %s does not exist, creating.', id)
asset = Item(id=id)
if acl:
asset.acl = acl
if mediaacl:
asset.mediaacl = mediaacl
if materialtype:
asset.materialtype = materialtype
if category:
asset.category = category
if rightscode:
asset.rightscode = rightscode
if tags:
for scheme, term in sorted(tags.items()):
asset.keywords.append(
AtomCategory(scheme=scheme, term=term)
)
asset = create_asset(asset, client=client)
except (HTTPClientError, HTTPServerError):
logging.error(u'Could not create asset %s, skipping.', id)
return
# Create payload if metadata is a dict
if type(metadata) is dict:
form = metadata.pop('form')
if old_payload is None:
models = MetadataFormCollection(client.GET(asset.models_link))
model_link = [model.self_link for model in models.entries if model.name == form][0]
model = Model(client.GET(model_link))
payload = model.to_payload()
else:
payload = old_payload
for name, value in metadata.items():
payload.set(name, value)
else:
payload = metadata
# Update Metadata if needed
if payload is not None and payload != old_payload:
logging.info(u'Updating metadata for asset %s.', asset.id)
for _ in range(3):
try:
asset.describedby_link.metadata = Payload(client.PUT(asset.describedby_link, payload))
break
except HTTPServerError:
logging.error(u'Could not update metadata for asset %s.', asset.id)
return
except HTTPClientError as e:
logging.info(u'Asset Metadata update failed %s', str(e))
if e.responstatus_code == 412:
logging.warn(u'Retrying...')
asset.parse(client.GET(asset.self_link))
else:
break
else:
logging.info(u'Updating metadata for asset %s not needed.', asset.id)
return asset
def start(self, data):
# Parse the PluginData
self.use(data)
# Verfify that we have metadata (otherwise this plugin makes little sense)
if self.asset.describedby_link.metadata is None:
self.fail("Asset has no metadata")
# Set the progress to 0 (this will show an empty progress bar in Studio)
self.update_progress(0)
logging.info("Media source: %s", self.source)
logging.info("Media destination: %s", self.destination)
logging.info("Setting up FXP...")
# Set up an FXP transfer (what is FXP? see
# https://en.wikipedia.org/wiki/File_eXchange_Protocol)
fxp = FXP(self.source, self.destination, debug=True)
logging.info("FXP copying...")
fxp.run()
fxp.quit()
logging.log("FXP done", None, 'ok')
# Update the progress to 99%, we're not there yet
self.update_progress(99)
# Construct a xml destiniation path
xml_destination = self.destination + ".xml"
logging.info("XML Destination: %s", xml_destination)
logging.info("Writing metadata...")
# Write the Asset Metadata to the xml destination path
ftp_write(xml_destination, asset.describedby_link.metadata.generate())
logging.log("Metadata done", None, 'ok')
# Set the progress to 100% and we're done
self.update_progress(100)
def start(self, data):
# Parse the PluginData
self.use(data)
# Verfify that we have metadata (otherwise this plugin makes little sense)
if self.asset.describedby_link.metadata is None:
self.fail("Asset has no metadata")
# Set the progress to 0 (this will show an empty progress bar in Studio)
self.update_progress(0)
logging.info("Media source: %s", self.source)
logging.info("Media destination: %s", self.destination)
logging.info("Setting up FXP...")
# Set up an FXP transfer (what is FXP? see
# https://en.wikipedia.org/wiki/File_eXchange_Protocol)
fxp = FXP(self.source, self.destination, debug=True)
logging.info("FXP copying...")
fxp.run()
fxp.quit()
logging.log("FXP done", None, 'ok')
# Update the progress to 99%, we're not there yet
self.update_progress(99)
# Construct a xml destiniation path
xml_destination = self.destination + ".xml"
logging.info("XML Destination: %s", xml_destination)
logging.info("Writing metadata...")
# Write the Asset Metadata to the xml destination path
ftp_write(xml_destination, self.asset.describedby_link.metadata.generate())
logging.log("Metadata done", None, 'ok')
# Set the progress to 100% and we're done
self.update_progress(100)
def start(self, message):
logging.info("Got callback.")
def process_asset_event(self, event):
asset = Item(event)
logging.info('Event for asset %s "%s"', asset.id, asset.title)
if callable(self.callback):
self.callback(asset)
# Run source.run once, which should call the workers' start method once or
# more. No parallelisation is done here
if issubclass(Flow.SOURCE, Once):
obj = source.run()
flow.start(obj)
# Run source.start multiple times as long as there are free workers in the
# pool. Stop when source.next() raises StopIteration.
elif issubclass(Flow.SOURCE, Iterable):
with Pool(workers=workers, join=True) as pool:
log_id = LogId()
for obj in source:
current_log_id = log_id.next()
pool.spawn(flow.start, obj, logger=logger, log_id=current_log_id)
logging.info("Spawned worker (%s).", str(current_log_id))
logging.info("Source is out of data.")
# Run source.start once and go to an idle loop. Source is typically an
# event listener of some kind and will call the callback upon
# external triggers.
elif issubclass(Flow.SOURCE, EventBased):
with Pool(workers=workers, join=True) as pool:
log_id = LogId()
def work(obj):
current_log_id = log_id.next()
pool.spawn(flow.start, obj, logger=logger, log_id=current_log_id)
logging.info("Spawned (%s)." % (str(current_log_id)))
def work(obj):
current_log_id = log_id.next()
pool.spawn(flow.start, obj, logger=logger, log_id=current_log_id)
logging.info("Spawned (%s)." % (str(current_log_id)))
def start(self, f):
# Queue up multiple events for the same file
with Locked(f.title):
logging.info('Processing XML file %s.', f.title)
if is_xml(f): # XML file
if f.media.filesize == 0:
logging.info('Skipping empty XML file %s.', f.title)
return
# Extract and translate the metadata from the XML
media_filename = None
if self.xml_format == 'default':
xml = ImportExport(self.client.GET(f.media.url))
metadata= (xml.describedby_link.metadata
if xml.describedby_link is not None else None)
if xml.content and xml.content.src:
media_filename = xml.content.src
asset_id = xml.id
# Custom Xml mode requires some further settings in the INI
elif self.xml_format == 'custom':
r = self.client.GET(f.media.url)
asset_id, media_filename, metadata = self.custom_parse(f.title, r.content)
# Create or Update a placeholder, including Metadata update from XML
asset = create_or_update_asset(
id=asset_id,
metadata=metadata, # may be a vizone.vdf.Payload or a dict
client=self.client,
)
# Remove the XML file, we're done with it
delete_unmanaged_file(
unmanaged_file=f,
client=self.client,
)
# Jump out here if this is not a placeholder
if asset.assetmediatype != 'placeholder':
logging.info('(%i) Asset is not a placeholder, skip import.')
return
# Check if a media file waas mentioned in atom:content/@src
if media_filename:
logging.info('Wants media file %s.', media_filename)
# Check if we have a MIN for it already
stored_info = self.store.get(media_filename)
if stored_info is not None and stored_info.get('type') == 'media':
# Start the import
import_unmanaged_file(
asset,
UriList([stored_info.get('link')]),
client=self.client,
)
self.store.delete(media_filename)
else:
logging.info('Remember media file %s -> asset %s.',
media_filename, asset.id)
self.store.put(media_filename,
{'type': 'asset', 'link': asset.self_link.href})
else: # Media file
# Check if there is an XML that mentioned this media file
stored_info = self.store.get(f.title)
if stored_info is not None and stored_info.get('type') == 'asset':
# Fetch the asset and import to it
asset = Item(self.client.GET(stored_info.get('link')))
import_unmanaged_file(
asset,
UriList([f.self_link.href]),
client=self.client,
)
self.store.delete(f.title)
elif stored_info is None:
logging.info('Remember media file %s -> unmanaged file %s.',
f.title, f.self_link.href)
self.store.put(f.title, {'type': 'media', 'link': f.self_link.href})
def start(self, message):
logging.info('Got callback.')
