def read_status(self, status_line): try: logging.log('Status line', status_line, 'info') if '%' in status_line: with_percent, rest = status_line.split('%', 1) status, value = with_percent.split(' ') self.update_progress(value) except Exception as e: logging.error("Error parsing status line from arftp: " + str(e))
def convert(self, raw_value, client): """ Perform conversion configured when contructing the object. Args: raw_value (unicode): The raw string to parse client (vizone.client.Instance): The HTTP client to use when looking up dictionary terms. Returns: object """ if raw_value is None or raw_value == "": return None if self.type == "string": return raw_value elif self.type == "integer": return int(raw_value) elif self.type == "float": return float(raw_value) elif self.type == "iso": return Timestamp(raw_value) elif self.type == "date": d = datetime.strptime(raw_value, self.format).date() t = Timestamp(d.isoformat()) return t elif self.type == "time": d = datetime.strptime(raw_value, self.format).time() t = Timestamp(d.isoformat()) return t elif self.type == "datetime": d = datetime.strptime(raw_value, self.format) t = Timestamp(d.isoformat()) if not t.has_tz(): t = t.assume(self.default_timezone) return t.utc() elif self.type == "dictionary": try: MultiParser.CacheLock.acquire() if self.source in MultiParser.DictionaryCache.keys(): dictionary = MultiParser.DictionaryCache[self.source] else: try: dictionary = Dictionary(client.GET(self.source)) MultiParser.DictionaryCache[self.source] = dictionary except HTTPClientError: raise ValueError("Bad dictionary link: %s" % self.source) finally: MultiParser.CacheLock.release() try: return [ term for term in dictionary.entries if term.key == raw_value ].pop(0) except IndexError: logging.error('Key "%s" missing in dictionary "%s".' % (raw_value, self.source))
def read_status(self, status_line): try: logging.log('Status line', status_line, 'info') if '%' in status_line: with_percent, rest = status_line.split('%', 1) status, value = with_percent.split(' ') self.update_progress(value) except e: logging.error("Error parsing status line from arftp: " + str(e))
def convert(self, raw_value, client): """ Perform conversion configured when contructing the object. Args: raw_value (unicode): The raw string to parse client (vizone.client.Instance): The HTTP client to use when looking up dictionary terms. Returns: object """ if raw_value is None or raw_value == "": return None if self.type == "string": return raw_value elif self.type == "integer": return int(raw_value) elif self.type == "float": return float(raw_value) elif self.type == "iso": return Timestamp(raw_value) elif self.type == "date": d = datetime.strptime(raw_value, self.format).date() t = Timestamp(d.isoformat()) return t elif self.type == "time": d = datetime.strptime(raw_value, self.format).time() t = Timestamp(d.isoformat()) return t elif self.type == "datetime": d = datetime.strptime(raw_value, self.format) t = Timestamp(d.isoformat()) if not t.has_tz(): t = t.assume(self.default_timezone) return t.utc() elif self.type == "dictionary": try: MultiParser.CacheLock.acquire() if self.source in MultiParser.DictionaryCache.keys(): dictionary = MultiParser.DictionaryCache[self.source] else: try: dictionary = Dictionary(client.GET(self.source)) MultiParser.DictionaryCache[self.source] = dictionary except HTTPClientError: raise ValueError("Bad dictionary link: %s" % self.source) finally: MultiParser.CacheLock.release() try: return [term for term in dictionary.entries if term.key == raw_value].pop(0) except IndexError: logging.error('Key "%s" missing in dictionary "%s".' % (raw_value, self.source))
def import_unmanaged_file(asset, uri_list, client=None): """ Start an Unmanaged File import to a given Asset Entry Args: asset (vizone.payload.asset.Item): The Asset Entry to import to uri_list (vizone.urilist.UriList): A URI List containing the link to the media client (Optional[vizone.client.Instance]): A Viz One client to use (None means the default) Returns: bool: True if successful, False on error """ logging.info(u'Importing file %s to %s.', uri_list.generate().strip(), asset.id) try: client.POST(asset.import_unmanaged_link, uri_list) return True except HTTPClientError: logging.error('Unable to import unmanaged file "%s" to asset %s', uri_list.generate().strip(), asset.id) return False
def start(self, asset, conflict=False): # Fetch metadata with a fresh etag try: old_metadata = Payload(self.client.GET(asset.describedby_link)) # If no metadata is found, we cannot do any metadata mapping except HTTPClientError as e: logging.error(u'Asset Metadata fetching failed %s', str(e)) return # If there was an internal server error (5xx), wait and retry except HTTPServerError as e: logging.error(u'Asset Metadata fetching failed %s', str(e)) logging.warn(u'Retrying in 10 seconds...') time.sleep(10) raise Retry # Copy metadata and operate on the copy, so we can compare them later new_metadata = Payload(old_metadata.generate()) scope = { 'asset': asset, 'metadata': new_metadata, } for field, expr in self.mappings.items(): logging.info('%s = %s', field, expr) new_metadata.set(field, eval(expr, scope)) logging.log(u'Resulting payload', new_metadata.generate(), 'xml') # Only update if we produced any differences, else we will have a circle of updates if new_metadata != old_metadata: logging.info(u'Updating metadata...') self.client.PUT( asset.describedby_link, new_metadata, etag=old_metadata.response_etag ) logging.info(u'Updated metadata.') else: logging.info(u'No changes to metadata.')
def start(self, asset, conflict=False): # Fetch metadata with a fresh etag try: old_metadata = Payload(self.client.GET(asset.describedby_link)) # If no metadata is found, we cannot do any metadata mapping except HTTPClientError as e: logging.error(u'Asset Metadata fetching failed %s', str(e)) return # If there was an internal server error (5xx), wait and retry except HTTPServerError as e: logging.error(u'Asset Metadata fetching failed %s', str(e)) logging.warn(u'Retrying in 10 seconds...') time.sleep(10) raise Retry # Copy metadata and operate on the copy, so we can compare them later new_metadata = Payload(old_metadata.generate()) scope = { 'asset': asset, 'metadata': new_metadata, } for field, expr in self.mappings.items(): logging.info('%s = %s', field, expr) new_metadata.set(field, eval(expr, scope)) logging.log(u'Resulting payload', new_metadata.generate(), 'xml') # Only update if we produced any differences, else we will have a circle of updates if new_metadata != old_metadata: logging.info(u'Updating metadata...') self.client.PUT(asset.describedby_link, new_metadata, etag=old_metadata.response_etag) logging.info(u'Updated metadata.') else: logging.info(u'No changes to metadata.')
def create_or_update_asset( id=None, metadata=None, acl=None, mediaacl=None, tags=None, materialtype=None, category=None, rightscode=None, client=None): """ Creates or Updates an Item with a given ``id``. Metadata updates will be retried three times if there are conflicts. Args: id (Optional[unicode]): An asset id or "site identity" metadata (Optional[vizone.vdf.Payload]): The metadata to update to. Can be a dict with a 'form' field too. acl (Optional[vizone.vizone.payload.user_group.Acl]): Specify a ACL when creating the Asset mediaacl (Optional[vizone.vizone.payload.user_group.Acl]): Specify a Media ACL when creating the Asset tags (Optional[dict]): scheme => term dictionary for custom tags when creating the Asset materialtype (Optional[unicode]): Set the Material Type to this when creating the Asset category (Optional[unicode]): Set the Category to this when creating the Asset rightscode (Optional[unicode]): Set the Rights Code to this when creating the Asset client (Optional[vizone.client.Instance]): A Viz One client to use (None means the default) Returns: vizone.payload.asset.Item: The updated or created Asset Entry """ old_payload = None client = client or get_default_instance() # Create or Update Placeholder try: asset = get_asset_by_id(id, headers={'X-Inline': 'describedby'}, client=client) old_payload = asset.describedby_link.metadata except HTTPClientError: try: logging.info(u'Item %s does not exist, creating.', id) asset = Item(id=id) if acl: asset.acl = acl if mediaacl: asset.mediaacl = mediaacl if materialtype: asset.materialtype = materialtype if category: asset.category = category if rightscode: asset.rightscode = rightscode if tags: for scheme, term in sorted(tags.items()): asset.keywords.append( AtomCategory(scheme=scheme, term=term) ) asset = create_asset(asset, client=client) except (HTTPClientError, HTTPServerError): logging.error(u'Could not create asset %s, skipping.', id) return # Create payload if metadata is a dict if type(metadata) is dict: form = metadata.pop('form') if old_payload is None: models = MetadataFormCollection(client.GET(asset.models_link)) model_link = [model.self_link for model in models.entries if model.name == form][0] model = Model(client.GET(model_link)) payload = model.to_payload() else: payload = old_payload for name, value in metadata.items(): payload.set(name, value) else: payload = metadata # Update Metadata if needed if payload is not None and payload != old_payload: logging.info(u'Updating metadata for asset %s.', asset.id) for _ in range(3): try: asset.describedby_link.metadata = Payload(client.PUT(asset.describedby_link, payload)) break except HTTPServerError: logging.error(u'Could not update metadata for asset %s.', asset.id) return except HTTPClientError as e: logging.info(u'Asset Metadata update failed %s', str(e)) if e.responstatus_code == 412: logging.warn(u'Retrying...') asset.parse(client.GET(asset.self_link)) else: break else: logging.info(u'Updating metadata for asset %s not needed.', asset.id) return asset