def test_import_base_jobmeta(self):
from gnmvidispine.vs_item import VSItem
i = VSItem(host=self.fake_host,port=self.fake_port,user=self.fake_user,passwd=self.fake_passwd)
result = i.import_base(jobMetadata={'keyone': 'valueone','keytwo': 'valuetwo'})
self.assertEqual({'essence': 'false', 'jobmetadata': ['keyone=valueone', 'keytwo=valuetwo'],
'priority': 'MEDIUM','tag': 'original', 'thumbnails': 'true'},
result)
def get(self, request, itemid=None):
import re
from gnmvidispine.vs_item import VSItem
from django.core import cache
from django.conf import settings
#not necessary, as this is already done by the url parser
# if not re.match(r'\w{2}-\d+$',itemid):
# return HttpResponseBadRequest("Item ID invalid")
c = cache.get_cache('default')
rtn = c.get('gnmuploadprofiler:item:{0}'.format(itemid))
if rtn is None:
item = VSItem(url=settings.VIDISPINE_URL,
port=settings.VIDISPINE_PORT,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD)
item.populate(itemid, specificFields=self.interesting_fields)
rtn = {}
for f in self.interesting_fields:
rtn[f] = item.get(f)
c.set('gnmuploadprofiler:item:{0}'.format(itemid), rtn)
return Response(rtn)
def get(self, request, itemid):
from gnmvidispine.vs_item import VSItem
from django.conf import settings
import traceback
try:
itemref = VSItem(url=settings.VIDISPINE_URL,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD,
run_as=request.user.username)
itemref.populate(itemid, specificFields=['title'])
rtn = []
for s in itemref.shapes():
info = {
"shapetag": s.tag(),
"shapeid": s.name,
"rules": map(lambda x: x.as_dict(),
s.storage_rules().rules())
}
rtn.append(info)
return Response(rtn, status=200)
except Exception as e:
return Response({
'status': 'error',
'error': str(e),
'trace': traceback.format_exc()
})
def delete_rule_from_item(request):
from django.http import HttpResponseRedirect
from django.conf import settings
from gnmvidispine.vs_item import VSItem, VSNotFound
i = VSItem(url=settings.VIDISPINE_URL,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD,
run_as=request.user.username)
try:
i.populate(request.POST['itemid'])
except VSNotFound as e:
return render(request, 'gnmlibrarytool/delete_failed.html',
{'error': e})
try:
shape = i.get_shape(request.POST['delete'])
except VSNotFound as e:
return render(request, 'gnmlibrarytool/delete_failed.html',
{'error': e})
try:
shape.delete_storage_rule()
except VSNotFound as e:
return render(request, 'gnmlibrarytool/delete_failed.html',
{'error': e})
return HttpResponseRedirect(
request.META.get('HTTP_REFERER') + "#shortcutmv_storage_rule_menu")
def test_toxml(self):
fix = re.compile(r'((?<=>)(\n[\t]*)(?=[^<\t]))|(?<=[^>\t])(\n[\t]*)(?=<)')
#christ knows why it gets indented like this, but it does.
output_testdoc = fix.sub("","""<?xml version='1.0' encoding='UTF-8'?>
<ns0:MetadataListDocument xmlns:ns0="http://xml.vidispine.com/schema/vidispine">
<ns0:item id="VX-1234">
<ns0:metadata>
<ns0:timespan end="+INF" start="-INF">
<ns0:field>
<ns0:name>sometestfield</ns0:name>
<ns0:value>sometestvalue</ns0:value>
</ns0:field>
<ns0:field>
<ns0:name>someotherfield</ns0:name>
<ns0:value>valueone</ns0:value>
<ns0:value>valuetwo</ns0:value>
</ns0:field>
</ns0:timespan>
</ns0:metadata>
</ns0:item>
</ns0:MetadataListDocument>""")
from gnmvidispine.vs_item import VSItem
i = VSItem(host=self.fake_host, port=self.fake_port, user=self.fake_user, passwd=self.fake_passwd)
i.fromXML(self.testdoc)
self.assertEqual(fix.sub("",i.toXML()),output_testdoc)
def test_fromxml(self):
from gnmvidispine.vs_item import VSItem
i = VSItem(host=self.fake_host, port=self.fake_port, user=self.fake_user, passwd=self.fake_passwd)
i.fromXML(self.testdoc)
self.assertEqual(i.get("sometestfield"),"sometestvalue")
self.assertEqual(i.get("someotherfield", allowArray=True),["valueone","valuetwo"])
def process_premiere_fileref(filepath, server_path, vsproject, vs_pathmap=None, db=None, cfg=None):
"""
process an individual file reference from a project
:param filepath: file path to process
:param server_path: file path as it is seen from the server
:param project_database_id: database id of the premiere project
:param db: assetimporter database object
:param cfg: assetimporter configuration object
:return: item reference or None
"""
vsid = db.get_vidispine_id(server_path)
# this call will return None if the file is not from an asset folder, e.g. newswire
if vsid is None:
vsid = find_item_id_for_path(server_path, vs_pathmap)
if vsid is None:
lg.error("File {0} is found by Vidispine but has not been imported yet".format(server_path))
return None
item = VSItem(host=cfg.value('vs_host'),port=cfg.value('vs_port'),user=cfg.value('vs_user'),passwd=cfg.value('vs_password'))
item.populate(vsid,specificFields=['gnm_asset_category'])
if item.get('gnm_asset_category') is not None:
if item.get('gnm_asset_category').lower() == 'branding':
lg.info("File %s is branding, not adding to project" % (filepath, ))
return item
else:
lg.info("gnm_asset_category field empty so continuing")
lg.debug("Got filepath %s" % filepath)
fileid = db.fileId(server_path)
if fileid:
db.link_file_to_edit_project(fileid, vsproject)
lg.debug("Linked file %s with id %s to project with database ID %s" % (filepath, fileid, vsproject))
else:
raise NotInDatabaseError
return item
def reattach(self, itemid, collectionid):
coll = VSCollection(host=self.config.value('vs_host'), port=self.config.value('vs_port'),
user=self.config.value('vs_user'), passwd=self.config.value('vs_password'))
coll.name = collectionid
coll.addToCollection(itemid, type="item")
self.logger.info("Attached item {0} to {1}".format(itemid, collectionid))
self.logger.info("Updating management metadata for {0}".format(itemid))
completed = False
held = False
deletable = False
sensitive = False
deep_archive = False
if coll.get('gnm_project_status') == 'Completed':
completed = True
elif coll.get('gnm_project_status') == 'Held':
held = True
if coll.get('gnm_storage_rule_deletable') == 'storage_rule_deletable':
deletable = True
if coll.get('gnm_storage_rule_sensitive') == 'storage_rule_sensitive':
sensitive = True
if coll.get('gnm_storage_ruke_deep_archive') == 'storage_rule_deep_archive':
deep_archive = True
item = VSItem(host=self.options.vshost, port=self.options.vsport, user=self.options.vsuser, passwd=self.options.vspass)
item.name = itemid
item.set_metadata({
'gnm_storage_rule_projects_completed': ['storage_rule_projects_completed' if completed else ''],
'gnm_storage_rule_projects_held' : ['storage_rule_projects_held' if held else ''],
'gnm_storage_rule_deletable' : ['storage_rule_deletable' if deletable else ''],
'gnm_storage_rule_sensitive' : ['storage_rule_sensitive' if sensitive else ''],
'gnm_storage_rule_deep_archive' : ['storage_rule_deep_archive' if deep_archive else '']
})
self.logger.info("Updated metadata for item {0}".format(item.name))
def add_rule_to_item(request):
from xml.etree.cElementTree import fromstring
from gnmvidispine.vs_item import VSItem, VSNotFound
from gnmvidispine.vs_storage_rule import VSStorageRuleNew
from django.conf import settings
from django.http import HttpResponseRedirect
newrule = VSStorageRuleNew(url=settings.VIDISPINE_URL,user=settings.VIDISPINE_USERNAME,passwd=settings.VIDISPINE_PASSWORD,run_as=request.user.username)
try:
newrule.populate_from_xml(fromstring(request.POST['rulexml']))
except VSNotFound as e:
return render(request, 'gnmlibrarytool/rule_failed.html', {'error': e})
except SyntaxError as e:
return render(request, 'gnmlibrarytool/rule_failed.html', {'error': e})
i = VSItem(url=settings.VIDISPINE_URL,user=settings.VIDISPINE_USERNAME,passwd=settings.VIDISPINE_PASSWORD,run_as=request.user.username)
try:
i.populate(request.POST['itemid'])
except VSNotFound as e:
return render(request, 'gnmlibrarytool/rule_failed.html', {'error': e})
try:
shape = i.get_shape(request.POST['ruleshape'])
except VSNotFound as e:
return render(request, 'gnmlibrarytool/rule_failed.html', {'error': e})
try:
shape.add_storage_rule(newrule)
except VSNotFound as e:
return render(request, 'gnmlibrarytool/rule_failed.html', {'error': e})
return HttpResponseRedirect(request.META.get('HTTP_REFERER')+"#shortcutmv_storage_rule_menu")
def get(self, request, vs_item_id=None):
from django.conf import settings
from gnmvidispine.vs_item import VSItem, VSNotFound, VSException
from traceback import format_exc
from models import GridCapturePreset
from traceback import format_exc
from notification_handler import should_trigger, vs_field_list
item = VSItem(url=settings.VIDISPINE_URL,
port=settings.VIDISPINE_PORT,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD,
run_as=request.user.username)
try:
item.populate(vs_item_id, specificFields=vs_field_list())
n = should_trigger(item)
if n:
ps = GridCapturePreset.objects.get(pk=n)
return Response({
'status': 'ok',
'result': True,
'because': ps.info()
})
else:
return Response({'status': 'ok', 'result': False})
except VSNotFound as e:
return Response(
{
'status': 'error',
'exception':
'Vidispine item {0} not found'.format(vs_item_id)
},
status=404)
except VSException as e:
return Response(
{
'status': 'error',
'exception': 'Vidispine error {0}'.format(
e.__class__.__name__),
'detail': {
'type': e.exceptionType,
'context': e.exceptionContext,
'what': e.exceptionWhat,
'id': e.exceptionID
}
},
status=500)
except Exception as e:
return Response(
{
'status': 'error',
'exception': unicode(e),
'trace': format_exc()
},
status=500)
def test_populate(self):
with patch("gnmvidispine.vs_item.VSItem.request", return_value=ET.fromstring(self.testdoc)) as mock_request:
from gnmvidispine.vs_item import VSItem
i = VSItem(host=self.fake_host, port=self.fake_port, user=self.fake_user, passwd=self.fake_passwd)
i.populate("VX-1234")
mock_request.assert_called_once_with("/item/VX-1234/metadata",method="GET")
self.assertEqual(i.get("sometestfield"),"sometestvalue")
self.assertEqual(i.get("someotherfield",allowArray=True),["valueone","valuetwo"])
self.assertEqual(i.name,"VX-1234")
def test_remove_external_id(self):
"""
remove_external_id should call VS to delete external id
:return:
"""
with patch('gnmvidispine.vs_item.VSItem.request') as mock_request:
from gnmvidispine.vs_item import VSItem
i = VSItem(host="localhost",port=1234,user="******",passwd="secret")
i.name="VX-1234"
i.remove_external_id("08473AFA-E7D5-4B92-9C4F-81035523A492")
mock_request.assert_called_once_with("/item/VX-1234/external-id/08473AFA-E7D5-4B92-9C4F-81035523A492", method="DELETE")
def test_import_external_xml(self):
with patch('gnmvidispine.vs_item.VSItem.raw_request') as mock_request:
from gnmvidispine.vs_item import VSItem
testdoc = """<?xml version="1.0" encoding="UTF-8"?>
<external-metadata>
<somefieldname>value</somefieldname>
</external-metadata>
"""
i = VSItem(host=self.fake_host,port=self.fake_port,user=self.fake_user,passwd=self.fake_passwd)
i.name = "VX-345"
i.import_external_xml(testdoc,projection_name="myprojection")
mock_request.assert_called_once_with("/item/VX-345/metadata", method="PUT", matrix={'projection': 'myprojection'},body=testdoc)
def test_import_external_xml_with_unicode(self):
with patch('gnmvidispine.vs_item.VSItem.raw_request') as mock_request:
from gnmvidispine.vs_item import VSItem
testdoc = u"""<?xml version="1.0" encoding="UTF-8"?>
<external-metadata>
<somefieldname>Arséne Wenger est alleé en vacances</somefieldname>
<someotherfield>This — is a silly long -, as in — not -</someotherfield>
</external-metadata>
"""
i = VSItem(host=self.fake_host,port=self.fake_port,user=self.fake_user,passwd=self.fake_passwd)
i.name = "VX-345"
i.import_external_xml(testdoc,projection_name="myprojection")
mock_request.assert_called_once_with("/item/VX-345/metadata", method="PUT", matrix={'projection': 'myprojection'},body=testdoc)
def test_make_metadata_document(self):
from gnmvidispine.vs_item import VSItem
i = VSItem(host=self.fake_host,port=self.fake_port,user=self.fake_user,passwd=self.fake_passwd)
i.name = "VX-123"
testdoc = i._make_metadata_document({"field1": "value1","field2": "value2","field3": 3})
self.assertEqual(testdoc,"""<?xml version='1.0' encoding='UTF-8'?>
<MetadataDocument xmlns="http://xml.vidispine.com/schema/vidispine"><timespan end="+INF" start="-INF"><field><name>field2</name><value>value2</value></field><field><name>field3</name><value>3</value></field><field><name>field1</name><value>value1</value></field></timespan></MetadataDocument>""")
testdoc = i._make_metadata_document({"field1": ["value1","value2","value3"], "field2": "value2"})
self.assertEqual(testdoc,"""<?xml version='1.0' encoding='UTF-8'?>
<MetadataDocument xmlns="http://xml.vidispine.com/schema/vidispine"><timespan end="+INF" start="-INF"><field><name>field2</name><value>value2</value></field><field><name>field1</name><value>value1</value><value>value2</value><value>value3</value></field></timespan></MetadataDocument>""")
def get(self, request, vs_item_id=None):
from django.conf import settings
from gnmvidispine.vs_item import VSItem, VSNotFound
from models import GridMetadataFields
from traceback import format_exc
from notification_handler import do_meta_substitution, vs_field_list
item = VSItem(url=settings.VIDISPINE_URL,
port=settings.VIDISPINE_PORT,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD,
run_as=request.user.username)
try:
#log.debug("Looking up item {0}".format(vs_item_id))
fieldnames = vs_field_list()
#log.debug("Field list: {0}".format(fieldnames))
try:
item.populate(vs_item_id, specificFields=fieldnames)
except VSNotFound as e:
return Response(
{
'status': 'error',
'problem': 'Item not found',
'exception': e
},
status=404)
return Response({
'status': 'success',
'item_meta': do_meta_substitution(item, -1, 1),
'rights_meta': do_meta_substitution(item, -1, 2)
})
except Exception as e:
if settings.DEBUG:
return Response(
{
'status': 'error',
'exception': e,
'type': e.__class__,
'trace': format_exc()
},
status=500)
else:
return Response({
'status': 'error',
'exception': e,
},
status=500)
def reattach(self, itemid, collectionid):
coll = VSCollection(host=self.config.value('vs_host'),
port=self.config.value('vs_port'),
user=self.config.value('vs_user'),
passwd=self.config.value('vs_password'))
coll.name = collectionid
coll.addToCollection(itemid, type="item")
self.logger.info("Attached item {0} to {1}".format(
itemid, collectionid))
self.logger.info("Updating management metadata for {0}".format(itemid))
completed = False
held = False
deletable = False
sensitive = False
deep_archive = False
if coll.get('gnm_project_status') == 'Completed':
completed = True
elif coll.get('gnm_project_status') == 'Held':
held = True
if coll.get('gnm_storage_rule_deletable') == 'storage_rule_deletable':
deletable = True
if coll.get('gnm_storage_rule_sensitive') == 'storage_rule_sensitive':
sensitive = True
if coll.get('gnm_storage_ruke_deep_archive'
) == 'storage_rule_deep_archive':
deep_archive = True
item = VSItem(host=self.options.vshost,
port=self.options.vsport,
user=self.options.vsuser,
passwd=self.options.vspass)
item.name = itemid
item.set_metadata({
'gnm_storage_rule_projects_completed':
['storage_rule_projects_completed' if completed else ''],
'gnm_storage_rule_projects_held':
['storage_rule_projects_held' if held else ''],
'gnm_storage_rule_deletable':
['storage_rule_deletable' if deletable else ''],
'gnm_storage_rule_sensitive':
['storage_rule_sensitive' if sensitive else ''],
'gnm_storage_rule_deep_archive':
['storage_rule_deep_archive' if deep_archive else '']
})
self.logger.info("Updated metadata for item {0}".format(item.name))
def lookup_vidispine_item(credentials, item_id):
"""
Look up the containing collection in Vidispine, if the search in the Portal index fails
:param credentials: dictionary of Vidipsine credentials
:param item_id: item ID to look up
:return: String of the collection ID, or None if no collection was found.
"""
i = VSItem(host=credentials['host'],port=int(credentials['port']),user=credentials['user'],
passwd=credentials['password'])
try:
i.populate(item_id, specificFields=['title'])
except VSNotFound:
logger.error("Item {0} was not found in Vidispine".format(item_id))
return None
containers = i.get('__collection',allowArray=True)
if not isinstance(containers,list):
return [containers]
else:
return containers
def get_item_for_atomid(self, atomid):
"""
Returns a populated VSItem object for the master, or None if no such item exists
:param atomid:
:return:
"""
try:
item = VSItem(url=settings.VIDISPINE_URL, user=settings.VIDISPINE_USERNAME,passwd=settings.VIDISPINE_PASSWORD)
item.populate(atomid) #this will work if the item has an external id set to the atom id. this is done in `set_project_fields_for_master`
return item
except VSNotFound:
s = VSItemSearch(url=settings.VIDISPINE_URL,user=settings.VIDISPINE_USERNAME,passwd=settings.VIDISPINE_PASSWORD)
s.addCriterion({const.GNM_MASTERS_MEDIAATOM_ATOMID: atomid, const.GNM_TYPE: 'Master'})
result = s.execute()
if result.totalItems==0:
return None
elif result.totalItems==1:
return result.results(shouldPopulate=True).next()
else:
resultList = map(lambda item: item, result.results(shouldPopulate=False))
potential_master_ids = map(lambda item: item.name, resultList)
logger.warning("Multiple masters returned for atom ID {0}: {1}. Using the first.".format(atomid, potential_master_ids))
resultList[0].populate(resultList[0].name)
return resultList[0]
def process_premiere_fileref(filepath,
server_path,
vsproject,
vs_pathmap=None,
db=None,
cfg=None):
"""
process an individual file reference from a project
:param filepath: file path to process
:param server_path: file path as it is seen from the server
:param project_database_id: database id of the premiere project
:param db: assetimporter database object
:param cfg: assetimporter configuration object
:return: item reference or None
"""
vsid = db.get_vidispine_id(server_path)
# this call will return None if the file is not from an asset folder, e.g. newswire
if vsid is None:
vsid = find_item_id_for_path(server_path, vs_pathmap)
if vsid is None:
lg.error(
"File {0} is found by Vidispine but has not been imported yet".
format(server_path))
return None
item = VSItem(host=cfg.value('vs_host'),
port=cfg.value('vs_port'),
user=cfg.value('vs_user'),
passwd=cfg.value('vs_password'))
item.populate(vsid, specificFields=['gnm_asset_category'])
if item.get('gnm_asset_category') is not None:
if item.get('gnm_asset_category').lower() == 'branding':
lg.info("File %s is branding, not adding to project" %
(filepath, ))
return item
else:
lg.info("gnm_asset_category field empty so continuing")
lg.debug("Got filepath %s" % filepath)
fileid = db.fileId(server_path)
if fileid:
db.link_file_to_edit_project(fileid, vsproject)
lg.debug("Linked file %s with id %s to project with database ID %s" %
(filepath, fileid, vsproject))
else:
raise NotInDatabaseError
return item
def get(self, request, vs_item_id=None):
from django.conf import settings
from gnmvidispine.vs_item import VSItem, VSNotFound
from notification_handler import VIDISPINE_GRID_REF_FIELD
from notification_handler import vs_field_list
from pprint import pformat
#log.debug("Request data: {0}".format(pformat(request.__dict__)))
#log.debug("Request user: {0}".format(request.user))
item = VSItem(url=settings.VIDISPINE_URL,
port=settings.VIDISPINE_PORT,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD,
run_as=request.user.username)
try:
#log.debug("Looking up item {0}".format(vs_item_id))
item.populate(vs_item_id, specificFields=self.interesting_fields)
except VSNotFound as e:
return Response(
{
'status': 'error',
'problem': 'Item not found',
'exception': e
},
status=404)
meta = {}
for f in self.interesting_fields:
meta[f] = item.get(f, allowArray=True)
for f in vs_field_list():
meta[f] = item.get(f, allowArray=True)
gridrefs = item.get(VIDISPINE_GRID_REF_FIELD, allowArray=True)
if not isinstance(gridrefs, list):
gridrefs = [gridrefs]
meta[VIDISPINE_GRID_REF_FIELD] = gridrefs
return Response({
'status': 'success',
'item': vs_item_id,
'metadata': meta
})
def create_placeholder_for_atomid(atomid, title="unknown video", user="******", parent=None):
"""
Creates a placeholder and returns a VSItem object for it
:param atomid: atom ID string
:param title: title of the new video
:return: VSItem object
"""
from gnmvidispine.vs_metadata import VSMetadataReference
item = VSItem(url=settings.VIDISPINE_URL,user=settings.VIDISPINE_USERNAME,passwd=settings.VIDISPINE_PASSWORD)
project_name_attribs = parent.get_metadata_attributes(const.GNM_PROJECT_HEADLINE)
project_name_reference = VSMetadataReference(uuid=project_name_attribs[0].uuid)
commission_name_attribs = parent.get_metadata_attributes(const.GNM_COMMISSION_TITLE)
commission_name_ref = VSMetadataReference(uuid=commission_name_attribs[0].uuid)
metadata = {const.GNM_TYPE: 'Master',
'title': title,
const.GNM_MASTERS_WEBSITE_HEADLINE: title,
const.GNM_MASTERS_MEDIAATOM_ATOMID: atomid,
const.GNM_MASTERS_GENERIC_TITLEID: atomid,
const.GNM_ASSET_CATEGORY: "Master",
const.GNM_MASTERS_MEDIAATOM_UPLOADEDBY: user,
const.GNM_PROJECT_HEADLINE: project_name_reference,
const.GNM_COMMISSION_TITLE: commission_name_ref
}
userid = VSMixin.get_userid_for_email(user)
if userid is not None:
metadata.update({
const.GNM_MASTERS_GENERIC_OWNER: userid
})
item.createPlaceholder(metadata, group='Asset')
item.add_external_id(atomid)
return item
def test_import_base_no_args(self):
from gnmvidispine.vs_item import VSItem
i = VSItem(host=self.fake_host,port=self.fake_port,user=self.fake_user,passwd=self.fake_passwd)
result = i.import_base()
self.assertEqual({'essence': 'false','priority': 'MEDIUM','tag': 'original', 'thumbnails': 'true'}, result)
def process_premiere_project(filepath, db=None, cfg=None):
global vs_pathmap
lg.debug("---------------------------------")
lg.info("Premiere project: %s" % filepath)
collection_vsid = id_from_filepath(filepath)
lg.info("Project's Vidispine ID: %s" % collection_vsid)
vsproject = VSCollection(host=cfg.value('vs_host'),
port=cfg.value('vs_port'),
user=cfg.value('vs_user'),
passwd=cfg.value('vs_password'))
vsproject.setName(
collection_vsid
) #we don't need the actual metadata so don't bother getting it.
#pprint(vs_pathmap)
pp = PremiereProject()
try:
pp.load(filepath)
except Exception as e:
lg.error("Unable to read '%s': %s" % (filepath, e.message))
lg.error(traceback.format_exc())
print "Unable to read '%s': %s" % (filepath, e.message)
traceback.print_exc()
raven_client.captureException()
return (0, 0, 0)
lg.debug("determining project details and updating database...")
try:
projectDetails = pp.getParticulars()
project_id = db.upsert_edit_project(
os.path.dirname(filepath),
os.path.basename(filepath),
projectDetails['uuid'],
projectDetails['version'],
desc="Adobe Premiere Project",
opens_with=
"/Applications/Adobe Premiere Pro CC 2014/Adobe Premiere Pro CC 2014.app/Contents/MacOS/Adobe Premiere Pro CC 2014"
)
except ValueError as e:
project_id = db.log_project_issue(os.path.dirname(filepath),
os.path.basename(filepath),
problem="Invalid project file",
detail="{0}: {1} {2}".format(
e.__class__, str(e),
traceback.format_exc()))
lg.error("Unable to read project file '{0}' - {1}".format(
filepath, str(e)))
lg.error(traceback.format_exc())
raven_client.captureException()
return (0, 0, 0)
except KeyError as e:
project_id = db.log_project_issue(os.path.dirname(filepath),
os.path.basename(filepath),
problem="Invalid project file",
detail="{0}: {1} {2}".format(
e.__class__, str(e),
traceback.format_exc()))
db.insert_sysparam(
"warning",
"Unable to read project file '{0}' - {1}".format(filepath, str(e)))
lg.error("Unable to read project file '{0}' - {1}".format(
filepath, str(e)))
lg.error(traceback.format_exc())
raven_client.captureException()
return (0, 0, 0)
except InvalidDataError as e:
db.insert_sysparam(
"warning",
"Corrupted project file: {0} {1}".format(filepath, unicode(e)))
raven_client.captureException()
return (0, 0, 0)
total_files = 0
no_vsitem = 0
not_in_db = 0
lg.debug("looking for referenced media....")
for filepath in pp.getReferencedMedia():
total_files += 1
server_path = re.sub(u'^/Volumes', '/srv', filepath).encode('utf-8')
vsid = db.get_vidispine_id(server_path)
# this call will return None if the file is not from an asset folder, e.g. newswire
if vsid is None:
try:
vsid = find_item_id_for_path(server_path)
if vsid is None:
lg.error(
"File {0} is found by Vidispine but has not been imported yet"
.format(server_path))
continue
except VSNotFound:
lg.error(
"File {0} could not be found in either Vidispine or the asset importer database"
.format(server_path))
continue
item = VSItem(host=cfg.value('vs_host'),
port=cfg.value('vs_port'),
user=cfg.value('vs_user'),
passwd=cfg.value('vs_password'))
item.populate(vsid, specificFields=['gnm_asset_category'])
try:
if item.get('gnm_asset_category').lower() == 'branding':
lg.info("File %s is branding, not adding to project" %
(filepath, ))
continue
except AttributeError:
lg.warning("File %s has no value for gnm_asset_catetory" %
(filepath, ))
lg.debug("Got filepath %s" % filepath)
fileid = db.fileId(server_path)
if fileid:
try:
db.link_file_to_edit_project(fileid, project_id)
lg.debug("Linked file %s with id %s to project %s" %
(filepath, fileid, project_id))
except AlreadyLinkedError:
lg.info("File %s with id %s is already linked to project %s" %
(filepath, fileid, project_id))
continue
else:
not_in_db += 1
lg.warning("File %s could not be found in the database" % filepath)
continue
n = 0
found = False
#using this construct to avoid loading more data from VS than necessary. We simply check whether the ID exists
#in the parent collections list (cached on the item record) without lifting any more info out of VS
if vsproject.name in map(
lambda x: x.name,
item.parent_collections(shouldPopulate=False)):
lg.info("File %s is already in project %s" %
(filepath, vsproject.name))
continue
vsproject.addToCollection(
item=item
) #this call will apparently succeed if the item is already added to said collection, but it won't be added twice.
lg.info(
"Run complete. Out of a total of %d referenced files, %d did not have a Vidispine item and %d were not in the Asset Importer database"
% (total_files, no_vsitem, not_in_db))
return (total_files, no_vsitem, not_in_db)
def profile_item(itemname):
"""
Calculate the times for the various waypoints to publishing this item. Assumes that the item has actually been published.
:param itemname: Vidispine ID for the item to profile
:return: Saved database model record
"""
from gnmvidispine.vs_item import VSItem
from django.conf import settings
from datetime import timedelta
import dateutil.parser
from models import OutputTimings
from pprint import pprint
print("Doing output time profile for item {0}".format(itemname))
item = VSItem(url=settings.VIDISPINE_URL,port=settings.VIDISPINE_PORT,
user=settings.VIDISPINE_USERNAME,passwd=settings.VIDISPINE_PASSWORD)
item.populate(itemname)
changeset_list = item.metadata_changeset_list()
try:
profile_record = OutputTimings.objects.get(item_id=itemname)
logger.warning(u"Item {0} ({1}) has already been profiled!".format(itemname,item.get('title')))
print(u"Item {0} ({1}) has already been profiled!".format(itemname,item.get('title')))
except OutputTimings.DoesNotExist:
profile_record = OutputTimings()
profile_record.item_id = item.name
profile_record.commission = item.get('gnm_commission_title')
profile_record.project = item.get('gnm_project_headline')
profile_record.item_duration = item.get('durationSeconds')
#step one - created time
profile_record.created_time = dateutil.parser.parse(item.get('created'))
print("Created time is {0}".format(profile_record.created_time))
print("portal_ingested is {0}".format(item.get('portal_ingested')))
#when was the media last added?
current_version = item.get('portal_essence_version')
#for the time being, ignoring updates; therefore we're looking for the creation of version 1.
subset = changesets_for_fieldname('portal_essence_version',changeset_list)
lastchange = first_change_from_set(subset,fieldname='portal_essence_version')#,value=str(1))
print("change of portal_essence_version is {0}".format(unicode(lastchange)))
if lastchange is not None:
profile_record.version_created_time = lastchange.timestamp
profile_record.media_version = int(current_version) #int(lastchange.value)
subset = changesets_for_fieldname('shapeTag',changeset_list)
#lastchange = last_change_from_set(subset)
#print("last change of shapeTag is {0}".format(unicode(lastchange)))
lastchange = change_for_value('lowres',subset)
print("change of shapeTag to have lowres is {0}".format(unicode(lastchange)))
interval = lastchange.timestamp - profile_record.version_created_time
print("Time interval is {0}".format(interval))
profile_record.proxy_completed_interval = timedelta_to_float(interval)
subset = changesets_for_fieldname('gnm_master_website_uploadstatus',changeset_list)
#print "got subset {0}".format(subset)
readychange = change_for_value('Ready to Upload',subset)
print("trigger change was {0}".format(unicode(readychange)))
profile_record.upload_trigger_interval=timedelta_to_float(readychange.timestamp - profile_record.version_created_time)
if profile_record.upload_trigger_interval<0:
profile_record.upload_trigger_interval = timedelta_to_float(readychange.timestamp - profile_record.created_time)
#page_created_interval
subset = changesets_for_fieldname('gnm_master_website_edit_url',changeset_list)
#lastchange = last_change_from_set(subset)
lastchange = first_change_from_set(subset,value=re.compile('.+'))
print("first non-empty change of gnm_master_website_edit_url is {0}".format(lastchange))
profile_record.page_created_interval=timedelta_to_float(lastchange.timestamp - profile_record.version_created_time)
#last transcode
subset = changesets_for_fieldname('gnm_master_website_uploadlog',changeset_list)
firstchange = first_change_from_set(subset)
log = UploadLog()
log.parse_upload_log(item.get('gnm_master_website_uploadlog'),assumed_date=firstchange.timestamp)
# print("There were {0} upload runs total:".format(log.upload_run_count))
# for n in range(0,log.upload_run_count):
# pprint(log.upload_run(n))
first_run=log.upload_run(log.upload_run_count-1)
while True:
print("last transcode of the first run was at {0} ({1})".format(first_run[0]['timestamp'], first_run[0]['text']))
profile_record.final_transcode_completed_interval = timedelta_to_float(first_run[0]['timestamp'] - profile_record.version_created_time)
if profile_record.final_transcode_completed_interval >0:
break
first_run[0]['timestamp']+=timedelta(days=1)
#lastchange = last_change_from_set(subset)
#print("last change of gnm_master_website_uploadlog is {0}".format(lastchange))
#profile_record.final_transcode_completed_interval = timedelta_to_float(lastchange.timestamp - profile_record.version_created_time)
#page launch guess
launchguess = dateutil.parser.parse(item.get('gnm_master_publication_time'))
profile_record.page_launch_guess_interval = timedelta_to_float(launchguess - profile_record.version_created_time)
#page launch from capi
capi_data = lookup_by_octid(item.get('gnm_master_generic_titleid'))
profile_record.page_launch_capi_interval = timedelta_to_float(capi_data['webPublicationDate'] - profile_record.version_created_time)
profile_record.page_launch_pluto_lag = timedelta_to_float(launchguess - capi_data['webPublicationDate'])
#last timestamp, for sorting.
#profile_record.completed_time = launchguess
profile_record.completed_time = capi_data['webPublicationDate']
pprint(profile_record.__dict__)
profile_record.save()
return profile_record
def post(self, request):
from pprint import pformat
from gnmvidispine.vidispine_api import VSException
from gnmvidispine.vs_item import VSItem
from gnmvidispine.vs_collection import VSCollection
from gnmvidispine.vs_metadata import VSMetadata
from django.conf import settings
from portal.plugins.gnm_masters.models import VSMaster, MasterModel
from portal.plugins.gnm_projects.models import VSProject
from django.contrib.auth.models import User
import re
is_vsid = re.compile(r'^\w{2}-\d+')
try:
comm_id = request.POST['plutoconverter_commission_id_input']
proj_id = request.POST['plutoconverter_project_id_input']
item_id = request.POST['plutoconverter_item_id_input']
except StandardError as e:
return Response({'status': 'error', 'error': unicode(e)},status=400)
if not is_vsid.match(comm_id) or not is_vsid.match(proj_id) or not is_vsid.match(item_id):
return Response({'status': 'error', 'error': 'Not a valid Vidispine ID'},status=400)
try:
item = VSItem(user=settings.VIDISPINE_USERNAME, passwd=settings.VIDISPINE_PASSWORD, url=settings.VIDISPINE_URL)
item.populate(item_id)
if item.get('gnm_type') == 'master':
return Response({'status': 'error', 'error': '{0} is already a master'.format(item_id)},status=400)
project = VSCollection(user=settings.VIDISPINE_USERNAME, passwd=settings.VIDISPINE_PASSWORD, url=settings.VIDISPINE_URL)
project.populate(proj_id)
commission_id = project.get('__parent_collection')
logger.info("Project {0} belongs to commission {1}".format(project.name, commission_id))
commission = VSCollection(user=settings.VIDISPINE_USERNAME, passwd=settings.VIDISPINE_PASSWORD, url=settings.VIDISPINE_URL)
commission.populate(commission_id)
md_to_set = {
'gnm_commission_title' : commission.get('gnm_commission_title'),
'gnm_commission_workinggroup' : commission.get('gnm_commission_workinggroup'),
'gnm_project_headline' : project.get('gnm_project_headline'),
'gnm_master_website_headline' : item.get('title'),
'gnm_master_website_standfirst': item.get('gnm_asset_description'),
'gnm_master_website_byline' : item.get('gnm_asset_owner'),
# 'gnm_master_website_tags': breakout_tags(item.get('gnm_asset_user_keywords',allowArray=True),
# host=options.vshost,user=options.vsuser,passwd=options.vspasswd),
'gnm_type' : 'Master',
'gnm_master_language' : settings.LANGUAGE_CODE[0:2],
}
logger.info("Going to add {0} to project {1}".format(item_id,md_to_set))
project.addToCollection(item)
logger.info("Going to set metadata on {0}: {1}".format(item_id, md_to_set))
item.set_metadata(md_to_set)
logger.info("SUCCESS: item {0} has been converted to master".format(item_id))
admin_user = User.objects.get(username=settings.VIDISPINE_USERNAME)
vs_project = VSProject(proj_id, admin_user)
vs_master = VSMaster(item_id, admin_user)
vs_project.add_master(vs_master)
return Response({'status': 'success', 'itemid': item_id },status=200)
except VSException as e:
if raven_client is not None:
raven_client.captureException()
return Response({'status': 'error', 'error': "Vidispine said {0}".format(unicode(e))},status=500)
except StandardError as e:
if raven_client is not None:
raven_client.captureException()
return Response({'status': 'error', 'error': unicode(e)},status=500)
def create_link_for_main(item_id, shape_tag, obfuscate=True, is_update=False):
"""
Task to trigger the optional transcoding and upload of an item
:param item_id:
:return:
"""
from gnmvidispine.vs_item import VSItem
from uuid import uuid4
from base64 import b64encode
from models import DownloadableLink
link_model = DownloadableLink.objects.get(item_id=item_id,
shapetag=shape_tag)
if link_model.status == "Upload Queued" or link_model.status == "Transcoding":
allow_transcode = False
else:
allow_transcode = True
if is_update and link_model.status == "Failed":
raise RuntimeError("Create link failed, requires manual retry")
item_ref = VSItem(url=settings.VIDISPINE_URL,
port=settings.VIDISPINE_PORT,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD)
item_ref.populate(item_id, specificFields=[
'title'
]) #this will raise VSNotFound if the item_id is invalid
try:
shaperef = get_shape_for(item_ref,
shape_tag,
allow_transcode=allow_transcode)
except TranscodeRequested as e:
link_model.status = "Transcoding"
link_model.transcode_job = e.jobid
link_model.save()
create_link_for.apply_async(
(item_id, shape_tag),
kwargs={
'obfuscate': obfuscate,
'is_update': True
},
countdown=getattr(settings, "DOWNLOADABLE_LINK_RETRYTIME", 30))
return "Transcode requested"
except NeedsRetry as e:
#if it's still not present, call ourselves again in up to 30s time.
create_link_for.apply_async(
(item_id, shape_tag),
kwargs={
'obfuscate': obfuscate,
'is_update': True
},
countdown=getattr(settings, "DOWNLOADABLE_LINK_RETRYTIME", 30))
return "Still waiting for transcode"
link_model.status = "Uploading"
link_model.save()
if obfuscate:
filename = make_filename(b64encode(uuid4().get_bytes()))
else:
filename = make_filename(item_ref.get('title'))
try:
s3key = upload_to_s3(shaperef, filename=filename)
s3key.set_canned_acl("public-read")
except NeedsRetry as e:
#if it's still not present, call ourselves again in up to 30s time.
create_link_for.apply_async(
(item_id, shape_tag),
kwargs={
'obfuscate': obfuscate,
'is_update': True
},
countdown=getattr(settings, "DOWNLOADABLE_LINK_RETRYTIME", 30))
return str(e)
link_model.status = "Available"
link_model.public_url = s3key.generate_url(expires_in=0, query_auth=False)
link_model.s3_url = "s3://{0}/{1}".format(
settings.DOWNLOADABLE_LINK_BUCKET, s3key.key)
link_model.save()
return "Media available at {0}".format(link_model.public_url)
def test_import_to_shape(self):
from gnmvidispine.vs_item import VSItem
i = VSItem(host=self.fake_host,port=self.fake_port,user=self.fake_user,passwd=self.fake_passwd)
i.name = "VX-123"
i.sendAuthorized = MagicMock(return_value=self.MockedResponse(200, self.import_job_doc))
with self.assertRaises(ValueError):
i.import_to_shape() #expect ValueError if neither uri nor file ref
fake_uri="file:///path/to/newmedia.mxf"
quoted_uri=quote(fake_uri,"") #we are embedding a URI as a parameter with another URL so it must be double-encoded
i.import_to_shape(uri=fake_uri,shape_tag="shapetagname",priority="HIGH")
i.sendAuthorized.assert_called_with('POST',
'/API/item/VX-123/shape?priority=HIGH&essence=false&tag=shapetagname&thumbnails=true&uri={0}'.format(quoted_uri)
,"",{'Accept':'application/xml'}, rawData=False)
fake_uri = "file:///path/to/" + quote("media with spaces.mxf",safe="/")
quoted_uri = quote(fake_uri,"") # we are embedding a URI as a parameter with another URL so it must be double-encoded
i.import_to_shape(uri=fake_uri, shape_tag="shapetagname", priority="HIGH")
i.sendAuthorized.assert_called_with('POST',
'/API/item/VX-123/shape?priority=HIGH&essence=false&tag=shapetagname&thumbnails=true&uri={0}'.format(
quoted_uri)
, "", {'Accept': 'application/xml'}, rawData=False)
fake_uri = "file:///path/to/" + quote("media+with+plusses.mxf",safe="/+")
quoted_uri = quote(fake_uri,"") # we are embedding a URI as a parameter with another URL so it must be double-encoded
i.import_to_shape(uri=fake_uri, shape_tag="shapetagname", priority="HIGH")
i.sendAuthorized.assert_called_with('POST',
'/API/item/VX-123/shape?priority=HIGH&essence=false&tag=shapetagname&thumbnails=true&uri={0}'.format(
quoted_uri)
, "", {'Accept': 'application/xml'}, rawData=False)
def get_new_thumbnail(notification_data):
from django.conf import settings
import os.path
import traceback
import time
from gnmvidispine.vs_item import VSItem
from grid_api import GridImage
tempdir = "/tmp"
retry_delay = 5
if hasattr(settings, 'GRID_RETRY_DELAY'):
retry_delay = int(settings.GRID_RETRY_DELAY)
max_retries = 10
if hasattr(settings, 'GRID_MAX_RETRIES'):
max_retries = int(settings.GRID_MAX_RETRIES)
resp = VidispineResponseWrapper(notification_data,
url=settings.VIDISPINE_URL,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD)
logger.info("Notified of new thumbnail for {0}".format(resp.get('itemId')))
if resp.get('createThumbnails') != 'false':
#if we don't filter this out, we get crappy small thumbnails as well, but we only want nice big poster frames
logger.info(
"Got createThumbnails={0} (expecting false, i.e. Poster frame was created). Not continuing."
.format(resp.get('createThumbnails')))
return
total = 0
item = VSItem(url=settings.VIDISPINE_URL,
port=settings.VIDISPINE_PORT,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD)
logger.info("Looking up associated item {0}".format(resp.get('itemId')))
item.populate(resp.get('itemId'), specificFields=vs_field_list())
n = should_trigger(item)
if not n:
logger.info(
"Not triggering on item {0} because it does not match enabling profiles. Go to Grid Integration in the Portal admin panel if you think this is incorrect."
.format(item.name))
return
logger.info("Item {0} matched profile id {1} so continuing".format(
item.name, n))
for t in resp.thumbs().each():
filename = "{item}_{frm}.jpg".format(item=resp.get('itemId'),
frm=str(t.target_frame))
outpath = os.path.join(tempdir, filename)
logger.debug("Outputting to {0}".format(outpath))
with open(outpath, 'w') as f:
f.write(t.download())
img = get_and_upload_image(item, outpath, [])
setup_image_metadata(item, img, frame_number=t.target_frame)
retries = 0
while True:
try:
#if it's a 'recognised' 16x9 video size then supply a crop as a convenience
if img.width == 1920 and img.height == 1080:
img.make_crop(0, 0, 1920, 1080)
elif img.width == 1280 and img.height == 720:
img.make_crop(0, 0, 1280, 720)
break
except GridImage.CropsNotAvailable as e:
logger.warning(e)
retries += 1
if retries >= max_retries:
logger.error(
"Could not crop after {0} attempts, giving up".format(
retries))
break
time.sleep(retry_delay)
except StandardError as e:
logger.error(unicode(e))
logger.error(traceback.format_exc())
break
total += 1
try:
os.unlink(outpath)
except StandardError as e:
logger.warning(unicode(e))
logger.info("Handler completed for {0}, processed {1} thumbs".format(
resp.get('itemId'), total))