def process_premiere_fileref(filepath, server_path, vsproject, vs_pathmap=None, db=None, cfg=None):
"""
process an individual file reference from a project
:param filepath: file path to process
:param server_path: file path as it is seen from the server
:param project_database_id: database id of the premiere project
:param db: assetimporter database object
:param cfg: assetimporter configuration object
:return: item reference or None
"""
vsid = db.get_vidispine_id(server_path)
# this call will return None if the file is not from an asset folder, e.g. newswire
if vsid is None:
vsid = find_item_id_for_path(server_path, vs_pathmap)
if vsid is None:
lg.error("File {0} is found by Vidispine but has not been imported yet".format(server_path))
return None
item = VSItem(host=cfg.value('vs_host'),port=cfg.value('vs_port'),user=cfg.value('vs_user'),passwd=cfg.value('vs_password'))
item.populate(vsid,specificFields=['gnm_asset_category'])
if item.get('gnm_asset_category') is not None:
if item.get('gnm_asset_category').lower() == 'branding':
lg.info("File %s is branding, not adding to project" % (filepath, ))
return item
else:
lg.info("gnm_asset_category field empty so continuing")
lg.debug("Got filepath %s" % filepath)
fileid = db.fileId(server_path)
if fileid:
db.link_file_to_edit_project(fileid, vsproject)
lg.debug("Linked file %s with id %s to project with database ID %s" % (filepath, fileid, vsproject))
else:
raise NotInDatabaseError
return item
def test_fromxml(self):
from gnmvidispine.vs_item import VSItem
i = VSItem(host=self.fake_host, port=self.fake_port, user=self.fake_user, passwd=self.fake_passwd)
i.fromXML(self.testdoc)
self.assertEqual(i.get("sometestfield"),"sometestvalue")
self.assertEqual(i.get("someotherfield", allowArray=True),["valueone","valuetwo"])
def test_populate(self):
with patch("gnmvidispine.vs_item.VSItem.request", return_value=ET.fromstring(self.testdoc)) as mock_request:
from gnmvidispine.vs_item import VSItem
i = VSItem(host=self.fake_host, port=self.fake_port, user=self.fake_user, passwd=self.fake_passwd)
i.populate("VX-1234")
mock_request.assert_called_once_with("/item/VX-1234/metadata",method="GET")
self.assertEqual(i.get("sometestfield"),"sometestvalue")
self.assertEqual(i.get("someotherfield",allowArray=True),["valueone","valuetwo"])
self.assertEqual(i.name,"VX-1234")
def get(self, request, itemid=None):
import re
from gnmvidispine.vs_item import VSItem
from django.core import cache
from django.conf import settings
#not necessary, as this is already done by the url parser
# if not re.match(r'\w{2}-\d+$',itemid):
# return HttpResponseBadRequest("Item ID invalid")
c = cache.get_cache('default')
rtn = c.get('gnmuploadprofiler:item:{0}'.format(itemid))
if rtn is None:
item = VSItem(url=settings.VIDISPINE_URL,
port=settings.VIDISPINE_PORT,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD)
item.populate(itemid, specificFields=self.interesting_fields)
rtn = {}
for f in self.interesting_fields:
rtn[f] = item.get(f)
c.set('gnmuploadprofiler:item:{0}'.format(itemid), rtn)
return Response(rtn)
def process_premiere_fileref(filepath,
server_path,
vsproject,
vs_pathmap=None,
db=None,
cfg=None):
"""
process an individual file reference from a project
:param filepath: file path to process
:param server_path: file path as it is seen from the server
:param project_database_id: database id of the premiere project
:param db: assetimporter database object
:param cfg: assetimporter configuration object
:return: item reference or None
"""
vsid = db.get_vidispine_id(server_path)
# this call will return None if the file is not from an asset folder, e.g. newswire
if vsid is None:
vsid = find_item_id_for_path(server_path, vs_pathmap)
if vsid is None:
lg.error(
"File {0} is found by Vidispine but has not been imported yet".
format(server_path))
return None
item = VSItem(host=cfg.value('vs_host'),
port=cfg.value('vs_port'),
user=cfg.value('vs_user'),
passwd=cfg.value('vs_password'))
item.populate(vsid, specificFields=['gnm_asset_category'])
if item.get('gnm_asset_category') is not None:
if item.get('gnm_asset_category').lower() == 'branding':
lg.info("File %s is branding, not adding to project" %
(filepath, ))
return item
else:
lg.info("gnm_asset_category field empty so continuing")
lg.debug("Got filepath %s" % filepath)
fileid = db.fileId(server_path)
if fileid:
db.link_file_to_edit_project(fileid, vsproject)
lg.debug("Linked file %s with id %s to project with database ID %s" %
(filepath, fileid, vsproject))
else:
raise NotInDatabaseError
return item
def get(self, request, vs_item_id=None):
from django.conf import settings
from gnmvidispine.vs_item import VSItem, VSNotFound
from notification_handler import VIDISPINE_GRID_REF_FIELD
from notification_handler import vs_field_list
from pprint import pformat
#log.debug("Request data: {0}".format(pformat(request.__dict__)))
#log.debug("Request user: {0}".format(request.user))
item = VSItem(url=settings.VIDISPINE_URL,
port=settings.VIDISPINE_PORT,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD,
run_as=request.user.username)
try:
#log.debug("Looking up item {0}".format(vs_item_id))
item.populate(vs_item_id, specificFields=self.interesting_fields)
except VSNotFound as e:
return Response(
{
'status': 'error',
'problem': 'Item not found',
'exception': e
},
status=404)
meta = {}
for f in self.interesting_fields:
meta[f] = item.get(f, allowArray=True)
for f in vs_field_list():
meta[f] = item.get(f, allowArray=True)
gridrefs = item.get(VIDISPINE_GRID_REF_FIELD, allowArray=True)
if not isinstance(gridrefs, list):
gridrefs = [gridrefs]
meta[VIDISPINE_GRID_REF_FIELD] = gridrefs
return Response({
'status': 'success',
'item': vs_item_id,
'metadata': meta
})
def lookup_vidispine_item(credentials, item_id):
"""
Look up the containing collection in Vidispine, if the search in the Portal index fails
:param credentials: dictionary of Vidipsine credentials
:param item_id: item ID to look up
:return: String of the collection ID, or None if no collection was found.
"""
i = VSItem(host=credentials['host'],port=int(credentials['port']),user=credentials['user'],
passwd=credentials['password'])
try:
i.populate(item_id, specificFields=['title'])
except VSNotFound:
logger.error("Item {0} was not found in Vidispine".format(item_id))
return None
containers = i.get('__collection',allowArray=True)
if not isinstance(containers,list):
return [containers]
else:
return containers
def create_link_for_main(item_id, shape_tag, obfuscate=True, is_update=False):
"""
Task to trigger the optional transcoding and upload of an item
:param item_id:
:return:
"""
from gnmvidispine.vs_item import VSItem
from uuid import uuid4
from base64 import b64encode
from models import DownloadableLink
link_model = DownloadableLink.objects.get(item_id=item_id,
shapetag=shape_tag)
if link_model.status == "Upload Queued" or link_model.status == "Transcoding":
allow_transcode = False
else:
allow_transcode = True
if is_update and link_model.status == "Failed":
raise RuntimeError("Create link failed, requires manual retry")
item_ref = VSItem(url=settings.VIDISPINE_URL,
port=settings.VIDISPINE_PORT,
user=settings.VIDISPINE_USERNAME,
passwd=settings.VIDISPINE_PASSWORD)
item_ref.populate(item_id, specificFields=[
'title'
]) #this will raise VSNotFound if the item_id is invalid
try:
shaperef = get_shape_for(item_ref,
shape_tag,
allow_transcode=allow_transcode)
except TranscodeRequested as e:
link_model.status = "Transcoding"
link_model.transcode_job = e.jobid
link_model.save()
create_link_for.apply_async(
(item_id, shape_tag),
kwargs={
'obfuscate': obfuscate,
'is_update': True
},
countdown=getattr(settings, "DOWNLOADABLE_LINK_RETRYTIME", 30))
return "Transcode requested"
except NeedsRetry as e:
#if it's still not present, call ourselves again in up to 30s time.
create_link_for.apply_async(
(item_id, shape_tag),
kwargs={
'obfuscate': obfuscate,
'is_update': True
},
countdown=getattr(settings, "DOWNLOADABLE_LINK_RETRYTIME", 30))
return "Still waiting for transcode"
link_model.status = "Uploading"
link_model.save()
if obfuscate:
filename = make_filename(b64encode(uuid4().get_bytes()))
else:
filename = make_filename(item_ref.get('title'))
try:
s3key = upload_to_s3(shaperef, filename=filename)
s3key.set_canned_acl("public-read")
except NeedsRetry as e:
#if it's still not present, call ourselves again in up to 30s time.
create_link_for.apply_async(
(item_id, shape_tag),
kwargs={
'obfuscate': obfuscate,
'is_update': True
},
countdown=getattr(settings, "DOWNLOADABLE_LINK_RETRYTIME", 30))
return str(e)
link_model.status = "Available"
link_model.public_url = s3key.generate_url(expires_in=0, query_auth=False)
link_model.s3_url = "s3://{0}/{1}".format(
settings.DOWNLOADABLE_LINK_BUCKET, s3key.key)
link_model.save()
return "Media available at {0}".format(link_model.public_url)
def post(self, request):
from pprint import pformat
from gnmvidispine.vidispine_api import VSException
from gnmvidispine.vs_item import VSItem
from gnmvidispine.vs_collection import VSCollection
from gnmvidispine.vs_metadata import VSMetadata
from django.conf import settings
from portal.plugins.gnm_masters.models import VSMaster, MasterModel
from portal.plugins.gnm_projects.models import VSProject
from django.contrib.auth.models import User
import re
is_vsid = re.compile(r'^\w{2}-\d+')
try:
comm_id = request.POST['plutoconverter_commission_id_input']
proj_id = request.POST['plutoconverter_project_id_input']
item_id = request.POST['plutoconverter_item_id_input']
except StandardError as e:
return Response({'status': 'error', 'error': unicode(e)},status=400)
if not is_vsid.match(comm_id) or not is_vsid.match(proj_id) or not is_vsid.match(item_id):
return Response({'status': 'error', 'error': 'Not a valid Vidispine ID'},status=400)
try:
item = VSItem(user=settings.VIDISPINE_USERNAME, passwd=settings.VIDISPINE_PASSWORD, url=settings.VIDISPINE_URL)
item.populate(item_id)
if item.get('gnm_type') == 'master':
return Response({'status': 'error', 'error': '{0} is already a master'.format(item_id)},status=400)
project = VSCollection(user=settings.VIDISPINE_USERNAME, passwd=settings.VIDISPINE_PASSWORD, url=settings.VIDISPINE_URL)
project.populate(proj_id)
commission_id = project.get('__parent_collection')
logger.info("Project {0} belongs to commission {1}".format(project.name, commission_id))
commission = VSCollection(user=settings.VIDISPINE_USERNAME, passwd=settings.VIDISPINE_PASSWORD, url=settings.VIDISPINE_URL)
commission.populate(commission_id)
md_to_set = {
'gnm_commission_title' : commission.get('gnm_commission_title'),
'gnm_commission_workinggroup' : commission.get('gnm_commission_workinggroup'),
'gnm_project_headline' : project.get('gnm_project_headline'),
'gnm_master_website_headline' : item.get('title'),
'gnm_master_website_standfirst': item.get('gnm_asset_description'),
'gnm_master_website_byline' : item.get('gnm_asset_owner'),
# 'gnm_master_website_tags': breakout_tags(item.get('gnm_asset_user_keywords',allowArray=True),
# host=options.vshost,user=options.vsuser,passwd=options.vspasswd),
'gnm_type' : 'Master',
'gnm_master_language' : settings.LANGUAGE_CODE[0:2],
}
logger.info("Going to add {0} to project {1}".format(item_id,md_to_set))
project.addToCollection(item)
logger.info("Going to set metadata on {0}: {1}".format(item_id, md_to_set))
item.set_metadata(md_to_set)
logger.info("SUCCESS: item {0} has been converted to master".format(item_id))
admin_user = User.objects.get(username=settings.VIDISPINE_USERNAME)
vs_project = VSProject(proj_id, admin_user)
vs_master = VSMaster(item_id, admin_user)
vs_project.add_master(vs_master)
return Response({'status': 'success', 'itemid': item_id },status=200)
except VSException as e:
if raven_client is not None:
raven_client.captureException()
return Response({'status': 'error', 'error': "Vidispine said {0}".format(unicode(e))},status=500)
except StandardError as e:
if raven_client is not None:
raven_client.captureException()
return Response({'status': 'error', 'error': unicode(e)},status=500)
def profile_item(itemname):
"""
Calculate the times for the various waypoints to publishing this item. Assumes that the item has actually been published.
:param itemname: Vidispine ID for the item to profile
:return: Saved database model record
"""
from gnmvidispine.vs_item import VSItem
from django.conf import settings
from datetime import timedelta
import dateutil.parser
from models import OutputTimings
from pprint import pprint
print("Doing output time profile for item {0}".format(itemname))
item = VSItem(url=settings.VIDISPINE_URL,port=settings.VIDISPINE_PORT,
user=settings.VIDISPINE_USERNAME,passwd=settings.VIDISPINE_PASSWORD)
item.populate(itemname)
changeset_list = item.metadata_changeset_list()
try:
profile_record = OutputTimings.objects.get(item_id=itemname)
logger.warning(u"Item {0} ({1}) has already been profiled!".format(itemname,item.get('title')))
print(u"Item {0} ({1}) has already been profiled!".format(itemname,item.get('title')))
except OutputTimings.DoesNotExist:
profile_record = OutputTimings()
profile_record.item_id = item.name
profile_record.commission = item.get('gnm_commission_title')
profile_record.project = item.get('gnm_project_headline')
profile_record.item_duration = item.get('durationSeconds')
#step one - created time
profile_record.created_time = dateutil.parser.parse(item.get('created'))
print("Created time is {0}".format(profile_record.created_time))
print("portal_ingested is {0}".format(item.get('portal_ingested')))
#when was the media last added?
current_version = item.get('portal_essence_version')
#for the time being, ignoring updates; therefore we're looking for the creation of version 1.
subset = changesets_for_fieldname('portal_essence_version',changeset_list)
lastchange = first_change_from_set(subset,fieldname='portal_essence_version')#,value=str(1))
print("change of portal_essence_version is {0}".format(unicode(lastchange)))
if lastchange is not None:
profile_record.version_created_time = lastchange.timestamp
profile_record.media_version = int(current_version) #int(lastchange.value)
subset = changesets_for_fieldname('shapeTag',changeset_list)
#lastchange = last_change_from_set(subset)
#print("last change of shapeTag is {0}".format(unicode(lastchange)))
lastchange = change_for_value('lowres',subset)
print("change of shapeTag to have lowres is {0}".format(unicode(lastchange)))
interval = lastchange.timestamp - profile_record.version_created_time
print("Time interval is {0}".format(interval))
profile_record.proxy_completed_interval = timedelta_to_float(interval)
subset = changesets_for_fieldname('gnm_master_website_uploadstatus',changeset_list)
#print "got subset {0}".format(subset)
readychange = change_for_value('Ready to Upload',subset)
print("trigger change was {0}".format(unicode(readychange)))
profile_record.upload_trigger_interval=timedelta_to_float(readychange.timestamp - profile_record.version_created_time)
if profile_record.upload_trigger_interval<0:
profile_record.upload_trigger_interval = timedelta_to_float(readychange.timestamp - profile_record.created_time)
#page_created_interval
subset = changesets_for_fieldname('gnm_master_website_edit_url',changeset_list)
#lastchange = last_change_from_set(subset)
lastchange = first_change_from_set(subset,value=re.compile('.+'))
print("first non-empty change of gnm_master_website_edit_url is {0}".format(lastchange))
profile_record.page_created_interval=timedelta_to_float(lastchange.timestamp - profile_record.version_created_time)
#last transcode
subset = changesets_for_fieldname('gnm_master_website_uploadlog',changeset_list)
firstchange = first_change_from_set(subset)
log = UploadLog()
log.parse_upload_log(item.get('gnm_master_website_uploadlog'),assumed_date=firstchange.timestamp)
# print("There were {0} upload runs total:".format(log.upload_run_count))
# for n in range(0,log.upload_run_count):
# pprint(log.upload_run(n))
first_run=log.upload_run(log.upload_run_count-1)
while True:
print("last transcode of the first run was at {0} ({1})".format(first_run[0]['timestamp'], first_run[0]['text']))
profile_record.final_transcode_completed_interval = timedelta_to_float(first_run[0]['timestamp'] - profile_record.version_created_time)
if profile_record.final_transcode_completed_interval >0:
break
first_run[0]['timestamp']+=timedelta(days=1)
#lastchange = last_change_from_set(subset)
#print("last change of gnm_master_website_uploadlog is {0}".format(lastchange))
#profile_record.final_transcode_completed_interval = timedelta_to_float(lastchange.timestamp - profile_record.version_created_time)
#page launch guess
launchguess = dateutil.parser.parse(item.get('gnm_master_publication_time'))
profile_record.page_launch_guess_interval = timedelta_to_float(launchguess - profile_record.version_created_time)
#page launch from capi
capi_data = lookup_by_octid(item.get('gnm_master_generic_titleid'))
profile_record.page_launch_capi_interval = timedelta_to_float(capi_data['webPublicationDate'] - profile_record.version_created_time)
profile_record.page_launch_pluto_lag = timedelta_to_float(launchguess - capi_data['webPublicationDate'])
#last timestamp, for sorting.
#profile_record.completed_time = launchguess
profile_record.completed_time = capi_data['webPublicationDate']
pprint(profile_record.__dict__)
profile_record.save()
return profile_record
def process_premiere_project(filepath, db=None, cfg=None):
global vs_pathmap
lg.debug("---------------------------------")
lg.info("Premiere project: %s" % filepath)
collection_vsid = id_from_filepath(filepath)
lg.info("Project's Vidispine ID: %s" % collection_vsid)
vsproject = VSCollection(host=cfg.value('vs_host'),
port=cfg.value('vs_port'),
user=cfg.value('vs_user'),
passwd=cfg.value('vs_password'))
vsproject.setName(
collection_vsid
) #we don't need the actual metadata so don't bother getting it.
#pprint(vs_pathmap)
pp = PremiereProject()
try:
pp.load(filepath)
except Exception as e:
lg.error("Unable to read '%s': %s" % (filepath, e.message))
lg.error(traceback.format_exc())
print "Unable to read '%s': %s" % (filepath, e.message)
traceback.print_exc()
raven_client.captureException()
return (0, 0, 0)
lg.debug("determining project details and updating database...")
try:
projectDetails = pp.getParticulars()
project_id = db.upsert_edit_project(
os.path.dirname(filepath),
os.path.basename(filepath),
projectDetails['uuid'],
projectDetails['version'],
desc="Adobe Premiere Project",
opens_with=
"/Applications/Adobe Premiere Pro CC 2014/Adobe Premiere Pro CC 2014.app/Contents/MacOS/Adobe Premiere Pro CC 2014"
)
except ValueError as e:
project_id = db.log_project_issue(os.path.dirname(filepath),
os.path.basename(filepath),
problem="Invalid project file",
detail="{0}: {1} {2}".format(
e.__class__, str(e),
traceback.format_exc()))
lg.error("Unable to read project file '{0}' - {1}".format(
filepath, str(e)))
lg.error(traceback.format_exc())
raven_client.captureException()
return (0, 0, 0)
except KeyError as e:
project_id = db.log_project_issue(os.path.dirname(filepath),
os.path.basename(filepath),
problem="Invalid project file",
detail="{0}: {1} {2}".format(
e.__class__, str(e),
traceback.format_exc()))
db.insert_sysparam(
"warning",
"Unable to read project file '{0}' - {1}".format(filepath, str(e)))
lg.error("Unable to read project file '{0}' - {1}".format(
filepath, str(e)))
lg.error(traceback.format_exc())
raven_client.captureException()
return (0, 0, 0)
except InvalidDataError as e:
db.insert_sysparam(
"warning",
"Corrupted project file: {0} {1}".format(filepath, unicode(e)))
raven_client.captureException()
return (0, 0, 0)
total_files = 0
no_vsitem = 0
not_in_db = 0
lg.debug("looking for referenced media....")
for filepath in pp.getReferencedMedia():
total_files += 1
server_path = re.sub(u'^/Volumes', '/srv', filepath).encode('utf-8')
vsid = db.get_vidispine_id(server_path)
# this call will return None if the file is not from an asset folder, e.g. newswire
if vsid is None:
try:
vsid = find_item_id_for_path(server_path)
if vsid is None:
lg.error(
"File {0} is found by Vidispine but has not been imported yet"
.format(server_path))
continue
except VSNotFound:
lg.error(
"File {0} could not be found in either Vidispine or the asset importer database"
.format(server_path))
continue
item = VSItem(host=cfg.value('vs_host'),
port=cfg.value('vs_port'),
user=cfg.value('vs_user'),
passwd=cfg.value('vs_password'))
item.populate(vsid, specificFields=['gnm_asset_category'])
try:
if item.get('gnm_asset_category').lower() == 'branding':
lg.info("File %s is branding, not adding to project" %
(filepath, ))
continue
except AttributeError:
lg.warning("File %s has no value for gnm_asset_catetory" %
(filepath, ))
lg.debug("Got filepath %s" % filepath)
fileid = db.fileId(server_path)
if fileid:
try:
db.link_file_to_edit_project(fileid, project_id)
lg.debug("Linked file %s with id %s to project %s" %
(filepath, fileid, project_id))
except AlreadyLinkedError:
lg.info("File %s with id %s is already linked to project %s" %
(filepath, fileid, project_id))
continue
else:
not_in_db += 1
lg.warning("File %s could not be found in the database" % filepath)
continue
n = 0
found = False
#using this construct to avoid loading more data from VS than necessary. We simply check whether the ID exists
#in the parent collections list (cached on the item record) without lifting any more info out of VS
if vsproject.name in map(
lambda x: x.name,
item.parent_collections(shouldPopulate=False)):
lg.info("File %s is already in project %s" %
(filepath, vsproject.name))
continue
vsproject.addToCollection(
item=item
) #this call will apparently succeed if the item is already added to said collection, but it won't be added twice.
lg.info(
"Run complete. Out of a total of %d referenced files, %d did not have a Vidispine item and %d were not in the Asset Importer database"
% (total_files, no_vsitem, not_in_db))
return (total_files, no_vsitem, not_in_db)