def test_find_valid_projectid(self):
from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread
from asset_folder_importer.database import importer_db
with mock.patch('psycopg2.connect') as mock_connect:
db = importer_db("_test_Version_",
username="******",
password="******")
with mock.patch('httplib.HTTPConnection') as mock_connection:
logging.basicConfig(level=logging.ERROR)
logger = logging.getLogger("tester")
logger.setLevel(logging.ERROR)
i = ImporterThread(None,
None,
self.FakeConfig({
'footage_providers_config':
'{0}/../../footage_providers.yml'.format(
self.mydir)
}),
dbconn=db)
mock_connection.side_effect = lambda h, c: self.FakeConnection(
json.dumps({'status': 'notfound'}), 404)
result = i.ask_pluto_for_projectid(
"/path/to/my/assetfolder/with/subdirectories/media.mxf")
self.assertEqual(result, 'KP-1234')
result = i.ask_pluto_for_projectid(
"/path/to/my/assetfolder/media.mxf")
self.assertEqual(result, 'KP-1234')
def test_vs_inconsistency_error(self):
from gnmvidispine.vs_storage import VSStorage, VSFile
from gnmvidispine.vidispine_api import VSNotFound, HTTPError
from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread
from asset_folder_importer.asset_folder_vsingester.exceptions import VSFileInconsistencyError
from asset_folder_importer.database import importer_db
with mock.patch('psycopg2.connect') as mock_connect:
db = importer_db("_test_Version_", username="******", password="******")
mockstorage = mock.MagicMock(target=VSStorage)
mockstorage.fileForPath = mock.MagicMock(side_effect=VSNotFound)
mockstorage.create_file_entity = mock.MagicMock(side_effect=HTTPError(503,"GET","http://fake-url","Failed","I didn't expect the Spanish Inqusition",""))
mockfile = mock.MagicMock(target=VSFile)
with mock.patch('httplib.HTTPConnection') as mock_connection:
logging.basicConfig(level=logging.ERROR)
logger = logging.getLogger("tester")
logger.setLevel(logging.ERROR)
i = ImporterThread(None, None,
self.FakeConfig({
'footage_providers_config': '{0}/../../footage_providers.yml'.format(self.mydir)
}), dbconn=db)
i.st = mockstorage
with self.assertRaises(VSFileInconsistencyError) as raised_error:
i.attempt_file_import(mockfile,"path/to/testfile","/rootpath")
self.assertEqual(str(raised_error.exception),"path/to/testfile")
def test_check_mime(self):
from asset_folder_importer.asset_folder_sweeper.find_files import check_mime
from asset_folder_importer.database import importer_db
__version__ = 'database_test'
db = importer_db(__version__,
hostname=self.dbhost,
port=self.dbport,
username=self.dbuser,
password=self.dbpass,
dbname=self.dbname)
filepath = "/etc/hosts"
(statinfo, mimetype) = check_mime(filepath, db)
self.assertEqual(mimetype, "text/plain")
self.assertNotEqual(statinfo, None)
filepath = os.path.join(os.path.dirname(os.path.realpath(__file__)),
"1.jpeg")
(statinfo, mimetype) = check_mime(filepath, db)
self.assertEqual(mimetype, "image/jpeg")
def test__has_table(self):
__version__ = 'database_test'
db = importer_db(__version__, hostname=self.dbhost, port=self.dbport, username=self.dbuser,
password=self.dbpass, dbname=self.dbname)
self.assertEqual(db._has_table('system'),True)
self.assertEqual(db._has_table('files'),True)
self.assertEqual(db._has_table('hamster'),False)
def test__has_table(self):
__version__ = 'database_test'
db = importer_db(__version__,
hostname=self.dbhost,
port=self.dbport,
username=self.dbuser,
password=self.dbpass,
dbname=self.dbname)
self.assertEqual(db._has_table('system'), True)
self.assertEqual(db._has_table('files'), True)
self.assertEqual(db._has_table('hamster'), False)
def __init__(self, q, storageid, cfg, permission_script="/invalid/permissionscript",
graveyard_folder="/tmp", timeout=60, import_timeout=3600, close_file_timeout=300, logger=None, dbconn=None):
"""
Initialises a new ImporterThread
:param q: job queue to work from
:param storageid: Storage ID to import from
:param cfg: configuration object from config file and commandline
:param permission_script: location of the suid change-permissions script
:param graveyard_folder: graveyard folder to move invalid jobs to (defaults to /tmp)
:param timeout: time to wait for API calls to complete (default: 60s; this is also the loadbalancer timeout)
:param import_timeout: time to wait for a Vidispine import job before cancelling it as it is stalled (default: 1 hour)
:param close_file_timeout: time to wait on processing a job before sending vidispine a close-file request (see https://support.vidispine.com/support/tickets/1360) (default: 5mins)
:param logger: logger object to log to. For testing purposes, really; if None then a new logger will be created. (default: None)
:param dbconn: database connection. For testing purposes really; if None then a new database connection will be created (default: None)
"""
super(ImporterThread, self).__init__()
self.templateEnv = Environment(loader=PackageLoader('asset_folder_importer', 'metadata_templates'))
self.mdTemplate = self.templateEnv.get_template('vsasset.xml')
self.queue = q
self.st = VSStorage(host=cfg.value('vs_host'), port=cfg.value('vs_port'), user=cfg.value('vs_user'),
passwd=cfg.value('vs_password'))
if storageid is not None:
self.st.populate(storageid)
self.db = dbconn if dbconn is not None else importer_db(__version__, hostname=cfg.value('database_host'),
port=cfg.value('database_port'),
username=cfg.value('database_user'),
password=cfg.value('database_password'))
self.found = 0
self.withItems = 0
self.imported = 0
self.cfg = cfg
self.ignored = 0
self._permissionscript = permission_script
self.logger = logger if logger is not None else logging.getLogger("importer_thread")
self._timeout = timeout
self.graveyard_folder=graveyard_folder
self._importer_timeout = import_timeout
self._close_file_timeout = close_file_timeout
try:
providers_config_file = self.cfg.value('footage_providers_config', noraise=False)
except KeyError:
providers_config_file = '/etc/asset_folder_importer/footage_providers.yml'
self.providers_list = externalprovider.ExternalProviderList(providers_config_file)
def test_fileid_sizes(self):
"""
test that the database module doesn't throw exceptions on different sized id numbers
:return:
"""
small_file_id=379311582
with mock.patch('psycopg2.connect') as mock_connect:
db = importer_db("_test_Version_", username="******", password="******")
db.update_file_vidispine_id(small_file_id,'VX-1234')
db.update_file_ignore(small_file_id,True)
db.update_prelude_clip_fileref(1234,small_file_id)
large_file_id = 37931158242397423792L
db.update_file_vidispine_id(large_file_id, 'VX-1234')
db.update_file_ignore(large_file_id,True)
db.update_prelude_clip_fileref(1234, large_file_id)
def test_find_invalid_projectid(self):
from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread
from asset_folder_importer.database import importer_db
with mock.patch('psycopg2.connect') as mock_connect:
db = importer_db("_test_Version_",username="******",password="******")
with mock.patch('httplib.HTTPConnection') as mock_connection:
logging.basicConfig(level=logging.ERROR)
logger = logging.getLogger("tester")
logger.setLevel(logging.ERROR)
i = ImporterThread(None, None,
self.FakeConfig({
'footage_providers_config': '{0}/../../footage_providers.yml'.format(self.mydir)
}), dbconn=db)
mock_connection.side_effect = lambda h,c: self.FakeConnection(json.dumps({'status': 'notfound'}),404)
result = i.ask_pluto_for_projectid("/path/to/something/invalid/media.mxf")
self.assertEqual(result,None)
def test_import_tags(self):
from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread
from asset_folder_importer.database import importer_db
with mock.patch('psycopg2.connect') as mock_connect:
db = importer_db("_test_Version_",
username="******",
password="******")
i = ImporterThread(None,
None,
self.FakeConfig({
'footage_providers_config':
'{0}/../../footage_providers.yml'.format(
self.mydir)
}),
dbconn=db)
self.assertEqual(i.import_tags_for_fileref({'mime_type': 'video/mp4'}),
['lowres'])
self.assertEqual(
i.import_tags_for_fileref({'mime_type': 'video/quicktime'}),
['lowres'])
self.assertEqual(
i.import_tags_for_fileref({'mime_type': 'application/mxf'}),
['lowres'])
self.assertEqual(
i.import_tags_for_fileref({'mime_type': 'model/vnd.mts'}),
['lowres'])
self.assertEqual(
i.import_tags_for_fileref({'mime_type': 'image/jpeg'}),
['lowimage'])
self.assertEqual(
i.import_tags_for_fileref({'mime_type': 'image/tiff'}),
['lowimage'])
self.assertEqual(
i.import_tags_for_fileref({'mime_type': 'audio/aiff'}),
['lowaudio'])
self.assertEqual(i.import_tags_for_fileref({'mime_type': 'audio/wav'}),
['lowaudio'])
self.assertEqual(
i.import_tags_for_fileref({'mime_type': 'application/xml'}), None)
def test_potential_sidecar_filenames(self):
from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread
from asset_folder_importer.database import importer_db
with mock.patch('psycopg2.connect') as mock_connect:
db = importer_db("_test_Version_",
username="******",
password="******")
i = ImporterThread(None,
None,
self.FakeConfig({
'footage_providers_config':
'{0}/../../footage_providers.yml'.format(
self.mydir)
}),
dbconn=db)
result = map(
lambda x: x,
i.potentialSidecarFilenames("/path/to/myfile.mp4", isxdcam=False))
self.assertEqual(result, [
'/path/to/myfile.xml', '/path/to/myfile.xmp',
'/path/to/myfile.meta', '/path/to/myfile.XML',
'/path/to/myfile.XMP', '/path/to/myfile.mp4.xml',
'/path/to/myfile.mp4.xmp', '/path/to/myfile.mp4.meta',
'/path/to/myfile.mp4.XML', '/path/to/myfile.mp4.XMP'
])
result = map(
lambda x: x,
i.potentialSidecarFilenames("/path/to/myfile.mp4", isxdcam=True))
self.assertEqual(result, [
'/path/to/myfile.xml', '/path/to/myfile.xmp',
'/path/to/myfile.meta', '/path/to/myfile.XML',
'/path/to/myfile.XMP', '/path/to/myfile.mp4.xml',
'/path/to/myfile.mp4.xmp', '/path/to/myfile.mp4.meta',
'/path/to/myfile.mp4.XML', '/path/to/myfile.mp4.XMP',
'/path/to/myfileM01.xml', '/path/to/myfileM01.xmp',
'/path/to/myfileM01.meta', '/path/to/myfileM01.XML',
'/path/to/myfileM01.XMP'
])
def test_import_tags(self):
from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread
from asset_folder_importer.database import importer_db
with mock.patch('psycopg2.connect') as mock_connect:
db = importer_db("_test_Version_",username="******",password="******")
i = ImporterThread(None,None,
self.FakeConfig({
'footage_providers_config': '{0}/../../footage_providers.yml'.format(self.mydir)
}),dbconn=db)
self.assertEqual(i.import_tags_for_fileref({'mime_type': 'video/mp4'}),['lowres'])
self.assertEqual(i.import_tags_for_fileref({'mime_type': 'video/quicktime'}), ['lowres'])
self.assertEqual(i.import_tags_for_fileref({'mime_type': 'application/mxf'}), ['lowres'])
self.assertEqual(i.import_tags_for_fileref({'mime_type': 'model/vnd.mts'}),['lowres'])
self.assertEqual(i.import_tags_for_fileref({'mime_type': 'image/jpeg'}), ['lowimage'])
self.assertEqual(i.import_tags_for_fileref({'mime_type': 'image/tiff'}), ['lowimage'])
self.assertEqual(i.import_tags_for_fileref({'mime_type': 'audio/aiff'}), ['lowaudio'])
self.assertEqual(i.import_tags_for_fileref({'mime_type': 'audio/wav'}), ['lowaudio'])
self.assertEqual(i.import_tags_for_fileref({'mime_type': 'application/xml'}),None)
def test_check_mime(self):
from asset_folder_importer.asset_folder_sweeper.find_files import check_mime
from asset_folder_importer.database import importer_db
__version__ = 'database_test'
db = importer_db(__version__,hostname=self.dbhost,port=self.dbport,username=self.dbuser,
password=self.dbpass,dbname=self.dbname)
filepath = "/etc/hosts"
(statinfo, mimetype) = check_mime(filepath, db)
self.assertEqual(mimetype, "text/plain")
self.assertNotEqual(statinfo, None)
filepath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "1.jpeg")
(statinfo, mimetype) = check_mime(filepath, db)
self.assertEqual(mimetype, "image/jpeg")
def test_timeout_retries(self):
from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread, ImportStalled
from asset_folder_importer.database import importer_db
from time import time
with mock.patch('psycopg2.connect') as mock_connect:
db = importer_db("_test_Version_", username="******", password="******")
with mock.patch('httplib.HTTPConnection') as mock_connection:
fake_job = self.StalledJob()
mock_vsfile = mock.MagicMock(target=gnmvidispine.vs_storage.VSFile)
mock_vsfile.importToItem = mock.MagicMock(return_value=fake_job)
i=ImporterThread(None,None,self.FakeConfig({
'footage_providers_config': '{0}/../../footage_providers.yml'.format(self.mydir)
}),dbconn=db,import_timeout=4) #set importer timeout to 4s
start_time = time()
with self.assertRaises(ImportStalled):
i.do_real_import(mock_vsfile,"/path/to/filename","fake_xml",['tagone'])
self.assertGreaterEqual(time()-start_time,4) #should have taken at least 4 seconds
mock_vsfile.importToItem.assert_called_once_with("fake_xml",tags=['tagone'],priority="LOW",jobMetadata={'gnm_app': 'vsingester'})
fake_job.abort.assert_called_once_with()
def test_potential_sidecar_filenames(self):
from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread
from asset_folder_importer.database import importer_db
with mock.patch('psycopg2.connect') as mock_connect:
db = importer_db("_test_Version_",username="******",password="******")
i = ImporterThread(None,None,
self.FakeConfig({
'footage_providers_config': '{0}/../../footage_providers.yml'.format(self.mydir)
}),dbconn=db)
result = map(lambda x: x, i.potentialSidecarFilenames("/path/to/myfile.mp4", isxdcam=False))
self.assertEqual(result,['/path/to/myfile.xml', '/path/to/myfile.xmp', '/path/to/myfile.meta', '/path/to/myfile.XML',
'/path/to/myfile.XMP','/path/to/myfile.mp4.xml','/path/to/myfile.mp4.xmp','/path/to/myfile.mp4.meta',
'/path/to/myfile.mp4.XML','/path/to/myfile.mp4.XMP'])
result = map(lambda x: x, i.potentialSidecarFilenames("/path/to/myfile.mp4", isxdcam=True))
self.assertEqual(result,['/path/to/myfile.xml', '/path/to/myfile.xmp', '/path/to/myfile.meta',
'/path/to/myfile.XML', '/path/to/myfile.XMP', '/path/to/myfile.mp4.xml',
'/path/to/myfile.mp4.xmp', '/path/to/myfile.mp4.meta', '/path/to/myfile.mp4.XML',
'/path/to/myfile.mp4.XMP', '/path/to/myfileM01.xml', '/path/to/myfileM01.xmp',
'/path/to/myfileM01.meta', '/path/to/myfileM01.XML', '/path/to/myfileM01.XMP'])
def test_timeout_retries(self):
from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread, ImportStalled
from asset_folder_importer.database import importer_db
from time import time
with mock.patch('psycopg2.connect') as mock_connect:
db = importer_db("_test_Version_",
username="******",
password="******")
with mock.patch('httplib.HTTPConnection') as mock_connection:
fake_job = self.StalledJob()
mock_vsfile = mock.MagicMock(target=gnmvidispine.vs_storage.VSFile)
mock_vsfile.importToItem = mock.MagicMock(return_value=fake_job)
i = ImporterThread(None,
None,
self.FakeConfig({
'footage_providers_config':
'{0}/../../footage_providers.yml'.format(
self.mydir)
}),
dbconn=db,
import_timeout=4) #set importer timeout to 4s
start_time = time()
with self.assertRaises(ImportStalled):
i.do_real_import(mock_vsfile, "/path/to/filename", "fake_xml",
['tagone'])
self.assertGreaterEqual(time() - start_time,
4) #should have taken at least 4 seconds
mock_vsfile.importToItem.assert_called_once_with(
"fake_xml",
tags=['tagone'],
priority="LOW",
jobMetadata={'gnm_app': 'vsingester'})
fake_job.abort.assert_called_once_with()
def test_vs_inconsistency_error(self):
from gnmvidispine.vs_storage import VSStorage, VSFile
from gnmvidispine.vidispine_api import VSNotFound, HTTPError
from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread
from asset_folder_importer.asset_folder_vsingester.exceptions import VSFileInconsistencyError
from asset_folder_importer.database import importer_db
with mock.patch('psycopg2.connect') as mock_connect:
db = importer_db("_test_Version_",
username="******",
password="******")
mockstorage = mock.MagicMock(target=VSStorage)
mockstorage.fileForPath = mock.MagicMock(side_effect=VSNotFound)
mockstorage.create_file_entity = mock.MagicMock(side_effect=HTTPError(
503, "GET", "http://fake-url", "Failed",
"I didn't expect the Spanish Inqusition", ""))
mockfile = mock.MagicMock(target=VSFile)
with mock.patch('httplib.HTTPConnection') as mock_connection:
logging.basicConfig(level=logging.ERROR)
logger = logging.getLogger("tester")
logger.setLevel(logging.ERROR)
i = ImporterThread(None,
None,
self.FakeConfig({
'footage_providers_config':
'{0}/../../footage_providers.yml'.format(
self.mydir)
}),
dbconn=db)
i.st = mockstorage
with self.assertRaises(VSFileInconsistencyError) as raised_error:
i.attempt_file_import(mockfile, "path/to/testfile",
"/rootpath")
self.assertEqual(str(raised_error.exception), "path/to/testfile")
if options.configfile:
cfg=configfile(options.configfile)
else:
cfg=configfile("/etc/asset_folder_importer.cfg")
#Step three. Set up pools.
pool = ThreadPool(UpdateVsThread,initial_size=int(options.threads), config=cfg)
raven_client = raven.Client(dsn=cfg.value("sentry_dsn"))
#Step four. Scan the database table and update VS
#Now connect to db
logger.info("Connecting to database on %s" % cfg.value('database_host',noraise=True))
try:
db = importer_db(__version__,hostname=cfg.value('database_host'),port=cfg.value('database_port'),username=cfg.value('database_user'),password=cfg.value('database_password'))
db.check_schema_22()
lastruntime = db.lastrun_endtime()
lastruntimestamp = 0
except Exception as e:
raven_client.captureException()
raise
n=0
c=0
for fileref in db.deleted_files():
n+=1
if fileref['ignore']:
continue
c+=1
if c>100:
def __init__(self,
q,
storageid,
cfg,
permission_script="/invalid/permissionscript",
graveyard_folder="/tmp",
timeout=60,
import_timeout=3600,
close_file_timeout=300,
logger=None,
dbconn=None):
"""
Initialises a new ImporterThread
:param q: job queue to work from
:param storageid: Storage ID to import from
:param cfg: configuration object from config file and commandline
:param permission_script: location of the suid change-permissions script
:param graveyard_folder: graveyard folder to move invalid jobs to (defaults to /tmp)
:param timeout: time to wait for API calls to complete (default: 60s; this is also the loadbalancer timeout)
:param import_timeout: time to wait for a Vidispine import job before cancelling it as it is stalled (default: 1 hour)
:param close_file_timeout: time to wait on processing a job before sending vidispine a close-file request (see https://support.vidispine.com/support/tickets/1360) (default: 5mins)
:param logger: logger object to log to. For testing purposes, really; if None then a new logger will be created. (default: None)
:param dbconn: database connection. For testing purposes really; if None then a new database connection will be created (default: None)
"""
super(ImporterThread, self).__init__()
self.templateEnv = Environment(loader=PackageLoader(
'asset_folder_importer', 'metadata_templates'))
self.mdTemplate = self.templateEnv.get_template('vsasset.xml')
self.queue = q
self.st = VSStorage(host=cfg.value('vs_host'),
port=cfg.value('vs_port'),
user=cfg.value('vs_user'),
passwd=cfg.value('vs_password'))
if storageid is not None:
self.st.populate(storageid)
self.db = dbconn if dbconn is not None else importer_db(
__version__,
hostname=cfg.value('database_host'),
port=cfg.value('database_port'),
username=cfg.value('database_user'),
password=cfg.value('database_password'))
self.found = 0
self.withItems = 0
self.imported = 0
self.cfg = cfg
self.ignored = 0
self._permissionscript = permission_script
self.logger = logger if logger is not None else logging.getLogger(
"importer_thread")
self._timeout = timeout
self.graveyard_folder = graveyard_folder
self._importer_timeout = import_timeout
self._close_file_timeout = close_file_timeout
try:
providers_config_file = self.cfg.value('footage_providers_config',
noraise=False)
except KeyError:
providers_config_file = '/etc/asset_folder_importer/footage_providers.yml'
self.providers_list = externalprovider.ExternalProviderList(
providers_config_file)
cfg = configfile("/etc/asset_folder_importer.cfg")
if logfile is not None:
logging.basicConfig(filename=logfile,
format=LOGFORMAT,
level=main_log_level)
else:
logging.basicConfig(format=LOGFORMAT, level=main_log_level)
logging.info("-----------------------------------------------------------\n\n")
logging.info("Connecting to database on %s" %
cfg.value('database_host', noraise=True))
db = importer_db(__version__,
hostname=cfg.value('database_host'),
port=cfg.value('database_port'),
username=cfg.value('database_user'),
password=cfg.value('database_password'))
lastruntime = db.lastrun_endtime()
lastruntimestamp = 0
if lastruntime is None:
logging.error(
"It appears that another instance of premiere_get_referenced_media is already running."
)
if not options.force:
logging.error(
"Not continuing because --force is not specified on the commandline"
)
db.end_run("already_running")
