def test_find_valid_projectid(self): from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread from asset_folder_importer.database import importer_db with mock.patch('psycopg2.connect') as mock_connect: db = importer_db("_test_Version_", username="******", password="******") with mock.patch('httplib.HTTPConnection') as mock_connection: logging.basicConfig(level=logging.ERROR) logger = logging.getLogger("tester") logger.setLevel(logging.ERROR) i = ImporterThread(None, None, self.FakeConfig({ 'footage_providers_config': '{0}/../../footage_providers.yml'.format( self.mydir) }), dbconn=db) mock_connection.side_effect = lambda h, c: self.FakeConnection( json.dumps({'status': 'notfound'}), 404) result = i.ask_pluto_for_projectid( "/path/to/my/assetfolder/with/subdirectories/media.mxf") self.assertEqual(result, 'KP-1234') result = i.ask_pluto_for_projectid( "/path/to/my/assetfolder/media.mxf") self.assertEqual(result, 'KP-1234')
def test_vs_inconsistency_error(self): from gnmvidispine.vs_storage import VSStorage, VSFile from gnmvidispine.vidispine_api import VSNotFound, HTTPError from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread from asset_folder_importer.asset_folder_vsingester.exceptions import VSFileInconsistencyError from asset_folder_importer.database import importer_db with mock.patch('psycopg2.connect') as mock_connect: db = importer_db("_test_Version_", username="******", password="******") mockstorage = mock.MagicMock(target=VSStorage) mockstorage.fileForPath = mock.MagicMock(side_effect=VSNotFound) mockstorage.create_file_entity = mock.MagicMock(side_effect=HTTPError(503,"GET","http://fake-url","Failed","I didn't expect the Spanish Inqusition","")) mockfile = mock.MagicMock(target=VSFile) with mock.patch('httplib.HTTPConnection') as mock_connection: logging.basicConfig(level=logging.ERROR) logger = logging.getLogger("tester") logger.setLevel(logging.ERROR) i = ImporterThread(None, None, self.FakeConfig({ 'footage_providers_config': '{0}/../../footage_providers.yml'.format(self.mydir) }), dbconn=db) i.st = mockstorage with self.assertRaises(VSFileInconsistencyError) as raised_error: i.attempt_file_import(mockfile,"path/to/testfile","/rootpath") self.assertEqual(str(raised_error.exception),"path/to/testfile")
def test_check_mime(self): from asset_folder_importer.asset_folder_sweeper.find_files import check_mime from asset_folder_importer.database import importer_db __version__ = 'database_test' db = importer_db(__version__, hostname=self.dbhost, port=self.dbport, username=self.dbuser, password=self.dbpass, dbname=self.dbname) filepath = "/etc/hosts" (statinfo, mimetype) = check_mime(filepath, db) self.assertEqual(mimetype, "text/plain") self.assertNotEqual(statinfo, None) filepath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "1.jpeg") (statinfo, mimetype) = check_mime(filepath, db) self.assertEqual(mimetype, "image/jpeg")
def test__has_table(self): __version__ = 'database_test' db = importer_db(__version__, hostname=self.dbhost, port=self.dbport, username=self.dbuser, password=self.dbpass, dbname=self.dbname) self.assertEqual(db._has_table('system'),True) self.assertEqual(db._has_table('files'),True) self.assertEqual(db._has_table('hamster'),False)
def test__has_table(self): __version__ = 'database_test' db = importer_db(__version__, hostname=self.dbhost, port=self.dbport, username=self.dbuser, password=self.dbpass, dbname=self.dbname) self.assertEqual(db._has_table('system'), True) self.assertEqual(db._has_table('files'), True) self.assertEqual(db._has_table('hamster'), False)
def __init__(self, q, storageid, cfg, permission_script="/invalid/permissionscript", graveyard_folder="/tmp", timeout=60, import_timeout=3600, close_file_timeout=300, logger=None, dbconn=None): """ Initialises a new ImporterThread :param q: job queue to work from :param storageid: Storage ID to import from :param cfg: configuration object from config file and commandline :param permission_script: location of the suid change-permissions script :param graveyard_folder: graveyard folder to move invalid jobs to (defaults to /tmp) :param timeout: time to wait for API calls to complete (default: 60s; this is also the loadbalancer timeout) :param import_timeout: time to wait for a Vidispine import job before cancelling it as it is stalled (default: 1 hour) :param close_file_timeout: time to wait on processing a job before sending vidispine a close-file request (see https://support.vidispine.com/support/tickets/1360) (default: 5mins) :param logger: logger object to log to. For testing purposes, really; if None then a new logger will be created. (default: None) :param dbconn: database connection. For testing purposes really; if None then a new database connection will be created (default: None) """ super(ImporterThread, self).__init__() self.templateEnv = Environment(loader=PackageLoader('asset_folder_importer', 'metadata_templates')) self.mdTemplate = self.templateEnv.get_template('vsasset.xml') self.queue = q self.st = VSStorage(host=cfg.value('vs_host'), port=cfg.value('vs_port'), user=cfg.value('vs_user'), passwd=cfg.value('vs_password')) if storageid is not None: self.st.populate(storageid) self.db = dbconn if dbconn is not None else importer_db(__version__, hostname=cfg.value('database_host'), port=cfg.value('database_port'), username=cfg.value('database_user'), password=cfg.value('database_password')) self.found = 0 self.withItems = 0 self.imported = 0 self.cfg = cfg self.ignored = 0 self._permissionscript = permission_script self.logger = logger if logger is not None else logging.getLogger("importer_thread") self._timeout = timeout self.graveyard_folder=graveyard_folder self._importer_timeout = import_timeout self._close_file_timeout = close_file_timeout try: providers_config_file = self.cfg.value('footage_providers_config', noraise=False) except KeyError: providers_config_file = '/etc/asset_folder_importer/footage_providers.yml' self.providers_list = externalprovider.ExternalProviderList(providers_config_file)
def test_fileid_sizes(self): """ test that the database module doesn't throw exceptions on different sized id numbers :return: """ small_file_id=379311582 with mock.patch('psycopg2.connect') as mock_connect: db = importer_db("_test_Version_", username="******", password="******") db.update_file_vidispine_id(small_file_id,'VX-1234') db.update_file_ignore(small_file_id,True) db.update_prelude_clip_fileref(1234,small_file_id) large_file_id = 37931158242397423792L db.update_file_vidispine_id(large_file_id, 'VX-1234') db.update_file_ignore(large_file_id,True) db.update_prelude_clip_fileref(1234, large_file_id)
def test_find_invalid_projectid(self): from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread from asset_folder_importer.database import importer_db with mock.patch('psycopg2.connect') as mock_connect: db = importer_db("_test_Version_",username="******",password="******") with mock.patch('httplib.HTTPConnection') as mock_connection: logging.basicConfig(level=logging.ERROR) logger = logging.getLogger("tester") logger.setLevel(logging.ERROR) i = ImporterThread(None, None, self.FakeConfig({ 'footage_providers_config': '{0}/../../footage_providers.yml'.format(self.mydir) }), dbconn=db) mock_connection.side_effect = lambda h,c: self.FakeConnection(json.dumps({'status': 'notfound'}),404) result = i.ask_pluto_for_projectid("/path/to/something/invalid/media.mxf") self.assertEqual(result,None)
def test_import_tags(self): from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread from asset_folder_importer.database import importer_db with mock.patch('psycopg2.connect') as mock_connect: db = importer_db("_test_Version_", username="******", password="******") i = ImporterThread(None, None, self.FakeConfig({ 'footage_providers_config': '{0}/../../footage_providers.yml'.format( self.mydir) }), dbconn=db) self.assertEqual(i.import_tags_for_fileref({'mime_type': 'video/mp4'}), ['lowres']) self.assertEqual( i.import_tags_for_fileref({'mime_type': 'video/quicktime'}), ['lowres']) self.assertEqual( i.import_tags_for_fileref({'mime_type': 'application/mxf'}), ['lowres']) self.assertEqual( i.import_tags_for_fileref({'mime_type': 'model/vnd.mts'}), ['lowres']) self.assertEqual( i.import_tags_for_fileref({'mime_type': 'image/jpeg'}), ['lowimage']) self.assertEqual( i.import_tags_for_fileref({'mime_type': 'image/tiff'}), ['lowimage']) self.assertEqual( i.import_tags_for_fileref({'mime_type': 'audio/aiff'}), ['lowaudio']) self.assertEqual(i.import_tags_for_fileref({'mime_type': 'audio/wav'}), ['lowaudio']) self.assertEqual( i.import_tags_for_fileref({'mime_type': 'application/xml'}), None)
def test_potential_sidecar_filenames(self): from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread from asset_folder_importer.database import importer_db with mock.patch('psycopg2.connect') as mock_connect: db = importer_db("_test_Version_", username="******", password="******") i = ImporterThread(None, None, self.FakeConfig({ 'footage_providers_config': '{0}/../../footage_providers.yml'.format( self.mydir) }), dbconn=db) result = map( lambda x: x, i.potentialSidecarFilenames("/path/to/myfile.mp4", isxdcam=False)) self.assertEqual(result, [ '/path/to/myfile.xml', '/path/to/myfile.xmp', '/path/to/myfile.meta', '/path/to/myfile.XML', '/path/to/myfile.XMP', '/path/to/myfile.mp4.xml', '/path/to/myfile.mp4.xmp', '/path/to/myfile.mp4.meta', '/path/to/myfile.mp4.XML', '/path/to/myfile.mp4.XMP' ]) result = map( lambda x: x, i.potentialSidecarFilenames("/path/to/myfile.mp4", isxdcam=True)) self.assertEqual(result, [ '/path/to/myfile.xml', '/path/to/myfile.xmp', '/path/to/myfile.meta', '/path/to/myfile.XML', '/path/to/myfile.XMP', '/path/to/myfile.mp4.xml', '/path/to/myfile.mp4.xmp', '/path/to/myfile.mp4.meta', '/path/to/myfile.mp4.XML', '/path/to/myfile.mp4.XMP', '/path/to/myfileM01.xml', '/path/to/myfileM01.xmp', '/path/to/myfileM01.meta', '/path/to/myfileM01.XML', '/path/to/myfileM01.XMP' ])
def test_import_tags(self): from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread from asset_folder_importer.database import importer_db with mock.patch('psycopg2.connect') as mock_connect: db = importer_db("_test_Version_",username="******",password="******") i = ImporterThread(None,None, self.FakeConfig({ 'footage_providers_config': '{0}/../../footage_providers.yml'.format(self.mydir) }),dbconn=db) self.assertEqual(i.import_tags_for_fileref({'mime_type': 'video/mp4'}),['lowres']) self.assertEqual(i.import_tags_for_fileref({'mime_type': 'video/quicktime'}), ['lowres']) self.assertEqual(i.import_tags_for_fileref({'mime_type': 'application/mxf'}), ['lowres']) self.assertEqual(i.import_tags_for_fileref({'mime_type': 'model/vnd.mts'}),['lowres']) self.assertEqual(i.import_tags_for_fileref({'mime_type': 'image/jpeg'}), ['lowimage']) self.assertEqual(i.import_tags_for_fileref({'mime_type': 'image/tiff'}), ['lowimage']) self.assertEqual(i.import_tags_for_fileref({'mime_type': 'audio/aiff'}), ['lowaudio']) self.assertEqual(i.import_tags_for_fileref({'mime_type': 'audio/wav'}), ['lowaudio']) self.assertEqual(i.import_tags_for_fileref({'mime_type': 'application/xml'}),None)
def test_check_mime(self): from asset_folder_importer.asset_folder_sweeper.find_files import check_mime from asset_folder_importer.database import importer_db __version__ = 'database_test' db = importer_db(__version__,hostname=self.dbhost,port=self.dbport,username=self.dbuser, password=self.dbpass,dbname=self.dbname) filepath = "/etc/hosts" (statinfo, mimetype) = check_mime(filepath, db) self.assertEqual(mimetype, "text/plain") self.assertNotEqual(statinfo, None) filepath = os.path.join(os.path.dirname(os.path.realpath(__file__)), "1.jpeg") (statinfo, mimetype) = check_mime(filepath, db) self.assertEqual(mimetype, "image/jpeg")
def test_timeout_retries(self): from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread, ImportStalled from asset_folder_importer.database import importer_db from time import time with mock.patch('psycopg2.connect') as mock_connect: db = importer_db("_test_Version_", username="******", password="******") with mock.patch('httplib.HTTPConnection') as mock_connection: fake_job = self.StalledJob() mock_vsfile = mock.MagicMock(target=gnmvidispine.vs_storage.VSFile) mock_vsfile.importToItem = mock.MagicMock(return_value=fake_job) i=ImporterThread(None,None,self.FakeConfig({ 'footage_providers_config': '{0}/../../footage_providers.yml'.format(self.mydir) }),dbconn=db,import_timeout=4) #set importer timeout to 4s start_time = time() with self.assertRaises(ImportStalled): i.do_real_import(mock_vsfile,"/path/to/filename","fake_xml",['tagone']) self.assertGreaterEqual(time()-start_time,4) #should have taken at least 4 seconds mock_vsfile.importToItem.assert_called_once_with("fake_xml",tags=['tagone'],priority="LOW",jobMetadata={'gnm_app': 'vsingester'}) fake_job.abort.assert_called_once_with()
def test_potential_sidecar_filenames(self): from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread from asset_folder_importer.database import importer_db with mock.patch('psycopg2.connect') as mock_connect: db = importer_db("_test_Version_",username="******",password="******") i = ImporterThread(None,None, self.FakeConfig({ 'footage_providers_config': '{0}/../../footage_providers.yml'.format(self.mydir) }),dbconn=db) result = map(lambda x: x, i.potentialSidecarFilenames("/path/to/myfile.mp4", isxdcam=False)) self.assertEqual(result,['/path/to/myfile.xml', '/path/to/myfile.xmp', '/path/to/myfile.meta', '/path/to/myfile.XML', '/path/to/myfile.XMP','/path/to/myfile.mp4.xml','/path/to/myfile.mp4.xmp','/path/to/myfile.mp4.meta', '/path/to/myfile.mp4.XML','/path/to/myfile.mp4.XMP']) result = map(lambda x: x, i.potentialSidecarFilenames("/path/to/myfile.mp4", isxdcam=True)) self.assertEqual(result,['/path/to/myfile.xml', '/path/to/myfile.xmp', '/path/to/myfile.meta', '/path/to/myfile.XML', '/path/to/myfile.XMP', '/path/to/myfile.mp4.xml', '/path/to/myfile.mp4.xmp', '/path/to/myfile.mp4.meta', '/path/to/myfile.mp4.XML', '/path/to/myfile.mp4.XMP', '/path/to/myfileM01.xml', '/path/to/myfileM01.xmp', '/path/to/myfileM01.meta', '/path/to/myfileM01.XML', '/path/to/myfileM01.XMP'])
def test_timeout_retries(self): from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread, ImportStalled from asset_folder_importer.database import importer_db from time import time with mock.patch('psycopg2.connect') as mock_connect: db = importer_db("_test_Version_", username="******", password="******") with mock.patch('httplib.HTTPConnection') as mock_connection: fake_job = self.StalledJob() mock_vsfile = mock.MagicMock(target=gnmvidispine.vs_storage.VSFile) mock_vsfile.importToItem = mock.MagicMock(return_value=fake_job) i = ImporterThread(None, None, self.FakeConfig({ 'footage_providers_config': '{0}/../../footage_providers.yml'.format( self.mydir) }), dbconn=db, import_timeout=4) #set importer timeout to 4s start_time = time() with self.assertRaises(ImportStalled): i.do_real_import(mock_vsfile, "/path/to/filename", "fake_xml", ['tagone']) self.assertGreaterEqual(time() - start_time, 4) #should have taken at least 4 seconds mock_vsfile.importToItem.assert_called_once_with( "fake_xml", tags=['tagone'], priority="LOW", jobMetadata={'gnm_app': 'vsingester'}) fake_job.abort.assert_called_once_with()
def test_vs_inconsistency_error(self): from gnmvidispine.vs_storage import VSStorage, VSFile from gnmvidispine.vidispine_api import VSNotFound, HTTPError from asset_folder_importer.asset_folder_vsingester.importer_thread import ImporterThread from asset_folder_importer.asset_folder_vsingester.exceptions import VSFileInconsistencyError from asset_folder_importer.database import importer_db with mock.patch('psycopg2.connect') as mock_connect: db = importer_db("_test_Version_", username="******", password="******") mockstorage = mock.MagicMock(target=VSStorage) mockstorage.fileForPath = mock.MagicMock(side_effect=VSNotFound) mockstorage.create_file_entity = mock.MagicMock(side_effect=HTTPError( 503, "GET", "http://fake-url", "Failed", "I didn't expect the Spanish Inqusition", "")) mockfile = mock.MagicMock(target=VSFile) with mock.patch('httplib.HTTPConnection') as mock_connection: logging.basicConfig(level=logging.ERROR) logger = logging.getLogger("tester") logger.setLevel(logging.ERROR) i = ImporterThread(None, None, self.FakeConfig({ 'footage_providers_config': '{0}/../../footage_providers.yml'.format( self.mydir) }), dbconn=db) i.st = mockstorage with self.assertRaises(VSFileInconsistencyError) as raised_error: i.attempt_file_import(mockfile, "path/to/testfile", "/rootpath") self.assertEqual(str(raised_error.exception), "path/to/testfile")
if options.configfile: cfg=configfile(options.configfile) else: cfg=configfile("/etc/asset_folder_importer.cfg") #Step three. Set up pools. pool = ThreadPool(UpdateVsThread,initial_size=int(options.threads), config=cfg) raven_client = raven.Client(dsn=cfg.value("sentry_dsn")) #Step four. Scan the database table and update VS #Now connect to db logger.info("Connecting to database on %s" % cfg.value('database_host',noraise=True)) try: db = importer_db(__version__,hostname=cfg.value('database_host'),port=cfg.value('database_port'),username=cfg.value('database_user'),password=cfg.value('database_password')) db.check_schema_22() lastruntime = db.lastrun_endtime() lastruntimestamp = 0 except Exception as e: raven_client.captureException() raise n=0 c=0 for fileref in db.deleted_files(): n+=1 if fileref['ignore']: continue c+=1 if c>100:
def __init__(self, q, storageid, cfg, permission_script="/invalid/permissionscript", graveyard_folder="/tmp", timeout=60, import_timeout=3600, close_file_timeout=300, logger=None, dbconn=None): """ Initialises a new ImporterThread :param q: job queue to work from :param storageid: Storage ID to import from :param cfg: configuration object from config file and commandline :param permission_script: location of the suid change-permissions script :param graveyard_folder: graveyard folder to move invalid jobs to (defaults to /tmp) :param timeout: time to wait for API calls to complete (default: 60s; this is also the loadbalancer timeout) :param import_timeout: time to wait for a Vidispine import job before cancelling it as it is stalled (default: 1 hour) :param close_file_timeout: time to wait on processing a job before sending vidispine a close-file request (see https://support.vidispine.com/support/tickets/1360) (default: 5mins) :param logger: logger object to log to. For testing purposes, really; if None then a new logger will be created. (default: None) :param dbconn: database connection. For testing purposes really; if None then a new database connection will be created (default: None) """ super(ImporterThread, self).__init__() self.templateEnv = Environment(loader=PackageLoader( 'asset_folder_importer', 'metadata_templates')) self.mdTemplate = self.templateEnv.get_template('vsasset.xml') self.queue = q self.st = VSStorage(host=cfg.value('vs_host'), port=cfg.value('vs_port'), user=cfg.value('vs_user'), passwd=cfg.value('vs_password')) if storageid is not None: self.st.populate(storageid) self.db = dbconn if dbconn is not None else importer_db( __version__, hostname=cfg.value('database_host'), port=cfg.value('database_port'), username=cfg.value('database_user'), password=cfg.value('database_password')) self.found = 0 self.withItems = 0 self.imported = 0 self.cfg = cfg self.ignored = 0 self._permissionscript = permission_script self.logger = logger if logger is not None else logging.getLogger( "importer_thread") self._timeout = timeout self.graveyard_folder = graveyard_folder self._importer_timeout = import_timeout self._close_file_timeout = close_file_timeout try: providers_config_file = self.cfg.value('footage_providers_config', noraise=False) except KeyError: providers_config_file = '/etc/asset_folder_importer/footage_providers.yml' self.providers_list = externalprovider.ExternalProviderList( providers_config_file)
cfg = configfile("/etc/asset_folder_importer.cfg") if logfile is not None: logging.basicConfig(filename=logfile, format=LOGFORMAT, level=main_log_level) else: logging.basicConfig(format=LOGFORMAT, level=main_log_level) logging.info("-----------------------------------------------------------\n\n") logging.info("Connecting to database on %s" % cfg.value('database_host', noraise=True)) db = importer_db(__version__, hostname=cfg.value('database_host'), port=cfg.value('database_port'), username=cfg.value('database_user'), password=cfg.value('database_password')) lastruntime = db.lastrun_endtime() lastruntimestamp = 0 if lastruntime is None: logging.error( "It appears that another instance of premiere_get_referenced_media is already running." ) if not options.force: logging.error( "Not continuing because --force is not specified on the commandline" ) db.end_run("already_running")