def setUpClass(cls):
"""
Set up the test
"""
LOGGER.info("running '{}' tests...".format(cls.__name__))
# disable indexing during import
test_utils.set_index_active(False)
# import resources
test_utils.setup_test_storage()
OBJECT_XML_CACHE.clear()
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"partial-corpus.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip(
"{}/repository/fixtures/full-resources/"
"full-lang-description.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip(
"{}/repository/fixtures/full-resources/"
"full-lex-conceptual.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-corpus-text.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-tool-service.xml".format(ROOT_PATH))
# enable indexing
test_utils.set_index_active(True)
# update index
update_index.Command().handle(using=[
settings.TEST_MODE_NAME,
])
def setUpClass(cls):
"""
Set up the test
"""
LOGGER.info("running '{}' tests...".format(cls.__name__))
# disable indexing during import
test_utils.set_index_active(False)
# import resources
test_utils.setup_test_storage()
OBJECT_XML_CACHE.clear()
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"partial-corpus.xml".format(ROOT_PATH))
def tearDownClass(cls):
"""
Clean up the test
"""
LOGGER.info("finished '{}' tests".format(cls.__name__))
# disable indexing during import
test_utils.set_index_active(False)
test_utils.clean_resources_db()
test_utils.clean_storage()
OBJECT_XML_CACHE.clear()
# enable indexing
test_utils.set_index_active(True)
def tearDown(self):
"""
Clean up the test
"""
# disable indexing during import
test_utils.set_index_active(False)
test_utils.clean_resources_db()
test_utils.clean_storage()
OBJECT_XML_CACHE.clear()
# enable indexing
test_utils.set_index_active(True)
# update index
rebuild_index.Command().handle()
def setUp(self):
"""
Set up the test
"""
# disable indexing during import
test_utils.set_index_active(False)
# import resources
test_utils.setup_test_storage()
OBJECT_XML_CACHE.clear()
test_utils.import_xml_or_zip(RESOURCES_ZIP_FILE)
# enable indexing
test_utils.set_index_active(True)
# update index
rebuild_index.Command().handle()
def tearDownClass(cls):
"""
Clean up the test
"""
LOGGER.info("finished '{}' tests".format(cls.__name__))
# disable indexing during import
test_utils.set_index_active(False)
test_utils.clean_resources_db()
test_utils.clean_storage()
OBJECT_XML_CACHE.clear()
# enable indexing
test_utils.set_index_active(True)
# update index
update_index.Command().handle(using=[settings.TEST_MODE_NAME,])
def tearDownClass(cls):
"""
Clean up the test
"""
LOGGER.info("finished '{}' tests".format(cls.__name__))
# disable indexing during import
test_utils.set_index_active(False)
test_utils.clean_resources_db()
test_utils.clean_storage()
OBJECT_XML_CACHE.clear()
# enable indexing
test_utils.set_index_active(True)
# update index
from django.core.management import call_command
call_command('rebuild_index', interactive=False, using=TEST_MODE_NAME)
def tearDownClass(cls):
"""
Clean up the test
"""
LOGGER.info("finished '{}' tests".format(cls.__name__))
# disable indexing during import
test_utils.set_index_active(False)
test_utils.clean_resources_db()
test_utils.clean_storage()
OBJECT_XML_CACHE.clear()
# enable indexing
test_utils.set_index_active(True)
# update index
from django.core.management import call_command
call_command('rebuild_index', interactive=False, using=TEST_MODE_NAME)
def setUpClass(cls):
"""
Set up the test
"""
LOGGER.info("running '{}' tests...".format(cls.__name__))
# disable indexing during import
test_utils.set_index_active(False)
# import resources
test_utils.setup_test_storage()
OBJECT_XML_CACHE.clear()
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"partial-corpus.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip(
"{}/repository/fixtures/full-resources/"
"full-lang-description.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip(
"{}/repository/fixtures/full-resources/"
"full-lex-conceptual.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-corpus-text.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-corpus-image.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-corpus-audio.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-corpus-video.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip(
"{}/repository/fixtures/full-resources/"
"full-corpus-textngram.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip(
"{}/repository/fixtures/full-resources/"
"full-corpus-textnumerical.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-tool-service.xml".format(ROOT_PATH))
# enable indexing
test_utils.set_index_active(True)
# update index
from django.core.management import call_command
call_command('rebuild_index', interactive=False, using=TEST_MODE_NAME)
def setUpClass(cls):
"""
Set up the test
"""
LOGGER.info("running '{}' tests...".format(cls.__name__))
# disable indexing during import
test_utils.set_index_active(False)
# import resources
test_utils.setup_test_storage()
OBJECT_XML_CACHE.clear()
test_utils.import_xml_or_zip(RESOURCES_ZIP_FILE)
# enable indexing
test_utils.set_index_active(True)
# update index
from django.core.management import call_command
call_command('rebuild_index', interactive=False, using=TEST_MODE_NAME)
def setUpClass(cls):
"""
Set up the test
"""
LOGGER.info("running '{}' tests...".format(cls.__name__))
# disable indexing during import
test_utils.set_index_active(False)
# import resources
test_utils.setup_test_storage()
OBJECT_XML_CACHE.clear()
test_utils.import_xml_or_zip(RESOURCES_ZIP_FILE)
# enable indexing
test_utils.set_index_active(True)
# update index
from django.core.management import call_command
call_command('rebuild_index', interactive=False, using=TEST_MODE_NAME)
def setUpClass(cls):
"""
Set up the test
"""
LOGGER.info("running '{}' tests...".format(cls.__name__))
# disable indexing during import
test_utils.set_index_active(False)
# import resources
test_utils.setup_test_storage()
OBJECT_XML_CACHE.clear()
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"partial-corpus.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-lang-description.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-lex-conceptual.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-corpus-text.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-corpus-image.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-corpus-audio.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-corpus-video.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-corpus-textngram.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-corpus-textnumerical.xml".format(ROOT_PATH))
test_utils.import_xml_or_zip("{}/repository/fixtures/full-resources/"
"full-tool-service.xml".format(ROOT_PATH))
# enable indexing
test_utils.set_index_active(True)
# update index
from django.core.management import call_command
call_command('rebuild_index', interactive=False, using=TEST_MODE_NAME)
def import_resources(import_folder):
"""
Imports resources from the given folder.
"""
# Check that SOLR is running, or else all resources will stay at status INTERNAL:
from metashare.repository import verify_at_startup
verify_at_startup() # may raise Exception, which we don't want to catch.
# Disable verbose debug output for the import process...
settings.DEBUG = False
os.environ['DISABLE_INDEXING_DURING_IMPORT'] = 'True'
from metashare.repository.supermodel import OBJECT_XML_CACHE
# Clean cache before starting the import process.
OBJECT_XML_CACHE.clear()
# iterate over storage folder content
from django.core import serializers
from metashare.storage.models import MASTER, ALLOWED_ARCHIVE_EXTENSIONS
from metashare.repository.models import resourceInfoType_model
imported_resources = []
erroneous_descriptors = []
storage_path = os.path.join(import_folder, STORAGE_FOLDER)
for folder_name in os.listdir(storage_path):
folder_path = "{}/{}/".format(storage_path, folder_name)
if os.path.isdir(folder_path):
try:
print "importing from folder: '{0}'".format(folder_name)
# import storage object
so_filename = os.path.join(folder_path, STORAGE)
so_in = open(so_filename, "rb")
for obj in serializers.deserialize("xml", so_in):
print "importing storage object"
# storage.xml only contains a single storage object
storage_obj = obj.object
# this storage object is NOT saved!
# we only copy the relevant attributes from this storage
# object to the one at the resource!
so_in.close()
# import resource object
ro_filename = os.path.join(folder_path, RESOURCE)
ro_in = open(ro_filename, "rb")
for obj in serializers.deserialize("xml", ro_in):
print "importing resource object"
# resource.xml only contains a single resource object
res_obj = obj
# the deserialized object contains the ManyToMany attributes
# in m2m_data
ro_in.close()
# import resource from metadata.xml
res_filename = os.path.join(folder_path, METADATA)
temp_file = open(res_filename, 'rb')
xml_string = temp_file.read()
result = resourceInfoType_model.import_from_string(
xml_string, copy_status=MASTER)
if not result[0]:
msg = u''
if len(result) > 2:
msg = u'{}'.format(result[2])
raise Exception(msg)
res = result[0]
# update imported resource with imported resource object
# and storage object
_update_resource(res, res_obj, storage_obj)
# copy possible binaries archives
for archive_name in [
ARCHIVE_TPL.format(_ext)
for _ext in ALLOWED_ARCHIVE_EXTENSIONS
]:
archive_filename = os.path.join(folder_path, archive_name)
if os.path.isfile(archive_filename):
print "copying archive"
res_storage_path = '{0}/{1}/'.format(
settings.STORAGE_PATH,
res.storage_object.identifier)
shutil.copy(
archive_filename,
os.path.join(res_storage_path, archive_name))
# there can be at most one binary
break
imported_resources.append(res)
except Exception as problem:
from django import db
if isinstance(problem, db.utils.DatabaseError):
# reset database connection (required for PostgreSQL)
db.close_connection()
erroneous_descriptors.append((folder_name, problem))
print "Done. Successfully imported {0} resources into the database, " \
"errors occurred in {1} cases.".format(
len(imported_resources), len(erroneous_descriptors))
if len(erroneous_descriptors) > 0:
print "The following resources could not be imported:"
for descriptor, exception in erroneous_descriptors:
print "\t{}: {}".format(descriptor, exception)
# Be nice and cleanup cache...
_cache_size = sum([len(x) for x in OBJECT_XML_CACHE.values()])
OBJECT_XML_CACHE.clear()
print "Cleared OBJECT_XML_CACHE ({} bytes)".format(_cache_size)
from django.core.management import call_command
call_command('rebuild_index', interactive=False)
# Check that SOLR is running, or else all resources will stay at status INTERNAL:
from metashare.repository import verify_at_startup
verify_at_startup() # may raise Exception, which we don't want to catch.
# Disable verbose debug output for the import process...
settings.DEBUG = False
os.environ['DISABLE_INDEXING_DURING_IMPORT'] = 'True'
successful_restored = []
erroneous_restored = []
from metashare.repository.supermodel import OBJECT_XML_CACHE
from metashare.storage.models import restore_from_folder
# Clean cache before starting the import process.
OBJECT_XML_CACHE.clear()
# iterate over storage folder content
for folder_name in os.listdir(settings.STORAGE_PATH):
folder_path = os.path.join(settings.STORAGE_PATH, folder_name)
if os.path.isdir(folder_path):
# skip empty folders; it is assumed that this is not an error
if os.listdir(folder_path) == []:
continue
try:
print 'restoring from folder: "{0}"'.format(folder_name)
# only restore resources with global- and local-storage.json
_storage_global_path = '{0}/storage-global.json'.format(folder_path)
if not os.path.isfile(_storage_global_path):
print 'missing global json, skipping "{}"'.format(folder_name)
continue
def import_resources(import_folder):
"""
Imports resources from the given folder.
"""
# Check that SOLR is running, or else all resources will stay at status INTERNAL:
from metashare.repository import verify_at_startup
verify_at_startup() # may raise Exception, which we don't want to catch.
# Disable verbose debug output for the import process...
settings.DEBUG = False
os.environ['DISABLE_INDEXING_DURING_IMPORT'] = 'True'
from metashare.repository.supermodel import OBJECT_XML_CACHE
# Clean cache before starting the import process.
OBJECT_XML_CACHE.clear()
# iterate over storage folder content
from django.core import serializers
from metashare.storage.models import MASTER, ALLOWED_ARCHIVE_EXTENSIONS
from metashare.repository.models import resourceInfoType_model
imported_resources = []
erroneous_descriptors = []
storage_path = os.path.join(import_folder, STORAGE_FOLDER)
for folder_name in os.listdir(storage_path):
folder_path = "{}/{}/".format(storage_path, folder_name)
if os.path.isdir(folder_path):
try:
print "importing from folder: '{0}'".format(folder_name)
# import storage object
so_filename = os.path.join(folder_path, STORAGE)
so_in = open(so_filename, "rb")
for obj in serializers.deserialize("xml", so_in):
print "importing storage object"
# storage.xml only contains a single storage object
storage_obj = obj.object
# this storage object is NOT saved!
# we only copy the relevant attributes from this storage
# object to the one at the resource!
so_in.close()
# import resource object
ro_filename = os.path.join(folder_path, RESOURCE)
ro_in = open(ro_filename, "rb")
for obj in serializers.deserialize("xml", ro_in):
print "importing resource object"
# resource.xml only contains a single resource object
res_obj = obj
# the deserialized object contains the ManyToMany attributes
# in m2m_data
ro_in.close()
# import resource from metadata.xml
res_filename = os.path.join(folder_path, METADATA)
temp_file = open(res_filename, 'rb')
xml_string = temp_file.read()
result = resourceInfoType_model.import_from_string(
xml_string, copy_status=MASTER)
if not result[0]:
msg = u''
if len(result) > 2:
msg = u'{}'.format(result[2])
raise Exception(msg)
res = result[0]
# update imported resource with imported resource object
# and storage object
_update_resource(res, res_obj, storage_obj)
# copy possible binaries archives
for archive_name in [ARCHIVE_TPL.format(_ext)
for _ext in ALLOWED_ARCHIVE_EXTENSIONS]:
archive_filename = os.path.join(folder_path, archive_name)
if os.path.isfile(archive_filename):
print "copying archive"
res_storage_path = '{0}/{1}/'.format(
settings.STORAGE_PATH, res.storage_object.identifier)
shutil.copy(archive_filename,
os.path.join(res_storage_path, archive_name))
# there can be at most one binary
break
imported_resources.append(res)
except Exception as problem:
from django import db
if isinstance(problem, db.utils.DatabaseError):
# reset database connection (required for PostgreSQL)
db.close_connection()
erroneous_descriptors.append((folder_name, problem))
print "Done. Successfully imported {0} resources into the database, " \
"errors occurred in {1} cases.".format(
len(imported_resources), len(erroneous_descriptors))
if len(erroneous_descriptors) > 0:
print "The following resources could not be imported:"
for descriptor, exception in erroneous_descriptors:
print "\t{}: {}".format(descriptor, exception)
# Be nice and cleanup cache...
_cache_size = sum([len(x) for x in OBJECT_XML_CACHE.values()])
OBJECT_XML_CACHE.clear()
print "Cleared OBJECT_XML_CACHE ({} bytes)".format(_cache_size)
from django.core.management import call_command
call_command('rebuild_index', interactive=False)
# Check that SOLR is running, or else all resources will stay at status INTERNAL:
from metashare.repository import verify_at_startup
verify_at_startup() # may raise Exception, which we don't want to catch.
# Disable verbose debug output for the import process...
settings.DEBUG = False
os.environ['DISABLE_INDEXING_DURING_IMPORT'] = 'True'
successful_imports = []
erroneous_imports = []
from metashare.xml_utils import import_from_file
from metashare.storage.models import PUBLISHED, MASTER
from metashare.repository.supermodel import OBJECT_XML_CACHE
# Clean cache before starting the import process.
OBJECT_XML_CACHE.clear()
for filename in sys.argv[arg_num:]:
temp_file = open(filename, 'rb')
success, failure = import_from_file(temp_file, filename, PUBLISHED, MASTER)
successful_imports += success
erroneous_imports += failure
temp_file.close()
print "Done. Successfully imported {0} files into the database, errors " \
"occurred in {1} cases.".format(len(successful_imports), len(erroneous_imports))
if len(erroneous_imports) > 0:
print "The following files could not be imported:"
for descriptor, exception in erroneous_imports:
if isinstance(exception.args, basestring):
print "\t{}: {}".format(descriptor, ' '.join(exception.args))