jpg_thumb_files = os.listdir(jpg_thumb_directory)
jp2_lossy_files = os.listdir(jp2_lossy_directory)
jp2_lossless_files = os.listdir(jp2_lossless_directory)
fits_files = os.listdir(fits_directory)
name_space = u'macrepo'
'''
do ingest
'''
#put in the Italian topographical map object
try:
collection_label = u'15'
collection_pid = unicode(name_space + ':' + collection_label)
collection_policy = u'<collection_policy xmlns="http://www.islandora.ca" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="" xsi:schemaLocation="http://www.islandora.ca http://syn.lib.umanitoba.ca/collection_policy.xsd"> <content_models> <content_model dsid="ISLANDORACM" name="Islandora Collection Model ~ islandora:collectionCModel" namespace="islandora:1" pid="islandora:collectionCModel"/> <content_model dsid="ISLANDORACM" name="Islandora large image content model" namespace="macrepo:1" pid="islandora:sp_large_image_cmodel"/> </content_models> <search_terms/> <staging_area/> <relationship>isMemberOfCollection</relationship> </collection_policy> '
fedora.getObject(collection_pid)
except FedoraConnectionException, object_fetch_exception:
if object_fetch_exception.httpcode in [404]:
logging.info(name_space + ':itm missing, creating object.\n')
collection_object = fedora.createObject(collection_pid, label = collection_label)
#collection_policy
try:
collection_object.addDataStream(u'COLLECTION_POLICY', collection_policy, label=u'COLLECTION_POLICY',
mimeType=u'text/xml', controlGroup=u'X',
logMessage=u'Added basic COLLECTION_POLICY data.')
logging.info('Added COLLECTION_POLICY datastream to:' + collection_pid)
except FedoraConnectionException:
logging.error('Error in adding COLLECTION_POLICY datastream to:' + collection_pid + '\n')
#add relationships
collection_object_RELS_EXT = fedora_relationships.rels_ext(collection_object, fedora_model_namespace)
class islandoraSurrogate(object):
def __init__(self, pid, dsid, worker_id, language, logger, config):
self.init_config(config)
self.init_logger(logger)
self.init_properties(worker_id, pid, dsid, language)
if self.load_fedora_object() and self.check_fedora_object_dsid():
self.fetch_fedora_object_file()
def add_fedora_object_ds(self, ds_label = '', ds_mimeType = '', control_group = 'M', connect_tries = 3, ds_checksumType = None, ds_checksum = None):
return update_datastream(
self.fedora_object,
self.dsid,
self.converted_file_path,
label = ds_label,
mimeType = ds_mimeType,
controlGroup = control_group,
tries = connect_tries,
checksumType = ds_checksumType,
checksum = ds_checksum
)
def append_additional_encode_options(self, call_list, extra_options_variable, encoder_name):
extra_options = self.config.get(self.dsid, extra_options_variable)
if not extra_options == '':
self.logger.info('Worker %s appending extra options %s to %s for %s.', self.worker_id, self.convert_comma_separated_options_to_list(extra_options), encoder_name, self.dsid)
call_list.extend(self.convert_comma_separated_options_to_list(extra_options))
def check_fedora_object_dsid(self):
self.logger.info('Worker %s checking that %s does not exist in PID %s.', self.worker_id, self.dsid, self.pid)
if self.dsid in self.fedora_object:
self.duplicate = True
return False
else:
self.logger.info('Worker %s reports %s does not exist in PID %s.', self.worker_id, self.dsid, self.pid)
return True
def convert_comma_separated_options_to_list(self, options_string):
return re.split("[ ,]+", options_string)
def fetch_fedora_object_file(self):
self.temp_file_dir, temp_filename = get_datastream_as_file(self.fedora_object, 'OBJ', 'jpg')
self.temp_filepath = '/'.join( (self.temp_file_dir, temp_filename) )
self.logger.info('Worker %s pulled OBJ from %s to %s.', self.worker_id, self.pid, self.temp_filepath)
def init_config(self, config):
self.config = config
def init_fedora_connection(self):
try:
self.logger.info('Worker %s attempting to connect to Fedora server.', self.worker_id)
self.fedora_connection = Connection(
self.config.get('Fedora', 'fedora_url'),
username = self.config.get('Fedora', 'fedora_admin_user'),
password = self.config.get('Fedora', 'fedora_admin_password')
)
self.fedora_client = FedoraClient(self.fedora_connection)
self.logger.info('Worker %s connected to Fedora server.', self.worker_id)
return True
except:
self.logger.error('Worker %s failed to connect to Fedora server.', self.worker_id)
return False
def init_logger(self, logger):
self.logger = logger
def init_properties(self, worker_id, pid, dsid, language):
self.converted_file_path = False
self.duplicate = False
self.temp_filepath = False
self.worker_id = worker_id
self.pid = pid
self.dsid = dsid
self.language = language
def load_fedora_object(self):
if self.init_fedora_connection():
self.logger.info('Worker %s getting object %s from %s.', self.worker_id, self.pid, self.config.get('Fedora', 'fedora_url'))
self.fedora_object = self.fedora_client.getObject(self.pid)
return True
return False
def log_encode_begin(self, output_filename):
self.logger.info('Worker %s encoding %s %s surrogate to %s.', self.worker_id, self.pid, self.dsid, output_filename)
def log_encode_fail(self):
self.logger.error('Worker %s encoding %s surrogate of %s has failed.', self.worker_id, self.dsid, self.pid)
def log_encode_success(self):
self.logger.info('Worker %s encoding %s surrogate of %s has succeeded.', self.worker_id, self.dsid, self.pid)
def remove_temp_files(self):
try:
rmtree(self.temp_file_dir)
self.logger.info('Worker %s removing temporary %s files.', self.worker_id, self.pid)
except:
self.logger.info('Worker %s could not remove temporary %s files.', self.worker_id, self.pid)
tei_files = os.listdir(tei_directory)
tei_page_files = os.listdir(tei_page_directory)
jp2_files = os.listdir(jp2_directory)
pdf_files = os.listdir(pdf_directory)
name_space = u'HamiltonCivilWar'
#name_space = u'HamiltonCivilWar2'
'''
do ingest
'''
#put in the JapaneseSilentFilmCollection collection object
try:
collection_label = u'UnitedStatesCivilWarLetters'
collection_pid = unicode(name_space + ':' + collection_label)
collection_policy = u'<collection_policy xmlns="http://www.islandora.ca" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="" xsi:schemaLocation="http://www.islandora.ca"> <content_models> <content_model dsid="ISLANDORACM" name="Book Content Model" namespace="islandora:1" pid="islandora:bookCModel"></content_model> </content_models> <search_terms></search_terms> <staging_area></staging_area> <relationship>isMemberOf</relationship></collection_policy>'
fedora.getObject(collection_pid)
except FedoraConnectionException, object_fetch_exception:
if object_fetch_exception.httpcode in [404]:
logging.info(
name_space +
':UnitedStatesCivilWarLetters missing, creating object.\n')
collection_object = fedora.createObject(collection_pid,
label=collection_label)
#collection_policy
try:
collection_object.addDataStream(
u'COLLECTION_POLICY',
collection_policy,
label=u'COLLECTION_POLICY',
mimeType=u'text/xml',
controlGroup=u'X',
class StompFedora(ConnectionListener):
"""
A custom interface to the stomp.py client. See \link stomp::internal::connect::Connection \endlink
for more information on establishing a connection to a stomp server.
"""
def __init__(self, host='localhost', port=61613, user='', passcode='', fedora_url=''):
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
self.transaction_id = None
self.fc = fcrepo.connection.Connection(fedora_url, username = user, password = passcode)
self.client = FedoraClient(self.fc)
self.fedora_url = fedora_url
self.user = user
self.password = passcode
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug('%s: %s' % (header_key, headers[header_key]))
logging.debug(body)
def __get_content_models(self, pid):
"""
Get a list of content models that apply to the object.
"""
obj = self.client.getObject(pid)
if 'RELS-EXT' in obj:
ds = obj['RELS-EXT']
return obj, [elem['value'].split('/')[1] for elem in ds[NS.fedoramodel.hasModel]]
else:
return obj, []
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
logging.error("lost connection reconnect in %d sec..." % reconnect_wait)
signal.alarm(reconnect_wait)
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
self.__print_async("MESSSAGE", headers, body)
method = headers['methodName']
pid = headers['pid']
newheaders = { 'methodname':headers['methodName'], 'pid':headers['pid']}
if method in ['addDatastream', 'modifyDatastreamByValue', 'modifyDatastreamByReference', 'modifyObject']:
f = feedparser.parse(body)
tags = f['entries'][0]['tags']
dsids = [tag['term'] for tag in tags if tag['scheme'] == 'fedora-types:dsID']
dsid = ''
if dsids:
dsid = dsids[0]
newheaders['dsid'] = dsid
obj, content_models = self.__get_content_models(pid) # and 'OBJ' in dsIDs:
for content_model in content_models:
logging.info("/topic/fedora.contentmodel.%s %s" % (content_model, dsid))
self.send("/topic/fedora.contentmodel.%s" % content_model, newheaders, body)
elif method in ['ingest']:
obj, content_models = self.__get_content_models(pid)
for dsid in obj:
for content_model in content_models:
newheaders['dsid'] = dsid
logging.info("/topic/fedora.contentmodel.%s %s" % (content_model, dsid))
self.send("/topic/fedora.contentmodel.%s" % content_model, newheaders, body)
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers = { 'message-id' : args[1]})
else:
self.conn.ack(headers = { 'message-id' : args[1]}, transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args=None):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, headers, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination, message=message, headers=headers)
def subscribe(self, destination, ack='auto'):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def connect(self):
self.conn.start()
self.fc = fcrepo.connection.Connection(self.fedora_url, username = self.user, password = self.password)
self.client = FedoraClient(self.fc)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
class ContentModelListener(ConnectionListener):
'''
classdocs
'''
def __init__(self, content_models, host='localhost', port=61613, user='', passcode='', fedora_url=''):
'''
Constructor
'''
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
logging.info('Connecting to STOMP server %(host)s on port %(port)s.' % {'host': host, 'port': port})
self.transaction_id = None
logging.info("Connecting to Fedora server at %(url)s" % {'url': fedora_url})
self.fc = fcrepo.connection.Connection(fedora_url, username = user, password = passcode)
self.client = FedoraClient(self.fc)
self.fedora_url = fedora_url
self.username = user
self.password = passcode
# Create plugin manager
self.manager = PluginManager(categories_filter = {"FedoraMicroService": FedoraMicroService})
plugin_path = os.path.dirname(__file__)
self.manager.setPluginPlaces([plugin_path + "/plugins"])
logging.debug("Plugin path: " + plugin_path + "/plugins")
# Load plugins.
self.manager.locatePlugins()
self.manager.loadPlugins()
self.contentModels = {}
for plugin in self.manager.getPluginsOfCategory("FedoraMicroService"):
# plugin.plugin_object is an instance of the plubin
logging.info("Loading plugin: %(name)s for content model %(cmodel)s." % {'name': plugin.plugin_object.name, 'cmodel': plugin.plugin_object.content_model})
plugin.plugin_object.config = config
if type(plugin.plugin_object.content_model) == types.StringType:
content_models = [plugin.plugin_object.content_model]
else:
content_models = plugin.plugin_object.content_model
for content_model in content_models:
if content_model in self.contentModels:
self.contentModels[content_model].append(plugin.plugin_object)
else:
self.contentModels[content_model] = [plugin.plugin_object]
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
#logging.debug("\r \r", end='')
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug('%s: %s' % (header_key, headers[header_key]))
logging.debug(body)
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
logging.error("lost connection reconnect in %d sec..." % reconnect_wait)
signal.alarm(reconnect_wait)
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
try:
global TOPIC_PREFIX
self.__print_async('MESSAGE', headers, body)
pid = headers['pid']
dsid = headers['dsid']
obj = self.client.getObject(pid)
content_model = headers['destination'][len(TOPIC_PREFIX):]
if content_model in self.contentModels:
logging.info('Running rules for %(pid)s from %(cmodel)s.' % {'pid': obj.pid, 'cmodel': content_model} )
for plugin in self.contentModels[content_model]:
plugin.runRules(obj, dsid, body)
except FedoraConnectionException:
logging.warning('Object %s was not found.' % (pid))
except:
logging.error("an exception occurred: " + str(sys.exc_info()[0]))
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers = { 'message-id' : args[1]})
else:
self.conn.ack(headers = { 'message-id' : args[1]}, transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, correlation_id, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination, message=message, headers={'correlation-id': correlation_id})
def subscribe(self, destination, ack='auto'):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
def connect(self):
self.conn.start()
self.fc = fcrepo.connection.Connection(self.fedora_url, username = self.username, password = self.password)
self.client = FedoraClient(self.fc)
fedora_url = 'http://localhost:8080/fedora'
username = '******'
password = '******'
log_filename = 'script.log'
body = ''
pids = [
#pids to work on here
]
levels = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL,
'FATAL': logging.FATAL
}
logging.basicConfig(filename=log_filename, level=levels['INFO'])
fc = fcrepo.connection.Connection(fedora_url,
username=username,
password=password)
client = FedoraClient(fc)
for pid in pids:
obj = client.getObject(pid)
logging.info("Processing Pid: %s" % (obj.pid))
for dsid in obj:
co.runRules(obj, dsid, body)
class StompFedora(ConnectionListener):
"""
A custom interface to the stomp.py client. See \link stomp::internal::connect::Connection \endlink
for more information on establishing a connection to a stomp server.
"""
def __init__(self, host="localhost", port=61613, user="", passcode="", fedora_url=""):
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener("", self)
self.conn.start()
self.transaction_id = None
self.fc = fcrepo.connection.Connection(fedora_url, username=user, password=passcode)
self.client = FedoraClient(self.fc)
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug("%s: %s" % (header_key, headers[header_key]))
logging.debug(body)
def __get_content_models(self, pid):
"""
Get a list of content models that apply to the object.
"""
obj = self.client.getObject(pid)
if "RELS-EXT" in obj:
ds = obj["RELS-EXT"]
return [elem["value"].split("/")[1] for elem in ds[NS.fedoramodel.hasModel]]
else:
return []
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
sysout("lost connection")
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
self.__print_async("MESSSAGE", headers, body)
f = feedparser.parse(body)
method = headers["methodName"]
if method in ["addDatastream", "modifyDatastreamByValue", "modifyDatastreamByReference"]:
tags = f["entries"][0]["tags"]
pid = [tag["term"] for tag in tags if tag["scheme"] == "fedora-types:pid"][0]
dsIDs = [tag["term"] for tag in tags if tag["scheme"] == "fedora-types:dsID"]
content_models = self.__get_content_models(pid) # and 'OBJ' in dsIDs:
for content_model in content_models:
print "/topic/fedora.contentmodel.%s" % content_model
self.send("/topic/fedora.contentmodel.%s" % content_model, "", body)
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers={"message-id": args[1]})
else:
self.conn.ack(headers={"message-id": args[1]}, transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, correlation_id, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination, message=message, headers={"correlation-id": correlation_id})
def subscribe(self, destination, ack="auto"):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
from categories import FedoraMicroService
sys.path.append( 'plugins' )
from coalliance_coccOralHistoryCModel import coalliance_coccOralHistoryCModel
co = coalliance_coccOralHistoryCModel()
fedora_url = 'http://localhost:8080/fedora'
username = '******'
password = '******'
log_filename = 'script.log'
body = ''
pids = [
#pids to work on here
]
levels = {'DEBUG':logging.DEBUG, 'INFO': logging.INFO, 'WARNING': logging.WARNING, 'ERROR':logging
.ERROR, 'CRITICAL':logging.CRITICAL, 'FATAL':logging.FATAL}
logging.basicConfig(filename = log_filename, level = levels['INFO'])
fc = fcrepo.connection.Connection(fedora_url, username = username, password = password)
client = FedoraClient(fc)
for pid in pids:
obj = client.getObject(pid)
logging.info("Processing Pid: %s"%(obj.pid))
for dsid in obj:
co.runRules(obj, dsid, body)
class ContentModelListener(ConnectionListener):
'''
classdocs
'''
def __init__(self,
content_models,
host='localhost',
port=61613,
user='',
passcode='',
fedora_url=''):
'''
Constructor
'''
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
logging.info('Connecting to STOMP server %(host)s on port %(port)s.' %
{
'host': host,
'port': port
})
self.transaction_id = None
logging.info("Connecting to Fedora server at %(url)s" %
{'url': fedora_url})
self.fc = fcrepo.connection.Connection(fedora_url,
username=user,
password=passcode)
self.client = FedoraClient(self.fc)
# Create plugin manager
self.manager = PluginManager(
categories_filter={"FedoraMicroService": FedoraMicroService})
self.manager.setPluginPlaces(["plugins"])
# Load plugins.
self.manager.locatePlugins()
self.manager.loadPlugins()
self.contentModels = {}
for plugin in self.manager.getPluginsOfCategory("FedoraMicroService"):
# plugin.plugin_object is an instance of the plubin
logging.info(
"Loading plugin: %(name)s for content model %(cmodel)s." % {
'name': plugin.plugin_object.name,
'cmodel': plugin.plugin_object.content_model
})
plugin.plugin_object.config = config
if plugin.plugin_object.content_model in self.contentModels:
self.contentModels[plugin.plugin_object.content_model].append(
plugin.plugin_object)
else:
self.contentModels[plugin.plugin_object.content_model] = [
plugin.plugin_object
]
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
#logging.debug("\r \r", end='')
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug('%s: %s' % (header_key, headers[header_key]))
logging.debug(body)
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
global TOPIC_PREFIX
self.__print_async('MESSAGE', headers, body)
f = feedparser.parse(body)
tags = f['entries'][0]['tags']
pid = [
tag['term'] for tag in tags if tag['scheme'] == 'fedora-types:pid'
][0]
dsID = [
tag['term'] for tag in tags if tag['scheme'] == 'fedora-types:dsID'
][0]
obj = self.client.getObject(pid)
content_model = headers['destination'][len(TOPIC_PREFIX):]
if content_model in self.contentModels:
logging.info('Running rules for %(pid)s from %(cmodel)s.' % {
'pid': obj.pid,
'cmodel': content_model
})
for plugin in self.contentModels[content_model]:
plugin.runRules(obj, dsID)
return
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers={'message-id': args[1]})
else:
self.conn.ack(headers={'message-id': args[1]},
transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, correlation_id, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination,
message=message,
headers={'correlation-id': correlation_id})
def subscribe(self, destination, ack='auto'):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
class StompFedora(ConnectionListener):
"""
A custom interface to the stomp.py client. See \link stomp::internal::connect::Connection \endlink
for more information on establishing a connection to a stomp server.
"""
def __init__(self,
host='localhost',
port=61613,
user='',
passcode='',
fedora_url=''):
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
self.transaction_id = None
self.fc = fcrepo.connection.Connection(fedora_url,
username=user,
password=passcode)
self.client = FedoraClient(self.fc)
self.fedora_url = fedora_url
self.user = user
self.password = passcode
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug('%s: %s' % (header_key, headers[header_key]))
logging.debug(body)
def __get_content_models(self, pid):
"""
Get a list of content models that apply to the object.
"""
obj = self.client.getObject(pid)
if 'RELS-EXT' in obj:
ds = obj['RELS-EXT']
return obj, [
elem['value'].split('/')[1]
for elem in ds[NS.fedoramodel.hasModel]
]
else:
return obj, []
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
logging.error("lost connection reconnect in %d sec..." %
reconnect_wait)
signal.alarm(reconnect_wait)
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
self.__print_async("MESSSAGE", headers, body)
method = headers['methodName']
pid = headers['pid']
newheaders = {
'methodname': headers['methodName'],
'pid': headers['pid']
}
if method in [
'addDatastream', 'modifyDatastreamByValue',
'modifyDatastreamByReference', 'modifyObject'
]:
f = feedparser.parse(body)
tags = f['entries'][0]['tags']
dsids = [
tag['term'] for tag in tags
if tag['scheme'] == 'fedora-types:dsID'
]
dsid = ''
if dsids:
dsid = dsids[0]
newheaders['dsid'] = dsid
obj, content_models = self.__get_content_models(
pid) # and 'OBJ' in dsIDs:
for content_model in content_models:
logging.info("/topic/fedora.contentmodel.%s %s" %
(content_model, dsid))
self.send("/topic/fedora.contentmodel.%s" % content_model,
newheaders, body)
elif method in ['ingest']:
obj, content_models = self.__get_content_models(pid)
for dsid in obj:
for content_model in content_models:
newheaders['dsid'] = dsid
logging.info("/topic/fedora.contentmodel.%s %s" %
(content_model, dsid))
self.send("/topic/fedora.contentmodel.%s" % content_model,
newheaders, body)
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers={'message-id': args[1]})
else:
self.conn.ack(headers={'message-id': args[1]},
transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args=None):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, headers, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination,
message=message,
headers=headers)
def subscribe(self, destination, ack='auto'):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def connect(self):
self.conn.start()
self.fc = fcrepo.connection.Connection(self.fedora_url,
username=self.user,
password=self.password)
self.client = FedoraClient(self.fc)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
