def __init__(self,
content_models,
host='localhost',
port=61613,
user='',
passcode='',
fedora_url=''):
'''
Constructor
'''
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
logging.info('Connecting to STOMP server %(host)s on port %(port)s.' %
{
'host': host,
'port': port
})
self.transaction_id = None
logging.info("Connecting to Fedora server at %(url)s" %
{'url': fedora_url})
self.fc = fcrepo.connection.Connection(fedora_url,
username=user,
password=passcode)
self.client = FedoraClient(self.fc)
self.fedora_url = fedora_url
self.username = user
self.password = passcode
# Create plugin manager
self.manager = PluginManager(
categories_filter={"FedoraMicroService": FedoraMicroService})
plugin_path = os.path.dirname(__file__)
self.manager.setPluginPlaces([plugin_path + "/plugins"])
# Load plugins.
self.manager.locatePlugins()
self.manager.loadPlugins()
self.contentModels = {}
for plugin in self.manager.getPluginsOfCategory("FedoraMicroService"):
# plugin.plugin_object is an instance of the plubin
logging.info(
"Loading plugin: %(name)s for content model %(cmodel)s." % {
'name': plugin.plugin_object.name,
'cmodel': plugin.plugin_object.content_model
})
plugin.plugin_object.config = config
if type(plugin.plugin_object.content_model) == types.StringType:
content_models = [plugin.plugin_object.content_model]
else:
content_models = plugin.plugin_object.content_model
for content_model in content_models:
if content_model in self.contentModels:
self.contentModels[content_model].append(
plugin.plugin_object)
else:
self.contentModels[content_model] = [plugin.plugin_object]
def __init__(self, content_models, host='localhost', port=61613, user='', passcode='', fedora_url=''):
'''
Constructor
'''
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
logging.info('Connecting to STOMP server %(host)s on port %(port)s.' % {'host': host, 'port': port})
self.transaction_id = None
logging.info("Connecting to Fedora server at %(url)s" % {'url': fedora_url})
self.fc = fcrepo.connection.Connection(fedora_url, username = user, password = passcode)
self.client = FedoraClient(self.fc)
# Create plugin manager
self.manager = PluginManager(categories_filter = {"FedoraMicroService": FedoraMicroService})
self.manager.setPluginPlaces(["plugins"])
# Load plugins.
self.manager.locatePlugins()
self.manager.loadPlugins()
self.contentModels = {}
for plugin in self.manager.getPluginsOfCategory("FedoraMicroService"):
# plugin.plugin_object is an instance of the plubin
logging.info("Loading plugin: %(name)s for content model %(cmodel)s." % {'name': plugin.plugin_object.name, 'cmodel': plugin.plugin_object.content_model})
plugin.plugin_object.config = config
if plugin.plugin_object.content_model in self.contentModels:
self.contentModels[plugin.plugin_object.content_model].append(plugin.plugin_object)
else:
self.contentModels[plugin.plugin_object.content_model] = [plugin.plugin_object]
def __init__(self, host="localhost", port=61613, user="", passcode="", fedora_url=""):
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener("", self)
self.conn.start()
self.transaction_id = None
self.fc = fcrepo.connection.Connection(fedora_url, username=user, password=passcode)
self.client = FedoraClient(self.fc)
def get_client():
"""Returns an instance of fcrepo..client.FedoraClient with the connection
settings specified in settings.py"""
connection = Connection(FEDORA_INSTANCE,
username=RTV_FEDORA_USER,
password=RTV_FEDORA_PASSWORD)
return FedoraClient(connection)
def __init__(self,
host='localhost',
port=61613,
user='',
passcode='',
fedora_url=''):
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
self.transaction_id = None
self.fc = fcrepo.connection.Connection(fedora_url,
username=user,
password=passcode)
self.client = FedoraClient(self.fc)
self.fedora_url = fedora_url
self.user = user
self.password = passcode
def __init__(self, host='localhost', port=61613, user='', passcode='', fedora_url=''):
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
self.transaction_id = None
self.fc = fcrepo.connection.Connection(fedora_url, username = user, password = passcode)
self.client = FedoraClient(self.fc)
self.fedora_url = fedora_url
self.user = user
self.password = passcode
def init_fedora_connection(self):
try:
self.logger.info('Worker %s attempting to connect to Fedora server.', self.worker_id)
self.fedora_connection = Connection(
self.config.get('Fedora', 'fedora_url'),
username = self.config.get('Fedora', 'fedora_admin_user'),
password = self.config.get('Fedora', 'fedora_admin_password')
)
self.fedora_client = FedoraClient(self.fedora_connection)
self.logger.info('Worker %s connected to Fedora server.', self.worker_id)
return True
except:
self.logger.error('Worker %s failed to connect to Fedora server.', self.worker_id)
return False
def startFcrepo():
'''
helper function that starts up the fedora connection
'''
connection = Connection(fedoraUrl,
username=fedoraUserName,
password=fedoraPassword)
global fedora
try:
fedora = FedoraClient(connection)
except FedoraConnectionException:
logging.error('Error connecting to fedora, exiting' + '\n')
sys.exit()
return True
logFile = os.path.join(log_directory,'/big2/dc/Digital-Collections/archival-objects/mcmaster-ontario-conservation-reports' + time.strftime('%y_%m_%d') + '.log')
logging.basicConfig(filename=logFile, level=logging.DEBUG)
#get config
config = ConfigParser.ConfigParser()
config.read(os.path.join(source_directory,'mcmaster.cfg'))
#config.read(os.path.join(source_directory,'TEST.cfg'))
solrUrl = config.get('Solr','url')
fedoraUrl = config.get('Fedora','url')
fedoraUserName = config.get('Fedora', 'username')
fedoraPassword = config.get('Fedora','password')
#get fedora connection
connection = Connection(fedoraUrl, username=fedoraUserName, password=fedoraPassword)
try:
fedora=FedoraClient(connection)
except FedoraConnectionException:
logging.error('Error connecting to fedora, exiting'+'\n')
sys.exit()
#setup the directories
metadata_directory = os.path.join(source_directory, 'metadata')
if not os.path.isdir(metadata_directory):
logging.error('METADATA directory invalid \n')
sys.exit()
mods_directory = os.path.join(source_directory, 'metadata')
if not os.path.isdir(mods_directory):
logging.error('MODS directory invalid \n')
sys.exit()
class ContentModelListener(ConnectionListener):
'''
classdocs
'''
def __init__(self, content_models, host='localhost', port=61613, user='', passcode='', fedora_url=''):
'''
Constructor
'''
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
logging.info('Connecting to STOMP server %(host)s on port %(port)s.' % {'host': host, 'port': port})
self.transaction_id = None
logging.info("Connecting to Fedora server at %(url)s" % {'url': fedora_url})
self.fc = fcrepo.connection.Connection(fedora_url, username = user, password = passcode)
self.client = FedoraClient(self.fc)
self.fedora_url = fedora_url
self.username = user
self.password = passcode
# Create plugin manager
self.manager = PluginManager(categories_filter = {"FedoraMicroService": FedoraMicroService})
plugin_path = os.path.dirname(__file__)
self.manager.setPluginPlaces([plugin_path + "/plugins"])
logging.debug("Plugin path: " + plugin_path + "/plugins")
# Load plugins.
self.manager.locatePlugins()
self.manager.loadPlugins()
self.contentModels = {}
for plugin in self.manager.getPluginsOfCategory("FedoraMicroService"):
# plugin.plugin_object is an instance of the plubin
logging.info("Loading plugin: %(name)s for content model %(cmodel)s." % {'name': plugin.plugin_object.name, 'cmodel': plugin.plugin_object.content_model})
plugin.plugin_object.config = config
if type(plugin.plugin_object.content_model) == types.StringType:
content_models = [plugin.plugin_object.content_model]
else:
content_models = plugin.plugin_object.content_model
for content_model in content_models:
if content_model in self.contentModels:
self.contentModels[content_model].append(plugin.plugin_object)
else:
self.contentModels[content_model] = [plugin.plugin_object]
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
#logging.debug("\r \r", end='')
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug('%s: %s' % (header_key, headers[header_key]))
logging.debug(body)
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
logging.error("lost connection reconnect in %d sec..." % reconnect_wait)
signal.alarm(reconnect_wait)
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
try:
global TOPIC_PREFIX
self.__print_async('MESSAGE', headers, body)
pid = headers['pid']
dsid = headers['dsid']
obj = self.client.getObject(pid)
content_model = headers['destination'][len(TOPIC_PREFIX):]
if content_model in self.contentModels:
logging.info('Running rules for %(pid)s from %(cmodel)s.' % {'pid': obj.pid, 'cmodel': content_model} )
for plugin in self.contentModels[content_model]:
plugin.runRules(obj, dsid, body)
except FedoraConnectionException:
logging.warning('Object %s was not found.' % (pid))
except:
logging.error("an exception occurred: " + str(sys.exc_info()[0]))
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers = { 'message-id' : args[1]})
else:
self.conn.ack(headers = { 'message-id' : args[1]}, transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, correlation_id, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination, message=message, headers={'correlation-id': correlation_id})
def subscribe(self, destination, ack='auto'):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
def connect(self):
self.conn.start()
self.fc = fcrepo.connection.Connection(self.fedora_url, username = self.username, password = self.password)
self.client = FedoraClient(self.fc)
def connect(self):
self.conn.start()
self.fc = fcrepo.connection.Connection(self.fedora_url, username = self.username, password = self.password)
self.client = FedoraClient(self.fc)
#get config
config = ConfigParser.ConfigParser()
config.read(os.path.join(source_directory,'HAMILTON.cfg'))
#config.read(os.path.join(source_directory,'TEST.cfg'))
solrUrl=config.get('Solr','url')
fedoraUrl=config.get('Fedora','url')
fedoraUserName=config.get('Fedora', 'username')
fedoraPassword=config.get('Fedora','password')
#get fedora connection
connection = Connection(fedoraUrl,
username=fedoraUserName,
password=fedoraPassword)
try:
fedora=FedoraClient(connection)
except FedoraConnectionException:
logging.error('Error connecting to fedora, exiting \n')
sys.exit()
#setup the directories
mods_directory = os.path.join(source_directory, 'mods')
if not os.path.isdir(mods_directory):
logging.error('MODS directory invalid \n')
sys.exit()
data_directory = os.path.join(source_directory, 'data')
if not os.path.isdir(data_directory):
logging.error('Data directory invalid \n')
sys.exit()
def connect(self):
self.conn.start()
self.fc = fcrepo.connection.Connection(self.fedora_url,
username=self.username,
password=self.password)
self.client = FedoraClient(self.fc)
class StompFedora(ConnectionListener):
"""
A custom interface to the stomp.py client. See \link stomp::internal::connect::Connection \endlink
for more information on establishing a connection to a stomp server.
"""
def __init__(self, host='localhost', port=61613, user='', passcode='', fedora_url=''):
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
self.transaction_id = None
self.fc = fcrepo.connection.Connection(fedora_url, username = user, password = passcode)
self.client = FedoraClient(self.fc)
self.fedora_url = fedora_url
self.user = user
self.password = passcode
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug('%s: %s' % (header_key, headers[header_key]))
logging.debug(body)
def __get_content_models(self, pid):
"""
Get a list of content models that apply to the object.
"""
obj = self.client.getObject(pid)
if 'RELS-EXT' in obj:
ds = obj['RELS-EXT']
return obj, [elem['value'].split('/')[1] for elem in ds[NS.fedoramodel.hasModel]]
else:
return obj, []
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
logging.error("lost connection reconnect in %d sec..." % reconnect_wait)
signal.alarm(reconnect_wait)
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
self.__print_async("MESSSAGE", headers, body)
method = headers['methodName']
pid = headers['pid']
newheaders = { 'methodname':headers['methodName'], 'pid':headers['pid']}
if method in ['addDatastream', 'modifyDatastreamByValue', 'modifyDatastreamByReference', 'modifyObject']:
f = feedparser.parse(body)
tags = f['entries'][0]['tags']
dsids = [tag['term'] for tag in tags if tag['scheme'] == 'fedora-types:dsID']
dsid = ''
if dsids:
dsid = dsids[0]
newheaders['dsid'] = dsid
obj, content_models = self.__get_content_models(pid) # and 'OBJ' in dsIDs:
for content_model in content_models:
logging.info("/topic/fedora.contentmodel.%s %s" % (content_model, dsid))
self.send("/topic/fedora.contentmodel.%s" % content_model, newheaders, body)
elif method in ['ingest']:
obj, content_models = self.__get_content_models(pid)
for dsid in obj:
for content_model in content_models:
newheaders['dsid'] = dsid
logging.info("/topic/fedora.contentmodel.%s %s" % (content_model, dsid))
self.send("/topic/fedora.contentmodel.%s" % content_model, newheaders, body)
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers = { 'message-id' : args[1]})
else:
self.conn.ack(headers = { 'message-id' : args[1]}, transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args=None):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, headers, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination, message=message, headers=headers)
def subscribe(self, destination, ack='auto'):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def connect(self):
self.conn.start()
self.fc = fcrepo.connection.Connection(self.fedora_url, username = self.user, password = self.password)
self.client = FedoraClient(self.fc)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
from categories import FedoraMicroService
sys.path.append( 'plugins' )
from coalliance_coccOralHistoryCModel import coalliance_coccOralHistoryCModel
co = coalliance_coccOralHistoryCModel()
fedora_url = 'http://localhost:8080/fedora'
username = '******'
password = '******'
log_filename = 'script.log'
body = ''
pids = [
#pids to work on here
]
levels = {'DEBUG':logging.DEBUG, 'INFO': logging.INFO, 'WARNING': logging.WARNING, 'ERROR':logging
.ERROR, 'CRITICAL':logging.CRITICAL, 'FATAL':logging.FATAL}
logging.basicConfig(filename = log_filename, level = levels['INFO'])
fc = fcrepo.connection.Connection(fedora_url, username = username, password = password)
client = FedoraClient(fc)
for pid in pids:
obj = client.getObject(pid)
logging.info("Processing Pid: %s"%(obj.pid))
for dsid in obj:
co.runRules(obj, dsid, body)
#usage: 'python purge.py START_PID END_PID'
import sys
import logging, sys, os, ConfigParser, time, subprocess
from fcrepo.connection import Connection, FedoraConnectionException
from fcrepo.client import FedoraClient
#get config
config = ConfigParser.ConfigParser()
config.read('mcmaster.cfg')
fedoraUrl = config.get('Fedora','url')
fedoraUserName = config.get('Fedora', 'username')
fedoraPassword = config.get('Fedora','password')
START_PID = int(sys.argv[1])
END_PID = int(sys.argv[2])
c = Connection(fedoraUrl, username=fedoraUserName, password=fedoraPassword)
fc = FedoraClient(c)
for i in range((START_PID), (END_PID)+1):
pids = []
term = u'pid~macrepo:' + unicode(i) + '-*'
results = fc.searchObjects(term, ['pid'])
for r in results:
pids.append(r['pid'])
for pid in pids:
fc.deleteObject(pid[0])
print(pid)[0]
c.close()
class ContentModelListener(ConnectionListener):
'''
classdocs
'''
def __init__(self,
content_models,
host='localhost',
port=61613,
user='',
passcode='',
fedora_url=''):
'''
Constructor
'''
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
logging.info('Connecting to STOMP server %(host)s on port %(port)s.' %
{
'host': host,
'port': port
})
self.transaction_id = None
logging.info("Connecting to Fedora server at %(url)s" %
{'url': fedora_url})
self.fc = fcrepo.connection.Connection(fedora_url,
username=user,
password=passcode)
self.client = FedoraClient(self.fc)
# Create plugin manager
self.manager = PluginManager(
categories_filter={"FedoraMicroService": FedoraMicroService})
self.manager.setPluginPlaces(["plugins"])
# Load plugins.
self.manager.locatePlugins()
self.manager.loadPlugins()
self.contentModels = {}
for plugin in self.manager.getPluginsOfCategory("FedoraMicroService"):
# plugin.plugin_object is an instance of the plubin
logging.info(
"Loading plugin: %(name)s for content model %(cmodel)s." % {
'name': plugin.plugin_object.name,
'cmodel': plugin.plugin_object.content_model
})
plugin.plugin_object.config = config
if plugin.plugin_object.content_model in self.contentModels:
self.contentModels[plugin.plugin_object.content_model].append(
plugin.plugin_object)
else:
self.contentModels[plugin.plugin_object.content_model] = [
plugin.plugin_object
]
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
#logging.debug("\r \r", end='')
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug('%s: %s' % (header_key, headers[header_key]))
logging.debug(body)
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
global TOPIC_PREFIX
self.__print_async('MESSAGE', headers, body)
f = feedparser.parse(body)
tags = f['entries'][0]['tags']
pid = [
tag['term'] for tag in tags if tag['scheme'] == 'fedora-types:pid'
][0]
dsID = [
tag['term'] for tag in tags if tag['scheme'] == 'fedora-types:dsID'
][0]
obj = self.client.getObject(pid)
content_model = headers['destination'][len(TOPIC_PREFIX):]
if content_model in self.contentModels:
logging.info('Running rules for %(pid)s from %(cmodel)s.' % {
'pid': obj.pid,
'cmodel': content_model
})
for plugin in self.contentModels[content_model]:
plugin.runRules(obj, dsID)
return
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers={'message-id': args[1]})
else:
self.conn.ack(headers={'message-id': args[1]},
transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, correlation_id, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination,
message=message,
headers={'correlation-id': correlation_id})
def subscribe(self, destination, ack='auto'):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
#get config
config = ConfigParser.ConfigParser()
config.read(os.path.join(source_directory, 'HAMILTON.cfg'))
#config.read(os.path.join(source_directory,'TEST.cfg'))
solrUrl = config.get('Solr', 'url')
fedoraUrl = config.get('Fedora', 'url')
fedoraUserName = config.get('Fedora', 'username')
fedoraPassword = config.get('Fedora', 'password')
#get fedora connection
connection = Connection(fedoraUrl,
username=fedoraUserName,
password=fedoraPassword)
try:
fedora = FedoraClient(connection)
except FedoraConnectionException:
logging.error('Error connecting to fedora, exiting' + '\n')
sys.exit()
#setup the directories
mods_directory = os.path.join(source_directory, 'mods-xml')
if not os.path.isdir(mods_directory):
logging.error('MODS directory invalid \n')
sys.exit()
tei_directory = os.path.join(source_directory, 'tei-xml')
if not os.path.isdir(tei_directory):
logging.error('TEI directory invalid \n')
sys.exit()
def connectToFedora(url, user, pw):
"""
Attempt to create a connection to fedora using the supplied username and password. If the
connection succeeds, return the connected fedora client, otherwise return None. The calling
function should terminate if None is received.
"""
try:
connection = Connection(url, username=user, password=pw)
except Exception, ex:
print("Error while connecting to fedoraUrl: %s", ex.message)
return None
try:
return FedoraClient(connection)
except Exception, ex:
print("Exception while opening fedora client")
print(
"Check if fedora is running and your login information is correct")
return None
""" ====== MANAGING FEDORA OBJECTS ====== """
def createRelsExt(childObject,
parentPid,
contentModel,
extraNamespaces={},
extraRelationships={}):
logFile = os.path.join(log_directory,'/big2/dc/Digital-Collections/archival-objects/WWIIJUR' + time.strftime('%y_%m_%d') + '.log')
logging.basicConfig(filename=logFile, level=logging.DEBUG)
#get config
config = ConfigParser.ConfigParser()
config.read(os.path.join(source_directory,'mcmaster.cfg'))
#config.read(os.path.join(source_directory,'TEST.cfg'))
solrUrl = config.get('Solr','url')
fedoraUrl = config.get('Fedora','url')
fedoraUserName = config.get('Fedora', 'username')
fedoraPassword = config.get('Fedora','password')
#get fedora connection
connection = Connection(fedoraUrl, username=fedoraUserName, password=fedoraPassword)
try:
fedora=FedoraClient(connection)
except FedoraConnectionException:
logging.error('Error connecting to fedora, exiting'+'\n')
sys.exit()
#setup the directories
mods_directory = os.path.join(source_directory, '/big2/dc/Digital-Collections/archival-objects/WWIIJUR/mods')
if not os.path.isdir(mods_directory):
logging.error('MODS directory invalid \n')
sys.exit()
macrepo_directory = os.path.join(source_directory, '/big2/dc/Digital-Collections/archival-objects/WWIIJUR/macrepo')
if not os.path.isdir(macrepo_directory):
logging.error('MODS directory invalid \n')
sys.exit()
class islandoraSurrogate(object):
def __init__(self, pid, dsid, worker_id, language, logger, config):
self.init_config(config)
self.init_logger(logger)
self.init_properties(worker_id, pid, dsid, language)
if self.load_fedora_object() and self.check_fedora_object_dsid():
self.fetch_fedora_object_file()
def add_fedora_object_ds(self, ds_label = '', ds_mimeType = '', control_group = 'M', connect_tries = 3, ds_checksumType = None, ds_checksum = None):
return update_datastream(
self.fedora_object,
self.dsid,
self.converted_file_path,
label = ds_label,
mimeType = ds_mimeType,
controlGroup = control_group,
tries = connect_tries,
checksumType = ds_checksumType,
checksum = ds_checksum
)
def append_additional_encode_options(self, call_list, extra_options_variable, encoder_name):
extra_options = self.config.get(self.dsid, extra_options_variable)
if not extra_options == '':
self.logger.info('Worker %s appending extra options %s to %s for %s.', self.worker_id, self.convert_comma_separated_options_to_list(extra_options), encoder_name, self.dsid)
call_list.extend(self.convert_comma_separated_options_to_list(extra_options))
def check_fedora_object_dsid(self):
self.logger.info('Worker %s checking that %s does not exist in PID %s.', self.worker_id, self.dsid, self.pid)
if self.dsid in self.fedora_object:
self.duplicate = True
return False
else:
self.logger.info('Worker %s reports %s does not exist in PID %s.', self.worker_id, self.dsid, self.pid)
return True
def convert_comma_separated_options_to_list(self, options_string):
return re.split("[ ,]+", options_string)
def fetch_fedora_object_file(self):
self.temp_file_dir, temp_filename = get_datastream_as_file(self.fedora_object, 'OBJ', 'jpg')
self.temp_filepath = '/'.join( (self.temp_file_dir, temp_filename) )
self.logger.info('Worker %s pulled OBJ from %s to %s.', self.worker_id, self.pid, self.temp_filepath)
def init_config(self, config):
self.config = config
def init_fedora_connection(self):
try:
self.logger.info('Worker %s attempting to connect to Fedora server.', self.worker_id)
self.fedora_connection = Connection(
self.config.get('Fedora', 'fedora_url'),
username = self.config.get('Fedora', 'fedora_admin_user'),
password = self.config.get('Fedora', 'fedora_admin_password')
)
self.fedora_client = FedoraClient(self.fedora_connection)
self.logger.info('Worker %s connected to Fedora server.', self.worker_id)
return True
except:
self.logger.error('Worker %s failed to connect to Fedora server.', self.worker_id)
return False
def init_logger(self, logger):
self.logger = logger
def init_properties(self, worker_id, pid, dsid, language):
self.converted_file_path = False
self.duplicate = False
self.temp_filepath = False
self.worker_id = worker_id
self.pid = pid
self.dsid = dsid
self.language = language
def load_fedora_object(self):
if self.init_fedora_connection():
self.logger.info('Worker %s getting object %s from %s.', self.worker_id, self.pid, self.config.get('Fedora', 'fedora_url'))
self.fedora_object = self.fedora_client.getObject(self.pid)
return True
return False
def log_encode_begin(self, output_filename):
self.logger.info('Worker %s encoding %s %s surrogate to %s.', self.worker_id, self.pid, self.dsid, output_filename)
def log_encode_fail(self):
self.logger.error('Worker %s encoding %s surrogate of %s has failed.', self.worker_id, self.dsid, self.pid)
def log_encode_success(self):
self.logger.info('Worker %s encoding %s surrogate of %s has succeeded.', self.worker_id, self.dsid, self.pid)
def remove_temp_files(self):
try:
rmtree(self.temp_file_dir)
self.logger.info('Worker %s removing temporary %s files.', self.worker_id, self.pid)
except:
self.logger.info('Worker %s could not remove temporary %s files.', self.worker_id, self.pid)
#get config
config = ConfigParser.ConfigParser()
config.read(os.path.join(source_directory, 'HAMILTON.cfg'))
#config.read(os.path.join(source_directory,'TEST.cfg'))
solrUrl = config.get('Solr', 'url')
fedoraUrl = config.get('Fedora', 'url')
fedoraUserName = config.get('Fedora', 'username')
fedoraPassword = config.get('Fedora', 'password')
#get fedora connection
connection = Connection(fedoraUrl,
username=fedoraUserName,
password=fedoraPassword)
try:
fedora = FedoraClient(connection)
except FedoraConnectionException:
logging.error('Error connecting to fedora, exiting \n')
sys.exit()
#setup the directories
mods_directory = os.path.join(source_directory, 'mods')
if not os.path.isdir(mods_directory):
logging.error('MODS directory invalid \n')
sys.exit()
data_directory = os.path.join(source_directory, 'data')
if not os.path.isdir(data_directory):
logging.error('Data directory invalid \n')
sys.exit()
fedora_url = 'http://localhost:8080/fedora'
username = '******'
password = '******'
log_filename = 'script.log'
body = ''
pids = [
#pids to work on here
]
levels = {
'DEBUG': logging.DEBUG,
'INFO': logging.INFO,
'WARNING': logging.WARNING,
'ERROR': logging.ERROR,
'CRITICAL': logging.CRITICAL,
'FATAL': logging.FATAL
}
logging.basicConfig(filename=log_filename, level=levels['INFO'])
fc = fcrepo.connection.Connection(fedora_url,
username=username,
password=password)
client = FedoraClient(fc)
for pid in pids:
obj = client.getObject(pid)
logging.info("Processing Pid: %s" % (obj.pid))
for dsid in obj:
co.runRules(obj, dsid, body)
class StompFedora(ConnectionListener):
"""
A custom interface to the stomp.py client. See \link stomp::internal::connect::Connection \endlink
for more information on establishing a connection to a stomp server.
"""
def __init__(self, host="localhost", port=61613, user="", passcode="", fedora_url=""):
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener("", self)
self.conn.start()
self.transaction_id = None
self.fc = fcrepo.connection.Connection(fedora_url, username=user, password=passcode)
self.client = FedoraClient(self.fc)
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug("%s: %s" % (header_key, headers[header_key]))
logging.debug(body)
def __get_content_models(self, pid):
"""
Get a list of content models that apply to the object.
"""
obj = self.client.getObject(pid)
if "RELS-EXT" in obj:
ds = obj["RELS-EXT"]
return [elem["value"].split("/")[1] for elem in ds[NS.fedoramodel.hasModel]]
else:
return []
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
sysout("lost connection")
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
self.__print_async("MESSSAGE", headers, body)
f = feedparser.parse(body)
method = headers["methodName"]
if method in ["addDatastream", "modifyDatastreamByValue", "modifyDatastreamByReference"]:
tags = f["entries"][0]["tags"]
pid = [tag["term"] for tag in tags if tag["scheme"] == "fedora-types:pid"][0]
dsIDs = [tag["term"] for tag in tags if tag["scheme"] == "fedora-types:dsID"]
content_models = self.__get_content_models(pid) # and 'OBJ' in dsIDs:
for content_model in content_models:
print "/topic/fedora.contentmodel.%s" % content_model
self.send("/topic/fedora.contentmodel.%s" % content_model, "", body)
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers={"message-id": args[1]})
else:
self.conn.ack(headers={"message-id": args[1]}, transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, correlation_id, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination, message=message, headers={"correlation-id": correlation_id})
def subscribe(self, destination, ack="auto"):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
logging.basicConfig(filename=logFile, level=logging.DEBUG)
#get config
config = ConfigParser.ConfigParser()
config.read(os.path.join(source_directory,'mcmaster.cfg'))
solrUrl = config.get('Solr','url')
fedoraUrl = config.get('Fedora','url')
fedoraUserName = config.get('Fedora', 'username')
fedoraPassword = config.get('Fedora','password')
#get fedora connection
connection = Connection(fedoraUrl,
username=fedoraUserName,
password=fedoraPassword)
try:
fedora=FedoraClient(connection)
except FedoraConnectionException:
logging.error('Error connecting to fedora, exiting'+'\n')
sys.exit()
#setup the directories
mods_directory = os.path.join(source_directory, '/big2/dc/Digital-Collections/archival-objects/ITM/mods')
if not os.path.isdir(mods_directory):
logging.error('MODS directory invalid \n')
sys.exit()
tif_directory = os.path.join(source_directory, '/big2/dc/Digital-Collections/archival-objects/ITM/tif')
if not os.path.isdir(tif_directory):
logging.error('TIF directory invalid \n')
sys.exit()
class StompFedora(ConnectionListener):
"""
A custom interface to the stomp.py client. See \link stomp::internal::connect::Connection \endlink
for more information on establishing a connection to a stomp server.
"""
def __init__(self,
host='localhost',
port=61613,
user='',
passcode='',
fedora_url=''):
self.conn = Connection([(host, port)], user, passcode)
self.conn.set_listener('', self)
self.conn.start()
self.transaction_id = None
self.fc = fcrepo.connection.Connection(fedora_url,
username=user,
password=passcode)
self.client = FedoraClient(self.fc)
self.fedora_url = fedora_url
self.user = user
self.password = passcode
def __print_async(self, frame_type, headers, body):
"""
Utility function for printing messages.
"""
logging.debug(frame_type)
for header_key in headers.keys():
logging.debug('%s: %s' % (header_key, headers[header_key]))
logging.debug(body)
def __get_content_models(self, pid):
"""
Get a list of content models that apply to the object.
"""
obj = self.client.getObject(pid)
if 'RELS-EXT' in obj:
ds = obj['RELS-EXT']
return obj, [
elem['value'].split('/')[1]
for elem in ds[NS.fedoramodel.hasModel]
]
else:
return obj, []
def on_connecting(self, host_and_port):
"""
\see ConnectionListener::on_connecting
"""
self.conn.connect(wait=True)
def on_disconnected(self):
"""
\see ConnectionListener::on_disconnected
"""
logging.error("lost connection reconnect in %d sec..." %
reconnect_wait)
signal.alarm(reconnect_wait)
def on_message(self, headers, body):
"""
\see ConnectionListener::on_message
"""
self.__print_async("MESSSAGE", headers, body)
method = headers['methodName']
pid = headers['pid']
newheaders = {
'methodname': headers['methodName'],
'pid': headers['pid']
}
if method in [
'addDatastream', 'modifyDatastreamByValue',
'modifyDatastreamByReference', 'modifyObject'
]:
f = feedparser.parse(body)
tags = f['entries'][0]['tags']
dsids = [
tag['term'] for tag in tags
if tag['scheme'] == 'fedora-types:dsID'
]
dsid = ''
if dsids:
dsid = dsids[0]
newheaders['dsid'] = dsid
obj, content_models = self.__get_content_models(
pid) # and 'OBJ' in dsIDs:
for content_model in content_models:
logging.info("/topic/fedora.contentmodel.%s %s" %
(content_model, dsid))
self.send("/topic/fedora.contentmodel.%s" % content_model,
newheaders, body)
elif method in ['ingest']:
obj, content_models = self.__get_content_models(pid)
for dsid in obj:
for content_model in content_models:
newheaders['dsid'] = dsid
logging.info("/topic/fedora.contentmodel.%s %s" %
(content_model, dsid))
self.send("/topic/fedora.contentmodel.%s" % content_model,
newheaders, body)
def on_error(self, headers, body):
"""
\see ConnectionListener::on_error
"""
self.__print_async("ERROR", headers, body)
def on_connected(self, headers, body):
"""
\see ConnectionListener::on_connected
"""
self.__print_async("CONNECTED", headers, body)
def ack(self, args):
"""
Required Parameters:
message-id - the id of the message being acknowledged
Description:
Acknowledge consumption of a message from a subscription using client
acknowledgement. When a client has issued a subscribe with an 'ack' flag set to client
received from that destination will not be considered to have been consumed (by the server) until
the message has been acknowledged.
"""
if not self.transaction_id:
self.conn.ack(headers={'message-id': args[1]})
else:
self.conn.ack(headers={'message-id': args[1]},
transaction=self.transaction_id)
def abort(self, args):
"""
Description:
Roll back a transaction in progress.
"""
if self.transaction_id:
self.conn.abort(transaction=self.transaction_id)
self.transaction_id = None
def begin(self, args):
"""
Description
Start a transaction. Transactions in this case apply to sending and acknowledging
any messages sent or acknowledged during a transaction will be handled atomically based on teh
transaction.
"""
if not self.transaction_id:
self.transaction_id = self.conn.begin()
def commit(self, args):
"""
Description:
Commit a transaction in progress.
"""
if self.transaction_id:
self.conn.commit(transaction=self.transaction_id)
self.transaction_id = None
def disconnect(self, args=None):
"""
Description:
Gracefully disconnect from the server.
"""
try:
self.conn.disconnect()
except NotConnectedException:
pass
def send(self, destination, headers, message):
"""
Required Parametes:
destination - where to send the message
message - the content to send
Description:
Sends a message to a destination in the message system.
"""
self.conn.send(destination=destination,
message=message,
headers=headers)
def subscribe(self, destination, ack='auto'):
"""
Required Parameters:
destination - the name to subscribe to
Optional Parameters:
ack - how to handle acknowledgements for a message, either automatically (auto) or manually (client)
Description
Register to listen to a given destination. Like send, the subscribe command requires a destination
header indicating which destination to subscribe to. The ack parameter is optional, and defaults to auto.
"""
self.conn.subscribe(destination=destination, ack=ack)
def connect(self):
self.conn.start()
self.fc = fcrepo.connection.Connection(self.fedora_url,
username=self.user,
password=self.password)
self.client = FedoraClient(self.fc)
def unsubscribe(self, destination):
"""
Required Parameters:
destination - the name to unsubscribe from
Description:
Remove an existing subscription - so that the client no longer receives messages from that destination.
"""
self.conn.unsubscribe(destination)
