def main():
import os.path
from optparse import OptionParser
from lofar.common import dbcredentials
# Check the invocation arguments
parser = OptionParser("%prog [options] <path_to_jobfile.xml>",
description='Run the ingestpipeline on a single jobfile.')
parser.add_option('-q', '--broker', dest='broker', type='string', default=DEFAULT_BROKER, help='Address of the qpid broker, default: %default')
parser.add_option('--busname', dest='busname', type='string', default=DEFAULT_BUSNAME, help='Name of the bus exchange on the qpid broker on which the ingest notifications are published, default: %default')
parser.add_option("-u", "--user", dest="user", type="string", default=getpass.getuser(), help="username for to login on <host>, [default: %default]")
parser.add_option('-s', '--minimal-SIP', dest='minimal_SIP', action='store_true', help='create and upload a minimal SIP to the LTA catalogue when the normal SIP is not accepted.')
parser.add_option('-V', '--verbose', dest='verbose', action='store_true', help='verbose logging')
parser.add_option('-t', '--timeout', dest='globus_timeout', type='int', default=GLOBUS_TIMEOUT, help='number of seconds (default=%default) to wait for globus-url-copy to finish after the transfer is done (while lta-site is computing checksums)')
parser.add_option("-l", "--lta_credentials", dest="lta_credentials", type="string",
default='LTA' if isProductionEnvironment() else 'LTA_test',
help="Name of lofar credentials for lta user/pass (see ~/.lofar/dbcredentials) [default=%default]")
parser.add_option("-m", "--mom_credentials", dest="mom_credentials", type="string",
default='MoM_site' if isProductionEnvironment() else 'MoM_site_test',
help="Name of credentials for MoM user/pass (see ~/.lofar/dbcredentials) [default=%default]")
(options, args) = parser.parse_args()
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG if options.verbose else logging.INFO)
if len(args) != 1:
parser.print_help()
sys.exit(1)
try:
path = args[0]
if os.path.isfile(path):
job = parseJobXmlFile(path)
job['filename'] = path
logger.info("Parsed jobfile %s: %s", path, job)
ltacreds = dbcredentials.DBCredentials().get(options.lta_credentials)
ltaClient = LTAClient(ltacreds.user, ltacreds.password)
momcreds = dbcredentials.DBCredentials().get(options.mom_credentials)
momClient = MoMClient(momcreds.user, momcreds.password)
jobPipeline = IngestPipeline(job, momClient, ltaClient,
busname=options.busname,
broker=options.broker,
user=options.user,
globus_timeout=options.globus_timeout,
minimal_SIP=options.minimal_SIP)
jobPipeline.run()
exit(0)
else:
logger.info("No such file %s", path)
exit(1)
except Exception as e:
logger.error(e)
exit(1)
def __init__(self, user=None, password=None):
if user == None or password == None:
# (mis)use dbcredentials to read user/pass from disk
from lofar.common import dbcredentials
dbc = dbcredentials.DBCredentials()
creds = dbc.get('LTA' if isProductionEnvironment() else 'LTA_test')
user = creds.user
password = creds.password
url = LTA_BASE_URL % (user, password)
self.__rpc = xmlrpc.client.ServerProxy(url)
logger.info('LTAClient connected to: %s', self.__hidePassword(url))
def __init__(self, mom_base_url, user=None, password=None):
if user == None or password == None:
# (mis)use dbcredentials to read user/pass from disk
from lofar.common import dbcredentials
dbc = dbcredentials.DBCredentials()
creds = dbc.get(
'MoM_site' if isProductionEnvironment() else 'MoM_site_test')
user = creds.user
password = creds.password
self.mom_base_url = mom_base_url
self.__user = user
self.__password = password
self.session = None
self.__momURLlogin = self.mom_base_url + 'useradministration/user/systemlogin.do'
self.__momUR_security_check = self.mom_base_url + 'useradministration/user/j_security_check'
self.__momURLlogout = self.mom_base_url + 'useradministration/user/logout.do'
#!/usr/bin/python
import sys
from lofar.qpidinfrastructure.QPIDDB import qpidinfra
from lofar.common import dbcredentials
dbcreds = dbcredentials.DBCredentials().get("qpidinfra")
todb = qpidinfra(dbcreds)
tosearch = sys.stdin.readlines()
numlines = len(tosearch)
#fqdn=['lhn001.cep2.lofar',
# 'ccu001.control.lofar',
# 'sas001.control.lofar',
# 'cbm001.control.lofar',
# 'cbm001.control.lofar',
# 'cbm001.control.lofar', 'cbm001.control.lofar', 'cbm001.control.lofar', 'cbm001.control.lofar', 'cbm001.control.lofar', 'cbm001.control.lofar', 'cbm001.control.lofar', 'cbm001.control.lofar', 'cbm001.control.lofar', 'cbm001.control.lofar',
def to_hostname(s):
leafname = s.split(':')[0].split('.')[0].lower()
fqdn = "%s.control.lofar" % (leafname)
#print leafname[0:5]
if (leafname[0:5] == 'locus'):
fqdn = "%s.cep2.lofar" % (leafname)
if (leafname == 'lhn001'):
fqdn = "%s.cep2.lofar" % (leafname)
return fqdn
https://docs.djangoproject.com/en/1.10/ref/settings/
"""
import os
import ldap
from . import credentials
from lofar.common import dbcredentials
import logging
logger = logging.getLogger('django_auth_ldap')
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.INFO)
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
dbc = dbcredentials.DBCredentials()
restinterface_credentials = dbc.get("trigger_restinterface")
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = credentials.SECRET_KEY
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = ['*']
# Application definition
def main():
# make sure we run in UTC timezone
import os
os.environ['TZ'] = 'UTC'
from optparse import OptionParser
# Check the invocation arguments
parser = OptionParser(
"%prog [options]",
description=
'runs the ingest transfer server which picks up as many jobs as it can handle from the given --ingest_job_queuename and tranfers the dataproducts to the LTA, updates the LTA catalogue, and updates MoM'
)
parser.add_option('-b',
'--broker',
dest='broker',
type='string',
default=DEFAULT_BROKER,
help='Address of the qpid broker, default: %default')
parser.add_option("-p",
"--max_nr_of_parallel_jobs",
dest="max_nr_of_parallel_jobs",
type="int",
default=MAX_NR_OF_JOBS,
help="Name of the job queue. [default: %default]")
parser.add_option(
'-e',
'--exchange',
dest='exchange',
type='string',
default=DEFAULT_BUSNAME,
help='Name of the common bus exchange on the broker, default: %default'
)
parser.add_option(
"-u",
"--user",
dest="user",
type="string",
default=getpass.getuser(),
help="username for to login on data source host, [default: %default]")
parser.add_option(
"-l",
"--lta_credentials",
dest="lta_credentials",
type="string",
default='LTA' if isProductionEnvironment() else 'LTA_test',
help=
"Name of lofar credentials for lta user/pass (see ~/.lofar/dbcredentials) [default=%default]"
)
parser.add_option(
"-m",
"--mom_credentials",
dest="mom_credentials",
type="string",
default='MoM_site' if isProductionEnvironment() else 'MoM_site_test',
help=
"Name of credentials for MoM user/pass (see ~/.lofar/dbcredentials) [default=%default]"
)
parser.add_option('-V',
'--verbose',
dest='verbose',
action='store_true',
help='verbose logging')
(options, args) = parser.parse_args()
logging.basicConfig(
format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG if options.verbose else logging.INFO)
logger.info('*****************************************')
logger.info('Started ingest server on host %s', socket.gethostname())
logger.info('*****************************************')
logger.info("environment:")
for k in sorted(os.environ):
logger.info("%s=%s", k, os.environ[k])
logger.info('*****************************************')
ltacreds = dbcredentials.DBCredentials().get(options.lta_credentials)
momcreds = dbcredentials.DBCredentials().get(options.mom_credentials)
transfer_server = IngestTransferServer(
exchange=options.exchange,
broker=options.broker,
mom_credentials=momcreds,
lta_credentials=ltacreds,
max_nr_of_parallel_jobs=options.max_nr_of_parallel_jobs)
incoming_jobs_listener = BusListener(
IngestJobsForTransferHandler, {'transfer_server': transfer_server},
exchange=options.exchange,
routing_key="%s.#" % DEFAULT_INGEST_JOB_FOR_TRANSFER_SUBJECT)
with incoming_jobs_listener:
transfer_server.run()
site_id = site['id']
site_name = site['name']
siteStats[site_name] = {'site_id': site_id}
visits = self.executeQuery('''
select *
from scraper.site_scraper_last_directory_visit
where site_id = %s
and last_visit < %s
order by last_visit asc
''', [site_id, before_timestamp],
fetch=FETCH_ALL)
siteStats[site_name]['queue_length'] = len(visits)
if len(visits) > 0:
siteStats[site_name]['least_recent_visited_dir_id'] = visits[
0]['dir_id']
siteStats[site_name]['least_recent_visit'] = visits[0][
'last_visit']
return siteStats
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.INFO)
dbcreds = dbcredentials.DBCredentials().get('LTASO')
with LTAStorageDb(dbcreds, True) as db:
print(db.rootDirectoriesForSite(1))
print(db.dir_id(1, 'rootDir_0'))