def regulate():
with app.app_context():
# Get function and args of
jobs = scheduler.get_jobs()
# Make sure a service update job is running
reindex_services_jobs = [job for job in jobs if job.func == reindex_services]
if len(reindex_services_jobs) < 1:
scheduler.schedule(
scheduled_time=datetime.now(), # Time for first execution
func=reindex_services, # Function to be queued
interval=21600, # Time before the function is called again, in seconds (21600 == 1/4 of a day)
repeat=None, # Repeat this number of times (None means repeat forever)
result_ttl=40000 # How long to keep the results, in seconds
)
# Make sure each service has a ping job
stat_jobs = [unicode(job.args[0]) for job in jobs if job.func == ping_service_task]
# Get services that don't have jobs
services = [s for s in db.Service.find() if unicode(s._id) not in stat_jobs]
# Schedule the ones that do not
for s in services:
scheduler.schedule(
scheduled_time=datetime.now(), # Time for first execution
func=ping_service_task, # Function to be queued
args=(unicode(s._id),), # Arguments passed into function when executed
interval=s.interval, # Time before the function is called again, in seconds
repeat=None, # Repeat this number of times (None means repeat forever)
result_ttl=s.interval * 2 # How long to keep the results, in seconds
)
return "Regulated %s reindex jobs and %s ping jobs" % (len(reindex_services_jobs), len(stat_jobs))
def reindex_services():
region_map = { 'AOOS' : '1E96581F-6B73-45AD-9F9F-2CC3FED76EE6',
'CENCOOS' : 'BE483F24-52E7-4DDE-909F-EE8D4FF118EA',
'GCOOS' : 'E77E250D-2D65-463C-B201-535775D222C9',
'MARACOOS' : 'A26F8553-798B-4B1C-8755-1031D752F7C2',
'NANOOS' : 'C6F4754B-30DC-459E-883A-2AC79DA977AB',
'NERACOOS' : 'E13C88D9-3FF3-4232-A379-84B6A1D7083E',
'PacIOOS' : '78C0463E-2FCE-4AB2-A9C9-6A34BF261F52',
'SCOOS' : '20A3408F-9EC4-4B36-8E10-BBCDB1E81BDF',
'SECOORA' : 'E796C954-B248-4118-896C-42E6FAA6EDE9' }
services = { 'SOS' : 'urn:x-esri:specification:ServiceType:sos:url',
'WMS' : 'urn:x-esri:specification:ServiceType:wms:url',
'WCS' : 'urn:x-esri:specification:ServiceType:wcs:url',
'DAP' : 'urn:x-esri:specification:ServiceType:odp:url' }
endpoint = 'http://www.ngdc.noaa.gov/geoportal/csw' # NGDC Geoportal
c = csw.CatalogueServiceWeb(endpoint, timeout=60)
with app.app_context():
for region,uuid in region_map.iteritems():
# Setup uuid filter
uuid_filter = fes.PropertyIsEqualTo(propertyname='sys.siteuuid', literal="{%s}" % uuid)
# Make CSW request
c.getrecords2([uuid_filter], esn='full')
for name,record in c.records.iteritems():
for ref in record.references:
# We are only interested in the 'services'
if ref["scheme"] in services.values():
url = unicode(ref["url"])
s = db.Service.find_one({ 'data_provider' : unicode(region), 'url' : url })
if s is None:
s = db.Service()
s.url = url
s.data_provider = unicode(region)
s.service_id = unicode(name)
s.name = unicode(record.title)
s.service_type = unicode(next((k for k,v in services.items() if v == ref["scheme"])))
s.interval = 3600 # 1 hour
s.tld = unicode(urlparse(url).netloc)
s.updated = datetime.utcnow()
s.save()
def ping_service_task(service_id):
with app.app_context():
stat = db.Stat()
stat.service_id=ObjectId(service_id)
stat.ping_service()
stat.save()
def send(subject, sender, recipients, text_body, html_body):
with app.app_context():
msg = Message(subject, sender=sender, recipients=recipients)
msg.body = text_body
msg.html = html_body
mail.send(msg)