def status():
"""
Displays the current Baleen status and job listing
"""
version = baleen.get_version()
counts = {
'feeds': Feed.objects.count(),
'posts': Post.objects.count(),
'jobs': Job.objects.count(),
}
latest_job = Job.objects.order_by('-started').first()
latest_feed = Feed.objects.order_by('-updated').first()
latest_post = Post.objects.order_by('-id').first()
recent_jobs = Job.objects.order_by('-started').limit(10)
# load all data into job_status template
return render_template(
'status.html',
latest_job=latest_job,
latest_feed=latest_feed,
latest_post=latest_post,
version=version,
counts=counts,
dtfmt=WEB_UTC_DATETIME,
recent_jobs=recent_jobs
)
def latest_job():
# get the last job executed
db.connect()
version = baleen.get_version()
counts = [
db.Feed.objects.count(),
db.Post.objects.count(),
db.Job.objects.count()
]
latest_job = db.Job.objects.order_by('-started').first()
latest_feed = db.Feed.objects.order_by('-updated').first()
latest_post = db.Post.objects.order_by('-id').first()
td = datetime.datetime.now() - latest_job.started
running_time = str(td)
logitems = get_logs()
# load all data into job_status template
return render_template('job_status.html',
latest_job=latest_job,
latest_feed=latest_feed,
latest_post=latest_post,
version=version,
counts=counts,
running_time=running_time,
logitems=logitems)
def handle(self, args):
logger = IngestLogger()
logger.info("Starting Baleen v{} ingestion service every hour.".format(
baleen.get_version()))
schedule.every().hour.do(partial(self.ingest, args))
while True:
try:
schedule.run_pending()
time.sleep(1)
except (KeyboardInterrupt, SystemExit):
logger.info("Graceful shutdown of Baleen ingestion service.")
return ""
except Exception as e:
logger.critical(str(e))
return str(e)
def handle(self, args):
logger = IngestLogger()
logger.info(
"Starting Baleen v{} ingestion service every hour.".format(baleen.get_version())
)
schedule.every().hour.do(partial(self.ingest, args))
while True:
try:
schedule.run_pending()
time.sleep(1)
except (KeyboardInterrupt, SystemExit):
logger.info("Graceful shutdown of Baleen ingestion service.")
return ""
except Exception as e:
logger.critical(str(e))
return str(e)
def latest_job():
# get the last job executed
db.connect()
version = baleen.get_version()
counts = [db.Feed.objects.count(),db.Post.objects.count(),db.Job.objects.count()]
latest_job = db.Job.objects.order_by('-started').first()
latest_feed = db.Feed.objects.order_by('-updated').first()
latest_post = db.Post.objects.order_by('-id').first()
td = datetime.datetime.now() - latest_job.started
running_time = str(td)
logitems = get_logs()
# load all data into job_status template
return render_template('job_status.html',
latest_job=latest_job,
latest_feed=latest_feed,
latest_post=latest_post,
version=version,
counts=counts,
running_time=running_time,
logitems=logitems)
def status():
"""
Displays the current Baleen status and job listing
"""
version = baleen.get_version()
counts = {
'feeds': Feed.objects.count(),
'posts': Post.objects.count(),
'jobs': Job.objects.count(),
}
latest_job = Job.objects.order_by('-started').first()
latest_feed = Feed.objects.order_by('-updated').first()
latest_post = Post.objects.order_by('-id').first()
recent_jobs = Job.objects.order_by('-started').limit(10)
# load all data into job_status template
return render_template('status.html',
latest_job=latest_job,
latest_feed=latest_feed,
latest_post=latest_post,
version=version,
counts=counts,
dtfmt=WEB_UTC_DATETIME,
recent_jobs=recent_jobs)
def handle(self, args):
# Setup output and connect to database.
output = []
db.connect()
# Printout configuration details as necessary.
if args.config:
output.append("Configuration:")
output.append(str(settings))
output.append("")
output.append("Baleen v{} Status:".format(baleen.get_version()))
output.append(
"{} Feeds and {} Posts after {} Jobs".format(
db.Feed.objects.count(),
db.Post.objects.count(),
db.Job.objects.count(),
)
)
latest = db.Job.objects.order_by('-started').first()
output.extend([
"",
"Latest Job: ",
" Type: {} v{}".format(latest.name, latest.version),
" Job ID: {}".format(latest.jobid),
" Started: {}".format(latest.started.strftime(HUMAN_DATETIME))
])
if latest.finished:
if latest.failed:
output.append(" Failed: {}".format(latest.reason))
else:
output.append(" Finished: {}".format(latest.finished.strftime(HUMAN_DATETIME)))
output.append(" Counts:")
output.append(" " + "\n ".join(["{}: {}".format(*item) for item in list(latest.counts.items())]))
output.append(" Errors:")
output.append(" " + "\n ".join(["{}: {}".format(*item) for item in list(latest.errors.items())]))
else:
output.append(" Currently Running")
latest = db.Feed.objects.order_by('-updated').first()
output.extend([
"",
"Latest Feed: ",
" Title: \"{}\"".format(latest.title),
" eTag: \"{}\"".format(latest.etag),
" Modified: {}".format(latest.modified),
" Updated: {}".format(latest.updated.strftime(HUMAN_DATETIME)),
# u" Posts: {}".format(latest.count_posts()), # This is very slow need to fix.
])
latest = db.Post.objects.order_by('-id').first()
output.extend([
"",
"Latest Post: ",
" Title: \"{}\"".format(latest.title),
" Feed: \"{}\"".format(latest.feed.title),
" Fetched: {}".format(latest.created.strftime(HUMAN_DATETIME)),
])
return "\n".join(output).encode('utf-8', errors='replace')
def handle(self, args):
# Setup output and connect to database.
output = []
db.connect()
# Printout configuration details as necessary.
if args.config:
output.append(u"Configuration:")
output.append(unicode(settings))
output.append(u"")
output.append(u"Baleen v{} Status:".format(baleen.get_version()))
output.append(
u"{} Feeds and {} Posts after {} Jobs".format(
db.Feed.objects.count(), db.Post.objects.count(), db.Job.objects.count()
)
)
latest = db.Job.objects.order_by("-started").first()
output.extend(
[
u"",
u"Latest Job: ",
u" Type: {} v{}".format(latest.name, latest.version),
u" Job ID: {}".format(latest.jobid),
u" Started: {}".format(latest.started.strftime(HUMAN_DATETIME)),
]
)
if latest.finished:
if latest.failed:
output.append(u" Failed: {}".format(latest.reason))
else:
output.append(u" Finished: {}".format(latest.finished.strftime(HUMAN_DATETIME)))
output.append(u" Counts:")
output.append(u" " + u"\n ".join([u"{}: {}".format(*item) for item in latest.counts.items()]))
output.append(u" Errors:")
output.append(u" " + u"\n ".join([u"{}: {}".format(*item) for item in latest.errors.items()]))
else:
output.append(u" Currently Running")
latest = db.Feed.objects.order_by("-updated").first()
output.extend(
[
u"",
u"Latest Feed: ",
u' Title: "{}"'.format(latest.title),
u' eTag: "{}"'.format(latest.etag),
u" Modified: {}".format(latest.modified),
u" Updated: {}".format(latest.updated.strftime(HUMAN_DATETIME)),
# u" Posts: {}".format(latest.count_posts()), # This is very slow need to fix.
]
)
latest = db.Post.objects.order_by("-id").first()
output.extend(
[
u"",
u"Latest Post: ",
u' Title: "{}"'.format(latest.title),
u' Feed: "{}"'.format(latest.feed.title),
u" Fetched: {}".format(latest.created.strftime(HUMAN_DATETIME)),
]
)
return u"\n".join(output).encode("utf-8", errors="replace")
def test_version(self):
"""
Assert that the version is sane
"""
import baleen
self.assertEqual("0.1.0", baleen.get_version())