def post(self):
'''
Returns values as they are received
'''
response = echo_request(request)
log.debug(response)
return reply_success(**response)
def post(self):
'''
post urls directly to the celery task queue
'''
json_data = side_load('url', request.get_json())
data, errors = queues_schema.load(json_data)
if errors:
log.error({__class__: errors})
return reply_error(errors)
elif data:
reply = []
for each in data:
result = db.session.query(Queue).filter_by(
url=each.url).first()
if not result:
log.info('[RECEIVED] {0}'.format(each.url))
db.session.add(each)
reply.append(queue_task_schema.dump(each).data)
elif result:
log.debug('[DUPLICATE]: {0}'.format(each.url))
pass
db.session.commit()
for each in reply:
if not Supervisor.status(key='debug'):
send_to_celery(each['url'])
log.info('[TASKED] {0}'.format(each['url']))
else:
log.info('[DEBUG MODE ENABLED] {0}'.format(each['url']))
return reply_success(reply)
return reply_empty()
def get(self):
'''
returns a json object containing status values
'''
msg = []
status, _ = supervisor_schema.dump(Supervisor.status())
return reply_success(msg=msg, **status)
def get_one(self, _id):
'''
displays a specific content result by database ID.
if no such entry exists, a 404 error is raised.
'''
result = db.session.query(Queue).filter_by(id=_id).first()
if not result:
abort(404)
data, _ = queue_schema.dump(result)
return reply_success(data)
def get(self):
'''
displays statistics on database size, can also be used for
Redis values and uptime.
'''
q = self.queue_metrics()
c = self.content_metrics()
database = dict(queue=q, content=c)
i = self.idle_ratio(q)
ratio = dict(idle=i)
return reply_success(database=database, ratio=ratio)
def get(self, _id=None):
'''
show all url entries from the database
'''
if _id:
return self.get_one(_id)
args, _ = queue_args_schema.dump(request.get_json())
result = db.session.query(Queue).filter(Queue.status.in_( \
args.get('status'))).limit( \
args.get('limit')).all()
data, _ = queues_schema.dump(result)
return reply_success(msg=args, reply=data)
def get(self, _id=None):
'''
displays the stored content from the database
uses request body to provide query args
'''
if _id:
return self.get_one(_id)
args, _ = pagination_schema.dump(request.args)
result = db.session.query(Content).filter(
Content.id.between(args['begins'], args['ends'])).all()
data, _ = contents_schema.dump(result)
return reply_success(msg=args, reply=data)
def post(self):
'''
allows the status configuration values to be modified
modified values are echoed back in the `msg` list
# TODO: unify how we are calling the SupervisorSchemas
'''
data, errors = supervisor_schema.load(request.get_json())
if errors:
log.error({__class__: errors})
return reply_error(errors)
elif data:
changed = Supervisor.update(data)
msg = Supervisor.render_msg(changed)
status = Supervisor.status()
return reply_success(msg=msg, **status)
return reply_empty()
def put(self):
'''
releases any waiting tasks to the queue for processing
# TODO: figure out what is gonna be the best way to do this in a controlled manner
'''
args, _ = args_schema.dump(request.get_json())
result = db.session.query(Queue).filter_by(status='READY').limit(
args['limit']).all()
for each in result:
each.status = 'TASKED'
if not Supervisor.status(key='debug'):
send_to_celery(each.url)
log.debug('[RELEASED] {}'.format(each.url))
else:
log.debug('[DEBUG] {}'.format(each.url))
db.session.commit()
reply = queues_schema.dump(result).data
return reply_success(msg=args, reply=reply)
def post(self):
'''
stores scraped content to the database
'''
data, errors = content_schema.load(request.get_json())
if errors:
log.error({__class__: errors})
return reply_error(errors)
elif data:
result = db.session.query(Content).filter_by(
origin=data.origin).first()
if not result:
db.session.add(data)
db.session.commit()
reply, _ = content_schema.dump(data)
return reply_success(reply)
reply, _ = content_schema.dump(result)
return reply_conflict(reply)
return reply_empty()
def put(self):
'''
saves urls to queue database, used only by celery
'''
data, errors = queue_schema.load(request.get_json())
if errors:
log.error({__class__: errors})
return reply_error(errors)
elif data:
log.debug('[RETURNING] {0}'.format(queue_schema.dump(data)))
result = db.session.query(Queue).filter_by(url=data.url).first()
if result:
result.status = data.status
else:
log.warning('[UNEXPECTED] {0}'.format(queue_schema.dump(data)))
db.session.add(data)
db.session.commit()
reply, _ = queue_schema.dump(result)
return reply_success(reply)
return reply_empty()
def get(self):
routes = self.routes_command(sort='rule')
return reply_success(routes)