def count(self):
"""
Returns the number of records as an integer.
The result is not cached nor comes from cache, cache must be handled
by the server.
"""
clone = self._clone()
# Instantiation of clone.model is necessary because we can't set
# a staticmethod for get_resource_url_count and avoid to set it
# for all model without relying on get_resource_url_list
instance = clone.model()
resource = Resource(instance.get_resource_url_count(),
filters=ROA_FILTERS,
**ROA_SSL_ARGS)
try:
parameters = clone.query.parameters
logger.debug(
u"""Counting : "%s" through %s with parameters "%s" """ %
(clone.model.__name__, resource.uri,
force_unicode(parameters)))
response = resource.get(headers=self._get_http_headers(),
**parameters)
except Exception as e:
raise ROAException(e)
response = force_unicode(
response.body_string()).encode(DEFAULT_CHARSET)
data = self.model.get_parser().parse(StringIO(response))
return self.model.count_response(data)
def count(self):
"""
Returns the number of records as an integer.
The result is not cached nor comes from cache, cache must be handled
by the server.
"""
clone = self._clone()
# Instantiation of clone.model is necessary because we can't set
# a staticmethod for get_resource_url_count and avoid to set it
# for all model without relying on get_resource_url_list
instance = clone.model()
resource = Resource(instance.get_resource_url_count(),
filters=ROA_FILTERS, **ROA_SSL_ARGS)
try:
parameters = clone.query.parameters
logger.debug(u"""Counting : "%s" through %s with parameters "%s" """ % (
clone.model.__name__,
resource.uri,
force_unicode(parameters)))
response = resource.get(headers=self._get_http_headers(), **parameters)
except Exception as e:
raise ROAException(e)
response = force_unicode(response.body_string()).encode(DEFAULT_CHARSET)
data = self.model.get_parser().parse(StringIO(response))
return self.model.count_response(data)
def iterator(self):
"""
An iterator over the results from applying this QuerySet to the
remote web service.
"""
resource = Resource(self.model.get_resource_url_list(),
filters=ROA_FILTERS)
try:
parameters = self.query.parameters
logger.debug(u"""Requesting: "%s" through %s
with parameters "%s" """ % (
self.model.__name__,
resource.uri,
force_unicode(parameters)))
response = resource.get(headers=ROA_HEADERS, **parameters)
except ResourceNotFound:
return
except Exception as e:
raise ROAException(e)
response = force_unicode(response.body_string()).encode(DEFAULT_CHARSET)
stream = StringIO(response)
data = self.model.get_parser().parse(stream)
serializer = self.model.get_serializer(data=data)
if not serializer.is_valid():
raise ROAException('Invalid deserialization')
for obj in serializer.object:
obj = res.object
yield obj
def post_authenticated_data(data, url, username, password):
"""
Post basic authenticated data, using restkit
"""
auth = BasicAuth(username, password)
r = Resource(url, filters=[auth, ])
return r.post(payload=data).body_string(), None
def get_qr_queue(host=celeryconfig.SERVER_HOST):
if not is_bootstrapped:
bootstrap()
res = Resource(host, manager=manager)
auth_params = {'username':celeryconfig.ZPRINTER_USERNAME, 'api_key': celeryconfig.ZPRINTER_API_KEY}
r = res.get('/api/zebra_queue/', params_dict=auth_params)
json = simplejson.loads(r.body_string())
if len(printer_dict.keys()) == 0:
get_printers()
if len(json['objects']) > 0:
for instance in json['objects']:
uri = instance['resource_uri']
zpl_code= instance['zpl_code']
printer_uri = instance['destination_printer']
printer_ip = printer_dict[printer_uri]['ip_address']
printer_port = printer_dict[printer_uri]['port']
instance['fulfilled_date'] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.000")
res.put(uri, simplejson.dumps(instance), headers={'Content-Type': 'application/json'}, params_dict=auth_params)
do_send(printer_ip, printer_port, zpl_code, recv=False)
else:
logging.debug("no jobs")
def iterator(self):
"""
An iterator over the results from applying this QuerySet to the
remote web service.
"""
resource = Resource(self.model.get_resource_url_list(**self.query.filters),
filters=ROA_FILTERS, **ROA_SSL_ARGS)
try:
parameters = self.query.parameters
logger.debug(u"""Requesting: "%s" through %s with parameters "%s" """ % (
self.model.__name__,
resource.uri,
force_unicode(parameters)))
response = resource.get(headers=self._get_http_headers(), **parameters)
except ResourceNotFound:
return
except Exception as e:
raise ROAException(e)
response = force_unicode(response.body_string()).encode(DEFAULT_CHARSET)
# Deserializing objects:
data = self.model.get_parser().parse(StringIO(response))
# [] is the case of empty no-paginated result
if data != []:
serializer = self.model.get_serializer(data=data)
if not serializer.is_valid():
raise ROAException(u'Invalid deserialization for {} model: {}'.format(self.model, serializer.errors))
for obj in serializer.object:
yield obj
def zebra_alert_handler(socket, address):
try:
fileobj = socket.makefile()
while True:
line = fileobj.readline()
if not line:
break
fileobj.write(line)
fileobj.flush()
raw_msg = line.strip()
message_split = raw_msg.split(':')
if message_split[0] == 'ALERT' or message_split[0] == 'ERROR CONDITION':
#it's an alert/error,
is_cleared=False
elif message_split[0] == 'ALERT CLEARED' or message_split[0] == 'ERROR CLEARED':
is_cleared=True
else:
is_cleared=True
condition = message_split[1].split('[')[0].strip().lower()
printer_uri = identify_printer(address, printers)
res = Resource(celeryconfig.SERVER_HOST)
auth_params = {'username':celeryconfig.ZPRINTER_USERNAME, 'api_key': celeryconfig.ZPRINTER_API_KEY}
#r = res.get('/api/zebra_status/', params_dict=auth_params)
new_instance = dict()
new_instance['printer'] = printer_uri
new_instance['event_date'] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.000")
new_instance['status'] = condition
new_instance['is_cleared'] = is_cleared
res.post('api/zebra_status/', simplejson.dumps(new_instance), headers={'Content-Type': 'application/json'}, params_dict=auth_params)
except Exception, ex:
logging.error("Unknown exception, gobbling up: %s", ex)
def publish(namespace, name, version, description_file, tar_file, readme_file, readme_file_ext):
''' Publish a tarblob to the registry, if the request fails, an exception
is raised, which either triggers re-authentication, or is turned into a
return value by the decorators. (If successful, the decorated function
returns None)
'''
url = '%s/%s/%s/versions/%s' % (
Registry_Base_URL,
namespace,
name,
version
)
if readme_file_ext == '.md':
readme_section_name = 'readme.md'
elif readme_file_ext == '':
readme_section_name = 'readme'
else:
raise ValueError('unsupported readme type: "%s"' % readne_file_ext)
# description file is in place as text (so read it), tar file is a file
body = OrderedDict([('metadata',description_file.read()), ('tarball',tar_file), (readme_section_name, readme_file)])
headers = { }
body, headers = multipart_form_encode(body, headers, uuid.uuid4().hex)
auth = _registryAuthFilter()
resource = Resource(url, pool=connection_pool.getPool(), filters=[auth])
response = resource.put(
headers = headers,
payload = body
)
return None
def post_authenticated_data(data, url, username, password):
"""
Post basic authenticated data, using restkit
"""
auth = BasicAuth(username, password)
r = Resource(url, filters=[auth, ])
return r.post(payload=data).body_string(), None
def _run(self, action, headers=None, **kwargs):
"""
Requests API
"""
res = Resource(self.HOST)
payload = kwargs
payload.update({'action': action, 'api_token': self.api_token})
response = res.post(payload=payload, headers=headers)
if response.status_int != 200:
raise POEditorException(
status='fail',
error_code=response.status_int,
message=response.status
)
data = json.loads(response.body_string().decode('utf-8'))
if 'response' not in data:
raise POEditorException(
status='fail',
error_code=-1,
message=u'"response" key is not present'
)
if 'status' in data['response'] and \
data['response']['status'] != self.SUCCESS_CODE:
raise POEditorException(
error_code=data['response'].get('code'),
status=data['response']['status'],
message=data['response'].get('message')
)
return data
def count(self):
"""
Returns the number of records as an integer.
The result is not cached nor comes from cache, cache must be handled
by the server.
"""
clone = self._clone()
# Instantiation of clone.model is necessary because we can't set
# a staticmethod for get_resource_url_count and avoid to set it
# for all model without relying on get_resource_url_list
instance = clone.model()
resource = Resource(instance.get_resource_url_count(),
filters=ROA_FILTERS)
try:
parameters = clone.query.parameters
logger.debug(u"""Counting : "%s" through %s
with parameters "%s" """ % (
clone.model.__name__,
resource.uri,
force_unicode(parameters)))
response = resource.get(headers=ROA_HEADERS, **parameters)
except Exception as e:
raise ROAException(e)
cnt = 0
try:
cnt = int(response.body_string())
except ValueError: pass
return cnt
def _post_payload(host, path, data, **kwargs):
"""generic function to handle posting data
:rtype : return body of page
:param host: host to send data to
:param path: path to interact with
:param data: data to send
:param kwargs: additional keyword args
"""
if "http://" not in host:
host = "http://{0}".format(host)
res = Resource(host)
payload = "jsonParameters={0}".format(json.dumps(data))
for key, value in kwargs.items():
payload += '&{0}={1}'.format(key, json.dumps(value))
headers = {"Content-Type": "application/json"}
# print "path is %s" % path
page = res.post(path=path, payload=payload, headers=headers)
body = page.body_string()
if body:
body = json.loads(body)
if isinstance(body, dict) and "ERROR" in body:
raise HelixException(body["ERROR"])
# test what was returned, see if any exceptions need to be raise
# if not body:
# raise HelixException("body for path {0} is empty".format(path))
# else:
# print "BODY IS EMPTY FOR ", path
# print "BODY is %s." % body
return body
def testNew(self):
"""succeed in creating fill"""
path = "/resources/data/fills"
user = self.testfilluser
res = Resource(self.baseurl, filters=[user])
# perform request
output = res.request("POST",
path=path,
params_dict={
"fill_number": 100,
"air_begin_datetime": "010110080910",
"end_datetime": "110110080910",
"start_datetime": "010110080910",
"rotation_number": 100,
"bin_id": 14,
"hybrid_code": "100A",
"field_code": "100A",
"storage_bin_number": 100,
"storage_bin_code": "100A",
"pre_mc_count": 2,
"pre_mc": "33,31.2",
"post_mc_count": 3,
"post_mc": "17,16,17.2",
"bushels": 42
})
self.unauthorizedChecks([None, self.testuser], path, method="POST")
def auth_get_repo(user_password):
pool = ConnectionPool(factory=Connection)
serverurl = "https://api.github.com"
'''
#print 'Enter your username:'******'+')[0]
password = user_password.split('+')[1]
# Add your username and password here, or prompt for them
auth=BasicAuth(username, password)
# Use your basic auth to request a token
# This is just an example from http://developer.github.com/v3/
authreqdata = { "scopes": [ "public_repo" ], "note": "admin script" }
resource = Resource('https://api.github.com/authorizations', pool=pool, filters=[auth])
response = resource.post(headers={ "Content-Type": "application/json" }, payload=json.dumps(authreqdata))
token = json.loads(response.body_string())['token']
'''
"""
Once you have a token, you can pass that in the Authorization header
You can store this in a cache and throw away the user/password
This is just an example query. See http://developer.github.com/v3/
for more about the url structure
"""
token = '94038d59a46c5ea1aa4f11626a83cde3e8794668'
resource = Resource('https://api.github.com/user/repos', pool=pool)
headers = {'Content-Type': 'application/json'}
headers['Authorization'] = 'token %s' % token
response = resource.get(headers=headers)
repos = json.loads(response.body_string())
for each in repos:
git("clone", each['clone_url'])
def check_celery_health():
ret = {}
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
worker_status = ""
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
all_workers = {}
try:
t = cresource.get("api/workers").body_string()
all_workers = json.loads(t)
except Exception, ex:
pass
worker_ok = '<span class="label label-success">OK</span>'
worker_bad = '<span class="label label-important">Down</span>'
tasks_ok = 'label-success'
tasks_full = 'label-warning'
worker_info = []
for hostname, w in all_workers.items():
status_html = mark_safe(worker_ok if w['status'] else worker_bad)
tasks_class = tasks_full if w['running_tasks'] == w['concurrency'] else tasks_ok
tasks_html = mark_safe('<span class="label %s">%d / %d</span> :: %d' % (tasks_class, w['running_tasks'], w['concurrency'], w['completed_tasks']))
worker_info.append(' '.join([hostname, status_html, tasks_html]))
worker_status = '<br>'.join(worker_info)
def check_heartbeat():
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
t = cresource.get("api/workers", params_dict={
'status': True
}).body_string()
all_workers = json.loads(t)
bad_workers = []
expected_running, expected_stopped = parse_celery_workers(all_workers)
celery = Celery()
celery.config_from_object(settings)
worker_responses = celery.control.ping(timeout=10)
pings = parse_celery_pings(worker_responses)
for hostname in expected_running:
if hostname not in pings or not pings[hostname]:
bad_workers.append('* {} celery worker down'.format(hostname))
for hostname in expected_stopped:
if hostname in pings:
bad_workers.append(
'* {} celery worker is running when we expect it to be stopped.'
.format(hostname))
if bad_workers:
return ServiceStatus(False, '\n'.join(bad_workers))
is_alive = heartbeat.is_alive()
return ServiceStatus(is_alive, "OK" if is_alive else "DOWN")
def post_unauthenticated_data(data, url):
"""
Post basic unauthenticated data, using restkit instead of the post_data
method.
"""
r = Resource(url, filters=[])
return r.post(payload=data).body_string(), None
def auth_get_repo(user_password):
pool = ConnectionPool(factory=Connection)
serverurl="https://api.github.com"
'''
#print 'Enter your username:'******'+')[0]
password = user_password.split('+')[1]
# Add your username and password here, or prompt for them
auth=BasicAuth(username, password)
# Use your basic auth to request a token
# This is just an example from http://developer.github.com/v3/
authreqdata = { "scopes": [ "public_repo" ], "note": "admin script" }
resource = Resource('https://api.github.com/authorizations', pool=pool, filters=[auth])
response = resource.post(headers={ "Content-Type": "application/json" }, payload=json.dumps(authreqdata))
token = json.loads(response.body_string())['token']
'''
"""
Once you have a token, you can pass that in the Authorization header
You can store this in a cache and throw away the user/password
This is just an example query. See http://developer.github.com/v3/
for more about the url structure
"""
token = '94038d59a46c5ea1aa4f11626a83cde3e8794668'
resource = Resource('https://api.github.com/user/repos', pool=pool)
headers = {'Content-Type' : 'application/json' }
headers['Authorization'] = 'token %s' % token
response = resource.get(headers = headers)
repos = json.loads(response.body_string())
for each in repos:
git("clone", each['clone_url'])
def get_pull_request_by_label(self, user, repo, label):
resource = Resource("https://api.github.com/repos/%s/%s/pulls" %
(user, repo))
pulls = json.loads(resource.get(headers=self.headers).body_string())
pulls_by_label = filter(lambda p: p['head']['label'] == label, pulls)
return pulls_by_label # I hope there is no more than one
def _post_payload(self, path, data, **kwargs):
"""generic function to handle posting data
:rtype : return body of page
:param path: path to interact with
:param data: data to send
:param kwargs: additional keyword args
"""
res = Resource(self.host)
payload = "jsonParameters={0}".format(json.dumps(data))
for key, value in kwargs.items():
payload += '&{0}={1}'.format(key, json.dumps(value))
headers = {"Content-Type": "application/json"}
# print "path is %s" % path
page = res.post(path=path, payload=payload, headers=headers)
body = page.body_string()
if body:
body = json.loads(body)
if isinstance(body, dict) and "ERROR" in body:
raise HelixException(body["ERROR"])
# test what was returned, see if any exceptions need to be raise
# if not body:
# raise HelixException("body for path {0} is empty".format(path))
# else:
# print "BODY IS EMPTY FOR ", path
# print "BODY is %s." % body
return body
def _get_from_id_or_pk(self, id=None, pk=None, **kwargs):
"""
Returns an object given an id or pk, request directly with the
get_resource_url_detail method without filtering on ids
(as Django's ORM do).
"""
clone = self._clone()
# Instantiation of clone.model is necessary because we can't set
# a staticmethod for get_resource_url_detail and avoid to set it
# for all model without relying on get_resource_url_list
instance = clone.model()
if pk is None:
instance.id = id
else:
instance.pk = pk
resource = Resource(instance.get_resource_url_detail(), headers=ROA_HEADERS, filters=ROA_FILTERS, **kwargs)
try:
parameters = clone.query.parameters
logger.debug(
u"""Retrieving : "%s" through %s
with parameters "%s" """
% (clone.model.__name__, resource.uri, force_unicode(parameters))
)
response = resource.get(**parameters)
except Exception, e:
raise ROAException(e)
def getAuthData():
''' Poll the registry to get the result of a completed authentication
(which, depending on the authentication the user chose or was directed
to, will include a github or other access token)
'''
url = '%s/tokens' % (
Registry_Base_URL
)
headers = { }
auth = _registryAuthFilter()
resource = Resource(url, pool=connection_pool.getPool(), filters=[auth])
try:
logger.debug('poll for tokens...')
response = resource.get(
headers = headers
)
except restkit_errors.Unauthorized as e:
logger.debug(str(e))
return None
except restkit_errors.ResourceNotFound as e:
logger.debug(str(e))
return None
except restkit_errors.RequestFailed as e:
logger.debug(str(e))
return None
body = response.body_string()
logger.debug('auth data response: %s' % body);
r = {}
for token in ordered_json.loads(body):
if token['provider'] == 'github':
r['github'] = token['accessToken']
break
logger.debug('parsed auth tokens %s' % r);
return r
def check_celery_health():
ret = {}
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
worker_status = ""
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
all_workers = {}
try:
t = cresource.get("api/workers").body_string()
all_workers = json.loads(t)
except Exception, ex:
pass
worker_ok = '<span class="label label-success">OK</span>'
worker_bad = '<span class="label label-important">Down</span>'
tasks_ok = 'label-success'
tasks_full = 'label-warning'
worker_info = []
for hostname, w in all_workers.items():
status_html = mark_safe(worker_ok if w['status'] else worker_bad)
tasks_class = tasks_full if w['running_tasks'] == w[
'concurrency'] else tasks_ok
tasks_html = mark_safe(
'<span class="label %s">%d / %d</span> :: %d' %
(tasks_class, w['running_tasks'], w['concurrency'],
w['completed_tasks']))
worker_info.append(' '.join([hostname, status_html, tasks_html]))
worker_status = '<br>'.join(worker_info)
def __init__(self, bot):
IPlugin.__init__(self, bot)
self.data = {
'last-bug-created': 0,
'ignored-names': [
'/^Not\-[0-9]+/' # Notifico bots
]
}
self.LoadPluginData()
self.url = globalConfig.get('plugins.redmine.url', None)
if self.url is None:
logging.error('Redmine: Disabled.')
return
self.ignored = []
for ignoretok in self.data.get('ignored-names',['/^Not\-[0-9]/']):
if ignoretok.startwith('/') and ignoretok.endwith('/'):
self.ignored+=[re.compile(ignoretok[1:-1])]
else:
self.ignored+=[re.compile('^'+re.escape(ignoretok)+'$')]
self.auth = BasicAuth(globalConfig.get('plugins.redmine.apikey', None), str(random.random()))
self.project_id = globalConfig.get('plugins.redmine.project', None)
if self.project_id is None: logging.warning('Redmine: Not going to check for bug updates.')
self.bug_info_format = globalConfig.get('plugins.redmine.bug-info-format', 'Redmine #{ID} - {AUTHOR} - {STATUS} - {SUBJECT}{CRLF}{URL}')
self.new_bug_format = globalConfig.get('plugins.redmine.new-bug-format', 'NEW ISSUE: {URL} (#{ID}: {SUBJECT})')
self.resource = Resource(self.url, filters=[self.auth])
self.bug_regex = re.compile(r'#(\d+)\b')
self.lastCheck = 0
def post_unauthenticated_data(data, url):
"""
Post basic unauthenticated data, using restkit instead of the post_data
method.
"""
r = Resource(url, filters=[])
return r.post(payload=data).body_string(), None
def _run(self, action, headers=None, **kwargs):
"""
Requests API
"""
res = Resource(self.HOST)
payload = kwargs
payload.update({'action': action, 'api_token': self.api_token})
response = res.post(payload=payload, headers=headers)
if response.status_int != 200:
raise POEditorException(
status='fail',
error_code=response.status_int,
message=response.status
)
data = json.loads(response.body_string().decode('utf-8'))
if 'response' not in data:
raise POEditorException(
status='fail',
error_code=-1,
message=u'"response" key is not present'
)
if 'status' in data['response'] and \
data['response']['status'] != self.SUCCESS_CODE:
raise POEditorException(
error_code=data['response'].get('code'),
status=data['response']['status'],
message=data['response'].get('message')
)
return data
def testUpdateByUserForOthers(self):
"""fail to update alarms belonging to others"""
# TODO: improve test to attempt to update alarm belonging to another user instead of a higher level user
path = "/resources/alarms/100" # id taken from DB (belongs to a power user)
user = self.testuser
res = Resource(self.baseurl, filters=[user])
try:
exception = False
# perform request
output = res.request("PUT",
path=path,
params_dict = {"alarm_type_id": 14, # changeing from 2 to 1
"account_id": 106, # std user from the DB
"sensor_type_id": 100, # value from DB
"greater_than_p": True,
"value": 100
}
)
except Unauthorized:
exception = True
# check that update failed
self.assertTrue(exception, "User was able to update alarm for another user.")
def get_pull_request_by_branch(self, user, repo, branch):
resource = Resource("https://api.github.com/repos/%s/%s/pulls" %
(user, repo))
pulls = json.loads(resource.get(headers=self.headers).body_string())
pulls_by_branch = filter(lambda p: p['head']['ref']==branch, pulls)
return pulls_by_branch # I hope there is no more than one
def check_heartbeat():
celery_monitoring = getattr(settings, "CELERY_FLOWER_URL", None)
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
t = cresource.get("api/workers", params_dict={"status": True}).body_string()
all_workers = json.loads(t)
bad_workers = []
expected_running, expected_stopped = parse_celery_workers(all_workers)
celery = Celery()
celery.config_from_object(settings)
worker_responses = celery.control.ping(timeout=10)
pings = parse_celery_pings(worker_responses)
for hostname in expected_running:
if hostname not in pings or not pings[hostname]:
bad_workers.append("* {} celery worker down".format(hostname))
for hostname in expected_stopped:
if hostname in pings:
bad_workers.append("* {} celery worker is running when we expect it to be stopped.".format(hostname))
if bad_workers:
return ServiceStatus(False, "\n".join(bad_workers))
is_alive = heartbeat.is_alive()
return ServiceStatus(is_alive, "OK" if is_alive else "DOWN")
def system_ajax(request):
"""
Utility ajax functions for polling couch and celerymon
"""
type = request.GET.get('api', None)
task_limit = getattr(settings, 'CELERYMON_TASK_LIMIT', 12)
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
db = XFormInstance.get_db()
if type == "_active_tasks":
tasks = [] if is_bigcouch() else filter(
lambda x: x['type'] == "indexer", db.server.active_tasks())
#for reference structure is:
# tasks = [{'type': 'indexer', 'pid': 'foo', 'database': 'mock',
# 'design_document': 'mockymock', 'progress': 0,
# 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458,
# 'total_changes': 1023},
# {'type': 'indexer', 'pid': 'foo', 'database': 'mock',
# 'design_document': 'mockymock', 'progress': 70,
# 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458,
# 'total_changes': 1023}]
return json_response(tasks)
elif type == "_stats":
return json_response({})
elif type == "_logs":
pass
elif type == 'pillowtop':
return json_response(get_all_pillows_json())
elif type == 'stale_pillows':
es_index_status = [
check_case_es_index(interval=3),
check_xform_es_index(interval=3),
check_reportcase_es_index(interval=3),
check_reportxform_es_index(interval=3)
]
return json_response(es_index_status)
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
if type == "flower_poll":
ret = []
try:
t = cresource.get("api/tasks",
params_dict={
'limit': task_limit
}).body_string()
all_tasks = json.loads(t)
except Exception, ex:
all_tasks = {}
logging.error("Error with getting from celery_flower: %s" % ex)
for task_id, traw in all_tasks.items():
# it's an array of arrays - looping through [<id>, {task_info_dict}]
if 'name' in traw and traw['name']:
traw['name'] = '.'.join(traw['name'].split('.')[-2:])
else:
traw['name'] = None
ret.append(traw)
ret = sorted(ret, key=lambda x: x['succeeded'], reverse=True)
return HttpResponse(json.dumps(ret), mimetype='application/json')
def testDeleteByPowerUser(self):
"""Power user to succeed in deleting alarm"""
path = "/resources/alarms/113" # alarm belonging to a poweruser
user = self.testpoweruser
res = Resource(self.baseurl, filters=[user])
output = res.request("DELETE",path=path)
self.assertEqual(204,output.status_int,"Power user was unable to delete alarm")
def rest_call(url, method, data, headers, client_auth_key, client_auth_secret):
consumer = oauth2.Consumer(key=client_auth_key, secret=client_auth_secret)
auth = OAuthFilter('*',consumer)
resource = Resource(url, filters=[auth])
response = resource.request(method, payload=data, headers=headers, params=None)
json_string = response.body_string()
status = response.status
return status, json_string
def getJSONResource(self, path, user, params_dict=None):
name = user.credentials[0]
res = Resource(self.baseurl, filters=[user])
output = res.get(path, None, params_dict=params_dict)
self.assertEqual(200, output.status_int,
"Wrong response code: " + name)
response = json.loads(output.body_string())
return response
def newDevice(self):
path = "/resources/conf/devices"
user = self.testconfiguser
res = Resource(self.baseurl, filters=[user])
output = res.request("POST", path, params_dict=self.newparams)
self.assertEqual(200, output.status_int, "Wrong response code from post.")
xlink = json.loads(output.body_string())
return xlink
def setUp(self):
auth = BasicAuth('john', 'teste')
res = Resource(server, filters=[auth, ])
r = res.get('/authenticate')
data = loads(r.body_string())
self.auth = AuthToken(data.get('token'))
self.res = Resource(server, filters=[self.auth, ])
def __init__(self, settings=None):
if not settings:
settings = friendloc.settings
Resource.__init__(self,
settings.gisgraphy_url,
client_opts={'timeout':60},
)
self._mdist = {}
def create_token(self, user, password):
serverurl = "https://api.github.com"
auth = BasicAuth(user, password)
authreqdata = {"scopes": ["repo"], "note": "admin script"}
resource = Resource('https://api.github.com/authorizations',
pool=self.pool, filters=[auth])
response = resource.post(headers={"Content-Type": "application/json"},
payload=json.dumps(authreqdata))
self.token = json.loads(response.body_string())['token']
def get_epic_id(url, key, auth):
resource = Resource(url + ('/rest/api/latest/issue/%s?expand=names' % key), filters=[auth])
response = resource.get(headers={'Content-Type': 'application/json'})
if response.status_int == 200:
for field_id, field_name in json.loads(response.body_string())['names'].items():
if field_name == 'Epic Link':
return field_id
else:
return None
def _getTarball(url, into_directory):
'''unpack the specified tarball url into the specified directory'''
resource = Resource(url, pool=connection_pool.getPool(), follow_redirect=True)
response = resource.get(
headers = {'Authorization': 'token ' + settings.getProperty('github', 'authtoken')},
)
logger.debug('getting file: %s', url)
# TODO: there's an MD5 in the response headers, verify it
access_common.unpackTarballStream(response.body_stream(), into_directory)
def _get(self, url):
print "Running GET on URL:", url
auth = BasicAuth(self.login, self.password)
res = Resource(url, filters=[auth])
r = res.get()
if r['Content-Type'].startswith('application/json'):
return json.loads(r.body_string())
else:
return r.body_string()
def testAuthentication(self):
r = self.res.get('/')
self.assertEqual(200, r.status_int)
try:
res = Resource(server)
r = res.get('/')
self.assertTrue(False)
except Unauthorized, e:
self.assertTrue(True)
def doGithub(url, method="GET", payload=None):
# screw it
global headers, pool
resource = Resource(url, pool=pool)
if payload:
payload = json.dumps(payload)
response = resource.request(headers=headers, payload=payload, method=method)
return json.loads(response.body_string())
def testDeleteByUserForSelf(self):
"""succeed in deleting own alarm contact"""
path = "/resources/alarms/104" # alarm contact belonging to this user
user = self.testuser
res = Resource(self.baseurl, filters=[user])
output = res.request("DELETE",path=path)
self.assertEqual(204,output.status_int, "User was unable to delete alarm belonging to self.")
def __init__(self, endpoint, name, **kwargs):
if endpoint.endswith('/'):
endpoint = endpoint.rstrip('/')
if 'pool' not in kwargs:
kwargs['pool'] = ConnectionPool(factory=Connection)
self.json_default = kwargs.get('json_default', json_util.default)
self.json_object_hook = kwargs.get('json_object_hook',
json_util.object_hook)
self.resource = Resource(endpoint, **kwargs)
self.name = name
def testAuthentication(self):
r = self.res.get('/')
self.assertEqual(200, r.status_int)
try:
res = Resource(server)
r = res.get('/')
self.assertTrue(False)
except Unauthorized, e:
self.assertTrue(True)
def testDeleteByPowerUserForOthers(self):
"""succeed in deleting other's alarm contact"""
path = "/resources/alarms/112" # alarm contact belonging to another user
user = self.testpoweruser
res = Resource(self.baseurl, filters=[user])
output = res.request("DELETE",path=path)
self.assertEqual(204,output.status_int, "Power user was unable to delete alarm belonging to lower level user.")
def create_token(self, user, password):
serverurl = "https://api.github.com"
auth = BasicAuth(user, password)
authreqdata = {"scopes": ["repo"], "note": "admin script"}
resource = Resource('https://api.github.com/authorizations',
pool=self.pool,
filters=[auth])
response = resource.post(headers={"Content-Type": "application/json"},
payload=json.dumps(authreqdata))
self.token = json.loads(response.body_string())['token']
def system_ajax(request):
"""
Utility ajax functions for polling couch and celerymon
"""
type = request.GET.get('api', None)
task_limit = getattr(settings, 'CELERYMON_TASK_LIMIT', 12)
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
db = XFormInstance.get_db()
if type == "_active_tasks":
tasks = [] if is_bigcouch() else filter(lambda x: x['type'] == "indexer", db.server.active_tasks())
#for reference structure is:
# tasks = [{'type': 'indexer', 'pid': 'foo', 'database': 'mock',
# 'design_document': 'mockymock', 'progress': 0,
# 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458,
# 'total_changes': 1023},
# {'type': 'indexer', 'pid': 'foo', 'database': 'mock',
# 'design_document': 'mockymock', 'progress': 70,
# 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458,
# 'total_changes': 1023}]
return json_response(tasks)
elif type == "_stats":
return json_response({})
elif type == "_logs":
pass
elif type == 'pillowtop':
return json_response(get_all_pillows_json())
elif type == 'stale_pillows':
es_index_status = [
check_case_es_index(interval=3),
check_xform_es_index(interval=3),
check_reportcase_es_index(interval=3),
check_reportxform_es_index(interval=3)
]
return json_response(es_index_status)
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
if type == "flower_poll":
ret = []
try:
t = cresource.get("api/tasks", params_dict={'limit': task_limit}).body_string()
all_tasks = json.loads(t)
except Exception, ex:
all_tasks = {}
logging.error("Error with getting from celery_flower: %s" % ex)
for task_id, traw in all_tasks.items():
# it's an array of arrays - looping through [<id>, {task_info_dict}]
if 'name' in traw and traw['name']:
traw['name'] = '.'.join(traw['name'].split('.')[-2:])
else:
traw['name'] = None
ret.append(traw)
ret = sorted(ret, key=lambda x: x['succeeded'], reverse=True)
return HttpResponse(json.dumps(ret), mimetype = 'application/json')
def __init__(self, uri="http://127.0.0.1:5984", **client_opts):
"""Constructor for a `CouchdbResource` object.
CouchdbResource represent an HTTP resource to CouchDB.
@param uri: str, full uri to the server.
"""
client_opts['response_class'] = CouchDBResponse
Resource.__init__(self, uri=uri, **client_opts)
self.safe = ":/%"
def get_stats(celery_monitoring, status_only=False, refresh=False):
cresource = Resource(celery_monitoring, timeout=3)
endpoint = "api/workers"
params = {'refresh': 'true'} if refresh else {}
if status_only:
params['status'] = 'true'
try:
t = cresource.get(endpoint, params_dict=params).body_string()
return json.loads(t)
except Exception:
return {}
def get_stats(celery_monitoring, status_only=False, refresh=False):
cresource = Resource(celery_monitoring, timeout=3)
endpoint = "api/workers"
params = {'refresh': 'true'} if refresh else {}
if status_only:
params['status'] = 'true'
try:
t = cresource.get(endpoint, params_dict=params).body_string()
return json.loads(t)
except Exception:
return {}
def testDelete(self):
"""Succeed in deleting fill."""
path = "/resources/data/fills/100"
user = self.testpoweruser
res = Resource(self.baseurl, filters=[user])
output = res.request("DELETE", path=path)
self.assertEqual(204, output.status_int,
"Fill user was unable to delete fill.")
def rest_call(url, method, data, headers, client_auth_key, client_auth_secret):
consumer = oauth2.Consumer(key=client_auth_key, secret=client_auth_secret)
auth = OAuthFilter('*', consumer)
resource = Resource(url, filters=[auth])
response = resource.request(method,
payload=data,
headers=headers,
params=None)
json_string = response.body_string()
status = response.status
return status, json_string
def testLogin():
url = '%s/users/me' % (
Registry_Base_URL
)
headers = { }
auth = _registryAuthFilter()
resource = Resource(url, pool=connection_pool.getPool(), filters=[auth])
logger.debug('test login...')
response = resource.get(
headers = headers
)
def get_epic_id(url, key, auth):
resource = Resource(url + ('/rest/api/latest/issue/%s?expand=names' % key),
filters=[auth])
response = resource.get(headers={'Content-Type': 'application/json'})
if response.status_int == 200:
for field_id, field_name in json.loads(
response.body_string())['names'].items():
if field_name == 'Epic Link':
return field_id
else:
return None
def __init__(self, uri="http://127.0.0.1:5984", **client_opts):
"""Constructor for a `CouchdbResource` object.
CouchdbResource represent an HTTP resource to CouchDB.
@param uri: str, full uri to the server.
"""
client_opts['response_class'] = CouchdbResponse
Resource.__init__(self, uri=uri, **client_opts)
self.safe = ":/%"
def setUp(self):
auth = BasicAuth('john', 'teste')
res = Resource(server, filters=[
auth,
])
r = res.get('/authenticate')
data = loads(r.body_string())
self.auth = AuthToken(data.get('token'))
self.res = Resource(server, filters=[
self.auth,
])
def getJSON(self, url):
resource = Resource(url, filters=[auth])
response = resource.get(headers={'Content-Type': 'application/json'})
if response.status_int == 200:
# Not all resources will return 200 on success. There are other success status codes. Like 204. We've read
# the documentation for though and know what to expect here.
versions = json.loads(response.body_string())
return versions
else:
print response.status_int
# print response.
return None
def _delete_page(self, path):
"""delete page at a given path"""
retval = None
res = Resource(self.host)
page = res.delete(path)
data = page.body_string()
if data:
retval = json.loads(data)
return retval
def unauthorizedChecks(self, users, path, method="GET", params_dict=None):
""":param users: list of BasicAuths to try, expecting Unauthorized"""
for user in users:
name = "None"
if (user):
name = user.credentials[0]
exception = False
try:
res = Resource(self.baseurl, filters=[user])
output = res.request(method, path, params_dict=params_dict)
except Unauthorized:
exception = True
self.assertTrue(exception, "No unauthorized exception for " + name)
