def load_from_url(self, url):
"""
reaches out to a url and loads the profile
:param url:
:return: dict: profile
"""
print type(url)
if type(url) == list:
# if we are fed a list of urls
# resolve each one and merge them with the later url taking precedence
outputDict = {}
for u in url :
Base.info("Attempting to fetch profile at: %s" % u)
response = requests.get(u, verify=False)
response.raise_for_status()
print json.loads(str(response.text))
outputDict = dict(self.merge_profiles(outputDict, json.loads(str(response.text))))
pprint.pprint(outputDict)
return outputDict
else:
response = requests.get(url, verify=False)
response.raise_for_status()
return json.loads(str(response.text))
def process(self, build_file):
"""Process an individual build file and output JSON of result to stdout."""
# Reset build_env for each build file so that the variables declared in the
# build file or the files in includes through include_defs() don't pollute
# the namespace for subsequent build files.
build_env = copy.copy(self.root_build_env)
relative_path_to_build_file = relpath(build_file, self.project_root).replace('\\', '/')
build_env['BASE'] = relative_path_to_build_file[:self.len_suffix]
build_env['BUILD_FILE_DIRECTORY'] = os.path.dirname(build_file)
build_env['RULES'] = {}
# Copy BUILD_FILE_SYMBOL_TABLE over. This is the only dict that we need
# a sperate copy of since update_lazy_functions will modify it.
build_env['BUILD_FILE_SYMBOL_TABLE'] = copy.copy(
self.root_build_env['BUILD_FILE_SYMBOL_TABLE'])
# Re-apply build_env to the rules added in this file with
# @provide_for_build.
update_lazy_functions(build_env['LAZY_FUNCTIONS'], build_env)
execfile(os.path.join(self.project_root, build_file),
build_env['BUILD_FILE_SYMBOL_TABLE'])
values = build_env['RULES'].values()
if self.strip_none:
# Filter out keys with a value of "None" from the final rule definition.
values = strip_none_entries(values)
values.append({"__includes": [build_file] + build_env['INCLUDES']})
if self.server:
print json.dumps(values)
else:
for value in values:
print json.dumps(value)
def do_POST(self):
if 'content-length' in self.headers.dict:
length = int(self.headers.dict['content-length'])
else:
logger.warn('content length required')
self.send_error(400, 'content length required for post')
return
if 'content-type' not in self.headers.dict or self.headers.dict['content-type'] != 'text/json':
logger.warn('content type missing or non-json')
body = self.rfile.read(length)
try:
logger.debug('received: [%s]' % body)
data_in = json.loads(body)
except:
logger.warn('content does not parse')
self.send_error(400, 'content does not parse as valid json')
return
try:
data_out = handle_request(data_in, self.server)
reply = json.dumps(data_out)
except TouchFormsBadRequest, e:
self.send_error(400, str(e))
return
def do_POST(self):
if "content-length" in self.headers.dict:
length = int(self.headers.dict["content-length"])
else:
logger.warn("content length required")
self.send_error(400, "content length required for post")
return
if "content-type" not in self.headers.dict or self.headers.dict["content-type"] != "text/json":
logger.warn("content type missing or non-json")
body = self.rfile.read(length)
try:
logger.debug("received: [%s]" % body)
data_in = json.loads(body)
except:
logger.warn("content does not parse")
self.send_error(400, "content does not parse as valid json")
return
try:
data_out = handle_request(data_in, self.server)
reply = json.dumps(data_out)
except TouchFormsBadRequest, e:
self.send_error(400, str(e))
return
def execute(self, expr, forest) :
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try :
sock.connect((self.HOST, self.PORT))
f = sock.makefile()
print str(expr)
f.write(str(expr) + "\n")
f.write("\n")
f.flush()
#print type(forest)
print forest.toDict()
f.write("Batch 1.0 JSON 1.0\n")
f.write(json.dumps(forest.toDict()) + "\n")
header = f.readline()
received = f.readline()
#print "Header " + str(header)
#print "Received " + str(received)
f.close()
finally :
sock.close()
# Possible no dictionary received, so hold off on loading
if received :
received = json.loads(received)
else :
received = {}
new_forest = Forest(received)
#print str(new_forest)
return new_forest # Return the forest
def body_handler(body):
if resp.status_code == 200:
favs['favorites'] = json.loads(body.to_string())
else:
print "Failed to fetch favorites: %s" % body.to_string()
EventBus.send('log.event', "user.favorites.list.result")
EventBus.send('user.favorites.list.result', json.dumps(favs))
def createIndex(self, indexName):
if indexName not in self.runtime["indices"]:
if self.runtime["client"].admin().indices().prepareExists(indexName).execute().actionGet().exists:
logger.debug("Index \"%s\" already exists", indexName)
self.runtime["indices"][indexName] = time.time()
return False
else:
logger.info("Creating index %s", indexName)
if "index_settings" in self.config:
self.config["indexSettings"] = self.config["index_settings"]
if "type_mapping" in self.config:
self.config["typeMapping"] = self.config["type_mapping"]
try:
if "indexSettings" in self.config:
settingsJsonStr = json.dumps(self.config["indexSettings"])
logger.info("Index settings: %s", settingsJsonStr)
self.runtime["client"].admin().indices().prepareCreate(indexName).setSettings(settingsJsonStr).execute().actionGet()
else:
self.runtime["client"].admin().indices().prepareCreate(indexName).execute().actionGet()
except IndexAlreadyExistsException, ex:
logger.warning(ex)
logger.warning("Index %s already exists, this should be harmless", indexName)
if "typeMapping" in self.config:
mappingJsonStr = json.dumps(self.config["typeMapping"])
logger.info("Setting mapping for %s/%s - %s", indexName, self.config["type"], mappingJsonStr)
self.runtime["client"].admin().indices().preparePutMapping().setIndices(indexName).setType(self.config["type"]).setSource(mappingJsonStr).execute().actionGet()
self.runtime["indices"][indexName] = time.time()
logger.debug("Created index: \"%s\"", indexName)
return True
def processData(self, data):
logger.debug(data)
if (self.getInputProperty("messageTemplate") != None):
template = self.getInputProperty("messageTemplate")
for key in template:
data[key] = self.evaluateCycleExpression(template[key], data)
for output in self.output:
if (self.getOutputProperty(output, "messageTemplate") != None):
template = self.getOutputProperty(output, "messageTemplate")
for key in template:
data[key] = self.evaluateCycleExpression(template[key], data)
if "outputmodule" in self.output[output]:
outputType = "plugin"
else:
outputType = self.output[output]["class"]
#TODO: deal with ommited fields
if outputType == "plugin":
self.outputWriteDocument(output, data, False)
if outputType == "stdout":
codec = self.output[output]["codec"]
if codec == "json_lines":
print(json.dumps(data))
else:
print(data)
if outputType == "file":
codec = self.output[output]["codec"]
if codec == "json_lines":
filename = self.output[output]["filename"]
self.openfiles[filename].write(json.dumps(data).encode('UTF-8'))
self.openfiles[filename].write("\n")
def get_time_difference(curr_line, prev_line):
if "'" not in curr_line and "'" not in prev_line:
curr_event = json.loads(curr_line)
prev_event = json.loads(prev_line)
difference = (float(curr_event['down']) - float(prev_event['up'])) / 1000
return difference
else:
return 0
def update_ci_property(self, ci_id, ci_property, property_value):
if self.check_CI_exist(ci_id):
ci = self.get_ci(ci_id, 'json')
data = json.loads(ci)
data[ci_property] = property_value
self.update_ci(ci_id, json.dumps(data), 'json')
else:
raise Exception("Did not find ci with id [%s]" % ci_id)
def body_handler(body):
global curators
if resp.status_code == 200:
data = json.loads(body.to_string())
curators = []
for user in data['users']:
curators.append({'screen_name': user['screen_name'], 'id': user['id']})
fetching.unlock()
EventBus.send('log.event', "curators.list.result")
EventBus.send('curators.list.result', json.dumps(curators))
def sendData(hook, layer):
url = 'http://localhost:9000/geoserver/' + hook
features = []
for feature in layer.features():
features.append(json.loads(writeJSON(feature)))
req = urllib2.Request(url, json.dumps(features), {'Content-Type': 'application/json'})
handler = urllib2.urlopen(req)
handler.read()
handler.close()
def add_new_task(self, new_task_title, new_task_type, container_id):
xlr_api_url = '/tasks/%s' % container_id
content = { "title" : new_task_title, "taskType" : new_task_type }
xlr_response = self.http_request.post(xlr_api_url, json.dumps(content), contentType='application/json')
if xlr_response.isSuccessful():
new_task = json.loads(xlr_response.getResponse())
print "Created %s\n" % new_task_title
else:
print "Failed to create %s\n" % new_task_title
print xlr_response.errorDump()
sys.exit(1)
return new_task
def set_learned_threshold_service(request):
body = {
"threshold":.5,
}
json_body = json.dumps(body)
result = request.POST(WEBROOT+"/problem-plugin/set-learned-threshold",json_body)
data = json.loads(result.getText())
if data["success"] == False:
grinder.logger.error("set-learned-threshold -- " +str(data["error"]))
else:
grinder.logger.info("set-learned-threshold -- " + str(data["success"]))
def sendData(hook, layer):
url = 'http://localhost:8080/geowebsocket/websocket/geowebsocket'
features = []
for feature in layer.features():
features.append(json.loads(writeJSON(feature)))
body = dict()
body['event'] = hook
body['layer'] = layer.name
body['features'] = features
req = urllib2.Request(url, json.dumps(body), {'Content-Type': 'application/json'})
urllib2.urlopen(req)
def get_change_request_states(self):
servicenow_api_url = '/api/now/v1/table/%s?element=state&name=task&sysparm_fields=%s' % ('sys_choice','value,label')
response = self.httpRequest.get(servicenow_api_url, contentType = 'application/json')
if response.getStatus() == SN_RESULT_STATUS:
data = json.loads(response.getResponse())
return data['result']
self.throw_error(response)
def outputWriteDocument(self, output, data, force):
if self.getInputProperty("transform") != None:
matches = re.findall(self.regexTransform, self.getInputProperty("transform"))
if matches:
out = self.getInputProperty("transform")
for match in matches:
substitution = None
(dictionary,key) = match.split(".")
if dictionary == "$cycle":
substitution = self.getCycleProperty(key)
if dictionary == "$config":
substitution = self.getInputProperty(key)
if dictionary == "$data":
if key in data:
substitution = data[key]
else:
logger.warning("Found no key named \"%s\" in %s, your data was discarded", key, dictionary)
logger.warning(data)
return None
if substitution == None:
logger.warning("Found no value for %s.%s, your data was discarded", dictionary, key)
logger.warning(data)
return None
out = out.replace(str(match), str(substitution))
data = out
if "codec" in self.outplugin[output]["config"]:
codec = self.outplugin[output]["config"]["codec"]
if codec == "json_lines":
data = json.dumps(data).encode('UTF-8')
return self.outplugin[output]["instance"].writeDocument(data, force)
def _transitionIssue(self, issue_id, new_status):
issue_url = self._issueUrl(issue_id)
# Find possible transitions
request = self._createRequest()
response = request.get(issue_url + "/transitions?expand=transitions.fields", contentType='application/json')
if response.status != 200:
error(u"Unable to find transitions for issue {0}".format(self._link(issue_id)), response)
transitions = Json.loads(response.response)['transitions']
# Check transition
wanted_transaction = -1
for transition in transitions:
if transition['to']['name'].lower() == new_status.lower():
wanted_transaction = transition['id']
break
if wanted_transaction == -1:
error(u"Unable to find status {0} for issue {1}".format(new_status, self._link(issue_id)))
# Prepare POST body
transition_data = {
"transition": {
"id": wanted_transaction
}
}
# Perform transition
response = request.post(issue_url + "/transitions?expand=transitions.fields", self._serialize(transition_data), contentType='application/json')
if response.status != 204:
error(u"Unable to perform transition {0} for issue {1}".format(wanted_transaction, self._link(issue_id)), response)
def queryIssues(self, query, options=None):
if not query:
error('No JQL query provided.')
# Create POST body
content = {
'jql': query,
'startAt': 0,
'fields': ['summary', 'status', 'assignee']
}
# Do request
request = self._createRequest()
response = request.post('/rest/api/2/search', self._serialize(content), contentType='application/json')
# Parse result
if response.status == 200:
issues = {}
data = Json.loads(response.response)
for item in data['issues']:
issue = item['key']
issues[issue] = {
'issue' : issue,
'summary' : item['fields']['summary'],
'status' : item['fields']['status']['name'],
'assignee': item['fields']['assignee']['displayName'],
'link' : "{1}/browse/{0}".format(issue, self.jira_server['url'])
}
return issues
else:
error(u"Failed to execute search '{0}' in JIRA.".format(query), response)
def update_ci_to_repo(storeName, data):
Base.info("writing ci: %s to repo" % storeName)
global StorageTimestamp
# get the store
store = load_ci_from_repo(storeName, __ciType)
# set the properties on the ci to be updated
for k, v in data.items():
store.setProperty(k, json.dumps(v))
store.setProperty('modTime', time_stamp())
# write back to xlr
if get_counter_timestamp(storeName) == StorageTimestamp:
try:
__repositoryService.update(store)
return True
except com.xebialabs.deployit.jcr.RuntimeRepositoryException as e:
Base.error('Error detected while saving %s' % storeName)
Base.error('Error: %s' % e)
return False
except com.xebialabs.deployit.repository.ItemConflictException as e:
Base.error('Error detected while saving %s' % storeName)
Base.error('Error: %s' % e)
return False
else:
Base.error('deadlock collision detected while saving %s' % storeName)
return False
def parse_error_file(self, parseTask):
"""Extract errors from JSON file
"""
#jyson import
jysonModule = File.separator.join([parseTask.app.SCRIPTDIR, "tools", "jyson.jar"])
if jysonModule not in sys.path:
sys.path.append(jysonModule)
from com.xhaus.jyson import JysonCodec as json
data = json.loads(parseTask.app.errorsData)
for e in data["errors"]:
#lat, lon, errorId
lat = float(e[0])
lon = float(e[1])
errorId = e[2]
errorType = e[3] # osmose item
#osmId
osmId = e[6]
osmId = osmId.replace("node", "n")
osmId = osmId.replace("way", "w")
osmId = osmId.replace("relation", "r")
other = [errorType]
#desc
if e[8] != "":
desc = "%s %s" % (e[8], e[9])
else:
desc = e[9]
#bbox
bbox = parseTask.build_bbox(lat, lon)
#Append to errors
if errorType in parseTask.errors:
parseTask.errors[errorType].append((osmId, (lat, lon), bbox, errorId, desc, other))
return True
def queryForIssueIds(self, query):
if not query:
error('No JQL query provided.')
# Create POST body
content = {
'jql': query,
'startAt': 0,
'fields': ['summary'],
'maxResults': 1000
}
# Do request
request = self._createRequest()
response = request.post('/rest/api/2/search', self._serialize(content), contentType='application/json')
# Parse result
if response.status == 200:
data = Json.loads(response.response)
print "#### Issues found"
issueIds = []
for item in data['issues']:
issueIds.append(item['id'])
print u"* {0} - {1}".format(item['id'], item['key'])
print "\n"
return issueIds
else:
error(u"Failed to execute search '{0}' in JIRA.".format(query), response)
def checkQuery(self, query):
if not query:
error('No JQL query provided.')
# Create POST body
content = {
'jql': query,
'startAt': 0,
'fields': ['summary', 'status']
}
# Do request
request = self._createRequest()
response = request.post('/rest/api/2/search', self._serialize(content), contentType='application/json')
# Parse result
if response.status == 200:
data = Json.loads(response.response)
issues = {}
for item in data['issues']:
issue = item['key']
issues[issue] = (item['fields']['summary'], item['fields']['status']['name'])
return issues
else:
error(u"Failed to execute search '{0}' in JIRA.".format(query), response)
def map(key, value, context):
try:
payload = json.loads(value)
except:
e = sys.exc_info()[0]
context.write(Text("exception:%s" % e), Text(key))
return
record = False
try:
ch = payload['chromeHangs']
if type(ch) == dict:
record = len(ch['memoryMap']) > 0
except KeyError:
pass
try:
lw = payload['lateWrites']
if type(lw) == dict:
record = record or len(lw['memoryMap']) > 0
except KeyError:
pass
if record:
outkey = Text(value)
context.write(outkey, Text())
def do_POST(self):
delay()
if 'content-length' in self.headers.dict:
length = int(self.headers.dict['content-length'])
else:
logging.warn('content length required')
self.send_error(400, 'content length required for post')
return
if 'content-type' not in self.headers.dict or self.headers.dict['content-type'] != 'text/json':
logging.warn('content type missing or non-json')
body = self.rfile.read(length)
try:
logging.debug('received: [%s]' % body)
data_in = json.loads(body)
except:
logging.warn('content does not parse')
self.send_error(400, 'content does not parse as valid json')
return
try:
data_out = handle_request(data_in, extensions=self.server.extensions)
except (Exception, java.lang.Exception), e:
if isinstance(e, java.lang.Exception):
e.printStackTrace() #todo: log the java stacktrace
logging.exception('error handling request')
self.send_error(500, 'internal error handling request: %s: %s' % (type(e), str(e)))
return
def send_error(self, code, message=None, error_type=None, human_readable_message=None):
"""
Override send_error to always return JSON.
"""
# copied and pasted lots of this from the base class
# but had to override due to html escaping messing up
# the json format of the message
try:
short, long = self.responses[code]
except KeyError:
short, long = '???', '???'
if message is None:
message = short
if human_readable_message is None:
human_readable_message = message
explain = long
logger.exception("Status Code: %d, Message %s" % (code, message))
content = json.dumps({'status': 'error',
'error_type': error_type,
'code': code,
'message': message,
'human_readable_message': human_readable_message,
'explain': explain})
# if this is more than one line it messes up the response content
message = message.split("\n")[0] if message else ""
self.send_response(code, message.encode("ascii", "xmlcharrefreplace"))
self.send_header("Content-Type", self.error_content_type)
self.cross_origin_header()
self.send_header('Connection', 'close')
self.end_headers()
if self.command != 'HEAD' and code >= 200 and code not in (204, 304):
self.wfile.write(content.encode("utf-8"))
def tick(self):
# Burn through old messages, and only use the latest ones
# This will allow the controller to keep working if the simulation slows
# down. In the future they should be synchronized
while True:
if self.read_socket( self.sock_in ):
try:
data_in = json.loads( self.odom_str )
except JSONDecodeError:
break
self.sensor_data = data_in
else:
break
self.position.set([self.sensor_data["roll"]])
self.velocity.set([self.sensor_data["wx"]])
self.counter += 1
if self.counter % CONTROL_PERIOD == 0:
torque = np.array(self.torque.get())[0]
# Send the command to the simulator as a torque
data_out = '{"force":[0,0,0],"torque":[%f,0,0]}\n' % torque
self.sock_out.send( data_out )
def postgres_lookup_session_command(cursor, key):
postgres_lookup(cursor, POSTGRES_TABLE, 'sess_id', str(key), 'sess_json')
if cursor.rowcount is 0:
raise KeyError
value = cursor.fetchone()[0]
jsonobj = json.loads(value.decode('utf8'))
return jsonobj
def _get_records(f_100, sug_json):
try:
url = 'http://libris.kb.se/xsearch'
values = {'query' : 'forf:(%s) spr:swe' % f_100, 'format' : 'json'}
data = urllib.urlencode(values)
#print "XSEARCH URL: %s?%s" % (url, data)
reply = urllib2.urlopen(url + "?" + data)
response = reply.read().decode('utf-8')
#print "got response", response, type(response)
xresult = json.loads(response)['xsearch']
sug_json['records'] = xresult['records']
top_3 = xresult['list'][:3]
top_titles = {}
for p in top_3:
top_titles[p['identifier']] = unicode(p['title'])
sug_json['top_titles'] = top_titles
except:
print "exception in get_records"
0
return sug_json
def __init__(self, sd, query, output = 'json'):
"""
Execute the query and store the results.
"""
self._sd = sd;
self._query = query;
self._data = {
'output' : output,
'query' : query
}
#self.response = requests.get(self._sd.url, params = self._data)
#print "url = " + str(self._sd.url)
conn = httplib.HTTPConnection(self._sd.url)
#print "conn = " + str(conn)
#print "query = " + str(query)
_path = self._sd.sqpath + "?" + urllib.urlencode(self._data)
#print "path = " + str(_path)
conn.request("GET", _path );
response = conn.getresponse();
if response.status != 200:
print response.status, response.reason, "\n"
raise IOError, "HTTP GET %s not successful" % _path
head = response.msg
#print "head = " + str(head)
self.text = response.read()
#print "body = " + str(self.text)
self.json = json.loads(self.text)
#print "json = " + str(self.json)
conn.close()
def find_record(self, table_name, query):
if self.useOAuth: self.issue_token()
servicenow_api_url = '/api/now/v1/table/%s?%s&%s' % (table_name, query,
self.sysparms)
print "Service Now URL = %s " % (servicenow_api_url)
response = self.httpRequest.get(servicenow_api_url,
contentType='application/json',
headers=self.headers)
if self.useOAuth: self.revoke_token()
if response.getStatus() == SN_RESULT_STATUS:
data = json.loads(response.getResponse())
return data['result']
else:
print "find_record error %s" % (response)
self.throw_error(response)
def get_all_sprints(self, board):
if not board:
error("No board id provided")
request = self._createRequest()
response = request.get("/rest/agile/1.0/board/%s/sprint" % board["id"], contentType="application/json")
sprints = {}
if response.status != 200:
error(u"Unable to find sprints for board {0}".format(board["name"]), response)
sprints_json = Json.loads(response.response)['values']
for sprint_json in sprints_json:
sprints[sprint_json["name"]] = sprint_json["id"]
print "| %s | %s | %s | %s |" % (sprint_json["name"], sprint_json["id"],
sprint_json["startDate"] if sprint_json.has_key(
"startDate") else "not defined",
sprint_json["endDate"] if sprint_json.has_key(
"endDate") else "not defined")
return sprints
def createVersion(self, versionName, description, project):
data = {
"description": description,
"name": versionName,
"project": project,
"startDate": str(datetime.date.today())
}
createVersionUrl = "/rest/api/2/version"
request = self._createRequest()
response = request.post(createVersionUrl,
self._serialize(data),
contentType='application/json')
if response.status != 201:
error(
u"Unable to create version {0} for project {1}".format(
versionName, project), response)
return Json.loads(response.response)['id']
def run_jblock(filename, newdevice):
f = open(filename, 'r')
'''
except IOError:
print 'problem reading:' + filename
'''
print "opened file"
total_completed = 0
total_actions = 0
current_line = 1
'''
if not os.path.exists('./images'):
os.mkdir('./images')
newdevice.wake()
pathName = os.path.abspath('./images')
#print pathName
startShot = os.path.join(pathName, 'start.png')
screenshot = newdevice.takeSnapshot()
screenshot.writeToFile(startShot)
'''
prev_line = None
for line in f:
print str(current_line) + " - ",
current_line += 1
if prev_line is not None:
pause = get_time_pause(line, prev_line)
time.sleep(pause)
prev_line = line
total_actions += 1
single_quotes = line.find("\'")
if single_quotes == -1:
device_input = json.loads(line)
complete = run_input(device_input, newdevice,
False) # change to True to
if complete:
total_completed += 1
else:
action = str(device_input).replace(': u', ': ')
print 'could not replay action ' + str(action)
else:
print 'could not replay action ' + line
print str(total_completed) + '/' + str(
total_actions) + ' actions completed'
f.close()
'''
def get_template(self, template_name):
xlr_api_url = '/api/v1/templates?filter=%s' % urllib.quote(
template_name)
print "Going to use xlr_api_url: ", xlr_api_url
xlr_response = self.http_request.get(xlr_api_url,
contentType='application/json')
if xlr_response.isSuccessful():
data = json.loads(xlr_response.getResponse())
for template in data:
if template["title"] == template_name:
print "Found template %s with id %s" % (template_name,
template["id"])
return XLTemplate(template["id"],
[str(t) for t in template["tags"]])
print "Failed to find template in XL Release %s" % template_name
xlr_response.errorDump()
raise ServerError(str(xlr_response.getResponse()))
def checkCriticity(P_criticite,P_workitem):
global result
global description
data = json.loads(P_workitem)
customMap = data["customFields"]["Custom"]
indexCriticiteWorkitem = map(itemgetter('key'), customMap).index('criticality')
criticiteWorkitem = customMap[indexCriticiteWorkitem]["value"]["id"]
indexCriticite = map(itemgetter('polarion'), criticiteMap).index(criticiteWorkitem)
if criticiteMap[indexCriticite].get("jira") != P_criticite:
log.error('[' + defect_id + ']' + E4)
result = False
description = u'ERREUR : Incohérence de criticité entre JIRA ' + defect_id + ' (' + P_criticite + ') et le test Polarion ' + workitem_id + ' (' + criticiteWorkitem + ')'
sendMetrics(E4)
else:
result = True
def refresh_token(self, httpConnection, refreshToken):
servicenowUrl = "/oauth_token.do"
content = {}
content['grant_type'] = 'refresh_token'
content['client_id'] = httpConnection['clientId']
content['client_secret'] = httpConnection['clientSecret']
content['refresh_token'] = refreshToken
httpRequest = HttpRequest(httpConnection, None, None)
response = httpRequest.post(
servicenowUrl,
body=urllib.urlencode(content),
contentType='application/x-www-form-urlencoded')
if response.getStatus() == SN_RESULT_STATUS:
data = json.loads(response.getResponse())
return data
print "Unable to refresh token using %s" % refreshToken
self.throw_error(response)
def process_record(self):
contentx = {}
# First old stuff so the newer fields will overwrite.
if 'content' in self.task_vars.keys():
if content:
oldStuff = {}
oldStuff = json.loads(content)
for m, y in oldStuff.items():
contentx[m] = y
self.set_from_task_vars('shortDescription', contentx, 'short_description')
self.set_from_task_vars('description', contentx)
self.set_from_task_vars('assignmentGroup', contentx, 'assignment_group')
self.set_from_task_vars('assignedTo', contentx, 'assigned_to')
self.set_from_task_vars('priority', contentx)
self.set_from_task_vars('state', contentx)
self.set_from_task_vars('ciSysId', contentx, 'cmdb_ci')
self.set_from_task_vars('comments', contentx)
self.set_from_task_vars('changeRequest', contentx, 'change_request')
self.set_from_task_vars('workNotes', contentx, 'work_notes')
self.set_from_task_vars('storyPoints', contentx, 'story_points')
self.set_from_task_vars('epic', contentx)
self.set_from_task_vars('product', contentx)
self.set_from_task_vars('sprint', contentx)
self.set_from_task_vars('acceptanceCriteria', contentx, 'acceptance_criteria')
self.set_from_task_vars('taskType', contentx, 'type')
self.set_from_task_vars('plannedHours', contentx, 'planned_hours')
self.set_from_task_vars('story', contentx)
self.set_from_task_vars('impact', contentx)
self.set_from_task_vars('urgency', contentx)
self.set_from_task_vars('closeCode', contentx, 'close_code')
self.set_from_task_vars('closeNotes', contentx, 'close_notes')
# Also sending release info.
contentx['x_xlbv_xl_release_identifier'] = str(release.id)
contentx['x_xlbv_xl_release_state'] = str(release.status)
for k, v in self.task_vars['additionalFields'].items():
contentx[k] = v
response = self.sn_client.update_record(self.table_name, self.task_vars['sysId'], contentx,
getCurrentTask().getId())
return response
def create_token(self, httpConnection):
servicenow_oauth_url = "/oauth_token.do"
content = {}
content['grant_type'] = 'password'
content['client_id'] = httpConnection['clientId']
content['client_secret'] = httpConnection['clientSecret']
content['username'] = httpConnection['oauthUsername']
content['password'] = httpConnection['oauthPassword']
httpRequest = HttpRequest(httpConnection, None, None)
response = httpRequest.post(
servicenow_oauth_url,
body=urllib.urlencode(content),
contentType='application/x-www-form-urlencoded')
if response.getStatus() == SN_RESULT_STATUS:
data = json.loads(response.getResponse())
return data
print 'Could not get access token'
self.throw_error(response)
def delete_package(self, instance, package, olderThan, status):
endevorUrl = 'EndevorService/rest/%s/packages/%s' % (instance, package)
if olderThan:
endevorUrl = "%s&olderthan=%s" % (endevorUrl, olderThan)
if status:
endevorUrl = "%s&status=%s" % (endevorUrl, status)
print endevorUrl.replace('&', '?', 1)
response = self.httpRequest.delete(endevorUrl.replace('&', '?', 1),
contentType='application/json')
if response.getStatus() in HTTP_SUCCESS:
data = json.loads(response.getResponse())
print("Delete Package Return = %s" % data)
# TO-DO: determine structure of returned data
# return (data.returnCode, data.reasonCode, data.data['key'])
return ("0000", "0000", "Endevor delete result")
self.throw_error(response)
def getIssue(self, issueId):
if issueId == None or len(issueId) < 1:
error(u'IssueId can not be null')
request = self._createRequest()
url = '/issues/' + issueId + '.json'
try:
response = request.get(url,
content=None,
contentType=self.content_type,
headers=self._createHeaders())
if response.status == 200:
data = Json.loads(response.response)
return data
else:
error(u'Failed to get issue', response)
except ClientProtocolException:
raise Exception()
def getPageIdsByTitle(self, spaceKey, pageTitles):
print "Executing getPageIdsByTitle() in ConfluenceClient\n"
contentType = "application/json"
headers = {'Accept': 'application/json'}
pageIdList = []
for pageTitle in pageTitles:
searchByPageTitleUrl = '/rest/api/content?spaceKey=%s&title=%s' % (
spaceKey, pageTitle)
response = self.httpRequest.get(searchByPageTitleUrl,
contentType=contentType,
headers=headers,
quotePlus=True)
if response.getStatus() not in HTTP_SUCCESS:
self.throw_error(response)
result = json.loads(response.response)
for page in result['results']:
pageIdList.append(page['id'])
return pageIdList
class XFormRequestHandler(BaseHTTPRequestHandler):
error_content_type = "text/json"
def do_POST(self):
delay()
if 'content-length' in self.headers.dict:
length = int(self.headers.dict['content-length'])
else:
logging.warn('content length required')
self.send_error(400, 'content length required for post')
return
if 'content-type' not in self.headers.dict or self.headers.dict['content-type'] != 'text/json':
logging.warn('content type missing or non-json')
body = self.rfile.read(length)
try:
logging.debug('received: [%s]' % body)
data_in = json.loads(body)
except:
logging.warn('content does not parse')
self.send_error(400, 'content does not parse as valid json')
return
try:
data_out = handle_request(data_in, extensions=self.server.extensions)
except (Exception, java.lang.Exception), e:
if isinstance(e, java.lang.Exception):
e.printStackTrace() #todo: log the java stacktrace
logging.exception('error handling request')
self.send_error(500, 'internal error handling request: %s: %s' % (type(e), str(e)))
return
reply = json.dumps(data_out)
logging.debug('returned: [%s]' % reply)
delay()
self.send_response(200)
self.send_header('Content-Type', 'text/json; charset=utf-8')
self.end_headers()
self.wfile.write(reply.encode('utf-8'))
def add_dependency(self, dependency_release_id, gate_task_id):
# the internal api uses a rel-phase-task format instead of Applications/Rel/Phase/Task
# is there a cleaner way to do this??
# TODO move to public API once it is possible using the public API
internal_format_task_id = gate_task_id.replace('Applications/',
'').replace('/', '-')
xlr_api_url = '/gates/%s/dependencies' % internal_format_task_id
content = {"target": {"releaseId": dependency_release_id}}
xlr_response = self.http_request.post(xlr_api_url,
json.dumps(content),
contentType='application/json')
if xlr_response.isSuccessful():
print "Dependency added to Gate task\n"
else:
print "Failed to add dependency\n"
print xlr_response.errorDump()
sys.exit(1)
def getIssues(self, issueIds=None, projectId=None, otherFields=None):
request = self._createRequest()
issues = {}
filters = ''
url = '/issues.json'
if issueIds != None and len(issueIds) > 0:
filters = 'issue_id=' + ','.join(issueIds)
if projectId != None:
if len(filters) > 0:
filters += '&'
filters += 'project_id=' + projectId
if otherFields != None:
if len(filters) > 0:
filters += '&'
for loop in otherFields:
filters += urllib.quote_plus(loop) + '=' + urllib.quote_plus(
otherFields[loop]) + '&'
filters = filters[:-1]
if len(filters) > 0:
url += '?' + filters
try:
response = request.get(url,
content=None,
contentType=self.content_type,
headers=self._createHeaders())
if response.status == 200:
data = Json.loads(response.response)
for item in data['issues']:
id = item['id']
subject = item['subject']
issues[id] = subject
return issues
else:
error(u'Failed to get issues', response)
except ClientProtocolException:
raise Exception()
return issues
def _get_station_ids(bounds):
"""
queries within bounds and returns the json result as stations associative array
"""
url = 'http://www.water.ca.gov/waterdatalibrary/maps/stnlocation.cfc?method=greg&returnFormat=json&argumentCollection=%s&_cf_nodebug=true&_cf_nocache=true&_cf_clientid=C664A0B77D0DF4B9EF22623335665E8E&_cf_rc=11'
print 'Getting station ids for %s'%bounds
argumentCollection = '{"LatNorth":%f,"LatSouth":%f,"LonWest":%f,"LonEast":%f,"getGW":false,"getWQ":true,"getHY":false,"getUser":"******"}' % (bounds['LatNorth'], bounds['LatSouth'], bounds['LonWest'], bounds['LonEast'])
argumentCollection = urllib.quote(argumentCollection)
response = urllib.urlopen(url % (argumentCollection))
result = response.readlines()
response.close()
result = reduce(lambda x, y: x + y, result)
stations = json.loads(result);
station_ids=[]
for station in stations:
if station.has_key('point'):
for val in station['point']:
station_ids.append(station['point'][val]['STATIONNUMBER'])
return station_ids
def msglst(self, recipient, mode=None):
if mode:
self.send_command('MSGLST', '%s %s' % (recipient, mode) )
else:
self.send_command('MSGLST', recipient)
result = self.read_response()
if mode == 'json':
if result:
d = json.loads(result)
else:
d = []
else:
d = []
for line in result:
d.append(line.strip()[4:])
return d
def backin_package(self, instance, package, statement, element):
endevorUrl = 'EndevorService/rest/%s/packages/%s/Backin' % (instance,
package)
if statement:
endevorUrl = "%s&statement=%s" % (endevorUrl, statement)
if element:
endevorUrl = "%s&element=%s" % (endevorUrl, element)
print endevorUrl.replace('&', '?', 1)
response = self.httpRequest.put(endevorUrl.replace('&', '?', 1),
'{}',
contentType='application/json')
if response.getStatus() in HTTP_SUCCESS:
data = json.loads(response.getResponse())
print("Backin Package Return = %s" % data)
# TO-DO: determine structure of returned data
# return (data.returnCode, data.reasonCode, data.data['key'])
return ("0000", "0000", "Endevor backin result")
self.throw_error(response)
def _transitionIssue(self, issue_id, new_status):
issue_url = self._issueUrl(issue_id)
# Find possible transitions
request = self._createRequest()
response = request.get(issue_url +
"/transitions?expand=transitions.fields",
contentType='application/json')
if response.status != 200:
error(
u"Unable to find transitions for issue {0}".format(
self._link(issue_id)), response)
transitions = Json.loads(response.response)['transitions']
# Check transition
wanted_transaction = -1
for transition in transitions:
if transition['to']['name'].lower() == new_status.lower():
wanted_transaction = transition['id']
break
if wanted_transaction == -1:
error(u"Unable to find status {0} for issue {1}".format(
new_status, self._link(issue_id)))
# Prepare POST body
transition_data = {"transition": {"id": wanted_transaction}}
# Perform transition
response = request.post(issue_url +
"/transitions?expand=transitions.fields",
self._serialize(transition_data),
contentType='application/json')
if response.status != 204:
error(
u"Unable to perform transition {0} for issue {1}".format(
wanted_transaction, self._link(issue_id)), response)
def dump(self, mode=None):
self.send_command('DUMP', mode)
result = self.read_response()
if result == 'No entries':
return None
if mode == 'json':
d = json.loads(result)
else:
d = {}
for line in result:
if line[0] == '\t':
d[rcpt].append(line.strip()[4:])
else:
rcpt = line[:-1]
d[rcpt] = []
return d
def getWorkitem(P_id_workitem):
global workitem
global description
global WorkitemUpdatedDate
global WorkitemCreationDate
global result
try:
r = urllib2.urlopen("http://" + ip_dashboard + "/api-polarion-1/" + path_workitem + P_id_workitem)
workitem = r.read()
WorkitemCreationDate = getWorkitemCreationDate(workitem)
WorkitemUpdatedDate = getWorkitemUpdatedDate(workitem)
except urllib2.HTTPError, httpError:
print 'HTTPError'
try:
data = json.loads(httpError.read())
error = data["status"]["status_content"][0]["message"]
throwError(error, "ID-cas-de-test", E3, P_id_workitem)
except:
log.error('[' + defect_id + ']' + E8 + str(httpError))
description = description + E8 + str(httpError)
sendMetrics(str(httpError))
result = False
def wait_for_deploy(self, deploymentId):
url = '/api/deployments/%s' % deploymentId
response = self.httpRequest.get(url, headers=self.headers)
if response.getStatus() not in HTTP_SUCCESS:
self.throw_error(response)
deployment_details = json.loads(response.getResponse())
taskUrl = deployment_details["Links"]["Task"]
time.sleep(5)
task_details = self.get_task_details(taskUrl)
while not task_details["IsCompleted"]:
task_details = self.get_task_details(taskUrl)
time.sleep(5)
if task_details["FinishedSuccessfully"]:
print("Deployment finished successfully.")
else:
msg = "Deployment failed, errors: [%s]" % task_details["ErrorMessage"]
print(msg)
sys.exit(msg)
def queryIssues(self, query, options=None):
if not query:
error('No JQL query provided.')
# Create POST body
content = {
'jql': query,
'startAt': 0,
'fields': ['summary', 'status', 'assignee'],
'maxResults': 1000
}
# Do request
request = self._createRequest()
response = request.post('/rest/api/2/search',
self._serialize(content),
contentType='application/json')
# Parse result
if response.status == 200:
issues = {}
data = Json.loads(response.response)
for item in data['issues']:
issue = item['key']
assignee = "Unassigned" if (
item['fields']['assignee'] is None
) else item['fields']['assignee']['displayName']
issues[issue] = {
'issue': issue,
'summary': item['fields']['summary'],
'status': item['fields']['status']['name'],
'assignee': assignee,
'link': "{1}/browse/{0}".format(issue,
self.jira_server['url'])
}
return issues
else:
error(u"Failed to execute search '{0}' in JIRA.".format(query),
response)
def request(self,
method,
url,
headers,
content_type='application/json',
body=None):
if self.useOAuth:
self.issue_token()
if method == 'GET':
response = self.httpRequest.get(url,
contentType=content_type,
headers=headers)
elif method == 'PUT':
response = self.httpRequest.put(url,
body=body,
contentType=content_type,
headers=headers)
else:
response = self.httpRequest.post(url,
body=body,
contentType=content_type,
headers=headers)
if self.useOAuth:
self.revoke_token()
if response.getStatus() == SUCCESS_STATUS_CODE or response.getStatus(
) == RECORD_CREATED_STATUS:
try:
data = json.loads(response.getResponse())
return data['result'] if 'result' in data else data['records']
except:
print response.getResponse()
raise RuntimeError("Cannot convert to json")
else:
print response.getResponse()
self.throw_error(response)
def create_workitem(self, project_area, workitem_type, title, description,
properties):
context_id = self._get_context_id(project_area)
if context_id is None:
self._error('Project area "%s" not found.' % project_area)
self.logger.debug('create_workitem: context id: "%s"' % context_id)
print 'Creating work item in project "%s"' % context_id
contentType = "application/json"
headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
}
body = {'dc:title': title, 'dc:description': description}
body.update(properties)
url = WORKITEM_FACTORY_URI[workitem_type].replace(
'${context}', context_id)
self.logger.debug('create_workitem: body: %s' % body)
self.logger.debug('create_workitem: url: %s' % url)
response = self.httpRequest.post(url,
body,
contentType=contentType,
headers=headers)
self.logger.debug('create_workitem: response: %s' % repr(response))
if response.getStatus() not in HTTP_SUCCESS:
self.logger.error('create_workitem: status: %s' %
response.getStatus())
self._error('Unable to create workitem', response)
# extract identifier
self.logger.debug('create_workitem: parse response as json')
data = json.loads(response.response)
self.logger.info('create_workitem: data: %s' % data)
return data['dc:identifier']
def create_release(self, release_title, release_description, variables,
tags, template_id):
content = """
{"title":"%s","description":"%s","scheduledStartDate":"%sT23:58:00.000Z","dueDate":"%sT23:59:00.000Z","plannedDuration":null,"variables":%s,"tags":%s,"flag":{"status":"OK"},"templateId":"%s"}
""" % (release_title, release_description, date.today(), date.today(),
variables, tags, template_id)
print "Sending content %s" % content
xlr_api_url = '/releases'
xlr_response = self.http_request.post(xlr_api_url,
content,
contentType='application/json')
if xlr_response.isSuccessful():
data = json.loads(xlr_response.getResponse())
release_id = data["id"]
print "Created %s in XLR" % release_id
return release_id
else:
print "Failed to create release in XLR"
xlr_response.errorDump()
raise ServerError(str(xlr_response.getResponse()))
def createIssue(self, projectId, priorityId, subject):
request = self._createRequest()
newContent = {
'issue': {
'project_id': projectId,
'priority_id': priorityId,
'subject': subject
}
}
newContent = self._serialize(newContent)
try:
response = request.post('/issues.json',
newContent,
contentType=self.content_type,
headers=self._createHeaders())
if response.status == 201:
data = Json.loads(response.getResponse())
return data
else:
print u'Error creating issue: {0}'.format(response.errorDump())
except ClientProtocolException:
raise Exception()
def msgget(self, recipient, msgid, mode=None):
if mode:
self.send_command('MSGGET', '%s %d %s' % (recipient, msgid, mode) )
else:
self.send_command('MSGGET', '%s %d' % (recipient, msgid))
result = self.read_response()
if mode == 'json':
if result:
d = json.loads(result)
else:
d = {}
if mode == 'mime':
fp = email.parser.Parser()
print result
msg = fp.parsestr(result)
d = msg
else:
d = result
return d
def test_generate_payload(self):
self.tm.create_all_metrics(1)
thread = ingest.IngestThread(0)
payload = json.loads(thread.generate_payload(0, [[2, 3], [2, 4], [2, 5]]))
valid_payload = [{u'collectionTime': 0,
u'metricName': u'int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.3',
u'metricValue': 0,
u'tenantId': u'2',
u'ttlInSeconds': 172800,
u'unit': u'days'},
{u'collectionTime': 0,
u'metricName': u'int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.4',
u'metricValue': 0,
u'tenantId': u'2',
u'ttlInSeconds': 172800,
u'unit': u'days'},
{u'collectionTime': 0,
u'metricName': u'int.abcdefg.hijklmnop.qrstuvw.xyz.ABCDEFG.HIJKLMNOP.QRSTUVW.XYZ.abcdefg.hijklmnop.qrstuvw.xyz.met.5',
u'metricValue': 0,
u'tenantId': u'2',
u'ttlInSeconds': 172800,
u'unit': u'days'}]
self.assertEqual(payload, valid_payload)
def ship_package(self, instance, package, destination, option, prefix):
endevorUrl = 'EndevorService/rest/%s/packages/%s/Ship' % (instance,
package)
if destination:
endevorUrl = "%s&destination=%s" % (endevorUrl, destination)
if option:
endevorUrl = "%s&option=%s" % (endevorUrl, option)
if prefix:
endevorUrl = "%s&prefix=%s" % (endevorUrl, prefix)
print endevorUrl.replace('&', '?', 1)
response = self.httpRequest.put(endevorUrl.replace('&', '?', 1),
'{}',
contentType='application/json')
if response.getStatus() in HTTP_SUCCESS:
data = json.loads(response.getResponse())
print("Ship Package Return = %s" % data)
return (data['returnCode'], data['reasonCode'], data['messages'])
print("Return Code = %s" % data['returnCode'])
print("Reason Code = %s" % data['reasonCode'])
print("Message = %s" % data['messages'])
self.throw_error(response)
