def get_org_metadata(raw=False, selectBasedOnPackageXml=False, selectedIds=[], keyword=None, **kwargs):
# debug('getting org metadata!');
# debug(raw)
# debug(selectBasedOnPackageXml)
# debug(kwargs.get('package_location', None))
project = config.project
if project.get_is_metadata_indexed():
if raw:
org_metadata_raw = util.get_file_as_string(os.path.join(project.location,"config",".org_metadata"))
org_index = json.loads(org_metadata_raw)
if kwargs.get('package_location', None) != None:
project.select_metadata_based_on_package_xml(org_index, kwargs.get('package_location'))
elif selectBasedOnPackageXml:
project.select_metadata_based_on_package_xml(org_index)
elif len(selectedIds) > 0 or keyword != None:
if keyword != None:
crawlJson.setVisibility(org_index, keyword)
if len(selectedIds) > 0:
crawlJson.setChecked(org_index, selectedIds)
return json.dumps(org_index)
else:
org_index = util.parse_json_from_file(os.path.join(project.location,"config",".org_metadata"))
if selectBasedOnPackageXml:
project.select_metadata_based_on_package_xml(org_index)
elif len(selectedIds) > 0 or keyword != None:
if keyword != None:
crawlJson.setVisibility(org_index, keyword)
if len(selectedIds) > 0:
crawlJson.setChecked(org_index, selectedIds)
return org_index
else:
IndexMetadataCommand(params=self.params).execute()
org_index = util.parse_json_from_file(os.path.join(project.location,"config",".org_metadata"))
project.select_metadata_based_on_package_xml(org_index)
return org_index
def execute(self):
if 'script_name' in self.params: #running an apex script
self.params["body"] = util.get_file_as_string(
os.path.join(config.project.location, "apex-scripts",
self.params["script_name"]))
if 'debug_categories' not in self.params and not os.path.isfile(
os.path.join(config.project.location, "config",
".apex_script")):
self.params["debug_categories"] = [{
"category": "Apex_code",
"level": "DEBUG"
}]
elif os.path.isfile(
os.path.join(config.project.location, "config",
".apex_script")):
log_settings = util.parse_json_from_file(
os.path.join(config.project.location, "config",
".apex_script"))
categories = []
levels = log_settings["levels"]
for category in levels.keys():
categories.append({
"category": category,
"level": levels[category]
})
self.params["debug_categories"] = categories
elif 'debug_categories' not in self.params:
self.params["debug_categories"] = [{
"category": "Apex_code",
"level": "DEBUG"
}]
return_log = self.params.get("return_log", True)
execute_result = config.sfdc_client.execute_apex(self.params)
result = {
'column': execute_result['column'],
'compileProblem': execute_result['compileProblem'],
'compiled': execute_result['compiled'],
'exceptionMessage': execute_result['exceptionMessage'],
'exceptionStackTrace': execute_result['exceptionStackTrace'],
'line': execute_result['line'],
'success': execute_result['success'],
}
if 'log' in execute_result and return_log:
result['log'] = execute_result['log']
if result['success']:
log_apex = config.connection.get_plugin_client_setting(
'mm_log_anonymous_apex', False)
if log_apex:
location = config.project.log_anonymous_apex(
self.params['body'], execute_result['log'],
self.params.get("script_name", None))
result["log_location"] = location
return util.generate_response(result)
def execute(self):
if 'script_name' in self.params: #running an apex script
self.params["body"] = util.get_file_as_string(os.path.join(config.project.location,"apex-scripts",self.params["script_name"]))
if 'debug_categories' not in self.params and not os.path.isfile(os.path.join(config.project.location,"config",".apex_script")):
self.params["debug_categories"] = [
{
"category" : "Apex_code",
"level" : "DEBUG"
}
]
elif os.path.isfile(os.path.join(config.project.location,"config",".apex_script")):
log_settings = util.parse_json_from_file(os.path.join(config.project.location,"config",".apex_script"))
categories = []
levels = log_settings["levels"]
for category in levels.keys():
categories.append({
"category" : category,
"level" : levels[category]
})
self.params["debug_categories"] = categories
elif 'debug_categories' not in self.params:
self.params["debug_categories"] = [
{
"category" : "Apex_code",
"level" : "DEBUG"
}
]
return_log = self.params.get("return_log", True)
execute_result = config.sfdc_client.execute_apex(self.params)
result = {
'column' : execute_result['column'],
'compileProblem' : execute_result['compileProblem'],
'compiled' : execute_result['compiled'],
'exceptionMessage' : execute_result['exceptionMessage'],
'exceptionStackTrace' : execute_result['exceptionStackTrace'],
'line' : execute_result['line'],
'success' : execute_result['success'],
}
if 'log' in execute_result and return_log:
result['log'] = execute_result['log']
if result['success']:
log_apex = config.connection.get_plugin_client_setting('mm_log_anonymous_apex', False)
if log_apex:
location = config.project.log_anonymous_apex(self.params['body'], execute_result['log'], self.params.get("script_name", None))
result["log_location"] = location
return util.generate_response(result)
def retrieve(self, **kwargs):
# request = {
# 'RetrieveRequest': {
# 'unpackaged': {
# 'types': {
# 'ApexTrigger': '*'
# }
# },
# 'apiVersion': {
# 25.0
# }
# }
# }
# package = {
# 'unpackaged' : {
# 'types' : [
# {
# "members": "*",
# "name": "ApexClass"
# }
# ]
# }
# }
package_dict = None
request_payload = None
debug('retrieve request: ')
debug(kwargs['package'])
if 'package' in kwargs and type(kwargs['package']) is not dict:
#if package is location of package.xml, we'll parse the xml and create a request
package_dict = xmltodict.parse(util.get_file_as_string(kwargs['package']))
api_version = package_dict['Package']['version']
package_dict['unpackaged'] = package_dict.pop('Package')
package_dict['unpackaged'].pop('version')
package_dict['unpackaged'].pop("@xmlns", None)
package_dict['unpackaged'].pop("#text", None)
package_dict['apiVersion'] = api_version
types = package_dict['unpackaged']['types']
if type(types) is not list:
types = [types]
if type(package_dict['unpackaged']['types']) is not list:
package_dict['unpackaged']['types'] = [package_dict['unpackaged']['types']]
requested_types = []
if 'type' in kwargs and kwargs['type'] != None and kwargs['type'] != '': #if the request is for a certain type, only request that type
for i, val in enumerate(types):
if val['name'] == kwargs['type']:
requested_types.append(val)
package_dict['unpackaged']['types'] = requested_types
types = requested_types
for i, val in enumerate(types):
try:
package_dict['unpackaged']['types'][i].pop("#text", None)
except:
package_dict['unpackaged']['types'].pop("#text", None)
#if custom object is asterisked, we need to explictly retrieve standard objects
for t in package_dict['unpackaged']['types']:
if 'name' in t:
metadata_type_def = util.get_meta_type_by_name(t['name'])
if metadata_type_def is not None and 'inFolder' in metadata_type_def and metadata_type_def['inFolder']: #TODO: right now this skips retrieval of unknown types, we should use describe data
if 'members' in t and type(t['members']) is not list:
if t['members'] == "*" or t['members'] == []:
mlist = self.listMetadata(t['name'], False)
objs = []
for obj in mlist:
objs.append(obj['fullName'])
objs.sort()
t['members'] = objs
elif t['name'] == 'CustomObject':
if 'members' in t and type(t['members']) is not list:
if t['members'] == "*":
mlist = self.listMetadata('CustomObject', False)
objs = []
for obj in mlist:
if ('__c') not in mlist:
objs.append(obj['fullName'])
objs.append("*")
objs.sort()
t['members'] = objs
request_payload = package_dict
elif 'package' in kwargs and type(kwargs['package']) is dict:
package = kwargs['package']
if package == {}:
raise MMException('Invalid package')
if 'unpackaged' not in package:
#{ "ApexClass" : ["MultiselectControllerTest","MultiselectController"] }
type_array = []
for i, metadata_type in enumerate(package):
member_value = package[metadata_type]
type_array.append({ "name" : metadata_type, "members" : member_value })
package = {
'unpackaged' : {
'types' : type_array
},
'apiVersion' : util.SFDC_API_VERSION
}
#if custom object is asterisked, we need to explictly retrieve standard objects
for t in package['unpackaged']['types']:
debug('----> ')
debug(t)
if 'name' in t:
metadata_type_def = util.get_meta_type_by_name(t['name'])
debug(metadata_type_def)
if metadata_type_def is not None and 'inFolder' in metadata_type_def and metadata_type_def['inFolder']: #TODO: right now this skips retrieval of unknown types, we should use describe data
if 'members' in t and (t['members'] == "*" or t['members'] == []):
#list_request_name = self.__transformFolderMetadataNameForListRequest(t['name'])
#mlist = self.listMetadata(list_request_name, False)
mlist = self.listMetadataAdvanced(t['name'])
objs = []
for obj in mlist:
debug('---obj')
debug(obj)
objs.append(obj['title'])
if 'children' in obj and type(obj['children'] is list):
for child in obj['children']:
objs.append(obj['title']+"/"+child['title'])
objs.sort()
t['members'] = objs
elif t['name'] == 'CustomObject':
if 'members' in t and type(t['members']) is not list:
if t['members'] == "*":
mlist = self.listMetadata('CustomObject', False)
objs = []
for obj in mlist:
if ('__c') not in mlist:
objs.append(obj['fullName'])
objs.append("*")
objs.sort()
t['members'] = objs
request_payload = package
debug('---request payload---')
debug(request_payload)
result = self._handleResultTyping(self._sforce.service.retrieve(request_payload))
debug('result of retrieve: \n\n')
debug(result)
if result.done == False:
debug('---> result is not done')
if int(float(util.SFDC_API_VERSION)) > 30:
return self._waitForRetrieveRequest(result.id) # will loop until done
else:
self._waitForRetrieveRequest(result.id) # will loop until done
return self._getRetrieveBody(result.id)
else:
return result
def run(self):
for dirname, dirnames, filenames in os.walk(os.path.join(self.project_location,"src")):
for filename in filenames:
full_file_path = os.path.join(dirname, filename)
ext = util.get_file_extension_no_period(full_file_path)
if ext in apex_extensions_to_check:
self.apex_files_to_check.append(full_file_path)
elif ext in vf_extensions_to_check:
self.vf_files_to_check.append(full_file_path)
apex_parser_threads = []
vf_parser_threads = []
apex_file_chunks = list(util.grouper(8, self.apex_files_to_check))
vf_file_chunks = list(util.grouper(8, self.vf_files_to_check))
for files in apex_file_chunks:
apex_parser_thread = ApexParser(files)
apex_parser_threads.append(apex_parser_thread)
apex_parser_thread.start()
for thread in apex_parser_threads:
thread.join()
if thread.complete:
self.apex_parser_results.update(thread.result)
for files in vf_file_chunks:
vf_parser_thread = VfParser(files)
vf_parser_threads.append(vf_parser_thread)
vf_parser_thread.start()
for thread in vf_parser_threads:
thread.join()
if thread.complete:
self.vf_parser_results.update(thread.result)
#pp = pprint.PrettyPrinter(indent=2)
#pp.pprint(self.parser_results)
for file_name in self.vf_files_to_check:
parser_result = self.vf_parser_results[file_name]
base_name = os.path.basename(file_name)
self.vf_result[base_name] = {}
file_body = util.get_file_as_string(file_name)
### ACTION POLLERS
if "actionPollers" not in parser_result:
#print file_name
continue
action_pollers = parser_result["actionPollers"]
action_poller_matches = []
if len(action_pollers) > 0:
for p in action_pollers:
line_contents = ""
for lnum in range(p["location"]["row"], p["location"]["row"]+1):
line_contents += print_file_line(file_name, lnum)
p["lineNumber"] = p["location"]["row"]
p["line_contents"] = line_contents
action_poller_matches.append(p)
self.result["visualforce_statistics"]["action_poller"]["results"].append(
{
"file_name" : base_name,
"flagged" : len(action_poller_matches) > 0,
"matches" : action_poller_matches
}
)
self.action_poller_count += len(action_poller_matches)
### HARDCODED URLS
output_links = parser_result["outputLinks"]
link_matches = []
if len(output_links) > 0:
for p in output_links:
line_contents = ""
for lnum in range(p["location"]["row"], p["location"]["row"]+1):
line_contents += print_file_line(file_name, lnum)
p["lineNumber"] = p["location"]["row"]
p["line_contents"] = line_contents
link_matches.append(p)
self.result["visualforce_statistics"]["hardcoded_url"]["results"].append(
{
"file_name" : base_name,
"flagged" : len(link_matches) > 0,
"matches" : link_matches
}
)
self.hardcoded_link_count += len(link_matches)
## REFRESHERS
refreshers = re.finditer(js_refresh_pattern, file_body)
js_matches = []
meta_matches = []
for match in refreshers:
if match != None and "meta" in match.group(0):
match_string = match.group(0).replace("<", "")
meta_matches.append(match_string)
else:
match_string = match.group(0)
js_matches.append(match_string)
if len(js_matches) > 0:
self.result["visualforce_statistics"]["javascript_refresh"]["results"].append(
{
"file_name" : base_name,
"flagged" : len(js_matches) > 0,
"matches" : js_matches
}
)
self.javascript_refresh_count += len(js_matches)
if len(meta_matches) > 0:
self.result["visualforce_statistics"]["meta_refresh"]["results"].append(
{
"file_name" : base_name,
"flagged" : len(meta_matches) > 0,
"matches" : meta_matches
}
)
self.meta_refresh_count += len(meta_matches)
for file_name in self.apex_files_to_check:
parser_result = self.apex_parser_results[file_name]
base_name = os.path.basename(file_name)
self.apex_result[base_name] = {}
file_body = util.get_file_as_string(file_name)
### WITHOUT SHARING
without_sharings = re.finditer(without_sharing_pattern, file_body)
matches = []
for match in without_sharings:
matches.append(match.group(0))
if len(matches) > 0:
self.result["apex_statistics"]["without_sharing"]["results"].append(
{
"file_name" : base_name,
"flagged" : len(matches) > 0,
"matches" : matches
}
)
self.without_sharing_count += len(matches)
#print parser_result
if "forLoops" not in parser_result:
#print file_name
continue
for_loops = parser_result["forLoops"]
dml_statements = parser_result["dmlStatements"]
queries = parser_result["queries"]
methods = parser_result["methods"]
classes = parser_result["classes"]
#seealldata
see_all_data_matches = []
for m in methods:
if "annotations" in m and len(m["annotations"]) > 0:
for a in m["annotations"]:
if "pairs" in a:
for p in a["pairs"]:
if p["name"].lower() == "seealldata" and p["value"]["value"] == True:
line_contents = ""
for lnum in range(p["beginLine"], p["beginLine"]+2):
line_contents += print_file_line(file_name, lnum)
m["lineNumber"] = p["beginLine"]
m["line_contents"] = line_contents
see_all_data_matches.append(m)
for c in classes:
if "annotations" in c and len(c["annotations"]) > 0:
for a in c["annotations"]:
if "pairs" in a:
for p in a["pairs"]:
if p["name"].lower() == "seealldata" and p["value"]["value"] == True:
line_contents = ""
for lnum in range(p["beginLine"], p["beginLine"]+2):
line_contents += print_file_line(file_name, lnum)
c["lineNumber"] = p["beginLine"]
c["line_contents"] = line_contents
see_all_data_matches.append(c)
if len(see_all_data_matches) > 0:
self.result["apex_statistics"]["see_all_data_annotations"]["results"].append(
{
"file_name" : base_name,
"flagged" : len(see_all_data_matches) > 0,
"matches" : see_all_data_matches
}
)
self.see_all_data_count += len(see_all_data_matches)
#SOQL WITHOUT WHERE CLAUSES
no_where_clause_matches = []
negative_operator_matches = []
for query in queries:
line_number = query["lineNumber"]
lower_query = query["statement"].lower()
#if ' where ' not in lower_query:
if where_pattern.search(lower_query) == None:
no_where_clause_matches.append(query)
#if ' not like ' in lower_query or '!=' in lower_query:
if not_like_pattern.search(lower_query) != None or "!=" in lower_query:
negative_operator_matches.append(query)
if len(no_where_clause_matches) > 0:
self.result["apex_statistics"]["soql_no_where_clause"]["results"].append(
{
"file_name" : base_name,
"flagged" : len(no_where_clause_matches) > 0,
"matches" : no_where_clause_matches
}
)
self.no_where_clause_count += len(no_where_clause_matches)
if len(negative_operator_matches) > 0:
self.result["apex_statistics"]["soql_negative_operators"]["results"].append(
{
"file_name" : base_name,
"flagged" : len(negative_operator_matches) > 0,
"matches" : negative_operator_matches
}
)
self.negative_soql_count += len(negative_operator_matches)
### DML INSIDE ITERATORS
dml_matches = []
query_matches = []
if len(for_loops) > 0:
for dml in dml_statements:
line_number = dml["statement"]["beginLine"]
for loop in for_loops:
if loop[0] < line_number < loop[1]:
#this is a dml statement inside an iterator
line_contents = ""
for lnum in range(loop[0], loop[1]+1):
line_contents += print_file_line(file_name, lnum)
dml["lineNumber"] = loop[0]
dml["line_contents"] = line_contents
dml_matches.append(dml)
for query in queries:
line_number = query["lineNumber"]
for loop in for_loops:
if loop[0] < line_number < loop[1]:
#this is a soql statement inside an iterator
query["line_contents"] = print_file_line(file_name, line_number)
query_matches.append(query)
if len(dml_matches) > 0:
self.result["apex_statistics"]["dml_for_loop"]["results"].append(
{
"file_name" : base_name,
"flagged" : len(dml_matches) > 0,
"matches" : dml_matches
}
)
self.dml_for_loop_count += len(dml_matches)
if len(query_matches) > 0:
self.result["apex_statistics"]["soql_for_loop"]["results"].append(
{
"file_name" : base_name,
"flagged" : len(query_matches) > 0,
"matches" : query_matches
}
)
self.soql_for_loop_count += len(query_matches)
self.result["apex_statistics"]["without_sharing"]["count"] = self.without_sharing_count
self.result["apex_statistics"]["dml_for_loop"]["count"] = self.dml_for_loop_count
self.result["apex_statistics"]["soql_for_loop"]["count"] = self.soql_for_loop_count
self.result["apex_statistics"]["soql_negative_operators"]["count"] = self.negative_soql_count
self.result["apex_statistics"]["soql_no_where_clause"]["count"] = self.no_where_clause_count
self.result["apex_statistics"]["see_all_data_annotations"]["count"] = self.see_all_data_count
self.result["visualforce_statistics"]["action_poller"]["count"] = self.action_poller_count
self.result["visualforce_statistics"]["javascript_refresh"]["count"] = self.javascript_refresh_count
self.result["visualforce_statistics"]["meta_refresh"]["count"] = self.meta_refresh_count
self.result["visualforce_statistics"]["hardcoded_url"]["count"] = self.hardcoded_link_count
return self.result
def retrieve(self, **kwargs):
# request = {
# 'RetrieveRequest': {
# 'unpackaged': {
# 'types': {
# 'ApexTrigger': '*'
# }
# },
# 'apiVersion': {
# 25.0
# }
# }
# }
# package = {
# 'unpackaged' : {
# 'types' : [
# {
# "members": "*",
# "name": "ApexClass"
# }
# ]
# }
# }
package_dict = None
request_payload = None
debug('retrieve request: ')
debug(kwargs['package'])
if 'package' in kwargs and type(kwargs['package']) is not dict:
#if package is location of package.xml, we'll parse the xml and create a request
package_dict = xmltodict.parse(util.get_file_as_string(kwargs['package']))
api_version = package_dict['Package']['version']
package_dict['unpackaged'] = package_dict.pop('Package')
package_dict['unpackaged'].pop('version')
package_dict['unpackaged'].pop("@xmlns", None)
package_dict['unpackaged'].pop("#text", None)
package_dict['apiVersion'] = api_version
if 'types' not in package_dict['unpackaged']:
# raise MMException('Invalid package.xml. Please ensure your package ')
package_dict['unpackaged']['types'] = []
types = package_dict['unpackaged']['types']
if type(types) is not list:
types = [types]
if type(package_dict['unpackaged']['types']) is not list:
package_dict['unpackaged']['types'] = [package_dict['unpackaged']['types']]
requested_types = []
if 'type' in kwargs and kwargs['type'] != None and kwargs['type'] != '': #if the request is for a certain type, only request that type
for i, val in enumerate(types):
if val['name'] == kwargs['type']:
requested_types.append(val)
package_dict['unpackaged']['types'] = requested_types
types = requested_types
for i, val in enumerate(types):
try:
package_dict['unpackaged']['types'][i].pop("#text", None)
except:
package_dict['unpackaged']['types'].pop("#text", None)
#if custom object is asterisked, we need to explictly retrieve standard objects
for t in package_dict['unpackaged']['types']:
if 'name' in t:
metadata_type_def = util.get_meta_type_by_name(t['name'])
if metadata_type_def is not None and 'inFolder' in metadata_type_def and metadata_type_def['inFolder']: #TODO: right now this skips retrieval of unknown types, we should use describe data
if 'members' in t and type(t['members']) is not list:
if t['members'] == "*" or t['members'] == []:
mlist = self.listMetadata(t['name'], False)
objs = []
for obj in mlist:
objs.append(obj['fullName'])
objs.sort()
t['members'] = objs
elif t['name'] == 'CustomObject':
if 'members' in t and type(t['members']) is not list:
if t['members'] == "*":
mlist = self.listMetadata('CustomObject', False)
objs = []
for obj in mlist:
if ('__c') not in mlist:
objs.append(obj['fullName'])
objs.append("*")
objs.sort()
t['members'] = objs
request_payload = package_dict
elif 'package' in kwargs and type(kwargs['package']) is dict:
package = kwargs['package']
if package == {}:
raise MMException('Invalid package')
if 'unpackaged' not in package:
#{ "ApexClass" : ["MultiselectControllerTest","MultiselectController"] }
type_array = []
for i, metadata_type in enumerate(package):
member_value = package[metadata_type]
type_array.append({ "name" : metadata_type, "members" : member_value })
package = {
'unpackaged' : {
'types' : type_array
},
'apiVersion' : util.SFDC_API_VERSION
}
#if custom object is asterisked, we need to explictly retrieve standard objects
for t in package['unpackaged']['types']:
debug('----> ')
debug(t)
if 'name' in t:
metadata_type_def = util.get_meta_type_by_name(t['name'])
debug(metadata_type_def)
if metadata_type_def is not None and 'inFolder' in metadata_type_def and metadata_type_def['inFolder']: #TODO: right now this skips retrieval of unknown types, we should use describe data
if 'members' in t and (t['members'] == "*" or t['members'] == []):
#list_request_name = self.__transformFolderMetadataNameForListRequest(t['name'])
#mlist = self.listMetadata(list_request_name, False)
mlist = self.listMetadataAdvanced(t['name'])
objs = []
for obj in mlist:
debug('---obj')
debug(obj)
objs.append(obj['title'])
if 'children' in obj and type(obj['children'] is list):
for child in obj['children']:
objs.append(obj['title']+"/"+child['title'])
objs.sort()
t['members'] = objs
elif t['name'] == 'CustomObject':
if 'members' in t and type(t['members']) is not list:
if t['members'] == "*":
mlist = self.listMetadata('CustomObject', False)
objs = []
for obj in mlist:
if ('__c') not in mlist:
objs.append(obj['fullName'])
objs.append("*")
objs.sort()
t['members'] = objs
request_payload = package
debug('---request payload---')
debug(request_payload)
result = self._handleResultTyping(self._sforce.service.retrieve(request_payload))
debug('result of retrieve: \n\n')
debug(result)
if result.done == False:
debug('---> result is not done')
if int(float(util.SFDC_API_VERSION)) > 30:
return self._waitForRetrieveRequest(result.id) # will loop until done
else:
self._waitForRetrieveRequest(result.id) # will loop until done
return self._getRetrieveBody(result.id)
else:
return result
