def clone(self, repo_address: str, code_dir: str, token: str):
logger.info("\tCloning Hg: " + self.get_safe_address(repo_address, token) + " to: " + code_dir)
revision_id = None
hgapi.hg_clone(repo_address, code_dir)
#TODO: Determine date and id for Hg
return revision_id
def _make_fake_measurement(self, date: timezone) -> Measurement:
"""Make a fake measurement at the given time."""
logger.info(" Making fake measurement for %s" % date)
# Numbers are +/- 10% of values for SB
num_files = random.randint(300, 360)
num_files_in_core = random.randint(135, 165)
uloc = random.randint(27000, 33000)
useful_comments = random.randint(6500, 8000)
duplicate_uloc = random.randint(3400, 4200)
num_complex = random.randint(20, 26)
fake_dict = {
'date':
date,
'Architecture Type':
'Random',
'core':
False,
'Propagation Cost':
"%.1f" % random.uniform(35, 55),
'Useful Lines of Code (ULOC)':
uloc,
'Classes':
random.randint(450, 550),
'Files':
num_files,
'num_files_in_core':
num_files_in_core,
'Core Size':
"%.1f" % (100 * num_files_in_core / num_files),
'Overly Complex Files':
num_complex,
'percent_files_overly_complex':
"%.1f" % (100 * num_complex / num_files),
'useful_lines_of_comments':
useful_comments,
'Useful Comment Density':
"%.1f" % (100 * useful_comments / uloc),
'duplicate_uloc':
duplicate_uloc,
'percent_duplicate_uloc':
"%.1f" % (100 * duplicate_uloc / uloc),
'revision_id':
'no revision id',
'Components':
make_tree_map(fake_tree)
}
return Measurement.create_from_dict(self.repo, fake_dict)
def prep_for_analysis(self) -> str:
"""Situate files for analysis! Return the revision_id if applicable"""
code_dir, data_dir, und_dir = get_directories_for_project(self.repo.name, REPO_CODE_BASE_DIR,
REPO_REPORTS_BASE_DIR,
REPO_UNDERSTAND_BASE_DIR)
# Delete that directory if it exists
if os.path.isdir(code_dir):
logger.info("\tDeleting " + str(code_dir))
shutil.rmtree(code_dir, onerror=on_rm_error)
# Then get the latest code in there!
auth_address = get_auth_address(self.repo.address, self.repo.token)
return self.stage_code(auth_address, code_dir, self.repo.token)
def remove_directories_for_project(project_name, code_base_dir, data_base_dir,
und_base_dir):
""" Remove all of the directories that would be created for a project """
if project_name:
code_dir, data_dir, und_dir = get_directories_for_project(
project_name, code_base_dir, data_base_dir, und_base_dir)
if DEBUG_UNDERSTAND:
logger.warn("DEBUG enabled - not deleting source directories")
else:
logger.info("\tDeleting the following 2 directories:\n\t%s\n\t%s" %
(code_dir, data_dir))
if os.path.isdir(code_dir):
shutil.rmtree(code_dir, onerror=on_rm_error)
if os.path.isdir(data_dir):
shutil.rmtree(data_dir, onerror=on_rm_error)
def clone(self, repo_address: str, code_dir: str, token: str) -> str:
logger.info("\tCloning Git: " +
self.get_safe_address(repo_address, token) + " to: " +
code_dir)
revision_id = None
try:
repo = Repo.clone_from(repo_address, code_dir)
# If needed in future, use django.utils.timezone instead
# revision_date = datetime.fromtimestamp(repo.head.commit.committed_date, timezone.utc)
revision_id = str(repo.head.object.hexsha)
except Exception as e:
logger.error("Repo could not be cloned")
logger.exception(e)
raise
return revision_id
def get_metrics_for_project(project_name, data_dir, output):
if not os.path.isdir(data_dir):
raise RuntimeError(
"Could not access understand results in directory: " + data_dir)
# INTENTIONALLY USING FORWARD SLASH, THIS WORKS FOR WINDOWS
# DO NOT CHANGE TO OS.PATH STUFF. UNDERSTAND WANTS FORWARD SLASHES
# EVEN IN WINDOWS. -DJC 2018-06-08
index_file = data_dir + '/index.html'
if not os.path.isfile(index_file):
raise RuntimeError("Understand results not found: ", index_file)
with open(index_file) as htmlfile:
logger.info("\tGathering metrics: " + index_file)
output['Project Name'] = project_name
# grab <head><script>
block = html.tostring(html.parse(htmlfile).getroot()[0][0])
myStr = block.decode()
metrics = json.loads(myStr.split("metrics=")[1].split(';')[0])
# Convert from name/value tags to dictionary
metricsDict = {}
for m in metrics:
name = m['name']
val = m['value']
# If blank String, don't put it in so it's easier to catch later -djc 2018-03-19
if name != "Project Name" and val:
metricsDict[name] = val.replace('%', '')
# Remove percentage information and only show core or central as appropriate
core = True
archType = metricsDict['Architecture Type']
if archType == 'Hierarchical' or archType == 'Multi-Core':
core = False
output['Core'] = core
for key, value in metricsDict.items():
output[key] = value
return output
def create(self, validated_data):
# Remove token from print version
print_data = validated_data.copy()
print_data['token'] = "..."
log_msg = "Creating repo: " + str(print_data)
logger.info(log_msg)
# Clean up common little problems with address
address = validated_data.get("address")
if address:
if address.endswith("/"):
address = address[:-1]
if address.endswith(".git"):
address = address[:-4]
validated_data["address"] = address
# Figure out type of repo from address
validated_data["type"] = get_repo_type(address,
validated_data.get("token"))
# Add the current user as the only person that can access
# the new repo til they add more people
current_user_email = self.get_current_user_email()
if current_user_email:
validated_data["allowed_emails"] = [
current_user_email,
]
# Add in the log string
validated_data['log'] = log_msg
# Make the repo per usual
repo = Repository.objects.create(**validated_data)
create_history(str(repo.id), current_user_email)
return repo
def run_understand(project_name, lang, code_dir, data_dir, und_dir):
# Check for the environment variables
und = os.getenv(UND_ENV_VAR)
uperl = os.getenv(UPERL_ENV_VAR)
if not und or not uperl:
raise RuntimeError("UND and UPERL environment variables must be set!")
# Check that the language is supported
if lang not in SUPPORTED_LANGUAGES:
raise RuntimeError("Unsupported language: " + lang)
# Remove the understand db if it exists
# INTENTIONALLY USING FORWARD SLASH, THIS WORKS FOR WINDOWS.
# DO NOT CHANGE TO OS.PATH STUFF. UNDERSTAND WANTS FORWARD SLASHES
# EVEN IN WINDOWS. -DJC 2018-06-08
clean_project_name = get_clean_project_name(project_name)
und_db = und_dir + "/" + clean_project_name + ".udb"
if os.access(und_db, os.F_OK):
os.remove(und_db)
# Call Understand
logger.info("\tSource analysis: " + str(code_dir))
logger.info("\tUnderstand DB: " + str(und_db))
und_command = und \
+ " -quiet create -languages " + lang \
+ " add " + code_dir \
+ " analyze " + und_db
und_command_split = shlex.split(und_command)
logger.info("Und command is:" + str(und_command))
run_checking_stdout_for_license(und_command_split,
"Understand error analyzing source: ")
if os.access(und_db, os.F_OK):
logger.info("\tDatabase exists at " + str(und_db))
else:
logger.error("\tDatabase does not exist at " + str(und_db))
# Generate core metrics to the specified output
logger.info("\tCore metrics: " + code_dir)
uperl_command = uperl + " " + CBRI_PLUGIN_DIR + "CoreMetrics_v1.26.pl -db " + und_db + " -createMetrics "
if DEBUG_UNDERSTAND:
uperl_command += "-createTestFiles "
uperl_command += "-DuplicateMinLines 10 -outputDir " + data_dir
uperl_command_split = shlex.split(uperl_command)
logger.info("Uperl command is:" + str(uperl_command))
run_checking_stdout_for_license(uperl_command_split,
"Understand error creating metrics: ")
def stage_code(self, repo_address: str, code_dir: str, token: str) -> str:
repo_address = repo_address.replace('file://', '')
logger.info("Copying from " + str(repo_address) + " to " +
str(code_dir))
shutil.copytree(repo_address, code_dir)
return "No VCS"