def scan_in_sub_process(json_file):
try:
infos = json.loads(open(json_file, 'rb').read())
os.remove(json_file)
location = infos['location']
working_dir = infos['working_dir']
html_loc = os.path.join(working_dir, get_random_chars(5) + '.html')
if configure_scancode(SCANCODE_LOC):
scan_command = 'scancode --format=html-app --license --copyright ' + location + ' ' + html_loc
process = subprocess.Popen(SCANCODE_LOC + scan_command,
stdout=subprocess.PIPE,
shell=True)
out, err = process.communicate()
if err:
return_results(infos, 'ERROR')
else:
return_results(infos, 'FINISHED', working_dir)
except Exception, e:
decrease_current_count_by_one()
change_status_in_db('NOT STARTED', infos['location'])
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(
exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def configure_scancode(scancode_loc):
result = commands.getstatusoutput(scancode_loc + './scancode --help')
try:
if result:
return True
except Exception, e:
decrease_current_count_by_one
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def get_file_name(location):
try:
file_loc = os.path.dirname(location)
name = location.replace(file_loc, '')
return name
except Exception, e:
decrease_current_count_by_one()
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def get_file_name_with_no_ext(name):
try:
no_ext = name.split('.')[-2]
if no_ext.startswith('/'):
no_ext = no_ext[1:]
return no_ext
except Exception, e:
decrease_current_count_by_one()
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def create_zip_file(dirname, zip_name):
try:
if dirname and zip_name:
if '.zip' in zip_name:
zip_name = zip_name.replace('.zip', '')
shutil.make_archive(os.path.join('/tmp/', zip_name), 'zip', dirname)
except Exception, e:
decrease_current_count_by_one
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def configure_scancode(scancode_loc):
result = commands.getstatusoutput(scancode_loc + './scancode --help')
try:
if result:
return True
except Exception, e:
decrease_current_count_by_one
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(
exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def get_file_name(location):
try:
file_loc = os.path.dirname(location)
name = location.replace(file_loc, '')
return name
except Exception, e:
decrease_current_count_by_one()
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(
exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def get_file_name_with_no_ext(name):
try:
no_ext = name.split('.')[-2]
if no_ext.startswith('/'):
no_ext = no_ext[1:]
return no_ext
except Exception, e:
decrease_current_count_by_one()
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(
exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def create_zip_file(dirname, zip_name):
try:
if dirname and zip_name:
if '.zip' in zip_name:
zip_name = zip_name.replace('.zip', '')
shutil.make_archive(os.path.join('/tmp/', zip_name), 'zip',
dirname)
except Exception, e:
decrease_current_count_by_one
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(
exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def return_results(infos, message, working_dir):
try:
if working_dir:
zip_name = get_filename_from_url(infos['resultsurl'])
try:
create_zip_file(working_dir, zip_name)
delete_directory(working_dir)
except Exception, e:
write_log(str(e))
infos = get_infos_from_db(location=infos['location'])
count = get_current_count()
set_current_count(count-1)
change_status_in_db('FINISHED', infos['location'])
upload_to_s3(os.path.join('/tmp/', zip_name ))
else:
def return_results(infos, message, working_dir):
try:
if working_dir:
zip_name = get_filename_from_url(infos['resultsurl'])
try:
create_zip_file(working_dir, zip_name)
delete_directory(working_dir)
except Exception, e:
write_log(str(e))
infos = get_infos_from_db(location=infos['location'])
count = get_current_count()
set_current_count(count - 1)
change_status_in_db('FINISHED', infos['location'])
upload_to_s3(os.path.join('/tmp/', zip_name))
else:
def scan_in_sub_process(json_file):
try:
infos = json.loads(open(json_file, 'rb').read())
os.remove(json_file)
location = infos['location']
working_dir = infos['working_dir']
html_loc = os.path.join(working_dir, get_random_chars(5) + '.html')
if configure_scancode(SCANCODE_LOC):
scan_command = 'scancode --format=html-app --license --copyright '+location + ' ' + html_loc
process = subprocess.Popen(SCANCODE_LOC + scan_command, stdout=subprocess.PIPE, shell=True)
out, err = process.communicate()
if err:
return_results(infos, 'ERROR')
else:
return_results(infos, 'FINISHED', working_dir)
except Exception, e:
decrease_current_count_by_one()
change_status_in_db('NOT STARTED', infos['location'])
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
create_zip_file(working_dir, zip_name)
delete_directory(working_dir)
except Exception, e:
write_log(str(e))
infos = get_infos_from_db(location=infos['location'])
count = get_current_count()
set_current_count(count-1)
change_status_in_db('FINISHED', infos['location'])
upload_to_s3(os.path.join('/tmp/', zip_name ))
else:
change_status_in_db('ERROR', infos['location'])
except Exception, e:
decrease_current_count_by_one()
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def get_file_name(location):
try:
file_loc = os.path.dirname(location)
name = location.replace(file_loc, '')
return name
except Exception, e:
decrease_current_count_by_one()
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def get_file_name_with_no_ext(name):
try:
delete_directory(working_dir)
except Exception, e:
write_log(str(e))
infos = get_infos_from_db(location=infos['location'])
count = get_current_count()
set_current_count(count - 1)
change_status_in_db('FINISHED', infos['location'])
upload_to_s3(os.path.join('/tmp/', zip_name))
else:
change_status_in_db('ERROR', infos['location'])
except Exception, e:
decrease_current_count_by_one()
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(
exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))
def get_file_name(location):
try:
file_loc = os.path.dirname(location)
name = location.replace(file_loc, '')
return name
except Exception, e:
decrease_current_count_by_one()
exc_type, exc_value, exc_traceback = sys.exc_info()
error_msg = str(exc_type.__name__) + ' at line "' + str(
exc_traceback.tb_lineno) + '" in "' + __file__ + '"'
write_log(str(error_msg))