def __init__(self, repo_name):
repo_info_json_url = re.sub('repos_name', repo_name, InfoJSONObj.RAW_INFO_JSON_URL)
try:
self.repo_info_json = json.loads(URILoader.loadURI(repo_info_json_url, repo_name+"'s info.json file", "Switch to next repo."))
except ValueError:
raise Exception("ValueError: "+repo_name+"'s info.json has an illegal format. Please check!"+"Switch to next repo.")
except Exception as e:
raise e
self.item_list = []
for key in InfoJSONObj.KEY_LIST:
try:
if type(self.repo_info_json[key]) == list:
val = self.repo_info_json[key][0]
else:
val = self.repo_info_json[key]
self.item_list.append(JSONObj(key,val.strip()))
except:
raise ValueError("info.json missed some of the items below:\n"
"type, provider, software, language, category, promotion.")
def createLangIndex(*args):
outdir = args[0]
ext_path = args[1]
index_key = RAW_INDEX_KEY
langLogger = None
try:
lang_path = os.path.join(outdir,LANG_DIR)
os.mkdir(lang_path)
root_log_dir = os.path.join(outdir, LOG_DIR_NAME)
langLogger = Logger(os.path.join(root_log_dir,LOG_INFO),'langLogger')
langLogger.info("CreateLicenseIndex script start ...")
langLogger.info("Get extension index ...")
repos_set_json = loadExtJSONContent(ext_path)
repos_set_json_index = repos_set_json[index_key]
for lang_item in LANG_LIST:
fp_content = LANG_INDEX_PRE
langLogger.info("Start to get '"+lang_item+"' file")
i=0
for repo in repos_set_json_index:
try:
repo_name = repo["repository"]
i+=1
except Exception:
raise Exception("At least one repository in index file does not have repo name. Please check!")
repo_lang_uri = RAW_REPO_LANG_URI.format(repo_name, lang_item)
try:
repo_lang_content = URILoader.loadURI(repo_lang_uri, "language index file")
lang_json_str = LangPropObj.convertToJSONStr(repo_name, repo_lang_content, lang_item)
except Exception as e:
if 'HTTPError' in str(e):
# some repositories do not have lang file, by default use en file in index for extension file
summary = ""
if 'Summary' in repo[EXT_KEY].keys():
summary = repo[EXT_KEY]['Summary']
desc = ""
if 'Description' in repo[EXT_KEY].keys():
desc = repo[EXT_KEY]['Description']
lang_json_str = LangPropObj.generateJSONStr(repo_name,summary,desc)
else:
raise e
fp_content += lang_json_str
try:
fp_content = fp_content[0:-2]+'\n'+INDENT+']\n}'
fp = open(os.path.join(lang_path, lang_item+'.json'), 'w', encoding='utf-8')
fp.write(fp_content)
fp.close()
except Exception as e:
raise e
except Exception as e:
if langLogger!=None:
langLogger.error(str(e),e)
langLogger.info("CreatelangIndex action failed!")
raise e
finally:
if langLogger!=None:
langLogger.close()
def CreateIndex(*args):
print("size" + str(len(args)))
outdir = args[0]
product = args[1]
if product == "modeler":
repos_set_uri = RAW_REPOS_SET_URI.format('modeler', 'modeler')
index_key = RAW_INDEX_KEY.format('modeler')
elif product == 'stats':
# wrong spell of statistics
repos_set_uri = RAW_REPOS_SET_URI.format('statisitcs', 'stats')
index_key = RAW_INDEX_KEY.format('stats')
try:
lic_path = os.path.join(outdir, LICENSE_DIR)
os.mkdir(lic_path)
logger = Logger(os.path.join(lic_path, LOG_INFO))
logger.info("Script start ...")
except IOError as e:
raise IOError("IOError: Need permission to write in " + outdir)
try:
try:
repos_set_json = json.loads(
URILoader.loadURI(repos_set_uri, "index file"))
except ValueError as e:
raise Exception(
"ValueError: The {0} has an illegal format. Please check!\n\n"
.format("index file"))
except Exception as e:
raise e
try:
repos_set_json_index = repos_set_json[index_key]
except Exception as e:
raise e
license_obj_list = []
for repo in repos_set_json_index:
try:
repo_name = repo["repository"]
except Exception:
raise Exception(
"At least one repository in index file does not have repo name. Please check!\n\n"
)
repo_license_uri = RAW_LICENSE_URI.format(repo_name)
try:
repo_license_content = URILoader.loadURI(
repo_license_uri, "license file")
except Exception as e:
raise e
isExistedLicense = False
for item in license_obj_list:
if repo_license_content == item.getLicenseContent():
isExistedLicense = True
item.addRepoName(repo_name)
break
if not isExistedLicense:
addObj(repo_name, repo_license_content, license_obj_list)
print("Start to read license...")
index_content = "{\n" + INDENT + "\"license_index\": [\n"
for obj in license_obj_list:
index_item_str = LicenseIndexItemStr.getItemStr(obj)
index_content += index_item_str
license_fp = open(os.path.join(lic_path, obj.getLicenseName()),
'w')
license_fp.write(obj.getLicenseContent())
license_fp.close()
index_content = index_content[0:-2]
index_content += '\n' + INDENT + "]\n}"
print("Start to write license to text...")
index_fp = open(os.path.join(lic_path, 'license_index.json'), 'w')
index_fp.write(index_content)
except Exception as e:
logger.error(str(e))
def createLicenseIndex(*args):
outdir = args[0]
ext_path = args[1]
#if product.lower() == "modeler":
#repos_set_uri = RAW_REPOS_SET_URI.format('modeler','modeler')
#index_key = RAW_INDEX_KEY.format('modeler')
#elif product.lower() == 'stats':
# wrong spell of statistics
#repos_set_uri = RAW_REPOS_SET_URI.format('statisitcs','stats')
#index_key = RAW_INDEX_KEY.format('stats')
index_key = RAW_INDEX_KEY
licenseLogger = None
try:
license_obj_list = []
lic_path = os.path.join(outdir,LICENSE_DIR)
os.mkdir(lic_path)
root_log_dir = os.path.join(outdir, LOG_DIR_NAME)
licenseLogger = Logger(os.path.join(root_log_dir,LOG_INFO),'licenseLogger')
licenseLogger.info("CreateLicenseIndex script start ...")
try:
licenseLogger.info("Get extension list ...")
repos_set_json = loadExtJSONContent(ext_path)
repos_set_json_index = repos_set_json[index_key]
except Exception as e:
raise e
repo_index = 0
licenseLogger.info("Start to get license content ...")
for repo in repos_set_json_index:
try:
repo_name = repo["repository"]
except Exception:
raise Exception("At least one repository in index file does not have repo name. Please check!")
repo_license_uri = RAW_LICENSE_URI.format(repo_name)
try:
repo_license_content = URILoader.loadURI(repo_license_uri, "license file")
except Exception as e:
raise e
isExistedLicense = False
repo_index += 1
for item in license_obj_list:
if repo_license_content == item.getLicenseContent():
isExistedLicense = True
item.addRepoName(repo_name)
break
if not isExistedLicense:
addObj(repo_name, repo_license_content,license_obj_list)
lic_index = 0
index_content = "{\n"+INDENT+"\"license_index\": [\n";
for obj in license_obj_list:
index_item_str = LicenseIndexItemStr.getItemStr(obj)
index_content += index_item_str
lic_index += 1
licenseLogger.info(str(lic_index)+" license: save in file '"+obj.getLicenseName()+"'.")
licenseLogger.info("Repos use this license: "+LicenseIndexItemStr.convertListToString(obj.getRepoNameList()))
license_fp = open(os.path.join(lic_path,obj.getLicenseName()),'w',encoding = "utf-8")
license_fp.write(obj.getLicenseContent())
license_fp.close()
index_content = index_content[0:-2]
index_content += '\n' + INDENT + "]\n}"
index_fp = open(os.path.join(lic_path,LIC_INDEX_FILE),'w',encoding='utf-8')
index_fp.write(index_content)
licenseLogger.info("CreateLicenseIndex action succeeded!")
except Exception as e:
if licenseLogger!=None:
licenseLogger.error(str(e),e)
licenseLogger.info("CreateLicenseIndex action failed!")
raise e
finally:
if licenseLogger!=None:
licenseLogger.info("Totally get "+str(len(license_obj_list))+" type(s) of license from "+str(repo_index)+" repos!")
licenseLogger.close()
def createLangIndex(*args):
outdir = args[0]
ext_path = args[1]
# if product == "modeler":
# index_key = RAW_INDEX_KEY.format('modeler')
# elif product == 'stats':
# index_key = RAW_INDEX_KEY.format('stats')
index_key = RAW_INDEX_KEY
langLogger = None
try:
lang_path = os.path.join(outdir, LANG_DIR)
os.mkdir(lang_path)
root_log_dir = os.path.join(outdir, LOG_DIR_NAME)
langLogger = Logger(os.path.join(root_log_dir, LOG_INFO), "langLogger")
langLogger.info("CreateLicenseIndex script start ...")
langLogger.info("Get extension index ...")
repos_set_json = loadExtJSONContent(ext_path)
repos_set_json_index = repos_set_json[index_key]
for lang_item in LANG_LIST:
fp_content = LANG_INDEX_PRE
langLogger.info("Start to get '" + lang_item + "' file")
i = 0
for repo in repos_set_json_index:
try:
repo_name = repo["repository"]
i += 1
langLogger.info(lang_item + ": " + str(i) + " repo " + repo_name)
except Exception:
raise Exception("At least one repository in index file does not have repo name. Please check!")
repo_lang_uri = RAW_REPO_LANG_URI.format(repo_name, lang_item)
try:
repo_lang_content = URILoader.loadURI(repo_lang_uri, "language index file")
lang_json_str = LangPropObj.convertToJSONStr(repo_name, repo_lang_content, lang_item)
except Exception as e:
if "HTTPError" in str(e):
# some repositories do not have lang file, by default use en file in index for extension file
lang_json_str = LangPropObj.generateJSONStr(
repo_name, repo[EXT_KEY]["Summary"], repo[EXT_KEY]["Description"]
)
else:
raise e
fp_content += lang_json_str
try:
fp_content = fp_content[0:-2] + "\n" + INDENT + "]\n}"
fp = open(os.path.join(lang_path, lang_item + ".json"), "w", encoding="utf-8")
fp.write(fp_content)
fp.close()
except Exception as e:
raise e
except Exception as e:
if langLogger != None:
langLogger.error(str(e), e)
langLogger.info("CreatelangIndex action failed!")
raise e
finally:
if langLogger != None:
langLogger.close()
def CreateIndex(*args):
print("size"+str(len(args)))
outdir = args[0]
product = args[1]
if product == "modeler":
repos_set_uri = RAW_REPOS_SET_URI.format('modeler','modeler')
index_key = RAW_INDEX_KEY.format('modeler')
elif product == 'stats':
# wrong spell of statistics
repos_set_uri = RAW_REPOS_SET_URI.format('statisitcs','stats')
index_key = RAW_INDEX_KEY.format('stats')
try:
lic_path = os.path.join(outdir,LICENSE_DIR)
os.mkdir(lic_path)
logger = Logger(os.path.join(lic_path,LOG_INFO))
logger.info("Script start ...")
except IOError as e:
raise IOError("IOError: Need permission to write in "+outdir)
try:
try:
repos_set_json = json.loads(URILoader.loadURI(repos_set_uri, "index file"))
except ValueError as e:
raise Exception("ValueError: The {0} has an illegal format. Please check!\n\n".format("index file"))
except Exception as e:
raise e
try:
repos_set_json_index = repos_set_json[index_key]
except Exception as e:
raise e
license_obj_list = []
for repo in repos_set_json_index:
try:
repo_name = repo["repository"]
except Exception:
raise Exception("At least one repository in index file does not have repo name. Please check!\n\n")
repo_license_uri = RAW_LICENSE_URI.format(repo_name)
try:
repo_license_content = URILoader.loadURI(repo_license_uri, "license file")
except Exception as e:
raise e
isExistedLicense = False
for item in license_obj_list:
if repo_license_content == item.getLicenseContent():
isExistedLicense = True
item.addRepoName(repo_name)
break
if not isExistedLicense:
addObj(repo_name, repo_license_content,license_obj_list)
print("Start to read license...")
index_content = "{\n"+INDENT+"\"license_index\": [\n";
for obj in license_obj_list:
index_item_str = LicenseIndexItemStr.getItemStr(obj)
index_content += index_item_str
license_fp = open(os.path.join(lic_path,obj.getLicenseName()),'w')
license_fp.write(obj.getLicenseContent())
license_fp.close()
index_content = index_content[0:-2]
index_content += '\n' + INDENT + "]\n}"
print("Start to write license to text...")
index_fp = open(os.path.join(lic_path,'license_index.json'),'w')
index_fp.write(index_content)
except Exception as e:
logger.error(str(e))