def __init__(self, repo_name):       
     repo_info_json_url = re.sub('repos_name', repo_name, InfoJSONObj.RAW_INFO_JSON_URL)      
     
     try:
         self.repo_info_json = json.loads(URILoader.loadURI(repo_info_json_url, repo_name+"'s info.json file", "Switch to next repo."))
     except ValueError:
         raise Exception("ValueError: "+repo_name+"'s info.json has an illegal format. Please check!"+"Switch to next repo.") 
     except Exception as e:
         raise e    
     
     self.item_list = []
     for key in InfoJSONObj.KEY_LIST:
         try:
             if type(self.repo_info_json[key]) == list:
                 val = self.repo_info_json[key][0]
             else:
                 val = self.repo_info_json[key]
             self.item_list.append(JSONObj(key,val.strip()))
         except:
             raise ValueError("info.json missed some of the items below:\n"
                              "type, provider, software, language, category, promotion.")  
def createLangIndex(*args):
    outdir = args[0]
    ext_path = args[1]

    index_key = RAW_INDEX_KEY
    langLogger = None
    try:
        lang_path = os.path.join(outdir,LANG_DIR) 
        os.mkdir(lang_path)
        root_log_dir = os.path.join(outdir, LOG_DIR_NAME)  
        langLogger = Logger(os.path.join(root_log_dir,LOG_INFO),'langLogger')
        langLogger.info("CreateLicenseIndex script start ...")    
        
        langLogger.info("Get extension index  ...")
        repos_set_json = loadExtJSONContent(ext_path)
        repos_set_json_index = repos_set_json[index_key]   
        
        for lang_item in LANG_LIST:
            fp_content = LANG_INDEX_PRE
            langLogger.info("Start to get '"+lang_item+"' file")
            i=0
            for repo in repos_set_json_index:
                try:
                    repo_name = repo["repository"]
                    i+=1
                except Exception:
                    raise Exception("At least one repository in index file does not have repo name. Please check!")    
                
                repo_lang_uri = RAW_REPO_LANG_URI.format(repo_name, lang_item)
 
                try:
                    repo_lang_content = URILoader.loadURI(repo_lang_uri, "language index file")
                    lang_json_str = LangPropObj.convertToJSONStr(repo_name, repo_lang_content, lang_item)
                except Exception as e:
                    if 'HTTPError' in str(e):
                        # some repositories do not have lang file, by default use en file in index for extension file
                        summary = ""
                        if 'Summary' in repo[EXT_KEY].keys():
                            summary = repo[EXT_KEY]['Summary']
                        desc = ""    
                        if 'Description' in repo[EXT_KEY].keys():
                            desc = repo[EXT_KEY]['Description']
                        lang_json_str = LangPropObj.generateJSONStr(repo_name,summary,desc)
                    else:
                        raise e

                fp_content += lang_json_str
                
            try:                
                fp_content = fp_content[0:-2]+'\n'+INDENT+']\n}'
                fp = open(os.path.join(lang_path, lang_item+'.json'), 'w', encoding='utf-8') 
                fp.write(fp_content)
                fp.close()
            except Exception as e:
                raise e
                
    except Exception as e:
        if langLogger!=None:
            langLogger.error(str(e),e)
            langLogger.info("CreatelangIndex action failed!")
        raise e
    finally:
        if langLogger!=None:
            langLogger.close()
예제 #3
0
    def CreateIndex(*args):
        print("size" + str(len(args)))
        outdir = args[0]
        product = args[1]

        if product == "modeler":
            repos_set_uri = RAW_REPOS_SET_URI.format('modeler', 'modeler')
            index_key = RAW_INDEX_KEY.format('modeler')
        elif product == 'stats':
            # wrong spell of statistics
            repos_set_uri = RAW_REPOS_SET_URI.format('statisitcs', 'stats')
            index_key = RAW_INDEX_KEY.format('stats')

        try:
            lic_path = os.path.join(outdir, LICENSE_DIR)
            os.mkdir(lic_path)
            logger = Logger(os.path.join(lic_path, LOG_INFO))
            logger.info("Script start ...")
        except IOError as e:
            raise IOError("IOError: Need permission to write in " + outdir)

        try:
            try:
                repos_set_json = json.loads(
                    URILoader.loadURI(repos_set_uri, "index file"))
            except ValueError as e:
                raise Exception(
                    "ValueError: The {0} has an illegal format. Please check!\n\n"
                    .format("index file"))
            except Exception as e:
                raise e

            try:
                repos_set_json_index = repos_set_json[index_key]
            except Exception as e:
                raise e
            license_obj_list = []
            for repo in repos_set_json_index:
                try:
                    repo_name = repo["repository"]
                except Exception:
                    raise Exception(
                        "At least one repository in index file does not have repo name. Please check!\n\n"
                    )

                repo_license_uri = RAW_LICENSE_URI.format(repo_name)

                try:
                    repo_license_content = URILoader.loadURI(
                        repo_license_uri, "license file")
                except Exception as e:
                    raise e

                isExistedLicense = False
                for item in license_obj_list:
                    if repo_license_content == item.getLicenseContent():
                        isExistedLicense = True
                        item.addRepoName(repo_name)
                        break
                if not isExistedLicense:
                    addObj(repo_name, repo_license_content, license_obj_list)

            print("Start to read license...")
            index_content = "{\n" + INDENT + "\"license_index\": [\n"

            for obj in license_obj_list:
                index_item_str = LicenseIndexItemStr.getItemStr(obj)
                index_content += index_item_str
                license_fp = open(os.path.join(lic_path, obj.getLicenseName()),
                                  'w')
                license_fp.write(obj.getLicenseContent())
                license_fp.close()

            index_content = index_content[0:-2]
            index_content += '\n' + INDENT + "]\n}"
            print("Start to write license to text...")
            index_fp = open(os.path.join(lic_path, 'license_index.json'), 'w')
            index_fp.write(index_content)
        except Exception as e:
            logger.error(str(e))
def createLicenseIndex(*args):
    outdir = args[0]
    ext_path = args[1]        
    
    #if product.lower() == "modeler":
        #repos_set_uri = RAW_REPOS_SET_URI.format('modeler','modeler')
        #index_key = RAW_INDEX_KEY.format('modeler')
    #elif product.lower() == 'stats':
        # wrong spell of statistics
        #repos_set_uri = RAW_REPOS_SET_URI.format('statisitcs','stats') 
        #index_key = RAW_INDEX_KEY.format('stats')
    index_key = RAW_INDEX_KEY
    licenseLogger = None
    try:   
        license_obj_list = []        
        lic_path = os.path.join(outdir,LICENSE_DIR)   
        os.mkdir(lic_path)
        root_log_dir = os.path.join(outdir, LOG_DIR_NAME)  
        licenseLogger = Logger(os.path.join(root_log_dir,LOG_INFO),'licenseLogger')
        licenseLogger.info("CreateLicenseIndex script start ...")
        
        try:     
            licenseLogger.info("Get extension list  ...")
            repos_set_json = loadExtJSONContent(ext_path)
            repos_set_json_index = repos_set_json[index_key]
        except Exception as e:
            raise e
        
        repo_index = 0
        licenseLogger.info("Start to get license content ...")
        for repo in repos_set_json_index:
            try:
                repo_name = repo["repository"]
            except Exception:
                raise Exception("At least one repository in index file does not have repo name. Please check!")    
            
            repo_license_uri = RAW_LICENSE_URI.format(repo_name)
            
            try:     
                repo_license_content = URILoader.loadURI(repo_license_uri, "license file")
            except Exception as e:
                raise e
    
            isExistedLicense = False
            repo_index += 1
            for item in license_obj_list:
                if repo_license_content == item.getLicenseContent():
                    isExistedLicense = True
                    item.addRepoName(repo_name)
                    break   
            if not isExistedLicense:
                addObj(repo_name, repo_license_content,license_obj_list)
        
        lic_index = 0
        index_content = "{\n"+INDENT+"\"license_index\": [\n";           
        for obj in license_obj_list:
            index_item_str = LicenseIndexItemStr.getItemStr(obj)
            index_content += index_item_str
            lic_index += 1
            licenseLogger.info(str(lic_index)+" license: save in file '"+obj.getLicenseName()+"'.")
            licenseLogger.info("Repos use this license: "+LicenseIndexItemStr.convertListToString(obj.getRepoNameList()))
            
            license_fp = open(os.path.join(lic_path,obj.getLicenseName()),'w',encoding = "utf-8")
            license_fp.write(obj.getLicenseContent())
            license_fp.close()

        index_content = index_content[0:-2]
        index_content += '\n' + INDENT + "]\n}"
        index_fp = open(os.path.join(lic_path,LIC_INDEX_FILE),'w',encoding='utf-8')
        index_fp.write(index_content)
        licenseLogger.info("CreateLicenseIndex action succeeded!")
    except Exception as e:
        if licenseLogger!=None:
            licenseLogger.error(str(e),e)
            licenseLogger.info("CreateLicenseIndex action failed!")
        raise e
    finally:
        if licenseLogger!=None:
            licenseLogger.info("Totally get "+str(len(license_obj_list))+" type(s) of license from "+str(repo_index)+" repos!")
            licenseLogger.close()
예제 #5
0
def createLangIndex(*args):
    outdir = args[0]
    ext_path = args[1]

    # if product == "modeler":
    #    index_key = RAW_INDEX_KEY.format('modeler')
    # elif product == 'stats':
    #    index_key = RAW_INDEX_KEY.format('stats')
    index_key = RAW_INDEX_KEY
    langLogger = None
    try:
        lang_path = os.path.join(outdir, LANG_DIR)
        os.mkdir(lang_path)
        root_log_dir = os.path.join(outdir, LOG_DIR_NAME)
        langLogger = Logger(os.path.join(root_log_dir, LOG_INFO), "langLogger")
        langLogger.info("CreateLicenseIndex script start ...")

        langLogger.info("Get extension index  ...")
        repos_set_json = loadExtJSONContent(ext_path)
        repos_set_json_index = repos_set_json[index_key]

        for lang_item in LANG_LIST:
            fp_content = LANG_INDEX_PRE
            langLogger.info("Start to get '" + lang_item + "' file")
            i = 0
            for repo in repos_set_json_index:
                try:
                    repo_name = repo["repository"]
                    i += 1
                    langLogger.info(lang_item + ": " + str(i) + " repo " + repo_name)
                except Exception:
                    raise Exception("At least one repository in index file does not have repo name. Please check!")

                repo_lang_uri = RAW_REPO_LANG_URI.format(repo_name, lang_item)

                try:
                    repo_lang_content = URILoader.loadURI(repo_lang_uri, "language index file")
                    lang_json_str = LangPropObj.convertToJSONStr(repo_name, repo_lang_content, lang_item)
                except Exception as e:
                    if "HTTPError" in str(e):
                        # some repositories do not have lang file, by default use en file in index for extension file
                        lang_json_str = LangPropObj.generateJSONStr(
                            repo_name, repo[EXT_KEY]["Summary"], repo[EXT_KEY]["Description"]
                        )
                    else:
                        raise e

                fp_content += lang_json_str

            try:
                fp_content = fp_content[0:-2] + "\n" + INDENT + "]\n}"
                fp = open(os.path.join(lang_path, lang_item + ".json"), "w", encoding="utf-8")
                fp.write(fp_content)
                fp.close()
            except Exception as e:
                raise e

    except Exception as e:
        if langLogger != None:
            langLogger.error(str(e), e)
            langLogger.info("CreatelangIndex action failed!")
        raise e
    finally:
        if langLogger != None:
            langLogger.close()
예제 #6
0
    def CreateIndex(*args):
        print("size"+str(len(args)))
        outdir = args[0]
        product = args[1]
        
        if product == "modeler":
            repos_set_uri = RAW_REPOS_SET_URI.format('modeler','modeler')
            index_key = RAW_INDEX_KEY.format('modeler')
        elif product == 'stats':
            # wrong spell of statistics
            repos_set_uri = RAW_REPOS_SET_URI.format('statisitcs','stats') 
            index_key = RAW_INDEX_KEY.format('stats')
        
        try:
            lic_path = os.path.join(outdir,LICENSE_DIR)   
            os.mkdir(lic_path)
            logger = Logger(os.path.join(lic_path,LOG_INFO))
            logger.info("Script start ...")
        except IOError as e:  
            raise IOError("IOError: Need permission to write in "+outdir)
            
        try:   
            try:     
                repos_set_json = json.loads(URILoader.loadURI(repos_set_uri, "index file"))
            except ValueError as e:
                raise Exception("ValueError: The {0} has an illegal format. Please check!\n\n".format("index file"))
            except Exception as e:
                raise e
            
            try:
                repos_set_json_index = repos_set_json[index_key]
            except Exception as e:
                raise e
            license_obj_list = []
            for repo in repos_set_json_index:
                try:
                    repo_name = repo["repository"]
                except Exception:
                    raise Exception("At least one repository in index file does not have repo name. Please check!\n\n")    
                
                repo_license_uri = RAW_LICENSE_URI.format(repo_name)
                
                try:     
                    repo_license_content = URILoader.loadURI(repo_license_uri, "license file")
                except Exception as e:
                    raise e
        
                isExistedLicense = False
                for item in license_obj_list:
                    if repo_license_content == item.getLicenseContent():
                        isExistedLicense = True
                        item.addRepoName(repo_name)
                        break   
                if not isExistedLicense:
                    addObj(repo_name, repo_license_content,license_obj_list)
            
            print("Start to read license...") 
            index_content = "{\n"+INDENT+"\"license_index\": [\n";
            
            for obj in license_obj_list:
                index_item_str = LicenseIndexItemStr.getItemStr(obj)
                index_content += index_item_str
                license_fp = open(os.path.join(lic_path,obj.getLicenseName()),'w')
                license_fp.write(obj.getLicenseContent())
                license_fp.close()

            index_content = index_content[0:-2]
            index_content += '\n' + INDENT + "]\n}"
            print("Start to write license to text...") 
            index_fp = open(os.path.join(lic_path,'license_index.json'),'w')
            index_fp.write(index_content)
        except Exception as e:
            logger.error(str(e))