Exemple #1
0
else:
    import USCCrawler2 as USCC
    List = USCC.USCCrawl()

#Either launch the USC cleaner or load a checkpoint file if one exists.

if os.path.exists(Setup.jsonFile):
    #Load a dataframe from a json file.
    import Setup
    jsonFile = Setup.jsonFile
    uscDF = pd.read_json(jsonFile, orient='index')
else:

    import Cleaner
    #Run Cleaning script to generate dictionary
    Dictionary = Cleaner.Clean(List)
    #Convert dictionary into pandas dataframe.
    uscDF = pd.DataFrame.from_dict(Dictionary, orient='index')
    print("Cleaned Data")

#Either launch the UCLA Crawler and cleaner or load a checkpoint file if one exists.

if os.path.exists(Setup.UCLAClean):
    import json
    jsonFile = Setup.UCLAClean
    with open(jsonFile, 'r') as inFile:
        UCLADict = json.load(inFile)
else:
    import UCLACleaner
    import UCLAScraper