Пример #1
0
class ResearcherHH:
    """Main class for searching vacancies and analyze them."""

    def __init__(self, config_path: str = SETTINGS_PATH, no_parse: bool = False):
        self.settings = Settings(config_path, no_parse=no_parse)
        self.exchanger = Exchanger(config_path)
        self.collector: Optional[DataCollector] = None
        self.analyzer: Optional[Analyzer] = None
        self.predictor = Predictor()

    def update(self, **kwargs):
        self.settings.update_params(**kwargs)
        if not any(self.settings.rates.values()) or self.settings.update:
            print("[INFO]: Trying to get exchange rates from remote server...")
            self.exchanger.update_exchange_rates(self.settings.rates)
            self.exchanger.save_rates(self.settings.rates)

        print(f"[INFO]: Get exchange rates: {self.settings.rates}")
        self.collector = DataCollector(self.settings.rates)
        self.analyzer = Analyzer(self.settings.save_result)

    def __call__(self):
        print("[INFO]: Collect data from JSON. Create list of vacancies...")
        vacancies = self.collector.collect_vacancies(
            query=self.settings.options, refresh=self.settings.refresh, max_workers=self.settings.max_workers
        )
        print("[INFO]: Prepare dataframe...")
        df = self.analyzer.prepare_df(vacancies)
        print("\n[INFO]: Analyze dataframe...")
        self.analyzer.analyze_df(df)
        print("\n[INFO]: Predict None salaries...")
        # total_df = self.predictor.predict(df)
        # self.predictor.plot_results(total_df)
        print("[INFO]: Done! Exit()")
Пример #2
0
    def update(self, **kwargs):
        self.settings.update_params(**kwargs)
        if not any(self.settings.rates.values()) or self.settings.update:
            print("[INFO]: Trying to get exchange rates from remote server...")
            self.exchanger.update_exchange_rates(self.settings.rates)
            self.exchanger.save_rates(self.settings.rates)

        print(f"[INFO]: Get exchange rates: {self.settings.rates}")
        self.collector = DataCollector(self.settings.rates)
        self.analyzer = Analyzer(self.settings.save_result)
Пример #3
0
def test_init() -> None:
    analyzer = Analyzer("test")
    infos = analyzer.get()
    assert infos.get("name") == "test"
    assert infos.get("magic_number") is None
    assert infos.get("format") is None
    assert infos.get("bits") is None
    assert infos.get("endianness") is None
    assert infos.get("size") is None
    assert infos.get("content") is None
Пример #4
0
def run():
    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument(
        'repo',
        type=str,
        help='GitHub public repo',
    )
    arg_parser.add_argument(
        '-df',
        '--date_from',
        type=valid_date,
        help='Analyze from date - YYYY-MM-DD',
    )
    arg_parser.add_argument(
        '-dt',
        '--date_to',
        type=valid_date,
        help='Analyze to date - YYYY-MM-DD',
    )
    arg_parser.add_argument(
        '-b',
        '--branch',
        type=str,
        default='master',
        help='Branch name to analyze. Defaults to master',
    )

    args = arg_parser.parse_args()

    repo = args.repo
    date_from = args.date_from
    date_to = args.date_to
    branch = args.branch

    analyzer = Analyzer(
        repo=repo,
        date_from=date_from,
        date_to=date_to,
        branch=branch,
    )

    result = {}
    try:
        result = analyzer.get_stats()
    except RequestException as e:
        log.error(f'Error getting stats: {e}')

    log.info(json.dumps(result, indent=2))
Пример #5
0
def test_init_noParameters():
    exampleFile = "./data/example/testUser/StreamingHistory.json"
    analyzer = Analyzer()

    numberOfItemsStreamed = 29632

    assert exampleFile == analyzer.libraryFiles[0]
    assert numberOfItemsStreamed == len(analyzer.library)
Пример #6
0
class TestAnalyzerProcessText(unittest.TestCase):
    """Test for method process_text.

    Special functions:
        setUpClass -> Executed once bafore all test cases.
        setUp -> Executed once before each test function.
        test* -> Where the test should be.
        tearDown -> Executed once after each test function.
        tearDownClass -> Executed once after all test cases
    """
    @timeout_decorator.timeout(30)
    def setUp(self) -> None:
        """Set up variables for testing."""
        self._analyzer = Analyzer("", "")

    @timeout_decorator.timeout(30)
    def test_1(self) -> None:
        """Test 1."""
        self._analyzer.process_text("Hola Hola que tal")
Пример #7
0
class TestAnalyzerInit(unittest.TestCase):
    """Test for method __init__.

    Special functions:
        setUpClass -> Executed once bafore all test cases.
        setUp -> Executed once before each test function.
        test* -> Where the test should be.
        tearDown -> Executed once after each test function.
        tearDownClass -> Executed once after all test cases
    """
    @timeout_decorator.timeout(30)
    def setUp(self) -> None:
        """Set up variables for testing."""
        self._analyzer = Analyzer("test", "test")

    @timeout_decorator.timeout(30)
    def test_1(self) -> None:
        """Test 1."""
        self._analyzer.__init__("test", "test")
Пример #8
0
class TestAnalyzerParsePhotos(unittest.TestCase):
    """Test for method _parse_photos.

    Special functions:
        setUpClass -> Executed once bafore all test cases.
        setUp -> Executed once before each test function.
        test* -> Where the test should be.
        tearDown -> Executed once after each test function.
        tearDownClass -> Executed once after all test cases
    """
    @timeout_decorator.timeout(30)
    def setUp(self) -> None:
        """Set up variables for testing."""
        self._analyzer = Analyzer("test", '<a src="Test" /a>')

    @timeout_decorator.timeout(30)
    def test_1(self) -> None:
        """Test 1."""
        self._analyzer._parse_photos()
        self._analyzer.route["photos"] = ["Test"]
Пример #9
0
def test_init_multipleFilesSpecified():
    test_file = "./data/example/testUser1/StreamingHistory.json"
    test_file2 = "./data/example/testUser2/StreamingHistory.json"
    streamingHistoryFiles = [test_file, test_file2]
    analyzer = Analyzer(streamingHistoryFiles=streamingHistoryFiles)

    numberOfItemsStreamed = 8

    assert test_file == analyzer.libraryFiles[0]
    assert test_file2 == analyzer.libraryFiles[1]
    assert numberOfItemsStreamed == len(analyzer.library)
Пример #10
0
class TestAnalyzerParseGrades(unittest.TestCase):
    """Test for method _parse_grades.

    Special functions:
        setUpClass -> Executed once bafore all test cases.
        setUp -> Executed once before each test function.
        test* -> Where the test should be.
        tearDown -> Executed once after each test function.
        tearDownClass -> Executed once after all test cases
    """
    @timeout_decorator.timeout(30)
    def setUp(self) -> None:
        """Set up variables for testing."""
        self._analyzer = Analyzer("test", "<title> Una ruta 6a+</title>")

    @timeout_decorator.timeout(30)
    def test_1(self) -> None:
        """Test 1. Grade in title"""
        self._analyzer._parse_grades()
        self.assertEqual(self._analyzer.route["grade"]["climbing"], 615)
Пример #11
0
class TestAnalyzerParseName(unittest.TestCase):
    """Test for method _parse_name.

    Special functions:
        setUpClass -> Executed once bafore all test cases.
        setUp -> Executed once before each test function.
        test* -> Where the test should be.
        tearDown -> Executed once after each test function.
        tearDownClass -> Executed once after all test cases
    """

    @timeout_decorator.timeout(30)
    def setUp(self) -> None:
        """Set up variables for testing."""
        self._analyzer = Analyzer("test", "<title>Test</title>")

    @timeout_decorator.timeout(30)
    def test_1(self) -> None:
        """Test 1."""
        self._analyzer._parse_name()
        self.assertEqual(self._analyzer.route["name"], "Test")
Пример #12
0
class TestAnalyzerAnalyze(unittest.TestCase):
    """Test for method analyze.

    Special functions:
        setUpClass -> Executed once bafore all test cases.
        setUp -> Executed once before each test function.
        test* -> Where the test should be.
        tearDown -> Executed once after each test function.
        tearDownClass -> Executed once after all test cases
    """
    @timeout_decorator.timeout(30)
    def setUp(self) -> None:
        """Set up variables for testing."""
        self._analyzer = Analyzer("", "<title>Hola 6a+</title>")

    @timeout_decorator.timeout(30)
    def test_1(self) -> None:
        """Test 1."""
        self._analyzer.analyze()
        print(self._analyzer.route)
        self._analyzer.route["name"] = "Hola 6a+"
        self._analyzer.route["grade"]["climbing"] = 615
Пример #13
0
 def setUp(self) -> None:
     """Set up variables for testing."""
     self._analyzer = Analyzer("test", "test")
Пример #14
0
        # for log
        log_text += "\n\n     - DATA FILE : {0} at {1}".format(
            in_file_directory + in_file_name, str(start)) + '\n'

        ########################
        # Read The Config File #
        ########################

        print("Step 0. CONFIG VERIFICATION\n")

        # initialization - generating an Analyzer with all info to generate file handlers
        _Analyzer = Analyzer(in_file_directory, in_file_name, in_extension,
                             in_delimiter, in_file_missing_value,
                             in_file_quote, out_file_directory, out_file_name,
                             out_extension, out_delimiter,
                             out_file_missing_value, out_file_single_file,
                             out_file_separate_line, raw_data_structure,
                             clean_data_structure)

        # verification of admitted formats (in, out and data)
        if not _Analyzer.check_valid():
            stop_system(_Analyzer.error_message)

        ##########################
        # Structure the raw data #
        ##########################

        print("Step 1. STRUCTURING THE RAW DATA\n")

        _Analyzer.structure_raw_data()
Пример #15
0
 def setUp(self) -> None:
     """Set up variables for testing."""
     self._analyzer = Analyzer("test", "<title> Una ruta 6a+</title>")
Пример #16
0
    elif "generatortest.py" in argv[1]:
        FILE_PATH = argv[2]
    else:
        error_msg = ("Unexpected call from the command line: {}")
        raise SyntaxError(error_msg.format(" ".join(argv)))
else:
    error_msg = ("Please pass an arg for the path to a flair program.")
    raise SyntaxError(error_msg)

# store program into string variable
with open(FILE_PATH, "r") as flr:
    flairProgram = flr.read()

scanner = Scanner(flairProgram)
parser = Parser(scanner)
ast = parser.parse()
analyzer = Analyzer(ast)
symbolTable = analyzer.getSymbolTable()
generator = Generator(ast, symbolTable)
code = generator.generateCode()
if "/" in FILE_PATH:
    fileName = FILE_PATH[FILE_PATH.rindex("/") + 1:]
else:
    fileName = FILE_PATH
if "." in fileName:
    fileName = fileName[:fileName.rindex(".")]
fileName += ".tm"
with open(fileName, 'w+') as f:
    f.write(code)
print("TM code saved to file {}".format(fileName))
Пример #17
0
    phpsessid = 't4cjka7f4nedrmmq7dubipji16'

    # 日记内容保存路径
    content_file_path = 'data/content.txt'

    # 日记时间保存路径
    time_file_path = 'data/time.txt'

    # 词云图片保存路径
    content_png_path = 'data/content.png'

    # 统计时间柱状图保存路径
    time_png_path = 'data/time.png'

    # 爬取数据
    # url: 某个人的饭否主页
    # phpsessid: 登录饭否后浏览器 Cookie 中的值
    # content_file_path: 日记内容保存路径
    # time_file_path: 日记时间保存路径
    Collector.collect(url, phpsessid, content_file_path, time_file_path)

    # 分词,制作词云
    # content_file_path: 日记内容保存路径
    # content_png_path: 词云图片保存路径
    Analyzer.segment_and_visualize(content_file_path, content_png_path)

    # 统计发日记时间
    # time_file_path: 日记时间保存路径
    # time_png_path: 时间柱状图保存路径
    Analyzer.count_times(time_file_path, time_png_path)
Пример #18
0
from datetime import datetime

from src.analyzer import Analyzer

last_refresh = datetime.now()

if __name__ == '__main__':
    analyzer = Analyzer()
    while True:
        country = input('Country: ')
        state = input('State: ')
        if (datetime.now() - last_refresh).seconds > 60:
            analyzer.provider.refresh()
        try:
            analyzer.plot(country, state)
        except AssertionError:
            print('Not found.')
        except TypeError:
            print('Can\'t plot record.')
Пример #19
0
def analyzer():
    return Analyzer('test')
            print ("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
            print ("     Opening data file : {0}".format(in_file_directory + in_file_name))
            print ("   - Started: " + str(start))
            print ("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n")
            log_text += "\n\n     -DATA FILE : {0} at {1}".format(in_file_directory + in_file_name, str(start))+'\n'

            ########################
            # Generate the network #
            ########################

            print ("Step 0. INITIALIZATION\n")

            # initialization - generating an Analyzer with all info to generate file handlers
            _Analyzer = Analyzer(in_file_directory, in_file_name,
                                  out_file_directory,
                                  is_weighted, is_directed, full_analysis, weight_id, aggregate_number
                                )

            _Analyzer.generate_network()

            ########################
            # Launch the analysis  #
            ########################

            print ("Step 1. ANALYSIS\n")


            _Analyzer.launch_analysis()


            ########################
Пример #21
0
            print ("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
            print ("     Opening data file : {0}".format(in_file_directory + in_file_name))
            print ("   - Started: " + str(start))
            print ("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n")
            log_text += "\n\n     -DATA FILE : {0} at {1}".format(in_file_directory + in_file_name, str(start))+'\n'

            ########################
            # Generate the network #
            ########################

            print ("Step 0. INITIALIZATION\n")

            # initialization - generating an Analyzer with all info to generate file handlers
            _Analyzer = Analyzer(in_file_directory, in_file_name,
                                  out_file_directory,
                                  is_weighted, is_directed, full_analysis, weight_id, aggregate_number
                                )

            _Analyzer.generate_network()

            ########################
            # Launch the analysis  #
            ########################

            print ("Step 1. ANALYSIS\n")


            _Analyzer.launch_analysis()


            ########################
Пример #22
0
            print ("   - Started: " + str(start))
            print ("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n")

            # for log
            log_text += "\n\n     - DATA FILE : {0} at {1}".format(in_file_directory + in_file_name, str(start))+'\n'

            ########################
            # Read The Config File #
            ########################

            print ("Step 0. CONFIG VERIFICATION\n")

            # initialization - generating an Analyzer with all info to generate file handlers
            _Analyzer = Analyzer( in_file_directory, in_file_name, in_extension, in_delimiter,
                                    in_file_missing_value, in_file_quote,
                                    out_file_directory, out_file_name, out_extension, out_delimiter, 
                                    out_file_missing_value, 
                                    out_file_single_file, out_file_separate_line,
                                    raw_data_structure, clean_data_structure )

            # verification of admitted formats (in, out and data)
            if not _Analyzer.check_valid():
                    stop_system(_Analyzer.error_message)

            ##########################
            # Structure the raw data #
            ##########################

            print ("Step 1. STRUCTURING THE RAW DATA\n")

            _Analyzer.structure_raw_data()
            
Пример #23
0
 def setUp(self) -> None:
     """Set up variables for testing."""
     self._analyzer = Analyzer("test", '<a src="Test" /a>')
Пример #24
0
def main() -> None:
    analyzer = Analyzer(["./data/example/testUser/StreamingHistory.json"])
    payload: Mapping[str, Union[str, int]] = {}

    print("Example 1: getPopularArtists()")
    pa = analyzer.getPopularArtists()
    print(pa)

    print()
    print("Example 1a: getPopularArtists() with specific Daytime")
    payload = {"daytime": "morning"}
    pa = analyzer.getPopularArtists(payload=payload)
    print(pa)

    print()
    print("Example 1b: getPopularArtists() of Month July")
    payload = {"month": 7}
    pa = analyzer.getPopularArtists(payload=payload)
    print(pa)

    print()
    print("Example 1c: getPopularArtists() of Month July for podcasts")
    payload = {"month": 7, "media": "podcast"}
    pa = analyzer.getPopularArtists(payload=payload)
    print(pa)

    print()
    print(
        "Example 1d: getPopularArtists() of specified period (2019-03-03 to 2019-03-04) for music, count = 3"
    )
    payload = {
        "startYear": 2019,
        "startMonth": 3,
        "startDay": 3,
        "startHour": 0,
        "endYear": 2019,
        "endMonth": 3,
        "endDay": 4,
        "endHour": 0,
        "count": 3,
        "media": "music",
    }
    pa = analyzer.getPopularArtists(payload=payload)
    print(pa)

    print()
    print("Example 1e: getPopularArtists() for specific weekday (monday)")
    payload = {
        "weekday": "monday",
    }
    pa = analyzer.getPopularArtists(payload=payload)
    print(pa)

    print()
    print("Example 2: getPopularItems()")
    pi = analyzer.getPopularItems()
    print(pi)

    print()
    print("Example 2a: getPopularItems() for podcasts")
    payload = {
        "media": "podcast",
    }
    pi = analyzer.getPopularItems(payload=payload)
    print(pi)

    print()
    print("Example 2b: getPopularItems() for podcasts by time played in ms")
    payload = {
        "media": "podcast",
        "ratingCrit": "time",
    }
    pi = analyzer.getPopularItems(payload=payload)
    print(pi)

    print()
    print("Example 2c: getPopularItems() with keyword 'franz ferdinand'")
    pi = analyzer.getPopularItems(payload={"keyword": "Franz Ferdinand"})
    print(pi)

    print()
    print("Example 2c2: getPopularItems() with keyword 'acoustic'")
    pi = analyzer.getPopularItems(payload={"keyword": "acoustic"})
    print(pi)

    print()
    print("Example 3: getDataPerWeekday()")
    dpw = analyzer.getDataPerWeekday()
    print(dpw)

    print()
    print("Example 3a: getDataPerWeekday() with string as key")
    dpw = analyzer.getDataPerWeekday(weekdayFormat="")
    print(dpw)
Пример #25
0
def analyzer_TestUser():
    test_file = "./data/example/testUser/StreamingHistory.json"
    return Analyzer([test_file])
Пример #26
0
def analyzer_patched(mock_get):
    a = Analyzer('test')
    a._get = mock_get
    return a
Пример #27
0
from sys import argv, path
import os
path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.parser import Parser
from src.scanner import Scanner
from src.analyzer import Analyzer

# pass in arg for path to flair program
if len(argv) > 1:
    if "analyzertest.py" in argv[0]:
        FILE_PATH = argv[1]
    elif "analyzertest.py" in argv[1]:
        FILE_PATH = argv[2]
    else:
        error_msg = ("Unexpected call from the command line: {}")
        raise SyntaxError(error_msg.format(" ".join(argv)))
else:
    error_msg = ("Please pass an arg for the path to a flair program.")
    raise SyntaxError(error_msg)

# store program into string variable
with open(FILE_PATH, "r") as flr:
    flairProgram = flr.read()

scanner = Scanner(flairProgram)
parser = Parser(scanner)
analyzer = Analyzer(parser.parse())
symbolTable = analyzer.getSymbolTable()
print("Symbol Table:\n" + str(symbolTable))
Пример #28
0
def main():
    analyzer = Analyzer()
    analyzer.draw_rtt_graph(DatType.Normal)
    analyzer.write_info_csv()
    print("Complete!")
Пример #29
0
def main():
    #xmldb = XMLDatabase("https://bugzilla.gnome.org/xmlrpc.cgi", "gnome")
    mongodb = MongoDatabase("https://bugzilla.gnome.org/xmlrpc.cgi", "gnome")
    bugdb = BugzillaDB(mongodb)
    man = MongoAnalyzer()
    an = Analyzer(bugdb, man)