class ResearcherHH:
"""Main class for searching vacancies and analyze them."""
def __init__(self, config_path: str = SETTINGS_PATH, no_parse: bool = False):
self.settings = Settings(config_path, no_parse=no_parse)
self.exchanger = Exchanger(config_path)
self.collector: Optional[DataCollector] = None
self.analyzer: Optional[Analyzer] = None
self.predictor = Predictor()
def update(self, **kwargs):
self.settings.update_params(**kwargs)
if not any(self.settings.rates.values()) or self.settings.update:
print("[INFO]: Trying to get exchange rates from remote server...")
self.exchanger.update_exchange_rates(self.settings.rates)
self.exchanger.save_rates(self.settings.rates)
print(f"[INFO]: Get exchange rates: {self.settings.rates}")
self.collector = DataCollector(self.settings.rates)
self.analyzer = Analyzer(self.settings.save_result)
def __call__(self):
print("[INFO]: Collect data from JSON. Create list of vacancies...")
vacancies = self.collector.collect_vacancies(
query=self.settings.options, refresh=self.settings.refresh, max_workers=self.settings.max_workers
)
print("[INFO]: Prepare dataframe...")
df = self.analyzer.prepare_df(vacancies)
print("\n[INFO]: Analyze dataframe...")
self.analyzer.analyze_df(df)
print("\n[INFO]: Predict None salaries...")
# total_df = self.predictor.predict(df)
# self.predictor.plot_results(total_df)
print("[INFO]: Done! Exit()")
def update(self, **kwargs):
self.settings.update_params(**kwargs)
if not any(self.settings.rates.values()) or self.settings.update:
print("[INFO]: Trying to get exchange rates from remote server...")
self.exchanger.update_exchange_rates(self.settings.rates)
self.exchanger.save_rates(self.settings.rates)
print(f"[INFO]: Get exchange rates: {self.settings.rates}")
self.collector = DataCollector(self.settings.rates)
self.analyzer = Analyzer(self.settings.save_result)
def test_init() -> None:
analyzer = Analyzer("test")
infos = analyzer.get()
assert infos.get("name") == "test"
assert infos.get("magic_number") is None
assert infos.get("format") is None
assert infos.get("bits") is None
assert infos.get("endianness") is None
assert infos.get("size") is None
assert infos.get("content") is None
def run():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument(
'repo',
type=str,
help='GitHub public repo',
)
arg_parser.add_argument(
'-df',
'--date_from',
type=valid_date,
help='Analyze from date - YYYY-MM-DD',
)
arg_parser.add_argument(
'-dt',
'--date_to',
type=valid_date,
help='Analyze to date - YYYY-MM-DD',
)
arg_parser.add_argument(
'-b',
'--branch',
type=str,
default='master',
help='Branch name to analyze. Defaults to master',
)
args = arg_parser.parse_args()
repo = args.repo
date_from = args.date_from
date_to = args.date_to
branch = args.branch
analyzer = Analyzer(
repo=repo,
date_from=date_from,
date_to=date_to,
branch=branch,
)
result = {}
try:
result = analyzer.get_stats()
except RequestException as e:
log.error(f'Error getting stats: {e}')
log.info(json.dumps(result, indent=2))
def test_init_noParameters():
exampleFile = "./data/example/testUser/StreamingHistory.json"
analyzer = Analyzer()
numberOfItemsStreamed = 29632
assert exampleFile == analyzer.libraryFiles[0]
assert numberOfItemsStreamed == len(analyzer.library)
class TestAnalyzerProcessText(unittest.TestCase):
"""Test for method process_text.
Special functions:
setUpClass -> Executed once bafore all test cases.
setUp -> Executed once before each test function.
test* -> Where the test should be.
tearDown -> Executed once after each test function.
tearDownClass -> Executed once after all test cases
"""
@timeout_decorator.timeout(30)
def setUp(self) -> None:
"""Set up variables for testing."""
self._analyzer = Analyzer("", "")
@timeout_decorator.timeout(30)
def test_1(self) -> None:
"""Test 1."""
self._analyzer.process_text("Hola Hola que tal")
class TestAnalyzerInit(unittest.TestCase):
"""Test for method __init__.
Special functions:
setUpClass -> Executed once bafore all test cases.
setUp -> Executed once before each test function.
test* -> Where the test should be.
tearDown -> Executed once after each test function.
tearDownClass -> Executed once after all test cases
"""
@timeout_decorator.timeout(30)
def setUp(self) -> None:
"""Set up variables for testing."""
self._analyzer = Analyzer("test", "test")
@timeout_decorator.timeout(30)
def test_1(self) -> None:
"""Test 1."""
self._analyzer.__init__("test", "test")
class TestAnalyzerParsePhotos(unittest.TestCase):
"""Test for method _parse_photos.
Special functions:
setUpClass -> Executed once bafore all test cases.
setUp -> Executed once before each test function.
test* -> Where the test should be.
tearDown -> Executed once after each test function.
tearDownClass -> Executed once after all test cases
"""
@timeout_decorator.timeout(30)
def setUp(self) -> None:
"""Set up variables for testing."""
self._analyzer = Analyzer("test", '<a src="Test" /a>')
@timeout_decorator.timeout(30)
def test_1(self) -> None:
"""Test 1."""
self._analyzer._parse_photos()
self._analyzer.route["photos"] = ["Test"]
def test_init_multipleFilesSpecified():
test_file = "./data/example/testUser1/StreamingHistory.json"
test_file2 = "./data/example/testUser2/StreamingHistory.json"
streamingHistoryFiles = [test_file, test_file2]
analyzer = Analyzer(streamingHistoryFiles=streamingHistoryFiles)
numberOfItemsStreamed = 8
assert test_file == analyzer.libraryFiles[0]
assert test_file2 == analyzer.libraryFiles[1]
assert numberOfItemsStreamed == len(analyzer.library)
class TestAnalyzerParseGrades(unittest.TestCase):
"""Test for method _parse_grades.
Special functions:
setUpClass -> Executed once bafore all test cases.
setUp -> Executed once before each test function.
test* -> Where the test should be.
tearDown -> Executed once after each test function.
tearDownClass -> Executed once after all test cases
"""
@timeout_decorator.timeout(30)
def setUp(self) -> None:
"""Set up variables for testing."""
self._analyzer = Analyzer("test", "<title> Una ruta 6a+</title>")
@timeout_decorator.timeout(30)
def test_1(self) -> None:
"""Test 1. Grade in title"""
self._analyzer._parse_grades()
self.assertEqual(self._analyzer.route["grade"]["climbing"], 615)
class TestAnalyzerParseName(unittest.TestCase):
"""Test for method _parse_name.
Special functions:
setUpClass -> Executed once bafore all test cases.
setUp -> Executed once before each test function.
test* -> Where the test should be.
tearDown -> Executed once after each test function.
tearDownClass -> Executed once after all test cases
"""
@timeout_decorator.timeout(30)
def setUp(self) -> None:
"""Set up variables for testing."""
self._analyzer = Analyzer("test", "<title>Test</title>")
@timeout_decorator.timeout(30)
def test_1(self) -> None:
"""Test 1."""
self._analyzer._parse_name()
self.assertEqual(self._analyzer.route["name"], "Test")
class TestAnalyzerAnalyze(unittest.TestCase):
"""Test for method analyze.
Special functions:
setUpClass -> Executed once bafore all test cases.
setUp -> Executed once before each test function.
test* -> Where the test should be.
tearDown -> Executed once after each test function.
tearDownClass -> Executed once after all test cases
"""
@timeout_decorator.timeout(30)
def setUp(self) -> None:
"""Set up variables for testing."""
self._analyzer = Analyzer("", "<title>Hola 6a+</title>")
@timeout_decorator.timeout(30)
def test_1(self) -> None:
"""Test 1."""
self._analyzer.analyze()
print(self._analyzer.route)
self._analyzer.route["name"] = "Hola 6a+"
self._analyzer.route["grade"]["climbing"] = 615
def setUp(self) -> None:
"""Set up variables for testing."""
self._analyzer = Analyzer("test", "test")
# for log
log_text += "\n\n - DATA FILE : {0} at {1}".format(
in_file_directory + in_file_name, str(start)) + '\n'
########################
# Read The Config File #
########################
print("Step 0. CONFIG VERIFICATION\n")
# initialization - generating an Analyzer with all info to generate file handlers
_Analyzer = Analyzer(in_file_directory, in_file_name, in_extension,
in_delimiter, in_file_missing_value,
in_file_quote, out_file_directory, out_file_name,
out_extension, out_delimiter,
out_file_missing_value, out_file_single_file,
out_file_separate_line, raw_data_structure,
clean_data_structure)
# verification of admitted formats (in, out and data)
if not _Analyzer.check_valid():
stop_system(_Analyzer.error_message)
##########################
# Structure the raw data #
##########################
print("Step 1. STRUCTURING THE RAW DATA\n")
_Analyzer.structure_raw_data()
def setUp(self) -> None:
"""Set up variables for testing."""
self._analyzer = Analyzer("test", "<title> Una ruta 6a+</title>")
elif "generatortest.py" in argv[1]:
FILE_PATH = argv[2]
else:
error_msg = ("Unexpected call from the command line: {}")
raise SyntaxError(error_msg.format(" ".join(argv)))
else:
error_msg = ("Please pass an arg for the path to a flair program.")
raise SyntaxError(error_msg)
# store program into string variable
with open(FILE_PATH, "r") as flr:
flairProgram = flr.read()
scanner = Scanner(flairProgram)
parser = Parser(scanner)
ast = parser.parse()
analyzer = Analyzer(ast)
symbolTable = analyzer.getSymbolTable()
generator = Generator(ast, symbolTable)
code = generator.generateCode()
if "/" in FILE_PATH:
fileName = FILE_PATH[FILE_PATH.rindex("/") + 1:]
else:
fileName = FILE_PATH
if "." in fileName:
fileName = fileName[:fileName.rindex(".")]
fileName += ".tm"
with open(fileName, 'w+') as f:
f.write(code)
print("TM code saved to file {}".format(fileName))
phpsessid = 't4cjka7f4nedrmmq7dubipji16'
# 日记内容保存路径
content_file_path = 'data/content.txt'
# 日记时间保存路径
time_file_path = 'data/time.txt'
# 词云图片保存路径
content_png_path = 'data/content.png'
# 统计时间柱状图保存路径
time_png_path = 'data/time.png'
# 爬取数据
# url: 某个人的饭否主页
# phpsessid: 登录饭否后浏览器 Cookie 中的值
# content_file_path: 日记内容保存路径
# time_file_path: 日记时间保存路径
Collector.collect(url, phpsessid, content_file_path, time_file_path)
# 分词，制作词云
# content_file_path: 日记内容保存路径
# content_png_path: 词云图片保存路径
Analyzer.segment_and_visualize(content_file_path, content_png_path)
# 统计发日记时间
# time_file_path: 日记时间保存路径
# time_png_path: 时间柱状图保存路径
Analyzer.count_times(time_file_path, time_png_path)
from datetime import datetime
from src.analyzer import Analyzer
last_refresh = datetime.now()
if __name__ == '__main__':
analyzer = Analyzer()
while True:
country = input('Country: ')
state = input('State: ')
if (datetime.now() - last_refresh).seconds > 60:
analyzer.provider.refresh()
try:
analyzer.plot(country, state)
except AssertionError:
print('Not found.')
except TypeError:
print('Can\'t plot record.')
def analyzer():
return Analyzer('test')
print ("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
print (" Opening data file : {0}".format(in_file_directory + in_file_name))
print (" - Started: " + str(start))
print ("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n")
log_text += "\n\n -DATA FILE : {0} at {1}".format(in_file_directory + in_file_name, str(start))+'\n'
########################
# Generate the network #
########################
print ("Step 0. INITIALIZATION\n")
# initialization - generating an Analyzer with all info to generate file handlers
_Analyzer = Analyzer(in_file_directory, in_file_name,
out_file_directory,
is_weighted, is_directed, full_analysis, weight_id, aggregate_number
)
_Analyzer.generate_network()
########################
# Launch the analysis #
########################
print ("Step 1. ANALYSIS\n")
_Analyzer.launch_analysis()
########################
print ("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
print (" Opening data file : {0}".format(in_file_directory + in_file_name))
print (" - Started: " + str(start))
print ("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n")
log_text += "\n\n -DATA FILE : {0} at {1}".format(in_file_directory + in_file_name, str(start))+'\n'
########################
# Generate the network #
########################
print ("Step 0. INITIALIZATION\n")
# initialization - generating an Analyzer with all info to generate file handlers
_Analyzer = Analyzer(in_file_directory, in_file_name,
out_file_directory,
is_weighted, is_directed, full_analysis, weight_id, aggregate_number
)
_Analyzer.generate_network()
########################
# Launch the analysis #
########################
print ("Step 1. ANALYSIS\n")
_Analyzer.launch_analysis()
########################
print (" - Started: " + str(start))
print ("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n")
# for log
log_text += "\n\n - DATA FILE : {0} at {1}".format(in_file_directory + in_file_name, str(start))+'\n'
########################
# Read The Config File #
########################
print ("Step 0. CONFIG VERIFICATION\n")
# initialization - generating an Analyzer with all info to generate file handlers
_Analyzer = Analyzer( in_file_directory, in_file_name, in_extension, in_delimiter,
in_file_missing_value, in_file_quote,
out_file_directory, out_file_name, out_extension, out_delimiter,
out_file_missing_value,
out_file_single_file, out_file_separate_line,
raw_data_structure, clean_data_structure )
# verification of admitted formats (in, out and data)
if not _Analyzer.check_valid():
stop_system(_Analyzer.error_message)
##########################
# Structure the raw data #
##########################
print ("Step 1. STRUCTURING THE RAW DATA\n")
_Analyzer.structure_raw_data()
def setUp(self) -> None:
"""Set up variables for testing."""
self._analyzer = Analyzer("test", '<a src="Test" /a>')
def main() -> None:
analyzer = Analyzer(["./data/example/testUser/StreamingHistory.json"])
payload: Mapping[str, Union[str, int]] = {}
print("Example 1: getPopularArtists()")
pa = analyzer.getPopularArtists()
print(pa)
print()
print("Example 1a: getPopularArtists() with specific Daytime")
payload = {"daytime": "morning"}
pa = analyzer.getPopularArtists(payload=payload)
print(pa)
print()
print("Example 1b: getPopularArtists() of Month July")
payload = {"month": 7}
pa = analyzer.getPopularArtists(payload=payload)
print(pa)
print()
print("Example 1c: getPopularArtists() of Month July for podcasts")
payload = {"month": 7, "media": "podcast"}
pa = analyzer.getPopularArtists(payload=payload)
print(pa)
print()
print(
"Example 1d: getPopularArtists() of specified period (2019-03-03 to 2019-03-04) for music, count = 3"
)
payload = {
"startYear": 2019,
"startMonth": 3,
"startDay": 3,
"startHour": 0,
"endYear": 2019,
"endMonth": 3,
"endDay": 4,
"endHour": 0,
"count": 3,
"media": "music",
}
pa = analyzer.getPopularArtists(payload=payload)
print(pa)
print()
print("Example 1e: getPopularArtists() for specific weekday (monday)")
payload = {
"weekday": "monday",
}
pa = analyzer.getPopularArtists(payload=payload)
print(pa)
print()
print("Example 2: getPopularItems()")
pi = analyzer.getPopularItems()
print(pi)
print()
print("Example 2a: getPopularItems() for podcasts")
payload = {
"media": "podcast",
}
pi = analyzer.getPopularItems(payload=payload)
print(pi)
print()
print("Example 2b: getPopularItems() for podcasts by time played in ms")
payload = {
"media": "podcast",
"ratingCrit": "time",
}
pi = analyzer.getPopularItems(payload=payload)
print(pi)
print()
print("Example 2c: getPopularItems() with keyword 'franz ferdinand'")
pi = analyzer.getPopularItems(payload={"keyword": "Franz Ferdinand"})
print(pi)
print()
print("Example 2c2: getPopularItems() with keyword 'acoustic'")
pi = analyzer.getPopularItems(payload={"keyword": "acoustic"})
print(pi)
print()
print("Example 3: getDataPerWeekday()")
dpw = analyzer.getDataPerWeekday()
print(dpw)
print()
print("Example 3a: getDataPerWeekday() with string as key")
dpw = analyzer.getDataPerWeekday(weekdayFormat="")
print(dpw)
def analyzer_TestUser():
test_file = "./data/example/testUser/StreamingHistory.json"
return Analyzer([test_file])
def analyzer_patched(mock_get):
a = Analyzer('test')
a._get = mock_get
return a
from sys import argv, path
import os
path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from src.parser import Parser
from src.scanner import Scanner
from src.analyzer import Analyzer
# pass in arg for path to flair program
if len(argv) > 1:
if "analyzertest.py" in argv[0]:
FILE_PATH = argv[1]
elif "analyzertest.py" in argv[1]:
FILE_PATH = argv[2]
else:
error_msg = ("Unexpected call from the command line: {}")
raise SyntaxError(error_msg.format(" ".join(argv)))
else:
error_msg = ("Please pass an arg for the path to a flair program.")
raise SyntaxError(error_msg)
# store program into string variable
with open(FILE_PATH, "r") as flr:
flairProgram = flr.read()
scanner = Scanner(flairProgram)
parser = Parser(scanner)
analyzer = Analyzer(parser.parse())
symbolTable = analyzer.getSymbolTable()
print("Symbol Table:\n" + str(symbolTable))
def main():
analyzer = Analyzer()
analyzer.draw_rtt_graph(DatType.Normal)
analyzer.write_info_csv()
print("Complete!")
def main():
#xmldb = XMLDatabase("https://bugzilla.gnome.org/xmlrpc.cgi", "gnome")
mongodb = MongoDatabase("https://bugzilla.gnome.org/xmlrpc.cgi", "gnome")
bugdb = BugzillaDB(mongodb)
man = MongoAnalyzer()
an = Analyzer(bugdb, man)