def __init__(self):
self.topics_requester = Requester(properties.CONVO_URL_TOPICS)
self.answer_requester = Requester(properties.CONVO_URL_ANSWER)
self.question_requester = Requester(properties.CONVO_URL_QUESTION)
self.question_length_requester = Requester(properties.CONVO_URL_LENGTH)
self.topic_commands = ["topic", "topics", "t"]
self.current_topic = None
self.current_level = None
self.users = {}
pass
def main(args):
# Read in the config
config = None
with args.config_filename.open('r') as config_file:
config = toml.load(config_file)
# Initialize the logger
logger = build_logger(config)
# Load randomization options
seed = config["common"]["seed"]
shuffle = config["common"]["shuffle"]
# Seed the rng
random.seed(seed)
# Initialize proxy configuration
proxy = Proxy(config, args.tbb_path)
# Initialize a browser
with Requester(config) as requester:
# Load the dataset
urls = load_alexa_lazy(args.urls_path, "http")
# Navigate to all of the urls
for idx, url in enumerate(urls):
with proxy as _:
# TODO: log this somewhere auditable
# Log the request
logger.info("Navigating to {} ({}/{})".format(
url, idx, "UNKNOWN"))
# Request the URL
requester.request(url)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.title('Tk Chat App')
self.geometry('700x500')
self.requester = Requester()
self.menu = tk.Menu(self, bg='lightgrey', fg='black', tearoff=0)
self.friends_menu = tk.Menu(self.menu,
fg='black',
bg='lightgrey',
tearoff=0)
self.friends_menu.add_command(label='Add Friend',
command=self.show_add_friend_window)
self.menu.add_command(label='Add Friend', command=self.add_friends)
self.menu.add_cascade(label='Friends', menu=self.friends_menu)
self.show_login_screen()
self.avatar_menu = tk.Menu(self.menu,
fg='black',
bg='lightgrey',
tearoff=0)
self.avatar_menu.add_command(label='Change Avatar',
command=self.change_avatar)
def setUp(self):
requester = Requester()
requester.commandCrypto = CommandCrypto(None)
requester.commandQuit = CommandQuit(sys)
self.commandError = CommandError(None)
requester.commandError = self.commandError
self.requester = requester
def testGetProjectNameInvalidUserInput_minus_one(self):
if os.name == 'posix':
configFilePathName = '/storage/emulated/0/Android/data/ru.iiec.pydroid3/files/trans_file_cloud/test/transfiles.ini'
else:
configFilePathName = 'D:\\Development\\Python\\trans_file_cloud\\test\\transfiles.ini'
configManager = ConfigManager(configFilePathName)
rq = Requester(configManager)
# simulating user input
stdin = sys.stdin
# invalid user input of -1
sys.stdin = StringIO('-1\nQ')
stdout = sys.stdout
outputCapturingString = StringIO()
sys.stdout = outputCapturingString
rq.getProjectName(
None
) # will eat up what has been filled in stdin using StringIO above
sys.stdin = stdin
sys.stdout = stdout
self.assertEqual(
'Select project (Enter to loop or quit):\n\n1 transFileCloudTestProject\n2 transFileCloudProject\n3 transFileCloudFilePathNameProject\n4 cartesianAxesProject\n5 transFileCloudInvalidProject\n\nInvalid selection. Select project (Enter to loop or quit):\n\n1 transFileCloudTestProject\n2 transFileCloudProject\n3 transFileCloudFilePathNameProject\n4 cartesianAxesProject\n5 transFileCloudInvalidProject\n\n',
outputCapturingString.getvalue())
def testGetUserConfirmation_downloadFiles(self):
if os.name == 'posix':
configFilePathName = '/storage/emulated/0/Android/data/ru.iiec.pydroid3/files/trans_file_cloud/test/transfiles.ini'
else:
configFilePathName = 'D:\\Development\\Python\\trans_file_cloud\\test\\transfiles.ini'
configManager = ConfigManager(configFilePathName)
rq = Requester(configManager)
# simulating user input
stdin = sys.stdin
sys.stdin = StringIO('y')
stdout = sys.stdout
outputCapturingString = StringIO()
sys.stdout = outputCapturingString
cloudFileLst = [
'constants_2.py', 'filelister_2.py', 'testfilelister_2.py'
]
questionStr = 'vvv {} files will be transferred from the cloud and then moved to the correct dir and sub-dir of {}.\nIf you want to upload new modified files instead, type N'.format(
len(cloudFileLst), 'ru.iiec.pydroid3/files/trans_file_cloud')
doDownload, lastSynchTimeChoice = rq.getUserConfirmation(
questionStr, cloudFileLst)
sys.stdin = stdin
sys.stdout = stdout
self.assertEqual(
'\nconstants_2.py\nfilelister_2.py\ntestfilelister_2.py\n\nvvv 3 files will be transferred from the cloud and then moved to the correct dir and sub-dir of ru.iiec.pydroid3/files/trans_file_cloud.\nIf you want to upload new modified files instead, type N.\n\nContinue (Y/YK/N/Enter) ',
outputCapturingString.getvalue())
self.assertTrue(doDownload)
self.assertEqual('', lastSynchTimeChoice)
def __init__(self, master, user_one, user_two):
super().__init__()
self.master = master
self.user_one = user_one
self.user_two = user_two
self.requester = Requester()
self.running = True
self.last_checked_time = arrow.now().timestamp
def setup_subscriber_requester(context):
requester = Requester(context, ip, port)
print("Requester ready")
requester.send_string('SUB_PORT')
sub_port = requester.recv_string()
print(sub_port)
subscriber = Subscriber(context, ip, sub_port)
print("subscriber ready")
return requester, subscriber
def main():
"""
Instancie mes classes Requester et Scrapper, effectue une première requete puis transmet la réponse au scrapper
"""
requester = Requester()
scrapper = Scrapper(requester)
requested_response = requester.html_requester(constants.URL)
category_list = scrapper.get_category_list(requested_response)
scrapper.scrap_books_in_category(category_list, scrapper)
def __init__(self, base_url, page_url, proxies, gather_titles, search_text,
timeout, delay):
self.base_url = base_url
self.page_url = page_url
self.delay = delay # Delay in seconds between each HTTP Request
self.urls = set()
self.url_with_title = dict()
self.gather_titles = gather_titles
self.page_title = None
self.search_text = search_text
self.requester = Requester(page_url=page_url,
host=parse.urlparse(self.page_url).netloc,
proxies=proxies,
timeout=timeout)
def get_project_count(user, **q_args):
"""
query the API for projects associated with a given internal_id
"""
url_args = user
url_args.update(extra_args="&".join(
["{}={}".format(item[0], item[1]) for item in q_args.items()]))
try:
project = Requester(
url_template="http://{domain}/api/{api_version}/project/?"
"format=json&api_key={api_key}&username={username}&{extra_args}",
url_args=url_args)
except Exception, e:
print "{message}".format(message=e.message)
return False, None
def __init__(self, configFilePath=None, projectName=None):
"""
TransferFiles constructor.
@param configFilePath: used for unit testing only
@param projectName used for unit testing only
"""
if configFilePath == None:
# we are not unit testing ...
configFilePath = CONFIG_FILE_PATH_NAME
self.configManager = ConfigManager(configFilePath)
self.requester = Requester(self.configManager)
self.initTransferFileOnProject(configFilePath, projectName)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.title('Tk Chat')
self.geometry('700x500')
self.menu = tk.Menu(self, bg="lightgrey", fg="black", tearoff=0)
self.friends_menu = tk.Menu(self.menu, fg="black", bg="lightgrey", tearoff=0)
self.friends_menu.add_command(label="Add Friend", command=self.add_friend)
self.menu.add_cascade(label="Friends", menu=self.friends_menu)
self.requester = Requester()
self.show_login_screen()
def load_matches_to_db_and_celery(self):
r = Requester()
matches = r.get_player_matches(self.faceit_id, 50)
matches_data = [(match['id'], self.faceit_id, match['started'])
for match in matches]
sql = f"""
INSERT INTO matches (faceit_id, player_faceit_id, date_played) VALUES
(%s, %s, to_timestamp(%s)) ON CONFLICT DO NOTHING;
"""
con = psycopg2.connect(
"host=13.53.197.126 port=5432 dbname=csgo user=csgo password=csgo")
cur = con.cursor()
cur.executemany(sql, matches_data)
cur.close()
con.commit()
con.close()
for match in matches_data:
load_and_store_stats.delay(match[0], self.faceit_id)
def __init__(self, audioDownloaderGUI, audioDir, configMgr=None):
"""
:param audioDownloaderGUI: used for unit testing only !
"""
if os.name == 'posix':
configFilePathName = '/storage/emulated/0/audiodownloader.ini'
else:
configFilePathName = 'c:\\temp\\audiodownloader.ini'
if configMgr == None:
self.configMgr = ConfigManager(configFilePathName)
else:
self.configMgr = configMgr
self.requester = Requester(self.configMgr)
self.audioDownloaderGUI = audioDownloaderGUI
self.audioDownloader = YoutubeDlAudioDownloader(self, audioDir)
def __init__(self, printer, configMgr, priceRequester): self.configMgr = configMgr self.priceRequester = priceRequester self.crypCompTranslator = CrypCompExchanges() self.processor = Processor(self.configMgr, self.priceRequester, self.crypCompTranslator) self.requester = Requester(self.configMgr) self.commandPrice = CommandPrice(self.processor, self.configMgr) self.commandCrypto = CommandCrypto(self.processor) self.requester.commandPrice = self.commandPrice self.requester.commandCrypto = self.commandCrypto self.commandQuit = CommandQuit(sys) self.requester.commandQuit = self.commandQuit self.commandError = CommandError(None) self.requester.commandError = self.commandError self.printer = printer
def run():
Configuration.initialize()
Cracker.crt_workload = 4 # TODO get value from parameters, adjust from keyboard
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
Cracker.req = Requester(Configuration.apikey,
Comunicator.error_printer)
Cracker.resume_work()
Comunicator.initialize()
Comunicator.printer("Cracker initialized", reprint=False)
# Disable terminal echo
os.system("stty -echo")
try:
last_time = None
while True:
now_time = datetime.now()
if last_time is None or (now_time -
last_time).total_seconds() > 10:
last_time = now_time
Cracker.crack_existing_handshakes()
cmd = Comunicator.get_command()
if cmd is not None:
Cracker.parse_command(cmd)
sleep(0.1)
except Exception as e:
Configuration.dual_print(
Configuration.logger.critical,
"Caught unexpected exception: '%s'" % (traceback.format_exc()))
Cracker.clean_variables()
die(True, e)
finally:
# Reenable terminal echo
os.system("stty echo")
pass
def put_an_activity(activity_element, pk, url_args):
"NOTE: does not work!!!"
url_args.update(pk=pk)
try:
iati_id = activity_element.findall('iati-identifier')[0].text
project = Requester(
method='put',
url_template="http://{domain}/api/{api_version}/iati_activity/{pk}/?"
"format=xml&api_key={api_key}&username={username}",
url_args=url_args,
headers={
'content-type': 'application/xml',
'encoding': 'utf-8'
},
data=etree.tostring(activity_element),
accept_codes=[HttpNoContent.status_code])
except Exception, e:
return False, "{extra}", dict(iati_id=iati_id,
event=ERROR_EXCEPTION,
extra=e.message)
def run(self):
req = Requester()
proc = Processor()
pri = Printer()
commandCrypto = CommandCrypto(proc)
req.commandCrypto = commandCrypto
commandQuit = CommandQuit(sys)
req.commandQuit = commandQuit
commandError = CommandError(None)
req.commandError = commandError
while True:
command = req.request()
result = command.execute()
if result != '':
pri.print(result)
def post_an_activity(activity_element, user):
try:
iati_id = activity_element.findall('iati-identifier')[0].text
project = Requester(
method='post',
url_template="http://{domain}/api/{api_version}/iati_activity/"
"?format=xml&api_key={api_key}&username={username}",
url_args=user,
headers={
'content-type': 'application/xml',
'encoding': 'utf-8'
},
data=etree.tostring(activity_element),
accept_codes=[HttpCreated.status_code])
except Exception, e:
return False, "{extra}", dict(
iati_id=iati_id,
event=ERROR_EXCEPTION,
extra=e.message,
)
def __enter__(self):
"""
Ensures server is available
Requests and stores a client ID from the server
Gets a connection to the tcpdump daemon
:throws Exception: if the client ID request fails
"""
# Send requests to the URLs service until the status
# page returns a response
waiting = True
while waiting:
try:
self.logger.info("Attempting to contact work queue")
self.session.get("{}/status".format(self.work_url))
waiting = False
except Exception as _:
self.logger.info(
"Attempt to contact work queue failed. Retrying")
# Request a client ID
# TODO: look into renaming this "register"
self.logger.info("Registering client with server")
# TODO: work types as part of config
response = self.session.post(
"{}/client/add".format(self.work_url),
json={'work_types': ['tor', 'normal']})
# Parse response as json
response = response.json()
# Extract client id from response
if response['success']:
self.client_id = response['client_id']
else:
raise Exception(response['error'])
# Start up a connection to the tcpdump daemon
# TODO: parameterize socket path
self.tcpdump = TcpDump('/tmp/tcpdump.socket')
# Instantiate proxy object
self.proxy = Proxy(self.tbb_path, self.config["tor"])
# Instantiate requester object
self.requester = Requester(self.config["firefox"],
self.config["tor"]["port"])
return self
def run():
Comunicator.initialize()
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
Configuration.initialize()
Cracker.crt_workload = Configuration.hashcat_workload # TODO maybe adjust from keyboard
Cracker.req = Requester(Configuration.apikey, Comunicator.error_logger)
Cracker.resume_work()
Comunicator.printer("Cracker initialized", reprint=False)
try:
# Disable terminal echo
os.system("stty -echo")
last_time = None
while True:
now_time = datetime.now()
if last_time is None or (now_time -
last_time).total_seconds() > 10:
last_time = now_time
Cracker.do_work()
cmd = Comunicator.get_command()
if cmd is not None:
Cracker.parse_command(cmd)
sleep(0.1)
except Exception as e:
Cracker.clean_variables()
Comunicator.fatal_debug_printer(
"Caught unexpected exception: '%s' '%s'" %
(e, traceback.format_exc()))
finally:
# Reenable terminal echo
os.system("stty echo")
pass
def crawl(self, URL, depth=0):
if depth <= P.crawlerProp.depth:
print("DEPTH >> ", depth)
print("VISITING URL >> ", URL)
try:
requester = Requester(URL)
HTML = requester.getHtml()
parser = Parser(HTML)
links = parser.getTag('a')
#print(HTML)
words = F.extractWords(
parser.getTags(list(P.crawlerProp.atlas.keys())),
P.crawlerProp.pos_tag)
print(words)
depth += 1
for link in links:
if link is not None:
if Tag(link).hasKey('href'):
nURL = link['href'] if F.urlValid(
link['href']) else (
F.urlFix(URL, link['href']) if F.urlValid(
F.urlFix(URL, link['href'])) else None)
if self.__pass(nURL):
self.visited.append(nURL)
#print(nURL)
self.crawl(nURL, depth)
#else:
#print("SKIPPING URL NOT VALID >> ", nURL)
except:
print(Exception())
else:
print("REACHED DEPTH LIMIT FOR >> ", URL)
def data_received(self, data):
"""
Clients data ie Http/Https
Args:
data(bytes):Has the request headers and body
"""
requester = Requester(data)
try:
requester.connect()
requester.send_data()
response = requester.receive_data()
self.transport.write(response)
requester.close()
self.close_transport()
except Exception as e:
print("Error", e)
async def main():
# init queue
queue = asyncio.Queue()
queue.lock = False
# init DB connector
db_connector = GetUrls()
await db_connector.create_connections(LOGGER)
# init URL handler
requester = Requester(LOGGER, db_connector)
# Producer
await start_produce(requester, queue)
# Consumers
consumers = [asyncio.create_task(consume(thread, requester, queue)).add_done_callback(task_completed) \
for thread in range(1, settings.MAX_THREADS)]
await queue.join() # Implicitly awaits consumers
for cons in consumers:
cons.cancel()
def api_user(domain, username, password='', api_key=''):
user = dict(
domain=domain,
username=username,
api_version=API_VERSION,
)
if api_key:
user['api_key'] = api_key
return user
elif password:
auth = Requester(
method='post',
url_template="http://{domain}/auth/token/",
url_args=dict(domain=domain),
data=dict(username=username, password=password),
)
xml = auth.response.text
root = etree.fromstring(xml)
user['api_key'] = root.find("api_key").text
return user
else:
raise Exception("Either password or API key must be supplied")
def testGetUserConfirmation_uploadFiles(self):
if os.name == 'posix':
configFilePathName = '/storage/emulated/0/Android/data/ru.iiec.pydroid3/files/trans_file_cloud/test/transfiles.ini'
else:
configFilePathName = 'D:\\Development\\Python\\trans_file_cloud\\test\\transfiles.ini'
configManager = ConfigManager(configFilePathName)
rq = Requester(configManager)
# simulating user input
stdin = sys.stdin
sys.stdin = StringIO('y')
stdout = sys.stdout
outputCapturingString = StringIO()
sys.stdout = outputCapturingString
updatedFileNameLst = [
'constants_2.py', 'filelister_2.py', 'testfilelister_2.py'
]
updatedFilePathNameLst = [
'/project/constants_2.py', '/project/filelister_2.py',
'/project/test/testfilelister_2.py'
]
questionStr = '^^^ {} files were modified locally after {}\nand will be uploaded to the cloud.\nChoose P to display the path or U to update the last sync time'.format(
len(updatedFileNameLst), '2020-07-22 15:30:22')
doUpload, lastSynchTimeChoice = rq.getUserConfirmation(
questionStr, updatedFileNameLst, updatedFilePathNameLst)
sys.stdin = stdin
sys.stdout = stdout
self.assertEqual(
'\nconstants_2.py\nfilelister_2.py\ntestfilelister_2.py\n\n^^^ 3 files were modified locally after 2020-07-22 15:30:22\nand will be uploaded to the cloud.\nChoose P to display the path or U to update the last sync time.\n\nContinue (Y/N/P/U/Enter) ',
outputCapturingString.getvalue())
self.assertTrue(doUpload)
self.assertEqual('', lastSynchTimeChoice)
def __init__(self, printer, configMgr):
if os.name == 'posix':
FILE_PATH = '/sdcard/cryptopricer.ini'
else:
FILE_PATH = 'c:\\temp\\cryptopricer.ini'
self.configMgr = configMgr
self.priceRequester = PriceRequester()
self.crypCompTranslator = CrypCompExchanges()
self.processor = Processor(self.configMgr, self.priceRequester, self.crypCompTranslator)
self.requester = Requester(self.configMgr)
self.commandPrice = CommandPrice(self.processor, self.configMgr)
self.commandCrypto = CommandCrypto(self.processor)
self.requester.commandPrice = self.commandPrice
self.requester.commandCrypto = self.commandCrypto
self.commandQuit = CommandQuit(sys)
self.requester.commandQuit = self.commandQuit
self.commandError = CommandError(None)
self.requester.commandError = self.commandError
self.printer = printer
def parse(self, site):
'''parse info
Args:
html: text file for the page
site: newsWebsite class object
Returns:
usefulInfoDicts: array, dicts of each news's useful information
'''
print(f'---------\ngetting info from {site}')
# Get news blocks if it's needed
if not site.articleBlock:
print('tracing without blocks\n----------')
self.trace(self.bs, site)
else:
articleBlock = self.traceBlock(self.bs, site.articleBlock)
# Get js content using selenium
if articleBlock == None:
print(
'Trying js loader to get html\n!Warning: check if your tracers are good\n----------\n'
)
html = Requester().jsHtmlLoader(url=site.rootURL)
self.html_to_bs(html)
articleBlock = self.traceBlock(self.bs, site.articleBlock)
# print(f'Getting news article from block: {articleBlock}')
print('successfully get block')
self.trace(articleBlock, site)
return self.usefulInfoDicts
import time
from unittest import TestCase
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.select import Select
import Main
from requester import Requester
requester = Requester()
class TestWeb(TestCase):
def setUp(self):
Main.run()
self.browser = webdriver.Firefox()
self.browser.implicitly_wait(3)
def tearDown(self):
Main.terminate()
self.browser.quit()
def test_basic_sequence(self):
requester.make_request({
"account": {
"type": "admin",
"login": "******",
"password": "******"
},
"object": {