def testGetProjectNameInvalidUserInput_minus_one(self):
if os.name == 'posix':
configFilePathName = '/storage/emulated/0/Android/data/ru.iiec.pydroid3/files/trans_file_cloud/test/transfiles.ini'
else:
configFilePathName = 'D:\\Development\\Python\\trans_file_cloud\\test\\transfiles.ini'
configManager = ConfigManager(configFilePathName)
rq = Requester(configManager)
# simulating user input
stdin = sys.stdin
# invalid user input of -1
sys.stdin = StringIO('-1\nQ')
stdout = sys.stdout
outputCapturingString = StringIO()
sys.stdout = outputCapturingString
rq.getProjectName(
None
) # will eat up what has been filled in stdin using StringIO above
sys.stdin = stdin
sys.stdout = stdout
self.assertEqual(
'Select project (Enter to loop or quit):\n\n1 transFileCloudTestProject\n2 transFileCloudProject\n3 transFileCloudFilePathNameProject\n4 cartesianAxesProject\n5 transFileCloudInvalidProject\n\nInvalid selection. Select project (Enter to loop or quit):\n\n1 transFileCloudTestProject\n2 transFileCloudProject\n3 transFileCloudFilePathNameProject\n4 cartesianAxesProject\n5 transFileCloudInvalidProject\n\n',
outputCapturingString.getvalue())
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.title('Tk Chat App')
self.geometry('700x500')
self.requester = Requester()
self.menu = tk.Menu(self, bg='lightgrey', fg='black', tearoff=0)
self.friends_menu = tk.Menu(self.menu,
fg='black',
bg='lightgrey',
tearoff=0)
self.friends_menu.add_command(label='Add Friend',
command=self.show_add_friend_window)
self.menu.add_command(label='Add Friend', command=self.add_friends)
self.menu.add_cascade(label='Friends', menu=self.friends_menu)
self.show_login_screen()
self.avatar_menu = tk.Menu(self.menu,
fg='black',
bg='lightgrey',
tearoff=0)
self.avatar_menu.add_command(label='Change Avatar',
command=self.change_avatar)
def setUp(self):
requester = Requester()
requester.commandCrypto = CommandCrypto(None)
requester.commandQuit = CommandQuit(sys)
self.commandError = CommandError(None)
requester.commandError = self.commandError
self.requester = requester
def testGetUserConfirmation_downloadFiles(self):
if os.name == 'posix':
configFilePathName = '/storage/emulated/0/Android/data/ru.iiec.pydroid3/files/trans_file_cloud/test/transfiles.ini'
else:
configFilePathName = 'D:\\Development\\Python\\trans_file_cloud\\test\\transfiles.ini'
configManager = ConfigManager(configFilePathName)
rq = Requester(configManager)
# simulating user input
stdin = sys.stdin
sys.stdin = StringIO('y')
stdout = sys.stdout
outputCapturingString = StringIO()
sys.stdout = outputCapturingString
cloudFileLst = [
'constants_2.py', 'filelister_2.py', 'testfilelister_2.py'
]
questionStr = 'vvv {} files will be transferred from the cloud and then moved to the correct dir and sub-dir of {}.\nIf you want to upload new modified files instead, type N'.format(
len(cloudFileLst), 'ru.iiec.pydroid3/files/trans_file_cloud')
doDownload, lastSynchTimeChoice = rq.getUserConfirmation(
questionStr, cloudFileLst)
sys.stdin = stdin
sys.stdout = stdout
self.assertEqual(
'\nconstants_2.py\nfilelister_2.py\ntestfilelister_2.py\n\nvvv 3 files will be transferred from the cloud and then moved to the correct dir and sub-dir of ru.iiec.pydroid3/files/trans_file_cloud.\nIf you want to upload new modified files instead, type N.\n\nContinue (Y/YK/N/Enter) ',
outputCapturingString.getvalue())
self.assertTrue(doDownload)
self.assertEqual('', lastSynchTimeChoice)
def __init__(self, master, user_one, user_two):
super().__init__()
self.master = master
self.user_one = user_one
self.user_two = user_two
self.requester = Requester()
self.running = True
self.last_checked_time = arrow.now().timestamp
def setup_subscriber_requester(context):
requester = Requester(context, ip, port)
print("Requester ready")
requester.send_string('SUB_PORT')
sub_port = requester.recv_string()
print(sub_port)
subscriber = Subscriber(context, ip, sub_port)
print("subscriber ready")
return requester, subscriber
def __init__(self, tieba_name="steam", cookie=None):
"""
Initialize Crawler
:param tieba_name:
:param cookie:
"""
Requester.__init__(self, tieba_name, cookie)
self.__available_check()
def main():
"""
Instancie mes classes Requester et Scrapper, effectue une première requete puis transmet la réponse au scrapper
"""
requester = Requester()
scrapper = Scrapper(requester)
requested_response = requester.html_requester(constants.URL)
category_list = scrapper.get_category_list(requested_response)
scrapper.scrap_books_in_category(category_list, scrapper)
def __init__(self, base_url, page_url, proxies, gather_titles, search_text,
timeout, delay):
self.base_url = base_url
self.page_url = page_url
self.delay = delay # Delay in seconds between each HTTP Request
self.urls = set()
self.url_with_title = dict()
self.gather_titles = gather_titles
self.page_title = None
self.search_text = search_text
self.requester = Requester(page_url=page_url,
host=parse.urlparse(self.page_url).netloc,
proxies=proxies,
timeout=timeout)
def __init__(self, configFilePath=None, projectName=None):
"""
TransferFiles constructor.
@param configFilePath: used for unit testing only
@param projectName used for unit testing only
"""
if configFilePath == None:
# we are not unit testing ...
configFilePath = CONFIG_FILE_PATH_NAME
self.configManager = ConfigManager(configFilePath)
self.requester = Requester(self.configManager)
self.initTransferFileOnProject(configFilePath, projectName)
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.title('Tk Chat')
self.geometry('700x500')
self.menu = tk.Menu(self, bg="lightgrey", fg="black", tearoff=0)
self.friends_menu = tk.Menu(self.menu, fg="black", bg="lightgrey", tearoff=0)
self.friends_menu.add_command(label="Add Friend", command=self.add_friend)
self.menu.add_cascade(label="Friends", menu=self.friends_menu)
self.requester = Requester()
self.show_login_screen()
def main(args):
# Read in the config
config = None
with args.config_filename.open('r') as config_file:
config = toml.load(config_file)
# Initialize the logger
logger = build_logger(config)
# Load randomization options
seed = config["common"]["seed"]
shuffle = config["common"]["shuffle"]
# Seed the rng
random.seed(seed)
# Initialize proxy configuration
proxy = Proxy(config, args.tbb_path)
# Initialize a browser
with Requester(config) as requester:
# Load the dataset
urls = load_alexa_lazy(args.urls_path, "http")
# Navigate to all of the urls
for idx, url in enumerate(urls):
with proxy as _:
# TODO: log this somewhere auditable
# Log the request
logger.info("Navigating to {} ({}/{})".format(
url, idx, "UNKNOWN"))
# Request the URL
requester.request(url)
class TestRequester(TestCase):
def setUp(self):
self.requester = Requester()
@patch('requests.post', side_effect=ret_val)
def test_make_request(self, test_patch):
value = self.requester.make_request("message")
self.assertEqual(value[1], {'data': '"message"'})
def __init__(self):
self.topics_requester = Requester(properties.CONVO_URL_TOPICS)
self.answer_requester = Requester(properties.CONVO_URL_ANSWER)
self.question_requester = Requester(properties.CONVO_URL_QUESTION)
self.question_length_requester = Requester(properties.CONVO_URL_LENGTH)
self.topic_commands = ["topic", "topics", "t"]
self.current_topic = None
self.current_level = None
self.users = {}
pass
def load_matches_to_db_and_celery(self):
r = Requester()
matches = r.get_player_matches(self.faceit_id, 50)
matches_data = [(match['id'], self.faceit_id, match['started'])
for match in matches]
sql = f"""
INSERT INTO matches (faceit_id, player_faceit_id, date_played) VALUES
(%s, %s, to_timestamp(%s)) ON CONFLICT DO NOTHING;
"""
con = psycopg2.connect(
"host=13.53.197.126 port=5432 dbname=csgo user=csgo password=csgo")
cur = con.cursor()
cur.executemany(sql, matches_data)
cur.close()
con.commit()
con.close()
for match in matches_data:
load_and_store_stats.delay(match[0], self.faceit_id)
def __init__(self, audioDownloaderGUI, audioDir, configMgr=None):
"""
:param audioDownloaderGUI: used for unit testing only !
"""
if os.name == 'posix':
configFilePathName = '/storage/emulated/0/audiodownloader.ini'
else:
configFilePathName = 'c:\\temp\\audiodownloader.ini'
if configMgr == None:
self.configMgr = ConfigManager(configFilePathName)
else:
self.configMgr = configMgr
self.requester = Requester(self.configMgr)
self.audioDownloaderGUI = audioDownloaderGUI
self.audioDownloader = YoutubeDlAudioDownloader(self, audioDir)
def __init__(self, printer, configMgr, priceRequester): self.configMgr = configMgr self.priceRequester = priceRequester self.crypCompTranslator = CrypCompExchanges() self.processor = Processor(self.configMgr, self.priceRequester, self.crypCompTranslator) self.requester = Requester(self.configMgr) self.commandPrice = CommandPrice(self.processor, self.configMgr) self.commandCrypto = CommandCrypto(self.processor) self.requester.commandPrice = self.commandPrice self.requester.commandCrypto = self.commandCrypto self.commandQuit = CommandQuit(sys) self.requester.commandQuit = self.commandQuit self.commandError = CommandError(None) self.requester.commandError = self.commandError self.printer = printer
def __enter__(self):
"""
Ensures server is available
Requests and stores a client ID from the server
Gets a connection to the tcpdump daemon
:throws Exception: if the client ID request fails
"""
# Send requests to the URLs service until the status
# page returns a response
waiting = True
while waiting:
try:
self.logger.info("Attempting to contact work queue")
self.session.get("{}/status".format(self.work_url))
waiting = False
except Exception as _:
self.logger.info(
"Attempt to contact work queue failed. Retrying")
# Request a client ID
# TODO: look into renaming this "register"
self.logger.info("Registering client with server")
# TODO: work types as part of config
response = self.session.post(
"{}/client/add".format(self.work_url),
json={'work_types': ['tor', 'normal']})
# Parse response as json
response = response.json()
# Extract client id from response
if response['success']:
self.client_id = response['client_id']
else:
raise Exception(response['error'])
# Start up a connection to the tcpdump daemon
# TODO: parameterize socket path
self.tcpdump = TcpDump('/tmp/tcpdump.socket')
# Instantiate proxy object
self.proxy = Proxy(self.tbb_path, self.config["tor"])
# Instantiate requester object
self.requester = Requester(self.config["firefox"],
self.config["tor"]["port"])
return self
class SnapshotReport:
"""
Parent class for retrieving data from a snapshot service to generate a
report that won't be written to the service.
"""
def __init__(self, snapshot_service, config, inputs):
"""
Params
------
snapshot_service: string
Name of snapshot service. This should be a section in the |config|
param.
config: ConfigParser (from a config.ini file)
inputs: array of strings
These are snapshot headers that look like "=====cond vacationator".
These are what will be returned from "get_inputs".
"""
section = snapshot_service
host = config.get(section, 'host')
req_port = int(config.get(section, 'request_port'))
self.inputs = inputs
self.snapshot_requester = Requester(host, req_port)
return
def get_inputs(self):
"""
Returns snapshots of the inputs as an array in the same order as
specified on initialization.
"""
headers = self.get_headers()
result = []
for h in headers:
input_data = StringIO.StringIO(self.snapshot_requester.make_request(h))
input_sections = sectionize(input_data)
result.append(input_sections["data"])
return result
#===========================================================================
# Internal Functions
def get_headers(self):
"""
Returns a an array of "GET" requests for each input header. These can be
sent directly to the SnapshotService.
"""
result = ["=====GET %s" % h.split("=====")[1] for h in self.inputs]
return result
def crawl(self, URL, depth=0):
if depth <= P.crawlerProp.depth:
print("DEPTH >> ", depth)
print("VISITING URL >> ", URL)
try:
requester = Requester(URL)
HTML = requester.getHtml()
parser = Parser(HTML)
links = parser.getTag('a')
#print(HTML)
words = F.extractWords(
parser.getTags(list(P.crawlerProp.atlas.keys())),
P.crawlerProp.pos_tag)
print(words)
depth += 1
for link in links:
if link is not None:
if Tag(link).hasKey('href'):
nURL = link['href'] if F.urlValid(
link['href']) else (
F.urlFix(URL, link['href']) if F.urlValid(
F.urlFix(URL, link['href'])) else None)
if self.__pass(nURL):
self.visited.append(nURL)
#print(nURL)
self.crawl(nURL, depth)
#else:
#print("SKIPPING URL NOT VALID >> ", nURL)
except:
print(Exception())
else:
print("REACHED DEPTH LIMIT FOR >> ", URL)
async def detect_features(requester: Requester) -> List[Feature]:
returner = []
for feature in features:
if callable(feature.tests):
checks = [await feature.tests(requester)]
else:
checks = await asyncio.gather(
*[requester.check(test) for test in feature.tests])
returner.append((feature, all(checks)))
return returner
def request(self, type=None, data=None, additional=None):
""" Builds and sends request
Args:
type: Paths Enum
data: Dict of data for request
additional: Dict of data which takes the key and maps it to the keys value if set in string
Returns:
After sending the request it returns one of the following:
json decoded object: Response from api
false: Something went wrong with the call
Json Decode Object Example:
[
{
"error":{
"type":101,
"address":"/",
"description":
"link button not pressed"
}
}
]
Raises:
ExceptionNoToken:
Token isn't being set on Huuey()
"""
address_set = self.address is not None
type_set = type is not None
token_set = self.token is not None
if not address_set or not type_set:
return False
if type is Paths.Api:
url = "".join([self.address, type.value[1]])
else:
if not token_set:
raise ExceptionNoToken('Token missing from api call')
url = "".join([self.address, "/api/", self.token, type.value[1]])
if additional:
for key in additional:
url = url.replace(key, additional[key])
request = Requester.request(url=url, method=type.value[0], data=data)
return request
def testGetUserConfirmation_uploadFiles(self):
if os.name == 'posix':
configFilePathName = '/storage/emulated/0/Android/data/ru.iiec.pydroid3/files/trans_file_cloud/test/transfiles.ini'
else:
configFilePathName = 'D:\\Development\\Python\\trans_file_cloud\\test\\transfiles.ini'
configManager = ConfigManager(configFilePathName)
rq = Requester(configManager)
# simulating user input
stdin = sys.stdin
sys.stdin = StringIO('y')
stdout = sys.stdout
outputCapturingString = StringIO()
sys.stdout = outputCapturingString
updatedFileNameLst = [
'constants_2.py', 'filelister_2.py', 'testfilelister_2.py'
]
updatedFilePathNameLst = [
'/project/constants_2.py', '/project/filelister_2.py',
'/project/test/testfilelister_2.py'
]
questionStr = '^^^ {} files were modified locally after {}\nand will be uploaded to the cloud.\nChoose P to display the path or U to update the last sync time'.format(
len(updatedFileNameLst), '2020-07-22 15:30:22')
doUpload, lastSynchTimeChoice = rq.getUserConfirmation(
questionStr, updatedFileNameLst, updatedFilePathNameLst)
sys.stdin = stdin
sys.stdout = stdout
self.assertEqual(
'\nconstants_2.py\nfilelister_2.py\ntestfilelister_2.py\n\n^^^ 3 files were modified locally after 2020-07-22 15:30:22\nand will be uploaded to the cloud.\nChoose P to display the path or U to update the last sync time.\n\nContinue (Y/N/P/U/Enter) ',
outputCapturingString.getvalue())
self.assertTrue(doUpload)
self.assertEqual('', lastSynchTimeChoice)
def get_project_count(user, **q_args):
"""
query the API for projects associated with a given internal_id
"""
url_args = user
url_args.update(extra_args="&".join(
["{}={}".format(item[0], item[1]) for item in q_args.items()]))
try:
project = Requester(
url_template="http://{domain}/api/{api_version}/project/?"
"format=json&api_key={api_key}&username={username}&{extra_args}",
url_args=url_args)
except Exception, e:
print "{message}".format(message=e.message)
return False, None
def __init__(self, printer, configMgr):
if os.name == 'posix':
FILE_PATH = '/sdcard/cryptopricer.ini'
else:
FILE_PATH = 'c:\\temp\\cryptopricer.ini'
self.configMgr = configMgr
self.priceRequester = PriceRequester()
self.crypCompTranslator = CrypCompExchanges()
self.processor = Processor(self.configMgr, self.priceRequester, self.crypCompTranslator)
self.requester = Requester(self.configMgr)
self.commandPrice = CommandPrice(self.processor, self.configMgr)
self.commandCrypto = CommandCrypto(self.processor)
self.requester.commandPrice = self.commandPrice
self.requester.commandCrypto = self.commandCrypto
self.commandQuit = CommandQuit(sys)
self.requester.commandQuit = self.commandQuit
self.commandError = CommandError(None)
self.requester.commandError = self.commandError
self.printer = printer
def data_received(self, data):
"""
Clients data ie Http/Https
Args:
data(bytes):Has the request headers and body
"""
requester = Requester(data)
try:
requester.connect()
requester.send_data()
response = requester.receive_data()
self.transport.write(response)
requester.close()
self.close_transport()
except Exception as e:
print("Error", e)
class ListeningThread(threading.Thread):
def __init__(self, master, user_one, user_two):
super().__init__()
self.master = master
self.user_one = user_one
self.user_two = user_two
self.requester = Requester()
self.running = True
self.last_checked_time = arrow.now().timestamp
def run(self):
while self.running:
new_messages = self.requester.get_new_messages(self.last_checked_time, self.user_one, self.user_two)
self.last_checked_time = arrow.now()
for message in new_messages['messages']:
self.master.recieve_message(message['author'], message['message'])
time.sleep(2)
del self.master.listening_thread
return
def run(self):
self.clear_output()
self.append_datetime_to_output()
url = self.url_entry.get()
if not Validator.is_url_correct(url):
self.append_text_to_output("Please provide a correct url")
return
response = Requester.send_request(url)
state = response.key
source = response.value
if not state == "ok":
self.append_text_to_output(source)
return
result = TagCounter.get_tags(source)
self.append_dic_to_output(result)
def __init__(self):
super(CalendarPlugin, self).__init__()
self.first_day = self.last_day = self.numdays = None
builder = Gtk.Builder()
builder.add_from_file("calendar_view.glade")
handlers = {
"on_window_destroy": Gtk.main_quit,
"on_today_clicked": self.on_today_clicked,
"on_combobox_changed": self.on_combobox_changed,
"on_add_clicked": self.on_add_clicked,
"on_edit_clicked": self.on_edit_clicked,
"on_remove_clicked": self.on_remove_clicked,
"on_next_clicked": self.on_next_clicked,
"on_previous_clicked": self.on_previous_clicked,
}
builder.connect_signals(handlers)
self.window = builder.get_object("window")
self.window.__init__()
self.window.set_title("GTG - Calendar View")
self.window.connect("destroy", Gtk.main_quit)
# DataStore object
self.ds = DataStore()
self.req = Requester(self.ds)
self.ds.populate() # hard-coded tasks
self.today_button = builder.get_object("today")
self.header = builder.get_object("header")
self.controller = Controller(self, self.req)
vbox = builder.get_object("vbox")
vbox.add(self.controller)
vbox.reorder_child(self.controller, 1)
self.current_view = None
self.combobox = builder.get_object("combobox")
self.combobox.set_active(2)
self.statusbar = builder.get_object("statusbar")
self.window.show_all()
def run():
Configuration.initialize()
Cracker.crt_workload = 4 # TODO get value from parameters, adjust from keyboard
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
Cracker.req = Requester(Configuration.apikey,
Comunicator.error_printer)
Cracker.resume_work()
Comunicator.initialize()
Comunicator.printer("Cracker initialized", reprint=False)
# Disable terminal echo
os.system("stty -echo")
try:
last_time = None
while True:
now_time = datetime.now()
if last_time is None or (now_time -
last_time).total_seconds() > 10:
last_time = now_time
Cracker.crack_existing_handshakes()
cmd = Comunicator.get_command()
if cmd is not None:
Cracker.parse_command(cmd)
sleep(0.1)
except Exception as e:
Configuration.dual_print(
Configuration.logger.critical,
"Caught unexpected exception: '%s'" % (traceback.format_exc()))
Cracker.clean_variables()
die(True, e)
finally:
# Reenable terminal echo
os.system("stty echo")
pass
def __init__(self, snapshot_service, config, inputs):
"""
Params
------
snapshot_service: string
Name of snapshot service. This should be a section in the |config|
param.
config: ConfigParser (from a config.ini file)
inputs: array of strings
These are snapshot headers that look like "=====cond vacationator".
These are what will be returned from "get_inputs".
"""
section = snapshot_service
host = config.get(section, 'host')
req_port = int(config.get(section, 'request_port'))
self.inputs = inputs
self.snapshot_requester = Requester(host, req_port)
return
def post_an_activity(activity_element, user):
try:
iati_id = activity_element.findall('iati-identifier')[0].text
project = Requester(
method='post',
url_template="http://{domain}/api/{api_version}/iati_activity/"
"?format=xml&api_key={api_key}&username={username}",
url_args=user,
headers={
'content-type': 'application/xml',
'encoding': 'utf-8'
},
data=etree.tostring(activity_element),
accept_codes=[HttpCreated.status_code])
except Exception, e:
return False, "{extra}", dict(
iati_id=iati_id,
event=ERROR_EXCEPTION,
extra=e.message,
)
def put_an_activity(activity_element, pk, url_args):
"NOTE: does not work!!!"
url_args.update(pk=pk)
try:
iati_id = activity_element.findall('iati-identifier')[0].text
project = Requester(
method='put',
url_template="http://{domain}/api/{api_version}/iati_activity/{pk}/?"
"format=xml&api_key={api_key}&username={username}",
url_args=url_args,
headers={
'content-type': 'application/xml',
'encoding': 'utf-8'
},
data=etree.tostring(activity_element),
accept_codes=[HttpNoContent.status_code])
except Exception, e:
return False, "{extra}", dict(iati_id=iati_id,
event=ERROR_EXCEPTION,
extra=e.message)
def __init__(self, tieba_name="steam", cookie=None):
Requester.__init__(self, tieba_name, cookie)
self.ban_url = TIEBA_BAN_URL
self.tbs_re = re.compile(r'"tbs": "(\S+)"')
self.fid_re = re.compile(r"fid:'(\d+)'")
def __init__(self, tieba_name="steam", cookie=None):
Requester.__init__(self, tieba_name, cookie)
class CalendarPlugin(GObject.GObject):
"""
This class is a plugin to display tasks into a dedicated view, where tasks
can be selected, edited, moved around by dragging and dropping, etc.
"""
def __init__(self):
super(CalendarPlugin, self).__init__()
self.first_day = self.last_day = self.numdays = None
builder = Gtk.Builder()
builder.add_from_file("calendar_view.glade")
handlers = {
"on_window_destroy": Gtk.main_quit,
"on_today_clicked": self.on_today_clicked,
"on_combobox_changed": self.on_combobox_changed,
"on_add_clicked": self.on_add_clicked,
"on_edit_clicked": self.on_edit_clicked,
"on_remove_clicked": self.on_remove_clicked,
"on_next_clicked": self.on_next_clicked,
"on_previous_clicked": self.on_previous_clicked,
}
builder.connect_signals(handlers)
self.window = builder.get_object("window")
self.window.__init__()
self.window.set_title("GTG - Calendar View")
self.window.connect("destroy", Gtk.main_quit)
# DataStore object
self.ds = DataStore()
self.req = Requester(self.ds)
self.ds.populate() # hard-coded tasks
self.today_button = builder.get_object("today")
self.header = builder.get_object("header")
self.controller = Controller(self, self.req)
vbox = builder.get_object("vbox")
vbox.add(self.controller)
vbox.reorder_child(self.controller, 1)
self.current_view = None
self.combobox = builder.get_object("combobox")
self.combobox.set_active(2)
self.statusbar = builder.get_object("statusbar")
self.window.show_all()
def on_add_clicked(self, button=None, start_date=None, due_date=None):
"""
Adds a new task, with the help of a pop-up dialog
for entering the task title, start and due dates.
Redraw the calendar view after the changes.
"""
# only to make testing easier
if tests and not start_date and not due_date:
today = datetime.date.today()
start = random.choice(range(today.day, 31))
end = random.choice(range(start, 31))
start_date = str(today.year) + "-" + str(today.month) + "-" + str(start)
due_date = str(today.year) + "-" + str(today.month) + "-" + str(end)
####
dialog = TaskView(self.window, new=True)
dialog.set_task_title("My New Task")
if start_date:
dialog.set_start_date(start_date)
if due_date:
dialog.set_due_date(due_date)
response = dialog.run()
if response == Gtk.ResponseType.OK:
title = dialog.get_title()
start_date = dialog.get_start_date()
due_date = dialog.get_due_date()
color = random_color()
self.current_view.add_new_task(title, start_date, due_date, color)
self.statusbar.push(0, "Added task: %s" % title)
else:
self.statusbar.pop(0)
dialog.hide()
def on_edit_clicked(self, button=None, task_id=None):
"""
Edits the selected task, with the help of a pop-up dialog
for modifying the task title, start and due dates.
Redraw the calendar view after the changes.
"""
if not task_id:
task_id = self.current_view.get_selected_task()
task = self.req.get_task(task_id)
if task:
dialog = TaskView(self.window, task)
response = dialog.run()
if response == Gtk.ResponseType.OK:
title = dialog.get_title()
start_date = dialog.get_start_date()
due_date = dialog.get_due_date()
is_done = dialog.get_active()
self.current_view.edit_task(task.get_id(), title,
start_date, due_date, is_done)
self.statusbar.push(0, "Edited task: %s" % title)
else:
self.statusbar.pop(0)
dialog.hide()
def on_remove_clicked(self, button=None):
"""
Removes the selected task from the datastore and redraw the
calendar view.
"""
task = self.req.get_task(self.current_view.get_selected_task())
if task:
self.current_view.delete_task(task.get_id())
self.statusbar.push(0, "Deleted task: %s" % task.get_title())
else:
self.statusbar.pop(0)
def on_next_clicked(self, button, days=None):
""" Advances the dates being displayed by a given number of @days """
self.current_view.next(days)
self.content_update()
self.current_view.update()
def on_previous_clicked(self, button, days=None):
""" Regresses the dates being displayed by a given number of @days """
self.current_view.previous(days)
self.content_update()
self.current_view.update()
def on_today_clicked(self, button):
""" Show the day corresponding to today """
self.current_view.show_today()
self.content_update()
def on_combobox_changed(self, combo):
"""
User chose a combobox entry: change the view_type according to it
"""
view_type = combo.get_active_text()
self.controller.on_view_changed(view_type)
if self.current_view != self.controller.get_visible_view():
# diconnect signals from previous view
if self.current_view is not None:
self._disconnect_view_signals()
self.current_view = self.controller.get_visible_view()
# start listening signals from the new view
self._connect_view_signals()
self.content_update()
def on_dates_changed(self, widget=None):
""" Callback to update date-related objects in main window """
self.header.set_text(self.current_view.get_current_year())
self.today_button.set_sensitive(
not self.current_view.is_today_being_shown())
def content_update(self):
""" Performs all that is needed to update the content displayed """
self.on_dates_changed()
self.current_view.update()
def _connect_view_signals(self):
"""
Connect to signals emitted from current view to add/edit a task or when
dates displayed changed
"""
self.current_view.connect("on_edit_task", self.on_edit_clicked)
self.current_view.connect("on_add_task", self.on_add_clicked)
self.current_view.connect("dates-changed", self.on_dates_changed)
def _disconnect_view_signals(self):
"""
Disconnect signals emitted from current view to add/edit a task or
when dates displayed changed
"""
self.current_view.disconnect_by_func(self.on_edit_clicked)
self.current_view.disconnect_by_func(self.on_add_clicked)
self.current_view.disconnect_by_func(self.on_dates_changed)
def __init__(self):
self._tasks = {}
self.requester = Requester(self)
class DataStore(object):
def __init__(self):
self._tasks = {}
self.requester = Requester(self)
def has_task(self, tid):
if tid in self._tasks:
return True
return False
def get_requester(self):
return self.requester
def get_tasks_tree(self):
return self._tasks
def get_all_tasks(self):
""" Returns a list of strings: tasks ids """
return list(self._tasks.keys())
def get_task(self, tid):
if self.has_task(tid):
return self._tasks[tid]
else:
return None
def new_task(self):
tid = str(uuid.uuid4())
task = Task(tid, True)
self._tasks[tid] = task
return task
def push_task(self, task):
def adding(task):
self._tasks[task.get_id()] = task
if self.has_task(task.get_id()):
return False
else:
adding(task)
return True
def request_task_deletion(self, tid):
self.requester.delete_task(tid)
# if self.has_task(tid):
# del self._tasks[tid]
# return True
# else:
# return False
def populate(self):
# hard coded tasks to populate calendar view
# (title, start_date, due_date, done?, color)
today = datetime.date.today()
ex_tasks = [("task1", today, today, True, random_color()),
("task2", today + datetime.timedelta(days=5),
today + datetime.timedelta(days=5), False, random_color()),
("task3", today + datetime.timedelta(days=1),
today + datetime.timedelta(days=3), False, random_color()),
("task4", today + datetime.timedelta(days=3),
today + datetime.timedelta(days=4), True, random_color()),
("task5", today - datetime.timedelta(days=1),
today + datetime.timedelta(days=8), False, random_color()),
("task6: very long title",
today + datetime.timedelta(days=2),
today + datetime.timedelta(days=3), False, random_color()),
("task7", today + datetime.timedelta(days=5),
today + datetime.timedelta(days=15), False, random_color())
]
for i in range(0, len(ex_tasks)):
new_task = self.new_task()
new_task.set_title(ex_tasks[i][0])
new_task.set_start_date(ex_tasks[i][1])
new_task.set_due_date(ex_tasks[i][2])
if ex_tasks[i][3]:
new_task.set_status(Task.STA_DONE)
new_task.set_color(ex_tasks[i][4])
def get_random_task(self):
if self._tasks:
return random.choice(list(self._tasks.keys()))
return None
def main():
index_dict = {}
try:
pkl_file = open(indexer.Indexer.filename, "rb")
index_dict = pickle.load(pkl_file)
pkl_file.close()
except IOError:
print "Pickle file not found."
indx = indexer.Indexer(index_dict)
db_manager = dbmanager.dbmanager(DB_NAME)
logging.basicConfig(
filename=LOG_NAME, format="%(asctime)s:%(levelname)s:%(message)s", filemode="w", level=logging.WARN
)
frontier = ["http://www.theonion.com", "http://www.reddit.com", "https://en.wikipedia.org/wiki/Satire"]
visited = {}
domains = {}
db_visited = db_manager.get_visited()
db_frontier = db_manager.get_frontier()
frontier += db_frontier
# shuffle(frontier)
for url in db_visited:
print "Already visited: " + url
visited[url] = 1
current_threads = 0
threads = []
data = []
t_urls = []
for url in frontier:
if visited.get(url, None):
logging.info("Not requesting " + url + " because it has already been visited.")
continue
if domains.get(get_domain(url), 0) >= MAX_REQ_PER_DOMAIN:
logging.info("Not requesting " + url + " because max requests per domain has been exceeded.")
continue
if is_blacklisted(url):
logging.info("Not requesting " + url + " because it is blacklisted.")
continue
if current_threads < MAX_THREADS:
logging.info("Requesting " + url)
print "Requesting " + url + " as t=" + str(current_threads)
visited[url] = 1
urldom = get_domain(url)
if urldom in domains:
domains[urldom] += 1
else:
domains[urldom] = 1
d = []
data.append(d)
t_urls.append(url)
t = Requester(url, TIME_LIMIT, d, MAX_SIZE_BYTES)
t.start()
threads.append(t)
current_threads += 1
if (current_threads >= MAX_THREADS) or (url == frontier[-1]):
current_threads = 0
for t in threads:
t.join()
for i in range(len(t_urls)):
htmldata = ""
if data[i]:
htmldata = data[i][0]
db_manager.insert_visited(t_urls[i], len(htmldata))
page_urls = list(set(get_urls(t_urls[i], htmldata)))
indx.index_page(t_urls[i], htmldata)
db_manager.insert_frontier(page_urls, t_urls[i])
frontier += page_urls
output_pkl = open(indexer.Indexer.filename, "wb")
pickle.dump(indx.index, output_pkl)
output_pkl.close()
threads = []
data = []
t_urls = []
db_manager.close()
