def argument_value_convert(method, argument, value, rpc_version): """ Check and fix Transmission RPC issues with regards to methods, arguments and values. """ if method in ('torrent-add', 'torrent-get', 'torrent-set'): args = constants.TORRENT_ARGS[method[-3:]] elif method in ('session-get', 'session-set'): args = constants.SESSION_ARGS[method[-3:]] else: return ValueError('Method "{0}" not supported'.format(method)) if argument in args: info = args[argument] invalid_version = True while invalid_version: invalid_version = False replacement = None if rpc_version < info[1]: invalid_version = True replacement = info[3] if info[2] and info[2] <= rpc_version: invalid_version = True replacement = info[4] if invalid_version: if replacement: LOGGER.warning( 'Replacing requested argument "{0}" with "{1}".'.format(argument, replacement)) argument = replacement info = args[argument] else: raise ValueError( 'Method "{0}" Argument "{1}" does not exist in version {2:d}.'.format(method, argument, rpc_version)) return argument, TR_TYPE_MAP[info[0]](value) else: raise ValueError('Argument "%s" does not exists for method "%s".', (argument, method))
def insert_city(name, cursor, connection): """ :param name: cityName :param cursor: database cursor :param connection: database connection :return: inserted city's id both if already present or not """ try: cursor.execute("INSERT INTO city (cityName) " + "values ('{}')".format(name)) connection.commit() city_id = cursor.lastrowid LOGGER.debug("City {} inserted correctly, returning associated id {}".format(name, city_id)) return city_id except mysql.connector.IntegrityError: cursor.execute("SELECT idCity FROM City WHERE cityName = '{}'".format(name)) city_id = cursor.fetchone()[0] LOGGER.debug("City {} already present, returning associated id {}".format(name, city_id)) return city_id
def insert_user(author, link, cursor, connection): """ :param author: username :param link: link to facebook profile without standard prefix https://www.facebook.com :param cursor: database cursor :param connection: database connection :return: inserted user's id both if already present or not """ try: cursor.execute("INSERT INTO user (username, linkToProfile, alreadyVisited, sex, idCurrentCity) " + "values ('{}','{}', 0, NULL, NULL)".format(author, link)) connection.commit() user_id = cursor.lastrowid LOGGER.debug("User {} inserted correctly, returning associated id {}".format(author, user_id)) return cursor.lastrowid except mysql.connector.IntegrityError: cursor.execute("SELECT idUser FROM user WHERE linkToProfile = '{}'".format(link)) user_id = cursor.fetchone()[0] LOGGER.debug("User {} already present, returning associated id {} ".format(author, user_id)) return user_id
def insert_contact(user_id, contact, cursor, connection): """ :param user_id: :param contact: { "contact": "......", "type": "EMAIL" / "NUMBER" / "ecc" } :param connection: database connection :param cursor: database cursor """ try: cursor.execute("INSERT INTO contact (contact,idOfContactType) " + "values ('{}','{}')".format( contact["contact"], get_id_of_contact_type(contact["type"], cursor, connection) )) contact_id = cursor.lastrowid cursor.execute("INSERT INTO usercontacts (idContact,idUser) values ({},{})".format(contact_id, user_id)) LOGGER.debug("contact inserted correctly, returning associated id {}".format(contact_id)) connection.commit() except Exception as e: LOGGER.error("Error while inserting contact of user {} error {}".format(user_id, e))
def re_validate(self): """ checks that the last node in a random bucket is still alive and replace or delete it if it isn't """ while True: time.sleep(RE_VALIDATE_INTERVAL) # the last node in a random, non-empty bucket bi = 0 last = None idx_arr = [i for i in range(len(self.buckets))] random.shuffle(idx_arr) for bi in idx_arr: bucket = self.buckets[bi] if len(bucket.nodes) > 0: last = bucket.nodes.pop() break if last is not None: LOGGER.debug('{:5} revalidate {}'.format('', last)) # wait for a pong ret = self.server.ping(last).get() bucket = self.buckets[bi] if ret: # bump node bucket.nodes.insert(0, last) else: # pick a replacement if len(bucket.replace_cache) > 0: r = bucket.replace_cache.pop( random.randint(0, len(bucket.replace_cache) - 1)) if r: bucket.nodes.append(r)
def insert_location(location, cursor, connection): """ :param location: location name as string :param cursor: database cursor :param connection: database connection :return: inserted location's id both if already present or not """ if location == "NULL": return None try: cursor.execute("INSERT INTO location (locationName) values ('{}')".format(location)) connection.commit() location_id = cursor.lastrowid LOGGER.debug("Location {} inserted correctly, returning associated id ".format(location_id)) return cursor.lastrowid except mysql.connector.IntegrityError: cursor.execute("SELECT idLocation FROM location WHERE locationName = '{}'".format(location)) location_id = cursor.fetchone()[0] LOGGER.debug("Location {} already present, returning associated id {}".format(location, location_id)) return location_id
def ping(self, node, callback=None): """ send a ping request to the given node and return instantly invoke callback while reply arrives """ ping = PingNode(self.endpoint, node.endpoint, time.time() + K_EXPIRATION) message = self.wrap_packet(ping) msg_hash = message[:32] def reply_call(chunks): if chunks.pop().echo == msg_hash: if callback is not None: callback() return True ep = (node.endpoint.address.exploded, node.endpoint.udpPort) pending = self.add_pending(Pending(node, Pong.packet_type, reply_call)) self.sock.sendto(message, ep) LOGGER.info("{:5} {}@{}:{} (Ping)".format( '---->', binascii.hexlify(node.node_id)[:8], ep[0], ep[1])) return pending
def is_italian_location(location): location = location.lstrip().rstrip().replace(" ", "+") + "+wikipedia" LOGGER.debug("Testing location {}".format(location)) try: query = requests.get( "https://www.google.com/search?q={}".format(location)) wikipedia_page = requests.get( re.search('https://it.wikipedia.org/.+?(?=&)', query.text).group()) LOGGER.debug("wikipedia page {}".format(wikipedia_page)) except Exception as e: LOGGER.debug("Error while locating nation") LOGGER.debug("wikipedia page {}".format(wikipedia_page)) is_italian = ( '<span style="white-space:nowrap"><a href="/wiki/File:Flag_of_Italy.svg" class="image" title="Italia">' in wikipedia_page.text) LOGGER.debug("Location {} is italian? {}".format(location, is_italian)) return is_italian
def receive_find_neighbors(self, addr, pubkey, fn): remote_id = keccak256(pubkey) if time.time() - self.last_pong_received.get(remote_id, 0) > K_BOND_EXPIRATION: # lost origin or origin is off return target_id = keccak256(fn.target) closest = self.table.closest(target_id, BUCKET_SIZE) # sent neighbours in chunks ns = Neighbors([], time.time() + K_EXPIRATION) sent = False node_to = Node(EndPoint(addr[0], addr[1], addr[1]), pubkey) for c in closest: ns.nodes.append(c) if len(ns.nodes) == K_MAX_NEIGHBORS: self.send_sock(ns, node_to) LOGGER.info("{:5} {}@{}:{} {}".format( '---->', binascii.hexlify(node_to.node_id)[:8], addr[0], addr[1], ns)) ns.nodes = [] sent = True if len(ns.nodes) > 0 or not sent: self.send_sock(ns, node_to) LOGGER.info("{:5} {}@{}:{} {}".format( '---->', binascii.hexlify(node_to.node_id)[:8], addr[0], addr[1], ns))
def handle_reply(self, addr, pubkey, packet_type, packet, match_callback=None): remote_id = keccak256(pubkey) is_match = False for pending in self.pending_hold: if pending.is_alive and packet_type == pending.packet_type: if remote_id == pending.from_id: is_match = True pending.emit(packet) match_callback and match_callback() elif pending.ep is not None and pending.ep == addr: LOGGER.error('{:5} {}@{}:{} mismatch request {}'.format( '', binascii.hexlify(remote_id)[:8], addr[0], addr[1], binascii.hexlify(pending.from_id)[:8])) # is_match = True # pending.emit(packet) # match_callback and match_callback() # for bucket in self.table.buckets: # for node in bucket.nodes: # if node.node_id == pending.from_id: # node.set_pubkey(pubkey) if not is_match: LOGGER.warning('{:5} {}@{}:{} ({}) unsolicited response'.format( '<-//-', binascii.hexlify(remote_id)[:8], addr[0], addr[1], PACKET_TYPES.get(packet.packet_type)))
def check_citta(elem): citta = elem.find_elements_by_xpath(".//div[@class='_6a _6b']/span/a") city = "NULL" if len(citta) > 0: city = citta[0].text LOGGER.debug(city) return city
def get_id_of_contact_type(contact_type, cursor, connection): """ :param contact_type: contact type :param cursor: database cursor :param connection: connection cursor :return: id of contact type if found, creates one and returns it if not found """ try: cursor.execute("SELECT idContactType FROM contacttype WHERE contactType = '{}'".format(contact_type.upper())) contact_type_id = cursor.fetchone()[0] LOGGER.debug("returning id of contact type: {}".format(contact_type)) return contact_type_id except Exception as e: cursor.execute("INSERT INTO contacttype (contactType) " + "values ('{}')".format(contact_type)) connection.commit() contact_type_id = cursor.lastrowid LOGGER.debug("Contact type {} created correctly, returning associated id {}".format(contact_type , contact_type_id)) return contact_type_id
def _run(self): chunks = [] while self._box is not None: try: packet = self._box.get(timeout=self._timeout) chunks.append(packet) except Empty: hex_id = binascii.hexlify(self.from_id) LOGGER.warning("{:5} {}@{}:{} ({}) timeout".format( '<-//-', hex_id[:8], self.ep[0], self.ep[1], PACKET_TYPES.get(self._packet_type))) # timeout self._box = None return None except: # die self._box = None raise try: if self._callback(chunks): hex_id = binascii.hexlify(self.from_id) LOGGER.info("{:5} {}@{}:{} ({}) ok".format( '<----', hex_id[:8], self.ep[0], self.ep[1], PACKET_TYPES.get(self._packet_type))) # job done self._box = None return chunks except: # die self._box = None raise
def test_connection(): LOGGER.debug("Testing db connection") connection = get_db_connection(constants.DB_USER, constants.DB_PASSWORD, constants.DB_HOST, constants.DB_NAME) connection.close() LOGGER.debug("Connection established successfully")
def listen(self): LOGGER.info("{:5} listening...".format('')) while True: ready = select([self.sock], [], [], 1.0) if ready[0]: data, addr = self.sock.recvfrom(2048) # non-block data reading gevent.spawn(self.receive, data, addr)
def get_comments(element, tasto): comments = [] try: while True: actions = ActionChains(driver) time.sleep(0.5) if len(element.find_elements_by_xpath( ".//a[@class='_4sxc _42ft']")) == 0: break else: tasto = element.find_elements_by_xpath( ".//a[@class='_4sxc _42ft']") actions.move_to_element(tasto[0]).perform() tasto[0].click() element = element.find_element_by_xpath( selectors.get("comment_section")) element = element.find_elements_by_xpath(".//div[@class='_72vr']") cont = 0 cont_php = 0 for elem in element: try: cont += 1 href_commenti = elem.find_element_by_xpath( selectors.get("comment_author")).get_attribute('href') author = elem.find_element_by_xpath( selectors.get("comment_author")).text text = elem.find_element_by_xpath( selectors.get("comment_text")).text href_finale = href_account(href_commenti) if ".php" not in href_finale: comments.append({ "author": author, "text": text, "linkToProfile": href_finale }) else: cont_php += 1 except Exception as e: LOGGER.debug("{}".format(e)) print(str(cont) + "commenti") print(str(cont_php) + "profili.php") except Exception as e: LOGGER.debug("{}".format(e)) return comments
def insert_personal_data(profile_link, personal_data): """ :param profile_link: link to profile no standard prefix :param personal_data: { "sex": 1 if male 0 if female "NULL" if not known, "cityName": "...", "contacts": [ { "contact": "......", "type": "EMAIL"/"NUMBER" } ] "jobs": ["..."] } """ connection = None try: connection = get_db_connection(constants.DB_USER, constants.DB_PASSWORD, constants.DB_HOST, constants.DB_NAME) cursor = connection.cursor() user_id = get_user_id_from_link(profile_link, cursor) if personal_data["cityName"] != "NULL": city_id = insert_city( personal_data["cityName"].replace("'", " "), cursor, connection ) else: city_id = "NULL" update_user_query = "UPDATE user SET alreadyVisited = 1 , sex = {} , idCurrentCity = {} WHERE idUser = {}"\ .format(personal_data["sex"], city_id, user_id) cursor.execute(update_user_query) for contact in personal_data["contacts"]: insert_contact(user_id, contact.replace("'", " "), cursor, connection) for job in personal_data["jobs"]: insert_job(user_id, job.replace("'", " "), cursor, connection) connection.commit() except Exception as e: connection.rollback() LOGGER.error("Error while inserting personal data error: {}".format(e)) finally: connection.close()
def worked_day(self, day=date.today(), json_settings=DEFAULT_FACTORIAL_SETTINGS): """Mark today as worked day :param day: date to save the worked day, by default is today :param json_settings: string config filename """ with open(json_settings, 'r') as file: settings = json.load(file) work_settings_block = settings.get('work', {}) start_work = work_settings_block.get('start', '') end_work = work_settings_block.get('end', '') work_minutes_variation = work_settings_block.get( 'minutes_variation', 0) breaks = work_settings_block.get('breaks', []) already_work = self.get_day(year=day.year, month=day.month, day=day.day) if already_work: if work_settings_block.get('resave'): for worked_period in already_work: self.delete_worked_period(worked_period.get('id')) else: LOGGER.info('Day already sign') return add_worked_period_kwargs = { 'year': day.year, 'month': day.month, 'day': day.day, # Dynamic over loop fields 'start_hour': 0, 'start_minute': 0, 'end_hour': 0, 'end_minute': 0 } worked_periods = self.generate_worked_periods(start_work, end_work, work_minutes_variation, breaks) for worked_period in worked_periods: start_hour = worked_period.get('start_hour') start_minute = worked_period.get('start_minute') end_hour = worked_period.get('end_hour') end_minute = worked_period.get('end_minute') add_worked_period_kwargs.update({ 'start_hour': start_hour, 'start_minute': start_minute, 'end_hour': end_hour, 'end_minute': end_minute, }) if self.add_worked_period(**add_worked_period_kwargs): LOGGER.info( 'Saved worked period for the day {0:s} between {1:02d}:{2:02d} - {3:02d}:{4:02d}' .format(day.isoformat(), start_hour, start_minute, end_hour, end_minute))
def load(cls, data: str): LOGGER.info("[MSG-01] Parsing message...") card_str, _comma, rdata = data.partition(",") card_id = int(card_str) cp_cnt_str, _comma, rrdata = rdata.partition(",") cp_count = int(cp_cnt_str) val_list = rrdata.split(",") cp_list = val_list[0:cp_count] res_list = val_list[cp_count:] return cls(card_id, cp_list, res_list)
def check_lavori(elem): lavori = elem.find_elements_by_xpath( ".//li[@class='_43c8 _5f6p fbEditProfileViewExperience experience']") jobs = [] if len(lavori) > 0: for lavoro in lavori: job = lavoro.find_element_by_xpath( ".//div[@class='_2lzr _50f5 _50f7']/a").text jobs.append(job) LOGGER.debug("jobs") LOGGER.debug(jobs) return jobs
def check_sesso(elem): genere = elem.find_element_by_xpath( ".//li[@class='_3pw9 _2pi4 _2ge8 _3ms8']") sesso = genere.find_element_by_xpath( ".//div[@class='clearfix']/div/span").text LOGGER.debug(str(sesso) + " sesso") sex = "NULL" if len(sesso) > 0: if sesso == "Uomo": sex = 1 if sesso == "Donna": sex = 0 return sex
def load(data: str) -> None: fdata, _comma, check_sum = data.strip().partition("*") expected_sum = hex(int(f"0x{check_sum}", 16)) actual_sum = MessageReader.check_sum(fdata) if expected_sum == actual_sum: LOGGER.info("[READER] CheckSum Succeed!") msg_type, _comma, rdata = fdata.partition(",") return getattr(sys.modules[__name__], f"Message_{msg_type}").load(rdata) else: LOGGER.error( f"[READER] CheckSum Failed! Actual: {actual_sum}, Expected: {expected_sum}, Msg: {fdata}" )
def get_db_connection(user, password, host, database_name): """ :param user: :param password: :param host: :param database_name: :return: returns a connection to the db, note that it needs to be closed """ try: return mysql.connector.connect(user=user, password=password, host=host, database=database_name) except Exception as e: LOGGER.error("Error while creating database connection {}" .format(e))
def logout(self): """Logout invalidating that session, invalidating the cookie _factorial_session :return: bool """ response = self.session.delete(url=self.SESSION_URL) logout_correcty = response.status_code == http_client.NO_CONTENT LOGGER.info('Logout successfully {}'.format(logout_correcty)) self.session = requests.Session() path_file = os.path.join(self.SESSIONS_FOLDER, self.cookie_file) if os.path.exists(path_file): os.remove(path_file) logging.info('Logout: Removed cookies file') self.mates.clear() self.current_user = {} return logout_correcty
def __run_worker(self): LOGGER.info("[WORKER] Launched!") while (data := self.__queue.get()) != "kill": try: MessageReader.load(data) LOGGER.info(f"[WORKER] [{data}] successfully processed!") except Exception as exp: LOGGER.error("[WORKER] Data processing operation failed!") LOGGER.error(f"[WORKER] {exp}")
def get_users_not_visited(): """ :return: list of the profile's link of the users we haven't visited yet (the usual facebook prefix is omitted) """ connection = None try: connection = get_db_connection(constants.DB_USER, constants.DB_PASSWORD, constants.DB_HOST, constants.DB_NAME) cursor = connection.cursor() cursor.execute("SELECT linkToProfile FROM user WHERE alreadyVisited = 0") return cursor.fetchall() except Exception as e: LOGGER.error("Error while retrieving users not visited: {}".format(e)) finally: connection.close()
def add_worked_period(self, year, month, day, start_hour, start_minute, end_hour, end_minute): """Add the period as worked Example to create a worked period for the day 2019-07-31 from 7:30 to 15:30 - year 2019 - month 7 - day 31 - start_hour 7 - start_minute 30 - end_hour 15 - end_minute 30 :param year: integer :param month: integer :param day: integer :param start_hour: integer :param start_minute: integer :param end_hour: integer :param end_minute: integer :return bool: correctly saved """ # Check if are vacations calendar = self.get_calendar(year=year, month=month, is_leave=True) formatted_date = f'{year:04d}-{month:02d}-{day:02d}' for calendar_day in calendar: if calendar_day.get('date') == formatted_date: LOGGER.info( f"Can't sign today {formatted_date}, because are vacations" ) return False period = self.get_period(year=year, month=month) current_period = period[0] period_id = current_period['id'] payload = { 'clock_in': f'{start_hour}:{start_minute}', 'clock_out': f'{end_hour}:{end_minute}', 'day': day, 'period_id': period_id } response = self.session.post(self.SHIFT_URL, data=payload) self.check_status_code(response.status_code, http_client.CREATED) return True
def __init__(self, email, password, cookie_file=None): """Factorial client to automatically sign up the work :param email: (required) string, email to login on Factorial :param password: (required) string, password to login on Factorial :param cookie_file: (optional) string, file to save the cookies """ self.email = email self.password = password self.current_user = {} self.mates = [] self.session = requests.Session() # Be able to save the cookies on a file specified, or save each user on a different email for multi account self.cookie_file = cookie_file or hashlib.sha512( email.encode('utf-8')).hexdigest() cookie_path = os.path.join(self.SESSIONS_FOLDER, self.cookie_file) if os.path.exists(cookie_path): with open(cookie_path, "rb") as file: # TODO: Watch out the expiration of the cookie LOGGER.info('Getting the session from cookies files') self.session.cookies.update(pickle.load(file))
def find_neighbors(self, node, target_key): """ send a find neighbours request to the given node and waits until the node has sent up to k neighbours """ node_id = node.node_id if time.time() - self.last_ping_received.get(node_id, 0) > K_BOND_EXPIRATION: send_ping = self.ping(node) receive_ping = self.add_pending( Pending(node, PingNode.packet_type, lambda _: True)) # wait until endpoint proof is finished gevent.joinall([send_ping, receive_ping]) fn = FindNeighbors(target_key, time.time() + K_EXPIRATION) def reply_call(chunks): num_received = 0 for neighbors in chunks: num_received += len(neighbors.nodes) if num_received >= BUCKET_SIZE: return True pending = self.add_pending( Pending(node, Neighbors.packet_type, reply_call, timeout=2)) self.send_sock(fn, node) ep = (node.endpoint.address.exploded, node.endpoint.udpPort) LOGGER.info("{:5} {}@{}:{} (FN {})".format( '---->', binascii.hexlify(node.node_id)[:8], ep[0], ep[1], binascii.hexlify(keccak256(fn.target))[:8])) # block to wait for neighbours ret = pending.get() if ret: neighbor_nodes = [] for chunk in ret: for n in chunk.nodes: neighbor_nodes.append(n) return neighbor_nodes
def login(self): """Login on the factorial web :return: boolean if is logged in """ try: self.load_user_data() # Try to load the user info using the cookie, if can't login again using the username and password LOGGER.info('Already logged in, re-login is not needed') return True except UserNotLoggedIn: payload = { 'utf8': '✓', 'authenticity_token': self.generate_new_token(), 'user[email]': self.email, 'user[password]': self.password, 'user[remember_me]': "0", 'commit': 'Iniciar sesión' } response = self.session.post(url=self.SESSION_URL, data=payload) loggedin = response.status_code == http_client.CREATED if loggedin: LOGGER.info('Login successfully') # Load user data self.load_user_data() # Save the cookies if is logged in if not os.path.exists(self.SESSIONS_FOLDER): os.mkdir(self.SESSIONS_FOLDER) with open(os.path.join(self.SESSIONS_FOLDER, self.cookie_file), "wb") as file: pickle.dump(self.session.cookies, file) LOGGER.info('Sessions saved') return loggedin
def add_node(self, node): bucket = self.get_bucket(node) # exclude self if self.self_node.node_id == node.node_id: return # bucket contains the node, remove the old one, push the new one for n in list(bucket.nodes): if n.node_id == node.node_id: bucket.nodes.remove(n) bucket.nodes.insert(0, node) LOGGER.debug('{:5} bump {} in bucket #{}'.format( '', node, self.buckets.index(bucket))) return # bucket is full, push node to replace cache if len(bucket.nodes) >= BUCKET_SIZE: for rc in bucket.replace_cache: if rc.node_id == node.node_id: return push_node(bucket.replace_cache, node, BUCKET_SIZE) LOGGER.debug('{:5} push {} to replacement #{}'.format( '', node, self.buckets.index(bucket))) return # push node to bucket, delete node from replace cache push_node(bucket.nodes, node, BUCKET_SIZE) LOGGER.debug('{:5} push {} to bucket #{}'.format( '', node, self.buckets.index(bucket))) del_node(bucket.replace_cache, node) node.added_time = time.time()