def argument_value_convert(method, argument, value, rpc_version):
"""
Check and fix Transmission RPC issues with regards to methods, arguments and values.
"""
if method in ('torrent-add', 'torrent-get', 'torrent-set'):
args = constants.TORRENT_ARGS[method[-3:]]
elif method in ('session-get', 'session-set'):
args = constants.SESSION_ARGS[method[-3:]]
else:
return ValueError('Method "{0}" not supported'.format(method))
if argument in args:
info = args[argument]
invalid_version = True
while invalid_version:
invalid_version = False
replacement = None
if rpc_version < info[1]:
invalid_version = True
replacement = info[3]
if info[2] and info[2] <= rpc_version:
invalid_version = True
replacement = info[4]
if invalid_version:
if replacement:
LOGGER.warning(
'Replacing requested argument "{0}" with "{1}".'.format(argument, replacement))
argument = replacement
info = args[argument]
else:
raise ValueError(
'Method "{0}" Argument "{1}" does not exist in version {2:d}.'.format(method, argument, rpc_version))
return argument, TR_TYPE_MAP[info[0]](value)
else:
raise ValueError('Argument "%s" does not exists for method "%s".',
(argument, method))
def insert_city(name, cursor, connection):
"""
:param name: cityName
:param cursor: database cursor
:param connection: database connection
:return: inserted city's id both if already present or not
"""
try:
cursor.execute("INSERT INTO city (cityName) " +
"values ('{}')".format(name))
connection.commit()
city_id = cursor.lastrowid
LOGGER.debug("City {} inserted correctly, returning associated id {}".format(name, city_id))
return city_id
except mysql.connector.IntegrityError:
cursor.execute("SELECT idCity FROM City WHERE cityName = '{}'".format(name))
city_id = cursor.fetchone()[0]
LOGGER.debug("City {} already present, returning associated id {}".format(name, city_id))
return city_id
def insert_user(author, link, cursor, connection):
"""
:param author: username
:param link: link to facebook profile without standard prefix https://www.facebook.com
:param cursor: database cursor
:param connection: database connection
:return: inserted user's id both if already present or not
"""
try:
cursor.execute("INSERT INTO user (username, linkToProfile, alreadyVisited, sex, idCurrentCity) " +
"values ('{}','{}', 0, NULL, NULL)".format(author, link))
connection.commit()
user_id = cursor.lastrowid
LOGGER.debug("User {} inserted correctly, returning associated id {}".format(author, user_id))
return cursor.lastrowid
except mysql.connector.IntegrityError:
cursor.execute("SELECT idUser FROM user WHERE linkToProfile = '{}'".format(link))
user_id = cursor.fetchone()[0]
LOGGER.debug("User {} already present, returning associated id {} ".format(author, user_id))
return user_id
def insert_contact(user_id, contact, cursor, connection):
"""
:param user_id:
:param contact: {
"contact": "......",
"type": "EMAIL" / "NUMBER" / "ecc"
}
:param connection: database connection
:param cursor: database cursor
"""
try:
cursor.execute("INSERT INTO contact (contact,idOfContactType) " +
"values ('{}','{}')".format(
contact["contact"],
get_id_of_contact_type(contact["type"], cursor, connection)
))
contact_id = cursor.lastrowid
cursor.execute("INSERT INTO usercontacts (idContact,idUser) values ({},{})".format(contact_id, user_id))
LOGGER.debug("contact inserted correctly, returning associated id {}".format(contact_id))
connection.commit()
except Exception as e:
LOGGER.error("Error while inserting contact of user {} error {}".format(user_id, e))
def re_validate(self):
"""
checks that the last node in a random bucket is still alive
and replace or delete it if it isn't
"""
while True:
time.sleep(RE_VALIDATE_INTERVAL)
# the last node in a random, non-empty bucket
bi = 0
last = None
idx_arr = [i for i in range(len(self.buckets))]
random.shuffle(idx_arr)
for bi in idx_arr:
bucket = self.buckets[bi]
if len(bucket.nodes) > 0:
last = bucket.nodes.pop()
break
if last is not None:
LOGGER.debug('{:5} revalidate {}'.format('', last))
# wait for a pong
ret = self.server.ping(last).get()
bucket = self.buckets[bi]
if ret:
# bump node
bucket.nodes.insert(0, last)
else:
# pick a replacement
if len(bucket.replace_cache) > 0:
r = bucket.replace_cache.pop(
random.randint(0,
len(bucket.replace_cache) - 1))
if r:
bucket.nodes.append(r)
def insert_location(location, cursor, connection):
"""
:param location: location name as string
:param cursor: database cursor
:param connection: database connection
:return: inserted location's id both if already present or not
"""
if location == "NULL":
return None
try:
cursor.execute("INSERT INTO location (locationName) values ('{}')".format(location))
connection.commit()
location_id = cursor.lastrowid
LOGGER.debug("Location {} inserted correctly, returning associated id ".format(location_id))
return cursor.lastrowid
except mysql.connector.IntegrityError:
cursor.execute("SELECT idLocation FROM location WHERE locationName = '{}'".format(location))
location_id = cursor.fetchone()[0]
LOGGER.debug("Location {} already present, returning associated id {}".format(location, location_id))
return location_id
def ping(self, node, callback=None):
"""
send a ping request to the given node and return instantly
invoke callback while reply arrives
"""
ping = PingNode(self.endpoint, node.endpoint,
time.time() + K_EXPIRATION)
message = self.wrap_packet(ping)
msg_hash = message[:32]
def reply_call(chunks):
if chunks.pop().echo == msg_hash:
if callback is not None:
callback()
return True
ep = (node.endpoint.address.exploded, node.endpoint.udpPort)
pending = self.add_pending(Pending(node, Pong.packet_type, reply_call))
self.sock.sendto(message, ep)
LOGGER.info("{:5} {}@{}:{} (Ping)".format(
'---->',
binascii.hexlify(node.node_id)[:8], ep[0], ep[1]))
return pending
def is_italian_location(location):
location = location.lstrip().rstrip().replace(" ", "+") + "+wikipedia"
LOGGER.debug("Testing location {}".format(location))
try:
query = requests.get(
"https://www.google.com/search?q={}".format(location))
wikipedia_page = requests.get(
re.search('https://it.wikipedia.org/.+?(?=&)',
query.text).group())
LOGGER.debug("wikipedia page {}".format(wikipedia_page))
except Exception as e:
LOGGER.debug("Error while locating nation")
LOGGER.debug("wikipedia page {}".format(wikipedia_page))
is_italian = (
'<span style="white-space:nowrap"><a href="/wiki/File:Flag_of_Italy.svg" class="image" title="Italia">'
in wikipedia_page.text)
LOGGER.debug("Location {} is italian? {}".format(location, is_italian))
return is_italian
def receive_find_neighbors(self, addr, pubkey, fn):
remote_id = keccak256(pubkey)
if time.time() - self.last_pong_received.get(remote_id,
0) > K_BOND_EXPIRATION:
# lost origin or origin is off
return
target_id = keccak256(fn.target)
closest = self.table.closest(target_id, BUCKET_SIZE)
# sent neighbours in chunks
ns = Neighbors([], time.time() + K_EXPIRATION)
sent = False
node_to = Node(EndPoint(addr[0], addr[1], addr[1]), pubkey)
for c in closest:
ns.nodes.append(c)
if len(ns.nodes) == K_MAX_NEIGHBORS:
self.send_sock(ns, node_to)
LOGGER.info("{:5} {}@{}:{} {}".format(
'---->',
binascii.hexlify(node_to.node_id)[:8], addr[0], addr[1],
ns))
ns.nodes = []
sent = True
if len(ns.nodes) > 0 or not sent:
self.send_sock(ns, node_to)
LOGGER.info("{:5} {}@{}:{} {}".format(
'---->',
binascii.hexlify(node_to.node_id)[:8], addr[0], addr[1], ns))
def handle_reply(self,
addr,
pubkey,
packet_type,
packet,
match_callback=None):
remote_id = keccak256(pubkey)
is_match = False
for pending in self.pending_hold:
if pending.is_alive and packet_type == pending.packet_type:
if remote_id == pending.from_id:
is_match = True
pending.emit(packet)
match_callback and match_callback()
elif pending.ep is not None and pending.ep == addr:
LOGGER.error('{:5} {}@{}:{} mismatch request {}'.format(
'',
binascii.hexlify(remote_id)[:8], addr[0], addr[1],
binascii.hexlify(pending.from_id)[:8]))
# is_match = True
# pending.emit(packet)
# match_callback and match_callback()
# for bucket in self.table.buckets:
# for node in bucket.nodes:
# if node.node_id == pending.from_id:
# node.set_pubkey(pubkey)
if not is_match:
LOGGER.warning('{:5} {}@{}:{} ({}) unsolicited response'.format(
'<-//-',
binascii.hexlify(remote_id)[:8], addr[0], addr[1],
PACKET_TYPES.get(packet.packet_type)))
def check_citta(elem):
citta = elem.find_elements_by_xpath(".//div[@class='_6a _6b']/span/a")
city = "NULL"
if len(citta) > 0:
city = citta[0].text
LOGGER.debug(city)
return city
def get_id_of_contact_type(contact_type, cursor, connection):
"""
:param contact_type: contact type
:param cursor: database cursor
:param connection: connection cursor
:return: id of contact type if found, creates one and returns it if not found
"""
try:
cursor.execute("SELECT idContactType FROM contacttype WHERE contactType = '{}'".format(contact_type.upper()))
contact_type_id = cursor.fetchone()[0]
LOGGER.debug("returning id of contact type: {}".format(contact_type))
return contact_type_id
except Exception as e:
cursor.execute("INSERT INTO contacttype (contactType) " +
"values ('{}')".format(contact_type))
connection.commit()
contact_type_id = cursor.lastrowid
LOGGER.debug("Contact type {} created correctly, returning associated id {}".format(contact_type
, contact_type_id))
return contact_type_id
def _run(self):
chunks = []
while self._box is not None:
try:
packet = self._box.get(timeout=self._timeout)
chunks.append(packet)
except Empty:
hex_id = binascii.hexlify(self.from_id)
LOGGER.warning("{:5} {}@{}:{} ({}) timeout".format(
'<-//-', hex_id[:8], self.ep[0], self.ep[1],
PACKET_TYPES.get(self._packet_type)))
# timeout
self._box = None
return None
except:
# die
self._box = None
raise
try:
if self._callback(chunks):
hex_id = binascii.hexlify(self.from_id)
LOGGER.info("{:5} {}@{}:{} ({}) ok".format(
'<----', hex_id[:8], self.ep[0], self.ep[1],
PACKET_TYPES.get(self._packet_type)))
# job done
self._box = None
return chunks
except:
# die
self._box = None
raise
def test_connection():
LOGGER.debug("Testing db connection")
connection = get_db_connection(constants.DB_USER, constants.DB_PASSWORD, constants.DB_HOST, constants.DB_NAME)
connection.close()
LOGGER.debug("Connection established successfully")
def listen(self):
LOGGER.info("{:5} listening...".format(''))
while True:
ready = select([self.sock], [], [], 1.0)
if ready[0]:
data, addr = self.sock.recvfrom(2048)
# non-block data reading
gevent.spawn(self.receive, data, addr)
def get_comments(element, tasto):
comments = []
try:
while True:
actions = ActionChains(driver)
time.sleep(0.5)
if len(element.find_elements_by_xpath(
".//a[@class='_4sxc _42ft']")) == 0:
break
else:
tasto = element.find_elements_by_xpath(
".//a[@class='_4sxc _42ft']")
actions.move_to_element(tasto[0]).perform()
tasto[0].click()
element = element.find_element_by_xpath(
selectors.get("comment_section"))
element = element.find_elements_by_xpath(".//div[@class='_72vr']")
cont = 0
cont_php = 0
for elem in element:
try:
cont += 1
href_commenti = elem.find_element_by_xpath(
selectors.get("comment_author")).get_attribute('href')
author = elem.find_element_by_xpath(
selectors.get("comment_author")).text
text = elem.find_element_by_xpath(
selectors.get("comment_text")).text
href_finale = href_account(href_commenti)
if ".php" not in href_finale:
comments.append({
"author": author,
"text": text,
"linkToProfile": href_finale
})
else:
cont_php += 1
except Exception as e:
LOGGER.debug("{}".format(e))
print(str(cont) + "commenti")
print(str(cont_php) + "profili.php")
except Exception as e:
LOGGER.debug("{}".format(e))
return comments
def insert_personal_data(profile_link, personal_data):
"""
:param profile_link: link to profile no standard prefix
:param personal_data: {
"sex": 1 if male 0 if female "NULL" if not known,
"cityName": "...",
"contacts": [
{
"contact": "......",
"type": "EMAIL"/"NUMBER"
}
]
"jobs": ["..."]
}
"""
connection = None
try:
connection = get_db_connection(constants.DB_USER, constants.DB_PASSWORD, constants.DB_HOST, constants.DB_NAME)
cursor = connection.cursor()
user_id = get_user_id_from_link(profile_link, cursor)
if personal_data["cityName"] != "NULL":
city_id = insert_city(
personal_data["cityName"].replace("'", " "),
cursor,
connection
)
else:
city_id = "NULL"
update_user_query = "UPDATE user SET alreadyVisited = 1 , sex = {} , idCurrentCity = {} WHERE idUser = {}"\
.format(personal_data["sex"],
city_id,
user_id)
cursor.execute(update_user_query)
for contact in personal_data["contacts"]:
insert_contact(user_id, contact.replace("'", " "), cursor, connection)
for job in personal_data["jobs"]:
insert_job(user_id, job.replace("'", " "), cursor, connection)
connection.commit()
except Exception as e:
connection.rollback()
LOGGER.error("Error while inserting personal data error: {}".format(e))
finally:
connection.close()
def worked_day(self,
day=date.today(),
json_settings=DEFAULT_FACTORIAL_SETTINGS):
"""Mark today as worked day
:param day: date to save the worked day, by default is today
:param json_settings: string config filename
"""
with open(json_settings, 'r') as file:
settings = json.load(file)
work_settings_block = settings.get('work', {})
start_work = work_settings_block.get('start', '')
end_work = work_settings_block.get('end', '')
work_minutes_variation = work_settings_block.get(
'minutes_variation', 0)
breaks = work_settings_block.get('breaks', [])
already_work = self.get_day(year=day.year,
month=day.month,
day=day.day)
if already_work:
if work_settings_block.get('resave'):
for worked_period in already_work:
self.delete_worked_period(worked_period.get('id'))
else:
LOGGER.info('Day already sign')
return
add_worked_period_kwargs = {
'year': day.year,
'month': day.month,
'day': day.day,
# Dynamic over loop fields
'start_hour': 0,
'start_minute': 0,
'end_hour': 0,
'end_minute': 0
}
worked_periods = self.generate_worked_periods(start_work, end_work,
work_minutes_variation,
breaks)
for worked_period in worked_periods:
start_hour = worked_period.get('start_hour')
start_minute = worked_period.get('start_minute')
end_hour = worked_period.get('end_hour')
end_minute = worked_period.get('end_minute')
add_worked_period_kwargs.update({
'start_hour': start_hour,
'start_minute': start_minute,
'end_hour': end_hour,
'end_minute': end_minute,
})
if self.add_worked_period(**add_worked_period_kwargs):
LOGGER.info(
'Saved worked period for the day {0:s} between {1:02d}:{2:02d} - {3:02d}:{4:02d}'
.format(day.isoformat(), start_hour, start_minute,
end_hour, end_minute))
def load(cls, data: str):
LOGGER.info("[MSG-01] Parsing message...")
card_str, _comma, rdata = data.partition(",")
card_id = int(card_str)
cp_cnt_str, _comma, rrdata = rdata.partition(",")
cp_count = int(cp_cnt_str)
val_list = rrdata.split(",")
cp_list = val_list[0:cp_count]
res_list = val_list[cp_count:]
return cls(card_id, cp_list, res_list)
def check_lavori(elem):
lavori = elem.find_elements_by_xpath(
".//li[@class='_43c8 _5f6p fbEditProfileViewExperience experience']")
jobs = []
if len(lavori) > 0:
for lavoro in lavori:
job = lavoro.find_element_by_xpath(
".//div[@class='_2lzr _50f5 _50f7']/a").text
jobs.append(job)
LOGGER.debug("jobs")
LOGGER.debug(jobs)
return jobs
def check_sesso(elem):
genere = elem.find_element_by_xpath(
".//li[@class='_3pw9 _2pi4 _2ge8 _3ms8']")
sesso = genere.find_element_by_xpath(
".//div[@class='clearfix']/div/span").text
LOGGER.debug(str(sesso) + " sesso")
sex = "NULL"
if len(sesso) > 0:
if sesso == "Uomo":
sex = 1
if sesso == "Donna":
sex = 0
return sex
def load(data: str) -> None:
fdata, _comma, check_sum = data.strip().partition("*")
expected_sum = hex(int(f"0x{check_sum}", 16))
actual_sum = MessageReader.check_sum(fdata)
if expected_sum == actual_sum:
LOGGER.info("[READER] CheckSum Succeed!")
msg_type, _comma, rdata = fdata.partition(",")
return getattr(sys.modules[__name__],
f"Message_{msg_type}").load(rdata)
else:
LOGGER.error(
f"[READER] CheckSum Failed! Actual: {actual_sum}, Expected: {expected_sum}, Msg: {fdata}"
)
def get_db_connection(user, password, host, database_name):
"""
:param user:
:param password:
:param host:
:param database_name:
:return: returns a connection to the db, note that it needs to be closed
"""
try:
return mysql.connector.connect(user=user, password=password, host=host, database=database_name)
except Exception as e:
LOGGER.error("Error while creating database connection {}" .format(e))
def logout(self):
"""Logout invalidating that session, invalidating the cookie _factorial_session
:return: bool
"""
response = self.session.delete(url=self.SESSION_URL)
logout_correcty = response.status_code == http_client.NO_CONTENT
LOGGER.info('Logout successfully {}'.format(logout_correcty))
self.session = requests.Session()
path_file = os.path.join(self.SESSIONS_FOLDER, self.cookie_file)
if os.path.exists(path_file):
os.remove(path_file)
logging.info('Logout: Removed cookies file')
self.mates.clear()
self.current_user = {}
return logout_correcty
def __run_worker(self):
LOGGER.info("[WORKER] Launched!")
while (data := self.__queue.get()) != "kill":
try:
MessageReader.load(data)
LOGGER.info(f"[WORKER] [{data}] successfully processed!")
except Exception as exp:
LOGGER.error("[WORKER] Data processing operation failed!")
LOGGER.error(f"[WORKER] {exp}")
def get_users_not_visited():
"""
:return: list of the profile's link of the users we haven't visited yet (the usual facebook prefix is omitted)
"""
connection = None
try:
connection = get_db_connection(constants.DB_USER, constants.DB_PASSWORD, constants.DB_HOST, constants.DB_NAME)
cursor = connection.cursor()
cursor.execute("SELECT linkToProfile FROM user WHERE alreadyVisited = 0")
return cursor.fetchall()
except Exception as e:
LOGGER.error("Error while retrieving users not visited: {}".format(e))
finally:
connection.close()
def add_worked_period(self, year, month, day, start_hour, start_minute,
end_hour, end_minute):
"""Add the period as worked
Example to create a worked period for the day 2019-07-31 from 7:30 to 15:30
- year 2019
- month 7
- day 31
- start_hour 7
- start_minute 30
- end_hour 15
- end_minute 30
:param year: integer
:param month: integer
:param day: integer
:param start_hour: integer
:param start_minute: integer
:param end_hour: integer
:param end_minute: integer
:return bool: correctly saved
"""
# Check if are vacations
calendar = self.get_calendar(year=year, month=month, is_leave=True)
formatted_date = f'{year:04d}-{month:02d}-{day:02d}'
for calendar_day in calendar:
if calendar_day.get('date') == formatted_date:
LOGGER.info(
f"Can't sign today {formatted_date}, because are vacations"
)
return False
period = self.get_period(year=year, month=month)
current_period = period[0]
period_id = current_period['id']
payload = {
'clock_in': f'{start_hour}:{start_minute}',
'clock_out': f'{end_hour}:{end_minute}',
'day': day,
'period_id': period_id
}
response = self.session.post(self.SHIFT_URL, data=payload)
self.check_status_code(response.status_code, http_client.CREATED)
return True
def __init__(self, email, password, cookie_file=None):
"""Factorial client to automatically sign up the work
:param email: (required) string, email to login on Factorial
:param password: (required) string, password to login on Factorial
:param cookie_file: (optional) string, file to save the cookies
"""
self.email = email
self.password = password
self.current_user = {}
self.mates = []
self.session = requests.Session()
# Be able to save the cookies on a file specified, or save each user on a different email for multi account
self.cookie_file = cookie_file or hashlib.sha512(
email.encode('utf-8')).hexdigest()
cookie_path = os.path.join(self.SESSIONS_FOLDER, self.cookie_file)
if os.path.exists(cookie_path):
with open(cookie_path, "rb") as file:
# TODO: Watch out the expiration of the cookie
LOGGER.info('Getting the session from cookies files')
self.session.cookies.update(pickle.load(file))
def find_neighbors(self, node, target_key):
"""
send a find neighbours request to the given node and
waits until the node has sent up to k neighbours
"""
node_id = node.node_id
if time.time() - self.last_ping_received.get(node_id,
0) > K_BOND_EXPIRATION:
send_ping = self.ping(node)
receive_ping = self.add_pending(
Pending(node, PingNode.packet_type, lambda _: True))
# wait until endpoint proof is finished
gevent.joinall([send_ping, receive_ping])
fn = FindNeighbors(target_key, time.time() + K_EXPIRATION)
def reply_call(chunks):
num_received = 0
for neighbors in chunks:
num_received += len(neighbors.nodes)
if num_received >= BUCKET_SIZE:
return True
pending = self.add_pending(
Pending(node, Neighbors.packet_type, reply_call, timeout=2))
self.send_sock(fn, node)
ep = (node.endpoint.address.exploded, node.endpoint.udpPort)
LOGGER.info("{:5} {}@{}:{} (FN {})".format(
'---->',
binascii.hexlify(node.node_id)[:8], ep[0], ep[1],
binascii.hexlify(keccak256(fn.target))[:8]))
# block to wait for neighbours
ret = pending.get()
if ret:
neighbor_nodes = []
for chunk in ret:
for n in chunk.nodes:
neighbor_nodes.append(n)
return neighbor_nodes
def login(self):
"""Login on the factorial web
:return: boolean if is logged in
"""
try:
self.load_user_data()
# Try to load the user info using the cookie, if can't login again using the username and password
LOGGER.info('Already logged in, re-login is not needed')
return True
except UserNotLoggedIn:
payload = {
'utf8': '✓',
'authenticity_token': self.generate_new_token(),
'user[email]': self.email,
'user[password]': self.password,
'user[remember_me]': "0",
'commit': 'Iniciar sesión'
}
response = self.session.post(url=self.SESSION_URL, data=payload)
loggedin = response.status_code == http_client.CREATED
if loggedin:
LOGGER.info('Login successfully')
# Load user data
self.load_user_data()
# Save the cookies if is logged in
if not os.path.exists(self.SESSIONS_FOLDER):
os.mkdir(self.SESSIONS_FOLDER)
with open(os.path.join(self.SESSIONS_FOLDER, self.cookie_file),
"wb") as file:
pickle.dump(self.session.cookies, file)
LOGGER.info('Sessions saved')
return loggedin
def add_node(self, node):
bucket = self.get_bucket(node)
# exclude self
if self.self_node.node_id == node.node_id:
return
# bucket contains the node, remove the old one, push the new one
for n in list(bucket.nodes):
if n.node_id == node.node_id:
bucket.nodes.remove(n)
bucket.nodes.insert(0, node)
LOGGER.debug('{:5} bump {} in bucket #{}'.format(
'', node, self.buckets.index(bucket)))
return
# bucket is full, push node to replace cache
if len(bucket.nodes) >= BUCKET_SIZE:
for rc in bucket.replace_cache:
if rc.node_id == node.node_id:
return
push_node(bucket.replace_cache, node, BUCKET_SIZE)
LOGGER.debug('{:5} push {} to replacement #{}'.format(
'', node, self.buckets.index(bucket)))
return
# push node to bucket, delete node from replace cache
push_node(bucket.nodes, node, BUCKET_SIZE)
LOGGER.debug('{:5} push {} to bucket #{}'.format(
'', node, self.buckets.index(bucket)))
del_node(bucket.replace_cache, node)
node.added_time = time.time()
