def check_addr(self, domain, addr):
dom = self.db.FTN_domains[domain]
try:
return get_addr_id(self.db, dom, addr)
except FTNNoAddressInBase:
self.add_addr(dom, addr)
return get_addr_id(self.db, dom, addr)
def reset_subscription(self, target_domain, target_addr, subscriber_addr, timestamp):
target=get_addr_id(self.db, self.db.FTN_domains[target_domain], target_addr)
subscriber=get_addr_id(self.db, self.db.FTN_domains["node"], subscriber_addr)
reset_id=self.db.prepare("select min(id) from messages where destination=$1 and receivedtimestamp>=$2::text::timestamptz").first(target, timestamp)
if reset_id is None:
return "no messages after specified date"
try:
self.db.prepare("update subscriptions set lastsent=$3 where subscriber=$1 and target=$2")(subscriber, target, reset_id)
return "reset to %d"%reset_id
except:
print ("Exception in reset_subscription")
traceback.print_exc()
return "failed"
def restore_fechoes():
base_dir = ftnconfig.DOUTBOUND
for addr in os.listdir(base_dir):
addr_dir = os.path.join(base_dir, addr)
for f in os.listdir(addr_dir):
fname = os.path.join(addr_dir, f)
if os.path.islink(fname):
target = os.readlink(fname)
if os.path.exists(target):
print("good link")
continue
target_dir, target_file = os.path.split(target)
_, fecho = os.path.split(target_dir)
dest_id = ftnconfig.get_addr_id(db, db.FTN_domains["fileecho"],
fecho.upper())
newest_post = db.prepare(
"select max(post_time) from file_post where destination=$1 and filename ilike $2"
).first(dest_id, target_file)
file_id = db.prepare(
"select filedata from file_post where destination=$1 and filename ilike $2 and post_time=$3"
).first(dest_id, target_file, newest_post)
print(addr, f, fecho, dest_id, target_file, file_id)
tmpf = fname + "_"
tmpfd = open(tmpf, "wb")
save(db, file_id, tmpfd)
tmpfd.close()
os.rename(fname, fname + "--badlink")
os.rename(tmpf, fname)
def restore_fechoes():
base_dir = ftnconfig.DOUTBOUND
for addr in os.listdir(base_dir):
addr_dir = os.path.join(base_dir, addr)
for f in os.listdir(addr_dir):
fname=os.path.join(addr_dir, f)
if os.path.islink(fname):
target = os.readlink(fname)
if os.path.exists(target):
print ("good link")
continue
target_dir, target_file = os.path.split(target)
_, fecho = os.path.split(target_dir)
dest_id = ftnconfig.get_addr_id(db, db.FTN_domains["fileecho"], fecho.upper())
newest_post = db.prepare("select max(post_time) from file_post where destination=$1 and filename ilike $2").first(dest_id, target_file)
file_id = db.prepare("select filedata from file_post where destination=$1 and filename ilike $2 and post_time=$3").first(dest_id, target_file, newest_post)
print (addr, f, fecho, dest_id, target_file, file_id)
tmpf = fname+"_"
tmpfd=open(tmpf, "wb")
save(db, file_id, tmpfd)
tmpfd.close()
os.rename(fname, fname+"--badlink")
os.rename(tmpf, fname)
def remove_subscription(self, target_domain, target_addr, subscriber_addr):
target=get_addr_id(self.db, self.db.FTN_domains[target_domain], target_addr)
subscriber=get_addr_id(self.db, self.db.FTN_domains["node"], subscriber_addr)
check = self.db.prepare("select vital, lastsent from subscriptions where target=$1 and subscriber=$2")(target, subscriber)
if len(check):
if check[0][0]:
print ("removing vital subscription")
lastsent = check[0][1]
self.save_watermark(target, lastsent)
else:
return "not subscribed"
op=self.db.prepare("delete from subscriptions where target=$1 and subscriber=$2")
op(target, subscriber)
return "unsubscribed"
def check_link(db, address, password, forrobots):
"returns local address that is linked with remote address and verified with specified password"
# returns link_id, addr_id or None, myaddr_id or None
matching = []
link_exists = False
password = password or ""
for addr_id, myaddr_id, authinfo, link_id in db.prepare(
"select a.id, l.my, l.authentication, l.id "
"from links l, addresses a "
"where l.address=a.id and a.domain=$1 and a.text=$2")(
db.FTN_domains["node"], address):
link_exists = True
pw = ""
if forrobots:
if authinfo.find("RobotsPassword") is not None:
pw = authinfo.find("RobotsPassword").text
else:
if authinfo.find("ConnectPassword") is not None:
pw = authinfo.find("ConnectPassword").text
if check_pw(pw, password):
matching.append((link_id, addr_id, myaddr_id))
if len(matching) > 2:
raise Exception(
"two links for %s with same local address and password in database"
% address)
if link_exists:
if len(matching) == 0:
print("no match for specified password") # log bad password
return None, None, None
# return None addr_id as remote link cannot prove address validity
return matching[0][0], matching[0][1], matching[0][
2] # checked link with known local address
return None, \
db.prepare("select id from addresses where domain=$1 and text=$2").first(db.FTN_domains["node"], address), \
ftnconfig.get_addr_id(db, db.FTN_domains["node"], ftnconfig.ADDRESS) # no link, use default address
def check_link(db, address, password, forrobots):
"returns local address that is linked with remote address and verified with specified password"
# returns link_id, addr_id or None, myaddr_id or None
matching = []
link_exists = False
password = password or ""
for addr_id, myaddr_id, authinfo, link_id in db.prepare(
"select a.id, l.my, l.authentication, l.id "
"from links l, addresses a "
"where l.address=a.id and a.domain=$1 and a.text=$2")(db.FTN_domains["node"], address):
link_exists = True
pw = ""
if forrobots:
if authinfo.find("RobotsPassword") is not None:
pw = authinfo.find("RobotsPassword").text
else:
if authinfo.find("ConnectPassword") is not None:
pw = authinfo.find("ConnectPassword").text
if check_pw(pw, password):
matching.append((link_id, addr_id, myaddr_id))
if len(matching)>2:
raise Exception("two links for %s with same local address and password in database"%address)
if link_exists:
if len(matching)==0:
print ("no match for specified password") # log bad password
return None, None, None
# return None addr_id as remote link cannot prove address validity
return matching[0][0], matching[0][1], matching[0][2] # checked link with known local address
return None, \
db.prepare("select id from addresses where domain=$1 and text=$2").first(db.FTN_domains["node"], address), \
ftnconfig.get_addr_id(db, db.FTN_domains["node"], ftnconfig.ADDRESS) # no link, use default address
def add_subscription(self, vital, target_domain, target_addr, subscriber_addr, start=None):
""" vital: True, False - set new value; None - leave old or add False """
target=get_addr_id(self.db, self.db.FTN_domains[target_domain], target_addr)
subscriber=self.check_addr("node", subscriber_addr)
if start is None:
start=self.db.prepare("select max(id) from messages").first()
check = self.db.prepare("select id, vital, lastsent from subscriptions where target=$1 and subscriber=$2")(target, subscriber)
if len(check):
print ("#add_subscription: subscription exists")
oldid, oldvital, lastsent = check[0]
if vital is None or vital==oldvital:
print ("#add_subscription: nothing changed")
return "already subscribed"
if vital:
print ("#add_subscription: convert to vital")
self.db.prepare("update subscriptions set vital=true where id=$1")(oldid)
self.remove_watermark(target)
return "converted to vital"
else:
print ("#add_subscription: convert to non-vital")
self.save_watermark(target, lastsent)
self.db.prepare("update subscriptions set vital=false where id=$1")(oldid)
return "converted to non-vital"
else:
print ("#add_subscription: new subscription")
if vital:
start = self.get_watermark(target)
self.remove_watermark(target)
op=self.db.prepare("insert into subscriptions (vital, target, subscriber, lastsent) values ($1, $2, $3, $4)")(vital or False, target, subscriber, start)
return "subscribed from %d"%start
def save_message(self, sender, recipient, msgid, header, body, origcharset, recvfrom, processed=0, bulkload=False):
"""import msg as part of transaction.
if msg is correct then it is stored in base.
if msg fails validation then it will be saved in bad messages' directory """
if (len(body)+len(repr(header))) > MSGSIZELIMIT:
raise FTNExcessiveMessageSize(len(body)+len(repr(header)), MSGSIZELIMIT)
origdomname, origaddr = sender
destdomname, destaddr = recipient
#origdom=self.domains[origdomname]
destdom=self.domains[destdomname]
#print "-"*79
#sys.stdout.write(body.encode("utf-8"))
#print "="*79
# raise Exception("Verify senderaddress and destination address are listed. (area autocreate, destination is listed, ...)")
origid = self.check_addr(origdomname, origaddr) # allow autocreate for source
destid = self.check_addr(destdomname, destaddr) # check node in nodelist and area exists
# no autocreate. only on request and if exists in uplink lists
# database must have trigger to check address to be created for nodelist and area for echolist and uplinks
if recvfrom:
recvfrom_id = get_addr_id(self.db, self.FIDOADDR, recvfrom)
else:
recvfrom_id = None
if not bulkload:
# check domain's "verifysubscription" and if true refuse if not subscribed
r=self.db.prepare("select verifysubscriptions from domains where id=$1")(destdom)
if len(r) == 0:
raise Exception("invalid domain after checkaddress ??? %d"%destdom)
verify_subscription = r[0][0]
if verify_subscription:
# check if message received from subscribed to the destination's address
r=self.db.prepare("select count(id) from subscriptions where subscriber=$1 and target=$2")(recvfrom_id, destid)
# flaw: allow to flood with messages to non-existent addresses. should be checked with trigger in database
# (allow create node/point only if nodelisted, area only if listed on bone/uplinks)
if r[0][0]==0:
raise FTNNotSubscribed("%d/%s (id=%d)"%(self.FIDOADDR, recvfrom, recvfrom_id), "%d/%s (id=%d)"%(destdom, destaddr, destid))
print("posting allowed for %d (%d/%s) to %d/%s"%(recvfrom_id, self.FIDOADDR, recvfrom, destdom, destaddr))
if len(self.db.prepare("select id from messages where msgid=$1")(msgid)):
raise FTNDupMSGID(msgid)
if not self.Q_msginsert:
self.Q_msginsert = self.db.prepare("insert into messages (source, destination, msgid, header, body, origcharset, processed, receivedfrom, receivedtimestamp)"
" values ($1, $2, $3, $4, $5, $6, $7, $8, $9) returning id")
if not self.Q_update_addr_msg:
self.Q_update_addr_msg = self.db.prepare("update addresses set last=$2 where id=$1")
with postgresql.alock.ExclusiveLock(self.db, IMPORTLOCK): # lock per all table messages
# begin insert-lock
# guarantees that for greater id greater timestamp
timestamp = datetime.datetime.now(datetime.timezone.utc)
print ("Transaction state", self.x.state, str(timestamp))
new_msg=self.Q_msginsert(origid, destid, msgid, header, body, origcharset, processed, recvfrom_id, timestamp)[0][0]
print ("insert msg #", new_msg, "to address", destid)
self.Q_update_addr_msg(destid, new_msg)
# end insert-lock
self.last_message_for_address[destid] = new_msg
self.poller.add_one(destid)
#!/usr/local/bin/python3 -bb import ftnconfig import ftnexport db=ftnconfig.connectdb() echoes = ftnexport.get_all_targets(db, "echo") for echo in echoes: addr=ftnconfig.get_addr_id(db, db.FTN_domains["echo"], echo) print (echo, ftnexport.count_messages_to(db, addr))
if authinfo.find("RobotsPassword") is not None:
pw = authinfo.find("RobotsPassword").text
else:
if authinfo.find("ConnectPassword") is not None:
pw = authinfo.find("ConnectPassword").text
return my_id, pw
# Tests:
# check valid link
# check existing link with bad password
# check non-existent link with known address
# check non-existent link with UNknown address
# may_post
# subscribed link
# unsubscribed link
# link with autocreate to existing but not subscribed echo
# link without autocreate to non-existent echo
if __name__=="__main__":
db = ftnconfig.connectdb()
print("y",may_post(db, ftnconfig.get_addr_id(db, db.FTN_domains["node"], "2:5020/12000.1"), ("echo", "FLUID.LOCAL")))
print("n",may_post(db, ftnconfig.get_addr_id(db, db.FTN_domains["node"], "2:5020/4441"), ("echo", "FLUID.LOCAL")))
print("y",may_post(db, ftnconfig.get_addr_id(db, db.FTN_domains["node"], "2:5020/4441"), ("echo", "FLUID.KINO")))
print("y", check_link(db, "2:5020/12000.1", "<real passw>", False))
print("n", check_link(db, "2:5020/12000.1", "sfefe", False))
print("ok", check_link(db, "2:5030/585", "", False))
print("ok", check_link(db, "2:5011/1122", "", False))
sess.send_message(my_addr, ftnconfig.SYSOP, ("node", subscriber), robot, None, pw, "+"+area, sendmode="direct")
else:
print ("local subscription")
elif cmd == "query": # send areafix request
sess.send_message(my_addr, ftnconfig.SYSOP, ("node", subscriber), robot, None, pw, "%QUERY", sendmode="direct")
elif cmd == "list": # send areafix request
sess.send_message(my_addr, ftnconfig.SYSOP, ("node", subscriber), robot, None, pw, "%LIST", sendmode="direct")
elif cmd == "show": # show subscribed areas
if area==".": # show for link
for x in ftnexport.get_node_subscriptions(db, subscriber, domain):
print (x)
else: # show from area
tid = ftnconfig.get_addr_id(db, db.FTN_domains[domain], area)
print (tid)
for aid, vital, level in ftnexport.get_subscribers(db, tid):
print (ftnconfig.get_addr(db, aid))
elif cmd == "showuplink": # show areas subsribed as vital
if area==".": # show for link
for x in ftnexport.get_node_subscriptions(db, subscriber, domain, asuplink=True):
print (x)
else: # show from area
tid = ftnconfig.get_addr_id(db, db.FTN_domains[domain], area)
print (tid)
for x in ftnexport.get_subscribers(db, tid, True):
print (x)
elif cmd == "verifyuplink":
def count_messages_to(db, address):
return int(db.prepare("select count(*) from messages where destination=$1")(address)[0][0])
def get_matching_targets(db, targetdomain, mask):
mask = mask.replace("+", "++").replace("%", "+%").replace("_", "+_").replace("*", "%").replace("?", "_")
return [x[0] for x in db.prepare("select t.text from addresses t where t.domain=$1 and t.text like $2 escape '+'")\
(db.FTN_domains[targetdomain], mask)]
def get_subnodes(db, addr):
return [x[0] for x in db.prepare("select s.text from addresses n, addresses s where s.group=n.id and n.text=$1 and n.domain=$2")(addr, db.FTN_domains["node"])]
def get_supernode(db, addr):
r = db.prepare("select s.text from addresses n, addresses s where s.id=n.group and n.text=$1 and n.domain=$2")(addr, db.FTN_domains["node"])
if len(r):
return r[0][0]
else:
return None
if __name__ == "__main__":
import ftnconfig
# for s in get_node_subscriptions(connectdb(), "2:5020/4441", "echo"):
# if s.find("TEST")!=-1:
# print (s)
db = ftnconfig.connectdb()
print (get_subnodes(db, '2:5020/12000'))
for m in get_subscriber_messages_n_loopy(db, ftnconfig.get_addr_id(db, db.FTN_domains["node"], "2:5020/715"), 1):
print (m)
#!/usr/local/bin/python3 -bb
import ftnconfig
import sys
link_address = sys.argv[1]
print("pq://*****:*****@host/database")
db = ftnconfig.connectdb(
input("enter connection string for admin connection: "))
addr_id = ftnconfig.get_addr_id(db, db.FTN_domains["node"], link_address)
link_id = ftnconfig.get_link_id(db, link_address)
print(addr_id, link_id)
print(
db.prepare(
"select t.domain, t.text from subscriptions s, addresses t where s.subscriber=$1 and t.id=s.target"
)(addr_id))
assert (input("enter 'yes' to confirm: ") == "yes")
db.prepare("delete from subscriptions where subscriber=$1")(addr_id)
db.prepare("delete from links where id=$1")(link_id)
#!/usr/local/bin/python3 -bb
import re
import tempfile
import os
import ast
import xml.etree.ElementTree
import ftnconfig
import ftnexport
import ftnimport
import sys
import getopt
db = ftnconfig.connectdb()
my_id = ftnconfig.get_addr_id(db, db.FTN_domains["node"], ftnconfig.ADDRESS)
RE_s = re.compile("\s+")
RE_quote = re.compile("\s*(\S{0,2}?)(\>+)\s*(.*)$")
MAXW = 79
def quote(qname, block):
if len(block) and block[-1].startswith(" * "):
block.pop(-1)
if len(block) and block[-1].startswith("---"):
block.pop(-1)
if len(block) and block[-1].startswith("..."):
block.pop(-1)
while len(block) and block[-1] == "":
block.pop(-1)
#!/usr/local/bin/python3 -bb
import re
import tempfile
import os
import ast
import xml.etree.ElementTree
import ftnconfig
import ftnexport
import ftnimport
import sys
import getopt
db=ftnconfig.connectdb()
my_id = ftnconfig.get_addr_id (db, db.FTN_domains["node"], ftnconfig.ADDRESS)
RE_s = re.compile("\s+")
RE_quote = re.compile("\s*(\S{0,2}?)(\>+)\s*(.*)$")
MAXW=79
def quote(qname, block):
if len(block) and block[-1].startswith(" * "):
block.pop(-1)
if len(block) and block[-1].startswith("---"):
block.pop(-1)
if len(block) and block[-1].startswith("..."):
block.pop(-1)
while len(block) and block[-1]=="":
block.pop(-1)
while len(block) and block[0]=="":
#!/usr/local/bin/python3 -bb
import ftnconfig
import ftnexport
db = ftnconfig.connectdb()
echoes = ftnexport.get_all_targets(db, "echo")
for echo in echoes:
addr = ftnconfig.get_addr_id(db, db.FTN_domains["echo"], echo)
print(echo, ftnexport.count_messages_to(db, addr))
# check non-existent link with known address
# check non-existent link with UNknown address
# may_post
# subscribed link
# unsubscribed link
# link with autocreate to existing but not subscribed echo
# link without autocreate to non-existent echo
if __name__ == "__main__":
db = ftnconfig.connectdb()
print(
"y",
may_post(
db,
ftnconfig.get_addr_id(db, db.FTN_domains["node"],
"2:5020/12000.1"), ("echo", "FLUID.LOCAL")))
print(
"n",
may_post(
db, ftnconfig.get_addr_id(db, db.FTN_domains["node"],
"2:5020/4441"), ("echo", "FLUID.LOCAL")))
print(
"y",
may_post(
db, ftnconfig.get_addr_id(db, db.FTN_domains["node"],
"2:5020/4441"), ("echo", "FLUID.KINO")))
print("y", check_link(db, "2:5020/12000.1", "<real passw>", False))
print("n", check_link(db, "2:5020/12000.1", "sfefe", False))
print("ok", check_link(db, "2:5030/585", "", False))
print("ok", check_link(db, "2:5011/1122", "", False))
#!/usr/local/bin/python3
link='2:5020/1042'
domain = "fileecho"
import ftnconfig
import ftnimport
robot = ftnconfig.robotnames[domain]
db=ftnconfig.connectdb()
lid = ftnconfig.get_addr_id(db, db.FTN_domains["node"], link)
outp=[]
pw = ftnconfig.get_link_password(db, link, True)
#print(pw)
for echo in db.prepare("select a.text from subscriptions s, addresses a "
"where a.id=s.target and s.subscriber=$1 and a.domain=$2")(lid, db.FTN_domains[domain]):
outp.append("+"+echo[0]+"\n")
if len(outp)==0:
print ("nothing to relink")
else:
print ("".join(outp))
with ftnimport.session(db) as sess:
sess.send_message("Sergey Dorofeev", ("node", link),
def import_tic(db,
fullname,
expect_addr=None,
import_utime=None,
ticdata=None,
ignore_pw=False,
skip_access_check=False):
" specify older import_utime value to make imported file the status of aarchive "
# if "TO" is present
# get from links with matching from and to addresses and verify password
# if "TO" is absent
# get to and password from links by from. if two rows are fetched - refuse tic
#
# in both cases refuse tic if no row fetched - tics are allowed for password links only
if ticdata is None:
filepath, filename = os.path.split(fullname)
ticdata = read_tic(fullname)
else:
filepath = os.path.dirname(fullname)
tic_src = get_optional(ticdata, "FROM")
print("TIC from:", tic_src)
if tic_src is None:
tic_src = expect_addr
tic_dest = get_optional(ticdata, "TO")
print("TIC to", tic_dest)
if tic_src is None and tic_dest is None and skip_access_check:
print("Importing non-FTN file")
src_id = None
dest_id = None
else:
q = "select l.address, l.my, l.authentication from links l"
q_args = []
if tic_src:
src_id = ftnconfig.get_addr_id(db, db.FTN_domains["node"], tic_src)
q += (" and"
if q_args else " where") + " address=$%d" % (len(q_args) + 1)
q_args.append(src_id)
else:
src_id = None
if tic_dest:
dest_id = ftnconfig.get_addr_id(db, db.FTN_domains["node"],
tic_dest)
q += (" and"
if q_args else " where") + " my=$%d" % (len(q_args) + 1)
q_args.append(dest_id)
else:
dest_id = None
#print (q)
#print (q_args)
possible_links = db.prepare(q)(*q_args)
if len(possible_links) > 1:
raise WrongTic("ambiguos link %s->%s" %
(str(tic_src), str(tic_dest)))
if len(possible_links) == 0:
raise WrongTic("no matching link %s->%s" %
(str(tic_src), str(tic_dest)))
src_id, dest_id, authinfo = possible_links[0]
pw = authinfo.find("RobotsPassword").text
print("TIC src_id, dst_id, pw:", src_id, dest_id, pw)
if not ignore_pw:
tic_passw = get_single(ticdata, "PW")
if not ftnaccess.check_pw(pw, tic_passw):
raise WrongTic("invalid password [%s] for %s" %
(tic_passw, tic_src))
# source and destination verified, now try to find file
# but before we should check if link can post to specified area
area = get_single(ticdata, "AREA").upper() # FTN areas must be uppercase
print("TIC area:", area)
if not skip_access_check:
maypost = ftnaccess.may_post(db, src_id, ("fileecho", area))
if not maypost:
raise WrongTic("%s may not post to %s" % (tic_src, area))
fname = os.path.split(get_single(ticdata, "FILE"))[1]
try:
fsize = get_single(ticdata, "SIZE", int)
except BadTic:
fsize = None
fcrc = get_single(ticdata, "CRC", remove=False)
print("TIC name, size, crc:", fname, fsize, fcrc)
ffullname = find_matching_file(filepath, fname, fsize, fcrc)
if not os.path.exists(ffullname):
raise NoFile("file %s does not exists" % ffullname)
if fsize is not None and os.path.getsize(ffullname) != fsize:
raise NoFile("file %s size != %d" % (ffullname, fsize))
fsize, checksum = sz_crc32(ffullname)
if checksum != fcrc.upper():
raise NoFile("file %s crc32 %s != %s" % (ffullname, checksum, fcrc))
print("file matches")
# >>> LOCK FILEECHOES POSTINGS
if db.FECHOIMPORTLOCK is None:
db.FECHOIMPORTLOCK = db.prepare(
"select oid from pg_class where relname='file_post'").first()
with postgresql.alock.ExclusiveLock(db, db.FECHOIMPORTLOCK, 0):
# calculate hash
# verify if it exists in database
# if not, post as new (new blob, new name, new destination)
# if yes, register new name (if differ) and destination for file
# check if it is not duplicate tic
# select posting of same origin, area, filename, origin_record
# if any has same filesize and hash - compare content and drop duplicate
tic_origin = get_optional(ticdata, "ORIGIN")
if tic_origin:
with ftnimport.session(db) as sess:
tic_origin_id = sess.check_addr("node", tic_origin)
else:
tic_origin_id = None
area_id = ftnconfig.get_addr_id(db, db.FTN_domains["fileecho"], area)
try:
tic_originrec = get_first(ticdata, "PATH")
except BadTic as e:
print("PATH is missing, no dupe checking")
print(e)
tic_originrec = None
if tic_originrec:
print("check if tic is first %s %d %s %s" %
((tic_origin, area_id, fname, tic_originrec)))
for prev_f, prev_l, prev_h, prev_p in db.prepare(
"select f.id, f.length, f.sha512, p.id from files f inner join file_post p ON p.filedata=f.id "
"where p.origin=$1 and p.destination=$2 and p.filename=$3 and p.origin_record=$4"
)(tic_origin_id, area_id, fname, tic_originrec):
os.rename(ffullname, ffullname + ".dup")
if not fullname.endswith(".faketic"):
os.rename(fullname, fullname + ".dup")
raise DupPost("similar posting %d, abandom" % prev_p,
ffullname)
# tic with the same first record of PATH - the same posting
sha512 = hashlib.new("sha512")
f = open(ffullname, "rb")
while (True):
z = f.read(262144)
if not z:
break
sha512.update(z)
f.close()
print(sha512.hexdigest())
oldf_id = db.prepare("select id from files where sha512=$1").first(
sha512.digest())
if oldf_id is None:
print("new file content")
if fsize <= 262144:
print("save as bytea")
newf_id = db.prepare(
"insert into files (length, sha512, content) values ($1, $2, $3) returning id"
).first(fsize, sha512.digest(),
open(ffullname, "rb").read())
else:
print("save as large object")
with ftnimport.session(db) as sess:
lo = sess.db.prepare("select lo_create(0)").first()
print("created lo", lo, end='')
lo_handle = sess.db.prepare(
"select lo_open($1, 131072)").first(lo)
f = open(ffullname, "rb")
while (True):
z = f.read(262144)
if not z:
break
print(".", end='', flush=True)
if sess.db.prepare("select lowrite($1, $2)").first(
lo_handle, z) != len(z):
raise Exception(
"error writing file data to database")
f.close()
if sess.db.prepare("select lo_close($1)").first(
lo_handle) != 0:
raise Exception("error closing large object")
newf_id = db.prepare(
"insert into files (length, sha512, lo) values ($1, $2, $3) returning id"
).first(fsize, sha512.digest(), lo)
f_id = newf_id
else:
print("use old", oldf_id)
f_id = oldf_id
# add name for filedata
is_with_name = db.prepare(
"select id from files where $1 = ANY(names) and id=$2").first(
fname, f_id)
if not is_with_name:
fnameslen = int(
db.prepare(
"select array_upper(names, 1) from files where id=$1").
first(f_id) or 0)
db.prepare("update files set names[$1]=$2 where id=$3")(
fnameslen + 1, fname, f_id)
if import_utime is None:
utime = int(
time.mktime(time.gmtime())
) # convert_post time to float and use fractions if you have rate more than one file per some seconds
else:
utime = int(import_utime)
print("post_time=", utime)
db.prepare("insert into file_post (filedata, origin, destination, recv_from, recv_as, recv_timestamp, origin_record, filename, other, post_time) "
"values ($1, $2, $3, $4, $5, $6, $7, $8, $9, free_posttime($10))")\
(f_id, tic_origin_id, area_id, src_id, dest_id, datetime.datetime.now(datetime.timezone.utc), tic_originrec, fname, json.dumps(ticdata), utime)
print("inserted successfully")
print("unlink", ffullname)
os.unlink(ffullname)
if not fullname.endswith(".faketic"):
print("unlink", fullname)
os.unlink(fullname)
def import_tic(db, fullname, expect_addr=None, import_utime=None, ticdata=None, ignore_pw=False, skip_access_check=False):
" specify older import_utime value to make imported file the status of aarchive "
# if "TO" is present
# get from links with matching from and to addresses and verify password
# if "TO" is absent
# get to and password from links by from. if two rows are fetched - refuse tic
#
# in both cases refuse tic if no row fetched - tics are allowed for password links only
if ticdata is None:
filepath, filename = os.path.split(fullname)
ticdata = read_tic(fullname)
else:
filepath = os.path.dirname(fullname)
tic_src = get_optional(ticdata, "FROM")
print ("from", tic_src)
if tic_src is None:
tic_src=expect_addr
tic_dest = get_optional(ticdata, "TO")
print ("to", tic_dest)
if tic_src is None and tic_dest is None and skip_access_check:
print ("Importing non-FTN file")
src_id = None
dest_id = None
else:
q="select l.address, l.my, l.authentication from links l"
q_args=[]
if tic_src:
src_id = ftnconfig.get_addr_id(db, db.FTN_domains["node"], tic_src)
q += (" and" if q_args else " where") + " address=$%d"%(len(q_args)+1)
q_args.append(src_id)
else:
src_id = None
if tic_dest:
dest_id = ftnconfig.get_addr_id(db, db.FTN_domains["node"], tic_dest)
q += (" and" if q_args else " where") + " my=$%d"%(len(q_args)+1)
q_args.append(dest_id)
else:
dest_id = None
print (q)
print (q_args)
possible_links = db.prepare(q)(*q_args)
if len(possible_links) > 1:
raise WrongTic("ambiguos link %s->%s"%(str(tic_src),str(tic_dest)))
if len(possible_links) == 0:
raise WrongTic("no matching link %s->%s"%(str(tic_src),str(tic_dest)))
src_id, dest_id, authinfo = possible_links[0]
pw = authinfo.find("RobotsPassword").text
print (src_id, dest_id, pw)
if not ignore_pw:
tic_passw = get_single(ticdata, "PW")
if not ftnaccess.check_pw(pw, tic_passw):
raise WrongTic("invalid password [%s] for %s"%(tic_passw,tic_src))
# source and destination verified, now try to find file
# but before we should check if link can post to specified area
area = get_single(ticdata, "AREA").upper() # FTN areas must be uppercase
print (area)
if not skip_access_check:
maypost = ftnaccess.may_post(db, src_id, ("fileecho", area))
if not maypost:
raise WrongTic("%s may not post to %s"%(tic_src, area))
fname = os.path.split(get_single(ticdata, "FILE"))[1]
try:
fsize = get_single(ticdata, "SIZE", int)
except BadTic:
fsize = None
fcrc = get_single(ticdata, "CRC", remove=False)
print (fname, fsize, fcrc)
ffullname=find_file(fname, filepath)
if not os.path.exists(ffullname):
raise NoFile("file %s does not exists"%ffullname)
if fsize is not None and os.path.getsize(ffullname)!=fsize:
raise NoFile("file %s size != %d"%(ffullname, fsize))
fsize, checksum = sz_crc32(ffullname)
if checksum != fcrc.upper():
raise NoFile("file %s crc32 %s != %s"%(ffullname, checksum, fcrc))
print ("file matches")
# >>> LOCK FILEECHOES POSTINGS
if db.FECHOIMPORTLOCK is None:
db.FECHOIMPORTLOCK=db.prepare("select oid from pg_class where relname='file_post'").first()
with postgresql.alock.ExclusiveLock(db, db.FECHOIMPORTLOCK, 0):
# calculate hash
# verify if it exists in database
# if not, post as new (new blob, new name, new destination)
# if yes, register new name (if differ) and destination for file
# check if it is not duplicate tic
# select posting of same origin, area, filename, origin_record
# if any has same filesize and hash - compare content and drop duplicate
tic_origin = get_optional(ticdata, "ORIGIN")
if tic_origin:
with ftnimport.session(db) as sess:
tic_origin_id = sess.check_addr("node", tic_origin)
else:
tic_origin_id = None
area_id = ftnconfig.get_addr_id(db, db.FTN_domains["fileecho"], area)
try:
tic_originrec = get_first(ticdata, "PATH")
except BadTic as e:
print ("PATH is missing, no dupe checking")
print (e)
tic_originrec = None
if tic_originrec:
print("check if tic is first %s %d %s %s"%((tic_origin, area_id, fname, tic_originrec)))
for prev_f, prev_l, prev_h, prev_p in db.prepare("select f.id, f.length, f.sha512, p.id from files f inner join file_post p ON p.filedata=f.id "
"where p.origin=$1 and p.destination=$2 and p.filename=$3 and p.origin_record=$4")(tic_origin_id, area_id, fname, tic_originrec):
os.rename(ffullname, ffullname+".dup")
if not fullname.endswith(".faketic"):
os.rename(fullname, fullname+".dup")
raise DupPost("similar posting %d, abandom"%prev_p, ffullname)
# tic with the same first record of PATH - the same posting
sha512 = hashlib.new("sha512")
f=open(ffullname, "rb")
while(True):
z=f.read(262144)
if not z:
break
sha512.update(z)
f.close()
print(sha512.hexdigest())
oldf_id = db.prepare("select id from files where sha512=$1").first(sha512.digest())
if oldf_id is None:
print("new file content")
if fsize<=262144:
print ("save as bytea")
newf_id = db.prepare("insert into files (length, sha512, content) values ($1, $2, $3) returning id").first(fsize, sha512.digest(), open(ffullname, "rb").read())
else:
print ("save as large object")
with ftnimport.session(db) as sess:
lo=sess.db.prepare("select lo_create(0)").first()
print("created lo", lo,end='')
lo_handle=sess.db.prepare("select lo_open($1, 131072)").first(lo)
f=open(ffullname, "rb")
while(True):
z=f.read(262144)
if not z:
break
print(".", end='', flush=True)
if sess.db.prepare("select lowrite($1, $2)").first(lo_handle, z) != len(z):
raise Exception("error writing file data to database")
f.close()
if sess.db.prepare("select lo_close($1)").first(lo_handle) != 0:
raise Exception("error closing large object")
newf_id = db.prepare("insert into files (length, sha512, lo) values ($1, $2, $3) returning id").first(fsize, sha512.digest(), lo)
f_id = newf_id
else:
print("use old", oldf_id)
f_id = oldf_id
# add name for filedata
is_with_name = db.prepare("select id from files where $1 = ANY(names) and id=$2").first(fname, f_id)
if not is_with_name:
fnameslen = int(db.prepare("select array_upper(names, 1) from files where id=$1").first(f_id) or 0)
db.prepare("update files set names[$1]=$2 where id=$3")(fnameslen+1, fname, f_id)
if import_utime is None:
utime = int(time.mktime(time.gmtime())) # convert_post time to float and use fractions if you have rate more than one file per some seconds
else:
utime = int(import_utime)
print ("post_time=", utime)
db.prepare("insert into file_post (filedata, origin, destination, recv_from, recv_as, recv_timestamp, origin_record, filename, other, post_time) "
"values ($1, $2, $3, $4, $5, $6, $7, $8, $9, free_posttime($10))")\
(f_id, tic_origin_id, area_id, src_id, dest_id, datetime.datetime.now(datetime.timezone.utc), tic_originrec, fname, json.dumps(ticdata), utime)
print ("inserted successfully")
print ("unlink", ffullname)
os.unlink(ffullname)
if not fullname.endswith(".faketic"):
print ("unlink", fullname)
os.unlink(fullname)
#!/usr/local/bin/python3 -bb
import ftnconfig
import ftnexport
import ftnimport
db = ftnconfig.connectdb()
bydomain = {}
for domain in ["echo", "fileecho"]:
#print ("domain", domain)
for t in ftnexport.get_all_targets(db, domain):
tid = ftnconfig.get_addr_id(db, db.FTN_domains[domain], t)
ulist = []
for x in ftnexport.get_subscribers(db, tid, True):
ulist.append("%d|%s"%ftnconfig.get_addr(db, x[0])+"!%d"%x[2])
#print
ulist.sort()
bydomain.setdefault(domain, {}).setdefault(", ".join(ulist), []).append(t)
with ftnimport.session(db) as sess:
for d, byuplink in bydomain.items():
outp=[]
for u, targets in byuplink.items():
outp.append(" uplink = ["+u+"]\n")
targets.sort()
for t in targets:
outp.append(" "+t+"\n")
outp.append("\n")
#link='2:5020/1200'
domain = "echo"
purgelist = 'orphans'
import ftnconfig
import ftnimport
import ftnexport
import os
robot = ftnconfig.robotnames[domain]
db = ftnconfig.connectdb()
for area in open(purgelist):
area = area.strip().upper()
aid = ftnconfig.get_addr_id(db, db.FTN_domains[domain], area)
# 1. verify that there is nothing there
if domain == "fileecho":
print(area, os.listdir("/tank/home/fido/fareas/" + area.lower()))
elif domain == "echo":
count = db.prepare(
"select count(*) from messages where destination=$1").first(aid)
print(area, count)
if count:
continue
else:
1 / 0
# 2. send message to subscribers
# 3. remove subscriptions and address
def file_export(db, address, password, what):
""" This generator fetches messages from database and
yields objects, that contain the file information
and instructions how to commit to db inforamtion
about successful message delivery """
# first netmail
# then requested file
# then echoes
# then filebox
# and at last fileechoes
print("export to", repr(address), repr(password), repr(what))
if password != (get_link_password(db, address) or ""):
raise FTNWrongPassword()
print("password is correct" if password else "password is empty")
# WARNING!
# unprotected sessions never must do queries as it may result in leaking netmail
# if address of some hub is spoofed
addr_id = get_addr_id(db, db.FTN_domains["node"], address)
link_pkt_format = get_link_pkt_format(db, address)
link_bundler = get_link_bundler(db, address)
if password and ("netmail" in what):
explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["netmail"], addr_id)))
if explock.acquire(False):
try:
print ("exporting netmail")
# only vital subscriptions is processed
# non-vital (CC) should be processed just like echomail
# set password in netmail packets
p = pktpacker(link_pkt_format, ADDRESS, address, get_link_password(db, address) or '', lambda: get_pkt_n(db, get_link_id(db, address)), lambda: netmailcommitter())
#..firstly send pkts in outbound
for id_msg, src, dest, msgid, header, body, origcharset, recvfrom in get_subscriber_messages_n(db, addr_id, db.FTN_domains["node"]):
print("netmail %d recvfrom %d pack to %s"%(id_msg, recvfrom, repr(address)))
# if exporting to utf8z always use UTF-8
if link_pkt_format == "utf8z":
origcharset = "utf-8"
myvia = "PyFTN " + ADDRESS + " " + time.asctime()
srca=db.prepare("select domain, text from addresses where id=$1").first(src)
dsta=db.prepare("select domain, text from addresses where id=$1").first(dest)
try:
msg, msgcharset = denormalize_message(
(db.FTN_backdomains[srca[0]], srca[1]),
(db.FTN_backdomains[dsta[0]], dsta[1]),
msgid, header, body, origcharset, address, addvia = myvia)
except:
raise Exception("denormalization error on message id=%d"%id_msg+"\n"+traceback.format_exc())
try:
print ("export msg attributes", msg.attr)
except:
traceback.print_exception()
if 'AuditRequest' in ftn.attr.binary_to_text(msg.attr):
audit_reply = (db.FTN_backdomains[srca[0]], srca[1]), header.find("sendername").text, address, msg, msgcharset
else:
audit_reply = None
for x in p.add_item(msg, (id_msg, audit_reply)): # add ARQ flag
yield x
for x in p.flush():
yield x
del p
finally:
explock.release()
else:
print ("could not acquire netmail lock")
if "direct" in what: # available for unprotected sessions
# export messages with processed==8 and destination==addr_id
explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["netmail"], addr_id)))
if explock.acquire(False):
print ("exporting direct netmail")
# only vital subscriptions is processed
# non-vital (CC) should be processed just like echomail
# set password in netmail packets
link_id=get_link_id(db, address, withfailback=True)
p = pktpacker(link_pkt_format, ADDRESS, address, get_link_password(db, address) or '', lambda: get_pkt_n(db, link_id), lambda: netmailcommitter(newstatus=7))
#..firstly send pkts in outbound
for id_msg, src, dest, msgid, header, body, origcharset, recvfrom in get_direct_messages(db, addr_id):
print("direct netmail %d recvfrom %d pack to %s"%(id_msg, recvfrom, repr(address)))
# if exporting to utf8z always use UTF-8
if link_pkt_format == "utf8z":
origcharset = "utf-8"
myvia = "PyFTN " + ADDRESS + " DIRECT " + time.asctime()
srca=db.prepare("select domain, text from addresses where id=$1").first(src)
dsta=db.prepare("select domain, text from addresses where id=$1").first(dest)
try:
msg, msgcharset = denormalize_message(
(db.FTN_backdomains[srca[0]], srca[1]),
(db.FTN_backdomains[dsta[0]], dsta[1]),
msgid, header, body, origcharset, address, addvia = myvia)
except:
raise Exception("denormalization error on message id=%d"%id_msg+"\n"+traceback.format_exc())
try:
print ("export msg attributes", msg.attr)
except:
traceback.print_exception()
if 'AuditRequest' in ftn.attr.binary_to_text(msg.attr):
audit_reply = (db.FTN_backdomains[srca[0]], srca[1]), header.find("sendername").text, address, msg, msgcharset
else:
audit_reply = None
for x in p.add_item(msg, (id_msg, audit_reply)): # add ARQ flag
yield x
for x in p.flush():
yield x
del p
explock.release()
pass
if password and ("echomail" in what):
explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["echomail"], addr_id)))
if explock.acquire(False):
try:
print ("exporting echomail")
#..firstly send bundles in outbound
#
if link_bundler:
p = pktpacker(link_pkt_format, ADDRESS, address, get_link_password(db, address) or '', lambda: get_pkt_n(db, get_link_id(db, address)), lambda: echomailcommitter(),
bundlepacker(link_bundler, address, lambda: get_bundle_n(db, get_link_id(db, address)), lambda: echomailcommitter()))
else:
p = pktpacker(link_pkt_format, ADDRESS, address, get_link_password(db, address) or '', lambda: get_pkt_n(db, get_link_id(db, address)), lambda: echomailcommitter())
subscache = {}
for id_msg, xxsrc, dest, msgid, header, body, origcharset, recvfrom, withsubscr, processed in get_subscriber_messages_e(db, addr_id, db.FTN_domains["echo"]):
will_export = True # do we really must send message or just update last_sent pointer
#print("echomail %d"%id_msg, repr(dest))
#print("dest %d recvfrom %s subscr %s pack to %s"%(dest, repr(recvfrom), repr(withsubscr), address))
# ignore src - for echomail it is just recv_from
# if exporting to utf8z always use UTF-8
if link_pkt_format == "utf8z":
origcharset = "utf-8"
if recvfrom == addr_id:
#print ("Message from this link, will not export")
will_export = False
if processed == 5:
#print ("Archived message, will not export")
will_export = False
# check commuter
subscriber_comm = db.FTN_commuter.get(withsubscr)
if subscriber_comm is not None: # must check as None==None => no export at all
# get subscription through what message was received
recvfrom_subscription = db.prepare("select id from subscriptions where target=$1 and subscriber=$2").first(sub_tart, m_recvfrom)
recvfrom_comm = db.FTN_commuter.get(recvfrom_subscription)
if recvfrom_comm == subscriber_comm:
print("commuter %d - %d, will not export"%(withsubscr, recvfrom_subscription))
will_export = False
# continue # do not forward between subscriptions in one commuter group (e.g. two uplinks)
if dest in subscache:
subscribers = subscache[dest]
else:
subscribers = db.prepare("select a.domain, a.text from subscriptions s, addresses a where s.target=$1 and s.subscriber=a.id")(dest)
if not all([x[0]==db.FTN_domains["node"] for x in subscribers]):
raise FTNFail("subscribers from wrong domain for "+str(sub_targ))
# print(sub_id, sub_targ, "all subscribers:", [x[1] for x in subscribers])
subscribers = subscache[dest] = [x[1] for x in subscribers]
#print("subscribers:", repr(subscribers))
# if withsubscr not in subscribers:
# raise Exception("strange: exporting to non-existent subscription", withsubscr)
dsta = db.prepare("select domain, text from addresses where id=$1").first(dest)
# modify path and seen-by
# seen-by's - get list of all subscribers of this target; add subscribers list
#... if go to another zone remove path and seen-by's and only add seen-by's of that zone -> ftnexport
if will_export: # create MSG else do not bother
try:
msg, msgcharset = denormalize_message(
("node", ADDRESS),
(db.FTN_backdomains[dsta[0]], dsta[1]),
msgid, header, body, origcharset, address, addseenby=subscribers, addpath=ADDRESS)
except:
raise Exception("denormalization error on message id=%d"%id_msg+"\n"+traceback.format_exc())
for x in p.add_item((msg if will_export else None), (withsubscr, id_msg)):
yield x
for x in p.flush():
yield x
finally:
explock.release()
else:
print("could not acquire echomail lock")
if password and ("filebox" in what):
explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["filebox"], addr_id)))
if explock.acquire(False):
# ..send freq filebox
print ("exporting filebox")
dsend = addrdir(DOUTBOUND, address)
if os.path.isdir(dsend):
print ("exporting daemon outbound")
for f in os.listdir(dsend):
fname = os.path.join(dsend, f)
if os.path.isfile(fname):
obj = outfile()
obj.data = open(fname, "rb")
obj.filename = f
obj.length = os.path.getsize(fname)
yield obj, filecommitter(fname)
explock.release()
if password and ("fileecho" in what):
explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["fileecho"], addr_id)))
if explock.acquire(False):
# ..send fileechoes
print ("exporting fileechoes (nothing here)")
pass #1/0
explock.release()
return
