def __init__(self, db, address, password, filename, length):
print("from",address,"pw",password,"recv",filename,"len",length)
# based on length choose to process data in memory (check received data length)
# or save to disk
# packets
# bundles
# freqs
# tics
# files
self.db = db
self.address = address
link_id, addr_id, myaddr_id = ftnaccess.check_link(db, address, password, False)
if myaddr_id is None:
raise FTNWrongPassword()
self.protected = link_id is not None
savedir = inbound_dir(address, self.protected, True)
if not os.path.exists(savedir):
os.makedirs(savedir)
mod = 0
while True:
try:
self.file = os.path.join(savedir, modname(filename, mod))
self.fo = os.fdopen(os.open(self.file, os.O_CREAT | os.O_EXCL | os.O_WRONLY), "wb") # should add | os.O_BINARY
break
except OSError as e:
if e.args[0]!=17:
raise e
mod += 1
def fix(db, sess, src, srcname, destname, domain, password, msgid, cmdtext, is_local):
dom, text = db.prepare("select domain, text from addresses where id=$1").first(src)
if dom != db.FTN_domains["node"]:
raise FTNFail("not our domain")
print(text)
link_id, addr_id, myaddr_id = ftnaccess.check_link(db, text, password, True)
if myaddr_id is None:
reply=["wrong password"]
elif link_id is None:
reply=["unknown link"]
elif not is_local:
reply=["request delivered not directly, please consider changing the password"]
else:
reply=[]
for cmd in map(str.strip, cmdtext.upper().strip().split("\n")):
if cmd.startswith("---"):
break
elif len(cmd)==0:
continue
elif cmd.startswith("%PING"):
reply.append("NOP: "+cmd)
elif cmd.startswith("%QUERY"):
for area in ftnexport.get_node_subscriptions(db, text, domain):
reply.append(area)
elif cmd.startswith("%HELP"):
reply.append("Use following command:")
reply.append(" %QUERY -- list of subscribed areas")
reply.append(" %LIST -- list of all areas")
reply.append(" +areaname -- subscribe, or:")
reply.append(" areaname -- subscribe")
reply.append(" -areaname -- unsubscribe")
reply.append(" you can use ? to substitite any one character an area name")
reply.append(" or * to substitude any number of any characters in area name")
reply.append(" areaname must be UPPERCASE")
reply.append(" %RESET area YYYY-MM-DD -- reset area to specified date")
reply.append("")
elif cmd.startswith("%LIST"):
for area in ftnexport.get_all_targets(db, domain):
reply.append(area)
elif cmd.startswith("%AVAIL"):
reply.append("Sorry not implemented: "+cmd)
elif cmd.startswith("%PAUSE"):
reply.append("Sorry not implemented: "+cmd)
elif cmd.startswith("%RESET"):
reply += reset(db, sess, text, domain, cmd[7:])
elif cmd.startswith("-"):
reply += unsubscribe(db, sess, text, domain, cmd[1:])
elif cmd.startswith("+"):
reply += subscribe(db, sess, text, domain, cmd[1:])
elif cmd.startswith("%"):
reply.append("Unknown command: "+cmd)
print(cmd)
# 1/0
else:
reply+=subscribe(db, sess, text, domain, cmd)
reply.append("")
sess.send_message(ftnconfig.get_taddr(db, myaddr_id), destname+" Robot", ("node", text), srcname, msgid, "report", "\n".join(reply), sendmode="direct")
print(reply)
def fix(db, sess, src, srcname, destname, domain, password, msgid, cmdtext,
is_local):
dom, text = db.prepare(
"select domain, text from addresses where id=$1").first(src)
if dom != db.FTN_domains["node"]:
raise FTNFail("not our domain")
print("*fix for:", text)
link_id, addr_id, myaddr_id = ftnaccess.check_link(db, text, password,
True)
if myaddr_id is None:
reply = ["wrong password"]
raise (Exception("cannot find my address to reply to *fix message"))
elif link_id is None:
reply = ["unknown link"]
elif not is_local:
reply = [
"request delivered not directly, please consider changing the password"
]
else:
reply = []
for cmd in map(str.strip, cmdtext.upper().strip().split("\n")):
if cmd.startswith("---"):
break
elif len(cmd) == 0:
continue
elif cmd.startswith("%PING"):
reply.append("NOP: " + cmd)
elif cmd.startswith("%QUERY"):
for area in ftnexport.get_node_subscriptions(db, text, domain):
reply.append(area)
elif cmd.startswith("%HELP"):
reply.append("Use following command:")
reply.append(" %QUERY -- list of subscribed areas")
reply.append(" %LIST -- list of all areas")
reply.append(" +areaname -- subscribe, or:")
reply.append(" areaname -- subscribe")
reply.append(" -areaname -- unsubscribe")
reply.append(
" you can use ? to substitite any one character an area name"
)
reply.append(
" or * to substitude any number of any characters in area name"
)
reply.append(" areaname must be UPPERCASE")
reply.append(
" %RESET area YYYY-MM-DD -- reset area to specified date")
reply.append("")
elif cmd.startswith("%LIST"):
for area in ftnexport.get_all_targets(db, domain):
reply.append(area)
elif cmd.startswith("%AVAIL"):
reply.append("Sorry not implemented: " + cmd)
elif cmd.startswith("%PAUSE"):
reply.append("Sorry not implemented: " + cmd)
elif cmd.startswith("%RESET"):
reply += reset(db, sess, text, domain, cmd[7:])
elif cmd.startswith("-"):
reply += unsubscribe(db, sess, text, domain, cmd[1:])
elif cmd.startswith("+"):
reply += subscribe(db, sess, text, domain, cmd[1:])
elif cmd.startswith("%"):
reply.append("Unknown command: " + cmd)
print(cmd)
# 1/0
else:
reply += subscribe(db, sess, text, domain, cmd)
reply.append("")
print("*fix reply from:", myaddr_id, ftnconfig.get_taddr(db, myaddr_id))
sess.send_message(ftnconfig.get_taddr(db, myaddr_id),
destname + " Robot", ("node", text),
srcname,
msgid,
"report",
"\n".join(reply),
sendmode="direct")
print(reply)
def file_export(db, address, password, what):
""" This generator fetches messages from database and
yields objects, that contain the file information
and instructions how to commit to db inforamtion
about successful message delivery """
# first netmail
# then requested file
# then echoes
# then filebox
# and at last fileechoes
print("export to", repr(address), repr(password), repr(what))
link_id, addr_id, myaddr_id = ftnaccess.check_link(db, address, password, False)
if myaddr_id is None:
raise FTNWrongPassword()
print("password is correct" if link_id else "unprotected session", "local address", myaddr_id)
# WARNING!
# unprotected sessions never must do queries as it may result in leaking netmail
# if address of some hub is spoofed
myaddr_text = get_addr(db, myaddr_id)[1]
link_pkt_format, link_bundler = get_link_packing(db, link_id)
link_my_id, link_pw = ftnaccess.link_password(db, link_id, False)
if link_id and ("netmail" in what):
explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["netmail"], addr_id)))
if explock.acquire(False):
try:
print ("exporting netmail")
# only vital subscriptions is processed
# non-vital (CC) should be processed just like echomail
# set password in netmail packets
p = pktpacker(link_pkt_format, myaddr_text, address, link_pw or '', lambda: get_pkt_n(db, link_id), lambda: netmailcommitter())
#..firstly send pkts in outbound
for id_msg, src, dest, msgid, header, body, origcharset, recvfrom in get_subscriber_messages_n(db, addr_id, db.FTN_domains["node"]):
print("netmail %d recvfrom %d pack to %s"%(id_msg, recvfrom, repr(address)))
# if exporting to utf8z always use UTF-8
if link_pkt_format == "utf8z":
origcharset = "utf-8"
myvia = "PyFTN " + ADDRESS + " " + time.asctime()
srca=db.prepare("select domain, text from addresses where id=$1").first(src)
dsta=db.prepare("select domain, text from addresses where id=$1").first(dest)
try:
msg, msgcharset = denormalize_message(
(db.FTN_backdomains[srca[0]], srca[1]),
(db.FTN_backdomains[dsta[0]], dsta[1]),
msgid, header, body, origcharset, address, addvia = myvia)
except:
raise Exception("denormalization error on message id=%d"%id_msg+"\n"+traceback.format_exc())
try:
print ("export msg attributes", msg.attr)
except:
traceback.print_exception()
if 'AuditRequest' in ftn.attr.binary_to_text(msg.attr):
audit_reply = (db.FTN_backdomains[srca[0]], srca[1]), header.find("sendername").text, address, msg, msgcharset
else:
audit_reply = None
for x in p.add_item(msg, (id_msg, audit_reply)): # add ARQ flag
yield x
for x in p.flush():
yield x
del p
finally:
explock.release()
else:
print ("could not acquire netmail lock")
if "direct" in what: # available for unprotected sessions
# export messages with processed==8 and destination==addr_id
explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["netmail"], addr_id)))
if explock.acquire(False):
print ("exporting direct netmail")
# only vital subscriptions is processed
# non-vital (CC) should be processed just like echomail
# set password in netmail packets
p = pktpacker(link_pkt_format, myaddr_text, address, link_pw or '', lambda: get_pkt_n(db, link_id), lambda: netmailcommitter(newstatus=7))
#..firstly send pkts in outbound
for id_msg, src, dest, msgid, header, body, origcharset, recvfrom in get_direct_messages(db, addr_id):
print("direct netmail %d recvfrom %d pack to %s"%(id_msg, recvfrom, repr(address)))
# if exporting to utf8z always use UTF-8
if link_pkt_format == "utf8z":
origcharset = "utf-8"
myvia = "PyFTN " + ADDRESS + " DIRECT " + time.asctime()
srca=db.prepare("select domain, text from addresses where id=$1").first(src)
dsta=db.prepare("select domain, text from addresses where id=$1").first(dest)
try:
msg, msgcharset = denormalize_message(
(db.FTN_backdomains[srca[0]], srca[1]),
(db.FTN_backdomains[dsta[0]], dsta[1]),
msgid, header, body, origcharset, address, addvia = myvia)
except:
raise Exception("denormalization error on message id=%d"%id_msg+"\n"+traceback.format_exc())
try:
print ("export msg attributes", msg.attr)
except:
traceback.print_exception()
if 'AuditRequest' in ftn.attr.binary_to_text(msg.attr):
audit_reply = (db.FTN_backdomains[srca[0]], srca[1]), header.find("sendername").text, address, msg, msgcharset
else:
audit_reply = None
for x in p.add_item(msg, (id_msg, audit_reply)): # add ARQ flag
yield x
for x in p.flush():
yield x
del p
explock.release()
pass
if link_id and ("echomail" in what):
explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["echomail"], addr_id)))
if explock.acquire(False):
try:
print ("exporting echomail")
#..firstly send bundles in outbound
#
if link_bundler:
p = pktpacker(link_pkt_format, myaddr_text, address, link_pw or '', lambda: get_pkt_n(db, link_id), lambda: echomailcommitter(),
bundlepacker(link_bundler, address, lambda: get_bundle_n(db, link_id), lambda: echomailcommitter()))
else:
p = pktpacker(link_pkt_format, myaddr_text, address, link_pw or '', lambda: get_pkt_n(db, link_id), lambda: echomailcommitter())
subscache = {}
for id_msg, xxsrc, dest, msgid, header, body, origcharset, recvfrom, withsubscr, processed in get_subscriber_messages_e(db, addr_id, db.FTN_domains["echo"]):
# ??? "my" addr in subscription - is it used here
will_export = True # do we really must send message or just update last_sent pointer
#print("echomail %d"%id_msg, repr(dest))
#print("dest %d recvfrom %s subscr %s pack to %s"%(dest, repr(recvfrom), repr(withsubscr), address))
# ignore src - for echomail it is just recv_from
# if exporting to utf8z always use UTF-8
if link_pkt_format == "utf8z":
origcharset = "utf-8"
if recvfrom == addr_id:
#print ("Message from this link, will not export")
will_export = False
if processed == 5:
#print ("Archived message, will not export")
will_export = False
# check commuter - NOT TESTED
subscriber_comm = db.FTN_commuter.get(withsubscr)
if subscriber_comm is not None: # must check as None==None => no export at all
# get subscription through what message was received
recvfrom_subscription = db.prepare("select id from subscriptions where target=$1 and subscriber=$2").first(sub_tart, m_recvfrom)
recvfrom_comm = db.FTN_commuter.get(recvfrom_subscription)
if recvfrom_comm == subscriber_comm:
print("commuter %d - %d, will not export"%(withsubscr, recvfrom_subscription))
will_export = False
# continue # do not forward between subscriptions in one commuter group (e.g. two uplinks)
if dest in subscache:
subscribers = subscache[dest]
else:
subscribers = db.prepare("select a.domain, a.text from subscriptions s, addresses a where s.target=$1 and s.subscriber=a.id")(dest)
if not all([x[0]==db.FTN_domains["node"] for x in subscribers]):
raise FTNFail("subscribers from wrong domain for "+str(sub_targ))
# print(sub_id, sub_targ, "all subscribers:", [x[1] for x in subscribers])
subscribers = subscache[dest] = [x[1] for x in subscribers]
#print("subscribers:", repr(subscribers))
# if withsubscr not in subscribers:
# raise Exception("strange: exporting to non-existent subscription", withsubscr)
dsta = db.prepare("select domain, text from addresses where id=$1").first(dest)
# modify path and seen-by
# seen-by's - get list of all subscribers of this target; add subscribers list
#... if go to another zone remove path and seen-by's and only add seen-by's of that zone -> ftnexport
if will_export: # create MSG else do not bother
try:
msg, msgcharset = denormalize_message(
("node", ADDRESS),
(db.FTN_backdomains[dsta[0]], dsta[1]),
msgid, header, body, origcharset, address, addseenby=subscribers, addpath=ADDRESS)
except:
raise Exception("denormalization error on message id=%d"%id_msg+"\n"+traceback.format_exc())
for x in p.add_item((msg if will_export else None), (withsubscr, id_msg)):
yield x
for x in p.flush():
yield x
finally:
explock.release()
else:
print("could not acquire echomail lock")
if link_id and ("filebox" in what):
explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["filebox"], addr_id)))
if explock.acquire(False):
# ..send freq filebox
print ("exporting filebox")
dsend = addrdir(DOUTBOUND, address)
if os.path.isdir(dsend):
print ("exporting daemon outbound")
for f in os.listdir(dsend):
fname = os.path.join(dsend, f)
if os.path.isfile(fname):
obj = outfile()
obj.data = open(fname, "rb")
obj.filename = f
obj.length = os.path.getsize(fname)
yield obj, filecommitter(fname)
explock.release()
if link_id and ("fileecho" in what):
explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["fileecho"], addr_id)))
if explock.acquire(False):
try:
print ("exporting fileechoes for", address)
subscache = {}
tic_password = password
t = ticpacker(lambda: get_tic_n(db, link_id), lambda: ticcommitter(db))
latest_post = db.prepare("select max(post_time) from file_post").first()
fruitful = False
for fp_id, fp_filename, fp_destination, fp_recv_from, fp_recv_as, fp_post_time, fp_filedata, \
fp_origin, fp_other, withsubscr, file_length, file_content, file_lo in \
db.prepare("select fp.id, fp.filename, fp.destination, fp.recv_from, fp.recv_as, fp.post_time, "
"fp.filedata, fp.origin, fp.other, s.id, f.length, f.content, f.lo "
"from file_post fp, subscriptions s, files f "
"where exists(select * from subscriptions ss where ss.target=fp.destination and ss.subscriber=$1) and s.subscriber=$1 and "
"fp.post_time>(select lastsent from lastsent where subscriber=$1) and fp.destination=s.target and f.id=fp.filedata "
"order by fp.post_time")(addr_id):
print (fp_id, fp_filename, fp_destination)
will_export = True # do we really must send message or just update last_sent pointer
if fp_recv_from == addr_id:
#print ("Message from this link, will not export")
will_export = False
other = json.loads(fp_other)
del fp_other
seenby = set(other.get("SEENBY", []))
if address in seenby:
print ("already in seenby list, not sending")
will_export = False
# dupe (own address) in path should be checked at import
# check commuter - NOT TESTED
subscriber_comm = db.FTN_commuter.get(withsubscr)
if subscriber_comm is not None: # must check as None==None => no export at all
# get subscription through what message was received
recvfrom_subscription = db.prepare("select id from subscriptions where target=$1 and subscriber=$2").first(sub_tart, m_recvfrom)
recvfrom_comm = db.FTN_commuter.get(recvfrom_subscription)
if recvfrom_comm == subscriber_comm:
print("commuter %d - %d, will not export"%(withsubscr, recvfrom_subscription))
will_export = False
# continue # do not forward between subscriptions in one commuter group (e.g. two uplinks)
if fp_destination in subscache:
subscribers = subscache[fp_destination]
else:
subscribers = db.prepare("select a.domain, a.text from subscriptions s, addresses a where s.target=$1 and s.subscriber=a.id")(fp_destination)
if not all([x[0]==db.FTN_domains["node"] for x in subscribers]):
raise FTNFail("subscribers from wrong domain for "+str(sub_targ))
# print(sub_id, sub_targ, "all subscribers:", [x[1] for x in subscribers])
subscribers = subscache[fp_destination] = [x[1] for x in subscribers]
dsta = get_addr(db, fp_destination)
# modify path and seen-by
# seen-by's - get list of all subscribers of this target; add subscribers list
if will_export:
fruitful = True # something was exported
#print("add seen-by", subscribers, " add path", myaddr_text)
other.setdefault("PATH", []).append(myaddr_text+" "+str(fp_post_time)+" "+time.asctime(time.gmtime(fp_post_time))+" PyFTN")
seenby.update(subscribers)
other["SEENBY"]=list(seenby)
if "CRC" in other:
file_crc32 = other.pop("CRC")[0]
print ("filename %s length %d crc %s"%(fp_filename, file_length, file_crc32))
else:
if file_content:
file_crc32 = ftntic.sz_crc32s(file_content)[1]
else:
file_crc32 = ftntic.sz_crc32fd(lo.LOIOReader(db, file_lo))[1]
#if fp_origin is None:
# print("substitute empty origin with source")
# fp_origin = fp_recv_from
tic = ftntic.make_tic(myaddr_text, address, tic_password, dsta[1], get_addr(db, fp_origin)[1] if fp_origin else None,
fp_filename, file_length, file_crc32, other)
# code was migrated from echo export, so it cares about blocked exports to update
# per-subscription last-sents
for x in t.add_item(((tic, (fp_filename, file_length, file_content or (db, file_lo))) if will_export else None), (addr_id, fp_post_time)): # ordering by post_time
yield x
if not fruitful:
print("empty export, must update per-subscriber lastsent up to", latest_post)
for x in t.flush():
yield x
finally:
explock.release()
else:
print ("cannot acquire fileecho lock for", addr_id)
return
