def __init__(self, db, address, password, filename, length): print("from",address,"pw",password,"recv",filename,"len",length) # based on length choose to process data in memory (check received data length) # or save to disk # packets # bundles # freqs # tics # files self.db = db self.address = address link_id, addr_id, myaddr_id = ftnaccess.check_link(db, address, password, False) if myaddr_id is None: raise FTNWrongPassword() self.protected = link_id is not None savedir = inbound_dir(address, self.protected, True) if not os.path.exists(savedir): os.makedirs(savedir) mod = 0 while True: try: self.file = os.path.join(savedir, modname(filename, mod)) self.fo = os.fdopen(os.open(self.file, os.O_CREAT | os.O_EXCL | os.O_WRONLY), "wb") # should add | os.O_BINARY break except OSError as e: if e.args[0]!=17: raise e mod += 1
def fix(db, sess, src, srcname, destname, domain, password, msgid, cmdtext, is_local): dom, text = db.prepare("select domain, text from addresses where id=$1").first(src) if dom != db.FTN_domains["node"]: raise FTNFail("not our domain") print(text) link_id, addr_id, myaddr_id = ftnaccess.check_link(db, text, password, True) if myaddr_id is None: reply=["wrong password"] elif link_id is None: reply=["unknown link"] elif not is_local: reply=["request delivered not directly, please consider changing the password"] else: reply=[] for cmd in map(str.strip, cmdtext.upper().strip().split("\n")): if cmd.startswith("---"): break elif len(cmd)==0: continue elif cmd.startswith("%PING"): reply.append("NOP: "+cmd) elif cmd.startswith("%QUERY"): for area in ftnexport.get_node_subscriptions(db, text, domain): reply.append(area) elif cmd.startswith("%HELP"): reply.append("Use following command:") reply.append(" %QUERY -- list of subscribed areas") reply.append(" %LIST -- list of all areas") reply.append(" +areaname -- subscribe, or:") reply.append(" areaname -- subscribe") reply.append(" -areaname -- unsubscribe") reply.append(" you can use ? to substitite any one character an area name") reply.append(" or * to substitude any number of any characters in area name") reply.append(" areaname must be UPPERCASE") reply.append(" %RESET area YYYY-MM-DD -- reset area to specified date") reply.append("") elif cmd.startswith("%LIST"): for area in ftnexport.get_all_targets(db, domain): reply.append(area) elif cmd.startswith("%AVAIL"): reply.append("Sorry not implemented: "+cmd) elif cmd.startswith("%PAUSE"): reply.append("Sorry not implemented: "+cmd) elif cmd.startswith("%RESET"): reply += reset(db, sess, text, domain, cmd[7:]) elif cmd.startswith("-"): reply += unsubscribe(db, sess, text, domain, cmd[1:]) elif cmd.startswith("+"): reply += subscribe(db, sess, text, domain, cmd[1:]) elif cmd.startswith("%"): reply.append("Unknown command: "+cmd) print(cmd) # 1/0 else: reply+=subscribe(db, sess, text, domain, cmd) reply.append("") sess.send_message(ftnconfig.get_taddr(db, myaddr_id), destname+" Robot", ("node", text), srcname, msgid, "report", "\n".join(reply), sendmode="direct") print(reply)
def fix(db, sess, src, srcname, destname, domain, password, msgid, cmdtext, is_local): dom, text = db.prepare( "select domain, text from addresses where id=$1").first(src) if dom != db.FTN_domains["node"]: raise FTNFail("not our domain") print("*fix for:", text) link_id, addr_id, myaddr_id = ftnaccess.check_link(db, text, password, True) if myaddr_id is None: reply = ["wrong password"] raise (Exception("cannot find my address to reply to *fix message")) elif link_id is None: reply = ["unknown link"] elif not is_local: reply = [ "request delivered not directly, please consider changing the password" ] else: reply = [] for cmd in map(str.strip, cmdtext.upper().strip().split("\n")): if cmd.startswith("---"): break elif len(cmd) == 0: continue elif cmd.startswith("%PING"): reply.append("NOP: " + cmd) elif cmd.startswith("%QUERY"): for area in ftnexport.get_node_subscriptions(db, text, domain): reply.append(area) elif cmd.startswith("%HELP"): reply.append("Use following command:") reply.append(" %QUERY -- list of subscribed areas") reply.append(" %LIST -- list of all areas") reply.append(" +areaname -- subscribe, or:") reply.append(" areaname -- subscribe") reply.append(" -areaname -- unsubscribe") reply.append( " you can use ? to substitite any one character an area name" ) reply.append( " or * to substitude any number of any characters in area name" ) reply.append(" areaname must be UPPERCASE") reply.append( " %RESET area YYYY-MM-DD -- reset area to specified date") reply.append("") elif cmd.startswith("%LIST"): for area in ftnexport.get_all_targets(db, domain): reply.append(area) elif cmd.startswith("%AVAIL"): reply.append("Sorry not implemented: " + cmd) elif cmd.startswith("%PAUSE"): reply.append("Sorry not implemented: " + cmd) elif cmd.startswith("%RESET"): reply += reset(db, sess, text, domain, cmd[7:]) elif cmd.startswith("-"): reply += unsubscribe(db, sess, text, domain, cmd[1:]) elif cmd.startswith("+"): reply += subscribe(db, sess, text, domain, cmd[1:]) elif cmd.startswith("%"): reply.append("Unknown command: " + cmd) print(cmd) # 1/0 else: reply += subscribe(db, sess, text, domain, cmd) reply.append("") print("*fix reply from:", myaddr_id, ftnconfig.get_taddr(db, myaddr_id)) sess.send_message(ftnconfig.get_taddr(db, myaddr_id), destname + " Robot", ("node", text), srcname, msgid, "report", "\n".join(reply), sendmode="direct") print(reply)
def file_export(db, address, password, what): """ This generator fetches messages from database and yields objects, that contain the file information and instructions how to commit to db inforamtion about successful message delivery """ # first netmail # then requested file # then echoes # then filebox # and at last fileechoes print("export to", repr(address), repr(password), repr(what)) link_id, addr_id, myaddr_id = ftnaccess.check_link(db, address, password, False) if myaddr_id is None: raise FTNWrongPassword() print("password is correct" if link_id else "unprotected session", "local address", myaddr_id) # WARNING! # unprotected sessions never must do queries as it may result in leaking netmail # if address of some hub is spoofed myaddr_text = get_addr(db, myaddr_id)[1] link_pkt_format, link_bundler = get_link_packing(db, link_id) link_my_id, link_pw = ftnaccess.link_password(db, link_id, False) if link_id and ("netmail" in what): explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["netmail"], addr_id))) if explock.acquire(False): try: print ("exporting netmail") # only vital subscriptions is processed # non-vital (CC) should be processed just like echomail # set password in netmail packets p = pktpacker(link_pkt_format, myaddr_text, address, link_pw or '', lambda: get_pkt_n(db, link_id), lambda: netmailcommitter()) #..firstly send pkts in outbound for id_msg, src, dest, msgid, header, body, origcharset, recvfrom in get_subscriber_messages_n(db, addr_id, db.FTN_domains["node"]): print("netmail %d recvfrom %d pack to %s"%(id_msg, recvfrom, repr(address))) # if exporting to utf8z always use UTF-8 if link_pkt_format == "utf8z": origcharset = "utf-8" myvia = "PyFTN " + ADDRESS + " " + time.asctime() srca=db.prepare("select domain, text from addresses where id=$1").first(src) dsta=db.prepare("select domain, text from addresses where id=$1").first(dest) try: msg, msgcharset = denormalize_message( (db.FTN_backdomains[srca[0]], srca[1]), (db.FTN_backdomains[dsta[0]], dsta[1]), msgid, header, body, origcharset, address, addvia = myvia) except: raise Exception("denormalization error on message id=%d"%id_msg+"\n"+traceback.format_exc()) try: print ("export msg attributes", msg.attr) except: traceback.print_exception() if 'AuditRequest' in ftn.attr.binary_to_text(msg.attr): audit_reply = (db.FTN_backdomains[srca[0]], srca[1]), header.find("sendername").text, address, msg, msgcharset else: audit_reply = None for x in p.add_item(msg, (id_msg, audit_reply)): # add ARQ flag yield x for x in p.flush(): yield x del p finally: explock.release() else: print ("could not acquire netmail lock") if "direct" in what: # available for unprotected sessions # export messages with processed==8 and destination==addr_id explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["netmail"], addr_id))) if explock.acquire(False): print ("exporting direct netmail") # only vital subscriptions is processed # non-vital (CC) should be processed just like echomail # set password in netmail packets p = pktpacker(link_pkt_format, myaddr_text, address, link_pw or '', lambda: get_pkt_n(db, link_id), lambda: netmailcommitter(newstatus=7)) #..firstly send pkts in outbound for id_msg, src, dest, msgid, header, body, origcharset, recvfrom in get_direct_messages(db, addr_id): print("direct netmail %d recvfrom %d pack to %s"%(id_msg, recvfrom, repr(address))) # if exporting to utf8z always use UTF-8 if link_pkt_format == "utf8z": origcharset = "utf-8" myvia = "PyFTN " + ADDRESS + " DIRECT " + time.asctime() srca=db.prepare("select domain, text from addresses where id=$1").first(src) dsta=db.prepare("select domain, text from addresses where id=$1").first(dest) try: msg, msgcharset = denormalize_message( (db.FTN_backdomains[srca[0]], srca[1]), (db.FTN_backdomains[dsta[0]], dsta[1]), msgid, header, body, origcharset, address, addvia = myvia) except: raise Exception("denormalization error on message id=%d"%id_msg+"\n"+traceback.format_exc()) try: print ("export msg attributes", msg.attr) except: traceback.print_exception() if 'AuditRequest' in ftn.attr.binary_to_text(msg.attr): audit_reply = (db.FTN_backdomains[srca[0]], srca[1]), header.find("sendername").text, address, msg, msgcharset else: audit_reply = None for x in p.add_item(msg, (id_msg, audit_reply)): # add ARQ flag yield x for x in p.flush(): yield x del p explock.release() pass if link_id and ("echomail" in what): explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["echomail"], addr_id))) if explock.acquire(False): try: print ("exporting echomail") #..firstly send bundles in outbound # if link_bundler: p = pktpacker(link_pkt_format, myaddr_text, address, link_pw or '', lambda: get_pkt_n(db, link_id), lambda: echomailcommitter(), bundlepacker(link_bundler, address, lambda: get_bundle_n(db, link_id), lambda: echomailcommitter())) else: p = pktpacker(link_pkt_format, myaddr_text, address, link_pw or '', lambda: get_pkt_n(db, link_id), lambda: echomailcommitter()) subscache = {} for id_msg, xxsrc, dest, msgid, header, body, origcharset, recvfrom, withsubscr, processed in get_subscriber_messages_e(db, addr_id, db.FTN_domains["echo"]): # ??? "my" addr in subscription - is it used here will_export = True # do we really must send message or just update last_sent pointer #print("echomail %d"%id_msg, repr(dest)) #print("dest %d recvfrom %s subscr %s pack to %s"%(dest, repr(recvfrom), repr(withsubscr), address)) # ignore src - for echomail it is just recv_from # if exporting to utf8z always use UTF-8 if link_pkt_format == "utf8z": origcharset = "utf-8" if recvfrom == addr_id: #print ("Message from this link, will not export") will_export = False if processed == 5: #print ("Archived message, will not export") will_export = False # check commuter - NOT TESTED subscriber_comm = db.FTN_commuter.get(withsubscr) if subscriber_comm is not None: # must check as None==None => no export at all # get subscription through what message was received recvfrom_subscription = db.prepare("select id from subscriptions where target=$1 and subscriber=$2").first(sub_tart, m_recvfrom) recvfrom_comm = db.FTN_commuter.get(recvfrom_subscription) if recvfrom_comm == subscriber_comm: print("commuter %d - %d, will not export"%(withsubscr, recvfrom_subscription)) will_export = False # continue # do not forward between subscriptions in one commuter group (e.g. two uplinks) if dest in subscache: subscribers = subscache[dest] else: subscribers = db.prepare("select a.domain, a.text from subscriptions s, addresses a where s.target=$1 and s.subscriber=a.id")(dest) if not all([x[0]==db.FTN_domains["node"] for x in subscribers]): raise FTNFail("subscribers from wrong domain for "+str(sub_targ)) # print(sub_id, sub_targ, "all subscribers:", [x[1] for x in subscribers]) subscribers = subscache[dest] = [x[1] for x in subscribers] #print("subscribers:", repr(subscribers)) # if withsubscr not in subscribers: # raise Exception("strange: exporting to non-existent subscription", withsubscr) dsta = db.prepare("select domain, text from addresses where id=$1").first(dest) # modify path and seen-by # seen-by's - get list of all subscribers of this target; add subscribers list #... if go to another zone remove path and seen-by's and only add seen-by's of that zone -> ftnexport if will_export: # create MSG else do not bother try: msg, msgcharset = denormalize_message( ("node", ADDRESS), (db.FTN_backdomains[dsta[0]], dsta[1]), msgid, header, body, origcharset, address, addseenby=subscribers, addpath=ADDRESS) except: raise Exception("denormalization error on message id=%d"%id_msg+"\n"+traceback.format_exc()) for x in p.add_item((msg if will_export else None), (withsubscr, id_msg)): yield x for x in p.flush(): yield x finally: explock.release() else: print("could not acquire echomail lock") if link_id and ("filebox" in what): explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["filebox"], addr_id))) if explock.acquire(False): # ..send freq filebox print ("exporting filebox") dsend = addrdir(DOUTBOUND, address) if os.path.isdir(dsend): print ("exporting daemon outbound") for f in os.listdir(dsend): fname = os.path.join(dsend, f) if os.path.isfile(fname): obj = outfile() obj.data = open(fname, "rb") obj.filename = f obj.length = os.path.getsize(fname) yield obj, filecommitter(fname) explock.release() if link_id and ("fileecho" in what): explock = postgresql.alock.ExclusiveLock(db, ((EXPORTLOCK["fileecho"], addr_id))) if explock.acquire(False): try: print ("exporting fileechoes for", address) subscache = {} tic_password = password t = ticpacker(lambda: get_tic_n(db, link_id), lambda: ticcommitter(db)) latest_post = db.prepare("select max(post_time) from file_post").first() fruitful = False for fp_id, fp_filename, fp_destination, fp_recv_from, fp_recv_as, fp_post_time, fp_filedata, \ fp_origin, fp_other, withsubscr, file_length, file_content, file_lo in \ db.prepare("select fp.id, fp.filename, fp.destination, fp.recv_from, fp.recv_as, fp.post_time, " "fp.filedata, fp.origin, fp.other, s.id, f.length, f.content, f.lo " "from file_post fp, subscriptions s, files f " "where exists(select * from subscriptions ss where ss.target=fp.destination and ss.subscriber=$1) and s.subscriber=$1 and " "fp.post_time>(select lastsent from lastsent where subscriber=$1) and fp.destination=s.target and f.id=fp.filedata " "order by fp.post_time")(addr_id): print (fp_id, fp_filename, fp_destination) will_export = True # do we really must send message or just update last_sent pointer if fp_recv_from == addr_id: #print ("Message from this link, will not export") will_export = False other = json.loads(fp_other) del fp_other seenby = set(other.get("SEENBY", [])) if address in seenby: print ("already in seenby list, not sending") will_export = False # dupe (own address) in path should be checked at import # check commuter - NOT TESTED subscriber_comm = db.FTN_commuter.get(withsubscr) if subscriber_comm is not None: # must check as None==None => no export at all # get subscription through what message was received recvfrom_subscription = db.prepare("select id from subscriptions where target=$1 and subscriber=$2").first(sub_tart, m_recvfrom) recvfrom_comm = db.FTN_commuter.get(recvfrom_subscription) if recvfrom_comm == subscriber_comm: print("commuter %d - %d, will not export"%(withsubscr, recvfrom_subscription)) will_export = False # continue # do not forward between subscriptions in one commuter group (e.g. two uplinks) if fp_destination in subscache: subscribers = subscache[fp_destination] else: subscribers = db.prepare("select a.domain, a.text from subscriptions s, addresses a where s.target=$1 and s.subscriber=a.id")(fp_destination) if not all([x[0]==db.FTN_domains["node"] for x in subscribers]): raise FTNFail("subscribers from wrong domain for "+str(sub_targ)) # print(sub_id, sub_targ, "all subscribers:", [x[1] for x in subscribers]) subscribers = subscache[fp_destination] = [x[1] for x in subscribers] dsta = get_addr(db, fp_destination) # modify path and seen-by # seen-by's - get list of all subscribers of this target; add subscribers list if will_export: fruitful = True # something was exported #print("add seen-by", subscribers, " add path", myaddr_text) other.setdefault("PATH", []).append(myaddr_text+" "+str(fp_post_time)+" "+time.asctime(time.gmtime(fp_post_time))+" PyFTN") seenby.update(subscribers) other["SEENBY"]=list(seenby) if "CRC" in other: file_crc32 = other.pop("CRC")[0] print ("filename %s length %d crc %s"%(fp_filename, file_length, file_crc32)) else: if file_content: file_crc32 = ftntic.sz_crc32s(file_content)[1] else: file_crc32 = ftntic.sz_crc32fd(lo.LOIOReader(db, file_lo))[1] #if fp_origin is None: # print("substitute empty origin with source") # fp_origin = fp_recv_from tic = ftntic.make_tic(myaddr_text, address, tic_password, dsta[1], get_addr(db, fp_origin)[1] if fp_origin else None, fp_filename, file_length, file_crc32, other) # code was migrated from echo export, so it cares about blocked exports to update # per-subscription last-sents for x in t.add_item(((tic, (fp_filename, file_length, file_content or (db, file_lo))) if will_export else None), (addr_id, fp_post_time)): # ordering by post_time yield x if not fruitful: print("empty export, must update per-subscriber lastsent up to", latest_post) for x in t.flush(): yield x finally: explock.release() else: print ("cannot acquire fileecho lock for", addr_id) return