def pack_tcp_segment(self, payload='', flag=ACK):
'''
Generate TCP segment.
'''
# tcp header fields
tcp_source = self.local_port # source port
tcp_dest = self.remote_port # destination port
tcp_seq = self.tcp_seq_num
tcp_ack_seq = self.tcp_ack_num
tcp_doff = 5 #4 bit field, size of tcp header, 5 * 4 = 20 bytes
tcp_window = socket.htons (5840) # maximum allowed window size
tcp_check = 0
tcp_urg_ptr = 0
tcp_offset_res = (tcp_doff << 4) + 0
tcp_flags = flag
tcp_header = struct.pack(TCP_HDR_FMT, tcp_source, tcp_dest, tcp_seq, tcp_ack_seq, tcp_offset_res, tcp_flags, tcp_window, tcp_check, tcp_urg_ptr)
# pseudo header fields
source_address = socket.inet_aton(self.local_host)
dest_address = socket.inet_aton(self.remote_host)
placeholder = 0
protocol = socket.IPPROTO_TCP
tcp_length = len(tcp_header) + len(payload)
psh = struct.pack(PSH_FMT, source_address , dest_address , placeholder , protocol , tcp_length);
psh = psh + tcp_header + payload;
tcp_check = checksum(psh)
tcp_header = struct.pack(TCP_HDR_FMT[:-2], tcp_source, tcp_dest, tcp_seq, tcp_ack_seq, tcp_offset_res, tcp_flags, tcp_window) + struct.pack('H' , tcp_check) + struct.pack('!H' , tcp_urg_ptr)
return tcp_header + payload
def make_tcp_packet(sourceIp,destinationIp,sourcePort,destPort,seqNum,ackNum,finFlag,synFlag,pshFlag,ackFlag,user_data):
tcp_doff = 5 #4 bit field, size of tcp header, 5 * 4 = 20 bytes
# tcp flags
rstFlag = 0
urgFlag = 0
window = socket.htons (5840) # Sets the maximum allowed window size
check = 0
urgent_ptr = 0
tcp_offset_res = (tcp_doff << 4) + 0
flags = finFlag + (synFlag << 1) + (rstFlag << 2) + (pshFlag <<3) + (ackFlag << 4) + (urgFlag << 5)
# the ! in the pack format string means network order
dummyheader = pack('!HHLLBBHHH',sourcePort,destPort,seqNum,ackNum, tcp_offset_res,flags,window, check,urgent_ptr)
# now start constructing the packet
packet ='';
# pseudo header fields
# The source and destination IP addresses are required for this.
protocol = socket.IPPROTO_TCP
tcp_length = len(dummyheader) + len(user_data)
psh = pack('!4s4sBBH',socket.inet_aton(sourceIp),socket.inet_aton(destinationIp),0,protocol,tcp_length);
psh = psh + dummyheader + user_data;
# Passes the Pseudo header, the tcp header and the user data to the function checksum in util file.
# It returns the TCP checksum which is stored in variable tcp_check
finalChecksum = checksum(psh) # tcp_checksum
# make the tcp header again and fill the correct checksum - remember checksum is NOT in network byte order
# packs the tcp header, the calculated checksum and tcp_urg_ptr
finalHeader = pack('!HHLLBBH' , sourcePort, destPort, seqNum, ackNum, tcp_offset_res, flags,window) + pack('H' , finalChecksum) + pack('!H' , urgent_ptr)
return finalHeader;
def process_segment(self):
segment = self.recv_sock.recv(1024)
if util.checksum(segment) != 0:
return
header, payload = deserialize(segment)
self.log(header)
self.seq_num = header.seq_num
ack = header.seq_num + len(payload)
if ack in self.segments_acked:
return
self.bytes_received += len(payload)
self.segments_acked.add(ack)
self.ACK = 1 if header.seq_num == self.ack_num else 0
ack_header, ack_segment = self.make_segment(payload, header.FIN)
self.send_sock.sendto(ack_segment, self.send_addr)
if header.FIN:
self.done = True
return
self.store_payload(header.seq_num, payload)
seq_num, payload = self.fetch_next_payload()
while seq_num == self.ack_num:
self.file.write(payload)
self.ack_num += len(payload)
self.pop_next_payload()
seq_num, payload = self.fetch_next_payload()
def __init__(self, data, do_checksum=False):
offset = 0
self.level = bytes_to_int(data[offset:offset + 1])
offset += 1
self.name = bytes_to_int(data[offset:offset + 2])
offset += 2
self.type = bytes_to_int(data[offset:offset + 2])
offset += 2
att_length = bytes_to_int(data[offset:offset + 4])
offset += 4
self.data = data[offset:offset + att_length]
offset += att_length
att_checksum = bytes_to_int(data[offset:offset + 2])
offset += 2
self.length = offset
if do_checksum:
calc_checksum = checksum(self.data)
if calc_checksum != att_checksum:
logger.warn("Checksum: %s != %s" %
(calc_checksum, att_checksum))
else:
calc_checksum = att_checksum
# whether the checksum is ok
self.good_checksum = calc_checksum == att_checksum
def process_segment(self):
segment = self.recv_sock.recv(1024)
if util.checksum(segment) != 0:
return
header, payload = deserialize(segment)
self.log(header)
self.seq_num = header.seq_num
ack = header.seq_num + len(payload)
if ack in self.segments_acked:
return
self.bytes_received += len(payload)
self.segments_acked.add(ack)
self.ACK = 1 if header.seq_num == self.ack_num else 0
ack_header, ack_segment = self.make_segment(payload, header.FIN)
self.send_sock.sendto(ack_segment, self.send_addr)
if header.FIN:
self.done = True
return
self.store_payload(header.seq_num, payload)
seq_num, payload = self.fetch_next_payload()
while seq_num == self.ack_num:
self.file.write(payload)
self.ack_num += len(payload)
self.pop_next_payload()
seq_num, payload = self.fetch_next_payload()
def tcp_header(self, flags=ACK, payload=''):
# tcp header fields
tcp_source = self.localhostport
tcp_dest = self.remotePORT
tcp_seq = self.tcp_seq
tcp_ack_seq = self.tcp_ack_seq
tcp_doff = 5
tcp_window = self.tcp_adwind
tcp_check = 0
tcp_urg_ptr = 0
tcp_offset_res = (tcp_doff << 4) + 0
tcp_flags = flags
tcp_header = struct.pack('!HHLLBBHHH', tcp_source, tcp_dest, tcp_seq, tcp_ack_seq, tcp_offset_res, tcp_flags, tcp_window, tcp_check, tcp_urg_ptr)
# pseudo header fields
source_address = socket.inet_aton(self.localhostip)
dest_address = socket.inet_aton(self.remoteIP)
placeholder = 0
protocol = socket.IPPROTO_TCP
if len(payload) % 2 != 0:
payload += ' '
tcp_length = len(tcp_header) + len(payload)
psh = struct.pack('!4s4sBBH', source_address, dest_address, placeholder, protocol, tcp_length)
psh = psh + tcp_header + payload
tcp_check = util.checksum(psh)
tcp_header = struct.pack('!HHLLBBH', tcp_source, tcp_dest, tcp_seq, tcp_ack_seq, tcp_offset_res, tcp_flags, tcp_window) + struct.pack('H', tcp_check) + struct.pack('!H', tcp_urg_ptr)
return tcp_header + payload
def _tcp_check(self, payload):
# pseudo header fields
source_address = socket.inet_aton(self.local_host)
dest_address = socket.inet_aton(self.remote_host)
placeholder = 0
protocol = socket.IPPROTO_TCP
tcp_length = len(payload)
psh = struct.pack(PSH_FMT, source_address , dest_address , placeholder , protocol , tcp_length);
psh = psh + payload;
return checksum(psh)
def serialize(src_port, dst_port, seq_num, ack_num,
ACK=0, SYN=0, FIN=0, payload=''):
header = TCPSegmentHeader()
header.src_port = src_port
header.dst_port = dst_port
header.seq_num = seq_num
header.ack_num = ack_num
header.hdr_len = 5 # 20 bytes in 32-bit words
header.ACK = ACK
header.SYN = SYN
header.FIN = FIN
header.checksum = util.checksum(header.serialize() + payload)
return header, header.serialize() + payload
def close(self, pickleDir):
if self.csum == None:
self.csum = util.checksum(self.dbFile)
if self.pickleOk:
return
base, ext = os.path.splitext(self.dbFile)
base = os.path.basename(base)
picklePath = pickleDir + '/' + base + '-' + self.dataName + '.pkl'
output = open(picklePath, 'wb')
cPickle.dump(self, output)
output.close()
def construct_tcp_header(ip_saddr, ip_daddr, ip_protocol, tcp_sport,
payload_data, tcp_seq, tcp_ack_seq, flags):
# tcp_sport = 2338
tcp_dport = 80 # destination port
tcp_data_offset = 5 # 和ip header一样，没option field
# tcp flags
tcp_flag_urg = flags[0]
tcp_flag_ack = flags[1]
tcp_flag_psh = flags[2]
tcp_flag_rst = flags[3]
tcp_flag_syn = flags[4]
tcp_flag_fin = flags[5]
tcp_window_size = socket.htons(1500)
tcp_checksum = 0
tcp_urgent_ptr = 0
# pack small fields
tcp_offset_reserv = (tcp_data_offset << 4)
tcp_flags = tcp_flag_fin + (tcp_flag_syn << 1) + (tcp_flag_rst << 2) + (
tcp_flag_psh << 3) + (tcp_flag_ack << 4) + (tcp_flag_urg << 5)
# construct tcp header。
tcp_header = pack('!HHLLBBHHH', tcp_sport, tcp_dport, tcp_seq, tcp_ack_seq,
tcp_offset_reserv, tcp_flags, tcp_window_size,
tcp_checksum, tcp_urgent_ptr)
# pseudo ip header
psh_saddr = ip_saddr
psh_daddr = ip_daddr
psh_reserved = 0
psh_protocol = ip_protocol
psh_tcp_len = len(tcp_header) + len(payload_data)
psh = pack('!4s4sBBH', psh_saddr, psh_daddr, psh_reserved, psh_protocol,
psh_tcp_len)
# final check sum
chk = psh + tcp_header + payload_data
# 必要时追加1字节的padding
if len(chk) % 2 != 0:
chk += '\0'
tcp_checksum = checksum(chk)
# checksum again
tcp_header = pack('!HHLLBBHHH', tcp_sport, tcp_dport, tcp_seq, tcp_ack_seq,
tcp_offset_reserv, tcp_flags, tcp_window_size,
tcp_checksum, tcp_urgent_ptr)
return tcp_header
def process_ack(self):
segment = self.recv_sock.recv(1024)
if util.checksum(segment) != 0:
return
header, payload = deserialize(segment)
if header.seq_num not in self.segments_in_transit:
return
if header.FIN:
self.done = True
if header.seq_num == self.ack_num:
self.reset_timer()
self.ACK = 1
else:
self.ACK = 0
self.segments_in_transit.pop(header.seq_num)
if self.segments_in_transit:
seq_num, _ = self.next_segment()
self.ack_num = seq_num
self.update_stats()
def pack(self):
tcp_header = struct.pack(TCP_HDR_FMT, self.tcp_source, self.tcp_dest, self.tcp_seq, self.tcp_ack_seq, self.tcp_offset_res, self.tcp_flags, self.tcp_window, self.tcp_check, self.tcp_urg_ptr)
# pseudo header fields
source_address = socket.inet_aton(self.local_host)
dest_address = socket.inet_aton(self.remote_host)
placeholder = 0
protocol = socket.IPPROTO_TCP
tcp_length = len(tcp_header) + len(self.payload)
psh = struct.pack(PSH_FMT, source_address , dest_address , placeholder , protocol , tcp_length);
psh = psh + tcp_header + self.payload;
tcp_check = checksum(psh)
# print tcp_checksum
tcp_header = struct.pack(TCP_HDR_FMT[:-2], self.tcp_source, self.tcp_dest, self.tcp_seq, self.tcp_ack_seq, self.tcp_offset_res, self.tcp_flags, self.tcp_window) + struct.pack('H' , self.tcp_check) + struct.pack('!H' , self.tcp_urg_ptr)
self.raw = tcp_header + self.payload
def process_ack(self):
segment = self.recv_sock.recv(1024)
if util.checksum(segment) != 0:
return
header, payload = deserialize(segment)
if header.seq_num not in self.segments_in_transit:
return
if header.FIN:
self.done = True
if header.seq_num == self.ack_num:
self.reset_timer()
self.ACK = 1
else:
self.ACK = 0
self.segments_in_transit.pop(header.seq_num)
if self.segments_in_transit:
seq_num, _ = self.next_segment()
self.ack_num = seq_num
self.update_stats()
def serialize(src_port,
dst_port,
seq_num,
ack_num,
ACK=0,
SYN=0,
FIN=0,
payload=''):
header = TCPSegmentHeader()
header.src_port = src_port
header.dst_port = dst_port
header.seq_num = seq_num
header.ack_num = ack_num
header.hdr_len = 5 # 20 bytes in 32-bit words
header.ACK = ACK
header.SYN = SYN
header.FIN = FIN
header.checksum = util.checksum(header.serialize() + payload)
return header, header.serialize() + payload
def __init__(self, data, do_checksum=False):
offset = 0
self.level = bytes_to_int(data[offset: offset+1]); offset += 1
self.name = bytes_to_int(data[offset:offset+2]); offset += 2
self.type = bytes_to_int(data[offset:offset+2]); offset += 2
att_length = bytes_to_int(data[offset:offset+4]); offset += 4
self.data = data[offset:offset+att_length]; offset += att_length
att_checksum = bytes_to_int(data[offset:offset+2]); offset += 2
self.length = offset
if do_checksum:
calc_checksum = checksum(self.data)
if calc_checksum != att_checksum:
logger.warn("Checksum: %s != %s" % (calc_checksum, att_checksum))
else:
calc_checksum = att_checksum
# whether the checksum is ok
self.good_checksum = calc_checksum == att_checksum
def unpack_ip_datagram(self, datagram):
'''
Paser IP datagram
'''
# print len(datagram)
hdr_fields = struct.unpack(IP_HDR_FMT, datagram[:20])
ip_header_size = struct.calcsize(IP_HDR_FMT)
ip_ver_ihl = hdr_fields[0]
ip_ihl = ip_ver_ihl - (4 << 4)
if ip_ihl > 5:
opts_size = (self.ip_ihl - 5) * 4
ip_header_size += opts_size
ip_headers = datagram[:ip_header_size]
data = datagram[ip_header_size:hdr_fields[2]]
ip_check = checksum(ip_headers)
return IPDatagram(ip_daddr=socket.inet_ntoa(hdr_fields[-1]), ip_saddr=socket.inet_ntoa(hdr_fields[-2]),
ip_frag_off=hdr_fields[4], ip_id=hdr_fields[3], ip_tlen=hdr_fields[2], ip_check=ip_check, data=data)
def pack_ip_datagram(self, payload):
'''
Generate IP datagram.
`payload` is TCP segment
'''
ip_tos = 0
ip_tot_len = 20 + len(payload)
ip_id = self.ip_id #Id of this packet
ip_frag_off = 0
ip_ttl = 255
ip_proto = socket.IPPROTO_TCP
ip_check = 0
ip_saddr = socket.inet_aton (self.local_host)
ip_daddr = socket.inet_aton (self.remote_host)
ip_ihl_ver = (4 << 4) + 5
ip_header = struct.pack(IP_HDR_FMT, ip_ihl_ver, ip_tos, ip_tot_len, ip_id, ip_frag_off, ip_ttl, ip_proto, ip_check, ip_saddr, ip_daddr)
ip_check = checksum(ip_header)
ip_header = struct.pack(IP_HDR_FMT, ip_ihl_ver, ip_tos, ip_tot_len, ip_id, ip_frag_off, ip_ttl, ip_proto, ip_check, ip_saddr, ip_daddr)
# print struct.unpack(IP_HDR_FMT, ip_header)
return ip_header + payload
def open(dbFile, dataName, pickleDir):
base, ext = os.path.splitext(dbFile)
base = os.path.basename(base)
picklePath = pickleDir + '/' + base + '-' + dataName + '.pkl'
serials = None
try:
pickleFile = open(picklePath, 'r')
serials = cPickle.load(pickleFile)
if serials.dataName != dataName:
raise Exception('unexpected dataName')
if serials.csum != util.checksum(dbFile):
raise Exception('checksum mismatch: stale pickle?')
# This following members are ephemeral
serials.dbFile = dbFile
serials.pickleOk = True
pickleFile.close()
except IOError, e:
if e.errno != errno.ENOENT:
raise
serials = MtraceSerials(dbFile, dataName)
def get_feed(self) -> None:
"""Downloads new static GTFS data, checks if different than existing data,
unzips, and then generates additional csv files:
merge_trips_and_stops combines trips, stops, and stop_times to make route_stops_with_names
load_time_between_stops calculates time b/w each pair of adjacent stops using stop_times
"""
u.log.info('parser: Downloading GTFS static data from %s', self.url)
try:
new_data = requests.get(self.url, allow_redirects=True, timeout=10)
except requests.exceptions.RequestException as err:
raise u.UpdateFailed(f'{err}, failed to connect to {self.url}')
_zipfile = f'{u.STATIC_PATH}/static_data.zip'
with open(_zipfile, 'wb') as zip_out_stream:
zip_out_stream.write(new_data.content)
self.current_checksum = u.checksum(_zipfile)
with suppress(ResponseError):
if self.redis_server.exists('static_json'):
if self.current_checksum == self.latest_checksum:
raise u.UpdateFailed(
'Static data checksum matches previously parsed static data. No new data!')
_rawpath = f'{u.STATIC_PATH}/raw'
shutil.rmtree(_rawpath, ignore_errors=True)
u.log.info('parser: Extracting static GTFS zip')
try:
with zipfile.ZipFile(_zipfile, "r") as zip_ref:
zip_ref.extractall(_rawpath)
except zipfile.BadZipFile as err:
raise u.UpdateFailed(err)
self.get_additional_data()
self.merge_trips_and_stops()
def get_file(nextSeq,clientSeqNum):
#creating the file based on the fileName request by the user
object = open(fileName,'w')
i = 0
firstServerSeqNumber = nextSeq;
#taking the current time of the clock to implement the timer functionality
startTime = time.clock()
currentTime = time.clock()
checksumCorrect = True;
#while i < 3:
while ((currentTime - startTime) < 60):
currentTime = time.clock();
i = i +1
packet = receiveSocket.recvfrom(tcp_source_global)
#packet string from tuple
packet = packet[0]
ipHeader = packet[0:20]
header = unpack('!BBHHHBBH4s4s' , ipHeader)
version_ihl = header[0]
version = version_ihl >> 4
ihl = version_ihl & 0xF
ipHeaderLength = ihl * 4
protocol = header[6]
ipChecksum = header[7] #incoming checksum from the header
#Only process the data if the IP checksum on the IP header is correct
if(checksum(ipHeader) == 0):
#TCP protocol
tcp_header = packet[ipHeaderLength:ipHeaderLength+20]
tcph = unpack('!HHLLBBHHH' , tcp_header)
#unpacking the header to retreive header details
source_port = tcph[0]
dest_port = tcph[1]
sequence = tcph[2]
acknowledgement = tcph[3]
doff_reserved = tcph[4]
tcp_doff = doff_reserved >> 4
tcpFlag = tcph[5]
window = tcph[6]
urgPointer = tcph[7]
urgFlag = (tcpFlag >> 3)
ackFlag = (tcpFlag >> 4)
finFlag = (tcpFlag >> 8)
synFlag = (tcpFlag >> 7)
rstFlag = (tcpFlag >> 6)
pshFlag = (tcpFlag >> 5)
window = tcph[6]
#Getting the checksum in the incoming packet
incomingChecksum = tcph[7]
tcph_length = doff_reserved >> 4
if (socket.inet_ntoa(header[8]) == destinationIp) and (sequence == nextSeq) and (acknowledgement == clientSeqNum):
h_size = ipHeaderLength + tcph_length * 4
data_size = len(packet) - h_size
data = packet[h_size:]
dataLength = len(data)
#incase the length of the data is 6,it means the packet has been padded with Zeroes
if len(data) == 6:
dataLength = 0
nextSeq = sequence + dataLength
if (dataLength != 0):
#Call the verifyChecksum() to check if the checksum is correct
checksumcorrect = verifyChecksum(tcp_source_global,sourceIp,destinationIp,
incomingChecksum,sequence,acknowledgement,
doff_reserved,tcpFlag,window,data)
#get data from the header only if the checksum of the TCP header is correct
if(checksumCorrect):
#Sending the ACK for the packet after the verification of the IP and TCP checksum
sendAckOrFin(ackNumber = nextSeq,seqNum = clientSeqNum,ackFlag = 1,finFlag = 0)
if firstServerSeqNumber == sequence:
data = getDataFromFirstResponse(data);
object.write(data)
else:
object.write(data)
if (dataLength == 0) and (firstServerSeqNumber != sequence):
#Sending the ACK for the packet after the verification of the IP and TCP checksum
sendAckOrFin(ackNumber = nextSeq+1,seqNum = clientSeqNum ,ackFlag = 1,finFlag = 0)
#closing the sockets
receiveSocket.close();
sendSocket.close();
sys.exit()
