def _receive(self): while True: data = self._pipe.buffer pos = data.find(b"=") if pos >= 0: data = data[:pos] data = data.replace(b"\r", b"").replace(b"\n", b"") data = data.translate(self.TABLE) # TODO: check data size overflow self._crc = crc32(data, self._crc) self._file.write(data) if pos >= 0: # Escape character (equals sign) self._pipe.buffer = self._pipe.buffer[pos + 1:] while True: byte = yield from self._pipe.read_one() if byte not in b"\r\n": break # TODO: check for size overflow [byte] = byte data = bytes(((byte - 64 - 42) & bitmask(8),)) self._crc = crc32(data, self._crc) self._file.write(data) else: try: self._pipe.buffer = yield except EOFError: break
def readReg(reg): global crcerrorcount,crcerrorcountout tries = 5 okay = 0 while (okay != 1 and tries > 0): tries = tries -1 send = [0x02,reg,0,0,0,0] crcout = (binascii.crc32("".join( chr( val ) for val in send[0:6])) >> 16) & 0xFFFF resp = SPI.xfer(send+[(crcout>>8)&0xFF,crcout&0xFF,0,0,0,0,0,0,0,0,0]) val = (resp[10] << 24) + (resp[11] << 16) + (resp[12] << 8) + (resp[13]) crcin = (resp[14] << 8) + resp[15] vals = "".join( chr( val ) for val in resp[0:14] ) crcis = (binascii.crc32(vals) >> 16) & 0xFFFF okay = 1 if (crcin != crcis): crcerrorcount = crcerrorcount +1 okay = 0 #print("CRC ERROR ON RECV") #print("crcrep "+format(crcin,'02x')+" vs "+format(crcis,'02x')) if (resp[16] != 0xFF): crcerrorcountout = crcerrorcountout +1 okay = 0 #print("CRC SEND ERROR: "+format(resp[16],'02x')) #print("reg "+str(reg)+" "+str(val)) if (okay == 0): print("ERROR READING") return val
def process_request(self, request): if not request.META.has_key(self.header): raise ImproperlyConfigured('Requires SSL on frontend web server' ' to be enabled and properly configured' ' to pass SSL_* headers.') if request.META.get(self.header, None) == 'SUCCESS': (pem, serial, dn) = ( request.META.get(self.header_pem, ''), request.META.get(self.header_serial, ''), request.META.get(self.header_dn, '') ) users = map(lambda i: i.certificate.user, CertificateIndex.objects.filter( pem_index=crc32(pem)&0xffffffff, serial_index=crc32(serial)&0xffffffff, dn_index=crc32(dn)&0xffffffff ) ) if len(users) > 1: users = filter(lambda u: any(u.certificates.filter(pem=pem, serial=serial, dn=dn)), users) if not any(users): certificate, created = Certificate.objects.get_or_create(pem=pem, serial=serial, dn=dn) return if len(users) > 1: raise Exception('Too many users for certificate') user = users[0] if request.user.is_authenticated(): if request.user == user: return user = auth.authenticate(user=user) if user: request.user = user auth.login(request, user)
def compute_crc(buf, offset): before_buffer = (ctypes.c_uint8 * offset).from_buffer(buf) zero = (ctypes.c_uint8 * 4)() after_buffer = (ctypes.c_uint8 * (ctypes.sizeof(buf) - offset - 4)).from_buffer(buf, offset + 4) crc = binascii.crc32(before_buffer) crc = binascii.crc32(zero, crc) return binascii.crc32(after_buffer, crc)
def crc32_checksum(filepath, start=0, length=0): crc = 0 with open(filepath, 'rb') as fp: if start: fp.seek(start) bytes_read = 0 while True: data = fp.read(8192) if not data: break if length > 0: bytes_read += len(data) if bytes_read >= length: if bytes_read > length: data = data[:length - bytes_read] crc = binascii.crc32(data, crc) break crc = binascii.crc32(data, crc) return crc & 0xFFFFFFFF
def unique_id(self, seen=None, account_id=None): """ Get an unique ID for the transaction based on date, amount and raw. :param seen: if given, the method uses this dictionary as a cache to prevent several transactions with the same values to have the same unique ID. :type seen: :class:`dict` :param account_id: if given, add the account ID in data used to create the unique ID. Can be useful if you want your ID to be unique across several accounts. :type account_id: :class:`str` :returns: an unique ID encoded in 8 length hexadecimal string (for example ``'a64e1bc9'``) :rtype: :class:`str` """ crc = crc32(str(self.date)) crc = crc32(str(self.amount), crc) crc = crc32(re.sub('[ ]+', ' ', self.raw.encode("utf-8")), crc) if account_id is not None: crc = crc32(str(account_id), crc) if seen is not None: while crc in seen: crc = crc32("*", crc) seen.add(crc) return "%08x" % (crc & 0xffffffff)
def _process_chunk( self, from_buffer, to_buffer, chunk_buffer, output_file, id, chunk_type): """ Method called on each chunk Returns a dict with meta-data on chunk """ if chunk_type > CHUNK_TYPE_KEEP: chunk_len = len(chunk_buffer) else: chunk_len = 0 from_crc = binascii.crc32(from_buffer) to_crc = binascii.crc32(to_buffer) chunkheader = ChunkHeader(CHUNK_HEADER_MAGIC, CHUNK_HEADER_VERSION, chunk_type, id, chunk_len, 0, len(from_buffer), from_crc, len(to_buffer), to_crc) chunkheader.crc = binascii.crc32(bytearray(chunkheader)) arr = bytearray(chunkheader) assert len(arr) == CHUNK_HEADER_SIZE chunkfile = open(output_file, "wb") chunkfile.write(arr) if chunk_type > CHUNK_TYPE_KEEP: chunkfile.write(chunk_buffer) chunkfile.close() chunk_len += CHUNK_HEADER_SIZE return { "_id": id, "_type": chunk_type, "_name": os.path.basename(output_file), "crc_from": "0x%x" % (from_crc & 0xffffffff), "crc_to": "0x%x" % (to_crc & 0xffffffff), "crc_header": "0x%x" % (chunkheader.crc & 0xffffffff), "chunk_header": "0x%x" % (binascii.crc32( open( output_file, "rb").read()) & 0xffffffff), "size_output": chunk_len, "size_input": len(to_buffer), "ratio": "%0.5f" % (1.0 * chunk_len / len(to_buffer)), "delta": chunk_len - len(to_buffer)}
def save_max_drive(self, f): if lzari == None: raise error, ("The lzari module is needed to " " decompress MAX Drive saves.") iconsysname = "" icon_sys = self.get_icon_sys() if icon_sys != None: title = icon_sys_title(icon_sys, "ascii") if len(title[0]) > 0 and title[0][-1] != ' ': iconsysname = title[0] + " " + title[1].strip() else: iconsysname = title[0] + title[1].rstrip() s = "" dirent = self.dirent for i in range(dirent[2]): (ent, data) = self.get_file(i) if not mode_is_file(ent[0]): raise error, "Non-file in save file." s += struct.pack("<L32s", ent[2], ent[8]) s += data s += "\0" * (round_up(len(s) + 8, 16) - 8 - len(s)) length = len(s) progress = "compressing " + dirent[8] + ": " compressed = lzari.encode(s, progress) hdr = struct.pack("<12sL32s32sLLL", PS2SAVE_MAX_MAGIC, 0, dirent[8], iconsysname, len(compressed) + 4, dirent[2], length) crc = binascii.crc32(hdr) crc = binascii.crc32(compressed, crc) f.write(struct.pack("<12sL32s32sLLL", PS2SAVE_MAX_MAGIC, crc & 0xFFFFFFFF, dirent[8], iconsysname, len(compressed) + 4, dirent[2], length)) f.write(compressed) f.flush()
def compute_checksum(path): r"""Returns the checksum of the file pointed by path Parameters ---------- path : str The path to compute the checksum Returns ------- int The file checksum """ crc = 0 filepaths = [] if isdir(path): for name, dirs, files in walk(path): join_f = partial(join, name) filepaths.extend(list(map(join_f, files))) else: filepaths.append(path) for fp in filepaths: with open(fp, "Ub") as f: # Go line by line so we don't need to load the entire file for line in f: if crc is None: crc = crc32(line) else: crc = crc32(line, crc) # We need the & 0xffffffff in order to get the same numeric value across # all python versions and platforms return crc & 0xffffffff
def genkey(name: str, version: Version) -> int: """Return the key for `name` according to `version`. This is a low-level function that rarely needs to be used directly. """ n = name.encode(ENCODING, "strict").upper() if version is Version.TD or version is Version.RA: l = len(n) k = 0 i = 0 while i < l: a = 0 for j in range(4): a >>= 8 if i < l: a |= (n[i] << 24) i += 1 k = (k << 1 | k >> 31) + a & 4294967295 return k if version is Version.TS: l = len(n) a = l & -4 if l & 3: n += bytes((l - a,)) n += bytes((n[a],)) * (3 - (l & 3)) return binascii.crc32(n) if version is Version.RG: return binascii.crc32(n) raise TypeError("`version` must be a Version enumeration member")
def encode(self, id): body = self._encode_message_body() header = struct.pack(_HEADER_SPEC, self.msgtype, id, len(body)) raw_crc = crc32(body, crc32(header)) & 0xffffffff crc = struct.pack(_FOOTER_SPEC, raw_crc) logger.debug("header:%s body:%s crc:%s", repr(header), repr(body)[:30], repr(crc)) return b''.join((header, body, crc))
def packet_sender(self, p_type, id_value, value, type_v): # This is building packets, the explanation can be found in Robert's # documentation which contains the structure of each packet. But it is # unlikely you'll have to modify start/adress/length/record/index # Only s_id (type of ID, can be found in DriveManager), param_ID (the # actual ID, also found in Drive Manager) and number (value to write in # ID) will have to be changed if you want to do other simple tasks. # The 2685547530 'magic' number is given in the documentation start = pack('<L', 2685547530) adress = pack('<L', 0) s_id = pack('<L', p_type) length = pack('<L', 16) record = pack('<l', -1) param_id = pack('<L', id_value) index = pack('<L', 0) if type_v == 'int32': number = pack('<L', value) elif type_v == 'float': number = pack('<f', value) else: number = pack('<i', value) # The 3344495068/3344494807 are necessary to obtain a correct checksum # They have been found by sniffing packets, and bruteforcing numbers # until both checksum are the same. checksum = crc32(start + adress + s_id + length, 3344495068) checksum_start = pack('<L', checksum) checksum = crc32(record + param_id + index + number, 3344494807) checksum_data = pack('<L', checksum) packet = start + adress + s_id + length + checksum_start packet += checksum_data + record + param_id + index + number self.s.send(packet) data = self.s.recv(self.buffer) return data
def write_data(self, write_finished_cb): # First generate the header part header_data = struct.pack("<BIBB", 0xEB, self.pins, self.vid, self.pid) header_crc = crc32(header_data) & 0x0FF header_data += struct.pack("B", header_crc) # Now generate the elements part elem = "" logger.info(self.elements.keys()) for element in reversed(self.elements.keys()): elem_string = self.elements[element] # logger.info(">>>> {}".format(elem_string)) key_encoding = self._rev_element_mapping[element] elem += struct.pack("BB", key_encoding, len(elem_string)) elem += elem_string elem_data = struct.pack("BB", 0x00, len(elem)) elem_data += elem elem_crc = crc32(elem_data) & 0x0FF elem_data += struct.pack("B", elem_crc) data = header_data + elem_data # Write data p = "" for s in data: p += "0x{:02X} ".format(ord(s)) logger.info(p) self.mem_handler.write(self, 0x00, struct.unpack("B" * len(data), data)) self._write_finished_cb = write_finished_cb
def __compare_current_version_data_with_data(self, p_version_data, p_data): """ Compare a version data with some new data using crc32. @param p_version_data: version data @param p_data: new data """ return crc32(p_version_data) != crc32(p_data)
def calculate_crc32_checksum(data): if sys.version_info < (3, 0): return "%08x" % (binascii.crc32(data) & 0xffffffff) else: if isinstance(data, str): data = str.encode(data) return "%08x" % (binascii.crc32(data) & 0xffffffff)
def test_save(self): filelike = open('jieba.cache', 'rb').read() self.bucket.save('new_jieba.cache', filelike) contents = self.bucket.get_object_contents('new_jieba.cache') self.assertEqual(binascii.crc32(filelike) & 0xffffffff, binascii.crc32(contents) & 0xffffffff) self.bucket.delete_object('new_jieba.cache')
def write_data(self, write_finished_cb): # First generate the header part header_data = struct.pack('<BIBB', 0xEB, self.pins, self.vid, self.pid) header_crc = crc32(header_data) & 0x0ff header_data += struct.pack('B', header_crc) # Now generate the elements part elem = bytearray() logger.info(list(self.elements.keys())) for element in reversed(list(self.elements.keys())): elem_string = self.elements[element] # logger.info(">>>> {}".format(elem_string)) key_encoding = self._rev_element_mapping[element] elem += struct.pack('BB', key_encoding, len(elem_string)) elem += bytearray(elem_string.encode('ISO-8859-1')) elem_data = struct.pack('BB', 0x00, len(elem)) elem_data += elem elem_crc = crc32(elem_data) & 0x0ff elem_data += struct.pack('B', elem_crc) data = header_data + elem_data self.mem_handler.write(self, 0x00, struct.unpack('B' * len(data), data)) self._write_finished_cb = write_finished_cb
def json_search(request): search_query = Search(indexes=['rt_main'], config=SphinxitConfig) keyword = request.GET['keyword'] start = request.GET.get('start', 0) count = request.GET.get('count', 10) sort = request.GET.get('sort', '') category = request.GET.get('category', '') if request.GET.get('base64') == '1': keyword = keyword.decode('base64').decode('utf8') mckey = str(binascii.crc32((u'%s%s%s%s%s' % (keyword,start,count,sort,category)).encode('utf8')) & 0xFFFFFFFFL) cache = mc.get(mckey) if cache: print 'bingo', keyword.encode('utf8'), mckey return HttpResponse(cache) q = search_query.match(keyword) if category: q = q.filter(category__eq=binascii.crc32(category)&0xFFFFFFFFL) if sort == 'create_time': q = q.order_by('create_time', 'desc') if sort == 'length': q = q.order_by('length', 'desc') q = q.limit(start, count) q2 = search_query.match(keyword).select('category', Count()).group_by('category').named('cats') res = q.ask(subqueries=[q2]) jsp = JsonResponse(res) mc.set(mckey, jsp.content) return jsp
def __init__(self, ffi, preamble, tmpdir=None, modulename=None, ext_package=None, tag='', force_generic_engine=False, **kwds): self.ffi = ffi self.preamble = preamble if not modulename: flattened_kwds = ffiplatform.flatten(kwds) vengine_class = _locate_engine_class(ffi, force_generic_engine) self._vengine = vengine_class(self) self._vengine.patch_extension_kwds(kwds) self.kwds = kwds # if modulename: if tag: raise TypeError("can't specify both 'modulename' and 'tag'") else: key = '\x00'.join([sys.version[:3], __version__, preamble, flattened_kwds] + ffi._cdefsources) if sys.version_info >= (3,): key = key.encode('utf-8') k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff) k1 = k1.lstrip('0x').rstrip('L') k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff) k2 = k2.lstrip('0').rstrip('L') modulename = '_cffi_%s_%s%s%s' % (tag, self._vengine._class_key, k1, k2) suffix = _get_so_suffixes()[0] self.tmpdir = tmpdir or _caller_dir_pycache() self.sourcefilename = os.path.join(self.tmpdir, modulename + '.c') self.modulefilename = os.path.join(self.tmpdir, modulename + suffix) self.ext_package = ext_package self._has_source = False self._has_module = False
def client_thread(self): while True: self.lock.acquire() if self.mode == "a": self.event_to_send = len(self.events) - 1 if self.event_to_send != -1: print "Sending event: %s" % (self.format_event(event)) theevent = self.events[self.event_to_send] rawdata = theevent["data"].decode("string-escape") theevent["crc"] = crc32(rawdata) self.proxy.send_de(self.events[self.event_to_send]) del self.events[self.event_to_send] self.event_to_send = -1 newevents = self.proxy.getdebugq() for event in newevents: d = event["data"].decode("string-escape") dcrc = crc32(d) print "CRC: %08x for eventid:%s" % (dcrc, event["eventid"]) self.events.append(event) self.lock.release() time.sleep(.1)
def try_to_recover(): if response['offset'] == 0: # Nothing to recover return expected_crc = binascii.crc32(firmware[:response['offset']]) & 0xFFFFFFFF remainder = response['offset'] % response['max_size'] if expected_crc != response['crc']: # Invalid CRC. Remove corrupted data. response['offset'] -= remainder if remainder != 0 else response['max_size'] response['crc'] = \ binascii.crc32(firmware[:response['offset']]) & 0xFFFFFFFF return if (remainder != 0) and (response['offset'] != len(firmware)): # Send rest of the page. try: to_send = firmware[response['offset'] : response['offset'] + response['max_size'] - remainder] response['crc'] = self.__stream_data(data = to_send, crc = response['crc'], offset = response['offset']) response['offset'] += len(to_send) except ValidationException: # Remove corrupted data. response['offset'] -= remainder response['crc'] = \ binascii.crc32(firmware[:response['offset']]) & 0xFFFFFFFF return self.__execute() self._send_event(event_type=DfuEvent.PROGRESS_EVENT, progress=response['offset'])
def _sweep(self, sweep_config): response = self.alh.post("sensing/quickSweepBin", "dev %d conf %d ch %d:%d:%d" % ( sweep_config.config.device.id, sweep_config.config.id, sweep_config.start_ch, sweep_config.step_ch, sweep_config.stop_ch)) data = response[:-4] crc = response[-4:] their_crc = struct.unpack("i", crc[-4:])[0] our_crc = binascii.crc32(data) if their_crc != our_crc: # Firmware versions 2.29 only calculate CRC on the # first half of the response due to a bug our_crc = binascii.crc32(data[:len(data)/2]) if their_crc != our_crc: raise CRCError else: log.warning("working around broken CRC calculation! " "please upgrade node firmware") assert sweep_config.num_channels * 2 == len(data) result = [] for n in xrange(0, len(data), 2): datum = data[n:n+2] dbm = struct.unpack("h", datum)[0]*1e-2 result.append(dbm) return result
def recv_data(self): # self.log("Waiting for content length..." self.check_timeout(3) # s1, s2 = self.sock.recv(8), self.sock.recv(8) # #print 'recv', s1, s2 # content_length, crc32 = hexstring2int( s1 ), hexstring2int( s2 ) # #print content_length, crc32 content_length, crc32 = tuple([int(s) for s in self.readline().split()]) self.log("Content-Length: %d\n" % content_length) recvbytes = 0 content = [] # self.log("Receiving data...") while recvbytes < content_length: recvstring = self.sock.recv(min(content_length - recvbytes, 2048)) recvbytes += len(recvstring) self.log("Received: %d bytes (%3.1f%%)\r" % (recvbytes, (100.0 * recvbytes / content_length))) content.append(recvstring) self.log("Received: %d bytes. " % (recvbytes) + "\n") content = "".join(content) if crc32 != binascii.crc32(content): str = """\ expected crc %d, calculated crc %d expected content length %d, content length %d """ % ( crc32, binascii.crc32(content), content_length, len(content), ) raise IOError("CRC error while receiving data:\n" + str) return content
def tryParseHeader(self, buf): """ Try to parse a buffer as a GPT header, return None on failure """ if len(buf) < self._gpt_size: raise NoGPT("Failed to locate GPT") data = dict(zip( self._gpt_head_fmt.keys(), self._gpt_struct.unpack(buf[0:self._gpt_size]) )) if data['header'] != self._gpt_header: return None tmp = data['crc32'] data['crc32'] = 0 crc = crc32(self._gpt_struct.pack(*[data[k] for k in self._gpt_head_fmt.keys()])) data['crc32'] = tmp # just in case future ones are larger crc = crc32(buf[self._gpt_size:data['headerSize']], crc) crc &= 0xFFFFFFFF if crc != data['crc32']: verbose("Warning: Found GPT candidate with bad CRC") return None return data
def read_deflated(self, count = -1): if count < 1 or count > self.zinfo.file_size - self.byteswritten: count = self.zinfo.file_size - self.byteswritten if count < 1: return '' count = count self.fp.seek(self.zinfo.file_offset + self.bytesread) # First, fill up the buffer. while len(self.buffer) < count: bytes = self.fp.read(min(self.zinfo.compress_size - self.bytesread, 4096)) self.bytesread += len(bytes) result = self.dc.decompress(bytes) if len(result): self.buffer += result self.crc = binascii.crc32(result, self.crc) if self.bytesread == self.zinfo.compress_size: bytes = self.dc.decompress('Z') + self.dc.flush() if len(bytes): self.buffer += bytes self.crc = binascii.crc32(bytes, self.crc) self._finalize() retval = self.buffer[:count] self.byteswritten += len(retval) self.buffer = self.buffer[count:] return retval
def VerifyPy2Exe(): """Checks for presense of the modified zipextimporter.py. We no longer want the modified version, and need the original.""" # CRCs of the correct version, from both 'r', and 'rb' mode crcGood = [0xA56E66A6, 0x57925DA8] path = os.path.join(sys.prefix, u'Lib', u'site-packages', u'zipextimporter.py') # First we'll test using 'r' mode, this way if the line endings differ, # but the contents are the same, the crc will still be equal with open(os.path.join(scripts, u'zipextimporter.py'), 'r') as ins: crcBad = binascii.crc32(ins.read()) crcBad &= 0xFFFFFFFFL with open(path, 'r') as ins: crcTest = binascii.crc32(ins.read()) crcTest &= 0xFFFFFFFFL if crcTest == crcBad: # Definitely using the old modified version, need to reinstall return False if crcTest in crcGood: # Definitely using the un-modified version, good to go return True # Now check if the current file's crc in 'rb' mode matches a known "good" # crc. with open(path, 'rb') as ins: crcTest = binascii.crc32(ins.read()) crcTest &= 0xFFFFFFFFL if crcTest in crcGood: # Definitely using the un-modified version return True # Last test: see if the modified lines are present with open(path, 'r') as ins: return 'filename = fullname.replace(".","\\")' in ins.read()
def calc_hash(self, widget, file_path): selection = self.hash_tree_view.get_selection() model, it = selection.get_selected() model.clear() for hash_type in self.type_conf.keys(): if self.type_conf[hash_type] == '1' or self.type_conf[hash_type] == 'True':# If it's True we should calc this hash f = open(file_path, 'rb') if hash_type.lower() in ('md5', 'sha1', 'sha512', 'sha224', 'sha256', 'sha384'): # hashlib function function = "hashlib.{0}()".format(hash_type.lower()) m = eval(function) data = f.read(10240) while (len(data) != 0): m.update(data) data = f.read(10240) hash_value = str(m.hexdigest()) elif hash_type.lower() in ('crc32'): m = binascii.crc32("") & 0xffffffff data = f.read(10240) while (len(data) != 0): m = binascii.crc32(data, m) & 0xffffffff data = f.read(10240) hash_value = str(hex(m))[2:] selection = self.hash_tree_view.get_selection() model, it = selection.get_selected() model.append([hash_type, hash_value]) f.close() self.hash_tree_view.set_cursor(0) return
def recv_file(self, filename): # the reason for this function is to store everything into a file and not have # the whole content in the memory at the same time # recv_data() actually has to content twice in the memory at the same time! self.check_timeout(3) f = open(filename, "wb") content_length, crc32 = tuple([int(s) for s in self.readline().split()]) self.log("Content-Length: %d\n" % content_length) recvbytes = 0 content = [] # self.log("Receiving data...") recvstring = self.sock.recv(min(content_length - recvbytes, 2048)) recvbytes += len(recvstring) f.write(recvstring) crc = binascii.crc32(recvstring) while recvbytes < content_length: recvstring = self.sock.recv(min(content_length - recvbytes, 2048)) recvbytes += len(recvstring) f.write(recvstring) crc = binascii.crc32(recvstring, crc) f.close() self.log("Received: %d bytes. " % (recvbytes) + "\n") if crc32 != crc: str = """\ expected crc %d, calculated crc %d expected content length %d, content length %d """ % ( crc32, crc, content_length, recvbytes, ) raise IOError("CRC error while receiving data:\n" + str)
def update(self, publisher, **kwargs): """ Handle notifications from other classes we are observing. Typically this is a new event that needs to be placed into the debug queue. @param publisher: The object that sent the update """ #print "[*] Debugger: got update event. Not ignoring it." event = kwargs["event"] if event == "debug": # self.log.debug("Debugger: update: adding event to debug queue") de = DebugEvent() de.eventid = kwargs["eventid"] de.event = kwargs["event"] de.source = kwargs["src"] de.destination = kwargs["dest"] de.data = kwargs["thedata"] de.direction = kwargs["direction"] de.crc = binascii.crc32(kwargs["thedata"]) self.log.debug("Debug.update: newevent: %08x [eventid:%s]" % (binascii.crc32(kwargs["thedata"]), kwargs["eventid"])) self.debugq.put(de)
def get_ffi(): import cffi ffi = cffi.FFI() # Load C definitions dir_path = dirname(abspath(inspect.getfile(inspect.currentframe()))) decl_path = os.path.join(dir_path, 'decl.h') with codecs.open(decl_path, 'r', 'utf-8') as header: ffi.cdef(header.read()) # The modulename # Simplified version of what cffi does: remove kwargs and vengine preamble = "#include <git2.h>" key = [sys.version[:3], cffi.__version__, preamble] + ffi._cdefsources key = '\x00'.join(key) if sys.version_info >= (3,): key = key.encode('utf-8') k1 = hex(crc32(key[0::2]) & 0xffffffff).lstrip('0x').rstrip('L') k2 = hex(crc32(key[1::2]) & 0xffffffff).lstrip('0').rstrip('L') modulename = 'pygit2_cffi_%s%s' % (k1, k2) # Load extension module libgit2_bin, libgit2_include, libgit2_lib = get_libgit2_paths() C = ffi.verify(preamble, modulename=modulename, libraries=["git2"], include_dirs=[libgit2_include], library_dirs=[libgit2_lib]) # Ok return ffi, C
def hexdigest(this): result = binascii.crc32(this.content) & 0xFFFFFFFF return "%08X" % result
def crc32(self, data): return binascii.crc32(data) & 0xFFFFFFFF
def handle(self, *args, **options): files = options['files'] admin = User.objects.get(username='******') wells = set() tz = pytz.timezone('Europe/Amsterdam') for fname in files: logger.info('Importing data from {}'.format(fname)) df = pd.read_csv(fname, sep='\t', index_col=0, parse_dates=True, na_values=['-']) df.drop('Datum', axis=1, inplace=True) span = [tz.localize(df.index.min()), tz.localize(df.index.max())] start, stop = span screens = set() for col in df.columns: serial, _peilbuis, name = map(lambda x: x.strip(), re.split('[:-]', col)) series = df[col] logger.info(series.name) try: datalogger = Datalogger.objects.get(serial=serial) datasource = LoggerDatasource.objects.get( logger=datalogger) io = StringIO() io.write('Datum\t{}\n'.format(name)) series.to_csv(io, sep='\t', header=False) contents = io.getvalue() crc = abs(binascii.crc32(contents)) filename = 'Export_{}_{}_{:%Y%m%d}_{:%Y%m%d}'.format( serial, name, start, stop) sourcefile = SourceFile(name=filename, datasource=datasource, user=admin, crc=crc) sourcefile.file.save(name=filename, content=io, save=True) except Exception as ex: logger.error( 'Cannot create sourcefile for logger {}: {}'.format( serial, ex)) # find out where logger is # we could use the name from the header, but this is not equal to the id of the screen in the database query = LoggerPos.objects.filter(logger=datalogger) pos = None if query.count() == 1: pos = query.first() else: # TODO: klopt niet, de if-else hieronder query1 = query.filter(start_date__range=span) if query1.count == 1: pos = query1.first() else: query2 = query.filter(end_date__range=span) if query2.count == 1: pos = query2.first() if pos is None: logger.error( 'Cannot find installation for logger {}'.format( serial)) continue screens.add(pos.screen) logger.info('File import completed') if len(screens) > 0: logger.info('Updating time series') for screen in screens: series = screen.find_series() if series: series.update(start=start, stop=stop) wells.add(screen.well) if len(wells) > 0: logger.info('Updating well charts') make_wellcharts(None, None, wells) logger.info('Done.')
def crc32(domain): return binascii.crc32(domain.encode('utf-8'))
if port.read() != b'\x55': print('No ACK for ADDR. Exiting.') sys.exit(4) elif args.verbose: print(f'ADDR<- ACK') if args.verbose: print(f'DATA-> {binascii.hexlify(memory_view[cur_page:cur_page+pagesize])}') port.write(bytes(memory_view[cur_page:cur_page+pagesize])) if port.read() != b'\x55': print('No ACK for DATA. Exiting.') sys.exit(4) elif args.verbose: print(f'DATA<- ACK') chksum = binascii.crc32(memory_view[cur_page:cur_page+pagesize]) & 0xFFFFFFFF if args.verbose: print(f'CHK -> {chksum:08X}') port.write(struct.pack('<I', chksum)) if port.read() != b'\x77': print('No ACK for CHK. Exiting.') sys.exit(4) elif args.verbose: print(f'CHK <- ACK') if port.read() != b'\x55': print('Flash failed. Exiting.') sys.exit(4) print(f'Page {index}/{no_pages} written.') elif args.verbose: print('-'*80)
def get_crc32(s): return binascii.crc32(binascii.unhexlify(s.lower()))
def add_fib_at_start(arginput): input_file = arginput + ".bin" file_name_hex = arginput + "_fib.hex" file_name_bin = arginput + ".bin" # Read in hex file input_hex_file = intelhex.IntelHex() input_hex_file.padding = 0x00 input_hex_file.loadbin(input_file, offset=FLASH_BASE) output_hex_file = intelhex.IntelHex() output_hex_file.padding = 0x00 # Get the starting and ending address addresses = input_hex_file.addresses() addresses.sort() start_end_pairs = list(ranges(addresses)) regions = len(start_end_pairs) if regions == 1: start, end = start_end_pairs[0] else: start = min(min(start_end_pairs)) end = max(max(start_end_pairs)) assert start >= FLASH_BASE, ( "Error - start 0x%x less than begining of user\ flash area" % start) # Compute checksum over the range (don't include data at location of crc) size = end - start + 1 data = input_hex_file.tobinarray(start=start, size=size) crc32 = binascii.crc32(data) & 0xFFFFFFFF fw_rev = FW_REV checksum = (start + size + crc32 + fw_rev) & 0xFFFFFFFF print("Writing FIB: base 0x%08X, size 0x%08X, crc32 0x%08X, fw rev 0x%08X,\ checksum 0x%08X" % (start, size, crc32, fw_rev, checksum)) #expected initial values used by daplink to validate that it is a valid bin #file added as dummy values in this file because the fib area preceeds the #application area the bootloader will ignore these dummy values # 00 is stack pointer (RAM address) # 04 is Reset vector (FLASH address) # 08 NMI_Handler (FLASH address) # 0C HardFault_Handler(FLASH address) # 10 dummy dummy_sp = 0x3FFFFC00 dummy_reset_vector = 0x00003625 dummy_nmi_handler = 0x00003761 dummy_hardfault_handler = 0x00003691 dummy_blank = 0x00000000 #expected fib structure #typedef struct fib{ #uint32_t base; /**< Base offset of firmware, indicating what flash the # firmware is in. (will never be 0x11111111) */ #uint32_t size; /**< Size of the firmware */ #uint32_t crc; /**< CRC32 for firmware correctness check */ #uint32_t rev; /**< Revision number */ #uint32_t checksum; /**< Check-sum of information block */ #}fib_t, *fib_pt; fib_start = FIB_BASE dummy_fib_size = 20 fib_size = 20 trim_size = 24 user_code_start = FLASH_BASE trim_area_start = TRIM_BASE # Write FIB to the file in little endian output_hex_file[fib_start + 0] = (dummy_sp >> 0) & 0xFF output_hex_file[fib_start + 1] = (dummy_sp >> 8) & 0xFF output_hex_file[fib_start + 2] = (dummy_sp >> 16) & 0xFF output_hex_file[fib_start + 3] = (dummy_sp >> 24) & 0xFF output_hex_file[fib_start + 4] = (dummy_reset_vector >> 0) & 0xFF output_hex_file[fib_start + 5] = (dummy_reset_vector >> 8) & 0xFF output_hex_file[fib_start + 6] = (dummy_reset_vector >> 16) & 0xFF output_hex_file[fib_start + 7] = (dummy_reset_vector >> 24) & 0xFF output_hex_file[fib_start + 8] = (dummy_nmi_handler >> 0) & 0xFF output_hex_file[fib_start + 9] = (dummy_nmi_handler >> 8) & 0xFF output_hex_file[fib_start + 10] = (dummy_nmi_handler >> 16) & 0xFF output_hex_file[fib_start + 11] = (dummy_nmi_handler >> 24) & 0xFF output_hex_file[fib_start + 12] = (dummy_hardfault_handler >> 0) & 0xFF output_hex_file[fib_start + 13] = (dummy_hardfault_handler >> 8) & 0xFF output_hex_file[fib_start + 14] = (dummy_hardfault_handler >> 16) & 0xFF output_hex_file[fib_start + 15] = (dummy_hardfault_handler >> 24) & 0xFF output_hex_file[fib_start + 16] = (dummy_blank >> 0) & 0xFF output_hex_file[fib_start + 17] = (dummy_blank >> 8) & 0xFF output_hex_file[fib_start + 18] = (dummy_blank >> 16) & 0xFF output_hex_file[fib_start + 19] = (dummy_blank >> 24) & 0xFF # Write FIB to the file in little endian output_hex_file[fib_start + 20] = (start >> 0) & 0xFF output_hex_file[fib_start + 21] = (start >> 8) & 0xFF output_hex_file[fib_start + 22] = (start >> 16) & 0xFF output_hex_file[fib_start + 23] = (start >> 24) & 0xFF output_hex_file[fib_start + 24] = (size >> 0) & 0xFF output_hex_file[fib_start + 25] = (size >> 8) & 0xFF output_hex_file[fib_start + 26] = (size >> 16) & 0xFF output_hex_file[fib_start + 27] = (size >> 24) & 0xFF output_hex_file[fib_start + 28] = (crc32 >> 0) & 0xFF output_hex_file[fib_start + 29] = (crc32 >> 8) & 0xFF output_hex_file[fib_start + 30] = (crc32 >> 16) & 0xFF output_hex_file[fib_start + 31] = (crc32 >> 24) & 0xFF output_hex_file[fib_start + 32] = (fw_rev >> 0) & 0xFF output_hex_file[fib_start + 33] = (fw_rev >> 8) & 0xFF output_hex_file[fib_start + 34] = (fw_rev >> 16) & 0xFF output_hex_file[fib_start + 35] = (fw_rev >> 24) & 0xFF output_hex_file[fib_start + 36] = (checksum >> 0) & 0xFF output_hex_file[fib_start + 37] = (checksum >> 8) & 0xFF output_hex_file[fib_start + 38] = (checksum >> 16) & 0xFF output_hex_file[fib_start + 39] = (checksum >> 24) & 0xFF #pad the rest of the file for i in range(fib_start + dummy_fib_size + fib_size, trim_area_start): output_hex_file[i] = 0xFF # Read in configuration data from the config parameter in targets.json configData = Config('NCS36510') paramData = configData.get_target_config_data() for v in paramData.values(): if (v.name == "target.mac-addr-high"): mac_addr_high = int(v.value, 16) elif (v.name == "target.mac-addr-low"): mac_addr_low = int(v.value, 16) elif (v.name == "target.32KHz-clk-trim"): clk_32k_trim = int(v.value, 16) elif (v.name == "target.32MHz-clk-trim"): clk_32m_trim = int(v.value, 16) elif (v.name == "target.rssi-trim"): rssi = int(v.value, 16) elif (v.name == "target.txtune-trim"): txtune = int(v.value, 16) else: print("Not a valid param") output_hex_file[trim_area_start + 0] = mac_addr_low & 0xFF output_hex_file[trim_area_start + 1] = (mac_addr_low >> 8) & 0xFF output_hex_file[trim_area_start + 2] = (mac_addr_low >> 16) & 0xFF output_hex_file[trim_area_start + 3] = (mac_addr_low >> 24) & 0xFF output_hex_file[trim_area_start + 4] = mac_addr_high & 0xFF output_hex_file[trim_area_start + 5] = (mac_addr_high >> 8) & 0xFF output_hex_file[trim_area_start + 6] = (mac_addr_high >> 16) & 0xFF output_hex_file[trim_area_start + 7] = (mac_addr_high >> 24) & 0xFF output_hex_file[trim_area_start + 8] = clk_32k_trim & 0xFF output_hex_file[trim_area_start + 9] = (clk_32k_trim >> 8) & 0xFF output_hex_file[trim_area_start + 10] = (clk_32k_trim >> 16) & 0xFF output_hex_file[trim_area_start + 11] = (clk_32k_trim >> 24) & 0xFF output_hex_file[trim_area_start + 12] = clk_32m_trim & 0xFF output_hex_file[trim_area_start + 13] = (clk_32m_trim >> 8) & 0xFF output_hex_file[trim_area_start + 14] = (clk_32m_trim >> 16) & 0xFF output_hex_file[trim_area_start + 15] = (clk_32m_trim >> 24) & 0xFF output_hex_file[trim_area_start + 16] = rssi & 0xFF output_hex_file[trim_area_start + 17] = (rssi >> 8) & 0xFF output_hex_file[trim_area_start + 18] = (rssi >> 16) & 0xFF output_hex_file[trim_area_start + 19] = (rssi >> 24) & 0xFF output_hex_file[trim_area_start + 20] = txtune & 0xFF output_hex_file[trim_area_start + 21] = (txtune >> 8) & 0xFF output_hex_file[trim_area_start + 22] = (txtune >> 16) & 0xFF output_hex_file[trim_area_start + 23] = (txtune >> 24) & 0xFF # pad the rest of the area with 0xFF for i in range(trim_area_start + trim_size, user_code_start): output_hex_file[i] = 0xFF #merge two hex files output_hex_file.merge(input_hex_file, overlap='error') # Write out file(s) output_hex_file.tofile(file_name_hex, 'hex') output_hex_file.tofile(file_name_bin, 'bin')
def checkHash(packet): h = packet.hash packet.hash = 0 result = (h == binascii.crc32(packet.__serialize__()) & 0xffffffff) packet.hash = h return result
# coding: utf-8 import binascii print(binascii.crc32("The quick brown fox jumped over the lazy dog.".encode()))
jarFilePath = jarDirPath + "pixiedust.jar" dir = os.path.dirname(jarDirPath) if not os.path.exists(dir): os.makedirs(dir) def installPixiedustJar(): with pkg_resources.resource_stream(__name__, "resources/pixiedust.jar") as resJar: with open( jarFilePath, 'wb+' ) as installedJar: shutil.copyfileobj(resJar, installedJar) print("Pixiedust runtime updated. Please restart kernel") copyFile = True if os.path.isfile(jarFilePath): with open( jarFilePath, 'rb' ) as installedJar: installedCRC = binascii.crc32( installedJar.read() ) with pkg_resources.resource_stream(__name__, "resources/pixiedust.jar") as resJar: copyFile = installedCRC != binascii.crc32( resJar.read() ) if copyFile: installPixiedustJar() def checkVersion(): import json try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen def printHTML(html): from IPython.display import display, HTML
typeid = '0x' + typeid cleanline = nametype.group(1) + nametype.group( 3) + '= ' + nametype.group(4) cleanline = re.sub(r' [a-zA-Z0-9_]+\:flags\.[0-9]+\?true', '', cleanline) cleanline = cleanline.replace('<', ' ').replace('>', ' ').replace(' ', ' ') cleanline = re.sub(r'^ ', '', cleanline) cleanline = re.sub(r' $', '', cleanline) cleanline = cleanline.replace(':bytes ', ':string ') cleanline = cleanline.replace('?bytes ', '?string ') cleanline = cleanline.replace('{', '') cleanline = cleanline.replace('}', '') countTypeId = binascii.crc32(binascii.a2b_qp(cleanline)) if (countTypeId < 0): countTypeId += 2**32 countTypeId = '0x' + re.sub(r'^0x|L$', '', hex(countTypeId)) if (typeid != countTypeId): print('Warning: counted ' + countTypeId + ' mismatch with provided ' + typeid + ' (' + cleanline + ')') continue params = nametype.group(3) restype = nametype.group(4) if (restype.find('<') >= 0): templ = re.match(r'^([vV]ector<)([A-Za-z0-9\._]+)>$', restype) if (templ): vectemplate = templ.group(2) if (re.match(r'^[A-Z]', vectemplate)
def hashPacket(packet): packet.hash = 0 packet.hash = binascii.crc32(packet.__serialize__()) & 0xffffffff
def getCRC32(input, prevRes=0): from binascii import crc32 return crc32(input, prevRes)
def CRC32(buf): import binascii buf = (binascii.crc32(buf) & 0xFFFFFFFF) return str("%08X" % buf)
c = s[:BS] # our first cipher block # PNG signature, chunk size, our dummy chunk type p = PNGSIG + struct.pack(">I", size) + chunktype #let's generate our IV c = "".join([chr(ord(c[i]) ^ ord(p[i])) for i in range(BS)]) IV = AES.new(key, AES.MODE_ECB).decrypt(c) ctr = Counter.new(128, initial_value=int(IV.encode('hex'), 16)) result = AES.new(key, AES.MODE_CTR, counter=ctr).decrypt(result) result += pad(4 + len(t[8:])) #not fixing the CRC on the decrypted file - lazy :D result += struct.pack(">I", binascii.crc32(result[0xc:]) % 0x100000000) #we append the whole target image result += t[8:] result += pad(len(result)) ctr = Counter.new(128, initial_value=int(IV.encode('hex'), 16)) result = AES.new(key, AES.MODE_CTR, counter=ctr).encrypt(result) #write the CRC of the remaining of s at the end of our dummy block result += struct.pack(">I", binascii.crc32(result[len(s) - 8:]) % 0x100000000) result += s[-12:] # our IEND chunk #we have our result, key and IV #generate the result file
def thg_encode(self, args): """modulo referente a encode de estrings""" arg_mensage = args.split(" ") if arg_mensage[0] == "": print("""suporte encode: Este módulo fornece funções para codificar dados binários em caracteres ASCII imprimíveis e decodificar essas codificações de volta para dados binários. Ele fornece funções de codificação e decodificação para as codificações especificadas em RFC 3548 ,que define os algoritmos Base16, Base32 e Base64, e para as codificações Ascii85 e Base85 padrão de fato. a2b_uu b2a_uu a2b_base64 b2a_base64 a2b_qp b2a_qp a2b_hqx rledecode_hqx rlecode_hqx b2a_hqx crc_hqx crc32 b2a_hex a2b_hex hexlify unhexlify Charcode binary base62 basen bcd ur unicode_normalize qp_encoding encode type[2,16,32,64] str """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) elif arg_mensage[0] == "64": arg_mensage[1] = arg_mensage[1].encode('ascii') base64_bytes = base64.b64encode(arg_mensage[1]) by_to_st(base64_bytes) elif arg_mensage[0] == "32": arg_mensage[1] = arg_mensage[1].encode('ascii') b32encode_bytes = base64.b32encode(arg_mensage[1]) by_to_st(b32encode_bytes) elif arg_mensage[0] == "16": arg_mensage[1] = arg_mensage[1].encode('ascii') b16encode_bytes = base64.b16encode(arg_mensage[1]) by_to_st(b16encode_bytes) elif arg_mensage[0] == "a85encode": arg_mensage[1] = arg_mensage[1].encode('ascii') a85encode_bytes = base64.a85encode(arg_mensage[1]) by_to_st(a85encode_bytes) elif arg_mensage[0] == "b85encode": arg_mensage[1] = arg_mensage[1].encode('ascii') b85encode_bytes = base64.b85encode(arg_mensage[1]) by_to_st(b85encode_bytes) elif arg_mensage[0] == "a2b_uu": if arg_mensage[1] == "help": print( """{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED} Converta uma única linha de dados uuencodificados de volta em binários e retorne os dados binários. As linhas normalmente contêm 45 bytes (binários), exceto a última linha. Os dados da linha podem ser seguidos de espaços em branco.""".format( YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st((binascii.a2b_uu(arg_mensage[1]))) elif arg_mensage[0] == "a2b_base64": if arg_mensage[1] == "help": print( """{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED}Converta dados binários em uma linha de caracteres ASCII na codificação base64. O valor de retorno é a linha convertida, incluindo um caractere de nova linha. O comprimento dos dados deve ser de no máximo 57 para aderir ao padrão base64.""".format( YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st(binascii.a2b_base64(arg_mensage[1])) elif arg_mensage[0] == "b2a_base64": if arg_mensage[1] == "help": print( """{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED} Converta dados binários em uma linha de caracteres ASCII na codificação base64. O valor de retorno é a linha convertida, incluindo um caractere de nova linha. O comprimento dos dados deve ser de no máximo 57 para aderir ao padrão base64.""".format( YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st(binascii.b2a_base64(b'arg_mensage[1]')) elif arg_mensage[0] == "a2b_qp": if arg_mensage[1] == "help": print( """{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED}Converta um bloco de dados imprimíveis entre aspas de volta em binários e retorne os dados binários. Mais de uma linha pode ser passada por vez. Se o cabeçalho do argumento opcional estiver presente e verdadeiro, os sublinhados serão decodificados como espaços.""".format( YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st(binascii.a2b_qp(arg_mensage[1])) elif arg_mensage[0] == "b2a_qp": if arg_mensage[1] == "help": print( """{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED}Converta dados binários em uma (s) linha (s) de caracteres ASCII em codificação imprimível entre aspas. O valor de retorno é a (s) linha (s) convertida (s). Se o argumento opcional quotetabs estiver presente e verdadeiro, todas as tabulações e espaços serão codificados. Se o argumento opcional istext estiver presente e verdadeiro, as novas linhas não serão codificadas, mas os espaços em branco finais serão codificados. Se o cabeçalho do argumento opcional estiver presente e verdadeiro, os espaços serão codificados como sublinhados de acordo com RFC1522. Se o cabeçalho do argumento opcional estiver presente e for falso, os caracteres de nova linha também serão codificados; caso contrário, a conversão de alimentação de linha pode corromper o fluxo de dados binários.""".format( YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st(binascii.a2b_qp(arg_mensage[1].encode())) elif arg_mensage[0] == "a2b_hqx": if arg_mensage[1] == "help": print("""{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED}Converta dados ASCII formatados de binhex4 em binários, sem fazer a descompressão RLE. A string deve conter um número completo de bytes binários ou (no caso da última parte dos dados binhex4) ter os bits restantes zero. """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st(binascii.a2b_hqx(arg_mensage[1])) elif arg_mensage[0] == "rledecode_hqx": if arg_mensage[1] == "help": print( """{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED} Execute a descompressão RLE nos dados, de acordo com o padrão binhex4. O algoritmo usa 0x90 após um byte como um indicador de repetição, seguido por uma contagem. Uma contagem de 0 especifica um valor de byte de 0x90 . A rotina retorna os dados descompactados, a menos que os dados de entrada de dados terminem em um indicador de repetição órfão, caso em que a exceção Incompleta é levantada.""".format( YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st((binascii.rledecode_hqx(arg_mensage[1].encode()))) elif arg_mensage[0] == "rlecode_hqx": if arg_mensage[1] == "help": print( """{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED} Execute a compactação RLE no estilo binhex4 nos dados e retorne o resultado.""".format( YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st((binascii.rlecode_hqx(arg_mensage[1].encode()))) elif arg_mensage[0] == "b2a_hqx": if arg_mensage[1] == "help": print("""{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED} Execute a conversão hexbin4 binário para ASCII e retorne a string resultante. O argumento já deve ser codificado por RLE e ter um comprimento divisível por 3 (exceto possivelmente o último fragmento). """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st((binascii.b2a_hqx(arg_mensage[1].encode()))) elif arg_mensage[0] == "crc_hqx": if arg_mensage[1] == "help": print("""{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED} Calcule o valor binhex4 crc dos dados , começando com um crc inicial e retornando o resultado. """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st( (binascii.crc_hqx(arg_mensage[1].encode(), int(arg_mensage[2])))) elif arg_mensage[0] == "crc32": if arg_mensage[1] == "help": print("""{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED} Calcule CRC-32, a soma de verificação de dados de 32 bits, começando com um crc inicial. Isso é consistente com a soma de verificação do arquivo ZIP. Uma vez que o algoritmo é projetado para uso como um algoritmo de soma de verificação, não é adequado para uso como um algoritmo de hash geral. {YELLOW}Nota{YELLOW}{RED} Para gerar o mesmo valor numérico em todas as versões e plataformas Python, {RED}{BLUE}use crc32 (dados) & 0xffffffff{BLUE}{RED}. Se você estiver usando apenas a soma de verificação no formato binário compactado, isso não é necessário, pois o valor de retorno é a representação binária correta de 32 bits, independentemente do sinal. """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st((binascii.crc32(arg_mensage[1].encode()))) elif arg_mensage[0] == "hexlify": if arg_mensage[1] == "help": print("""{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED} Retorna a representação hexadecimal dos dados binários . Cada byte de dados é convertido na representação hexadecimal de 2 dígitos correspondente. A string resultante é, portanto, o dobro do comprimento dos dados . """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st( (binascii.hexlify(arg_mensage[1].encode(), arg_mensage[2].encode()))) elif arg_mensage[0] == "b2a_hex": if arg_mensage[1] == "help": print("""{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED} hex """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st( (binascii.b2a_hex(arg_mensage[1].encode(), int(arg_mensage[2])))) elif arg_mensage[0] == "unhexlify": if arg_mensage[1] == "help": print("""{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED} Retorna os dados binários representados pela string hexadecimal hexstr . Esta função é o inverso de b2a_hex () . hexstr deve conter um número par de dígitos hexadecimais (que podem ser maiúsculas ou minúsculas), caso contrário, um TypeError é gerado. """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st((binascii.unhexlify(arg_mensage[1].encode()))) elif arg_mensage[0] == "b2a_uu": if arg_mensage[1] == "help": print("""{YELLOW}a2b_uu{YELLOW}{BLUE} =>{BLUE}{RED} Converta dados binários em uma linha de caracteres ASCII, o valor de retorno é a linha convertida, incluindo um caractere de nova linha. O comprimento dos dados deve ser de no máximo 45. """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: by_to_st( (binascii.b2a_uu(arg_mensage[1].encode(), int(arg_mensage[2])))) elif arg_mensage[0] == "charcode": if arg_mensage[1] == "help": print("""{YELLOW}charcode{YELLOW}{BLUE} =>{BLUE}{RED}converte string em charcode """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: print(ord(arg_mensage[1].encode())) elif arg_mensage[0] == "binary": if arg_mensage[1] == "help": print("""{YELLOW}binary{YELLOW}{BLUE} =>{BLUE}{RED}converte string em binary """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: print(' '.join(format(ord(x), 'b') for x in arg_mensage[1])) elif arg_mensage[0] == "base62": if arg_mensage[1] == "help": print("""{YELLOW}base62{YELLOW}{BLUE} =>{BLUE}{RED}converte string em base62 """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: print(decode62(arg_mensage[1])) elif arg_mensage[0] == "basen": if arg_mensage[1] == "help": print("""{YELLOW}basen{YELLOW}{BLUE} =>{BLUE}{RED}converte decimal em basen """.format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: print(numpy.base_repr(int(arg_mensage[1]), base=int(arg_mensage[2]))) elif arg_mensage[0] == "url": try: if arg_mensage[1] == "help": print( """{YELLOW}url_encode{YELLOW}{BLUE} =>{BLUE}{RED}encode personalidado para url\nencode url_encode safa[] encoding""".format( YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: print(quote(arg_mensage[1], safe=arg_mensage[2], encoding=arg_mensage[3])) except IndexError: print("digite a sintaxe correta\nncode url_encode safa[] encoding\n ou use o comando help") elif arg_mensage[0] == "unicode_normalize": try: if arg_mensage[1] == "help": print("""{YELLOW}unicode_normalize{YELLOW}{BLUE} =>{BLUE}{RED}Transforme caracteres Unicode em uma das formas de normalização['NFC', 'NFKC', 'NFD','NFKD']\n {YELLOW}NFD{YELLOW}{BLUE} =>{BLUE}{RED}Normalisation Form Canonical Decomposition {YELLOW}NFC{YELLOW}{BLUE} =>{BLUE}{RED}Normalisation Form Canonical Composition {YELLOW}NFKD{YELLOW}{BLUE} =>{BLUE}{RED}Normalisation Form Compatibility Decomposition {YELLOW}NFKC{YELLOW}{BLUE} =>{BLUE}{RED}Normalisation Form Compatibility Composition encode unicode_normalize str encoding['NFC', 'NFKC', 'NFD','NFKD']\n""".format(YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: print(unicodedata.normalize(arg_mensage[1], arg_mensage[2])) except IndexError: print("digite a sintaxe correta\nncode url_encode safa[] encoding\n ou use o comando help") elif arg_mensage[0] == "qp_encoding": try: if arg_mensage[1] == "help": print("""{YELLOW}qp_encoding{YELLOW}{BLUE} =>{BLUE}{RED} Quoted-Printable, ou QP encoding, é uma codificação que usa caracteres ASCII imprimíveis (alfanuméricos e o sinal de igual '=') para transmitir dados de 8 bits em um caminho de dados de 7 bits ou, geralmente, em um meio que não é 8- um pouco limpo. É definido como uma codificação de transferência de conteúdo MIME para uso em e-mail. QP funciona usando o sinal de igual '=' como um caractere de escape. Ele também limita o comprimento da linha a 76, pois alguns softwares têm limites no comprimento da linha\nencode qp_encoding TXT encode""".format( YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: encoded = quopri.encodestring(arg_mensage[1].encode(arg_mensage[2])) print(encoded.decode()) except IndexError: print("digite a sintaxe correta\nencode qp_encoding é utf-16\n ou use o comando help") elif arg_mensage[0] == "idna": try: if arg_mensage[1] == "help": print( """{YELLOW}idna{YELLOW}{BLUE} =>{BLUE}{RED}encode personalidado para url\nencode url_encode safa[] encoding""".format( YELLOW=Fore.YELLOW, BLUE=Fore.BLUE, RED=Fore.RED)) else: print(idna.encode(arg_mensage[1]).decode(arg_mensage[2])) except IndexError: print("digite a sintaxe correta\nncode idna string encoding\n ou use o comando help") else: pass try: pass except IndexError: print("verificar a saida")
def hash_func(port): m = 6 size = 2**m hash_val = binascii.crc32(str(port).encode()) return hash_val % size
def RC4_main(filename, key): # 获得输入文件 try: fin = open(filename, 'rb') except: print('打开文件失败!') print(filename) # 打开输出文件 if filename[-4:] == '.RC4': eID = 1 key = key.encode() ofilename = filename[:-4] else: eID = 2 # key = input('输入加密密钥: ').encode() key = key.encode() ofilename = filename + '.RC4' key = Coding(CreatKey(key)) key = UpdataKey(key) fout = open(ofilename, 'wb') print(ofilename) # 解密 if eID == 1: # 读文件长度 filelen = struct.unpack('I', fin.read(4))[0] print('FlieLen =', filelen, '\n......') while 1: ps = fin.read(2) if not ps: break packsize = struct.unpack('H', ps)[0] # 读数据 dd = fin.read(packsize) # 解密 dd = Coding(dd) x = RC4(key, len(key), dd, len(dd)) key = UpdataKey(key) # crc crc = struct.unpack('I', fin.read(4))[0] if binascii.crc32(x) != crc: print('CRC32校验错误!', crc, binascii.crc32(x)) fout.write(x) # 裁剪末尾填充位 fout.truncate(filelen) # 加密 elif eID == 2: # 获得文件长度 fin.seek(0, 2) filelen = fin.tell() print('FlieLen =', filelen, '\n......') fin.seek(0, 0) fout.write(struct.pack('I', filelen)) while 1: # 读数据 dd = fin.read(65534) if not dd: # 文件结束 break # 末尾填充 srl = len(dd) if srl % 2: srl += 1 dd += b'\0' # crc crc = struct.pack('I', binascii.crc32(dd)) # 加密数据 dd = Coding(dd) x = RC4(key, len(key), dd, len(dd)) key = UpdataKey(key) # 写入文件 fout.write(struct.pack('H', srl)) fout.write(x) fout.write(crc) fin.close() fout.close() print('encode success')
def frame(cls, destination, command, data, index=0): packet = struct.pack('<IBB', destination, command, index) + data cs = binascii.crc32(packet) & ((1 << 32) - 1) cs_bytes = struct.pack('<I', cs) return cls.cobs_encode(packet + cs_bytes), cs_bytes
def calculate_crc32(response): return crc32(to_bytes(response.content)) & 0xffffffff
def docker_tagify(cid: str) -> str: """modify a challenge id so that it is an acceptable docker tag""" tag = "r8:" + re.sub(r"[^a-zA-Z0-9_.-]", "_", cid) if len(tag) > 128: tag = tag[:118] + hex(binascii.crc32(tag.encode())) return tag
def unframe(cls, packet): packet = cls.cobs_decode(packet) cs = binascii.crc32(packet) & ((1 << 32) - 1) if cs != 0x2144DF1C: raise DecodeError("BAD CRC: " + hex(cs), repr(packet)) return struct.unpack('<IBB', packet[0:6]) + (packet[6:-4], packet[-4:])
def crc32(s): return (~binascii.crc32(s, -1)) & 0xFFFFFFFF
def save_array_as_PNG(img, fname, kwArgCheck=None, ftype_req=-1): """ Save an array as a PNG image. img -- a 3D NumPy array of type uint8 with shape (ny,nx,3) fname -- output file name ftype_req -- filter type to be used The PNG specification defines 5 different possible filters which are numbered 0 to 4 (inclusively). Filter #0 is "no filtering". If the user defines "ftype_req" as one of the identifying integers then that filter will be used for the entire PNG file. If the user defines "ftype_req" as "-1" (or does not define "ftype_req" at all) then adaptive filtering will be used whereby an attempt is made to predict which filtering method will yield the smallest compressed stream. """ # Import standard modules ... import binascii import zlib # Import special modules ... try: import numpy except: raise Exception( "\"numpy\" is not installed; run \"pip install --user numpy\"" ) from None # Import sub-functions ... from .paeth_filter import paeth_filter # Check keyword arguments ... if kwArgCheck is not None: print( f"WARNING: \"{__name__}\" has been called with an extra positional argument" ) # Find image size ... ny, nx, nc = img.shape # Check image ... if not img.dtype == "uint8": raise TypeError("\"img\" must be a \"uint8\" array") from None if nc != 3: raise Exception("\"img\" must be a 3-channel array") from None # Try opening the PNG ... with open(fname, "wb") as fobj: # ********************************************************************** # * WRITE THE SIGNATURE * # ********************************************************************** fobj.write(binascii.unhexlify("89504E470D0A1A0A")) # ********************************************************************** # * CREATE "IHDR" CHUNK AND WRITE IT * # ********************************************************************** ihdr = bytearray() ihdr += numpy.uint32(13).byteswap().tobytes() # Length ihdr += bytearray("IHDR", encoding="ascii") # Chunk type ihdr += numpy.uint32(nx).byteswap().tobytes() # IHDR : Width ihdr += numpy.uint32(ny).byteswap().tobytes() # IHDR : Height ihdr += numpy.uint8(8).byteswap().tobytes() # IHDR : Bit depth ihdr += numpy.uint8(2).byteswap().tobytes() # IHDR : Colour type ihdr += numpy.uint8( 0).byteswap().tobytes() # IHDR : Compression method ihdr += numpy.uint8(0).byteswap().tobytes() # IHDR : Filter method ihdr += numpy.uint8(0).byteswap().tobytes() # IHDR : Interlace method ihdr += numpy.uint32(binascii.crc32( ihdr[4:])).byteswap().tobytes() # CRC-32 fobj.write(ihdr) del ihdr # ********************************************************************** # * CREATE "IDAT" CHUNK AND WRITE IT * # ********************************************************************** idat = bytearray() idat += numpy.uint32(0).byteswap().tobytes() # Length idat += bytearray("IDAT", encoding="ascii") # Chunk type stream = bytearray() # Loop over rows ... for iy in range(ny): row = numpy.zeros((5, nc, nx), dtype=numpy.uint8) # Calculate stream for "none" filter (if required) ... if ftype_req in [-1, 0]: ftype = 0 for ix in range(nx): row[ftype, :, ix] = img[iy, ix, :] # Calculate stream for "sub" filter (if required) ... if ftype_req in [-1, 1]: ftype = 1 for ix in range(nx): for ic in range(nc): if ix == 0: p1 = numpy.int16(0) else: p1 = img[iy, ix - 1, ic].astype(numpy.int16) diff = img[iy, ix, ic].astype(numpy.int16) - p1 diff = numpy.mod(diff, 256) row[ftype, ic, ix] = diff.astype(numpy.uint8) # Calculate stream for "up" filter (if required) ... if ftype_req in [-1, 2]: ftype = 2 for ix in range(nx): for ic in range(nc): if iy == 0: p1 = numpy.int16(0) else: p1 = img[iy - 1, ix, ic].astype(numpy.int16) diff = img[iy, ix, ic].astype(numpy.int16) - p1 diff = numpy.mod(diff, 256) row[ftype, ic, ix] = diff.astype(numpy.uint8) # Calculate stream for "average" filter (if required) ... if ftype_req in [-1, 3]: ftype = 3 for ix in range(nx): for ic in range(nc): if ix == 0: p1 = numpy.int16(0) else: p1 = img[iy, ix - 1, ic].astype(numpy.int16) if iy == 0: p2 = numpy.int16(0) else: p2 = img[iy - 1, ix, ic].astype(numpy.int16) diff = img[iy, ix, ic].astype(numpy.int16) - ( (p1 + p2) // numpy.int16(2)) diff = numpy.mod(diff, 256) row[ftype, ic, ix] = diff.astype(numpy.uint8) # Calculate stream for "Paeth" filter (if required) ... if ftype_req in [-1, 4]: ftype = 4 for ix in range(nx): for ic in range(nc): if ix == 0: p1 = numpy.int16(0) else: p1 = img[iy, ix - 1, ic].astype(numpy.int16) if iy == 0: p2 = numpy.int16(0) else: p2 = img[iy - 1, ix, ic].astype(numpy.int16) if ix == 0 or iy == 0: p3 = numpy.int16(0) else: p3 = img[iy - 1, ix - 1, ic].astype(numpy.int16) diff = img[iy, ix, ic].astype(numpy.int16) - paeth_filter( p1, p2, p3).astype(numpy.int16) diff = numpy.mod(diff, 256) row[ftype, ic, ix] = diff.astype(numpy.uint8) # Figure out which stream to use ... if ftype_req == -1: tmp1 = numpy.uint64(255 * nx) for ftype in range(5): tmp2 = row[ftype, :, :].astype(numpy.uint64).sum() if tmp2 < tmp1: tmp1 = tmp2 ftype_best = ftype else: ftype_best = ftype_req # Use the best/requested stream for this row ... stream += numpy.uint8(ftype_best).byteswap().tobytes() for ix in range(nx): stream += row[ftype_best, :, ix].byteswap().tobytes() # Clean up ... del row idat += zlib.compress(stream, 9) # IDAT : Data idat[0:4] = numpy.uint32(len(idat[8:])).byteswap().tobytes() # Length idat += numpy.uint32(binascii.crc32( idat[4:])).byteswap().tobytes() # CRC-32 fobj.write(idat) del idat # ********************************************************************** # * CREATE "IEND" CHUNK AND WRITE IT * # ********************************************************************** iend = bytearray() iend += numpy.uint32(0).byteswap().tobytes() # Length iend += bytearray("IEND", encoding="ascii") # Chunk type iend += numpy.uint32(binascii.crc32( iend[4:])).byteswap().tobytes() # CRC-32 fobj.write(iend) del iend
def processList(self): import time # begin loop through of email list for line in exchangeDigest.itemArray: # get individual email xml = self.getItemXML(line[0], line[1]) attr = exchangeDigest.c.service.ResolveNames(__inject={'msg': xml}) data = attr.GetItemResponseMessage.Items.Message a = str(data.DateTimeSent) ''' try: strptime = time.strptime() except AttributeError: import time ''' dt_obj = time.strptime(a, "%Y-%m-%d %H:%M:%S") exchangeDigest.timeSent = str(int(time.mktime(dt_obj))) try: body = self.cleanText(data.Body.value) body_md5 = str(hashlib.md5(body).hexdigest()) except AttributeError: body = '' body_md5 = '' sql = "select post_id from phpbb_posts where post_time = %s and post_checksum = %s" cursor.execute(sql, (exchangeDigest.timeSent, body_md5)) rows = cursor.fetchall() copy = 0 if (len(rows) > 0): copy = 1 # print data.From.Mailbox.Name if (copy == 0): try: emailAddress = str(data.From.Mailbox.EmailAddress) except UnicodeEncodeError: a = data.From.Mailbox.EmailAddress b = unicode(a) emailAddress = b.encode("utf-8") except AttributeError: emailAddress = '*****@*****.**' tmp = emailAddress.partition('@') if (tmp[2] == 'yourdomain.com'): exchangeDigest.From = tmp[0].lower() else: exchangeDigest.From = emailAddress sql = "select * from phpbb_users where username = '******'" cursor.execute(sql) rows = cursor.fetchall() if (len(rows) > 0): #User exists for row in rows: user_info = row else: #User does not exist try: test = urllib2.urlopen( "http://www.yourdomain.com/images/staff/" + exchangeDigest.From + ".jpg") user_avatar = "http://www.yourdomain.com/images/staff/" + exchangeDigest.From + ".jpg" except urllib2.HTTPError: user_avatar = "images/avatars/gallery/baby_cute_face.jpg" email_hash = str(binascii.crc32(emailAddress)) salt = str( hashlib.md5(exchangeDigest.timeSent).hexdigest()) sql = "insert into phpbb_users (user_type, group_id, user_permissions, user_perm_from, user_ip, user_regdate, username, username_clean, user_password, user_passchg, user_pass_convert, user_email, user_email_hash, user_birthday, user_lastvisit, user_lastmark, user_lastpost_time, user_lastpage, user_last_confirm_key, user_last_search, user_warnings, user_last_warning, user_login_attempts, user_inactive_reason, user_inactive_time, user_posts, user_lang, user_timezone, user_dst, user_dateformat, user_style, user_rank, user_colour, user_new_privmsg, user_unread_privmsg, user_last_privmsg, user_message_rules, user_full_folder, user_emailtime, user_topic_show_days, user_topic_sortby_type, user_topic_sortby_dir, user_post_show_days, user_post_sortby_type, user_post_sortby_dir, user_notify, user_notify_pm, user_notify_type, user_allow_pm, user_allow_viewonline, user_allow_viewemail, user_allow_massemail, user_options, user_avatar, user_avatar_type, user_avatar_width, user_avatar_height, user_sig, user_sig_bbcode_uid, user_sig_bbcode_bitfield, user_from, user_icq, user_aim, user_yim, user_msnm, user_jabber, user_website, user_occ, user_interests, user_actkey, user_newpasswd, user_form_salt, user_new, user_reminded, user_reminded_time, user_digest_filter_type, user_digest_format, user_digest_max_display_words, user_digest_max_posts, user_digest_min_words, user_digest_new_posts_only, user_digest_pm_mark_read, user_digest_remove_foes, user_digest_reset_lastvisit, user_digest_send_hour_gmt, user_digest_send_on_no_posts, user_digest_show_mine, user_digest_show_pms, user_digest_sortby, user_digest_type, user_digest_has_ever_unsubscribed, user_digest_no_post_text) values(0, 2, '00000000006xrqeiww\n\n\nzik0zi000000\nzik0zi000000\nzik0zi000000', 0, '10.1.12.70', %s, %s, %s, '', %s, 0, %s, %s, '', %s, %s, 0, '', '', 0, 0, 0, 0, 0, 0, 0, 'en', '-10.00', 0, 'D M d, Y g:i a', 1, 0, '', 0, 0, 0, 0, -3, 0, 0, 't', 'd', 0, 't', 'a', 0, 1, 0, 1, 1, 1, 1, 230271, %s, 2, 90, 90, '', '', '', '', '', '', '', '', '', '', '', '', '', '', %s, 1, 0, 0, 'ALL', 'HTML', 0, 0, 0, 0, 0, 0, 1, 0.00, 0, 1, 1, 'board', 'NONE', 0 ,0)" cursor.execute( sql, (exchangeDigest.timeSent, exchangeDigest.From, exchangeDigest.From, exchangeDigest.timeSent, emailAddress, email_hash, exchangeDigest.timeSent, exchangeDigest.timeSent, user_avatar, salt)) sql = "select * from phpbb_users where username = '******'" cursor.execute(sql) rows = cursor.fetchall() for row in rows: user_info = row exchangeDigest.user_id = str(user_info[0]) try: exchangeDigest.subj = str(data.Subject) except UnicodeEncodeError: a = data.Subject b = unicode(a) exchangeDigest.subj = b.encode("utf-8") if (body != ''): has_attachments = '0' if (str(data.HasAttachments) == 'True'): has_attachments = '1' sql = "insert into phpbb_topics (forum_id, icon_id, topic_attachment, topic_approved, topic_reported, topic_title, topic_poster, topic_time, topic_time_limit, topic_views, topic_replies, topic_replies_real, topic_status, topic_type, topic_first_post_id, topic_first_poster_name, topic_first_poster_colour, topic_last_post_id, topic_last_poster_id, topic_last_poster_name, topic_last_poster_colour, topic_last_post_subject, topic_last_post_time, topic_last_view_time, topic_moved_id, topic_bumped, topic_bumper, poll_title, poll_start, poll_length, poll_max_options, poll_last_vote, poll_vote_change) values(%s, 0, %s, 1, 0, %s, %s, %s, 0, 0, 0, 0, 0, 0, %s, %s, '', %s, %s, %s, '', %s, %s, %s, 0, 0, 0, '', 0, 0, 1, 0, 0)" cursor.execute( sql, (self.forum_id, has_attachments, exchangeDigest.subj, exchangeDigest.user_id, exchangeDigest.timeSent, exchangeDigest.post_id, exchangeDigest.From, exchangeDigest.post_id, exchangeDigest.user_id, exchangeDigest.From, exchangeDigest.subj, exchangeDigest.timeSent, exchangeDigest.timeSent)) topic_id = str(cursor.lastrowid) sql = "insert into phpbb_posts (topic_id, forum_id, poster_id, icon_id, poster_ip, post_time, post_approved, post_reported, enable_bbcode, enable_smilies, enable_magic_url, enable_sig, post_username, post_subject, post_text, post_checksum, post_attachment, bbcode_bitfield, bbcode_uid, post_postcount, post_edit_time, post_edit_reason, post_edit_user, post_edit_count, post_edit_locked) values(%s, %s, %s, 0, '10.1.12.70', %s, 1, 0, 1, 1, 1, 1, '', %s, %s, %s, %s, '', '', 0, 0, '', 0, 0, 0)" cursor.execute( sql, (topic_id, self.forum_id, exchangeDigest.user_id, exchangeDigest.timeSent, exchangeDigest.subj, body, body_md5, has_attachments)) exchangeDigest.post_id = str(cursor.lastrowid) sql = "update phpbb_users set user_posts = user_posts+1 where user_id = %s" cursor.execute(sql, (exchangeDigest.user_id)) if (has_attachments == '1'): try: for attachment in data.Attachments.FileAttachment: try: id = str(line.AttachmentId._Id) self.writeFile(id, exchangeDigest.post_id, topic_id, exchangeDigest.user_id, exchangeDigest.timeSent) except AttributeError: id = str( getattr(attachment[1], "_Id", None)) mimetype = str( getattr(attachment[1], "ContentType", None)) if (id != "None"): self.writeFile(id, exchangeDigest.post_id, topic_id, exchangeDigest.user_id, exchangeDigest.timeSent) except AttributeError: error = AttributeError
def test_same_as_binascii_crc32(self): foo = b'abcdefghijklmnop' crc = 2486878355 self.assertEqual(binascii.crc32(foo), crc) self.assertEqual(zlib.crc32(foo), crc) self.assertEqual(binascii.crc32(b'spam'), zlib.crc32(b'spam'))
def scrapy_comment_user_ifeng(guid, username, adminid, address): ''' 凤凰网个人页面爬取 http://comment.ifeng.com/get? job=7 & format=json & pagesize=20 & _1460705534 & guid=65969467 & p=1 ''' username_decode = unquote(username) siteid = 2 # 判断用户是否存在 sql = 'select userid,adminid from system_site_user where siteid=%s and username=%s' r = mysql_cursor.query(sql, siteid, username_decode) if r: if int(adminid) != int(r[0]['adminid']): print '网站帐号存在,且adminid不符' processlog('auto_scrapyuser', 1, 'scrapy_comment_user_ifeng', '网站帐号存在,添加人不匹配,现:%s, 原:%s' % (adminid, r[0]['adminid'])) return print '网站帐号存在' userid = r[0]['userid'] setAddressStatus(userid, 1) else: processlog('auto_scrapyuser', 1, 'scrapy_comment_user_ifeng', '网站帐号不存在,添加:%s' % username) crc32_address = crc32(address) & 0xffffffff sql = 'insert into system_site_user(`siteid`, `username`,`createtime`, `adminid`, `address`, `crc32address`, `status`) values(%s, %s, now(), %s, %s, %s, 1)' userid = mysql_cursor.execute_lastrowid(sql, siteid, username_decode, adminid, address, crc32_address) headers = { 'Host': 'comment.news.163.com', 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate', 'Connection': 'keep-alive' } # 默认爬取6页 for page in xrange(scrapy_page): url = 'http://comment.ifeng.com/get?job=7&format=json&pagesize=20&guid=%s&p=%s' % ( guid, page) req = requests.get(url, headers=headers, timeout=timeout) if req.status_code == 200: data = json.loads(req.text) comments = data['comments'] for comment in comments: _url = comment['doc_url'] # 判断url是否支持 res = r'://(.*?)/' ret = re.findall(res, _url) if ret: shorturl = ret[0] if shorturl in [ 'news.ifeng.com', ]: title = comment['doc_name'] #帖子标题 content = comment['comment_contents'] createTime = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(int(comment['create_time']))) commentId = comment['comment_id'] nickname = comment['uname'] url_post = 'http://gentie.ifeng.com/view.html?docUrl=' + quote( _url.encode('utf8')) + '&docName=' + quote( title.encode('utf8')) #判断帖子是否保存过 sql = 'select postid from system_url_posts where `commentIds`=%s and createTime=%s and `adminid`=%s' r = mysql_cursor.query(sql, commentId, createTime, adminid) if not r: #判断url是否添加过 crc32_url = crc32(url_post) & 0xffffffff sql = 'select urlid from system_url_list where `crc32url`=%s and adminid=%s' ret = mysql_cursor.query(sql, crc32_url, adminid) if ret: #添加过 urlid = ret[0]['urlid'] else: sql = 'insert into system_url_list(`siteid`, `title`, `url`, `crc32url`, `addtime`,`status`, `adminid`) values(%s, %s, %s, %s, now(), 1, %s)' urlid = mysql_cursor.execute_lastrowid( sql, siteid, title, url_post, crc32_url, adminid) processlog('auto_scrapyuser', 1, 'scrapy_comment_user_ifeng', 'url未添加过,添加url,urlid:%s' % urlid) try: #保存帖子 sql = 'insert into system_url_posts(`urlid`, `userid`, `commentIds`, `content`, `nickname`'\ ', `createTime`, `adminid`) values(%s,%s,%s,%s,%s,%s,%s)' postid = mysql_cursor.execute_lastrowid( sql, urlid, userid, commentId, content, nickname, createTime, adminid) print '保存帖子: %s; postid :%s ; adminid : %s' % ( nickname, postid, adminid) processlog( 'auto_scrapyuser', 1, 'scrapy_comment_user_ifeng', '保存帖子: %s; postid :%s ; adminid : %s' % (nickname, postid, adminid)) except Exception, e: if 'Data too long for column' in str(e): processlog( 'auto_scrapyuser', 1, 'scrapy_comment_user_ifeng', '帖子内容过长,重新截取写入,urlid:%s' % urlid) content = content[:255] #保存帖子 sql = 'insert into system_url_posts(`urlid`, `userid`, `commentIds`, `content`, `nickname`'\ ', `createTime`, `adminid`) values(%s,%s,%s,%s,%s,%s,%s)' postid = mysql_cursor.execute_lastrowid( sql, urlid, userid, commentId, content, nickname, createTime, adminid) print '保存帖子: %s; postid :%s ; adminid : %s' % ( nickname, postid, adminid) processlog( 'auto_scrapyuser', 1, 'scrapy_comment_user_ifeng', '保存帖子: %s; postid :%s ; adminid : %s' % (nickname, postid, adminid)) # 更新site_user状态 setAddressStatus(userid, 0) else: print '帖子已经添加过: commentId:%s' % commentId # processlog('auto_scrapyuser', 1, 'scrapy_comment_user_ifeng', '帖子已经添加过: commentId:%s' % commentId) #如果到最后一页,退出循环 total = data['count'] if (page + 1) * 20 >= total: break else: print req.text
def test_get_metadata_checksums_correctly(self): parts = self._get_written_line().decode('ascii').strip().split(' ') expected_checksum = '{0:08x}'.format( crc32(' '.join(parts[3:]).encode('utf-8')) & 0xffffffff) checksum = parts[2] self.assertEqual(expected_checksum, checksum)
def getpwuuid(self, uuid): tmp = (self.domain_name or "").encode('utf-8') (checksum, uid) = parse_uuid2(uuid) if crc32(tmp) != checksum: return None return self._convert_user(self.server.getpwuid(uid))
def scrapy_comment_user_163(username, adminid, address): ''' 网易用户所有跟贴的爬取 ''' username_decode = base64.b64decode(username) siteid = mysql_cursor.query( 'select siteid from system_site_list where shorturl="comment.news.163.com"' )[0]['siteid'] # 判断用户是否存在 sql = 'select userid,adminid from system_site_user where siteid=%s and username=%s ' r = mysql_cursor.query(sql, int(siteid), username_decode) if r: if int(adminid) != int(r[0]['adminid']): print '网站帐号存在,添加人不匹配' processlog('auto_scrapyuser', 1, 'scrapy_comment_user_163', '网站帐号存在,添加人不匹配,现:%s, 原:%s' % (adminid, r[0]['adminid'])) return userid = r[0]['userid'] setAddressStatus(userid, 1) else: processlog('auto_scrapyuser', 1, 'scrapy_comment_user_163', '网站帐号不存在,添加:%s,userid:%s' % (username, adminid)) crc32_address = crc32(address) & 0xffffffff sql = 'insert into system_site_user(`siteid`, `username`,`createtime`, `adminid`, `address`, `crc32address`, `status`) values(%s, %s, now(), %s, %s, %s, 1)' userid = mysql_cursor.execute_lastrowid(sql, siteid, username_decode, adminid, address, crc32_address) headers = { 'Host': 'comment.news.163.com', 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:45.0) Gecko/20100101 Firefox/45.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate', 'Connection': 'keep-alive' } #默认爬取6页 for page in xrange(scrapy_page): url = 'http://comment.api.163.com/api/v1/products/a2869674571f77b5a0867c3d71db5856/users/0/comments?username=%s&offset=%s&limit=30&ibc=newspc' % ( username, page) req = requests.get(url, headers=headers, timeout=timeout) if req.status_code == 200: data = json.loads(req.text) threads = data['threads'] urllist = [] for k, v in threads.items(): param = {} _url = v['url'] # 判断url是否支持 res = r'://(.*?)/' ret = re.findall(res, _url) if ret: shorturl = ret[0] if shorturl in [ 'news.163.com', ]: boardId = v['boardId'] param['docId'] = v['docId'] param['title'] = v['title'] param[ 'url'] = 'http://comment.news.163.com/' + boardId + '/' + v[ 'docId'] + '.html' urllist.append(param) else: processlog('auto_scrapyuser', 1, 'crapy_comment_user_163', 'url不支持:%s' % _url) comments = data['comments'] for k, v in comments.items(): url_post = '' title = '' for u in urllist: if u['docId'] == k.split('_')[0]: url_post = u['url'] title = u['title'] buildLevel = v['buildLevel'] # 判断是否含有nickname, 是否是最外层的评论 if url_post and title and v['user'].has_key( 'nickname') and buildLevel == 1: nickname = v['user']['nickname'] commentId = v['commentId'] createTime = v['createTime'] content = v['content'].encode('utf8') #判断帖子是否保存过 sql = 'select postid from system_url_posts where `commentIds`=%s and createTime=%s and `adminid`=%s' r = mysql_cursor.query(sql, commentId, createTime, adminid) if not r: #判断url是否添加过 crc32_url = crc32(url_post) & 0xffffffff sql = 'select urlid from system_url_list where `crc32url`=%s and `adminid`=%s' ret = mysql_cursor.query(sql, crc32_url, adminid) if ret: #添加过 urlid = ret[0]['urlid'] else: sql = 'insert into system_url_list(`siteid`, `title`, `url`, `crc32url`, `addtime`, `status`, `adminid`) values(%s,%s,%s,%s,now(),1, %s)' urlid = mysql_cursor.execute_lastrowid( sql, siteid, title, url_post, crc32_url, adminid) processlog('auto_scrapyuser', 1, 'scrapy_comment_user_163', 'url未添加过,添加url,urlid:%s' % urlid) #保存帖子 try: sql = 'insert into system_url_posts(`urlid`, `userid`, `commentIds`, `content`, `nickname`'\ ', `createTime`, `adminid`) values(%s,%s,%s,%s,%s,%s,%s)' postid = mysql_cursor.execute_lastrowid( sql, urlid, userid, commentId, content, nickname, createTime, adminid) print '保存帖子: %s; postid :%s ; adminid : %s' % ( nickname, postid, adminid) processlog( 'auto_scrapyuser', 1, 'scrapy_comment_user_163', '保存帖子: %s; postid :%s ; adminid : %s' % (nickname, postid, adminid)) except Exception, e: # 有的字符集无法保存 if 'Incorrect string value:' in str(e): print '存在表情,无法保存content, nickname:%s' % nickname processlog( 'auto_scrapyuser', 0, 'scrapy_comment_user_163', '存在表情,无法保存content, nickname:%s' % nickname) elif 'Data too long for column' in str(e): processlog('auto_scrapyuser', 1, 'scrapy_comment_user_163', '帖子内容过长,重新截取写入,urlid:%s' % urlid) content = content[:255] sql = 'insert into system_url_posts(`urlid`, `userid`, `commentIds`, `content`, `nickname`'\ ', `createTime`, `adminid`) values(%s,%s,%s,%s,%s,%s,%s)' postid = mysql_cursor.execute_lastrowid( sql, urlid, userid, commentId, content, nickname, createTime, adminid) print '保存帖子: %s; postid :%s ; adminid : %s' % ( nickname, postid, adminid) processlog( 'auto_scrapyuser', 1, 'scrapy_comment_user_163', '保存帖子: %s; postid :%s ; adminid : %s' % (nickname, postid, adminid)) else: print e processlog('auto_scrapyuser', 0, 'scrapy_comment_user_163', str(e)) # 更新site_user状态 setAddressStatus(userid, 0) else: print '帖子保存过:postid:%s' % r[0]['postid'] # processlog('auto_scrapyuser', 1, 'scrapy_comment_user_163', '帖子保存过:postid:%s' % r[0]['postid']) #如果到最后一页,退出循环 total = data['total'] if (page + 1) * 30 >= total: break else: print req.text