Пример #1
0
    def push(self, data, allow_none=False):
        if data is None and allow_none is False:
            raise ValueError(
                "Code push has no data (use allow_none=True to push empty data)"
            )

        # Checks the hex-digest of the current iteration to see if anything changes
        # and will not update if nothing has occurred to the output
        if md5sum(getattr(self.current, 'code', None)) == md5sum(data):
            return self.current

        # Build the new item
        next = Iteration(data, from_id=self.active_iteration)
        self.iterations.append(next)

        to_id = len(self.iterations) - 1

        # Update to "to_id" for our old iteration to point to the new one
        # but only do this if we had an old iteration
        if bound_check(self.iterations, self.active_iteration):
            self.iterations[self.active_iteration].branch(to_id)

        # Update our active iteration tracker
        self.active_iteration = to_id

        # Returns the current item (added from this push)
        return self.current
Пример #2
0
 def checkMatch(self, imgpath, txtpath, confpath):
     # return True
     in1md5 = utils.md5sum(imgpath)
     in2md5 = utils.md5sum(txtpath)
     ta1md5 = utils.getJsonAttr('ImageMD5', None, confpath)
     ta2md5 = utils.getJsonAttr('TextMD5', None, confpath)
     return in1md5 == ta1md5 and in2md5 == ta2md5
Пример #3
0
	def __init__(self, path, plist=None):

		#init path for bundle
		self.bundle = None

		#if its a directory (e.g. an app)
		# ->get binary (from info.plist)
		if os.path.isdir(path):

			#save bundle path
			self.bundle = path

			#get path
			self.path = utils.getBinaryFromBundle(path)

		else:

			#save path
			self.path = path

		#convert file path to utf-8 if needed
		if isinstance(self.path, unicode):

			#convert
			self.path = self.path.encode('utf-8')

		#save plist
		# ->this will be set for launch daemons/agents
		self.plist = plist

		#compute/save name
		self.name = os.path.split(self.path)[1]

		#compute/save hash
		self.hash = utils.md5sum(self.path)

		#compute/save size
		self.size = os.path.getsize(self.path)

		#init whitelist info
		self.whitelistInfo = None

		#init signing authorities
		self.signingAuthorities = None

		#check if its whitelisted
		# ->hash is key for whitelist info
		if self.hash in whitelist.whitelistedFiles:

			#grab whitelist info
		 	self.whitelistInfo = whitelist.whitelistedFiles[self.hash]

		#init
		self.signatureStatus = utils.errSecCSUnsigned

		#check if signed and if so, by apple
		# note: sets class's signatureStatus and signingAuthorities iVars
		self.initSigningStatus()

		return
Пример #4
0
 def validate_segment_md5sums(self):
     if not self.download.check_segment_md5sums:
         return True
     corrupt_segments = 0
     intervals = sorted(self.completed.items())
     pbar = ProgressBar(widgets=[
         'Checksumming {}: '.format(self.download.ID), Percentage(), ' ',
         Bar(marker='#', left='[', right=']'), ' ', ETA()])
     with mmap_open(self.download.path) as data:
         for interval in pbar(intervals):
             log.debug('Checking segment md5: {}'.format(interval))
             if not interval.data or 'md5sum' not in interval.data:
                 log.error(STRIP(
                     """User opted to check segment md5sums on restart.
                     Previous download did not record segment
                     md5sums (--no-segment-md5sums)."""))
                 return
             chunk = data[interval.begin:interval.end]
             checksum = md5sum(chunk)
             if checksum != interval.data.get('md5sum'):
                 log.debug('Redownloading corrupt segment {}, {}.'.format(
                     interval, checksum))
                 corrupt_segments += 1
                 self.completed.remove(interval)
     if corrupt_segments:
         log.warn('Redownloading {} currupt segments.'.format(
             corrupt_segments))
Пример #5
0
def main(args):
    zipfilepath = args.zip
    if zipfilepath is None:
        print "pass arguements correctly!"
        exit(-1)
    xmlfilepath = args.xmlfile
    zip_path = zipfilepath
    if utils.valid_file(zip_path) is not True:
        print "bad zip"
        exit(-1)
    data_for_all_files = []
    path_to_extract = utils.random_temp_path(TEMP_DIR)
    utils.extractor(zip_path, path_to_extract)
    list_of_all_files = utils.getListOfFiles(path_to_extract)
    
    for path_to_file in list_of_all_files:
        uid = utils.get_uuid()
        filename = utils.stripfilepath(path_to_file)
        rel_path = utils.get_relative_path(path_to_file, path_to_extract)
        md5hash = utils.md5sum(path_to_file)
        filesize = utils.get_file_size(filepath=path_to_file)
        data = FileDetails(file_uuid=uid, file_name=filename, file_full_path=path_to_file, relative_path=rel_path, file_md5hash=md5hash, file_size=filesize)
        data_for_all_files.append(data)
    
    XS.XMLSerialize(data_for_all_files, xmlfilepath)
    utils.cleanup(path_to_extract)
    exit(0)
Пример #6
0
def s2e_concolic_testcase_watcher(seedbox, testcases):
    setproctitle.setproctitle('S2E test case watcher')

    processed = set()
    count = 0

    i = adapters.InotifyTree(testcases, mask=IN_CLOSE_WRITE)
    for event in i.event_gen():
        # auto join child process
        if event is None:
            continue

        (_, _, path, filename) = event
        full_name = os.path.join(path,filename)

        md5 = md5sum(full_name)
        if md5 in processed:
            continue
        processed.add(md5)
        count += 1

        dst_file= 'id:{:06d},{}'.format(count, filename)
        dst = '{}/{}'.format(seedbox, dst_file)
        shutil.copyfile(full_name, dst)
        print '[{}] -> [{}]'.format(filename, dst_file)
Пример #7
0
def validate():
    """ Uploads a file to a draft area then tests if it is valid.
        Expects:
            index_view_slug:   unique name of tif file to lookup(string)
            file:              in request.files for upload
    """
    index_view_slug = request.form.get('index_view_slug')

    if not index_view_slug or 'file' not in request.files:
        abort(400)

    if index_view_slug.endswith('_raster'):
        index_view_slug = index_view_slug[:-7]

    draft_file = os.path.join(conf.DRAFT_DIR,
                              '{}.tif'.format(index_view_slug))
    request.files['file'].save(draft_file)
    result = {'errors': []}
    try:
        valid = is_valid(
            index_view_slug, base_path=conf.DRAFT_DIR
        )
        result = {
            'md5': md5sum(draft_file),
            'valid': valid
        }
        os.remove(draft_file)
    except NotImplementedError as e:
        result['errors'].append(e.message)

    return jsonify(result)
Пример #8
0
    def process_coverage(self, contents, input_id, bb_dict):
        """ insert into coverage table """
        nodes = list()
        hashes = [x[3] for x in contents]
        for filename, tbs in self._mmap_tb.iteritems():
            # skip if the hash does not exist in database, else get the database index
            if not md5sum(filename) in hashes:
                continue

            for tb, value in tbs.iteritems():  # iterate through tbs
                bb_idx = bb_dict.get(tb)
                if bb_idx:
                    nodes.append([bb_idx, value])
                else:
                    # if no full match found , try to match both bb's end address and start address
                    # this is because bb condition 2 is not handled (dynamic tb spans multiple bbs)
                    # this should be rare, so looping through the dict for now
                    for bb in bb_dict.iterkeys():
                        if tb[0] >= bb[0] and tb[0] < bb[1]:
                            nodes.append([bb_dict[bb], value])
                        if tb[1] > bb[0] and tb[1] <= bb[1]:
                            nodes.append([bb_dict[bb], value])

        # print 'number of coverage inserted: {}'.format(len(nodes))
        gen = ('{}\t{}\n'.format(x[0], x[1]) for x in nodes)
        bio = BytesIO(''.join(gen))
        self._db.insert_coverage(bio, input_id, self._project_id)
Пример #9
0
def publish():
    """ Uploads a file to a draft area then moves it to data directory if valid.
        Expects:
            index_view_slug:   unique name of tif file to lookup(string)
            file:              in request.files for upload
    """

    index_view_slug = request.form.get('index_view_slug')

    if not index_view_slug or 'file' not in request.files:
        abort(400)

    if index_view_slug.endswith('_raster'):
        index_view_slug = index_view_slug[:-7]

    draft_file = os.path.join(conf.DRAFT_DIR,
                              '{}.tif'.format(index_view_slug))
    request.files['file'].save(draft_file)
    result = {'errors': []}
    try:
        result['valid'] = is_valid(index_view_slug, base_path=conf.DRAFT_DIR)
        if result['valid']:
            live_file = make_live(draft_file)
            if os.path.exists(live_file):
                result = {
                    'md5': md5sum(live_file),
                }

    #except Exception as e:
    except NotImplementedError as e:
        result['errors'].append(e.message)

    return jsonify(result)
Пример #10
0
def get_md5sum(request, share, subpath):
    from utils import md5sum
    file_path = os.path.join(share.get_path(), subpath)
    try:
        return json_response({'md5sum': md5sum(file_path), 'path': subpath})
    except Exception, e:
        return json_error(str(e))
Пример #11
0
 def run(self, data):
     # Initialize our output with our input data
     output = data
     
     # Check if we match (we need to know this later)
     # If we have no matcher, we just skip this matching step
     matched = (not self.matcher) or self.match(data)
     
     if matched:
         output = self.execute(data)
         
         if utils.md5sum(output) != utils.md5sum(data) and self.recursive_on_change:
             _, output = self.run(output)
     
     # The recursion will handle our output and our matched variable
     # will contain the match if we ever matched (not when the final case fails)
     return matched, output
Пример #12
0
    def process_edge(self, contents, edge_set, bb_dict, input_id):
        """ update edge table according to dynamic analyze results """

        nodes = list()

        hashes = [x[3] for x in contents]

        for filename, edges in self._mmap_edges.iteritems():
            # skip if the hash does not exist in database, else get the database index
            try:
                content_id = contents[hashes.index(md5sum(filename))][0]
            except ValueError:
                continue

            for edge in edges:
                # map basic blocks with their database index
                from_bb = edge[0]
                to_bb = edge[1]

                # first node and last node, not insert into edge table
                if from_bb is None or to_bb is None:
                    continue

                idx_from_bb = bb_dict.get(from_bb)
                idx_to_bb = bb_dict.get(to_bb)

                # # if no full match found for idx_from_bb, try to match bb's end address
                # # this is because bb condition 2 is not handled (dynamic tb spans multiple bbs)
                # # this should be rare, so looping through the dict for now
                # if idx_from_bb is None:
                #     for bb in bb_dict.iterkeys():
                #         if from_bb[1] <= bb[1] and from_bb[1] > bb[0]:
                #             idx_from_bb = bb_dict[bb]
                #             break

                # # same with to_bb, try to match bb's start address
                # if idx_to_bb is None:
                #     for bb in bb_dict.iterkeys():
                #         if to_bb[0] >= bb[0] and to_bb[0] < bb[1]:
                #             idx_to_bb = bb_dict[bb]
                #             break

                # still can't find, it might be because the basic block does not
                # belong to any of the memory mapped file in the project content table
                # ignore for now...
                if idx_from_bb is None or idx_to_bb is None:
                    continue

                edge_with_idx = (idx_from_bb, idx_to_bb)
                if edge_with_idx not in edge_set:
                    # insert new edge
                    nodes.append(edge_with_idx)

            # print 'number of edges inserted: {}'.format(len(nodes))
            gen = ('{}\t{}\t{}\n'.format(self._project_id, x[0], x[1])
                   for x in nodes)
            bio = BytesIO(''.join(gen))
            self._db.insert_edge(bio, input_id)
Пример #13
0
def update_md5sum(key):
    # This view is a hack. We need proper SQL migrations instead of this
    if key == app.config['PERIODIC_KEY']:
        for post in JobPost.query.all():
            if not post.md5sum:
                post.md5sum = md5sum(post.email)
        db.session.commit()
        return Response("Updated md5sum for all posts.\n",
                        content_type='text/plain; charset=utf-8')
    else:
        abort(403)
Пример #14
0
    def fill(self, piece_id, buf, md5=None):
        with self._lock:
            start = piece_id * self.piece_size
            self.fileob.seek(start)
            self.fileob.write(buf)
            self.fileob.flush()
            if md5:
                piece_md5 = md5
            else:
                piece_md5 = md5sum(buf)

            self.filled(piece_id, md5=piece_md5)
Пример #15
0
def run_fuzz(argv):
    ''' run the test '''
    # check the requirements of cim_fuzz
    check_req(argv)

    # working directory name

    working_dir = '{}/cb_{}'.format(
        argv['cbhome'], str(datetime.now().strftime('%Y-%m-%d-%H%M%S.%f')))

    check_dir(working_dir)

    # 1. download from remote or get the file with path
    file_name = url_get_file(argv['uri'], working_dir)

    # 2. unzip the file
    dir_name = unzip(file_name, working_dir)
    argv['tar_file'] = file_name
    print_sep()
    print '[C]: working directory: [{}]'.format(dir_name)

    # prepare the experiment directory structure
    dir_struct = prepare_dir(dir_name)

    # get the architecture of the test binary
    binary = '{}/cb'.format(dir_struct['binary'])
    bin_arch = get_file_arch(binary)[0]
    if bin_arch not in ['32bit', '64bit']:
        print '[C]: unsupported file arch!, exiting now'
        exit(0)
    if bin_arch == '64bit':
        argv['arch'] = 'x86_64'
    if bin_arch == '32bit':
        argv['arch'] = 'i386'

    # first 8 bytes of md5 as file id
    argv['md5sum'] = md5sum(binary)
    argv['fid'] = argv['md5sum'][:8]
    check_dir('{}/{}'.format(dir_struct['out_afl'], argv['fid']))

    print "out_afl:{}/{}".format(dir_struct['out_afl'], argv['fid'])

    # save command to cimfuzz.cmd file
    argv.pop('func', None)
    with open('{}/cimfuzz.cmd'.format(dir_name), 'w') as fuzz_cmd:
        json.dump(argv, fuzz_cmd)

    # globals for flask
    if argv['afl_only']:
        execute_aflonly(dir_struct, argv)
    else:
        execute(dir_struct, argv)
Пример #16
0
    def save_action(file):
        if fnmatch.fnmatch(file, pattern):
            if db.is_existed(file):
                return

            name = os.path.basename(file).lower()
            hash = md5sum(file)
            db.insert_file(hash, name, file)

            cache['current'] += 1
            if cache['current'] % 2000 == 0:
                logging.info('Processing: {} / {}'.format(
                    cache['current'], total_count))
                db.commit_changes()
Пример #17
0
def signin(db):
	"""
		登陆
	"""
	passwd = request.POST.get('passwd','')
	db.execute('SELECT PASSWD FROM USER')
	v, = db.getone()
	if v != passwd:
		return template("error",msg="密码错误")
	#-设定cookie
	token = md5sum('%s-%s' % (now(),random.random()*now()))
	db.execute('INSERT INTO COOKIE(TOKEN) VALUES(%s)',token)
	d = urlparse.urlparse(CFG.domain).netloc.split(':')[0]
	response.set_cookie('token',token,path="/",expires=int(now()/1000)+3600*24*365,domain=d)
	redirect("/",301)
Пример #18
0
def signin(db):
    """
		登陆
	"""
    passwd = request.POST.get('passwd', '')
    db.execute('SELECT PASSWD FROM USER')
    v, = db.getone()
    if v != passwd:
        return template("error", msg="密码错误")
    #-设定cookie
    token = md5sum('%s-%s' % (now(), random.random() * now()))
    db.execute('INSERT INTO COOKIE(TOKEN) VALUES(%s)', token)
    d = urlparse.urlparse(CFG.domain).netloc.split(':')[0]
    response.set_cookie('token',
                        token,
                        path="/",
                        expires=int(now() / 1000) + 3600 * 24 * 365,
                        domain=d)
    redirect("/", 301)
Пример #19
0
    def process_file(self, filename):
        """ insert binary entry into table content """
        ins_file = ('insert into file (hash, file) '
                    'values (%s, %s) on conflict do nothing returning id;')

        # calculate md5
        md5 = md5sum(filename)

        with open(filename, 'rb') as f_src:
            src_content = f_src.read()

        content = (md5, psycopg2.Binary(src_content))
        file_id, rowcount = self._db.execute(ins_file, content, fetchone=True)
        if rowcount == 0:
            query = 'select id from file where hash = %s;'
            file_id, _ = self._db.execute(query, (md5, ),
                                          fetchone=True,
                                          commit=False)

        return file_id[0]
Пример #20
0
def get_md5_for_index_view_slug():
    """ Gets the md5 value for a tif matching the slug parameter or -1 if
    not found.

        Expects:
            index_view_slug:   unique name of tif file to lookup(string)
    """
    index_view_slug = request.args.get('index_view_slug')

    if not index_view_slug:
        abort(400)

    value = -1
    try:
        raster_file = get_raster_file_path(index_view_slug)
        value = md5sum(raster_file)

    except IOError:
        pass

    result = {
        'md5': value
    }
    return jsonify(result)
Пример #21
0
def packing(node, parameters, compiler_replace_maps):

    package = node.get_package_name()
    version_git = node.get_version()
    packing = node.is_packing()
    if not packing:
        logging.warning('Skiping package: %s' % package)
        return 0

    manager = node.get_version_manager()
    if manager == "git":
        build_modes = node.get_build_modes()
        for plat, build_mode in product(platforms, build_modes):
            workspace = node.get_workspace(plat)
            build_directory = os.path.join(
                os.getcwd(), node.get_build_directory(plat, build_mode))
            revision_git = hash_version.get_last_changeset(build_directory,
                                                           short=False)
            version_old = node.get_version()
            version_git = hash_version.to_cmaki_version(
                build_directory, revision_git)
            logging.info('[git] Renamed version from %s to %s' %
                         (version_old, version_git))

            # renombrar package-version-platform/package-version

            workspace = node.get_workspace(plat)
            source_folder = node.get_base_folder()
            oldversion = node.get_version()
            try:
                node.set_version(version_git)
                new_workspace = node.get_workspace(plat)
                new_source_folder = node.get_base_folder()

                # changed version ?
                if source_folder != new_source_folder:
                    utils.move_folder_recursive(
                        os.path.join(workspace, source_folder),
                        os.path.join(workspace, new_source_folder))
                    utils.move_folder_recursive(workspace, new_workspace)
            finally:
                node.set_version(oldversion)

    node.set_version(version_git)
    version = node.get_version()

    # regenerate autoscripts with new version
    node.generate_scripts_headers(compiler_replace_maps)

    precmd = ''
    if utils.is_windows():
        precmd = 'cmake -E '

    folder_3rdparty = parameters.third_party_dir
    output_3rdparty = os.path.join(folder_3rdparty, node.get_base_folder())
    utils.trymkdir(output_3rdparty)

    folder_mark = os.path.join(parameters.prefix, node.get_base_folder())
    utils.trymkdir(folder_mark)

    utils.superverbose(
        parameters,
        '*** [%s] Generation cmakefiles *** %s' % (package, output_3rdparty))
    errors = node.generate_cmakefiles(platforms, output_3rdparty,
                                      compiler_replace_maps)
    logging.debug('errors generating cmakefiles: %d' % errors)
    node.ret += abs(errors)

    for plat in platforms:
        utils.superverbose(
            parameters, '*** [%s (%s)] Generating package .tar.gz (%s) ***' %
            (package, version, plat))
        workspace = node.get_workspace(plat)
        utils.trymkdir(workspace)
        with utils.working_directory(workspace):

            if utils.is_windows():
                utils.safe_system('del /s *.ilk')
                utils.safe_system('del /s *.exp')

            source_folder = node.get_base_folder()
            prefix_package = os.path.join(parameters.prefix,
                                          '%s.tar.gz' % workspace)
            prefix_package_cmake = os.path.join(parameters.prefix,
                                                '%s-cmake.tar.gz' % workspace)
            prefix_package_md5 = os.path.join(output_3rdparty,
                                              '%s.md5' % workspace)

            logging.info(
                'generating package %s from source %s' %
                (prefix_package, os.path.join(os.getcwd(), source_folder)))
            logging.info('generating md5file %s' % prefix_package_md5)

            # packing install
            gen_targz = "%star zcvf %s %s" % (precmd, prefix_package,
                                              source_folder)

            node.ret += abs(node.safe_system(gen_targz, compiler_replace_maps))
            if not os.path.exists(prefix_package):
                logging.error('No such file: {}'.format(prefix_package))
                return False

            # calculate md5 file
            package_md5 = utils.md5sum(prefix_package)
            logging.debug("new package {}, with md5sum {}".format(
                prefix_package, package_md5))
            with open(prefix_package_md5, 'wt') as f:
                f.write('%s\n' % package_md5)

    # marker is a empty file
    # prefix_package_marker = os.path.join(folder_mark, '%s.cache' % get_identifier('ALL'))
    # logging.info('generating marker %s' % prefix_package_marker)
    # open(prefix_package_marker, 'a').close()

    # packing cmakefiles (more easy distribution)
    if not parameters.no_packing_cmakefiles:
        for plat in platforms:
            base_folder = node.get_base_folder()
            prefix_package_cmake = os.path.join(
                parameters.prefix, '%s-%s-cmake.tar.gz' % (base_folder, plat))
            with utils.working_directory(folder_3rdparty):
                logging.debug('working dir: %s' % folder_3rdparty)
                # packing install
                logging.info('generating package cmake %s' %
                             prefix_package_cmake)
                gen_targz_cmake = '{}tar zcvf {} {}'.format(
                    precmd, prefix_package_cmake, node.get_base_folder())
                node.ret += abs(
                    node.safe_system(gen_targz_cmake, compiler_replace_maps))

    # finish well
    return True
Пример #22
0
def execute_aflonly(dir_struct, argv):
    try:
        container = run_afl(dir_struct, argv)
        prev_cvg = dict()

        seedbox_dir = os.path.join(dir_struct['out_afl'], argv['fid'], 'seedbox', 'queue')
        s2e_testcase_dir = '{}/testcases'.format(dir_struct['out_s2e'])

        print 'watching [{}]'.format(s2e_testcase_dir)
        check_dir(seedbox_dir)
        check_dir(s2e_testcase_dir)

        s2e_testcase_handler = Process(target=s2e_concolic_testcase_watcher, args=[seedbox_dir, s2e_testcase_dir])
        s2e_testcase_handler.start()

        processes = list()
        q = Queue()
        while True:
            print 'sleep'
            time.sleep(60)
            # updated = False
            # watch for afl fuzzer_stats change
            cur_cvg = get_cvg(dir_struct, argv)

            # __import__('pprint').pprint(cur_cvg)

            # if not prev_cvg:
            #     prev_cvg = cur_cvg
            #     continue


            # for node, cvg in cur_cvg.iteritems():
            #     if prev_cvg[node] != cvg:
            #         updated = True
            #         break

            # if updated:
            #     continue

            if len(processes) >= argv['num_s2e']:
                print argv['num_s2e']
                for p in processes:
                    print p
                active_children()
                processes[:] = [p for p in processes if p[0].is_alive()]
                continue

            # launch s2e with driller searcher, with .cur_input from first node:
            s2e_node = random.choice(cur_cvg.keys())
            seed = os.path.join(dir_struct['out_afl'], argv['fid'], s2e_node, '.cur_input')
            tmp_input_file = os.path.join(dir_struct['in_s2e'], 'seed')

            shutil.copyfile(seed, tmp_input_file)
            input_hash = md5sum(tmp_input_file)
            input_file = os.path.join(dir_struct['in_s2e'], input_hash)
            shutil.copyfile(tmp_input_file, input_file)

            process = Process(target=launch_driller, args=[input_file, input_hash, dir_struct, argv, q])
            # process.daemon=True
            process.start()
            qemu = q.get(timeout=5)
            processes.append((process,qemu))

    except KeyboardInterrupt:
        for p in processes:
            kill_process(p[0])
            kill_process(p[1])
        docker.from_env().containers.get(container).kill()
        kill_process(s2e_testcase_handler)
Пример #23
0
    def __init__(self, path, plist=None, parent=None):

        #init path for bundle
        self.bundle = None

        #if its a directory (e.g. an app bundle)
        # ->get binary (from app's Info.plist)
        if os.path.isdir(path):

            #save bundle path
            self.bundle = path

            #get path
            self.path = utils.getBinaryFromBundle(path)

            #if binary could not be found
            # ->default to 'unknown'
            if not self.path:

                #just set to something...
                self.path = '<unknown>'

        #path is to file
        # ->just save into class var
        else:

            #save
            self.path = path

        #convert file path to utf-8 if needed
        if isinstance(self.path, unicode):

            #convert
            self.path = self.path.encode('utf-8')

        #save plist
        # ->this will be set for launch daemons/agents, inserted dylibs, etc
        self.plist = plist

        #compute/save name
        self.name = os.path.split(self.path)[1]

        #compute/save hash
        self.hash = utils.md5sum(self.path)

        #init whitelist flag
        self.isWhitelisted = False

        #check if its whitelisted
        # ->path is key
        if self.path in whitelist.whitelistedFiles:

            #check if hash is in white list
            self.isWhitelisted = (self.hash
                                  in whitelist.whitelistedFiles[self.path])

        #init
        self.signatureStatus = None

        #init signing authorities
        self.signingAuthorities = None

        #check if signed and if so, by apple
        # note: sets class's signatureStatus and signingAuthorities iVars
        self.initSigningStatus()

        return
Пример #24
0
def editjob(hashid, key, form=None, post=None, validated=False):
    if form is None:
        form = forms.ListingForm(request.form)
        form.job_type.choices = [(ob.id, ob.title) for ob in JobType.query.filter_by(public=True).order_by('seq')]
        form.job_category.choices = [(ob.id, ob.title) for ob in JobCategory.query.filter_by(public=True).order_by('seq')]
    if post is None:
        post = JobPost.query.filter_by(hashid=hashid).first_or_404()
    if key != post.edit_key:
        abort(403)
    # Don't allow email address to be changed once its confirmed
    if request.method == 'POST' and post.status >= POSTSTATUS.PENDING:
        form.poster_email.data = post.email
    if request.method == 'POST' and (validated or form.validate()):
        form_description = bleach.clean(form.job_description.data, tags=ALLOWED_TAGS)
        form_description = bleach.linkify(form_description)
        form_perks = bleach.linkify(bleach.clean(form.job_perks_description.data, tags=ALLOWED_TAGS)) if form.job_perks.data else ''
        form_how_to_apply = form.job_how_to_apply.data
        form_email_domain = get_email_domain(form.poster_email.data)
        form_words = get_word_bag(u' '.join((form_description, form_perks, form_how_to_apply)))

        similar = False
        for oldpost in JobPost.query.filter(JobPost.email_domain == form_email_domain).filter(
                                            JobPost.status > POSTSTATUS.PENDING).filter(
                                            JobPost.datetime > datetime.utcnow() - agelimit).all():
            if oldpost.id != post.id:
                if oldpost.words:
                    s = SequenceMatcher(None, form_words, oldpost.words)
                    if s.ratio() > 0.6:
                        similar = True
                        break

        if similar:
            flash("This listing is very similar to an earlier listing. You may not relist the same job "
                "in less than %d days. If you believe this to be an error, please email us at %s." % (agelimit.days,
                app.config['SUPPORT_EMAIL']), category='interactive')
        else:
            post.headline = form.job_headline.data
            post.type_id = form.job_type.data
            post.category_id = form.job_category.data
            post.location = form.job_location.data
            post.relocation_assist = form.job_relocation_assist.data
            post.description = form_description
            post.perks = form_perks
            post.how_to_apply = form_how_to_apply
            post.company_name = form.company_name.data
            post.company_url = form.company_url.data
            post.email = form.poster_email.data
            post.email_domain = form_email_domain
            post.md5sum = md5sum(post.email)
            post.hr_contact = form.hr_contact.data
            # To protect from gaming, don't allow words to be removed in edited listings once the post
            # has been confirmed. Just add the new words.
            if post.status >= POSTSTATUS.CONFIRMED:
                prev_words = post.words or ''
            else:
                prev_words = u''
            post.words = get_word_bag(u' '.join((prev_words, form_description, form_perks, form_how_to_apply)))

            if request.files['company_logo']:
                # The form's validator saved the processed logo in g.company_logo.
                thumbnail = g.company_logo
                logofilename = uploaded_logos.save(thumbnail, name='%s.' % post.hashid)
                post.company_logo = logofilename
            else:
                if form.company_logo_remove.data:
                    post.company_logo = None

            db.session.commit()
            userkeys = session.get('userkeys', [])
            userkeys.append(post.edit_key)
            session['userkeys'] = userkeys
            session.permanent = True
            return redirect(url_for('jobdetail', hashid=post.hashid), code=303)
    elif request.method == 'POST':
        flash("Please correct the indicated errors", category='interactive')
    elif request.method == 'GET':
        # Populate form from model
        form.job_headline.data = post.headline
        form.job_type.data = post.type_id
        form.job_category.data = post.category_id
        form.job_location.data = post.location
        form.job_relocation_assist.data = post.relocation_assist
        form.job_description.data = post.description
        form.job_perks.data = True if post.perks else False
        form.job_perks_description.data = post.perks
        form.job_how_to_apply.data = post.how_to_apply
        form.company_name.data = post.company_name
        form.company_url.data = post.company_url
        form.poster_email.data = post.email
        form.hr_contact.data = int(post.hr_contact or False)

    return render_template('postjob.html', form=form, no_email=post.status > POSTSTATUS.DRAFT)
Пример #25
0
	def __init__(self, path, plist=None, parent=None):

		#init path for bundle
		self.bundle = None

		#if its a directory (e.g. an app bundle)
		# ->get binary (from app's Info.plist)
		if os.path.isdir(path):

			#save bundle path
			self.bundle = path

			#get path
			self.path = utils.getBinaryFromBundle(path)

			#if binary could not be found
			# ->default to 'unknown'
			if not self.path:

				#just set to something...
				self.path = '<unknown>'

		#path is to file
		# ->just save into class var
		else:

			#save
			self.path = path

		#convert file path to utf-8 if needed
		if isinstance(self.path, unicode):

			#convert
			self.path = self.path.encode('utf-8')

		#save plist
		# ->this will be set for launch daemons/agents, inserted dylibs, etc
		self.plist = plist

		#compute/save name
		self.name = os.path.split(self.path)[1]

		#compute/save hash
		self.hash = utils.md5sum(self.path)

		#compute/save size
		self.size = os.path.getsize(self.path)

		#init whitelist flag
		self.isWhitelisted = False

		#init signing authorities
		self.signingAuthorities = None

		#check if its whitelisted
		# ->path is key
		if self.path in whitelist.whitelistedFiles:

			#check if hash is in white list
		 	self.isWhitelisted = (self.hash in whitelist.whitelistedFiles[self.path])

		#init
		self.signatureStatus = utils.errSecCSUnsigned

		#check if signed and if so, by apple
		# note: sets class's signatureStatus and signingAuthorities iVars
		self.initSigningStatus()

		return
Пример #26
0
      return result


if __name__ == "__main__":
  import os
  import random
  import sys
  import utils
  reload(sys)
  sys.setdefaultencoding('utf-8')
  
  from simplejson import *

  l = os.listdir(".")
  #l = [ "abcd" ]
  random.shuffle(l)
  a = bst(l)
  random.shuffle(l)
  je = JSONEncoder(ensure_ascii=False)
  jd = JSONDecoder()

  json = dumps(l, ensure_ascii = False)
  l1 = loads(json)

  for i in l1:
    fn = a.extract(i)
    md5 = utils.md5sum(fn)
    print "%s %s" % (md5, fn)
  #print a.as_list()

Пример #27
0
    def process_input(path, filename, count):
        setproctitle('S2E test case watcher: processing [{}]'.format(filename))
        # 1. update interested table with testcase info
        src = os.path.join(path, filename)
        try:
            with open(src, 'rb') as f_src:
                src_content = f_src.read()
            md5 = md5sum(src)
        except Exception as excp:
            print getattr(excp, 'message', repr(excp))
            return

        conn = psycopg2.connect(host=db_config['host'],
                                port=db_config['port'],
                                database=db_config['database'],
                                user=db_config['user'],
                                password=db_config['password'])
        cur = conn.cursor()

        id_file = 0
        try:
            query = ('insert into file (hash, file) '
                     'values (%s, %s) on conflict do nothing returning id;')
            cur.execute(query, (md5, psycopg2.Binary(src_content)))
            conn.commit()
            id_file = cur.fetchone()
        except KeyboardInterrupt:
            raise
        except psycopg2.IntegrityError as excp:
            print getattr(excp, 'message', repr(excp))
            conn.rollback()
        except Exception as excp:
            print getattr(excp, 'message', repr(excp))
            conn.rollback()

        # only update interested table if the testcase is generated by concolic mode
        if not filename.startswith('testcase'):
            # l_fn = ['s2e', 'input', idx_input, idx_interested, idx_basic_block, md5_short]
            l_fn = filename.split('_')
            if cur.description is None or id_file is None:
                query = 'select id from file where hash = %s;'
                cur.execute(query, (md5, ))
                id_file = cur.fetchone()

            if id_file is not None:
                query = ('update interested set '
                         'idfile = %s, '
                         'uri = %s, '
                         'update_time=now(), '
                         'status = 3 '
                         'where id = %s returning id;')
                cur.execute(query, (id_file[0], src, l_fn[3]))
                rowcount = cur.rowcount

                if rowcount != 1:
                    conn.rollback()
                else:
                    conn.commit()

        conn.close()

        # copy file from s2e test case output directory to AFL seedbox
        if not filename.startswith('testcase'):
            dst_file = 'id:{:06d},{},{},{},{}'.format(count, l_fn[2], l_fn[3],
                                                      l_fn[4], l_fn[5])
            dst = '{}/{}'.format(seedbox, dst_file)
        else:
            dst_file = 'id:{:06d},{}'.format(count, filename)
            dst = '{}/{}'.format(seedbox, dst_file)

        # lastly copy file to seedbox
        print '[W][S2E]: [{}] -> [{}]'.format(basename(src), basename(dst))
        shutil.copyfile(src, dst)
Пример #28
0
def packing(node, parameters, compiler_replace_maps):

    package = node.get_package_name()
    version_git = node.get_version()
    packing = node.is_packing()
    if not packing:
        logging.warning('Skiping package: %s' % package)
        return 0

    manager = node.get_version_manager()
    if manager == "git":
        build_modes = node.get_build_modes()
        for plat, build_mode in product(platforms, build_modes):
            build_directory = os.path.join(os.getcwd(), node.get_build_directory(plat, build_mode))
            revision_git = hash_version.get_last_changeset(build_directory, short=False)
            version_old = node.get_version()
            version_git = hash_version.to_cmaki_version(build_directory, revision_git)
            logging.info('[git] Renamed version from %s to %s' % (version_old, version_git))

            current_workspace = node.get_binary_workspace(plat)
            current_base = node.get_base_folder()
            oldversion = node.get_version()
            try:
                node.set_version(version_git)
                updated_workspace = node.get_binary_workspace(plat)
                updated_base = node.get_base_folder()

                current_base2 = os.path.join(current_workspace, current_base)
                updated_base2 = os.path.join(current_workspace, updated_base)
                logging.debug("from: %s" % current_base2)
                logging.debug("to: %s" % updated_base2)
                if current_base != updated_base:
                    utils.move_folder_recursive(current_base2, updated_base2)
                    logging.debug('-- copy from: {}, {}'.format(current_workspace, os.path.exists(current_workspace)))
                    logging.debug('-- copy to: {}, {}'.format(updated_workspace, os.path.exists(updated_workspace)))
                    utils.move_folder_recursive(current_workspace, updated_workspace)
            finally:
                node.set_version(oldversion)

    node.set_version(version_git)
    version = node.get_version()

    # regenerate autoscripts with new version
    node.generate_scripts_headers(compiler_replace_maps)

    # # generate versions.cmake
    node.generate_3rdpartyversion(parameters.prefix)

    precmd = ''
    if utils.is_windows():
        precmd = 'cmake -E '

    folder_3rdparty = parameters.third_party_dir
    output_3rdparty = os.path.join(folder_3rdparty, node.get_base_folder())
    utils.trymkdir(output_3rdparty)

    folder_mark = os.path.join(parameters.prefix, node.get_base_folder())
    utils.trymkdir(folder_mark)

    utils.superverbose(parameters, '*** [%s] Generation cmakefiles *** %s' % (package, output_3rdparty))
    errors = node.generate_cmakefiles(platforms, output_3rdparty, compiler_replace_maps)
    logging.debug('errors generating cmakefiles: %d' % errors)
    node.ret += abs(errors)

    for plat in platforms:
        utils.superverbose(parameters, '*** [%s (%s)] Generating package .tar.gz (%s) ***' % (package, version, plat))
        workspace = node.get_workspace(plat)
        current_workspace = node.get_binary_workspace(plat)
        utils.trymkdir(current_workspace)
        with utils.working_directory(current_workspace):

            logging.info('working directory: {}'.format(current_workspace))

            if utils.is_windows():
                utils.safe_system('del /s *.ilk')
                utils.safe_system('del /s *.exp')

            current_base = node.get_base_folder()
            prefix_package = os.path.join(parameters.prefix, '%s.tar.gz' % workspace)
            prefix_package_md5 = os.path.join(output_3rdparty, '%s.md5' % workspace)

            logging.info('generating package %s from source %s' % (prefix_package, os.path.join(os.getcwd(), current_base)))
            logging.info('generating md5file %s' % prefix_package_md5)
            print_folder(current_base)

            # packing install
            gen_targz = "%star zcvf %s %s" % (precmd, prefix_package, current_base)

            node.ret += abs( node.safe_system(gen_targz, compiler_replace_maps) )
            if not os.path.exists(prefix_package):
                logging.error('No such file: {}'.format(prefix_package))
                return False

            # calculate md5 file
            package_md5 = utils.md5sum(prefix_package)
            logging.debug("new package {}, with md5sum {}".format(prefix_package, package_md5))
            with open(prefix_package_md5, 'wt') as f:
                f.write('%s\n' % package_md5)

    # packing cmakefiles (more easy distribution)
    if not parameters.no_packing_cmakefiles:
        for plat in platforms:
            current_base = node.get_base_folder()
            prefix_package_cmake = os.path.join(parameters.prefix, '%s-%s-cmake.tar.gz' % (current_base, plat))
            with utils.working_directory(folder_3rdparty):

                logging.info('working directory: {}'.format(folder_3rdparty))

                logging.debug('working dir: %s' % folder_3rdparty)
                logging.info('generating package cmake %s' % prefix_package_cmake)
                print_folder(current_base)

                gen_targz_cmake = '{}tar zcvf {} {}'.format(precmd, prefix_package_cmake, current_base)
                node.ret += abs( node.safe_system(gen_targz_cmake, compiler_replace_maps) )
                if not os.path.exists(prefix_package_cmake):
                    logging.error('No such file: {}'.format(prefix_package_cmake))
                    return False

    # finish well
    return True
Пример #29
0
 def get_hash_from_file(self):
     """Calculate and set md5 hash from path."""
     self.hash = utils.md5sum(self.path)
Пример #30
0
def afl_watcher(argv):
    setproctitle('AFL test case watcher')

    def process_input(path, filename):
        setproctitle('analyzing [{}:{}]'.format(basename(dirname(path)),
                                                filename))

        db_config = argv['db_config']
        project_id = argv['project_id']
        qemu = argv['qemu']
        basedir = argv['basedir']

        analyzer = DynamicAnalyzer(db_config=db_config,
                                   qemu=qemu,
                                   basedir=basedir,
                                   project_id=project_id)
        analyzer.analyze_dynamic(os.path.join(path, filename))

    def process_stats(path, filename):
        setproctitle('Processing fuzzer_stats for node {}'.format(
            basename(path)))

        db_config = argv['db_config']
        project_id = argv['project_id']

        conn = psycopg2.connect(host=db_config['host'],
                                port=db_config['port'],
                                database=db_config['database'],
                                user=db_config['user'],
                                password=db_config['password'])
        cur = conn.cursor()

        with open(os.path.join(path, filename)) as f_stats:
            content = f_stats.readlines()

        stats_dict = dict(
            map(str.strip, line.split(':', 1)) for line in content)

        stats_dict['idproject'] = project_id

        try:
            columns = stats_dict.keys()
            values = [stats_dict[column] for column in columns]
            exclude_columns = ['EXCLUDED.' + column for column in columns]

            query = ('insert into afl_stats '
                     '(%s) values %s '
                     'on conflict (idproject, afl_banner) DO UPDATE SET '
                     '(%s) = (%s) '
                     'returning id;')

            cur.execute(
                query,
                (AsIs(', '.join(columns)), tuple(values),
                 AsIs(', '.join(columns)), AsIs(', '.join(exclude_columns))))

            conn.commit()
            id_file = cur.fetchone()
        except psycopg2.IntegrityError as excp:
            print getattr(excp, 'message', repr(excp))
            conn.rollback()
        except Exception as excp:
            print getattr(excp, 'message', repr(excp))
            conn.rollback()

        conn.close()

    def worker(queue):
        setproctitle('AFL dynamic analyze dispatcher')
        processes = []
        MAX_PROCESSES = 20

        while True:
            try:
                # max number of child processes
                while active_children():
                    processes[:] = [p for p in processes if p.is_alive()]
                    if len(processes) < MAX_PROCESSES:
                        break
                    time.sleep(0.1)

                path, filename = queue.get()

                p_input = Process(target=process_input, args=[path, filename])
                p_input.start()
                processes.append(p_input)
            except KeyboardInterrupt:
                raise
            except Exception as excp:
                print getattr(excp, 'message', repr(excp))
                continue

    watch_dir = argv['out_afl']
    max_testcase_size = argv.get('max_testcase_size', 1024 * 1024 * 50)
    queue = Queue()
    processed = set()
    tracker = set()

    worker = Process(target=worker, args=[queue])
    worker.start()

    i = adapters.InotifyTree(watch_dir, mask=IN_CLOSE_WRITE)
    for event in i.event_gen():
        if event is None:
            continue

        (_, _, path, filename) = event

        # filter #1, most often: the evenet is not inside queue directory
        dir_base = basename(path)
        if not dir_base == 'queue':
            # possibly its the fuzzer statistics
            if filename == 'fuzzer_stats':
                p_stats = Process(target=process_stats, args=[path, filename])
                p_stats.start()
            continue

        # filter #2, there is a subdirectory inside queue
        if not filename.startswith('id:'):
            continue

        # filter #3, do not analyze seedbox
        node = basename(dirname(path))
        if node == 'seedbox':
            continue

        # filter #4, for some reason, *MOST* of the test case AFL created,
        # IN_CLOSE_WRITE event will fire twice, workaround by only handling
        # the event every second time.
        if not (path, filename) in tracker:
            tracker.add((path, filename))
            continue
        tracker.remove((path, filename))

        current = []
        # XXX since the tracker set keeps growing, clear the set when reaches
        # 100 records by try to put them into queue for processing
        if len(tracker) > 100:
            while tracker:
                current.append(tracker.pop())

        # always put current event file if reach here
        current.append((path, filename))

        for c_path, c_filename in current:
            # filter #5, different nodes can generate test case with same hash
            md5 = md5sum(os.path.join(c_path, c_filename))
            if md5 in processed:
                continue
            processed.add(md5)

            f_size = os.stat(os.path.join(c_path, c_filename)).st_size
            if f_size > max_testcase_size:
                print 'TEST CASE FILE SIZE TOO LARGE FOR FILE: '
                print '{}:{} ({}) NOT SYNC INTO DATABASE'.format(
                    node, c_filename, f_size)
                continue

            queue.put((c_path, c_filename))

            print '[W][AFL][{}][{: >8}]: [{}] {}'.format(
                len(processed), basename(dirname(c_path)), c_filename, md5)
        active_children()
Пример #31
0
    def process_basic_block(self, contents):
        """ update basic_block table according to dynamic analyze results """
        ###### tb -> tranlation block which is colected from qemu's dynamic trace
        ###### bb -> basic block information recorded in database

        hashes = [x[3] for x in contents]

        for filename, tbs in self._mmap_tb.iteritems():
            # skip if the hash does not exist in database, else get the database index
            try:
                content_id = contents[hashes.index(md5sum(filename))][0]
            except ValueError:
                continue

            query = ('select b.id, b.start_addr, b.end_addr '
                     'from basic_block b '
                     'where b.idcontent = %s '
                     'and b.status = 1 '
                     'order by b.start_addr asc;')
            rows, _ = self._db.execute(query, (content_id, ), commit=False)

            # since the full match should be the majority of the case
            bb_dict = OrderedDict()
            for row in rows:
                bb_dict[(int(row[1]), int(row[2]))] = row[0]

            rows = None

            # loop through tbs and rows
            for tb in tbs.iterkeys():
                (status_code, bb) = self.get_tb_status(tb, bb_dict)

                if status_code == 1:
                    # insert new record into basic_block table
                    query = (
                        'insert into basic_block '
                        '(idcontent, start_addr, end_addr) '
                        'values (%s, %s, %s) on conflict do nothing returning id;'
                    )
                    self._db.execute(query, (content_id, tb[0], tb[1]))
                if status_code == 2:
                    # XXX: do nothing for now
                    pass
                if status_code == 3:
                    # split the original into 3 parts, update edge accordingly
                    # new_block_1 = (bb[0], tb[0])
                    # new_block_2 = (tb[0], tb[1])
                    # new_block_3 = (tb[1], bb[1])
                    bb_idx = bb_dict.get(bb)
                    query = (
                        'insert into basic_block '
                        '(idcontent, start_addr, end_addr) '
                        'values (%s, %s, %s) on conflict do nothing returning id;'
                    )
                    idx_1, _ = self._db.execute(query,
                                                (content_id, bb[0], tb[0]),
                                                fetchone=True)
                    idx_2, _ = self._db.execute(query,
                                                (content_id, tb[0], tb[1]),
                                                fetchone=True)
                    idx_3, _ = self._db.execute(query,
                                                (content_id, tb[1], bb[1]),
                                                fetchone=True)

                    # update original edges, to_bb change to idx_1, from_bb change to idx_3
                    # also insert new edges: new_block_1->new_block_2 and new_block_2 -> new_block_3
                    query = ('insert into edge '
                             '(idproject, from_bb, to_bb) '
                             'values (%s, %s, %s) '
                             'on conflict do nothing returning id;')

                    self._db.execute(query, (self._project_id, idx_1, idx_2))
                    self._db.execute(query, (self._project_id, idx_2, idx_3))

                    query = (
                        'update edge set from_bb = %s where from_bb = %s returning id;'
                    )
                    self._db.execute(query, (idx_3[0], bb_idx))

                    query = (
                        'update edge set to_bb = %s where to_bb = %s returning id;'
                    )
                    self._db.execute(query, (idx_1[0], bb_idx))

                    # update the original basic block's status to inactive
                    query = (
                        'update basic_block set status = 2 where id = %s returning id;'
                    )
                    self._db.execute(query, (bb_idx, ))
                if status_code == 4:
                    # split the original into 2 parts, update edge accordingly
                    # new_block_1 = (bb[0], tb[0])
                    # new_block_2 = (tb[0], tb[1])
                    bb_idx = bb_dict.get(bb)
                    query = (
                        'insert into basic_block '
                        '(idcontent, start_addr, end_addr) '
                        'values (%s, %s, %s) on conflict do nothing returning id;'
                    )
                    idx_1, _ = self._db.execute(query,
                                                (content_id, bb[0], tb[0]),
                                                fetchone=True)
                    idx_2, _ = self._db.execute(query,
                                                (content_id, tb[0], tb[1]),
                                                fetchone=True)

                    # update original edges, to_bb change to idx_1, from_bb change to idx_2
                    query = (
                        'update edge set from_bb = %s where from_bb = %s returning id;'
                    )
                    self._db.execute(query, (idx_2[0], bb_idx))

                    query = (
                        'update edge set to_bb = %s where to_bb = %s returning id;'
                    )
                    self._db.execute(query, (idx_1[0], bb_idx))

                    # update the original basic block's status to inactive
                    query = (
                        'update basic_block set status = 2 where id = %s returning id;'
                    )
                    self._db.execute(query, (bb_idx, ))