def push(self, data, allow_none=False):
if data is None and allow_none is False:
raise ValueError(
"Code push has no data (use allow_none=True to push empty data)"
)
# Checks the hex-digest of the current iteration to see if anything changes
# and will not update if nothing has occurred to the output
if md5sum(getattr(self.current, 'code', None)) == md5sum(data):
return self.current
# Build the new item
next = Iteration(data, from_id=self.active_iteration)
self.iterations.append(next)
to_id = len(self.iterations) - 1
# Update to "to_id" for our old iteration to point to the new one
# but only do this if we had an old iteration
if bound_check(self.iterations, self.active_iteration):
self.iterations[self.active_iteration].branch(to_id)
# Update our active iteration tracker
self.active_iteration = to_id
# Returns the current item (added from this push)
return self.current
def checkMatch(self, imgpath, txtpath, confpath):
# return True
in1md5 = utils.md5sum(imgpath)
in2md5 = utils.md5sum(txtpath)
ta1md5 = utils.getJsonAttr('ImageMD5', None, confpath)
ta2md5 = utils.getJsonAttr('TextMD5', None, confpath)
return in1md5 == ta1md5 and in2md5 == ta2md5
def __init__(self, path, plist=None):
#init path for bundle
self.bundle = None
#if its a directory (e.g. an app)
# ->get binary (from info.plist)
if os.path.isdir(path):
#save bundle path
self.bundle = path
#get path
self.path = utils.getBinaryFromBundle(path)
else:
#save path
self.path = path
#convert file path to utf-8 if needed
if isinstance(self.path, unicode):
#convert
self.path = self.path.encode('utf-8')
#save plist
# ->this will be set for launch daemons/agents
self.plist = plist
#compute/save name
self.name = os.path.split(self.path)[1]
#compute/save hash
self.hash = utils.md5sum(self.path)
#compute/save size
self.size = os.path.getsize(self.path)
#init whitelist info
self.whitelistInfo = None
#init signing authorities
self.signingAuthorities = None
#check if its whitelisted
# ->hash is key for whitelist info
if self.hash in whitelist.whitelistedFiles:
#grab whitelist info
self.whitelistInfo = whitelist.whitelistedFiles[self.hash]
#init
self.signatureStatus = utils.errSecCSUnsigned
#check if signed and if so, by apple
# note: sets class's signatureStatus and signingAuthorities iVars
self.initSigningStatus()
return
def validate_segment_md5sums(self):
if not self.download.check_segment_md5sums:
return True
corrupt_segments = 0
intervals = sorted(self.completed.items())
pbar = ProgressBar(widgets=[
'Checksumming {}: '.format(self.download.ID), Percentage(), ' ',
Bar(marker='#', left='[', right=']'), ' ', ETA()])
with mmap_open(self.download.path) as data:
for interval in pbar(intervals):
log.debug('Checking segment md5: {}'.format(interval))
if not interval.data or 'md5sum' not in interval.data:
log.error(STRIP(
"""User opted to check segment md5sums on restart.
Previous download did not record segment
md5sums (--no-segment-md5sums)."""))
return
chunk = data[interval.begin:interval.end]
checksum = md5sum(chunk)
if checksum != interval.data.get('md5sum'):
log.debug('Redownloading corrupt segment {}, {}.'.format(
interval, checksum))
corrupt_segments += 1
self.completed.remove(interval)
if corrupt_segments:
log.warn('Redownloading {} currupt segments.'.format(
corrupt_segments))
def main(args):
zipfilepath = args.zip
if zipfilepath is None:
print "pass arguements correctly!"
exit(-1)
xmlfilepath = args.xmlfile
zip_path = zipfilepath
if utils.valid_file(zip_path) is not True:
print "bad zip"
exit(-1)
data_for_all_files = []
path_to_extract = utils.random_temp_path(TEMP_DIR)
utils.extractor(zip_path, path_to_extract)
list_of_all_files = utils.getListOfFiles(path_to_extract)
for path_to_file in list_of_all_files:
uid = utils.get_uuid()
filename = utils.stripfilepath(path_to_file)
rel_path = utils.get_relative_path(path_to_file, path_to_extract)
md5hash = utils.md5sum(path_to_file)
filesize = utils.get_file_size(filepath=path_to_file)
data = FileDetails(file_uuid=uid, file_name=filename, file_full_path=path_to_file, relative_path=rel_path, file_md5hash=md5hash, file_size=filesize)
data_for_all_files.append(data)
XS.XMLSerialize(data_for_all_files, xmlfilepath)
utils.cleanup(path_to_extract)
exit(0)
def s2e_concolic_testcase_watcher(seedbox, testcases):
setproctitle.setproctitle('S2E test case watcher')
processed = set()
count = 0
i = adapters.InotifyTree(testcases, mask=IN_CLOSE_WRITE)
for event in i.event_gen():
# auto join child process
if event is None:
continue
(_, _, path, filename) = event
full_name = os.path.join(path,filename)
md5 = md5sum(full_name)
if md5 in processed:
continue
processed.add(md5)
count += 1
dst_file= 'id:{:06d},{}'.format(count, filename)
dst = '{}/{}'.format(seedbox, dst_file)
shutil.copyfile(full_name, dst)
print '[{}] -> [{}]'.format(filename, dst_file)
def validate():
""" Uploads a file to a draft area then tests if it is valid.
Expects:
index_view_slug: unique name of tif file to lookup(string)
file: in request.files for upload
"""
index_view_slug = request.form.get('index_view_slug')
if not index_view_slug or 'file' not in request.files:
abort(400)
if index_view_slug.endswith('_raster'):
index_view_slug = index_view_slug[:-7]
draft_file = os.path.join(conf.DRAFT_DIR,
'{}.tif'.format(index_view_slug))
request.files['file'].save(draft_file)
result = {'errors': []}
try:
valid = is_valid(
index_view_slug, base_path=conf.DRAFT_DIR
)
result = {
'md5': md5sum(draft_file),
'valid': valid
}
os.remove(draft_file)
except NotImplementedError as e:
result['errors'].append(e.message)
return jsonify(result)
def process_coverage(self, contents, input_id, bb_dict):
""" insert into coverage table """
nodes = list()
hashes = [x[3] for x in contents]
for filename, tbs in self._mmap_tb.iteritems():
# skip if the hash does not exist in database, else get the database index
if not md5sum(filename) in hashes:
continue
for tb, value in tbs.iteritems(): # iterate through tbs
bb_idx = bb_dict.get(tb)
if bb_idx:
nodes.append([bb_idx, value])
else:
# if no full match found , try to match both bb's end address and start address
# this is because bb condition 2 is not handled (dynamic tb spans multiple bbs)
# this should be rare, so looping through the dict for now
for bb in bb_dict.iterkeys():
if tb[0] >= bb[0] and tb[0] < bb[1]:
nodes.append([bb_dict[bb], value])
if tb[1] > bb[0] and tb[1] <= bb[1]:
nodes.append([bb_dict[bb], value])
# print 'number of coverage inserted: {}'.format(len(nodes))
gen = ('{}\t{}\n'.format(x[0], x[1]) for x in nodes)
bio = BytesIO(''.join(gen))
self._db.insert_coverage(bio, input_id, self._project_id)
def publish():
""" Uploads a file to a draft area then moves it to data directory if valid.
Expects:
index_view_slug: unique name of tif file to lookup(string)
file: in request.files for upload
"""
index_view_slug = request.form.get('index_view_slug')
if not index_view_slug or 'file' not in request.files:
abort(400)
if index_view_slug.endswith('_raster'):
index_view_slug = index_view_slug[:-7]
draft_file = os.path.join(conf.DRAFT_DIR,
'{}.tif'.format(index_view_slug))
request.files['file'].save(draft_file)
result = {'errors': []}
try:
result['valid'] = is_valid(index_view_slug, base_path=conf.DRAFT_DIR)
if result['valid']:
live_file = make_live(draft_file)
if os.path.exists(live_file):
result = {
'md5': md5sum(live_file),
}
#except Exception as e:
except NotImplementedError as e:
result['errors'].append(e.message)
return jsonify(result)
def get_md5sum(request, share, subpath):
from utils import md5sum
file_path = os.path.join(share.get_path(), subpath)
try:
return json_response({'md5sum': md5sum(file_path), 'path': subpath})
except Exception, e:
return json_error(str(e))
def run(self, data):
# Initialize our output with our input data
output = data
# Check if we match (we need to know this later)
# If we have no matcher, we just skip this matching step
matched = (not self.matcher) or self.match(data)
if matched:
output = self.execute(data)
if utils.md5sum(output) != utils.md5sum(data) and self.recursive_on_change:
_, output = self.run(output)
# The recursion will handle our output and our matched variable
# will contain the match if we ever matched (not when the final case fails)
return matched, output
def process_edge(self, contents, edge_set, bb_dict, input_id):
""" update edge table according to dynamic analyze results """
nodes = list()
hashes = [x[3] for x in contents]
for filename, edges in self._mmap_edges.iteritems():
# skip if the hash does not exist in database, else get the database index
try:
content_id = contents[hashes.index(md5sum(filename))][0]
except ValueError:
continue
for edge in edges:
# map basic blocks with their database index
from_bb = edge[0]
to_bb = edge[1]
# first node and last node, not insert into edge table
if from_bb is None or to_bb is None:
continue
idx_from_bb = bb_dict.get(from_bb)
idx_to_bb = bb_dict.get(to_bb)
# # if no full match found for idx_from_bb, try to match bb's end address
# # this is because bb condition 2 is not handled (dynamic tb spans multiple bbs)
# # this should be rare, so looping through the dict for now
# if idx_from_bb is None:
# for bb in bb_dict.iterkeys():
# if from_bb[1] <= bb[1] and from_bb[1] > bb[0]:
# idx_from_bb = bb_dict[bb]
# break
# # same with to_bb, try to match bb's start address
# if idx_to_bb is None:
# for bb in bb_dict.iterkeys():
# if to_bb[0] >= bb[0] and to_bb[0] < bb[1]:
# idx_to_bb = bb_dict[bb]
# break
# still can't find, it might be because the basic block does not
# belong to any of the memory mapped file in the project content table
# ignore for now...
if idx_from_bb is None or idx_to_bb is None:
continue
edge_with_idx = (idx_from_bb, idx_to_bb)
if edge_with_idx not in edge_set:
# insert new edge
nodes.append(edge_with_idx)
# print 'number of edges inserted: {}'.format(len(nodes))
gen = ('{}\t{}\t{}\n'.format(self._project_id, x[0], x[1])
for x in nodes)
bio = BytesIO(''.join(gen))
self._db.insert_edge(bio, input_id)
def update_md5sum(key):
# This view is a hack. We need proper SQL migrations instead of this
if key == app.config['PERIODIC_KEY']:
for post in JobPost.query.all():
if not post.md5sum:
post.md5sum = md5sum(post.email)
db.session.commit()
return Response("Updated md5sum for all posts.\n",
content_type='text/plain; charset=utf-8')
else:
abort(403)
def fill(self, piece_id, buf, md5=None):
with self._lock:
start = piece_id * self.piece_size
self.fileob.seek(start)
self.fileob.write(buf)
self.fileob.flush()
if md5:
piece_md5 = md5
else:
piece_md5 = md5sum(buf)
self.filled(piece_id, md5=piece_md5)
def run_fuzz(argv):
''' run the test '''
# check the requirements of cim_fuzz
check_req(argv)
# working directory name
working_dir = '{}/cb_{}'.format(
argv['cbhome'], str(datetime.now().strftime('%Y-%m-%d-%H%M%S.%f')))
check_dir(working_dir)
# 1. download from remote or get the file with path
file_name = url_get_file(argv['uri'], working_dir)
# 2. unzip the file
dir_name = unzip(file_name, working_dir)
argv['tar_file'] = file_name
print_sep()
print '[C]: working directory: [{}]'.format(dir_name)
# prepare the experiment directory structure
dir_struct = prepare_dir(dir_name)
# get the architecture of the test binary
binary = '{}/cb'.format(dir_struct['binary'])
bin_arch = get_file_arch(binary)[0]
if bin_arch not in ['32bit', '64bit']:
print '[C]: unsupported file arch!, exiting now'
exit(0)
if bin_arch == '64bit':
argv['arch'] = 'x86_64'
if bin_arch == '32bit':
argv['arch'] = 'i386'
# first 8 bytes of md5 as file id
argv['md5sum'] = md5sum(binary)
argv['fid'] = argv['md5sum'][:8]
check_dir('{}/{}'.format(dir_struct['out_afl'], argv['fid']))
print "out_afl:{}/{}".format(dir_struct['out_afl'], argv['fid'])
# save command to cimfuzz.cmd file
argv.pop('func', None)
with open('{}/cimfuzz.cmd'.format(dir_name), 'w') as fuzz_cmd:
json.dump(argv, fuzz_cmd)
# globals for flask
if argv['afl_only']:
execute_aflonly(dir_struct, argv)
else:
execute(dir_struct, argv)
def save_action(file):
if fnmatch.fnmatch(file, pattern):
if db.is_existed(file):
return
name = os.path.basename(file).lower()
hash = md5sum(file)
db.insert_file(hash, name, file)
cache['current'] += 1
if cache['current'] % 2000 == 0:
logging.info('Processing: {} / {}'.format(
cache['current'], total_count))
db.commit_changes()
def signin(db):
"""
登陆
"""
passwd = request.POST.get('passwd','')
db.execute('SELECT PASSWD FROM USER')
v, = db.getone()
if v != passwd:
return template("error",msg="密码错误")
#-设定cookie
token = md5sum('%s-%s' % (now(),random.random()*now()))
db.execute('INSERT INTO COOKIE(TOKEN) VALUES(%s)',token)
d = urlparse.urlparse(CFG.domain).netloc.split(':')[0]
response.set_cookie('token',token,path="/",expires=int(now()/1000)+3600*24*365,domain=d)
redirect("/",301)
def signin(db):
"""
登陆
"""
passwd = request.POST.get('passwd', '')
db.execute('SELECT PASSWD FROM USER')
v, = db.getone()
if v != passwd:
return template("error", msg="密码错误")
#-设定cookie
token = md5sum('%s-%s' % (now(), random.random() * now()))
db.execute('INSERT INTO COOKIE(TOKEN) VALUES(%s)', token)
d = urlparse.urlparse(CFG.domain).netloc.split(':')[0]
response.set_cookie('token',
token,
path="/",
expires=int(now() / 1000) + 3600 * 24 * 365,
domain=d)
redirect("/", 301)
def process_file(self, filename):
""" insert binary entry into table content """
ins_file = ('insert into file (hash, file) '
'values (%s, %s) on conflict do nothing returning id;')
# calculate md5
md5 = md5sum(filename)
with open(filename, 'rb') as f_src:
src_content = f_src.read()
content = (md5, psycopg2.Binary(src_content))
file_id, rowcount = self._db.execute(ins_file, content, fetchone=True)
if rowcount == 0:
query = 'select id from file where hash = %s;'
file_id, _ = self._db.execute(query, (md5, ),
fetchone=True,
commit=False)
return file_id[0]
def get_md5_for_index_view_slug():
""" Gets the md5 value for a tif matching the slug parameter or -1 if
not found.
Expects:
index_view_slug: unique name of tif file to lookup(string)
"""
index_view_slug = request.args.get('index_view_slug')
if not index_view_slug:
abort(400)
value = -1
try:
raster_file = get_raster_file_path(index_view_slug)
value = md5sum(raster_file)
except IOError:
pass
result = {
'md5': value
}
return jsonify(result)
def packing(node, parameters, compiler_replace_maps):
package = node.get_package_name()
version_git = node.get_version()
packing = node.is_packing()
if not packing:
logging.warning('Skiping package: %s' % package)
return 0
manager = node.get_version_manager()
if manager == "git":
build_modes = node.get_build_modes()
for plat, build_mode in product(platforms, build_modes):
workspace = node.get_workspace(plat)
build_directory = os.path.join(
os.getcwd(), node.get_build_directory(plat, build_mode))
revision_git = hash_version.get_last_changeset(build_directory,
short=False)
version_old = node.get_version()
version_git = hash_version.to_cmaki_version(
build_directory, revision_git)
logging.info('[git] Renamed version from %s to %s' %
(version_old, version_git))
# renombrar package-version-platform/package-version
workspace = node.get_workspace(plat)
source_folder = node.get_base_folder()
oldversion = node.get_version()
try:
node.set_version(version_git)
new_workspace = node.get_workspace(plat)
new_source_folder = node.get_base_folder()
# changed version ?
if source_folder != new_source_folder:
utils.move_folder_recursive(
os.path.join(workspace, source_folder),
os.path.join(workspace, new_source_folder))
utils.move_folder_recursive(workspace, new_workspace)
finally:
node.set_version(oldversion)
node.set_version(version_git)
version = node.get_version()
# regenerate autoscripts with new version
node.generate_scripts_headers(compiler_replace_maps)
precmd = ''
if utils.is_windows():
precmd = 'cmake -E '
folder_3rdparty = parameters.third_party_dir
output_3rdparty = os.path.join(folder_3rdparty, node.get_base_folder())
utils.trymkdir(output_3rdparty)
folder_mark = os.path.join(parameters.prefix, node.get_base_folder())
utils.trymkdir(folder_mark)
utils.superverbose(
parameters,
'*** [%s] Generation cmakefiles *** %s' % (package, output_3rdparty))
errors = node.generate_cmakefiles(platforms, output_3rdparty,
compiler_replace_maps)
logging.debug('errors generating cmakefiles: %d' % errors)
node.ret += abs(errors)
for plat in platforms:
utils.superverbose(
parameters, '*** [%s (%s)] Generating package .tar.gz (%s) ***' %
(package, version, plat))
workspace = node.get_workspace(plat)
utils.trymkdir(workspace)
with utils.working_directory(workspace):
if utils.is_windows():
utils.safe_system('del /s *.ilk')
utils.safe_system('del /s *.exp')
source_folder = node.get_base_folder()
prefix_package = os.path.join(parameters.prefix,
'%s.tar.gz' % workspace)
prefix_package_cmake = os.path.join(parameters.prefix,
'%s-cmake.tar.gz' % workspace)
prefix_package_md5 = os.path.join(output_3rdparty,
'%s.md5' % workspace)
logging.info(
'generating package %s from source %s' %
(prefix_package, os.path.join(os.getcwd(), source_folder)))
logging.info('generating md5file %s' % prefix_package_md5)
# packing install
gen_targz = "%star zcvf %s %s" % (precmd, prefix_package,
source_folder)
node.ret += abs(node.safe_system(gen_targz, compiler_replace_maps))
if not os.path.exists(prefix_package):
logging.error('No such file: {}'.format(prefix_package))
return False
# calculate md5 file
package_md5 = utils.md5sum(prefix_package)
logging.debug("new package {}, with md5sum {}".format(
prefix_package, package_md5))
with open(prefix_package_md5, 'wt') as f:
f.write('%s\n' % package_md5)
# marker is a empty file
# prefix_package_marker = os.path.join(folder_mark, '%s.cache' % get_identifier('ALL'))
# logging.info('generating marker %s' % prefix_package_marker)
# open(prefix_package_marker, 'a').close()
# packing cmakefiles (more easy distribution)
if not parameters.no_packing_cmakefiles:
for plat in platforms:
base_folder = node.get_base_folder()
prefix_package_cmake = os.path.join(
parameters.prefix, '%s-%s-cmake.tar.gz' % (base_folder, plat))
with utils.working_directory(folder_3rdparty):
logging.debug('working dir: %s' % folder_3rdparty)
# packing install
logging.info('generating package cmake %s' %
prefix_package_cmake)
gen_targz_cmake = '{}tar zcvf {} {}'.format(
precmd, prefix_package_cmake, node.get_base_folder())
node.ret += abs(
node.safe_system(gen_targz_cmake, compiler_replace_maps))
# finish well
return True
def execute_aflonly(dir_struct, argv):
try:
container = run_afl(dir_struct, argv)
prev_cvg = dict()
seedbox_dir = os.path.join(dir_struct['out_afl'], argv['fid'], 'seedbox', 'queue')
s2e_testcase_dir = '{}/testcases'.format(dir_struct['out_s2e'])
print 'watching [{}]'.format(s2e_testcase_dir)
check_dir(seedbox_dir)
check_dir(s2e_testcase_dir)
s2e_testcase_handler = Process(target=s2e_concolic_testcase_watcher, args=[seedbox_dir, s2e_testcase_dir])
s2e_testcase_handler.start()
processes = list()
q = Queue()
while True:
print 'sleep'
time.sleep(60)
# updated = False
# watch for afl fuzzer_stats change
cur_cvg = get_cvg(dir_struct, argv)
# __import__('pprint').pprint(cur_cvg)
# if not prev_cvg:
# prev_cvg = cur_cvg
# continue
# for node, cvg in cur_cvg.iteritems():
# if prev_cvg[node] != cvg:
# updated = True
# break
# if updated:
# continue
if len(processes) >= argv['num_s2e']:
print argv['num_s2e']
for p in processes:
print p
active_children()
processes[:] = [p for p in processes if p[0].is_alive()]
continue
# launch s2e with driller searcher, with .cur_input from first node:
s2e_node = random.choice(cur_cvg.keys())
seed = os.path.join(dir_struct['out_afl'], argv['fid'], s2e_node, '.cur_input')
tmp_input_file = os.path.join(dir_struct['in_s2e'], 'seed')
shutil.copyfile(seed, tmp_input_file)
input_hash = md5sum(tmp_input_file)
input_file = os.path.join(dir_struct['in_s2e'], input_hash)
shutil.copyfile(tmp_input_file, input_file)
process = Process(target=launch_driller, args=[input_file, input_hash, dir_struct, argv, q])
# process.daemon=True
process.start()
qemu = q.get(timeout=5)
processes.append((process,qemu))
except KeyboardInterrupt:
for p in processes:
kill_process(p[0])
kill_process(p[1])
docker.from_env().containers.get(container).kill()
kill_process(s2e_testcase_handler)
def __init__(self, path, plist=None, parent=None):
#init path for bundle
self.bundle = None
#if its a directory (e.g. an app bundle)
# ->get binary (from app's Info.plist)
if os.path.isdir(path):
#save bundle path
self.bundle = path
#get path
self.path = utils.getBinaryFromBundle(path)
#if binary could not be found
# ->default to 'unknown'
if not self.path:
#just set to something...
self.path = '<unknown>'
#path is to file
# ->just save into class var
else:
#save
self.path = path
#convert file path to utf-8 if needed
if isinstance(self.path, unicode):
#convert
self.path = self.path.encode('utf-8')
#save plist
# ->this will be set for launch daemons/agents, inserted dylibs, etc
self.plist = plist
#compute/save name
self.name = os.path.split(self.path)[1]
#compute/save hash
self.hash = utils.md5sum(self.path)
#init whitelist flag
self.isWhitelisted = False
#check if its whitelisted
# ->path is key
if self.path in whitelist.whitelistedFiles:
#check if hash is in white list
self.isWhitelisted = (self.hash
in whitelist.whitelistedFiles[self.path])
#init
self.signatureStatus = None
#init signing authorities
self.signingAuthorities = None
#check if signed and if so, by apple
# note: sets class's signatureStatus and signingAuthorities iVars
self.initSigningStatus()
return
def editjob(hashid, key, form=None, post=None, validated=False):
if form is None:
form = forms.ListingForm(request.form)
form.job_type.choices = [(ob.id, ob.title) for ob in JobType.query.filter_by(public=True).order_by('seq')]
form.job_category.choices = [(ob.id, ob.title) for ob in JobCategory.query.filter_by(public=True).order_by('seq')]
if post is None:
post = JobPost.query.filter_by(hashid=hashid).first_or_404()
if key != post.edit_key:
abort(403)
# Don't allow email address to be changed once its confirmed
if request.method == 'POST' and post.status >= POSTSTATUS.PENDING:
form.poster_email.data = post.email
if request.method == 'POST' and (validated or form.validate()):
form_description = bleach.clean(form.job_description.data, tags=ALLOWED_TAGS)
form_description = bleach.linkify(form_description)
form_perks = bleach.linkify(bleach.clean(form.job_perks_description.data, tags=ALLOWED_TAGS)) if form.job_perks.data else ''
form_how_to_apply = form.job_how_to_apply.data
form_email_domain = get_email_domain(form.poster_email.data)
form_words = get_word_bag(u' '.join((form_description, form_perks, form_how_to_apply)))
similar = False
for oldpost in JobPost.query.filter(JobPost.email_domain == form_email_domain).filter(
JobPost.status > POSTSTATUS.PENDING).filter(
JobPost.datetime > datetime.utcnow() - agelimit).all():
if oldpost.id != post.id:
if oldpost.words:
s = SequenceMatcher(None, form_words, oldpost.words)
if s.ratio() > 0.6:
similar = True
break
if similar:
flash("This listing is very similar to an earlier listing. You may not relist the same job "
"in less than %d days. If you believe this to be an error, please email us at %s." % (agelimit.days,
app.config['SUPPORT_EMAIL']), category='interactive')
else:
post.headline = form.job_headline.data
post.type_id = form.job_type.data
post.category_id = form.job_category.data
post.location = form.job_location.data
post.relocation_assist = form.job_relocation_assist.data
post.description = form_description
post.perks = form_perks
post.how_to_apply = form_how_to_apply
post.company_name = form.company_name.data
post.company_url = form.company_url.data
post.email = form.poster_email.data
post.email_domain = form_email_domain
post.md5sum = md5sum(post.email)
post.hr_contact = form.hr_contact.data
# To protect from gaming, don't allow words to be removed in edited listings once the post
# has been confirmed. Just add the new words.
if post.status >= POSTSTATUS.CONFIRMED:
prev_words = post.words or ''
else:
prev_words = u''
post.words = get_word_bag(u' '.join((prev_words, form_description, form_perks, form_how_to_apply)))
if request.files['company_logo']:
# The form's validator saved the processed logo in g.company_logo.
thumbnail = g.company_logo
logofilename = uploaded_logos.save(thumbnail, name='%s.' % post.hashid)
post.company_logo = logofilename
else:
if form.company_logo_remove.data:
post.company_logo = None
db.session.commit()
userkeys = session.get('userkeys', [])
userkeys.append(post.edit_key)
session['userkeys'] = userkeys
session.permanent = True
return redirect(url_for('jobdetail', hashid=post.hashid), code=303)
elif request.method == 'POST':
flash("Please correct the indicated errors", category='interactive')
elif request.method == 'GET':
# Populate form from model
form.job_headline.data = post.headline
form.job_type.data = post.type_id
form.job_category.data = post.category_id
form.job_location.data = post.location
form.job_relocation_assist.data = post.relocation_assist
form.job_description.data = post.description
form.job_perks.data = True if post.perks else False
form.job_perks_description.data = post.perks
form.job_how_to_apply.data = post.how_to_apply
form.company_name.data = post.company_name
form.company_url.data = post.company_url
form.poster_email.data = post.email
form.hr_contact.data = int(post.hr_contact or False)
return render_template('postjob.html', form=form, no_email=post.status > POSTSTATUS.DRAFT)
def __init__(self, path, plist=None, parent=None):
#init path for bundle
self.bundle = None
#if its a directory (e.g. an app bundle)
# ->get binary (from app's Info.plist)
if os.path.isdir(path):
#save bundle path
self.bundle = path
#get path
self.path = utils.getBinaryFromBundle(path)
#if binary could not be found
# ->default to 'unknown'
if not self.path:
#just set to something...
self.path = '<unknown>'
#path is to file
# ->just save into class var
else:
#save
self.path = path
#convert file path to utf-8 if needed
if isinstance(self.path, unicode):
#convert
self.path = self.path.encode('utf-8')
#save plist
# ->this will be set for launch daemons/agents, inserted dylibs, etc
self.plist = plist
#compute/save name
self.name = os.path.split(self.path)[1]
#compute/save hash
self.hash = utils.md5sum(self.path)
#compute/save size
self.size = os.path.getsize(self.path)
#init whitelist flag
self.isWhitelisted = False
#init signing authorities
self.signingAuthorities = None
#check if its whitelisted
# ->path is key
if self.path in whitelist.whitelistedFiles:
#check if hash is in white list
self.isWhitelisted = (self.hash in whitelist.whitelistedFiles[self.path])
#init
self.signatureStatus = utils.errSecCSUnsigned
#check if signed and if so, by apple
# note: sets class's signatureStatus and signingAuthorities iVars
self.initSigningStatus()
return
return result
if __name__ == "__main__":
import os
import random
import sys
import utils
reload(sys)
sys.setdefaultencoding('utf-8')
from simplejson import *
l = os.listdir(".")
#l = [ "abcd" ]
random.shuffle(l)
a = bst(l)
random.shuffle(l)
je = JSONEncoder(ensure_ascii=False)
jd = JSONDecoder()
json = dumps(l, ensure_ascii = False)
l1 = loads(json)
for i in l1:
fn = a.extract(i)
md5 = utils.md5sum(fn)
print "%s %s" % (md5, fn)
#print a.as_list()
def process_input(path, filename, count):
setproctitle('S2E test case watcher: processing [{}]'.format(filename))
# 1. update interested table with testcase info
src = os.path.join(path, filename)
try:
with open(src, 'rb') as f_src:
src_content = f_src.read()
md5 = md5sum(src)
except Exception as excp:
print getattr(excp, 'message', repr(excp))
return
conn = psycopg2.connect(host=db_config['host'],
port=db_config['port'],
database=db_config['database'],
user=db_config['user'],
password=db_config['password'])
cur = conn.cursor()
id_file = 0
try:
query = ('insert into file (hash, file) '
'values (%s, %s) on conflict do nothing returning id;')
cur.execute(query, (md5, psycopg2.Binary(src_content)))
conn.commit()
id_file = cur.fetchone()
except KeyboardInterrupt:
raise
except psycopg2.IntegrityError as excp:
print getattr(excp, 'message', repr(excp))
conn.rollback()
except Exception as excp:
print getattr(excp, 'message', repr(excp))
conn.rollback()
# only update interested table if the testcase is generated by concolic mode
if not filename.startswith('testcase'):
# l_fn = ['s2e', 'input', idx_input, idx_interested, idx_basic_block, md5_short]
l_fn = filename.split('_')
if cur.description is None or id_file is None:
query = 'select id from file where hash = %s;'
cur.execute(query, (md5, ))
id_file = cur.fetchone()
if id_file is not None:
query = ('update interested set '
'idfile = %s, '
'uri = %s, '
'update_time=now(), '
'status = 3 '
'where id = %s returning id;')
cur.execute(query, (id_file[0], src, l_fn[3]))
rowcount = cur.rowcount
if rowcount != 1:
conn.rollback()
else:
conn.commit()
conn.close()
# copy file from s2e test case output directory to AFL seedbox
if not filename.startswith('testcase'):
dst_file = 'id:{:06d},{},{},{},{}'.format(count, l_fn[2], l_fn[3],
l_fn[4], l_fn[5])
dst = '{}/{}'.format(seedbox, dst_file)
else:
dst_file = 'id:{:06d},{}'.format(count, filename)
dst = '{}/{}'.format(seedbox, dst_file)
# lastly copy file to seedbox
print '[W][S2E]: [{}] -> [{}]'.format(basename(src), basename(dst))
shutil.copyfile(src, dst)
def packing(node, parameters, compiler_replace_maps):
package = node.get_package_name()
version_git = node.get_version()
packing = node.is_packing()
if not packing:
logging.warning('Skiping package: %s' % package)
return 0
manager = node.get_version_manager()
if manager == "git":
build_modes = node.get_build_modes()
for plat, build_mode in product(platforms, build_modes):
build_directory = os.path.join(os.getcwd(), node.get_build_directory(plat, build_mode))
revision_git = hash_version.get_last_changeset(build_directory, short=False)
version_old = node.get_version()
version_git = hash_version.to_cmaki_version(build_directory, revision_git)
logging.info('[git] Renamed version from %s to %s' % (version_old, version_git))
current_workspace = node.get_binary_workspace(plat)
current_base = node.get_base_folder()
oldversion = node.get_version()
try:
node.set_version(version_git)
updated_workspace = node.get_binary_workspace(plat)
updated_base = node.get_base_folder()
current_base2 = os.path.join(current_workspace, current_base)
updated_base2 = os.path.join(current_workspace, updated_base)
logging.debug("from: %s" % current_base2)
logging.debug("to: %s" % updated_base2)
if current_base != updated_base:
utils.move_folder_recursive(current_base2, updated_base2)
logging.debug('-- copy from: {}, {}'.format(current_workspace, os.path.exists(current_workspace)))
logging.debug('-- copy to: {}, {}'.format(updated_workspace, os.path.exists(updated_workspace)))
utils.move_folder_recursive(current_workspace, updated_workspace)
finally:
node.set_version(oldversion)
node.set_version(version_git)
version = node.get_version()
# regenerate autoscripts with new version
node.generate_scripts_headers(compiler_replace_maps)
# # generate versions.cmake
node.generate_3rdpartyversion(parameters.prefix)
precmd = ''
if utils.is_windows():
precmd = 'cmake -E '
folder_3rdparty = parameters.third_party_dir
output_3rdparty = os.path.join(folder_3rdparty, node.get_base_folder())
utils.trymkdir(output_3rdparty)
folder_mark = os.path.join(parameters.prefix, node.get_base_folder())
utils.trymkdir(folder_mark)
utils.superverbose(parameters, '*** [%s] Generation cmakefiles *** %s' % (package, output_3rdparty))
errors = node.generate_cmakefiles(platforms, output_3rdparty, compiler_replace_maps)
logging.debug('errors generating cmakefiles: %d' % errors)
node.ret += abs(errors)
for plat in platforms:
utils.superverbose(parameters, '*** [%s (%s)] Generating package .tar.gz (%s) ***' % (package, version, plat))
workspace = node.get_workspace(plat)
current_workspace = node.get_binary_workspace(plat)
utils.trymkdir(current_workspace)
with utils.working_directory(current_workspace):
logging.info('working directory: {}'.format(current_workspace))
if utils.is_windows():
utils.safe_system('del /s *.ilk')
utils.safe_system('del /s *.exp')
current_base = node.get_base_folder()
prefix_package = os.path.join(parameters.prefix, '%s.tar.gz' % workspace)
prefix_package_md5 = os.path.join(output_3rdparty, '%s.md5' % workspace)
logging.info('generating package %s from source %s' % (prefix_package, os.path.join(os.getcwd(), current_base)))
logging.info('generating md5file %s' % prefix_package_md5)
print_folder(current_base)
# packing install
gen_targz = "%star zcvf %s %s" % (precmd, prefix_package, current_base)
node.ret += abs( node.safe_system(gen_targz, compiler_replace_maps) )
if not os.path.exists(prefix_package):
logging.error('No such file: {}'.format(prefix_package))
return False
# calculate md5 file
package_md5 = utils.md5sum(prefix_package)
logging.debug("new package {}, with md5sum {}".format(prefix_package, package_md5))
with open(prefix_package_md5, 'wt') as f:
f.write('%s\n' % package_md5)
# packing cmakefiles (more easy distribution)
if not parameters.no_packing_cmakefiles:
for plat in platforms:
current_base = node.get_base_folder()
prefix_package_cmake = os.path.join(parameters.prefix, '%s-%s-cmake.tar.gz' % (current_base, plat))
with utils.working_directory(folder_3rdparty):
logging.info('working directory: {}'.format(folder_3rdparty))
logging.debug('working dir: %s' % folder_3rdparty)
logging.info('generating package cmake %s' % prefix_package_cmake)
print_folder(current_base)
gen_targz_cmake = '{}tar zcvf {} {}'.format(precmd, prefix_package_cmake, current_base)
node.ret += abs( node.safe_system(gen_targz_cmake, compiler_replace_maps) )
if not os.path.exists(prefix_package_cmake):
logging.error('No such file: {}'.format(prefix_package_cmake))
return False
# finish well
return True
def get_hash_from_file(self):
"""Calculate and set md5 hash from path."""
self.hash = utils.md5sum(self.path)
def afl_watcher(argv):
setproctitle('AFL test case watcher')
def process_input(path, filename):
setproctitle('analyzing [{}:{}]'.format(basename(dirname(path)),
filename))
db_config = argv['db_config']
project_id = argv['project_id']
qemu = argv['qemu']
basedir = argv['basedir']
analyzer = DynamicAnalyzer(db_config=db_config,
qemu=qemu,
basedir=basedir,
project_id=project_id)
analyzer.analyze_dynamic(os.path.join(path, filename))
def process_stats(path, filename):
setproctitle('Processing fuzzer_stats for node {}'.format(
basename(path)))
db_config = argv['db_config']
project_id = argv['project_id']
conn = psycopg2.connect(host=db_config['host'],
port=db_config['port'],
database=db_config['database'],
user=db_config['user'],
password=db_config['password'])
cur = conn.cursor()
with open(os.path.join(path, filename)) as f_stats:
content = f_stats.readlines()
stats_dict = dict(
map(str.strip, line.split(':', 1)) for line in content)
stats_dict['idproject'] = project_id
try:
columns = stats_dict.keys()
values = [stats_dict[column] for column in columns]
exclude_columns = ['EXCLUDED.' + column for column in columns]
query = ('insert into afl_stats '
'(%s) values %s '
'on conflict (idproject, afl_banner) DO UPDATE SET '
'(%s) = (%s) '
'returning id;')
cur.execute(
query,
(AsIs(', '.join(columns)), tuple(values),
AsIs(', '.join(columns)), AsIs(', '.join(exclude_columns))))
conn.commit()
id_file = cur.fetchone()
except psycopg2.IntegrityError as excp:
print getattr(excp, 'message', repr(excp))
conn.rollback()
except Exception as excp:
print getattr(excp, 'message', repr(excp))
conn.rollback()
conn.close()
def worker(queue):
setproctitle('AFL dynamic analyze dispatcher')
processes = []
MAX_PROCESSES = 20
while True:
try:
# max number of child processes
while active_children():
processes[:] = [p for p in processes if p.is_alive()]
if len(processes) < MAX_PROCESSES:
break
time.sleep(0.1)
path, filename = queue.get()
p_input = Process(target=process_input, args=[path, filename])
p_input.start()
processes.append(p_input)
except KeyboardInterrupt:
raise
except Exception as excp:
print getattr(excp, 'message', repr(excp))
continue
watch_dir = argv['out_afl']
max_testcase_size = argv.get('max_testcase_size', 1024 * 1024 * 50)
queue = Queue()
processed = set()
tracker = set()
worker = Process(target=worker, args=[queue])
worker.start()
i = adapters.InotifyTree(watch_dir, mask=IN_CLOSE_WRITE)
for event in i.event_gen():
if event is None:
continue
(_, _, path, filename) = event
# filter #1, most often: the evenet is not inside queue directory
dir_base = basename(path)
if not dir_base == 'queue':
# possibly its the fuzzer statistics
if filename == 'fuzzer_stats':
p_stats = Process(target=process_stats, args=[path, filename])
p_stats.start()
continue
# filter #2, there is a subdirectory inside queue
if not filename.startswith('id:'):
continue
# filter #3, do not analyze seedbox
node = basename(dirname(path))
if node == 'seedbox':
continue
# filter #4, for some reason, *MOST* of the test case AFL created,
# IN_CLOSE_WRITE event will fire twice, workaround by only handling
# the event every second time.
if not (path, filename) in tracker:
tracker.add((path, filename))
continue
tracker.remove((path, filename))
current = []
# XXX since the tracker set keeps growing, clear the set when reaches
# 100 records by try to put them into queue for processing
if len(tracker) > 100:
while tracker:
current.append(tracker.pop())
# always put current event file if reach here
current.append((path, filename))
for c_path, c_filename in current:
# filter #5, different nodes can generate test case with same hash
md5 = md5sum(os.path.join(c_path, c_filename))
if md5 in processed:
continue
processed.add(md5)
f_size = os.stat(os.path.join(c_path, c_filename)).st_size
if f_size > max_testcase_size:
print 'TEST CASE FILE SIZE TOO LARGE FOR FILE: '
print '{}:{} ({}) NOT SYNC INTO DATABASE'.format(
node, c_filename, f_size)
continue
queue.put((c_path, c_filename))
print '[W][AFL][{}][{: >8}]: [{}] {}'.format(
len(processed), basename(dirname(c_path)), c_filename, md5)
active_children()
def process_basic_block(self, contents):
""" update basic_block table according to dynamic analyze results """
###### tb -> tranlation block which is colected from qemu's dynamic trace
###### bb -> basic block information recorded in database
hashes = [x[3] for x in contents]
for filename, tbs in self._mmap_tb.iteritems():
# skip if the hash does not exist in database, else get the database index
try:
content_id = contents[hashes.index(md5sum(filename))][0]
except ValueError:
continue
query = ('select b.id, b.start_addr, b.end_addr '
'from basic_block b '
'where b.idcontent = %s '
'and b.status = 1 '
'order by b.start_addr asc;')
rows, _ = self._db.execute(query, (content_id, ), commit=False)
# since the full match should be the majority of the case
bb_dict = OrderedDict()
for row in rows:
bb_dict[(int(row[1]), int(row[2]))] = row[0]
rows = None
# loop through tbs and rows
for tb in tbs.iterkeys():
(status_code, bb) = self.get_tb_status(tb, bb_dict)
if status_code == 1:
# insert new record into basic_block table
query = (
'insert into basic_block '
'(idcontent, start_addr, end_addr) '
'values (%s, %s, %s) on conflict do nothing returning id;'
)
self._db.execute(query, (content_id, tb[0], tb[1]))
if status_code == 2:
# XXX: do nothing for now
pass
if status_code == 3:
# split the original into 3 parts, update edge accordingly
# new_block_1 = (bb[0], tb[0])
# new_block_2 = (tb[0], tb[1])
# new_block_3 = (tb[1], bb[1])
bb_idx = bb_dict.get(bb)
query = (
'insert into basic_block '
'(idcontent, start_addr, end_addr) '
'values (%s, %s, %s) on conflict do nothing returning id;'
)
idx_1, _ = self._db.execute(query,
(content_id, bb[0], tb[0]),
fetchone=True)
idx_2, _ = self._db.execute(query,
(content_id, tb[0], tb[1]),
fetchone=True)
idx_3, _ = self._db.execute(query,
(content_id, tb[1], bb[1]),
fetchone=True)
# update original edges, to_bb change to idx_1, from_bb change to idx_3
# also insert new edges: new_block_1->new_block_2 and new_block_2 -> new_block_3
query = ('insert into edge '
'(idproject, from_bb, to_bb) '
'values (%s, %s, %s) '
'on conflict do nothing returning id;')
self._db.execute(query, (self._project_id, idx_1, idx_2))
self._db.execute(query, (self._project_id, idx_2, idx_3))
query = (
'update edge set from_bb = %s where from_bb = %s returning id;'
)
self._db.execute(query, (idx_3[0], bb_idx))
query = (
'update edge set to_bb = %s where to_bb = %s returning id;'
)
self._db.execute(query, (idx_1[0], bb_idx))
# update the original basic block's status to inactive
query = (
'update basic_block set status = 2 where id = %s returning id;'
)
self._db.execute(query, (bb_idx, ))
if status_code == 4:
# split the original into 2 parts, update edge accordingly
# new_block_1 = (bb[0], tb[0])
# new_block_2 = (tb[0], tb[1])
bb_idx = bb_dict.get(bb)
query = (
'insert into basic_block '
'(idcontent, start_addr, end_addr) '
'values (%s, %s, %s) on conflict do nothing returning id;'
)
idx_1, _ = self._db.execute(query,
(content_id, bb[0], tb[0]),
fetchone=True)
idx_2, _ = self._db.execute(query,
(content_id, tb[0], tb[1]),
fetchone=True)
# update original edges, to_bb change to idx_1, from_bb change to idx_2
query = (
'update edge set from_bb = %s where from_bb = %s returning id;'
)
self._db.execute(query, (idx_2[0], bb_idx))
query = (
'update edge set to_bb = %s where to_bb = %s returning id;'
)
self._db.execute(query, (idx_1[0], bb_idx))
# update the original basic block's status to inactive
query = (
'update basic_block set status = 2 where id = %s returning id;'
)
self._db.execute(query, (bb_idx, ))
