def obfuscate(input_path, output_path, obfuscation_combinations, no_of_variants): ''' Iterates through each c file in a folder or just a single c file and calls the variant function. Arguments: input_path {str} -- Non-obfuscated c file/files path output_path {str} -- Obfuscated c file path obfuscation_combinations {list} -- the list of combinations of obfuscation techniques no_of_variants {int} -- The number of variants for each obfuscation ''' input_files_path = fs.ls(input_path, EXT['c']) for input_path in input_files_path: input_file = fs.details(input_path) output_dir_path = os.path.join(output_path, input_file['name']) fs.mkdir(output_dir_path, False) variant(input_path, output_dir_path, obfuscation_combinations, no_of_variants) fs.rmdirs(output_dir_path, obfuscation_combinations)
def variant(original_input_path, output_path, obfuscation_combinations, no_of_variants=1): ''' Iterates through the number of variatns and obfuscation list to generate the necessary output path and calls the obscure function. Arguments: original_input_path {str} -- Single non-obfuscated file path output_path {str} -- obfuscated c file path Keyword Arguments: obfuscation_combinations {dict} -- List of combinations of each obfuscation technique (default: {{}}) no_of_variants {int} -- The number of variants to be generated (default: {1}) ''' for index in range(1, no_of_variants + 1): for combination in obfuscation_combinations: if re.search(RE_OBFUSCATION, combination): v_n = VN file_name = '' input_path = original_input_path for obfuscation in combination: file_name += obfuscation target_path = os.path.join(output_path, file_name) fs.mkdir(target_path, False) input_path = obscure(input_path, output_path, obfuscation.upper(), file_name, str(index), v_n) v_n += 1
def file_iterator(input_path, output_path, tool, func_name, func_args): ''' Recursively iterate c files from a given path. Call given function on each file. Arguments: input_path {str} -- Input path of source files output_path {str} -- Output path to store test results tool {str} -- Tool to run on each file iteration func_name {str} -- Name of function to call in each iternation func_args {dict} -- Arguments requried for function call ''' input_files_path = fs.ls(input_path, EXT['c']) new_output_path = fs.mkdir(output_path) csv_header = get_csv_header(tool, func_args['credentials']) analysis_file_path = os.path.join(new_output_path, FILE_NAME['analysis']) fs.write_csv(analysis_file_path, [csv_header]) for input_file_path in input_files_path: input_file = fs.details(input_file_path) output_dir_path = os.path.join(new_output_path, input_file['name']) new_output_dir_path = fs.mkdir(output_dir_path) # TODO: Remove file permissions on copy copy2(input_file_path, new_output_dir_path) test_results = func_name(input_file_path, new_output_dir_path, func_args) fs.append_csv(analysis_file_path, test_results)
def run_tha_test(isolated_hash, storage, cache, leak_temp_dir, result_json, root_dir, hard_timeout, grace_period, extra_args): """Downloads the dependencies in the cache, hardlinks them into a temporary directory and runs the executable from there. A temporary directory is created to hold the output files. The content inside this directory will be uploaded back to |storage| packaged as a .isolated file. Arguments: isolated_hash: the SHA-1 of the .isolated file that must be retrieved to recreate the tree of files to run the target executable. storage: an isolateserver.Storage object to retrieve remote objects. This object has a reference to an isolateserver.StorageApi, which does the actual I/O. cache: an isolateserver.LocalCache to keep from retrieving the same objects constantly by caching the objects retrieved. Can be on-disk or in-memory. leak_temp_dir: if true, the temporary directory will be deliberately leaked for later examination. result_json: file path to dump result metadata into. If set, the process exit code is always 0 unless an internal error occured. root_dir: directory to the path to use to create the temporary directory. If not specified, a random temporary directory is created. hard_timeout: kills the process if it lasts more than this amount of seconds. grace_period: number of seconds to wait between SIGTERM and SIGKILL. extra_args: optional arguments to add to the command stated in the .isolate file. Returns: Process exit code that should be used. """ # run_isolated exit code. Depends on if result_json is used or not. result = map_and_run(isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout, grace_period, extra_args) logging.info('Result:\n%s', tools.format_json(result, dense=True)) if result_json: # We've found tests to delete 'work' when quitting, causing an exception # here. Try to recreate the directory if necessary. work_dir = os.path.dirname(result_json) if not fs.isdir(work_dir): fs.mkdir(work_dir) tools.write_json(result_json, result, dense=True) # Only return 1 if there was an internal error. return int(bool(result['internal_failure'])) # Marshall into old-style inline output. if result['outputs_ref']: data = { 'hash': result['outputs_ref']['isolated'], 'namespace': result['outputs_ref']['namespace'], 'storage': result['outputs_ref']['isolatedserver'], } sys.stdout.flush() print('[run_isolated_out_hack]%s[/run_isolated_out_hack]' % tools.format_json(data, dense=True)) sys.stdout.flush() return result['exit_code'] or int(bool(result['internal_failure']))
def test_rmtree_unicode(self): subdir = os.path.join(self.tempdir, "hi") fs.mkdir(subdir) filepath = os.path.join(subdir, u"\u0627\u0644\u0635\u064A\u0646\u064A\u0629") with fs.open(filepath, "wb") as f: f.write("hi") # In particular, it fails when the input argument is a str. file_path.rmtree(str(subdir))
def test_file_to_metadata_path_case_simple(self): # Ensure the symlink dest is saved in the right path case. subdir = os.path.join(self.cwd, u'subdir') fs.mkdir(subdir) linkdir = os.path.join(self.cwd, u'linkdir') fs.symlink('subDir', linkdir) actual = isolated_format.file_to_metadata(linkdir.upper(), False) self.assertEqual({'l': u'subdir'}, actual)
def test_rmtree_unicode(self): subdir = os.path.join(self.tempdir, 'hi') fs.mkdir(subdir) filepath = os.path.join( subdir, u'\u0627\u0644\u0635\u064A\u0646\u064A\u0629') with fs.open(filepath, 'wb') as f: f.write('hi') # In particular, it fails when the input argument is a str. file_path.rmtree(str(subdir))
def test_cleanup_unexpected(self): # cleanup() delete unexpected file in the cache directory. fs.mkdir(self.cache_dir) with fs.open(os.path.join(self.cache_dir, u'junk'), 'w') as f: f.write('random') cache = self.get_cache(_get_policies()) self.assertEqual(['junk'], fs.listdir(cache.cache_dir)) self.assertEqual(True, cache.cleanup()) self.assertEqual([cache.STATE_FILE], fs.listdir(cache.cache_dir))
def test_hard_link_mode(self): # Creates a hard link, see if the file mode changed on the node or the # directory entry. dir_foo = os.path.join(self.tempdir, 'foo') file_bar = os.path.join(dir_foo, 'bar') file_link = os.path.join(dir_foo, 'link') fs.mkdir(dir_foo, 0777) write_content(file_bar, 'bar') file_path.hardlink(file_bar, file_link) self.assertFileMode(file_bar, 0100666) self.assertFileMode(file_link, 0100666) file_path.set_read_only(file_bar, True) self.assertMaskedFileMode(file_bar, 0100444) self.assertMaskedFileMode(file_link, 0100444)
def test_hard_link_mode(self): # Creates a hard link, see if the file mode changed on the node or the # directory entry. dir_foo = os.path.join(self.tempdir, 'foo') file_bar = os.path.join(dir_foo, 'bar') file_link = os.path.join(dir_foo, 'link') fs.mkdir(dir_foo, 0o777) write_content(file_bar, b'bar') file_path.hardlink(file_bar, file_link) self.assertFileMode(file_bar, 0o100666) self.assertFileMode(file_link, 0o100666) file_path.set_read_only(file_bar, True) self.assertMaskedFileMode(file_bar, 0o100444) self.assertMaskedFileMode(file_link, 0o100444)
def test_delete_rd_rf(self): # Confirms that a RO file in a RO directory can't be deleted. dir_foo = os.path.join(self.tempdir, 'foo') file_bar = os.path.join(dir_foo, 'bar') fs.mkdir(dir_foo, 0o777) write_content(file_bar, b'bar') file_path.set_read_only(dir_foo, True) file_path.set_read_only(file_bar, True) self.assertMaskedFileMode(dir_foo, 0o40555) self.assertMaskedFileMode(file_bar, 0o100444) with self.assertRaises(OSError): # It fails for different reason depending on the OS. See the test cases # above. fs.remove(file_bar)
def test_delete_rd_rf(self): # Confirms that a RO file in a RO directory can't be deleted. dir_foo = os.path.join(self.tempdir, 'foo') file_bar = os.path.join(dir_foo, 'bar') fs.mkdir(dir_foo, 0777) write_content(file_bar, 'bar') file_path.set_read_only(dir_foo, True) file_path.set_read_only(file_bar, True) self.assertMaskedFileMode(dir_foo, 040555) self.assertMaskedFileMode(file_bar, 0100444) with self.assertRaises(OSError): # It fails for different reason depending on the OS. See the test cases # above. fs.remove(file_bar)
def test_delete_wd_rf(self): # Confirms that a RO file in a RW directory can be deleted on non-Windows. dir_foo = os.path.join(self.tempdir, 'foo') file_bar = os.path.join(dir_foo, 'bar') fs.mkdir(dir_foo, 0o777) write_content(file_bar, b'bar') file_path.set_read_only(dir_foo, False) file_path.set_read_only(file_bar, True) self.assertFileMode(dir_foo, 0o40777) self.assertMaskedFileMode(file_bar, 0o100444) if sys.platform == 'win32': # On Windows, a read-only file can't be deleted. with self.assertRaises(OSError): fs.remove(file_bar) else: fs.remove(file_bar)
def test_delete_wd_rf(self): # Confirms that a RO file in a RW directory can be deleted on non-Windows. dir_foo = os.path.join(self.tempdir, 'foo') file_bar = os.path.join(dir_foo, 'bar') fs.mkdir(dir_foo, 0777) write_content(file_bar, 'bar') file_path.set_read_only(dir_foo, False) file_path.set_read_only(file_bar, True) self.assertFileMode(dir_foo, 040777) self.assertMaskedFileMode(file_bar, 0100444) if sys.platform == 'win32': # On Windows, a read-only file can't be deleted. with self.assertRaises(OSError): fs.remove(file_bar) else: fs.remove(file_bar)
def test_symlink_absolute(self): # A symlink to an absolute path is valid. # /dir # /dir/file # /ld -> /dir # /lf -> /ld/file dirpath = os.path.join(self.tempdir, 'dir') filepath = os.path.join(dirpath, 'file') fs.mkdir(dirpath) write_content(filepath, b'hello') linkfile = os.path.join(self.tempdir, 'lf') linkdir = os.path.join(self.tempdir, 'ld') dstfile = os.path.join(linkdir, 'file') fs.symlink(dstfile, linkfile) fs.symlink(dirpath, linkdir) self.assertEqual(True, fs.islink(linkfile)) self.assertEqual(True, fs.islink(linkdir)) self.assertEqual(dstfile, fs.readlink(linkfile)) self.assertEqual(dirpath, fs.readlink(linkdir)) self.assertEqual(['file'], fs.listdir(linkdir)) # /lf resolves to /dir/file. with fs.open(linkfile) as f: self.assertEqual('hello', f.read()) # Ensures that followlinks is respected in walk(). expected = [ (self.tempdir, ['dir', 'ld'], ['lf']), (dirpath, [], ['file']), ] actual = [ (r, sorted(d), sorted(f)) for r, d, f in sorted(fs.walk(self.tempdir, followlinks=False)) ] self.assertEqual(expected, actual) expected = [ (self.tempdir, ['dir', 'ld'], ['lf']), (dirpath, [], ['file']), (linkdir, [], ['file']), ] actual = [ (r, sorted(d), sorted(f)) for r, d, f in sorted(fs.walk(self.tempdir, followlinks=True)) ] self.assertEqual(expected, actual)
def test_cleanup_incorrect_link(self): # cleanup() repairs broken symlink in named/. cache = self.get_cache(_get_policies()) self._add_one_item(cache, 1) self._add_one_item(cache, 2) fs.remove(os.path.join(self.cache_dir, cache.NAMED_DIR, u'1')) fs.remove(os.path.join(self.cache_dir, cache.NAMED_DIR, u'2')) fs.symlink('invalid_dest', os.path.join(self.cache_dir, cache.NAMED_DIR, u'1')) fs.mkdir(os.path.join(self.cache_dir, cache.NAMED_DIR, u'2')) cache = self.get_cache(_get_policies()) self.assertEqual( ['1', '2'], sorted(fs.listdir(os.path.join(cache.cache_dir, cache.NAMED_DIR)))) self.assertEqual(True, cache.cleanup()) self.assertEqual([], fs.listdir( os.path.join(cache.cache_dir, cache.NAMED_DIR)))
def test_expand_symlinks_path_case(self): # Ensures that the resulting path case is fixed on case insensitive file # system. fs.symlink('dest', os.path.join(self.cwd, u'link')) fs.mkdir(os.path.join(self.cwd, u'Dest')) fs.open(os.path.join(self.cwd, u'Dest', u'file.txt'), 'w').close() relfile, symlinks = isolated_format._expand_symlinks( self.cwd, u'.') self.assertEqual((u'.', []), (relfile, symlinks)) relfile, symlinks = isolated_format._expand_symlinks( self.cwd, u'link') self.assertEqual((u'Dest', [u'link']), (relfile, symlinks)) relfile, symlinks = isolated_format._expand_symlinks( self.cwd, u'link/File.txt') self.assertEqual((u'Dest/file.txt', [u'link']), (relfile, symlinks))
def fuck_yourself(self): # set ipv6 address if needed if self.use_ipv6: ipv6 = ipv6_addr() if not ipv6: ptl_error("cannot get ipv6 address") self.str_ipv6 = "&ipv6=%s" % urlencode(ipv6) # generate a port number if not given if not self.port: self.port = randint(MIN_PORT, MAX_PORT) # generate client key self.client_key = client_key() client_info = BT_CLIENTS[self.client_id] # generate peer_id : based on client chosen prefix = client_info["prefix"] pid = peer_id(prefix) self.quoted_peer_id = urlencode(pid) # generate http header[user-agent] : based on client chosen user_agent = client_info["user-agent"] self.headers.update({"User-Agent": user_agent}) # supports scrape? self.scrapable = not self.no_scrape and client_info["scrape"] # create directories if not exist for up_down in (UP, DOWN): mkdir(DIR[up_down]) log.setLevel(DEBUG) log.debug("ptliar started, version: %s" % __version__) log.info("verbose : %s" % (self.logging_level == DEBUG)) log.info("ipv6 : %s" % self.use_ipv6) log.info("zero_rate : %s" % self.use_zero_rate) log.info("timer : %s" % ptime(self.timer)) log.info("max up bandwidth : %s/s" % psize(self.max_up_speed)) log.info("max down bandwidth : %s/s" % psize(self.max_down_speed)) log.info("max torrent speed : %s/s" % psize(self.max_torrent_speed)) log.info("fake bt client : %s" % self.client_id)
def test_rmtree_win(self): # Mock our sleep for faster test case execution. sleeps = [] self.mock(time, 'sleep', sleeps.append) self.mock(sys, 'stderr', StringIO.StringIO()) # Open a child process, so the file is locked. subdir = os.path.join(self.tempdir, 'to_be_deleted') fs.mkdir(subdir) script = 'import time; open(\'a\', \'w\'); time.sleep(60)' proc = subprocess.Popen([sys.executable, '-c', script], cwd=subdir) try: # Wait until the file exist. while not fs.isfile(os.path.join(subdir, 'a')): self.assertEqual(None, proc.poll()) file_path.rmtree(subdir) self.assertEqual([2, 4, 2], sleeps) # sys.stderr.getvalue() would return a fair amount of output but it is # not completely deterministic so we're not testing it here. finally: proc.wait()
def test_delete_rd_wf(self): # Confirms that a Rw file in a RO directory can be deleted on Windows only. dir_foo = os.path.join(self.tempdir, 'foo') file_bar = os.path.join(dir_foo, 'bar') fs.mkdir(dir_foo, 0777) write_content(file_bar, 'bar') file_path.set_read_only(dir_foo, True) file_path.set_read_only(file_bar, False) self.assertMaskedFileMode(dir_foo, 040555) self.assertFileMode(file_bar, 0100666) if sys.platform == 'win32': # A read-only directory has a convoluted meaning on Windows, it means that # the directory is "personalized". This is used as a signal by Windows # Explorer to tell it to look into the directory for desktop.ini. # See http://support.microsoft.com/kb/326549 for more details. # As such, it is important to not try to set the read-only bit on # directories on Windows since it has no effect other than trigger # Windows Explorer to look for desktop.ini, which is unnecessary. fs.remove(file_bar) else: with self.assertRaises(OSError): fs.remove(file_bar)
def test_delete_rd_wf(self): # Confirms that a Rw file in a RO directory can be deleted on Windows only. dir_foo = os.path.join(self.tempdir, 'foo') file_bar = os.path.join(dir_foo, 'bar') fs.mkdir(dir_foo, 0o777) write_content(file_bar, b'bar') file_path.set_read_only(dir_foo, True) file_path.set_read_only(file_bar, False) self.assertMaskedFileMode(dir_foo, 0o40555) self.assertFileMode(file_bar, 0o100666) if sys.platform == 'win32': # A read-only directory has a convoluted meaning on Windows, it means that # the directory is "personalized". This is used as a signal by Windows # Explorer to tell it to look into the directory for desktop.ini. # See http://support.microsoft.com/kb/326549 for more details. # As such, it is important to not try to set the read-only bit on # directories on Windows since it has no effect other than trigger # Windows Explorer to look for desktop.ini, which is unnecessary. fs.remove(file_bar) else: with self.assertRaises(OSError): fs.remove(file_bar)
def test_file_to_metadata_path_case_complex(self): # Ensure the symlink dest is saved in the right path case. This includes 2 # layers of symlinks. basedir = os.path.join(self.cwd, u'basebir') fs.mkdir(basedir) linkeddir2 = os.path.join(self.cwd, u'linkeddir2') fs.mkdir(linkeddir2) linkeddir1 = os.path.join(basedir, u'linkeddir1') fs.symlink('../linkedDir2', linkeddir1) subsymlinkdir = os.path.join(basedir, u'symlinkdir') fs.symlink('linkedDir1', subsymlinkdir) actual = isolated_format.file_to_metadata(subsymlinkdir.upper(), True, False) self.assertEqual({'l': u'linkeddir1'}, actual) actual = isolated_format.file_to_metadata(linkeddir1.upper(), True, False) self.assertEqual({'l': u'../linkeddir2'}, actual)
def test_load_corrupted_state(self): # cleanup() handles a broken state file. fs.mkdir(self.cache_dir) c = local_caching.NamedCache with fs.open(os.path.join(self.cache_dir, c.STATE_FILE), 'w') as f: f.write('}}}}') fs.makedirs(os.path.join(self.cache_dir, '1'), 0777) cache = self.get_cache(_get_policies()) self._add_one_item(cache, 1) self.assertTrue( fs.exists(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1'))) self.assertTrue( fs.islink(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1'))) self.assertEqual([], cache.trim()) self.assertTrue( fs.exists(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1'))) self.assertTrue( fs.islink(os.path.join(cache.cache_dir, cache.NAMED_DIR, '1'))) self.assertEqual(True, cache.cleanup()) self.assertEqual( sorted([cache.NAMED_DIR, cache.STATE_FILE, cache._lru[u'1'][0]]), sorted(fs.listdir(cache.cache_dir)))
def test_upgrade(self): # Make sure upgrading works. This is temporary as eventually all bots will # be updated. now = time.time() fs.mkdir(self.cache_dir) fs.mkdir(os.path.join(self.cache_dir, 'f1')) with fs.open(os.path.join(self.cache_dir, 'f1', 'hello'), 'wb') as f: f.write('world') # v1 old = { 'version': 2, 'items': [ ['cache1', ['f1', now]], ], } c = local_caching.NamedCache with fs.open(os.path.join(self.cache_dir, c.STATE_FILE), 'w') as f: json.dump(old, f) # It automatically upgrades to v2. cache = self.get_cache(_get_policies()) expected = {u'cache1': ((u'f1', len('world')), now)} self.assertEqual(expected, dict(cache._lru._items.iteritems())) self.assertEqual([u'f1', cache.STATE_FILE], sorted(fs.listdir(cache.cache_dir)))
def test_symlink_input_absolute_path(self): # A symlink is outside of the checkout, it should be treated as a normal # directory. # .../src # .../src/out -> .../tmp/foo # .../tmp # .../tmp/foo src = os.path.join(self.cwd, u'src') src_out = os.path.join(src, u'out') tmp = os.path.join(self.cwd, u'tmp') tmp_foo = os.path.join(tmp, u'foo') fs.mkdir(src) fs.mkdir(tmp) fs.mkdir(tmp_foo) # The problem was that it's an absolute path, so it must be considered a # normal directory. fs.symlink(tmp, src_out) fs.open(os.path.join(tmp_foo, u'bar.txt'), 'w').close() relfile, symlinks = isolated_format._expand_symlinks( src, u'out/foo/bar.txt') self.assertEqual((u'out/foo/bar.txt', []), (relfile, symlinks))
def test_cleanup_unexpected_named(self): # cleanup() deletes unexpected symlink and directory in named/. fs.mkdir(self.cache_dir) c = local_caching.NamedCache fs.mkdir(os.path.join(self.cache_dir, c.NAMED_DIR)) p = os.path.join(self.cache_dir, c.NAMED_DIR, u'junk_file') with fs.open(p, 'w') as f: f.write('random') fs.mkdir(os.path.join(self.cache_dir, c.NAMED_DIR, u'junk_dir')) fs.symlink('invalid_dest', os.path.join(self.cache_dir, c.NAMED_DIR, u'junk_link')) cache = self.get_cache(_get_policies()) self.assertEqual([cache.NAMED_DIR], fs.listdir(cache.cache_dir)) self.assertEqual( ['junk_dir', 'junk_file', 'junk_link'], sorted(fs.listdir(os.path.join(cache.cache_dir, cache.NAMED_DIR)))) self.assertEqual(True, cache.cleanup()) self.assertEqual([cache.NAMED_DIR, cache.STATE_FILE], sorted(fs.listdir(cache.cache_dir))) self.assertEqual([], fs.listdir( os.path.join(cache.cache_dir, cache.NAMED_DIR)))
def test_file_to_metadata_path_case_collapse(self): # Ensure setting the collapse_symlink option doesn't include the symlinks basedir = os.path.join(self.cwd, u'basedir') fs.mkdir(basedir) subdir = os.path.join(basedir, u'subdir') fs.mkdir(subdir) linkdir = os.path.join(basedir, u'linkdir') fs.mkdir(linkdir) foo_file = os.path.join(subdir, u'Foo.txt') fs.open(foo_file, 'w').close() sym_file = os.path.join(basedir, u'linkdir', u'Sym.txt') fs.symlink('../subdir/Foo.txt', sym_file) actual = isolated_format.file_to_metadata(sym_file, True, True) actual['h'] = isolated_format.hash_file(sym_file, ALGO) expected = { # SHA-1 of empty string 'h': 'da39a3ee5e6b4b0d3255bfef95601890afd80709', 'm': 256, 's': 0, } self.assertEqual(expected, actual)
def run(argv): ''' Selects which tool to be used by user. Arguments: argv {list} -- List of arguements ''' if len(argv) <= 1: print_help() sys.exit(1) try: args = PARSER.parse_args() except BaseException: sys.exit(1) tool = args.option if args.output: output_path = os.path.abspath(args.output) else: print_help() exit(1) if tool == GENERATE: codes = args.codes passwords = args.passwords generate(output_path, codes, passwords) elif tool == OBFUSCATE: num_variants = args.num_variants input_path = os.path.abspath(args.input) fs.mkdir(output_path, False) obfuscation_combinations = args.obfuscation_list for obs in obfuscation_combinations: if not re.search(RE_OBFUSCATION, obs): print( 'One more more items in the list does not contain [A, C, D, V]') print('Error at: ' + obs) sys.exit(1) obfuscate( input_path, output_path, obfuscation_combinations, num_variants) elif tool == RUN: input_path = os.path.abspath(args.input) fn_args = { 'credentials': credentials(args), 'levels': args.optimization_levels, } file_iterator(input_path, output_path, RUN, run_code, fn_args) elif (tool == ANGR or tool == KLEE or tool == ALL): input_path = os.path.abspath(args.input) fn_args = { 'tool': tool, 'stdin': stdin(args), 'options': se_options(args), 'credentials': credentials(args), } file_iterator(input_path, output_path, tool, run_se, fn_args)
def run_tha_test( isolated_hash, storage, cache, leak_temp_dir, result_json, root_dir, hard_timeout, grace_period, extra_args): """Downloads the dependencies in the cache, hardlinks them into a temporary directory and runs the executable from there. A temporary directory is created to hold the output files. The content inside this directory will be uploaded back to |storage| packaged as a .isolated file. Arguments: isolated_hash: the SHA-1 of the .isolated file that must be retrieved to recreate the tree of files to run the target executable. storage: an isolateserver.Storage object to retrieve remote objects. This object has a reference to an isolateserver.StorageApi, which does the actual I/O. cache: an isolateserver.LocalCache to keep from retrieving the same objects constantly by caching the objects retrieved. Can be on-disk or in-memory. leak_temp_dir: if true, the temporary directory will be deliberately leaked for later examination. result_json: file path to dump result metadata into. If set, the process exit code is always 0 unless an internal error occured. root_dir: directory to the path to use to create the temporary directory. If not specified, a random temporary directory is created. hard_timeout: kills the process if it lasts more than this amount of seconds. grace_period: number of seconds to wait between SIGTERM and SIGKILL. extra_args: optional arguments to add to the command stated in the .isolate file. Returns: Process exit code that should be used. """ # run_isolated exit code. Depends on if result_json is used or not. result = map_and_run( isolated_hash, storage, cache, leak_temp_dir, root_dir, hard_timeout, grace_period, extra_args) logging.info('Result:\n%s', tools.format_json(result, dense=True)) if result_json: # We've found tests to delete 'work' when quitting, causing an exception # here. Try to recreate the directory if necessary. work_dir = os.path.dirname(result_json) if not fs.isdir(work_dir): fs.mkdir(work_dir) tools.write_json(result_json, result, dense=True) # Only return 1 if there was an internal error. return int(bool(result['internal_failure'])) # Marshall into old-style inline output. if result['outputs_ref']: data = { 'hash': result['outputs_ref']['isolated'], 'namespace': result['outputs_ref']['namespace'], 'storage': result['outputs_ref']['isolatedserver'], } sys.stdout.flush() print( '[run_isolated_out_hack]%s[/run_isolated_out_hack]' % tools.format_json(data, dense=True)) sys.stdout.flush() return result['exit_code'] or int(bool(result['internal_failure']))
def CMDreproduce(parser, args): """Runs a task locally that was triggered on the server. This running locally the same commands that have been run on the bot. The data downloaded will be in a subdirectory named 'work' of the current working directory. You can pass further additional arguments to the target command by passing them after --. """ options, args = parser.parse_args(args) extra_args = [] if not args: parser.error('Must specify exactly one task id.') if len(args) > 1: if args[1] == '--': if len(args) > 2: extra_args = args[2:] else: extra_args = args[1:] url = options.swarming + '/_ah/api/swarming/v1/task/%s/request' % args[0] request = net.url_read_json(url) if not request: print >> sys.stderr, 'Failed to retrieve request data for the task' return 1 workdir = unicode(os.path.abspath('work')) if not fs.isdir(workdir): fs.mkdir(workdir) properties = request['properties'] env = None if properties.get('env'): env = os.environ.copy() logging.info('env: %r', properties['env']) for i in properties['env']: key = i['key'].encode('utf-8') if not i['value']: env.pop(key, None) else: env[key] = i['value'].encode('utf-8') if properties.get('inputs_ref'): # Create the tree. with isolateserver.get_storage( properties['inputs_ref']['isolatedserver'], properties['inputs_ref']['namespace']) as storage: bundle = isolateserver.fetch_isolated( properties['inputs_ref']['isolated'], storage, isolateserver.MemoryCache(file_mode_mask=0700), workdir, False) command = bundle.command if bundle.relative_cwd: workdir = os.path.join(workdir, bundle.relative_cwd) else: command = properties['command'] try: return subprocess.call(command + extra_args, env=env, cwd=workdir) except OSError as e: print >> sys.stderr, 'Failed to run: %s' % ' '.join(command) print >> sys.stderr, str(e) return 1
def CMDreproduce(parser, args): """Runs a task locally that was triggered on the server. This running locally the same commands that have been run on the bot. The data downloaded will be in a subdirectory named 'work' of the current working directory. You can pass further additional arguments to the target command by passing them after --. """ parser.add_option( '--output-dir', metavar='DIR', default='', help='Directory that will have results stored into') options, args = parser.parse_args(args) extra_args = [] if not args: parser.error('Must specify exactly one task id.') if len(args) > 1: if args[1] == '--': if len(args) > 2: extra_args = args[2:] else: extra_args = args[1:] url = options.swarming + '/_ah/api/swarming/v1/task/%s/request' % args[0] request = net.url_read_json(url) if not request: print >> sys.stderr, 'Failed to retrieve request data for the task' return 1 workdir = unicode(os.path.abspath('work')) if fs.isdir(workdir): parser.error('Please delete the directory \'work\' first') fs.mkdir(workdir) properties = request['properties'] env = None if properties.get('env'): env = os.environ.copy() logging.info('env: %r', properties['env']) for i in properties['env']: key = i['key'].encode('utf-8') if not i['value']: env.pop(key, None) else: env[key] = i['value'].encode('utf-8') if properties.get('inputs_ref'): # Create the tree. with isolateserver.get_storage( properties['inputs_ref']['isolatedserver'], properties['inputs_ref']['namespace']) as storage: bundle = isolateserver.fetch_isolated( properties['inputs_ref']['isolated'], storage, isolateserver.MemoryCache(file_mode_mask=0700), workdir) command = bundle.command if bundle.relative_cwd: workdir = os.path.join(workdir, bundle.relative_cwd) command.extend(properties.get('extra_args') or []) # https://github.com/luci/luci-py/blob/master/appengine/swarming/doc/Magic-Values.md new_command = run_isolated.process_command(command, options.output_dir) if not options.output_dir and new_command != command: parser.error('The task has outputs, you must use --output-dir') command = new_command else: command = properties['command'] try: return subprocess.call(command + extra_args, env=env, cwd=workdir) except OSError as e: print >> sys.stderr, 'Failed to run: %s' % ' '.join(command) print >> sys.stderr, str(e) return 1