def encode_video(experiment_name, video_name): # TODO: security lol dirname = '~/results/{}'.format(experiment_name) input_name = os.path.join(dirname, video_name) output_name = input_name.replace('.mjpeg', '.mp4') with pushd(dirname): os.system('ffmpeg -y -i {} {}'.format(input_name, output_name)) os.system('rm {}'.format(input_name))
def test_main(self, mock_clone, mock_post): try: readmesfix.main('test.tsv') self.assertTrue(mock_clone.called) self.assertEqual(2, mock_post.call_count) with tempfile.TemporaryDirectory() as temp_dir, util.pushd( temp_dir): repo = git.Repo.clone_from(TEST_REPO_ABSOLUTE_PATH, '.') diff = subprocess.run(['git', 'diff', 'HEAD~'], check=True, stdout=subprocess.PIPE).stdout repo.git.reset('--hard', 'HEAD~') repo.remotes['origin'].push(force=True) with open('test.diff', 'rb') as gold_diff_file: self.assertEqual(gold_diff_file.read(), diff) finally: with util.pushd('test-repo.git'): subprocess.run( 'git -c gc.reflogExpire=0 -c gc.reflogExpireUnreachable=0 -c gc.rerereresolved=0 ' '-c gc.rerereunresolved=0 -c gc.pruneExpire=now gc "$@"', shell=True, check=True, stdout=subprocess.PIPE)
def build( self, namespace: argparse.Namespace, image: str, build_config: config.ImageBuildConfig): ''' :param namespace: Namespace passed in via CLI. :param image: The image to build. :param build_config: the image build configuration. :raises: subprocess.CalledProcessError ''' LOGGER.info('Build image') LOGGER.info('Dry Run: %s', namespace.dry_run) image_dir = util.get_image_dir(namespace.path, image) image = build_config.image.full_name build_args = builder.get_build_args(build_config) command = util.Command(['buildah', 'bud', '-t']) command.add_arg(image) command.add_args_list('--build-arg', build_args) version = build_config.image.tag_build.version if version: command.add_args('--build-arg', f'VERSION={version}') # add build context command.add_arg('.') LOGGER.info('Image name: %s', image) LOGGER.info('Command: %s', ' '.join(command)) if namespace.dry_run: return # build with util.pushd(image_dir): subprocess.check_call(command) image_tag = build_config.image.tag if namespace.tag_latest and image_tag != 'latest': self.tag_latest(image) if namespace.push: self.push( namespace=namespace, image=image)
def iterate_targets(root): """Iterate over the targets generated based on root directory tree""" root = os.path.abspath(root) logging.info('sourcedir=%s', root) target_rules_filename = Settings.get('targets')['filename'] for path, files in util.walk(root): with util.pushd(os.path.join(root, path)): relpath = Path.clean(path.replace(root, '')) logging.info('Parsing folder: $sourcedir/%s', relpath) if not files: logging.debug('No files found') elif target_rules_filename in files: target_rules_filepath = os.path.join(path, target_rules_filename) target_rules = util.load_yaml_or_json(target_rules_filepath) for gyp_target in target_rules.get('targets', []): yield Target(relpath, files, gyp_target) else: yield Target(relpath, files)
def build(self, namespace: argparse.Namespace, image: str, build_config: config.ImageBuildConfig): ''' :param namespace: Namespace passed in via CLI. :param image: The image to build. :param build_config: the image build configuration. :raises: subprocess.CalledProcessError ''' LOGGER.info('Build image') image_dir = util.get_image_dir(namespace.path, image) image = build_config.image.full_name build_args = builder.get_build_args(build_config) version = build_config.image.tag_build.version if version: build_args.append(f'VERSION={version}') command = self._create_command(namespace=namespace, action='build', build_args=build_args) image_names_output = [image] image_tag = build_config.image.tag if namespace.tag_latest and image_tag != 'latest': image_latest = util.set_image_tag_latest(image=image) image_names_output.append(image_latest) names_output = ','.join([f'name={i}' for i in image_names_output]) command.add_args( name='--output', value=f'type=image,{names_output},push={namespace.push}') LOGGER.info('Image name: %s', image) LOGGER.info('Command: %s', ' '.join(command)) if namespace.dry_run: return with util.pushd(image_dir): subprocess.check_call(command)
#!/usr/bin/env python from util import pushd, run_server, run, example_root with pushd(example_root): with pushd('shogun'): with run_server('jubaclassifier', '-f', 'shogun.json'): run('ruby', 'ruby/shogun.rb') with pushd('gender'): with run_server('jubaclassifier', '-f', 'gender.json'): run('ruby', 'ruby/gender.rb') with pushd('movielens'): with run_server('jubarecommender', '-f', 'config.json'): with pushd('ruby'): run('ruby', 'ml_update.rb') run('ruby', 'ml_analysis.rb') with pushd('npb_similar_player'): with run_server('jubarecommender', '-f', 'npb_similar_player.json'): run('ruby', 'update.rb') run('ruby', 'analyze.rb') with pushd('rent'): with run_server('jubaregression', '-f', 'rent.json'): run('ruby', 'ruby/train.rb', 'dat/rent-data.csv') run('ruby', 'ruby/test.rb', input='19.9\n2\n22\n4\nW\n') with pushd('language_detection'):
#!/usr/bin/env python from util import pushd, run_server, run, example_root with pushd(example_root): with pushd('shogun'): with run_server('jubaclassifier', '-f', 'shogun.json'): with pushd('java'): run('mvn', 'clean') run('sh', 'run.sh') with pushd('gender'): with run_server('jubaclassifier', '-f', 'gender.json'): with pushd('java'): run('mvn', 'clean') run('sh', 'run.sh') with pushd('twitter_streaming_location'): with run_server('jubaclassifier', '-f', 'twitter_streaming_location.json'): pass with pushd('movielens'): with run_server('jubarecommender', '-f', 'config.json'): with pushd('java'): run('mvn', 'clean') run('sh', 'run-update.sh') run('sh', 'run-analyze.sh')
#!/usr/bin/env python import os from util import pushd, run_server, run, example_root with pushd(example_root): with pushd('gender'): with run_server('jubaclassifier', '-f', 'gender.json'): with pushd('cpp'): run('make') run('./gender') with pushd('malware_classification'): pass with pushd('movielens'): with run_server('jubarecommender', '-f', 'config.json'): with pushd('cpp'): run('python', 'waf', 'configure') run('python', 'waf') run('./build/ml_update') run('./build/ml_analysis')
def main(dataset_path): global inside_code_block, last_valid_fence with open(dataset_path) as file: number_of_lines = sum(1 for _ in file) file.seek(0) for (repo_name, ) in tqdm.tqdm(csv.reader(file), total=number_of_lines): with tempfile.TemporaryDirectory() as temp_dir, util.pushd( temp_dir): # noinspection PyBroadException try: repo = git.Repo.clone_from( f'[email protected]:{repo_name}.git', '.', depth=1, origin='upstream') markdown_paths = set(insensitive_glob('**/*.md', recursive=True)) \ | set(insensitive_glob('**/*.mkdn?', recursive=True)) \ | set(insensitive_glob('**/*.mdown', recursive=True)) \ | set(insensitive_glob('**/*.markdown', recursive=True)) markdown_paths = { path for path in markdown_paths if os.path.isfile(path) } if markdown_paths: # Gets stuck otherwise paths_with_crlf = crlf_paths(markdown_paths) with fileinput.input(markdown_paths, inplace=True) as markdown_file: use_crlf = False for line in markdown_file: if fileinput.isfirstline(): inside_code_block = False last_valid_fence = None use_crlf = markdown_file.filename( ) in paths_with_crlf if use_crlf and line and line[-1] == '\n': line = line[:-1] + '\r\n' CODE_BLOCK_FENCE_BACK_TICKS_RE.sub( detect_code_block_back_ticks_fence, line) CODE_BLOCK_FENCE_TILDES_RE.sub( detect_code_block_tildes_fence, line) print(HEADING_WITHOUT_SPACE_RE.sub( heading_fix, line), end='') if repo.index.diff(None): repo.git.add('.') repo.git.commit(m="Fix broken Markdown headings") response = requests.post( f'https://api.github.com/repos/{repo_name}/forks', params=AUTH_PARAMS) response_dict = json.loads(response.text) if response.status_code == 202: repo.create_remote( 'origin', response_dict['ssh_url']).push() create_pr( repo_name, response_dict["default_branch"], f'{response_dict["owner"]["login"]}:{response_dict["default_branch"]}' ) else: print( f"There was an error forking {repo_name}: {response_dict}" ) except Exception: print(traceback.format_exc())
#!/usr/bin/env python import os from util import pushd, run, example_root with pushd(example_root): with pushd('movielens'): if not os.path.exists('dat/ml-100k'): run('mkdir', '-p', 'dat') with pushd('dat'): run('wget', 'http://www.grouplens.org/system/files/ml-100k.zip') run('unzip', 'ml-100k.zip')
#!/usr/bin/env python from util import pushd, run_server, run, example_root with pushd(example_root): with pushd('shogun'): with run_server('jubaclassifier', '-f', 'shogun.json'): with pushd('ruby'): run('ruby', 'shogun.rb') with pushd('gender'): with run_server('jubaclassifier', '-f', 'gender.json'): with pushd('ruby'): run('ruby', 'gender.rb') with pushd('movielens'): with run_server('jubarecommender', '-f', 'config.json'): with pushd('ruby'): run('ruby', 'ml_update.rb') run('ruby', 'ml_analysis.rb') with pushd('npb_similar_player'): with run_server('jubarecommender', '-f', 'npb_similar_player.json'): with pushd('ruby'): run('ruby', 'update.rb') run('ruby', 'analyze.rb') with pushd('rent'): with run_server('jubaregression', '-f', 'rent.json'): with pushd('ruby'):
#!/usr/bin/env python from util import pushd, run_server, run, example_root with pushd(example_root): with pushd('shogun'): with run_server('jubaclassifier', '-f', 'shogun.json'): with pushd('python'): run('python', 'shogun.py') with pushd('gender'): with run_server('jubaclassifier', '-f', 'gender.json'): with pushd('python'): run('python', 'gender.py') with pushd('twitter_streaming_lang'): with run_server('jubaclassifier', '-f', 'twitter_streaming_lang.json'): pass with pushd('twitter_streaming_location'): with run_server('jubaclassifier', '-f', 'twitter_streaming_location.json'): pass with pushd('movielens'): with run_server('jubarecommender', '-f', 'config.json'): with pushd('python'): run('python', 'ml_update.py') run('python', 'ml_analysis.py')
#!/usr/bin/env python from util import pushd, run_server, run, example_root with pushd(example_root): with pushd('shogun'): with run_server('jubaclassifier', '-f', 'shogun.json'): run('python', 'python/shogun.py') with pushd('gender'): with run_server('jubaclassifier', '-f', 'gender.json'): run('python', 'python/gender.py') with pushd('twitter_streaming_lang'): with run_server('jubaclassifier', '-f', 'twitter_streaming_lang.json'): pass with pushd('twitter_streaming_location'): with run_server('jubaclassifier', '-f', 'twitter_streaming_location.json'): pass with pushd('movielens'): with run_server('jubarecommender', '-f', 'config.json'): with pushd('python'): run('python', 'ml_update.py') run('python', 'ml_analysis.py') with pushd('npb_similar_player'): with run_server('jubarecommender', '-f', 'npb_similar_player.json'): run('python', 'update.py')