def encode_video(experiment_name, video_name):
# TODO: security lol
dirname = '~/results/{}'.format(experiment_name)
input_name = os.path.join(dirname, video_name)
output_name = input_name.replace('.mjpeg', '.mp4')
with pushd(dirname):
os.system('ffmpeg -y -i {} {}'.format(input_name, output_name))
os.system('rm {}'.format(input_name))
def test_main(self, mock_clone, mock_post):
try:
readmesfix.main('test.tsv')
self.assertTrue(mock_clone.called)
self.assertEqual(2, mock_post.call_count)
with tempfile.TemporaryDirectory() as temp_dir, util.pushd(
temp_dir):
repo = git.Repo.clone_from(TEST_REPO_ABSOLUTE_PATH, '.')
diff = subprocess.run(['git', 'diff', 'HEAD~'],
check=True,
stdout=subprocess.PIPE).stdout
repo.git.reset('--hard', 'HEAD~')
repo.remotes['origin'].push(force=True)
with open('test.diff', 'rb') as gold_diff_file:
self.assertEqual(gold_diff_file.read(), diff)
finally:
with util.pushd('test-repo.git'):
subprocess.run(
'git -c gc.reflogExpire=0 -c gc.reflogExpireUnreachable=0 -c gc.rerereresolved=0 '
'-c gc.rerereunresolved=0 -c gc.pruneExpire=now gc "$@"',
shell=True,
check=True,
stdout=subprocess.PIPE)
def build(
self,
namespace: argparse.Namespace,
image: str,
build_config: config.ImageBuildConfig):
'''
:param namespace: Namespace passed in via CLI.
:param image: The image to build.
:param build_config: the image build configuration.
:raises: subprocess.CalledProcessError
'''
LOGGER.info('Build image')
LOGGER.info('Dry Run: %s', namespace.dry_run)
image_dir = util.get_image_dir(namespace.path, image)
image = build_config.image.full_name
build_args = builder.get_build_args(build_config)
command = util.Command(['buildah', 'bud', '-t'])
command.add_arg(image)
command.add_args_list('--build-arg', build_args)
version = build_config.image.tag_build.version
if version:
command.add_args('--build-arg', f'VERSION={version}')
# add build context
command.add_arg('.')
LOGGER.info('Image name: %s', image)
LOGGER.info('Command: %s', ' '.join(command))
if namespace.dry_run:
return
# build
with util.pushd(image_dir):
subprocess.check_call(command)
image_tag = build_config.image.tag
if namespace.tag_latest and image_tag != 'latest':
self.tag_latest(image)
if namespace.push:
self.push(
namespace=namespace,
image=image)
def iterate_targets(root):
"""Iterate over the targets generated based on root directory tree"""
root = os.path.abspath(root)
logging.info('sourcedir=%s', root)
target_rules_filename = Settings.get('targets')['filename']
for path, files in util.walk(root):
with util.pushd(os.path.join(root, path)):
relpath = Path.clean(path.replace(root, ''))
logging.info('Parsing folder: $sourcedir/%s', relpath)
if not files:
logging.debug('No files found')
elif target_rules_filename in files:
target_rules_filepath = os.path.join(path, target_rules_filename)
target_rules = util.load_yaml_or_json(target_rules_filepath)
for gyp_target in target_rules.get('targets', []):
yield Target(relpath, files, gyp_target)
else:
yield Target(relpath, files)
def build(self, namespace: argparse.Namespace, image: str,
build_config: config.ImageBuildConfig):
'''
:param namespace: Namespace passed in via CLI.
:param image: The image to build.
:param build_config: the image build configuration.
:raises: subprocess.CalledProcessError
'''
LOGGER.info('Build image')
image_dir = util.get_image_dir(namespace.path, image)
image = build_config.image.full_name
build_args = builder.get_build_args(build_config)
version = build_config.image.tag_build.version
if version:
build_args.append(f'VERSION={version}')
command = self._create_command(namespace=namespace,
action='build',
build_args=build_args)
image_names_output = [image]
image_tag = build_config.image.tag
if namespace.tag_latest and image_tag != 'latest':
image_latest = util.set_image_tag_latest(image=image)
image_names_output.append(image_latest)
names_output = ','.join([f'name={i}' for i in image_names_output])
command.add_args(
name='--output',
value=f'type=image,{names_output},push={namespace.push}')
LOGGER.info('Image name: %s', image)
LOGGER.info('Command: %s', ' '.join(command))
if namespace.dry_run:
return
with util.pushd(image_dir):
subprocess.check_call(command)
#!/usr/bin/env python
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('shogun'):
with run_server('jubaclassifier', '-f', 'shogun.json'):
run('ruby', 'ruby/shogun.rb')
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
run('ruby', 'ruby/gender.rb')
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('ruby'):
run('ruby', 'ml_update.rb')
run('ruby', 'ml_analysis.rb')
with pushd('npb_similar_player'):
with run_server('jubarecommender', '-f', 'npb_similar_player.json'):
run('ruby', 'update.rb')
run('ruby', 'analyze.rb')
with pushd('rent'):
with run_server('jubaregression', '-f', 'rent.json'):
run('ruby', 'ruby/train.rb', 'dat/rent-data.csv')
run('ruby', 'ruby/test.rb', input='19.9\n2\n22\n4\nW\n')
with pushd('language_detection'):
#!/usr/bin/env python
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('shogun'):
with run_server('jubaclassifier', '-f', 'shogun.json'):
with pushd('java'):
run('mvn', 'clean')
run('sh', 'run.sh')
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
with pushd('java'):
run('mvn', 'clean')
run('sh', 'run.sh')
with pushd('twitter_streaming_location'):
with run_server('jubaclassifier', '-f', 'twitter_streaming_location.json'):
pass
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('java'):
run('mvn', 'clean')
run('sh', 'run-update.sh')
run('sh', 'run-analyze.sh')
#!/usr/bin/env python
import os
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
with pushd('cpp'):
run('make')
run('./gender')
with pushd('malware_classification'):
pass
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('cpp'):
run('python', 'waf', 'configure')
run('python', 'waf')
run('./build/ml_update')
run('./build/ml_analysis')
def main(dataset_path):
global inside_code_block, last_valid_fence
with open(dataset_path) as file:
number_of_lines = sum(1 for _ in file)
file.seek(0)
for (repo_name, ) in tqdm.tqdm(csv.reader(file),
total=number_of_lines):
with tempfile.TemporaryDirectory() as temp_dir, util.pushd(
temp_dir):
# noinspection PyBroadException
try:
repo = git.Repo.clone_from(
f'[email protected]:{repo_name}.git',
'.',
depth=1,
origin='upstream')
markdown_paths = set(insensitive_glob('**/*.md', recursive=True)) \
| set(insensitive_glob('**/*.mkdn?', recursive=True)) \
| set(insensitive_glob('**/*.mdown', recursive=True)) \
| set(insensitive_glob('**/*.markdown', recursive=True))
markdown_paths = {
path
for path in markdown_paths if os.path.isfile(path)
}
if markdown_paths: # Gets stuck otherwise
paths_with_crlf = crlf_paths(markdown_paths)
with fileinput.input(markdown_paths,
inplace=True) as markdown_file:
use_crlf = False
for line in markdown_file:
if fileinput.isfirstline():
inside_code_block = False
last_valid_fence = None
use_crlf = markdown_file.filename(
) in paths_with_crlf
if use_crlf and line and line[-1] == '\n':
line = line[:-1] + '\r\n'
CODE_BLOCK_FENCE_BACK_TICKS_RE.sub(
detect_code_block_back_ticks_fence, line)
CODE_BLOCK_FENCE_TILDES_RE.sub(
detect_code_block_tildes_fence, line)
print(HEADING_WITHOUT_SPACE_RE.sub(
heading_fix, line),
end='')
if repo.index.diff(None):
repo.git.add('.')
repo.git.commit(m="Fix broken Markdown headings")
response = requests.post(
f'https://api.github.com/repos/{repo_name}/forks',
params=AUTH_PARAMS)
response_dict = json.loads(response.text)
if response.status_code == 202:
repo.create_remote(
'origin', response_dict['ssh_url']).push()
create_pr(
repo_name, response_dict["default_branch"],
f'{response_dict["owner"]["login"]}:{response_dict["default_branch"]}'
)
else:
print(
f"There was an error forking {repo_name}: {response_dict}"
)
except Exception:
print(traceback.format_exc())
#!/usr/bin/env python
import os
from util import pushd, run, example_root
with pushd(example_root):
with pushd('movielens'):
if not os.path.exists('dat/ml-100k'):
run('mkdir', '-p', 'dat')
with pushd('dat'):
run('wget', 'http://www.grouplens.org/system/files/ml-100k.zip')
run('unzip', 'ml-100k.zip')
#!/usr/bin/env python
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('shogun'):
with run_server('jubaclassifier', '-f', 'shogun.json'):
with pushd('ruby'):
run('ruby', 'shogun.rb')
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
with pushd('ruby'):
run('ruby', 'gender.rb')
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('ruby'):
run('ruby', 'ml_update.rb')
run('ruby', 'ml_analysis.rb')
with pushd('npb_similar_player'):
with run_server('jubarecommender', '-f', 'npb_similar_player.json'):
with pushd('ruby'):
run('ruby', 'update.rb')
run('ruby', 'analyze.rb')
with pushd('rent'):
with run_server('jubaregression', '-f', 'rent.json'):
with pushd('ruby'):
#!/usr/bin/env python
import os
from util import pushd, run, example_root
with pushd(example_root):
with pushd('movielens'):
if not os.path.exists('dat/ml-100k'):
run('mkdir', '-p', 'dat')
with pushd('dat'):
run('wget',
'http://www.grouplens.org/system/files/ml-100k.zip')
run('unzip', 'ml-100k.zip')
#!/usr/bin/env python
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('shogun'):
with run_server('jubaclassifier', '-f', 'shogun.json'):
with pushd('python'):
run('python', 'shogun.py')
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
with pushd('python'):
run('python', 'gender.py')
with pushd('twitter_streaming_lang'):
with run_server('jubaclassifier', '-f', 'twitter_streaming_lang.json'):
pass
with pushd('twitter_streaming_location'):
with run_server('jubaclassifier', '-f',
'twitter_streaming_location.json'):
pass
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('python'):
run('python', 'ml_update.py')
run('python', 'ml_analysis.py')
#!/usr/bin/env python
from util import pushd, run_server, run, example_root
with pushd(example_root):
with pushd('shogun'):
with run_server('jubaclassifier', '-f', 'shogun.json'):
run('python', 'python/shogun.py')
with pushd('gender'):
with run_server('jubaclassifier', '-f', 'gender.json'):
run('python', 'python/gender.py')
with pushd('twitter_streaming_lang'):
with run_server('jubaclassifier', '-f', 'twitter_streaming_lang.json'):
pass
with pushd('twitter_streaming_location'):
with run_server('jubaclassifier', '-f', 'twitter_streaming_location.json'):
pass
with pushd('movielens'):
with run_server('jubarecommender', '-f', 'config.json'):
with pushd('python'):
run('python', 'ml_update.py')
run('python', 'ml_analysis.py')
with pushd('npb_similar_player'):
with run_server('jubarecommender', '-f', 'npb_similar_player.json'):
run('python', 'update.py')
