def create_pr(repo, from_branch, to_branch): print(f"Creating PR for {from_branch}") utils.subprocess_check_output([ "gh", "pr", "create", "-R", repo, "--base", to_branch, "--head", from_branch, "--title", "Update CSV framework coverage reports", "--body", "This PR changes the CSV framework coverage reports." ])
def get_previous_run_id(repo, run_id, pr_number): """ Gets the previous run id for a given workflow run, considering that the previous workflow run needs to come from the same PR. """ # Get branch and repo from run: this_run = utils.subprocess_check_output([ "gh", "api", "-X", "GET", f"repos/{repo}/actions/runs/{run_id}", "--jq", "{ head_branch: .head_branch, head_repository: .head_repository.full_name }" ]) this_run = json.loads(this_run) pr_branch = this_run["head_branch"] pr_repo = this_run["head_repository"] # Get all previous runs that match branch, repo and workflow name: output = utils.subprocess_check_output([ "gh", "api", "-X", "GET", f"repos/{repo}/actions/runs", "-f", "event=pull_request", "-f", "status=success", "-f", f"branch='{pr_branch}'", "--paginate", "--jq", f'[.workflow_runs.[] | select(.head_repository.full_name=="{pr_repo}" and .name=="{artifacts_workflow_name}")] | sort_by(.id) | reverse | [.[].id]' ]) ids = [] for l in [json.loads(l) for l in output.splitlines()]: for id in l: ids.append(id) if ids[0] != int(run_id): raise Exception( f"Expected to find {run_id} in the list of matching runs.") for previous_run_id in ids[1:]: utils.download_artifact(repo, "pr", "prev_run_pr", previous_run_id) try: with open("prev_run_pr/NR") as file: prev_pr_number = int(file.read()) print(f"PR number: {prev_pr_number}") finally: if os.path.isdir("prev_run_pr"): shutil.rmtree("prev_run_pr") # the previous run needs to be coming from the same PR: if pr_number == prev_pr_number: return int(previous_run_id) raise Exception("Couldn't find previous run.")
def split_video(video_path, folder_path, splits): """ video_path => full path to video \n folder_path => full path to folder \n splits => array of dictionaries containing keys of start and duration which are used for splitting """ basename = os.path.split(video_path)[-1] extension = ''.join(pathlib.Path(basename).suffixes) videoname = basename.split('.')[0] split_video = ['ffmpeg', '-nostdin', '-y', '-i', video_path] for i, split in enumerate(splits): file_name = f'{videoname}-{i}{extension}' file_path = os.path.join(folder_path, file_name) split_video += ['-ss', split['start'], '-t', split['duration'], '-codec', 'copy', file_path] ret, failed = subprocess_check_output(split_video) outputpath = os.path.join(folder_path, f'output{extension}') ret, failed = subprocess_check_output(split_video) return ret, failed
def combine_videos(output_path, folder_path): files = [] for file in os.listdir(folder_path): file_name = os.path.join(folder_path, file) extension = ''.join(pathlib.Path(file_name).suffixes) if 'mp4' in extension: files.append(os.path.join(folder_path, file)) files = natsorted(files) print(files) ffmpeg_txt = os.path.join(folder_path, 'ffmpeg_split.txt') with open(ffmpeg_txt, 'w') as tf: for file in files: tf.write(f'file {file}\n') join_videos = ['ffmpeg', '-y', '-f', 'concat', '-safe', '0', '-i', ffmpeg_txt, '-c', 'copy', output_path] ret, failed = subprocess_check_output(join_videos) return ret, failed
def get_pr_number(repo, owner, from_branch, to_branch): ids = utils.subprocess_check_output([ "gh", "api", "-X", "GET", f"repos/{repo}/pulls", "-f", f"name={to_branch}", "-f", f"head={owner}:{from_branch}", "--jq", "[.[].number]" ]) ids = json.loads(ids) if len(ids) > 1: print(f"Found more than one PR that matches the branches. {ids}", file=sys.stderr) sys.exit(1) if len(ids) == 1: print(f"Matching PR found: {ids[0]}") return ids[0] return 0
def get_output(arr): r = utils.subprocess_check_output(arr) return r.strip("\n'")
working_dir = "" if len(sys.argv) > 1: working_dir = sys.argv[1] else: print("Working directory is not specified") exit(1) found_diff = False overwrite_files() os.chdir(working_dir) already_open_pr = get_pr_number(repo, owner, branch_name, main) try: utils.subprocess_check_output(["git", "diff", "--exit-code"]) print("No differences found") found_diff = False except: print("Differences found") found_diff = True if not found_diff: if already_open_pr != 0: # close the PR utils.subprocess_run( ["gh", "pr", "close", str(already_open_pr), "-R", repo]) else: utils.subprocess_run(["git", "config", "user.name", "github-actions[bot]"]) utils.subprocess_run([
print("====================================== warning: no notes for " + str(univ2lect)) else: print('skipping notes') if TRIM_TRACKING is True: if len(fielddat['tracking_data']) > 4: in_file = fielddat['tracking_data'] dl_path = f'{tmpdir}/{in_file}' full_key = f'{univ2lect_key}/{in_file}' s3_download_file(bucket, full_key, dl_path) subprocess_check_output([ 'python3', os.path.join(thispath, 'trim_tracking_json.py'), tmpdir + fielddat['tracking_data'], '-ss', str(args.trimstart), '-te', str(args.trimend) ]) out_file = fielddat['tracking_data'][:-len('.json')] + '_trimmed.json' out_path = f'{tmpdir}/{out_file}' assert os.path.isfile(out_path), out_path getorpostcontent( 'tracking_data', fielddat['tracking_data'][:-len('.json')] + '_trimmed.json') full_key = f'{univ2lect_key}/{out_file}' s3_upload_file(bucket, full_key, out_path) else: print(
def download_and_trim_contours(bucket:str, key: str, localpath: str, trimstart:float=-1., trimend:float=-1., trimduration:float=-1., \ outdir:str='', refilter:bool=False, pickle_params:str=''): s3_download_folder(bucket, key, localpath) if trimend < 0. and trimduration < 0.: trimend = int(1e9) trimduration = int(1e9) else: assert not (trimend > 0. and trimduration > 0.), 'use one or the other' if trimduration > 0.: trimend = trimstart + trimduration else: trimduration = trimend - trimstart assert trimduration > 0., str(trimend)+', '+str(trimstart) tsfileend = '_timestamps.png' #---------------------- tmpdir = localpath while tmpdir.endswith('/'): tmpdir = tmpdir[:-1] subprocess_call(['mkdir','-p', tmpdir]) #---------------------- basecf = localpath.split('/')[-1] backupbaseblockfold = copy(basecf) if len(outdir) < 1: ival = None if '_' in basecf: try: ival = int(basecf.split('_')[-1]) except ValueError: pass if isinstance(ival,int): ival += 1 basecf = '_'.join(basecf.split('_')[:-1])+'_'+str(ival) # TODO: s3 head to check if this folder exists already print("ayyy: "+str(ival)) else: basecf += '_0' outdir = basecf assert '/' not in outdir, str(outdir) print("outdir: "+str(outdir)) localoutdir = os.path.join(os.path.dirname(tmpdir), outdir) subprocess_call(['mkdir','-p', localoutdir]) univ2lect = '/'.join(key.split('/')[:-2]) full_key = f'{univ2lect}/{outdir}' print("localoutdir: "+str(localoutdir)) print("s3 out key: "+str(full_key)) pngs = [os.path.join(tmpdir,ff) for ff in os.listdir(tmpdir) if ff.endswith(tsfileend)] assert len(pngs) > 0, str(pngs)+'\n'+str(list(os.listdir(tmpdir))) subtrme = int(round(float(trimstart))) maxtime = int(round(float(trimend ))) syncme = [] for pngf in pngs: img = read_timestamps_image(pngf) assert isinstance(img,np.ndarray), str(type(img)) assert len(img.shape) == 2, str(img.shape) describe(os.path.basename(pngf), img) if np.amin(img) > int(round(trimend)): continue # TODO: if we know erase times, remove contours which were erased before "trimstart" img[img<subtrme] = subtrme img[img>maxtime] = maxtime # TODO: this is needlessly destructive; but fixes potential frontend bugs (seek past end of video?) img -= subtrme writer = png.Writer(width=img.shape[1], height=img.shape[0], greyscale=True, alpha=False, bitdepth=16, compression=5) outfname = os.path.join(localoutdir,os.path.basename(pngf)) syncme.append(outfname) with open(outfname, 'wb') as openedfile: writer.write(openedfile, img) for fpth in syncme: oldimgf = os.path.join(os.path.dirname(fpth), os.path.basename(fpth)[:-len(tsfileend)]) oldfs = [os.path.join(tmpdir,ff) for ff in os.listdir(tmpdir) if not ff.endswith(tsfileend) and ff.startswith(os.path.basename(oldimgf))] assert len(oldfs) == 1, str(oldimgf)+'\n'+str(oldfs) oldfs = oldfs[0] subprocess_check_output(['cp', oldfs, localoutdir+'/']) metafiles = [os.path.join(tmpdir,ff) for ff in os.listdir(tmpdir) if ff.lower().startswith('meta') and ff.lower().endswith('.json')] for ff in metafiles: print("in meta.json, replacing \'"+str(backupbaseblockfold)+"\' with \'"+str(outdir)+"\'") replace_folder_in_metafile(ff, localoutdir+'/'+os.path.basename(ff), backupbaseblockfold, outdir) # TODO: ADD THIS FUNCTIONALITY BACK test_it = False if refilter and test_it is True: filterfile = '/evt/interactive-writing-segmentation/filter_keyframes.py' assert os.path.isfile(filterfile), filterfile fargs = ['python',filterfile,localoutdir,'--were_transparency_on_s3','--overwrite_in_place'] if len(pickle_params) > 1: fargs += ['--autorun','--pickle_params',pickle_params] assert 0 == subprocess_call(fargs) print("syncing resulting folder") while full_key.endswith('/'): full_key = full_key[:-1] s3_upload_folder(bucket, full_key, localoutdir) return outdir