def evaluate(args):
device = torch.device("cuda" if args.cuda else "cpu")
preprocessor = torch.load(args.preprocessor_file)
loader_config = dict(
preprocessor=preprocessor,
batch_size=args.batch_size,
device=device,
)
eval_dataloader = create_dataloader(args.eval_file, **loader_config, shuffle=False)
checkpoint = torch.load(args.checkpoint_file)
model = build_model(
word_vocab_size=len(preprocessor.vocabs["word"]),
pretrained_word_vocab_size=len(preprocessor.vocabs["pretrained_word"]),
postag_vocab_size=len(preprocessor.vocabs["postag"]),
n_deprels=len(preprocessor.vocabs["deprel"]),
)
model.load_state_dict(checkpoint["model"])
model.to(device)
trainer = create_trainer(model)
trainer.add_callback(utils.training.PrintCallback(printer=logger.info))
deprel_map = {v: k for k, v in preprocessor.vocabs["deprel"].mapping.items()}
trainer.add_callback(EvaluateCallback(args.eval_file, deprel_map, args.verbose), priority=0)
with logging_redirect_tqdm(loggers=[logger]):
trainer.evaluate(eval_dataloader)
def build(ctx: click.Context, directory: str, zenodo: bool, no_strict: bool, force: bool):
"""Build all databases."""
# if no_strict and zenodo:
# click.secho("Must be strict before uploading", fg="red")
# sys.exit(1)
with logging_redirect_tqdm():
click.secho("Collecting metadata and building", fg="cyan", bold=True)
# note that this is the only one that needs a force=force
ctx.invoke(metadata, directory=directory, no_strict=no_strict, force=force)
click.secho("Alternate Identifiers", fg="cyan", bold=True)
ctx.invoke(alts, directory=directory, zenodo=zenodo, no_strict=no_strict)
click.secho("Synonyms", fg="cyan", bold=True)
ctx.invoke(synonyms, directory=directory, zenodo=zenodo, no_strict=no_strict)
click.secho("Xrefs", fg="cyan", bold=True)
ctx.invoke(xrefs, directory=directory, zenodo=zenodo, no_strict=no_strict)
click.secho("Names", fg="cyan", bold=True)
ctx.invoke(names, directory=directory, zenodo=zenodo, no_strict=no_strict)
click.secho("Definitions", fg="cyan", bold=True)
ctx.invoke(definitions, directory=directory, zenodo=zenodo, no_strict=no_strict)
click.secho("Properties", fg="cyan", bold=True)
ctx.invoke(properties, directory=directory, zenodo=zenodo, no_strict=no_strict)
click.secho("Relations", fg="cyan", bold=True)
ctx.invoke(relations, directory=directory, zenodo=zenodo, no_strict=no_strict)
click.secho("Typedefs", fg="cyan", bold=True)
ctx.invoke(typedefs, directory=directory, zenodo=zenodo, no_strict=no_strict)
click.secho("Species", fg="cyan", bold=True)
ctx.invoke(species, directory=directory, zenodo=zenodo, no_strict=no_strict)
def test_should_inherit_console_logger_formatter(self):
logger = logging.Logger('test')
formatter = logging.Formatter('custom: %(message)s')
console_handler = logging.StreamHandler(sys.stderr)
console_handler.setFormatter(formatter)
logger.handlers = [console_handler]
with logging_redirect_tqdm(loggers=[logger]):
assert logger.handlers[0].formatter == formatter
def test_should_not_remove_stream_handlers_not_for_stdout_or_stderr(self):
logger = logging.Logger('test')
stream_handler = logging.StreamHandler(StringIO())
logger.addHandler(stream_handler)
with logging_redirect_tqdm(loggers=[logger]):
assert len(logger.handlers) == 2
assert logger.handlers[0] == stream_handler
assert isinstance(logger.handlers[1], TqdmLoggingHandler)
assert logger.handlers == [stream_handler]
def test_should_remove_and_restore_console_handlers(self):
logger = logging.Logger('test')
stderr_console_handler = logging.StreamHandler(sys.stderr)
stdout_console_handler = logging.StreamHandler(sys.stderr)
logger.handlers = [stderr_console_handler, stdout_console_handler]
with logging_redirect_tqdm(loggers=[logger]):
assert len(logger.handlers) == 1
assert isinstance(logger.handlers[0], TqdmLoggingHandler)
assert logger.handlers == [
stderr_console_handler, stdout_console_handler
]
def species(directory: str, zenodo: bool, no_strict: bool, force: bool):
"""Make the prefix-identifier-species dump."""
with logging_redirect_tqdm():
paths = db_output_helper(
_iter_species,
"species",
("prefix", "identifier", "species"),
strict=not no_strict,
force=force,
directory=directory,
)
if zenodo:
# see https://zenodo.org/record/5334738
update_zenodo(SPECIES_RECORD, paths)
async def run(self, py_path, wait=False):
response = await self.send_command_and_get_response(
"program_modechange", {"mode": "download"}
)
with open(py_path, "rb") as demo:
program = demo.read()
chunk_size = 512
chunks = [
program[i : i + chunk_size] for i in range(0, len(program), chunk_size)
]
while response is None or "transferid" not in response:
response = await self.send_command_and_get_response(
"start_write_program",
{
"meta": {
"created": 0,
"modified": 0,
"project_id": "Pybricksdev_",
"project_id": "Pybricksdev_",
"name": "Pybricksdev_____",
"type": "python",
},
"size": len(program),
"slotid": 0,
},
)
transferid = response["transferid"]
with logging_redirect_tqdm(), tqdm(
total=len(program), unit="B", unit_scale=True
) as pbar:
for chunk in chunks:
response = await self.send_command_and_get_response(
"write_package",
{
"data": base64.b64encode(chunk).decode("ascii"),
"transferid": transferid,
},
)
pbar.update(len(chunk))
await asyncio.sleep(0.5)
response = await self.send_command_and_get_response(
"program_execute", {"slotid": 0}
)
print(response)
def synonyms(directory: str, zenodo: bool, force: bool, no_strict: bool):
"""Make the prefix-identifier-synonym dump."""
with logging_redirect_tqdm():
paths = db_output_helper(
_iter_synonyms,
"synonyms",
("prefix", "identifier", "synonym"),
directory=directory,
force=force,
strict=not no_strict,
skip_set={"kegg.pathway", "kegg.genes", "kegg.genome"},
)
if zenodo:
# see https://zenodo.org/record/4021482
update_zenodo(SYNONYMS_RECORD, paths)
def properties(directory: str, zenodo: bool, force: bool, no_strict: bool):
"""Make the properties dump."""
with logging_redirect_tqdm():
paths = db_output_helper(
_iter_properties,
"properties",
("prefix", "identifier", "property", "value"),
directory=directory,
force=force,
strict=not no_strict,
summary_detailed=(0, 2), # second column corresponds to property type
)
if zenodo:
# see https://zenodo.org/record/4625172
update_zenodo(PROPERTIES_RECORD, paths)
def alts(directory: str, zenodo: bool, force: bool, no_strict: bool):
"""Make the prefix-alt-id dump."""
with logging_redirect_tqdm():
paths = db_output_helper(
_iter_alts,
"alts",
("prefix", "identifier", "alt"),
directory=directory,
force=force,
strict=not no_strict,
skip_set={"kegg.pathway", "kegg.genes", "kegg.genome", "umls"},
)
if zenodo:
# see https://zenodo.org/record/4021476
update_zenodo(ALTS_DATA_RECORD, paths)
def definitions(directory: str, zenodo: bool, no_strict: bool, force: bool):
"""Make the prefix-identifier-definition dump."""
with logging_redirect_tqdm():
paths = db_output_helper(
_iter_definitions,
"definitions",
("prefix", "identifier", "definition"),
strict=not no_strict,
force=force,
directory=directory,
skip_set={"kegg.pathway", "kegg.genes", "kegg.genome", "umls"},
)
if zenodo:
# see https://zenodo.org/record/4637061
update_zenodo(DEFINITIONS_RECORD, paths)
def xrefs(directory: str, zenodo: bool, force: bool, no_strict: bool): # noqa: D202
"""Make the prefix-identifier-xref dump."""
with logging_redirect_tqdm():
paths = db_output_helper(
_iter_xrefs,
"xrefs",
("prefix", "identifier", "xref_prefix", "xref_identifier", "provenance"),
directory=directory,
force=force,
strict=not no_strict,
summary_detailed=(0, 2), # second column corresponds to xref prefix
)
if zenodo:
# see https://zenodo.org/record/4021477
update_zenodo(JAVERT_RECORD, paths)
def typedefs(directory: str, zenodo: bool, no_strict: bool, force: bool):
"""Make the typedef prefix-identifier-name dump."""
with logging_redirect_tqdm():
paths = db_output_helper(
_iter_typedefs,
"typedefs",
("prefix", "typedef_prefix", "identifier", "name"),
strict=not no_strict,
force=force,
directory=directory,
use_gzip=False,
skip_set={"ncbigene", "kegg.pathway", "kegg.genes", "kegg.genome"},
)
if zenodo:
# see https://zenodo.org/record/4644013
update_zenodo(TYPEDEFS_RECORD, paths)
def train(args):
if args.seed is not None:
utils.random.seed_everything(args.seed)
device = torch.device("cuda" if args.cuda else "cpu")
preprocessor = Preprocessor()
preprocessor.build_vocab(args.train_file, cache_dir=args.cache_dir)
if args.embed_file:
preprocessor.load_embeddings(args.embed_file, cache_dir=args.cache_dir)
loader_config = dict(
preprocessor=preprocessor,
batch_size=args.batch_size,
device=device,
cache_dir=args.cache_dir,
)
train_dataloader = create_dataloader(args.train_file, **loader_config, shuffle=True)
eval_dataloader = None
if args.eval_file:
eval_dataloader = create_dataloader(args.eval_file, **loader_config, shuffle=False)
model = build_model(
word_vocab_size=len(preprocessor.vocabs["word"]),
pretrained_word_vocab_size=len(preprocessor.vocabs["pretrained_word"]),
postag_vocab_size=len(preprocessor.vocabs["postag"]),
pretrained_word_embeddings=preprocessor.pretrained_word_embeddings,
n_deprels=len(preprocessor.vocabs["deprel"]),
)
model.to(device)
trainer = create_trainer(
model, lr=args.learning_rate, max_steps=args.max_steps, eval_interval=args.eval_interval
)
trainer.add_callback(utils.training.PrintCallback(printer=logger.info))
if eval_dataloader:
deprel_map = {v: k for k, v in preprocessor.vocabs["deprel"].mapping.items()}
trainer.add_callback(EvaluateCallback(args.eval_file, deprel_map), priority=0)
if args.save_dir:
torch.save(preprocessor, os.path.join(args.save_dir, "preprocessor.pt"))
trainer.add_callback(
utils.training.SaveCallback(args.save_dir, monitor="eval/UAS", mode="max")
)
with logging_redirect_tqdm(loggers=[logger]):
trainer.fit(train_dataloader, eval_dataloader)
async def run(self, py_path, wait=True, print_output=True):
"""Run a Pybricks MicroPython script on the hub and print output.
Arguments:
py_path (str):
Path to MicroPython script.
wait (bool):
Whether to wait for any output until the program completes.
print_output(bool):
Whether to print the standard output.
"""
# Reset output buffer
self.log_file = None
self.output = []
self.print_output = print_output
# Compile the script to mpy format
mpy = await compile_file(py_path)
# Get length of file and send it as bytes to hub
length = len(mpy).to_bytes(4, byteorder='little')
await self.send_message(length)
# Divide script in chunks of bytes
n = 100
chunks = [mpy[i:i + n] for i in range(0, len(mpy), n)]
# Send the data chunk by chunk
with logging_redirect_tqdm(), tqdm(total=len(mpy),
unit='B',
unit_scale=True) as pbar:
for chunk in chunks:
await self.send_message(chunk)
pbar.update(len(chunk))
# Optionally wait for the program to finish
if wait:
await asyncio.sleep(0.2)
await self.wait_until_state_is_not(self.RUNNING)
def relations(directory: str, zenodo: bool, force: bool, no_strict: bool):
"""Make the relation dump."""
with logging_redirect_tqdm():
paths = db_output_helper(
_iter_relations,
"relations",
(
"source_prefix",
"source_identifier",
"relation_prefix",
"relation_identifier",
"target_prefix",
"target_identifier",
),
directory=directory,
force=force,
strict=not no_strict,
summary_detailed=(0, 2, 3), # second column corresponds to relation type
)
if zenodo:
# see https://zenodo.org/record/4625167
update_zenodo(RELATIONS_RECORD, paths)
async def _download_queue(
queue: asyncio.Queue,
session: RetryClient,
stats: dict,
params: DownloadParams,
progressbar: tqdm_asyncio = None,
logger: logging.Logger = None,
):
"""Consumes items from download queue
Args:
queue (asyncio.Queue): Queue of items
session (RetryClient): RetryClient aiohttp session object
params (DownloadParams): Download parameter dict
logger (logging.Logger): Logger object
"""
while True:
batch = await queue.get()
for sample in batch:
failed = False
try:
success = await download_single(sample, session, params)
except Exception as e:
with logging_redirect_tqdm(loggers=[logger]):
logger.error(e.request_info.url,
extra={"status": e.status})
failed = True
if failed:
stats["failed"] += 1
elif not success:
stats["skipped"] += 1
else:
stats["success"] += 1
progressbar.set_postfix(stats=stats, refresh=True)
progressbar.update(1)
queue.task_done()
def names(
directory: str,
zenodo: bool,
no_strict: bool,
force: bool,
skip_below: Optional[str],
skip_below_exclusive: bool,
):
"""Make the prefix-identifier-name dump."""
with logging_redirect_tqdm():
paths = db_output_helper(
_iter_names,
"names",
("prefix", "identifier", "name"),
strict=not no_strict,
force=force,
directory=directory,
skip_below=skip_below,
skip_below_inclusive=not skip_below_exclusive,
)
if zenodo:
# see https://zenodo.org/record/4020486
update_zenodo(OOH_NA_NA_RECORD, paths)
async def run(self, py_path, wait=True, print_output=True):
# Reset output buffer
self.log_file = None
self.output = []
self.print_output = print_output
# Compile the script to mpy format
self.script_dir, _ = os.path.split(py_path)
mpy = await compile_file(py_path)
try:
self.loading = True
self.user_program_stopped.clear()
# Get length of file and send it as bytes to hub
length = len(mpy).to_bytes(4, byteorder='little')
await self.send_block(length)
# Divide script in chunks of bytes
n = 100
chunks = [mpy[i:i + n] for i in range(0, len(mpy), n)]
# Send the data chunk by chunk
with logging_redirect_tqdm(), tqdm(total=len(mpy),
unit='B',
unit_scale=True) as pbar:
for chunk in chunks:
await self.send_block(chunk)
pbar.update(len(chunk))
finally:
self.loading = False
if wait:
await self.user_program_stopped.wait()
await asyncio.sleep(0.3)
def update_database():
with logging_redirect_tqdm():
install_mp_handler()
latest_update = Update.get_latest_update(success=False)
if latest_update.status not in [
Update.Status.ERROR, Update.Status.SUCCESS
]:
print(
'Last update revision=%s is still not ended (or hang or crashed). '
'Would you like to start new update (y) continue previous (N)?'
% latest_update.id)
response = input()
if response.lower() == 'y':
latest_update.status = 'error'
latest_update.save()
latest_update = Update.objects.create(
status=Update.Status.IN_PROGRESS)
else:
latest_update = Update.objects.create(
status=Update.Status.IN_PROGRESS)
# scraping level 3 indexes first
level_3_koatuu = list(
set([koatuu for koatuu in _get_indexes() if koatuu.level <= 2]))
# sort unique ids then (stable sort, so level is still sorted)
level_3_koatuu.sort(key=attrgetter('unique_id'))
# sort level from 1 to 3 keeping stable unique_id
level_3_koatuu.sort(key=attrgetter('level'))
if latest_update.latest_koatuu:
logging.info("Searching for the latest koatuu scraped")
latest_koatuu_obj = next(
(koatuu for koatuu in level_3_koatuu
if koatuu.unique_id == latest_update.latest_koatuu), None)
if latest_koatuu_obj is None:
level_3_koatuu = []
else:
logging.info("Found latest koatuu scraped %s",
latest_koatuu_obj)
level_3_koatuu = level_3_koatuu[level_3_koatuu.
index(latest_koatuu_obj):]
logging.info('Koatuu to scrape only %s', len(level_3_koatuu))
if level_3_koatuu:
_download_and_insert(latest_update, level_3_koatuu)
logging.info('All insert l1 operations ended')
# process level 4 indexes only for regions where parcels
# number is more than 100000
annotated = Landuse.objects.all().values('koatuu').filter(
revision=latest_update.id).annotate(
total=Count('koatuu')).order_by('-total')
level_4_koatuu = []
all_koatuu = list(set([koatuu for koatuu in _get_indexes()]))
for result in annotated:
if result['total'] < 100000:
continue
koatuu_obj = next(koatuu for koatuu in all_koatuu
if koatuu.unique_id == str(result['koatuu']))
if koatuu_obj.level == 2:
level_3_koatuus = [
*set([
koatuu for koatuu in all_koatuu if koatuu.level == 3
and str(koatuu.parent) == koatuu_obj.unique_id
])
]
level_4_koatuu.extend(level_3_koatuus)
for level_3_koatuu in level_3_koatuus:
level_4_koatuu.extend([
*set([
koatuu
for koatuu in all_koatuu if koatuu.level == 4
and str(koatuu.parent) == level_3_koatuu.unique_id
])
])
if koatuu_obj.level == 3:
level_4_koatuu.extend([
*set([
koatuu for koatuu in all_koatuu if koatuu.level == 4
and str(koatuu.parent) == koatuu_obj.unique_id
])
])
level_4_koatuu.sort(key=attrgetter('unique_id'))
_download_and_insert(latest_update, level_4_koatuu)
# detecting changes to create analysis table
create_changeset(
revision=Update.objects.get(id=latest_update.id),
previous=Update.objects.get(id=Update.get_latest_update().id),
)
# everything is ok => success status
Update.objects.filter(id=latest_update.id).update(
status=Update.Status.SUCCESS)
def bidsparticipants(rawfolder: str, bidsfolder: str, keys: str, bidsmapfile: str='bidsmap.yaml', dryrun: bool=False) -> None:
"""
Main function that processes all the subjects and session in the sourcefolder to (re)generate the particpants.tsv file in the BIDS folder.
:param rawfolder: The root folder-name of the sub/ses/data/file tree containing the source data files
:param bidsfolder: The name of the BIDS root folder
:param keys: The keys that are extracted fro mthe source data when populating the participants.tsv file
:param bidsmapfile: The name of the bidsmap YAML-file. If the bidsmap pathname is relative (i.e. no "/" in the name) then it is assumed to be located in bidsfolder/code/bidscoin
:param dryrun: Boolean to just display the participants info
:return: Nothing
"""
# Input checking & defaults
rawfolder = Path(rawfolder).resolve()
bidsfolder = Path(bidsfolder).resolve()
# Start logging
if dryrun:
bidscoin.setup_logging()
else:
bidscoin.setup_logging(bidsfolder/'code'/'bidscoin'/'bidsparticipants.log')
LOGGER.info('')
LOGGER.info(f"-------------- START bidsparticipants {bidscoin.version()} ------------")
LOGGER.info(f">>> bidsparticipants sourcefolder={rawfolder} bidsfolder={bidsfolder} bidsmap={bidsmapfile}")
# Get the bidsmap sub-/ses-prefix from the bidsmap YAML-file
bidsmap,_ = bids.load_bidsmap(Path(bidsmapfile), bidsfolder/'code'/'bidscoin')
subprefix = bidsmap['Options']['bidscoin']['subprefix']
sesprefix = bidsmap['Options']['bidscoin']['sesprefix']
# Get the table & dictionary of the subjects that have been processed
participants_tsv = bidsfolder/'participants.tsv'
participants_json = participants_tsv.with_suffix('.json')
if participants_tsv.is_file():
participants_table = pd.read_csv(participants_tsv, sep='\t')
participants_table.set_index(['participant_id'], verify_integrity=True, inplace=True)
else:
participants_table = pd.DataFrame()
participants_table.index.name = 'participant_id'
if participants_json.is_file():
with participants_json.open('r') as json_fid:
participants_dict = json.load(json_fid)
else:
participants_dict = {'participant_id': {'Description': 'Unique participant identifier'}}
# Get the list of subjects
subjects = bidscoin.lsdirs(bidsfolder, 'sub-*')
if not subjects:
LOGGER.warning(f"No subjects found in: {bidsfolder}")
# Remove obsolete participants from the participants table
for participant in participants_table.index:
if participant not in [sub.name for sub in subjects]:
participants_table = participants_table.drop(participant)
# Loop over all subjects in the bids-folder and add them to the participants table
with logging_redirect_tqdm():
for n, subject in enumerate(tqdm(subjects, unit='subject', leave=False), 1):
LOGGER.info(f"------------------- Subject {n}/{len(subjects)} -------------------")
personals = dict()
subject = rawfolder/subject.name.replace('sub-', subprefix.replace('*','')) # TODO: This assumes e.g. that the subject-ids in the rawfolder did not contain BIDS-invalid characters (such as '_')
sessions = bidscoin.lsdirs(subject, (sesprefix if sesprefix!='*' else '') + '*')
if not subject.is_dir():
LOGGER.error(f"Could not find source-folder: {subject}")
continue
if not sessions:
sessions = [subject]
for session in sessions:
# Only take data from the first session -> BIDS specification
subid, sesid = bids.DataSource(session/'dum.my', subprefix='sub-', sesprefix='ses-').subid_sesid()
if sesprefix and sesid and 'session_id' not in personals:
personals['session_id'] = sesid
participants_dict['session_id'] = {'Description': 'Session identifier'}
# Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file
sesfolders, unpacked = bids.unpack(session)
for sesfolder in sesfolders:
# Update / append the personal source data
LOGGER.info(f"Scanning session: {sesfolder}")
success = scanpersonals(bidsmap, sesfolder, personals)
# Clean-up the temporary unpacked data
if unpacked:
shutil.rmtree(sesfolder)
if success:
break
# Store the collected personals in the participant_table. TODO: Check that only values that are consistent over sessions go in the participants.tsv file, otherwise put them in a sessions.tsv file
for key in keys:
if key not in participants_dict:
participants_dict[key] = dict(LongName = 'Long (unabbreviated) name of the column',
Description = 'Description of the the column',
Levels = dict(Key='Value (This is for categorical variables: a dictionary of possible values (keys) and their descriptions (values))'),
Units = 'Measurement units. [<prefix symbol>]<unit symbol> format following the SI standard is RECOMMENDED')
participants_table.loc[subid, key] = personals.get(key)
# Write the collected data to the participant files
LOGGER.info(f"Writing subject data to: {participants_tsv}")
if not dryrun:
participants_table.replace('','n/a').to_csv(participants_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
LOGGER.info(f"Writing subject data dictionary to: {participants_json}")
if not dryrun:
with participants_json.open('w') as json_fid:
json.dump(participants_dict, json_fid, indent=4)
print(participants_table)
LOGGER.info('-------------- FINISHED! ------------')
LOGGER.info('')
bidscoin.reporterrors()
def analyze(
self,
detector, # fer.FER instance
display: bool = False,
output: str = "csv",
frequency: Optional[int] = None,
max_results: int = None,
save_fps: Optional[int] = None,
video_id: Optional[str] = None,
save_frames: bool = True,
save_video: bool = True,
annotate_frames: bool = True,
zip_images: bool = True,
detection_box: Optional[dict] = None,
) -> list:
"""Recognize facial expressions in video using `detector`.
Args:
detector (fer.FER): facial expression recognizer
display (bool): show images with cv2.imshow
output (str): csv or pandas
frequency (int): inference on every nth frame (higher number is faster)
max_results (int): number of frames to run inference before stopping
save_fps (bool): inference frequency = video fps // save_fps
video_id (str): filename for saving
save_frames (bool): saves frames to directory
save_video (bool): saves output video
annotate_frames (bool): add emotion labels
zip_images (bool): compress output
detection_box (dict): dict with bounding box for subimage (xmin, xmax, ymin, ymax)
Returns:
data (list): list of results
"""
frames_emotions = []
if frequency is None:
frequency = 1
else:
frequency = int(frequency)
self.display = display
self.save_frames = save_frames
self.save_video = save_video
self.annotate_frames = annotate_frames
results_nr = 0
# Open video
assert self.cap.open(self.filepath), "Video capture not opening"
self.__emotions = detector._get_labels().items()
self.cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
pos_frames = self.cap.get(cv2.CAP_PROP_POS_FRAMES)
assert int(pos_frames) == 0, "Video not at index 0"
self.frameCount = 0
height, width = (
int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)),
int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)),
)
fps = self.cap.get(cv2.CAP_PROP_FPS)
length = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT))
assert fps and length, "File {} not loaded".format(self.filepath)
if save_fps is not None:
frequency = fps // save_fps
log.info("Saving every {} frames".format(frequency))
log.info("{:.2f} fps, {} frames, {:.2f} seconds".format(
fps, length, length / fps))
if self.save_frames:
os.makedirs(self.outdir, exist_ok=True)
log.info(f"Making directories at {self.outdir}")
root, ext = os.path.splitext(os.path.basename(self.filepath))
outfile = os.path.join(self.outdir, f"{root}_output{ext}")
if save_video:
self.videowriter = self._save_video(outfile, fps, width, height)
with logging_redirect_tqdm():
pbar = tqdm(total=length, unit="frames")
while self.cap.isOpened():
ret, frame = self.cap.read()
if not ret: # end of video
break
if frame is None:
log.warn("Empty frame")
continue
if self.frameCount % frequency != 0:
self.frameCount += 1
continue
if detection_box is not None:
frame = self._crop(frame, detection_box)
# Get faces and detect emotions; coordinates are for unpadded frame
try:
faces = detector.detect_emotions(frame)
except Exception as e:
log.error(e)
break
# Offset detection_box to include padding
if detection_box is not None:
faces = self._offset_detection_box(faces, detection_box)
self._increment_frames(frame, faces, video_id, root)
if cv2.waitKey(1) & 0xFF == ord("q"):
break
if faces:
frames_emotions.append(faces)
results_nr += 1
if max_results and results_nr > max_results:
break
pbar.update(1)
pbar.close()
self._close_video(outfile, save_frames, zip_images)
return self.to_format(frames_emotions, output)
def bidsmapper(rawfolder: str, bidsfolder: str, bidsmapfile: str, templatefile: str, plugins: list, subprefix: str, sesprefix: str, store: bool=False, noedit: bool=False, force: bool=False) -> None:
"""
Main function that processes all the subjects and session in the sourcefolder
and that generates a maximally filled-in bidsmap.yaml file in bidsfolder/code/bidscoin.
Folders in sourcefolder are assumed to contain a single dataset.
:param rawfolder: The root folder-name of the sub/ses/data/file tree containing the source data files
:param bidsfolder: The name of the BIDS root folder
:param bidsmapfile: The name of the bidsmap YAML-file
:param templatefile: The name of the bidsmap template YAML-file
:param plugins: Optional list of plugins that should be used (overrules the list in the study/template bidsmaps)
:param subprefix: The prefix common for all source subject-folders
:param sesprefix: The prefix common for all source session-folders
:param store: If True, the provenance samples will be stored
:param noedit: The bidseditor will not be launched if True
:param force: If True, the previous bidsmap and logfiles will be deleted
:return:
"""
# Input checking
rawfolder = Path(rawfolder).resolve()
bidsfolder = Path(bidsfolder).resolve()
bidsmapfile = Path(bidsmapfile)
templatefile = Path(templatefile)
bidscoinfolder = bidsfolder/'code'/'bidscoin'
# Start logging
if force:
(bidscoinfolder/'bidsmapper.log').unlink(missing_ok=True)
bidscoin.setup_logging(bidscoinfolder/'bidsmapper.log')
LOGGER.info('')
LOGGER.info('-------------- START BIDSmapper ------------')
LOGGER.info(f">>> bidsmapper sourcefolder={rawfolder} bidsfolder={bidsfolder} bidsmap={bidsmapfile} "
f"template={templatefile} plugins={plugins} subprefix={subprefix} sesprefix={sesprefix} store={store} force={force}")
# Get the heuristics for filling the new bidsmap
bidsmap_old, bidsmapfile = bids.load_bidsmap(bidsmapfile, bidscoinfolder, plugins)
template, _ = bids.load_bidsmap(templatefile, bidscoinfolder, plugins)
# Create the new bidsmap as a copy / bidsmap skeleton with no datatype entries (i.e. bidsmap with empty lists)
if force:
bidsmapfile.unlink(missing_ok=True)
bidsmap_old = {}
if bidsmap_old:
bidsmap_new = copy.deepcopy(bidsmap_old)
else:
bidsmap_new = copy.deepcopy(template)
template['Options'] = bidsmap_new['Options'] # Always use the options of the new bidsmap
bidscoindatatypes = bidsmap_new['Options']['bidscoin'].get('datatypes',[])
unknowndatatypes = bidsmap_new['Options']['bidscoin'].get('unknowntypes',[])
ignoredatatypes = bidsmap_new['Options']['bidscoin'].get('ignoretypes',[])
for dataformat in bidsmap_new:
if dataformat in ('Options','PlugIns'): continue # Handle legacy bidsmaps (-> 'PlugIns')
for datatype in bidscoindatatypes + unknowndatatypes + ignoredatatypes:
if bidsmap_new[dataformat].get(datatype):
bidsmap_new[dataformat][datatype] = None
# Store/retrieve the empty or user-defined sub-/ses-prefix
subprefix, sesprefix = setprefix(bidsmap_new, subprefix, sesprefix)
# Start with an empty skeleton if we didn't have an old bidsmap
if not bidsmap_old:
bidsmap_old = copy.deepcopy(bidsmap_new)
bidsmapfile = bidscoinfolder/'bidsmap.yaml'
# Import the data scanning plugins
plugins = [bidscoin.import_plugin(plugin, ('bidsmapper_plugin',)) for plugin in bidsmap_new['Options']['plugins']]
plugins = [plugin for plugin in plugins if plugin] # Filter the empty items from the list
if not plugins:
LOGGER.warning(f"The plugins listed in your bidsmap['Options'] did not have a usable `bidsmapper_plugin` function, nothing to do")
LOGGER.info('-------------- FINISHED! ------------')
LOGGER.info('')
return
# Loop over all subjects and sessions and built up the bidsmap entries
subjects = bidscoin.lsdirs(rawfolder, (subprefix if subprefix!='*' else '') + '*')
if not subjects:
LOGGER.warning(f'No subjects found in: {rawfolder/subprefix}*')
with logging_redirect_tqdm():
for n, subject in enumerate(tqdm(subjects, unit='subject', leave=False), 1):
sessions = bidscoin.lsdirs(subject, (sesprefix if sesprefix!='*' else '') + '*')
if not sessions or (subject/'DICOMDIR').is_file():
sessions = [subject]
for session in sessions:
LOGGER.info(f"Mapping: {session} (subject {n}/{len(subjects)})")
# Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file
sesfolders, unpacked = bids.unpack(session)
for sesfolder in sesfolders:
if store:
store = {'source': sesfolder.parent.parent.parent.parent if unpacked else rawfolder.parent, 'target': bidscoinfolder/'provenance'}
else:
store = {}
# Run the bidsmapper plugins
for module in plugins:
LOGGER.info(f"Executing plugin: {Path(module.__file__).name} -> {sesfolder}")
module.bidsmapper_plugin(sesfolder, bidsmap_new, bidsmap_old, template, store)
# Clean-up the temporary unpacked data
if unpacked:
shutil.rmtree(sesfolder)
# Save the new study bidsmap in the bidscoinfolder or launch the bidseditor UI_MainWindow
if noedit:
bids.save_bidsmap(bidsmapfile, bidsmap_new)
else:
LOGGER.info('Opening the bidseditor')
app = QApplication(sys.argv)
app.setApplicationName(f"{bidsmapfile} - BIDS editor {localversion}")
mainwin = bidseditor.MainWindow(bidsfolder, bidsmap_new, template)
mainwin.show()
messagebox = QMessageBox(mainwin)
messagebox.setText(f"The bidsmapper has finished scanning {rawfolder}\n\n"
f"Please carefully check all the different BIDS output names "
f"and BIDScoin options and (re)edit them to your needs.\n\n"
f"You can always redo this step later by re-running the "
f"bidsmapper or by just running the bidseditor tool\n\n"
f"{versionmessage}")
messagebox.setWindowTitle('About the BIDS-mapping workflow')
messagebox.setIconPixmap(QtGui.QPixmap(str(bidseditor.BIDSCOIN_LOGO)).scaled(150, 150, QtCore.Qt.KeepAspectRatio, QtCore.Qt.SmoothTransformation))
messagebox.setWindowFlags(messagebox.windowFlags() & ~QtCore.Qt.WindowMinMaxButtonsHint)
messagebox.show()
app.exec()
LOGGER.info('-------------- FINISHED! -------------------')
LOGGER.info('')
bidscoin.reporterrors()
def update(force: bool):
"""Update the data file."""
with logging_redirect_tqdm():
_update(force=force)
def medeface(bidsdir: str, pattern: str, maskpattern: str, subjects: list,
force: bool, output: str, cluster: bool, nativespec: str,
kwargs: dict):
"""
:param bidsdir: The bids-directory with the (multi-echo) subject data
:param pattern: Globlike search pattern (relative to the subject/session folder) to select the echo-images that need to be defaced, e.g. 'anat/*_T1w*'
:param maskpattern: Globlike search pattern (relative to the subject/session folder) to select the images from which the defacemask is computed, e.g. 'anat/*_part-mag_*_T2starw*'. If not given then 'pattern' is used
:param subjects: List of sub-# identifiers to be processed (the sub- prefix can be left out). If not specified then all sub-folders in the bidsfolder will be processed
:param force: If True then images will be processed, regardless if images have already been defaced (i.e. if {"Defaced": True} in the json sidecar file)
:param output: Determines where the defaced images are saved. It can be the name of a BIDS datatype folder, such as 'anat', or of the derivatives folder, i.e. 'derivatives'. If output is left empty then the original images are replaced by the defaced images
:param cluster: Flag to submit the deface jobs to the high-performance compute (HPC) cluster
:param nativespec: DRMAA native specifications for submitting deface jobs to the HPC cluster
:param kwargs: Additional arguments (in dict/json-style) that are passed to pydeface. See examples for usage
:return:
"""
# Input checking
bidsdir = Path(bidsdir).resolve()
if not maskpattern:
maskpattern = pattern
# Start logging
bidscoin.setup_logging(bidsdir / 'code' / 'bidscoin' / 'deface.log')
LOGGER.info('')
LOGGER.info('------------ START multi-echo deface ----------')
LOGGER.info(
f">>> medeface bidsfolder={bidsdir} pattern={pattern} subjects={subjects} output={output}"
f" cluster={cluster} nativespec={nativespec} {kwargs}")
# Get the list of subjects
if not subjects:
subjects = bidscoin.lsdirs(bidsdir, 'sub-*')
if not subjects:
LOGGER.warning(f"No subjects found in: {bidsdir/'sub-*'}")
else:
subjects = [
'sub-' + subject.replace('sub-', '') for subject in subjects
] # Make sure there is a "sub-" prefix
subjects = [
bidsdir / subject for subject in subjects
if (bidsdir / subject).is_dir()
]
# Prepare the HPC pydeface job submission
with drmaa.Session() as pbatch:
if cluster:
jt = pbatch.createJobTemplate()
jt.jobEnvironment = os.environ
jt.remoteCommand = shutil.which('pydeface')
jt.nativeSpecification = nativespec
jt.joinFiles = True
# Loop over bids subject/session-directories to first get all the echo-combined deface masks
for n, subject in enumerate(subjects, 1):
sessions = bidscoin.lsdirs(subject, 'ses-*')
if not sessions:
sessions = [subject]
for session in sessions:
LOGGER.info('--------------------------------------')
LOGGER.info(f"Processing ({n}/{len(subjects)}): {session}")
datasource = bids.DataSource(session / 'dum.my',
subprefix='sub-',
sesprefix='ses-')
subid, sesid = datasource.subid_sesid()
# Read the echo-images that will be combined to compute the deface mask
echofiles = sorted([
match for match in session.glob(maskpattern)
if '.nii' in match.suffixes
])
if not echofiles:
LOGGER.info(
f'No mask files found for: {session}/{maskpattern}')
continue
# Check the json "Defaced" field to see if it has already been defaced
if not force:
with echofiles[0].with_suffix('').with_suffix(
'.json').open('r') as fid:
jsondata = json.load(fid)
if jsondata.get('Defaced'):
LOGGER.info(
f"Skipping already defaced images: {[str(echofile) for echofile in echofiles]}"
)
continue
LOGGER.info(
f'Loading mask files: {[str(echofile) for echofile in echofiles]}'
)
echos = [nib.load(echofile) for echofile in echofiles]
# Create a temporary echo-combined image
tmpfile = session / 'tmp_echocombined_deface.nii'
combined = nib.Nifti1Image(
np.mean([echo.get_fdata() for echo in echos], axis=0),
echos[0].affine, echos[0].header)
combined.to_filename(tmpfile)
# Deface the echo-combined image
LOGGER.info(
f"Creating a deface-mask from the echo-combined image: {tmpfile}"
)
if cluster:
jt.args = [
str(tmpfile), '--outfile',
str(tmpfile), '--force'
] + [
item for pair in [[f"--{key}", val]
for key, val in kwargs.items()]
for item in pair
]
jt.jobName = f"pydeface_{subid}_{sesid}"
jobid = pbatch.runJob(jt)
LOGGER.info(
f"Your deface job has been submitted with ID: {jobid}")
else:
pdu.deface_image(str(tmpfile),
str(tmpfile),
force=True,
forcecleanup=True,
**kwargs)
if cluster:
LOGGER.info('Waiting for the deface jobs to finish...')
pbatch.synchronize(jobIds=[pbatch.JOB_IDS_SESSION_ALL],
timeout=pbatch.TIMEOUT_WAIT_FOREVER,
dispose=True)
pbatch.deleteJobTemplate(jt)
# Loop again over bids subject/session-directories to apply the deface masks and write meta-data
with logging_redirect_tqdm():
for n, subject in enumerate(
tqdm(subjects, unit='subject', leave=False), 1):
sessions = bidscoin.lsdirs(subject, 'ses-*')
if not sessions:
sessions = [subject]
for session in sessions:
LOGGER.info('--------------------------------------')
LOGGER.info(f"Processing ({n}/{len(subjects)}): {session}")
datasource = bids.DataSource(session / 'dum.my',
subprefix='sub-',
sesprefix='ses-')
subid, sesid = datasource.subid_sesid()
# Read the temporary defacemask
tmpfile = session / 'tmp_echocombined_deface.nii'
if not tmpfile.is_file():
LOGGER.info(f'No {tmpfile} file found')
continue
defacemask = nib.load(tmpfile).get_fdata(
) != 0 # The original defacemask is saved in a temporary folder so it may be deleted -> use the defaced image to infer the mask
tmpfile.unlink()
# Process the echo-images that need to be defaced
for echofile in sorted([
match for match in session.glob(pattern)
if '.nii' in match.suffixes
]):
# Construct the output filename and relative path name (used in BIDS)
echofile_rel = echofile.relative_to(session).as_posix()
if not output:
outputfile = echofile
outputfile_rel = echofile_rel
elif output == 'derivatives':
outputfile = bidsdir / 'derivatives' / 'deface' / subid / sesid / echofile.parent.name / echofile.name
outputfile_rel = outputfile.relative_to(
bidsdir).as_posix()
else:
outputfile = session / output / echofile.name
outputfile_rel = outputfile.relative_to(
session).as_posix()
outputfile.parent.mkdir(parents=True, exist_ok=True)
# Apply the defacemask
LOGGER.info(
f'Applying deface mask on: {echofile} -> {outputfile_rel}'
)
echoimg = nib.load(echofile)
outputimg = nib.Nifti1Image(
echoimg.get_fdata() * defacemask, echoimg.affine,
echoimg.header)
outputimg.to_filename(outputfile)
# Overwrite or add a json sidecar-file
inputjson = echofile.with_suffix('').with_suffix('.json')
outputjson = outputfile.with_suffix('').with_suffix(
'.json')
if inputjson.is_file() and inputjson != outputjson:
if outputjson.is_file():
LOGGER.info(
f"Overwriting the json sidecar-file: {outputjson}"
)
outputjson.unlink()
else:
LOGGER.info(
f"Adding a json sidecar-file: {outputjson}")
shutil.copyfile(inputjson, outputjson)
# Add a custom "Defaced" field to the json sidecar-file
with outputjson.open('r') as output_fid:
data = json.load(output_fid)
data['Defaced'] = True
with outputjson.open('w') as output_fid:
json.dump(data, output_fid, indent=4)
# Update the IntendedFor fields in the fieldmap sidecar-files NB: IntendedFor must be relative to the subject folder
if output and output != 'derivatives' and (
session / 'fmap').is_dir():
for fmap in (session / 'fmap').glob('*.json'):
with fmap.open('r') as fmap_fid:
fmap_data = json.load(fmap_fid)
intendedfor = fmap_data['IntendedFor']
if isinstance(intendedfor, str):
intendedfor = [intendedfor]
if (Path(sesid) /
echofile_rel).as_posix() in intendedfor:
LOGGER.info(
f"Updating 'IntendedFor' to {Path(sesid)/outputfile_rel} in {fmap}"
)
fmap_data['IntendedFor'] = intendedfor + [
(Path(sesid) / outputfile_rel).as_posix()
]
with fmap.open('w') as fmap_fid:
json.dump(fmap_data, fmap_fid, indent=4)
# Update the scans.tsv file
if (bidsdir / '.bidsignore').is_file():
bidsignore = (bidsdir /
'.bidsignore').read_text().splitlines()
else:
bidsignore = []
bidsignore.append('derivatives/')
scans_tsv = session / f"{subid}{bids.add_prefix('_',sesid)}_scans.tsv"
if output and output + '/' not in bidsignore and scans_tsv.is_file(
):
LOGGER.info(f"Adding {outputfile_rel} to {scans_tsv}")
scans_table = pd.read_csv(scans_tsv,
sep='\t',
index_col='filename')
scans_table.loc[outputfile_rel] = scans_table.loc[
echofile_rel]
scans_table.sort_values(by=['acq_time', 'filename'],
inplace=True)
scans_table.to_csv(scans_tsv,
sep='\t',
encoding='utf-8')
LOGGER.info('-------------- FINISHED! -------------')
LOGGER.info('')
update_zenodo(PROPERTIES_RECORD, paths)
@main.command()
@verbose_option
@directory_option
@zenodo_option
@force_option
@no_strict_option
def xrefs(directory: str, zenodo: bool, force: bool, no_strict: bool): # noqa: D202
"""Make the prefix-identifier-xref dump."""
with logging_redirect_tqdm():
paths = db_output_helper(
_iter_xrefs,
"xrefs",
("prefix", "identifier", "xref_prefix", "xref_identifier", "provenance"),
directory=directory,
force=force,
strict=not no_strict,
summary_detailed=(0, 2), # second column corresponds to xref prefix
)
if zenodo:
# see https://zenodo.org/record/4021477
update_zenodo(JAVERT_RECORD, paths)
if __name__ == "__main__":
logging.captureWarnings(True)
with logging_redirect_tqdm():
main()
def echocombine(bidsdir: str, pattern: str, subjects: list, output: str, algorithm: str, weights: list, force: bool=False):
"""
:param bidsdir: The bids-directory with the (multi-echo) subject data
:param pattern: Globlike recursive search pattern (relative to the subject/session folder) to select the first echo of the images that need to be combined, e.g. '*task-*echo-1*'
:param subjects: List of sub-# identifiers to be processed (the sub- prefix can be left out). If not specified then all sub-folders in the bidsfolder will be processed
:param output: Determines where the output is saved. It can be the name of a BIDS datatype folder, such as 'func', or of the derivatives folder, i.e. 'derivatives'. If output = [the name of the input datatype folder] then the original echo images are replaced by one combined image. If output is left empty then the combined image is saved in the input datatype folder and the original echo images are moved to the {unknowndatatype} folder
:param algorithm: Combination algorithm, either 'PAID', 'TE' or 'average'
:param weights: Weights for each echo
:param force: Boolean to overwrite existing ME target files
:return:
"""
# Input checking
bidsdir = Path(bidsdir).resolve()
# Start logging
bidscoin.setup_logging(bidsdir/'code'/'bidscoin'/'echocombine.log')
LOGGER.info('')
LOGGER.info(f"--------- START echocombine ---------")
LOGGER.info(f">>> echocombine bidsfolder={bidsdir} pattern={pattern} subjects={subjects} output={output}"
f" algorithm={algorithm} weights={weights}")
# Get the list of subjects
if not subjects:
subjects = bidscoin.lsdirs(bidsdir, 'sub-*')
if not subjects:
LOGGER.warning(f"No subjects found in: {bidsdir/'sub-*'}")
else:
subjects = ['sub-' + subject.replace('sub-', '') for subject in subjects] # Make sure there is a "sub-" prefix
subjects = [bidsdir/subject for subject in subjects if (bidsdir/subject).is_dir()]
# Loop over bids subject/session-directories
with logging_redirect_tqdm():
for n, subject in enumerate(tqdm(subjects, unit='subject', leave=False), 1):
sessions = bidscoin.lsdirs(subject, 'ses-*')
if not sessions:
sessions = [subject]
for session in sessions:
LOGGER.info('-------------------------------------')
LOGGER.info(f"Combining echos for ({n}/{len(subjects)}): {session}")
subid, sesid = bids.DataSource(session/'dum.my', subprefix='sub-', sesprefix='ses-').subid_sesid()
# Search for multi-echo matches
for match in sorted([match for match in session.rglob(pattern) if '.nii' in match.suffixes]):
# Check if it is normal/BIDS multi-echo data or that the echo-number is appended to the acquisition label (as done in BIDScoin)
if '_echo-' in match.name:
echonr = bids.get_bidsvalue(match, 'echo')
mepattern = bids.get_bidsvalue(match, 'echo', '*') # The pattern that selects all echos
cename = match.name.replace(f"_echo-{echonr}", '') # The combined-echo output filename
elif '_acq-' in match.name and bids.get_bidsvalue(match, 'acq').split('e')[-1].isnumeric():
acq, echonr = bids.get_bidsvalue(match, 'acq').rsplit('e',1)
mepattern = bids.get_bidsvalue(match, 'acq', acq + 'e*') # The pattern that selects all echos
cename = match.name.replace(f"_acq-{acq}e{echonr}", f"_acq-{acq}") # The combined-echo output filename
LOGGER.info(f"No 'echo' key-value pair found in the filename, using the 'acq-{acq}e{echonr}' pair instead (BIDScoin-style)")
else:
LOGGER.warning(f"No 'echo' encoding found in the filename, skipping: {match}")
continue
echos = sorted(match.parent.glob(mepattern.name))
newechos = [echo.parents[1]/unknowndatatype/echo.name for echo in echos]
if len(echos) == 1:
LOGGER.warning(f"Only one echo image found, nothing to do for: {match}")
continue
# Construct the combined-echo output filename and check if that file already exists
datatype = match.parent.name
if not output:
cefile = session/datatype/cename
elif output == 'derivatives':
cefile = bidsdir/'derivatives'/'multiecho'/subid/sesid/datatype/cename
else:
cefile = session/output/cename
cefile.parent.mkdir(parents=True, exist_ok=True)
if cefile.is_file() and not force:
LOGGER.warning(f"Outputfile {cefile} already exists, skipping: {match}")
continue
# Combine the multi-echo images
me.me_combine(mepattern, cefile, algorithm, weights, saveweights=False)
# (Re)move the original multi-echo images
if not output:
for echo, newecho in zip(echos, newechos):
LOGGER.info(f"Moving original echo image: {echo} -> {newecho}")
newecho.parent.mkdir(parents=True, exist_ok=True)
echo.replace(newecho)
echo.with_suffix('').with_suffix('.json').replace(newecho.with_suffix('').with_suffix('.json'))
elif output == datatype:
for echo in echos:
LOGGER.info(f"Removing original echo image: {echo}")
echo.unlink()
echo.with_suffix('').with_suffix('.json').unlink()
# Construct the path names relative to the session folder (as in the scans.tsv file)
oldechos_rel = [echo.relative_to(session).as_posix() for echo in echos]
newechos_rel = [echo.relative_to(session).as_posix() for echo in echos + newechos if echo.is_file()]
if output == 'derivatives':
cefile_rel = '' # A remote folder cannot be specified as IntendedFor :-(
else:
cefile_rel = cefile.relative_to(session).as_posix()
# Update the IntendedFor fields of the fieldmaps (i.e. remove the old echos, add the echo-combined image and, optionally, the new echos)
if output != 'derivatives' and (session/'fmap').is_dir():
for fmap in (session/'fmap').glob('*.json'):
with fmap.open('r') as fmap_fid:
metadata = json.load(fmap_fid)
intendedfor = metadata.get('IntendedFor', [])
if isinstance(intendedfor, str):
intendedfor = [intendedfor]
if sesid: # NB: IntendedFor is relative to the subject folder
intendedfor = [file.split(sesid+'/',1)[1] for file in intendedfor]
if oldechos_rel[0] in intendedfor:
LOGGER.info(f"Updating 'IntendedFor' in {fmap}")
relfiles = [file for file in intendedfor if file not in oldechos_rel] + newechos_rel + [cefile_rel]
metadata['IntendedFor'] = [(Path(sesid)/relfile).as_posix() for relfile in relfiles]
with fmap.open('w') as fmap_fid:
json.dump(metadata, fmap_fid, indent=4)
# Update the scans.tsv file
if (bidsdir/'.bidsignore').is_file():
bidsignore = (bidsdir/'.bidsignore').read_text().splitlines()
else:
bidsignore = [unknowndatatype + '/']
scans_tsv = session/f"{subid}{bids.add_prefix('_', sesid)}_scans.tsv"
if scans_tsv.is_file():
scans_table = pd.read_csv(scans_tsv, sep='\t', index_col='filename')
if oldechos_rel[0] in scans_table.index:
scans_table.loc['oldrow'] = scans_table.loc[oldechos_rel[0]]
elif 'acq_time' in scans_table:
with cefile.with_suffix('').with_suffix('.json').open('r') as fid:
metadata = json.load(fid)
date = scans_table.iloc[0]['acq_time'].split('T')[0]
scans_table.loc['oldrow', 'acq_time'] = f"{date}T{metadata.get('AcquisitionTime')}"
else:
scans_table.loc['oldrow'] = None
if output+'/' not in bidsignore + ['derivatives/'] and cefile.parent.name in bids.bidsdatatypes:
LOGGER.info(f"Adding '{cefile_rel}' to '{scans_tsv}'")
scans_table.loc[cefile_rel] = scans_table.loc['oldrow']
for echo in oldechos_rel + newechos_rel:
if echo in scans_table.index and not (session/echo).is_file():
LOGGER.info(f"Removing '{echo}' from '{scans_tsv}'")
scans_table.drop(echo, inplace=True)
elif echo not in scans_table.index and (session/echo).is_file() and echo.split('/')[0] in bids.bidsdatatypes:
LOGGER.info(f"Adding '{echo}' to '{scans_tsv}'")
scans_table.loc[echo] = scans_table.loc['oldrow'] # NB: Assuming that the echo-rows are all identical
scans_table.drop('oldrow', inplace=True)
scans_table.sort_values(by=['acq_time','filename'], inplace=True)
scans_table.replace('','n/a').to_csv(scans_tsv, sep='\t', encoding='utf-8', na_rep='n/a')
for scan in scans_table.index:
if not (session/scan).is_file():
LOGGER.warning(f"Found non-existent file '{scan}' in '{scans_tsv}'")
LOGGER.info('-------------- FINISHED! -------------')
LOGGER.info('')
def cli(
input_csvs: List[TextIOWrapper],
out_dir: Path,
csdap_api_url: str,
username: str,
password: str,
verbosity: int,
concurrency: int,
scene_ids: List[str],
asset_types: List[str],
):
"""
The CSDAP Bulk Download tool intends to make it easy to download many
assets from an order placed within the CSDAP system.
\b
The Assets CSV must contain a header row with the following columns:
- collection_id
- scene_id
- asset_type
A user has the option to filter the csv file and only download a subset
of files based on scene_id or asset_type.
Note that a user is only granted access to download each file once.
For more information on CSDAP, please visit https://csdap.earthdata.nasa.gov.
For support, contact [email protected].
"""
setup_logger(verbosity)
csdap = CsdapClient(csdap_api_url)
token = csdap.get_auth_token(username, password)
with concurrent.futures.ThreadPoolExecutor(
max_workers=concurrency, thread_name_prefix="CsdapDownload"
) as executor, logging_redirect_tqdm():
logger.debug(
"Creating threadpool with max_workers of %s", executor._max_workers
)
future_to_path = {}
def log_results(future):
path = future_to_path.pop(future)
try:
logger.info("%s: %s", path, future.result())
except Exception as exc:
if verbosity:
logger.exception("%s generated an exception: %s" % (path, exc))
else:
logger.warn("%s: Failed to download", path)
for input_csv in input_csvs:
api_version = 2
for row in csv.DictReader(input_csv):
if "order_id" in row and api_version == 2:
logger.warn("Detected legacy CSV.")
api_version = 1
base = Path(
row["order_id"] if api_version == 1 else row["collection_id"]
)
path = base / row["scene_id"] / row["asset_type"]
# Filter rows
if scene_ids and row["scene_id"].lower() not in scene_ids:
logger.debug("Skipping %s, does not pass scene_id filter", path)
continue
if asset_types and row["asset_type"].lower() not in asset_types:
logger.debug("Skipping %s, does not pass asset_type filter", path)
continue
# Schedule work
future = executor.submit(
csdap.download_file,
path=path,
out_dir=out_dir,
token=token,
endpoint_version=api_version,
)
future_to_path[future] = path
future.add_done_callback(log_results)
# To avoid the memory overhead of scheduling the entire CSV as futures,
# we wait will for some futures to complete before scheduling more
if len(future_to_path) >= 2 * executor._max_workers:
logger.debug(
"Waiting for some downloads to finish before continuing to "
"process CSV rows..."
)
concurrent.futures.wait(
future_to_path, return_when=concurrent.futures.FIRST_COMPLETED
)
# Log outstanding futures
logger.debug(
"All CSVs processed, waiting for remaining %s downloads to complete",
len(future_to_path),
)
concurrent.futures.wait(
future_to_path, return_when=concurrent.futures.ALL_COMPLETED
)
click.echo("Complete.")
def find_decoding_function_features(
vw,
functions,
disable_progress=False) -> Tuple[Dict[int, Dict], Dict[int, str]]:
decoding_candidate_functions: DefaultDict[
int, Dict] = collections.defaultdict(dict)
library_functions: Dict[int, str] = dict()
pbar = tqdm.tqdm
if disable_progress:
logger.info("identifying decoding function features...")
# do not use tqdm to avoid unnecessary side effects when caller intends
# to disable progress completely
pbar = lambda s, *args, **kwargs: s
functions = sorted(functions)
n_funcs = len(functions)
pb = pbar(functions,
desc="finding decoding function features",
unit=" functions",
postfix="skipped 0 library functions")
with logging_redirect_tqdm(), redirecting_print_to_tqdm():
for f in pb:
function_address = int(f)
if is_thunk_function(vw, function_address):
continue
if viv_utils.flirt.is_library_function(vw, function_address):
# TODO handle j_j_j__free_base (lib function wrappers), e.g. 0x140035AF0 in d2ca76...
# TODO ignore function called to by library functions
function_name = viv_utils.get_function_name(
vw, function_address)
logger.debug("skipping library function 0x%x (%s)",
function_address, function_name)
library_functions[function_address] = function_name
n_libs = len(library_functions)
percentage = 100 * (n_libs / n_funcs)
if isinstance(pb, tqdm.tqdm):
pb.set_postfix_str("skipped %d library functions (%d%%)" %
(n_libs, percentage))
continue
f = viv_utils.Function(vw, function_address)
function_data = {"meta": get_function_meta(f), "features": list()}
# meta data features
function_data["features"].append(
BlockCount(function_data["meta"].get("block_count")))
function_data["features"].append(
InstructionCount(
function_data["meta"].get("instruction_count")))
function_data["features"].append(
Arguments(function_data["meta"].get("api",
[]).get("arguments")))
for feature in extract_function_features(f):
function_data["features"].append(feature)
for bb in f.basic_blocks:
for feature in extract_basic_block_features(f, bb):
function_data["features"].append(feature)
for insn in bb.instructions:
for feature in extract_insn_features(f, bb, insn):
function_data["features"].append(feature)
for feature in abstract_features(function_data["features"]):
function_data["features"].append(feature)
function_data["score"] = get_function_score_weighted(
function_data["features"])
logger.debug("analyzed function 0x%x - total score: %f",
function_address, function_data["score"])
for feat in function_data["features"]:
logger.trace(" %s", feat)
decoding_candidate_functions[function_address] = function_data
return decoding_candidate_functions, library_functions
def bidscoiner(rawfolder: str,
bidsfolder: str,
subjects: list = (),
force: bool = False,
participants: bool = False,
bidsmapfile: str = 'bidsmap.yaml') -> None:
"""
Main function that processes all the subjects and session in the sourcefolder and uses the
bidsmap.yaml file in bidsfolder/code/bidscoin to cast the data into the BIDS folder.
:param rawfolder: The root folder-name of the sub/ses/data/file tree containing the source data files
:param bidsfolder: The name of the BIDS root folder
:param subjects: List of selected subjects / participants (i.e. sub-# names / folders) to be processed (the sub- prefix can be removed). Otherwise all subjects in the sourcefolder will be selected
:param force: If True, subjects will be processed, regardless of existing folders in the bidsfolder. Otherwise existing folders will be skipped
:param participants: If True, subjects in particpants.tsv will not be processed (this could be used e.g. to protect these subjects from being reprocessed), also when force=True
:param bidsmapfile: The name of the bidsmap YAML-file. If the bidsmap pathname is relative (i.e. no "/" in the name) then it is assumed to be located in bidsfolder/code/bidscoin
:return: Nothing
"""
# Input checking & defaults
rawfolder = Path(rawfolder).resolve()
bidsfolder = Path(bidsfolder).resolve()
bidsmapfile = Path(bidsmapfile)
# Start logging
bidscoin.setup_logging(bidsfolder / 'code' / 'bidscoin' / 'bidscoiner.log')
LOGGER.info('')
LOGGER.info(
f"-------------- START BIDScoiner {localversion}: BIDS {bidscoin.bidsversion()} ------------"
)
LOGGER.info(
f">>> bidscoiner sourcefolder={rawfolder} bidsfolder={bidsfolder} subjects={subjects} force={force} participants={participants} bidsmap={bidsmapfile}"
)
# Create a code/bidscoin subfolder
(bidsfolder / 'code' / 'bidscoin').mkdir(parents=True, exist_ok=True)
# Create a dataset description file if it does not exist
dataset_file = bidsfolder / 'dataset_description.json'
generatedby = [{
"Name": "BIDScoin",
"Version": localversion,
"CodeURL": "https://github.com/Donders-Institute/bidscoin"
}]
if not dataset_file.is_file():
LOGGER.info(f"Creating dataset description file: {dataset_file}")
dataset_description = {
"Name":
"REQUIRED. Name of the dataset",
"GeneratedBy":
generatedby,
"BIDSVersion":
str(bidscoin.bidsversion()),
"DatasetType":
"raw",
"License":
"RECOMMENDED. The license for the dataset. The use of license name abbreviations is RECOMMENDED for specifying a license. The corresponding full license text MAY be specified in an additional LICENSE file",
"Authors": [
"OPTIONAL. List of individuals who contributed to the creation/curation of the dataset"
],
"Acknowledgements":
"OPTIONAL. Text acknowledging contributions of individuals or institutions beyond those listed in Authors or Funding",
"HowToAcknowledge":
"OPTIONAL. Instructions how researchers using this dataset should acknowledge the original authors. This field can also be used to define a publication that should be cited in publications that use the dataset",
"Funding":
["OPTIONAL. List of sources of funding (grant numbers)"],
"EthicsApprovals": [
"OPTIONAL. List of ethics committee approvals of the research protocols and/or protocol identifiers"
],
"ReferencesAndLinks": [
"OPTIONAL. List of references to publication that contain information on the dataset, or links",
"https://github.com/Donders-Institute/bidscoin"
],
"DatasetDOI":
"OPTIONAL. The Document Object Identifier of the dataset (not the corresponding paper)"
}
else:
with dataset_file.open('r') as fid:
dataset_description = json.load(fid)
if 'BIDScoin' not in [
generatedby_['Name']
for generatedby_ in dataset_description.get('GeneratedBy', [])
]:
LOGGER.info(f"Adding {generatedby} to {dataset_file}")
dataset_description['GeneratedBy'] = dataset_description.get(
'GeneratedBy', []) + generatedby
with dataset_file.open('w') as fid:
json.dump(dataset_description, fid, indent=4)
# Create a README file if it does not exist
readme_file = bidsfolder / 'README'
if not readme_file.is_file():
LOGGER.info(f"Creating README file: {readme_file}")
readme_file.write_text(
f"A free form text ( README ) describing the dataset in more details that SHOULD be provided\n\n"
f"The raw BIDS data was created using BIDScoin {localversion}\n"
f"All provenance information and settings can be found in ./code/bidscoin\n"
f"For more information see: https://github.com/Donders-Institute/bidscoin\n"
)
# Get the bidsmap heuristics from the bidsmap YAML-file
bidsmap, _ = bids.load_bidsmap(bidsmapfile,
bidsfolder / 'code' / 'bidscoin')
dataformats = [
dataformat for dataformat in bidsmap
if dataformat and dataformat not in ('Options', 'PlugIns')
] # Handle legacy bidsmaps (-> 'PlugIns')
if not bidsmap:
LOGGER.error(
f"No bidsmap file found in {bidsfolder}. Please run the bidsmapper first and/or use the correct bidsfolder"
)
return
# Load the data conversion plugins
plugins = [
bidscoin.import_plugin(plugin, ('bidscoiner_plugin', ))
for plugin, options in bidsmap['Options']['plugins'].items()
]
plugins = [plugin for plugin in plugins
if plugin] # Filter the empty items from the list
if not plugins:
LOGGER.warning(
f"The plugins listed in your bidsmap['Options'] did not have a usable `bidscoiner_plugin` function, nothing to do"
)
LOGGER.info('-------------- FINISHED! ------------')
LOGGER.info('')
return
# Append options to the .bidsignore file
bidsignore_items = [
item.strip()
for item in bidsmap['Options']['bidscoin']['bidsignore'].split(';')
]
bidsignore_file = bidsfolder / '.bidsignore'
if bidsignore_items:
LOGGER.info(f"Writing {bidsignore_items} entries to {bidsignore_file}")
if bidsignore_file.is_file():
bidsignore_items += bidsignore_file.read_text().splitlines()
with bidsignore_file.open('w') as bidsignore:
for item in set(bidsignore_items):
bidsignore.write(item + '\n')
# Get the table & dictionary of the subjects that have been processed
participants_tsv = bidsfolder / 'participants.tsv'
if participants_tsv.is_file():
participants_table = pd.read_csv(participants_tsv, sep='\t')
participants_table.set_index(['participant_id'],
verify_integrity=True,
inplace=True)
else:
participants_table = pd.DataFrame()
participants_table.index.name = 'participant_id'
# Get the list of subjects
subprefix = bidsmap['Options']['bidscoin']['subprefix'].replace('*', '')
sesprefix = bidsmap['Options']['bidscoin']['sesprefix'].replace('*', '')
if not subjects:
subjects = bidscoin.lsdirs(
rawfolder, (subprefix if subprefix != '*' else '') + '*')
if not subjects:
LOGGER.warning(f"No subjects found in: {rawfolder/subprefix}*")
else:
subjects = [
rawfolder / (subprefix + re.sub(f"^{subprefix}", '', subject))
for subject in subjects
] # Make sure there is a sub-prefix
# Loop over all subjects and sessions and convert them using the bidsmap entries
with logging_redirect_tqdm():
for n, subject in enumerate(
tqdm(subjects, unit='subject', leave=False), 1):
LOGGER.info(
f"------------------- Subject {n}/{len(subjects)} -------------------"
)
if participants and subject.name in list(participants_table.index):
LOGGER.info(
f"Skipping subject: {subject} ({n}/{len(subjects)})")
continue
if not subject.is_dir():
LOGGER.warning(
f"The '{subject}' subject folder does not exist")
continue
sessions = bidscoin.lsdirs(
subject, (sesprefix if sesprefix != '*' else '') + '*')
if not sessions or (subject / 'DICOMDIR').is_file():
sessions = [subject]
for session in sessions:
# Unpack the data in a temporary folder if it is tarballed/zipped and/or contains a DICOMDIR file
sesfolders, unpacked = bids.unpack(session)
for sesfolder in sesfolders:
# Check if we should skip the session-folder
datasource = bids.get_datasource(
sesfolder, bidsmap['Options']['plugins'])
if not datasource.dataformat:
LOGGER.info(
f"No coinable datasources found in '{sesfolder}'")
continue
subid = bidsmap[datasource.dataformat]['subject']
sesid = bidsmap[datasource.dataformat]['session']
subid, sesid = datasource.subid_sesid(
subid, sesid if sesid else '')
bidssession = bidsfolder / subid / sesid # TODO: Support DICOMDIR with multiple subjects (as in PYDICOMDIR)
if not force and bidssession.is_dir():
datatypes = []
for dataformat in dataformats:
for datatype in bidscoin.lsdirs(
bidssession
): # See what datatypes we already have in the bids session-folder
if datatype.iterdir(
) and bidsmap[dataformat].get(
datatype.name
): # See if we are going to add data for this datatype
datatypes.append(datatype.name)
if datatypes:
LOGGER.info(
f"Skipping processed session: {bidssession} already has {datatypes} data (you can carefully use the -f option to overrule)"
)
continue
LOGGER.info(f"Coining datasources in: {sesfolder}")
if bidssession.is_dir():
LOGGER.warning(
f"Existing BIDS output-directory found, which may result in duplicate data (with increased run-index). Make sure {bidssession} was cleaned-up from old data before (re)running the bidscoiner"
)
bidssession.mkdir(parents=True, exist_ok=True)
# Run the bidscoiner plugins
for module in plugins:
LOGGER.info(
f"Executing plugin: {Path(module.__file__).name}")
module.bidscoiner_plugin(sesfolder, bidsmap,
bidssession)
# Add the special fieldmap metadata (IntendedFor, B0FieldIdentifier, TE, etc)
addmetadata(bidssession, subid, sesid)
# Clean-up the temporary unpacked data
if unpacked:
shutil.rmtree(sesfolder)
# Re-read the participants_table and store the collected personals in the json sidecar-file
if participants_tsv.is_file():
participants_table = pd.read_csv(participants_tsv, sep='\t')
participants_table.set_index(['participant_id'],
verify_integrity=True,
inplace=True)
participants_json = participants_tsv.with_suffix('.json')
participants_dict = {}
if participants_json.is_file():
with participants_json.open('r') as json_fid:
participants_dict = json.load(json_fid)
if not participants_dict.get('participant_id'):
participants_dict['participant_id'] = {
'Description': 'Unique participant identifier'
}
if not participants_dict.get(
'session_id') and 'session_id' in participants_table.columns:
participants_dict['session_id'] = {'Description': 'Session identifier'}
newkey = False
for col in participants_table.columns:
if col not in participants_dict:
newkey = True
participants_dict[col] = dict(
LongName='Long (unabbreviated) name of the column',
Description='Description of the the column',
Levels=dict(
Key=
'Value (This is for categorical variables: a dictionary of possible values (keys) and their descriptions (values))'
),
Units=
'Measurement units. [<prefix symbol>]<unit symbol> format following the SI standard is RECOMMENDED'
)
# Write the collected data to the participant files
if newkey:
LOGGER.info(f"Writing subject meta data to: {participants_json}")
with participants_json.open('w') as json_fid:
json.dump(participants_dict, json_fid, indent=4)
LOGGER.info('-------------- FINISHED! ------------')
LOGGER.info('')
bidscoin.reporterrors()
