async def collect_output(self, token: Token, output_dir: Text) -> Token: if isinstance(token.job, MutableSequence): return token.update(await asyncio.gather(*[ asyncio.create_task(self.collect_output(t, output_dir)) for t in token.value ])) elif isinstance(token.value, MutableSequence): token_list = await asyncio.gather(*[ asyncio.create_task( self.collect_output(token.update(t), output_dir)) for t in token.value ]) return token.update([t.value for t in token_list]) context = self.port.step.context path_processor = utils.get_path_processor(self.port.step) src_path = token.value src_job = context.scheduler.get_job(token.job) dest_path = os.path.join(output_dir, path_processor.basename(src_path)) # Transfer file to local destination await self.port.step.context.data_manager.transfer_data( src=src_path, src_job=src_job, dst=dest_path, dst_job=None, writable=True) # Update token return token.update(dest_path)
async def update_token(self, job: Job, token: Token) -> Token: if isinstance(token.job, MutableSequence): return await super().update_token(job, token) if token.value is None and self.default_value is not None: token = token.update(await self._build_token_value( job, self.default_value)) if self.port_type == 'Any' or self.port_type is None: if (isinstance(self.port, InputPort) and isinstance( self.port.dependee.token_processor, CWLTokenProcessor) and self.port.dependee.token_processor.port_type != 'Any' and self.port.dependee.token_processor.port_type is not None): self.port_type = self.port.dependee.token_processor.port_type else: self.port_type = utils.infer_type_from_token(token.value) if isinstance(token.value, MutableMapping) and token.value.get('class') in [ 'File', 'Directory' ]: context = self.get_context() src_job = context.scheduler.get_job(token.job) if isinstance(token.value, MutableSequence): elements = [] for element in token.value: elements.append(await self._update_file_token( job, src_job, element)) return token.update(elements) elif token.value is not None: return token.update(await self._update_file_token( job, src_job, token.value)) else: return token else: return token.update(await self._build_token_value(job, token.value))
async def collect_output(self, token: Token, output_dir: Text) -> Token: if isinstance(token.job, MutableSequence): return token.update(await asyncio.gather(*[ asyncio.create_task( self.collect_output( t if isinstance(t, Token) else token.update(t), output_dir)) for t in token.value ])) else: return token
async def _recover_token(self, job: Job, resources: MutableSequence[Text], token: Token) -> Token: if isinstance(token.value, MutableSequence): elements = [] for t in token.value: elements.append(await self._recover_token_value( job, resources, token, t)) return token.update(elements) else: return token.update(await self._recover_token_value( job, resources, token, token.value))
async def collect_output(self, token: Token, output_dir: Text) -> Token: if isinstance(token.job, MutableSequence): return await super().collect_output(token, output_dir) self._check_list(token.value) token_tasks = [] for i, v in enumerate(token.value): token_tasks.append( asyncio.create_task( self.processor.collect_output(token.update(v), output_dir))) return token.update( [t.value for t in await asyncio.gather(*token_tasks)])
async def update_token(self, job: Job, token: Token) -> Token: if isinstance(token.job, MutableSequence): return await super().update_token(job, token) self._check_list(token.value) token_tasks = [] for i, processor in enumerate(self.processors): if i < len(token.value): partial_token = token.update(token.value[i]) token_tasks.append( asyncio.create_task( processor.update_token(job, partial_token))) return token.update( [t.value for t in await asyncio.gather(*token_tasks)])
async def collect_output(self, token: Token, output_dir: Text) -> Token: if isinstance(token.job, MutableSequence): return await super().collect_output(token, output_dir) self._check_list(token.value) output_tasks = [] for i, processor in enumerate(self.processors): if i < len(token.value): partial_token = token.update(token.value[i]) output_tasks.append( asyncio.create_task( processor.collect_output(partial_token, output_dir))) return token.update( [t.value for t in await asyncio.gather(*output_tasks)])
async def collect_output(self, token: Token, output_dir: Text) -> Token: if isinstance(token.job, MutableSequence): return await super().collect_output(token, output_dir) self._check_dict(token.value) output_tasks = [] for key, processor in self.processors.items(): if key in token.value: partial_token = token.update(token.value[key]) output_tasks.append( asyncio.create_task( processor.collect_output(partial_token, output_dir))) return token.update( dict( zip(token.value.keys(), [t.value for t in await asyncio.gather(*output_tasks)])))
async def update_token(self, job: Job, token: Token) -> Token: if isinstance(token.job, MutableSequence): update_tasks = [] for t in token.value: update_tasks.append( asyncio.create_task(self.processor.update_token(job, t))) return token.update(await asyncio.gather(*update_tasks)) self._check_list(token.value) token_tasks = [] for i, v in enumerate(token.value): token_tasks.append( asyncio.create_task( self.processor.update_token(job, token.update(v)))) return token.update( [t.value for t in await asyncio.gather(*token_tasks)])
async def update_token(self, job: Job, token: Token) -> Token: if isinstance(token.job, MutableSequence): return await super().update_token(job, token) self._check_dict(token.value) token_tasks = {} for key, processor in self.processors.items(): if key in token.value: partial_token = token.update(token.value[key]) token_tasks[key] = asyncio.create_task( processor.update_token(job, partial_token)) return token.update( dict( zip(token_tasks.keys(), [ t.value for t in await asyncio.gather(*token_tasks.values()) ])))
def get_related_resources(self, token: Token) -> Set[Text]: self._check_list(token.value) related_resources = set() for v in token.value: related_resources.update( self.processor.get_related_resources(token.update(v))) return related_resources
async def update_token(self, job: Job, token: Token) -> Token: if isinstance(token.job, MutableSequence): return token.update(await asyncio.gather(*[ asyncio.create_task(self.update_token(job, t)) for t in token.value ])) else: return token
async def recover_token(self, job: Job, resources: MutableSequence[Text], token: Token) -> Token: if isinstance(token.job, MutableSequence): return token.update(await asyncio.gather(*[ asyncio.create_task(self.recover_token(job, resources, t)) for t in token.value ])) return token
def get_related_resources(self, token: Token) -> Set[Text]: self._check_dict(token.value) related_resources = set() for key, processor in self.processors.items(): if key in token.value: partial_token = token.update(token.value[key]) related_resources.update( processor.get_related_resources(partial_token)) return related_resources
def get_related_resources(self, token: Token) -> Set[Text]: self._check_list(token.value) related_resources = set() for i, processor in enumerate(self.processors): if i < len(token.value): partial_token = token.update(token.value[i]) related_resources.update( processor.get_related_resources(partial_token)) return related_resources
async def collect_output(self, token: Token, output_dir: Text) -> Token: if isinstance(token.job, MutableSequence): return await super().collect_output(token, output_dir) return token.update( executor.postload(compiler=self.compiler, name=self.port.name, value=([dill.loads(v) for v in token.value] if isinstance(token.value, MutableSequence) else dill.loads(token.value)), serializer=self.serializer))
async def update_token(self, job: Job, token: Token) -> Token: if isinstance(token.job, MutableSequence): return token.update(await asyncio.gather(*[ asyncio.create_task(self.update_token(job, t)) for t in token.value ])) elif isinstance(token.value, MutableSequence): token_list = await asyncio.gather(*[ asyncio.create_task(self.update_token(job, token.update(t))) for t in token.value ]) return token.update([t.value for t in token_list]) src_job = self.get_context().scheduler.get_job(token.job) path_processor = utils.get_path_processor(self.port.step) token_value = dill.loads(token.value) if isinstance( token.value, bytes) else token.value dest_path = path_processor.join(job.input_directory, os.path.basename(token_value)) await self.port.step.context.data_manager.transfer_data( src=token_value, src_job=src_job, dst=dest_path, dst_job=job) return token.update(dill.dumps(dest_path))
def get_related_resources(self, token: Token) -> Set[Text]: if isinstance(token.job, MutableSequence) or isinstance( token.value, MutableSequence): resources = set() for t in token.value: resources.update( self.get_related_resources(t if isinstance( token.job, MutableSequence) else token.update(t))) context = self.port.step.context src_job = context.scheduler.get_job(token.job) resources = set(src_job.get_resources() if src_job else []) data_locations = set() for resource in resources: data_locations.update( context.data_manager.get_data_locations(resource, token.value)) resources.update({loc.resource for loc in data_locations}) return resources
async def collect_output(self, token: Token, output_dir: Text) -> Token: if isinstance(token.job, MutableSequence) or self.port_type not in [ 'File', 'Directory' ]: return await super().collect_output(token, output_dir) if token.value is not None and self.port_type in ['File', 'Directory']: context = self.get_context() output_collector = BaseJob(name=random_name(), step=BaseStep(name=random_name(), context=context), inputs=[], input_directory=output_dir) return token.update(await self._update_file_token( job=output_collector, src_job=context.scheduler.get_job(token.job), token_value=token.value, load_listing=LoadListing.deep_listing, writable=True)) else: return token
async def _replace_token(job: Job, token_processor: TokenProcessor, old_token: Token, new_token: Token): if isinstance(old_token.job, MutableSequence): token_value = [] if isinstance(token_processor, ListTokenProcessor): for (t, tp) in zip(old_token.value, token_processor.processors): token_value.append(await _replace_token(job, tp, t, new_token)) elif isinstance(token_processor, MapTokenProcessor): for t in old_token.value: token_value.append(await _replace_token(job, token_processor.processor, t, new_token)) else: for t in old_token.value: token_value.append(await _replace_token(job, token_processor, t, new_token)) return old_token.update(token_value) elif new_token.job == old_token.job: return await token_processor.update_token(job, new_token) else: return old_token
async def update_token(self, job: Job, token: Token) -> Token: if token.value is None: token = token.update(self.default_value) return await super().update_token(job, token)