def display_number(v): # type: (int) -> Text if defs.OpSchema.is_infinite(v): return '∞' return Text(v)
def _init_job(self, joborder, **kwargs): # type: (Dict[Text, Text], **Any) -> Builder """ kwargs: eval_timeout: javascript evaluation timeout use_container: do/don't use Docker when DockerRequirement hint provided make_fs_access: make an FsAccess() object with given basedir basedir: basedir for FsAccess docker_outdir: output directory inside docker for this job docker_tmpdir: tmpdir inside docker for this job docker_stagedir: stagedir inside docker for this job outdir: outdir on host for this job tmpdir: tmpdir on host for this job stagedir: stagedir on host for this job select_resources: callback to select compute resources """ builder = Builder() builder.job = cast(Dict[Text, Union[Dict[Text, Any], List, Text]], copy.deepcopy(joborder)) # Validate job order try: fillInDefaults(self.tool[u"inputs"], builder.job) normalizeFilesDirs(builder.job) validate.validate_ex(self.names.get_name("input_record_schema", ""), builder.job, strict=False, logger=_logger_validation_warnings) except (validate.ValidationException, WorkflowException) as e: raise WorkflowException("Invalid job input record:\n" + Text(e)) builder.files = [] builder.bindings = CommentedSeq() builder.schemaDefs = self.schemaDefs builder.names = self.names builder.requirements = self.requirements builder.hints = self.hints builder.resources = {} builder.timeout = kwargs.get("eval_timeout") builder.debug = kwargs.get("debug") builder.mutation_manager = kwargs.get("mutation_manager") dockerReq, is_req = self.get_requirement("DockerRequirement") builder.make_fs_access = kwargs.get("make_fs_access") or StdFsAccess builder.fs_access = builder.make_fs_access(kwargs["basedir"]) loadListingReq, _ = self.get_requirement("http://commonwl.org/cwltool#LoadListingRequirement") if loadListingReq: builder.loadListing = loadListingReq.get("loadListing") if dockerReq and kwargs.get("use_container"): builder.outdir = builder.fs_access.realpath( dockerReq.get("dockerOutputDirectory") or kwargs.get("docker_outdir") or "/var/spool/cwl") builder.tmpdir = builder.fs_access.realpath(kwargs.get("docker_tmpdir") or "/tmp") builder.stagedir = builder.fs_access.realpath(kwargs.get("docker_stagedir") or "/var/lib/cwl") else: builder.outdir = builder.fs_access.realpath(kwargs.get("outdir") or tempfile.mkdtemp()) builder.tmpdir = builder.fs_access.realpath(kwargs.get("tmpdir") or tempfile.mkdtemp()) builder.stagedir = builder.fs_access.realpath(kwargs.get("stagedir") or tempfile.mkdtemp()) if self.formatgraph: for i in self.tool["inputs"]: d = shortname(i["id"]) if d in builder.job and i.get("format"): checkFormat(builder.job[d], builder.do_eval(i["format"]), self.formatgraph) builder.bindings.extend(builder.bind_input(self.inputs_record_schema, builder.job)) if self.tool.get("baseCommand"): for n, b in enumerate(aslist(self.tool["baseCommand"])): builder.bindings.append({ "position": [-1000000, n], "datum": b }) if self.tool.get("arguments"): for i, a in enumerate(self.tool["arguments"]): lc = self.tool["arguments"].lc.data[i] fn = self.tool["arguments"].lc.filename builder.bindings.lc.add_kv_line_col(len(builder.bindings), lc) if isinstance(a, dict): a = copy.copy(a) if a.get("position"): a["position"] = [a["position"], i] else: a["position"] = [0, i] builder.bindings.append(a) elif ("$(" in a) or ("${" in a): cm = CommentedMap(( ("position", [0, i]), ("valueFrom", a) )) cm.lc.add_kv_line_col("valueFrom", lc) cm.lc.filename = fn builder.bindings.append(cm) else: cm = CommentedMap(( ("position", [0, i]), ("datum", a) )) cm.lc.add_kv_line_col("datum", lc) cm.lc.filename = fn builder.bindings.append(cm) builder.bindings.sort(key=lambda a: a["position"]) builder.resources = self.evalResources(builder, kwargs) return builder
def __init__(self, toolpath_object, pos, **kwargs): # type: (Dict[Text, Any], int, **Any) -> None if "id" in toolpath_object: self.id = toolpath_object["id"] else: self.id = "#step" + Text(pos) kwargs["requirements"] = ( kwargs.get("requirements", []) + toolpath_object.get("requirements", []) + get_overrides( kwargs.get("overrides", []), self.id).get("requirements", [])) kwargs["hints"] = kwargs.get("hints", []) + toolpath_object.get( "hints", []) try: if isinstance(toolpath_object["run"], dict): self.embedded_tool = kwargs.get("makeTool")( toolpath_object["run"], **kwargs) else: self.embedded_tool = load_tool( toolpath_object["run"], kwargs.get("makeTool"), kwargs, enable_dev=kwargs.get("enable_dev"), strict=kwargs.get("strict"), fetcher_constructor=kwargs.get("fetcher_constructor"), resolver=kwargs.get("resolver"), overrides=kwargs.get("overrides")) except validate.ValidationException as v: raise WorkflowException( u"Tool definition %s failed validation:\n%s" % (toolpath_object["run"], validate.indent(str(v)))) validation_errors = [] self.tool = toolpath_object = copy.deepcopy(toolpath_object) bound = set() for stepfield, toolfield in (("in", "inputs"), ("out", "outputs")): toolpath_object[toolfield] = [] for n, step_entry in enumerate(toolpath_object[stepfield]): if isinstance(step_entry, six.string_types): param = CommentedMap() # type: CommentedMap inputid = step_entry else: param = CommentedMap(six.iteritems(step_entry)) inputid = step_entry["id"] shortinputid = shortname(inputid) found = False for tool_entry in self.embedded_tool.tool[toolfield]: frag = shortname(tool_entry["id"]) if frag == shortinputid: #if the case that the step has a default for a parameter, #we do not want the default of the tool to override it step_default = None if "default" in param and "default" in tool_entry: step_default = param["default"] param.update(tool_entry) if step_default is not None: param["default"] = step_default found = True bound.add(frag) break if not found: if stepfield == "in": param["type"] = "Any" else: validation_errors.append( SourceLine(self.tool["out"], n).makeError( "Workflow step output '%s' does not correspond to" % shortname(step_entry)) + "\n" + SourceLine(self.embedded_tool.tool, "outputs"). makeError(" tool output (expected '%s')" % ("', '".join([ shortname(tool_entry["id"]) for tool_entry in self.embedded_tool.tool[toolfield] ])))) param["id"] = inputid param.lc.line = toolpath_object[stepfield].lc.data[n][0] param.lc.col = toolpath_object[stepfield].lc.data[n][1] param.lc.filename = toolpath_object[stepfield].lc.filename toolpath_object[toolfield].append(param) missing = [] for i, tool_entry in enumerate(self.embedded_tool.tool["inputs"]): if shortname(tool_entry["id"]) not in bound: if "null" not in tool_entry[ "type"] and "default" not in tool_entry: missing.append(shortname(tool_entry["id"])) if missing: validation_errors.append( SourceLine(self.tool, "in").makeError( "Step is missing required parameter%s '%s'" % ("s" if len(missing) > 1 else "", "', '".join(missing)))) if validation_errors: raise validate.ValidationException("\n".join(validation_errors)) super(WorkflowStep, self).__init__(toolpath_object, **kwargs) if self.embedded_tool.tool["class"] == "Workflow": (feature, _) = self.get_requirement("SubworkflowFeatureRequirement") if not feature: raise WorkflowException( "Workflow contains embedded workflow but SubworkflowFeatureRequirement not in requirements" ) if "scatter" in self.tool: (feature, _) = self.get_requirement("ScatterFeatureRequirement") if not feature: raise WorkflowException( "Workflow contains scatter but ScatterFeatureRequirement not in requirements" ) inputparms = copy.deepcopy(self.tool["inputs"]) outputparms = copy.deepcopy(self.tool["outputs"]) scatter = aslist(self.tool["scatter"]) method = self.tool.get("scatterMethod") if method is None and len(scatter) != 1: raise validate.ValidationException( "Must specify scatterMethod when scattering over multiple inputs" ) inp_map = {i["id"]: i for i in inputparms} for s in scatter: if s not in inp_map: raise validate.ValidationException( SourceLine(self.tool, "scatter").makeError( u"Scatter parameter '%s' does not correspond to an input parameter of this " u"step, expecting '%s'" % (shortname(s), "', '".join( shortname(k) for k in inp_map.keys())))) inp_map[s]["type"] = { "type": "array", "items": inp_map[s]["type"] } if self.tool.get("scatterMethod") == "nested_crossproduct": nesting = len(scatter) else: nesting = 1 for r in range(0, nesting): for op in outputparms: op["type"] = {"type": "array", "items": op["type"]} self.tool["inputs"] = inputparms self.tool["outputs"] = outputparms
def get_view(self, name: _t.Text, **options): # pylint: disable=redefined-outer-name,too-many-statements mixin_class: _t.Type[NestedViewMixin] = NestedViewMixin if hasattr(self.view, 'create'): if self.kwargs.get('allow_append', False): mixin_class = NestedWithAppendMixin else: mixin_class = NestedWithoutAppendMixin tp = name.split('_')[-1] if tp == 'detail': detail = True elif tp == 'list': detail = False else: detail = getattr(self.view, options['nested_sub']).detail manager_name = self.kwargs.get('manager_name', self.name) class NestedView(mixin_class, self.view): # type: ignore __slots__ = ('nested_detail', ) # pylint: disable=class-variable-slots-conflict __doc__ = self.view.__doc__ format_kwarg = None queryset_filters = self.queryset_filters NestedView.__name__ = self.view.__name__ # type: ignore NestedView.nested_detail = detail # type: ignore def nested_view_function_wrapper(view_obj, request, *args, **kwargs): kwargs.update(options) view_obj.nested_parent_object = view_obj.get_object() nested_append_arg = view_obj.nested_append_arg nested_request_arg = view_obj.nested_arg nested_parent_object = view_obj.nested_parent_object if nested_append_arg: nested_id = getattr(nested_parent_object, nested_append_arg, None) else: nested_id = None if callable(manager_name): nested_manager = manager_name(nested_parent_object) else: if hasattr(nested_parent_object, manager_name): nested_manager = getattr(nested_parent_object, manager_name) else: view_manager_function_name = f'get_manager_{manager_name}' nested_manager_func = getattr(view_obj, view_manager_function_name) nested_manager = nested_manager_func(nested_parent_object) NestedView.__name__ = self.view.__name__ NestedView.master_view = view_obj NestedView.lookup_field = nested_append_arg NestedView.lookup_url_kwarg = nested_request_arg NestedView.nested_detail = detail NestedView.nested_allow_append = view_obj.nested_allow_append NestedView.nested_append_arg = nested_append_arg NestedView.nested_request_arg = nested_request_arg NestedView.nested_parent_object = nested_parent_object NestedView.nested_id = nested_id NestedView.nested_manager = nested_manager getattr(view_obj, 'nested_allow_check', lambda *args, **kwargs: None)() return nested_view_function(view_obj, NestedView, request, *args, **kwargs) nested_view_function_wrapper.__name__ = name nested_view_function_wrapper.__doc__ = self.view.__doc__ nested_view_function_wrapper._nested_view = self.view # type: ignore return name, nested_view_function_wrapper
def __init__(self, toolpath_object, **kwargs): # type: (Dict[Text, Any], **Any) -> None """ kwargs: metadata: tool document metadata requirements: inherited requirements hints: inherited hints loader: schema_salad.ref_resolver.Loader used to load tool document avsc_names: CWL Avro schema object used to validate document strict: flag to determine strict validation (fail on unrecognized fields) """ self.metadata = kwargs.get("metadata", {}) # type: Dict[Text,Any] self.names = None # type: avro.schema.Names global SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY # pylint: disable=global-statement if SCHEMA_FILE is None: get_schema("v1.0") SCHEMA_ANY = cast(Dict[Text, Any], SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/salad#Any"]) SCHEMA_FILE = cast(Dict[Text, Any], SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/cwl#File"]) SCHEMA_DIR = cast(Dict[Text, Any], SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/cwl#Directory"]) names = schema_salad.schema.make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY], schema_salad.ref_resolver.Loader({}))[0] if isinstance(names, avro.schema.SchemaParseException): raise names else: self.names = names self.tool = toolpath_object self.requirements = kwargs.get("requirements", []) + self.tool.get("requirements", []) self.hints = kwargs.get("hints", []) + self.tool.get("hints", []) self.formatgraph = None # type: Graph if "loader" in kwargs: self.formatgraph = kwargs["loader"].graph self.doc_loader = kwargs["loader"] self.doc_schema = kwargs["avsc_names"] checkRequirements(self.tool, supportedProcessRequirements) self.validate_hints(kwargs["avsc_names"], self.tool.get("hints", []), strict=kwargs.get("strict")) self.schemaDefs = {} # type: Dict[Text,Dict[Text, Any]] sd, _ = self.get_requirement("SchemaDefRequirement") if sd: sdtypes = sd["types"] av = schema_salad.schema.make_valid_avro(sdtypes, {t["name"]: t for t in avroize_type(sdtypes)}, set()) for i in av: self.schemaDefs[i["name"]] = i avro.schema.make_avsc_object(av, self.names) # Build record schema from inputs self.inputs_record_schema = { "name": "input_record_schema", "type": "record", "fields": []} # type: Dict[Text, Any] self.outputs_record_schema = { "name": "outputs_record_schema", "type": "record", "fields": []} # type: Dict[Text, Any] for key in ("inputs", "outputs"): for i in self.tool[key]: c = copy.copy(i) c["name"] = shortname(c["id"]) del c["id"] if "type" not in c: raise validate.ValidationException(u"Missing `type` in parameter `%s`" % c["name"]) if "default" in c and "null" not in aslist(c["type"]): c["type"] = ["null"] + aslist(c["type"]) else: c["type"] = c["type"] c["type"] = avroize_type(c["type"], c["name"]) if key == "inputs": self.inputs_record_schema["fields"].append(c) elif key == "outputs": self.outputs_record_schema["fields"].append(c) try: self.inputs_record_schema = cast(Dict[unicode, Any], schema_salad.schema.make_valid_avro(self.inputs_record_schema, {}, set())) avro.schema.make_avsc_object(self.inputs_record_schema, self.names) except avro.schema.SchemaParseException as e: raise validate.ValidationException(u"Got error `%s` while processing inputs of %s:\n%s" % (Text(e), self.tool["id"], json.dumps(self.inputs_record_schema, indent=4))) try: self.outputs_record_schema = cast(Dict[unicode, Any], schema_salad.schema.make_valid_avro(self.outputs_record_schema, {}, set())) avro.schema.make_avsc_object(self.outputs_record_schema, self.names) except avro.schema.SchemaParseException as e: raise validate.ValidationException(u"Got error `%s` while processing outputs of %s:\n%s" % (Text(e), self.tool["id"], json.dumps(self.outputs_record_schema, indent=4)))
def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move"): # type: (List[Text], MutableMapping[Text, Text], bool, Text) -> None scr, _ = get_feature(self, "ShellCommandRequirement") shouldquote = None # type: Callable[[Any], Any] if scr: shouldquote = lambda x: False else: shouldquote = needs_shell_quoting_re.search _logger.info( u"[job %s] %s$ %s%s%s%s", self.name, self.outdir, " \\\n ".join([ shellescape.quote(Text(arg)) if shouldquote(Text(arg)) else Text(arg) for arg in (runtime + self.command_line) ]), u' < %s' % self.stdin if self.stdin else '', u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '', u' 2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '') outputs = {} # type: Dict[Text,Text] try: stdin_path = None if self.stdin: stdin_path = self.pathmapper.reversemap(self.stdin)[1] stderr_path = None if self.stderr: abserr = os.path.join(self.outdir, self.stderr) dnerr = os.path.dirname(abserr) if dnerr and not os.path.exists(dnerr): os.makedirs(dnerr) stderr_path = abserr stdout_path = None if self.stdout: absout = os.path.join(self.outdir, self.stdout) dn = os.path.dirname(absout) if dn and not os.path.exists(dn): os.makedirs(dn) stdout_path = absout commands = [Text(x) for x in (runtime + self.command_line)] job_script_contents = None # type: Text builder = getattr(self, "builder", None) # type: Builder if builder is not None: job_script_contents = builder.build_job_script(commands) rcode = _job_popen( commands, stdin_path=stdin_path, stdout_path=stdout_path, stderr_path=stderr_path, env=env, cwd=self.outdir, job_script_contents=job_script_contents, ) if self.successCodes and rcode in self.successCodes: processStatus = "success" elif self.temporaryFailCodes and rcode in self.temporaryFailCodes: processStatus = "temporaryFail" elif self.permanentFailCodes and rcode in self.permanentFailCodes: processStatus = "permanentFail" elif rcode == 0: processStatus = "success" else: processStatus = "permanentFail" if self.generatefiles["listing"]: relink_initialworkdir(self.generatemapper, self.outdir, self.builder.outdir, inplace_update=self.inplace_update) outputs = self.collect_outputs(self.outdir) outputs = bytes2str_in_dicts(outputs) # type: ignore except OSError as e: if e.errno == 2: if runtime: _logger.error(u"'%s' not found", runtime[0]) else: _logger.error(u"'%s' not found", self.command_line[0]) else: _logger.exception("Exception while running job") processStatus = "permanentFail" except WorkflowException as e: _logger.error(u"[job %s] Job error:\n%s" % (self.name, e)) processStatus = "permanentFail" except Exception as e: _logger.exception("Exception while running job") processStatus = "permanentFail" if processStatus != "success": _logger.warning(u"[job %s] completed %s", self.name, processStatus) else: _logger.info(u"[job %s] completed %s", self.name, processStatus) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"[job %s] %s", self.name, json.dumps(outputs, indent=4)) self.output_callback(outputs, processStatus) if self.stagedir and os.path.exists(self.stagedir): _logger.debug(u"[job %s] Removing input staging directory %s", self.name, self.stagedir) shutil.rmtree(self.stagedir, True) if rm_tmpdir: _logger.debug(u"[job %s] Removing temporary directory %s", self.name, self.tmpdir) shutil.rmtree(self.tmpdir, True)
def get_cuda_version(config): # type: (ConfigTree) -> (Text, Text) # we assume os.environ already updated the config['agent.cuda_version'] & config['agent.cudnn_version'] cuda_version = config['agent.cuda_version'] cudnn_version = config['agent.cudnn_version'] if cuda_version and cudnn_version: return normalize_cuda_version( cuda_version), normalize_cuda_version(cudnn_version) if not cuda_version and is_windows_platform(): try: cuda_vers = [ int(k.replace('CUDA_PATH_V', '').replace('_', '')) for k in os.environ.keys() if k.startswith('CUDA_PATH_V') ] cuda_vers = max(cuda_vers) if cuda_vers > 40: cuda_version = cuda_vers except: pass if not cuda_version: try: try: nvcc = 'nvcc.exe' if is_windows_platform() else 'nvcc' if is_windows_platform() and 'CUDA_PATH' in os.environ: nvcc = os.path.join(os.environ['CUDA_PATH'], nvcc) output = Argv(nvcc, '--version').get_output() except OSError: raise CudaNotFound('nvcc not found') match = re.search(r'release (.{3})', output).group(1) cuda_version = Text(int(float(match) * 10)) except: pass if not cuda_version: try: try: output = Argv('nvidia-smi', ).get_output() except OSError: raise CudaNotFound('nvcc not found') match = re.search(r'CUDA Version: ([0-9]+).([0-9]+)', output) match = match.group(1) + '.' + match.group(2) cuda_version = Text(int(float(match) * 10)) except: pass if not cudnn_version: try: cuda_lib = which('nvcc') if is_windows_platform: cudnn_h = path.sep.join( cuda_lib.split(path.sep)[:-2] + ['include', 'cudnn.h']) else: cudnn_h = path.join( path.sep, *(cuda_lib.split(path.sep)[:-2] + ['include', 'cudnn.h'])) cudnn_major, cudnn_minor = None, None try: include_file = open(cudnn_h) except OSError: raise CudaNotFound('Could not read cudnn.h') with include_file: for line in include_file: if 'CUDNN_MAJOR' in line: cudnn_major = line.split()[-1] if 'CUDNN_MINOR' in line: cudnn_minor = line.split()[-1] if cudnn_major and cudnn_minor: break cudnn_version = cudnn_major + (cudnn_minor or '0') except: pass return (normalize_cuda_version(cuda_version or 0), normalize_cuda_version(cudnn_version or 0))
def send_messages( data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int] ) -> int: (tot_messages, personals_pairs, options, output, random_seed) = data random.seed(random_seed) with open("var/test_messages.json", "r") as infile: dialog = ujson.load(infile) random.shuffle(dialog) texts = itertools.cycle(dialog) recipient_streams = [ klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] # type: List[int] recipient_huddles = [ h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE) ] # type: List[int] huddle_members = {} # type: Dict[int, List[int]] for h in recipient_huddles: huddle_members[h] = [ s.user_profile.id for s in Subscription.objects.filter(recipient_id=h) ] num_messages = 0 random_max = 1000000 recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]] while num_messages < tot_messages: saved_data = {} # type: Dict[str, Any] message = Message() message.sending_client = get_client('populate_db') message.content = next(texts) randkey = random.randint(1, random_max) if (num_messages > 0 and random.randint(1, random_max) * 100. / random_max < options["stickyness"]): # Use an old recipient message_type, recipient_id, saved_data = recipients[num_messages - 1] if message_type == Recipient.PERSONAL: personals_pair = saved_data['personals_pair'] random.shuffle(personals_pair) elif message_type == Recipient.STREAM: message.subject = saved_data['subject'] message.recipient = get_recipient_by_id(recipient_id) elif message_type == Recipient.HUDDLE: message.recipient = get_recipient_by_id(recipient_id) elif (randkey <= random_max * options["percent_huddles"] / 100.): message_type = Recipient.HUDDLE message.recipient = get_recipient_by_id( random.choice(recipient_huddles)) elif (randkey <= random_max * (options["percent_huddles"] + options["percent_personals"]) / 100.): message_type = Recipient.PERSONAL personals_pair = random.choice(personals_pairs) random.shuffle(personals_pair) elif (randkey <= random_max * 1.0): message_type = Recipient.STREAM message.recipient = get_recipient_by_id( random.choice(recipient_streams)) if message_type == Recipient.HUDDLE: sender_id = random.choice(huddle_members[message.recipient.id]) message.sender = get_user_profile_by_id(sender_id) elif message_type == Recipient.PERSONAL: message.recipient = Recipient.objects.get( type=Recipient.PERSONAL, type_id=personals_pair[0]) message.sender = get_user_profile_by_id(personals_pair[1]) saved_data['personals_pair'] = personals_pair elif message_type == Recipient.STREAM: stream = Stream.objects.get(id=message.recipient.type_id) # Pick a random subscriber to the stream message.sender = random.choice( Subscription.objects.filter( recipient=message.recipient)).user_profile message.subject = stream.name + Text(random.randint(1, 3)) saved_data['subject'] = message.subject # Spoofing time not supported with threading if options['threads'] != 1: message.pub_date = timezone_now() else: # Distrubutes 80% of messages starting from 5 days ago, over a period # of 3 days. Then, distributes remaining messages over past 24 hours. spoofed_date = timezone_now() - timezone_timedelta(days=5) if (num_messages < tot_messages * 0.8): # Maximum of 3 days ahead, convert to minutes time_ahead = 3 * 24 * 60 time_ahead //= int(tot_messages * 0.8) else: time_ahead = 24 * 60 time_ahead //= int(tot_messages * 0.2) spoofed_minute = random.randint(time_ahead * num_messages, time_ahead * (num_messages + 1)) spoofed_date += timezone_timedelta(minutes=spoofed_minute) message.pub_date = spoofed_date # We disable USING_RABBITMQ here, so that deferred work is # executed in do_send_message_messages, rather than being # queued. This is important, because otherwise, if run-dev.py # wasn't running when populate_db was run, a developer can end # up with queued events that reference objects from a previous # life of the database, which naturally throws exceptions. settings.USING_RABBITMQ = False do_send_messages([{'message': message}]) settings.USING_RABBITMQ = True recipients[num_messages] = (message_type, message.recipient.id, saved_data) num_messages += 1 return tot_messages
def get_image( docker_requirement, # type: Dict[Text, Text] pull_image, # type: bool force_pull=False, # type: bool tmp_outdir_prefix=DEFAULT_TMP_PREFIX # type: Text ): # type: (...) -> bool """ Retrieve the relevant Docker container image. Returns True upon success """ found = False if "dockerImageId" not in docker_requirement \ and "dockerPull" in docker_requirement: docker_requirement["dockerImageId"] = docker_requirement[ "dockerPull"] with _IMAGES_LOCK: if docker_requirement["dockerImageId"] in _IMAGES: return True for line in subprocess.check_output( ["docker", "images", "--no-trunc", "--all"]).decode('utf-8').splitlines(): try: match = re.match(r"^([^ ]+)\s+([^ ]+)\s+([^ ]+)", line) split = docker_requirement["dockerImageId"].split(":") if len(split) == 1: split.append("latest") elif len(split) == 2: # if split[1] doesn't match valid tag names, it is a part of repository if not re.match(r'[\w][\w.-]{0,127}', split[1]): split[0] = split[0] + ":" + split[1] split[1] = "latest" elif len(split) == 3: if re.match(r'[\w][\w.-]{0,127}', split[2]): split[0] = split[0] + ":" + split[1] split[1] = split[2] del split[2] # check for repository:tag match or image id match if (match and ((split[0] == match.group(1) and split[1] == match.group(2)) or docker_requirement["dockerImageId"] == match.group(3))): found = True break except ValueError: pass if (force_pull or not found) and pull_image: cmd = [] # type: List[Text] if "dockerPull" in docker_requirement: cmd = ["docker", "pull", str(docker_requirement["dockerPull"])] _logger.info(Text(cmd)) subprocess.check_call(cmd, stdout=sys.stderr) found = True elif "dockerFile" in docker_requirement: dockerfile_dir = str( tempfile.mkdtemp(prefix=tmp_outdir_prefix)) with open(os.path.join(dockerfile_dir, "Dockerfile"), "wb") as dfile: dfile.write( docker_requirement["dockerFile"].encode('utf-8')) cmd = [ "docker", "build", "--tag=%s" % str(docker_requirement["dockerImageId"]), dockerfile_dir ] _logger.info(Text(cmd)) subprocess.check_call(cmd, stdout=sys.stderr) found = True elif "dockerLoad" in docker_requirement: cmd = ["docker", "load"] _logger.info(Text(cmd)) if os.path.exists(docker_requirement["dockerLoad"]): _logger.info(u"Loading docker image from %s", docker_requirement["dockerLoad"]) with open(docker_requirement["dockerLoad"], "rb") as dload: loadproc = subprocess.Popen(cmd, stdin=dload, stdout=sys.stderr) else: loadproc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=sys.stderr) assert loadproc.stdin is not None _logger.info(u"Sending GET request to %s", docker_requirement["dockerLoad"]) req = requests.get(docker_requirement["dockerLoad"], stream=True) size = 0 for chunk in req.iter_content(1024 * 1024): size += len(chunk) _logger.info("\r%i bytes", size) loadproc.stdin.write(chunk) loadproc.stdin.close() rcode = loadproc.wait() if rcode != 0: raise WorkflowException( "Docker load returned non-zero exit status %i" % (rcode)) found = True elif "dockerImport" in docker_requirement: cmd = [ "docker", "import", str(docker_requirement["dockerImport"]), str(docker_requirement["dockerImageId"]) ] _logger.info(Text(cmd)) subprocess.check_call(cmd, stdout=sys.stderr) found = True if found: with _IMAGES_LOCK: _IMAGES.add(docker_requirement["dockerImageId"]) return found
def __new__(cls, val): # type: (Text) -> Any assert "@" in val, "Email is not valid" return Text.__new__(cls, val) # type: ignore