def display_number(v): # type: (int) -> Text
if defs.OpSchema.is_infinite(v):
return '∞'
return Text(v)
def _init_job(self, joborder, **kwargs):
# type: (Dict[Text, Text], **Any) -> Builder
"""
kwargs:
eval_timeout: javascript evaluation timeout
use_container: do/don't use Docker when DockerRequirement hint provided
make_fs_access: make an FsAccess() object with given basedir
basedir: basedir for FsAccess
docker_outdir: output directory inside docker for this job
docker_tmpdir: tmpdir inside docker for this job
docker_stagedir: stagedir inside docker for this job
outdir: outdir on host for this job
tmpdir: tmpdir on host for this job
stagedir: stagedir on host for this job
select_resources: callback to select compute resources
"""
builder = Builder()
builder.job = cast(Dict[Text, Union[Dict[Text, Any], List,
Text]], copy.deepcopy(joborder))
# Validate job order
try:
fillInDefaults(self.tool[u"inputs"], builder.job)
normalizeFilesDirs(builder.job)
validate.validate_ex(self.names.get_name("input_record_schema", ""), builder.job,
strict=False, logger=_logger_validation_warnings)
except (validate.ValidationException, WorkflowException) as e:
raise WorkflowException("Invalid job input record:\n" + Text(e))
builder.files = []
builder.bindings = CommentedSeq()
builder.schemaDefs = self.schemaDefs
builder.names = self.names
builder.requirements = self.requirements
builder.hints = self.hints
builder.resources = {}
builder.timeout = kwargs.get("eval_timeout")
builder.debug = kwargs.get("debug")
builder.mutation_manager = kwargs.get("mutation_manager")
dockerReq, is_req = self.get_requirement("DockerRequirement")
builder.make_fs_access = kwargs.get("make_fs_access") or StdFsAccess
builder.fs_access = builder.make_fs_access(kwargs["basedir"])
loadListingReq, _ = self.get_requirement("http://commonwl.org/cwltool#LoadListingRequirement")
if loadListingReq:
builder.loadListing = loadListingReq.get("loadListing")
if dockerReq and kwargs.get("use_container"):
builder.outdir = builder.fs_access.realpath(
dockerReq.get("dockerOutputDirectory") or kwargs.get("docker_outdir") or "/var/spool/cwl")
builder.tmpdir = builder.fs_access.realpath(kwargs.get("docker_tmpdir") or "/tmp")
builder.stagedir = builder.fs_access.realpath(kwargs.get("docker_stagedir") or "/var/lib/cwl")
else:
builder.outdir = builder.fs_access.realpath(kwargs.get("outdir") or tempfile.mkdtemp())
builder.tmpdir = builder.fs_access.realpath(kwargs.get("tmpdir") or tempfile.mkdtemp())
builder.stagedir = builder.fs_access.realpath(kwargs.get("stagedir") or tempfile.mkdtemp())
if self.formatgraph:
for i in self.tool["inputs"]:
d = shortname(i["id"])
if d in builder.job and i.get("format"):
checkFormat(builder.job[d], builder.do_eval(i["format"]), self.formatgraph)
builder.bindings.extend(builder.bind_input(self.inputs_record_schema, builder.job))
if self.tool.get("baseCommand"):
for n, b in enumerate(aslist(self.tool["baseCommand"])):
builder.bindings.append({
"position": [-1000000, n],
"datum": b
})
if self.tool.get("arguments"):
for i, a in enumerate(self.tool["arguments"]):
lc = self.tool["arguments"].lc.data[i]
fn = self.tool["arguments"].lc.filename
builder.bindings.lc.add_kv_line_col(len(builder.bindings), lc)
if isinstance(a, dict):
a = copy.copy(a)
if a.get("position"):
a["position"] = [a["position"], i]
else:
a["position"] = [0, i]
builder.bindings.append(a)
elif ("$(" in a) or ("${" in a):
cm = CommentedMap((
("position", [0, i]),
("valueFrom", a)
))
cm.lc.add_kv_line_col("valueFrom", lc)
cm.lc.filename = fn
builder.bindings.append(cm)
else:
cm = CommentedMap((
("position", [0, i]),
("datum", a)
))
cm.lc.add_kv_line_col("datum", lc)
cm.lc.filename = fn
builder.bindings.append(cm)
builder.bindings.sort(key=lambda a: a["position"])
builder.resources = self.evalResources(builder, kwargs)
return builder
def __init__(self, toolpath_object, pos, **kwargs):
# type: (Dict[Text, Any], int, **Any) -> None
if "id" in toolpath_object:
self.id = toolpath_object["id"]
else:
self.id = "#step" + Text(pos)
kwargs["requirements"] = (
kwargs.get("requirements", []) +
toolpath_object.get("requirements", []) + get_overrides(
kwargs.get("overrides", []), self.id).get("requirements", []))
kwargs["hints"] = kwargs.get("hints", []) + toolpath_object.get(
"hints", [])
try:
if isinstance(toolpath_object["run"], dict):
self.embedded_tool = kwargs.get("makeTool")(
toolpath_object["run"], **kwargs)
else:
self.embedded_tool = load_tool(
toolpath_object["run"],
kwargs.get("makeTool"),
kwargs,
enable_dev=kwargs.get("enable_dev"),
strict=kwargs.get("strict"),
fetcher_constructor=kwargs.get("fetcher_constructor"),
resolver=kwargs.get("resolver"),
overrides=kwargs.get("overrides"))
except validate.ValidationException as v:
raise WorkflowException(
u"Tool definition %s failed validation:\n%s" %
(toolpath_object["run"], validate.indent(str(v))))
validation_errors = []
self.tool = toolpath_object = copy.deepcopy(toolpath_object)
bound = set()
for stepfield, toolfield in (("in", "inputs"), ("out", "outputs")):
toolpath_object[toolfield] = []
for n, step_entry in enumerate(toolpath_object[stepfield]):
if isinstance(step_entry, six.string_types):
param = CommentedMap() # type: CommentedMap
inputid = step_entry
else:
param = CommentedMap(six.iteritems(step_entry))
inputid = step_entry["id"]
shortinputid = shortname(inputid)
found = False
for tool_entry in self.embedded_tool.tool[toolfield]:
frag = shortname(tool_entry["id"])
if frag == shortinputid:
#if the case that the step has a default for a parameter,
#we do not want the default of the tool to override it
step_default = None
if "default" in param and "default" in tool_entry:
step_default = param["default"]
param.update(tool_entry)
if step_default is not None:
param["default"] = step_default
found = True
bound.add(frag)
break
if not found:
if stepfield == "in":
param["type"] = "Any"
else:
validation_errors.append(
SourceLine(self.tool["out"], n).makeError(
"Workflow step output '%s' does not correspond to"
% shortname(step_entry)) + "\n" +
SourceLine(self.embedded_tool.tool, "outputs").
makeError(" tool output (expected '%s')" %
("', '".join([
shortname(tool_entry["id"])
for tool_entry in
self.embedded_tool.tool[toolfield]
]))))
param["id"] = inputid
param.lc.line = toolpath_object[stepfield].lc.data[n][0]
param.lc.col = toolpath_object[stepfield].lc.data[n][1]
param.lc.filename = toolpath_object[stepfield].lc.filename
toolpath_object[toolfield].append(param)
missing = []
for i, tool_entry in enumerate(self.embedded_tool.tool["inputs"]):
if shortname(tool_entry["id"]) not in bound:
if "null" not in tool_entry[
"type"] and "default" not in tool_entry:
missing.append(shortname(tool_entry["id"]))
if missing:
validation_errors.append(
SourceLine(self.tool, "in").makeError(
"Step is missing required parameter%s '%s'" %
("s" if len(missing) > 1 else "", "', '".join(missing))))
if validation_errors:
raise validate.ValidationException("\n".join(validation_errors))
super(WorkflowStep, self).__init__(toolpath_object, **kwargs)
if self.embedded_tool.tool["class"] == "Workflow":
(feature,
_) = self.get_requirement("SubworkflowFeatureRequirement")
if not feature:
raise WorkflowException(
"Workflow contains embedded workflow but SubworkflowFeatureRequirement not in requirements"
)
if "scatter" in self.tool:
(feature, _) = self.get_requirement("ScatterFeatureRequirement")
if not feature:
raise WorkflowException(
"Workflow contains scatter but ScatterFeatureRequirement not in requirements"
)
inputparms = copy.deepcopy(self.tool["inputs"])
outputparms = copy.deepcopy(self.tool["outputs"])
scatter = aslist(self.tool["scatter"])
method = self.tool.get("scatterMethod")
if method is None and len(scatter) != 1:
raise validate.ValidationException(
"Must specify scatterMethod when scattering over multiple inputs"
)
inp_map = {i["id"]: i for i in inputparms}
for s in scatter:
if s not in inp_map:
raise validate.ValidationException(
SourceLine(self.tool, "scatter").makeError(
u"Scatter parameter '%s' does not correspond to an input parameter of this "
u"step, expecting '%s'" %
(shortname(s), "', '".join(
shortname(k) for k in inp_map.keys()))))
inp_map[s]["type"] = {
"type": "array",
"items": inp_map[s]["type"]
}
if self.tool.get("scatterMethod") == "nested_crossproduct":
nesting = len(scatter)
else:
nesting = 1
for r in range(0, nesting):
for op in outputparms:
op["type"] = {"type": "array", "items": op["type"]}
self.tool["inputs"] = inputparms
self.tool["outputs"] = outputparms
def get_view(self, name: _t.Text, **options):
# pylint: disable=redefined-outer-name,too-many-statements
mixin_class: _t.Type[NestedViewMixin] = NestedViewMixin
if hasattr(self.view, 'create'):
if self.kwargs.get('allow_append', False):
mixin_class = NestedWithAppendMixin
else:
mixin_class = NestedWithoutAppendMixin
tp = name.split('_')[-1]
if tp == 'detail':
detail = True
elif tp == 'list':
detail = False
else:
detail = getattr(self.view, options['nested_sub']).detail
manager_name = self.kwargs.get('manager_name', self.name)
class NestedView(mixin_class, self.view): # type: ignore
__slots__ = ('nested_detail', ) # pylint: disable=class-variable-slots-conflict
__doc__ = self.view.__doc__
format_kwarg = None
queryset_filters = self.queryset_filters
NestedView.__name__ = self.view.__name__ # type: ignore
NestedView.nested_detail = detail # type: ignore
def nested_view_function_wrapper(view_obj, request, *args, **kwargs):
kwargs.update(options)
view_obj.nested_parent_object = view_obj.get_object()
nested_append_arg = view_obj.nested_append_arg
nested_request_arg = view_obj.nested_arg
nested_parent_object = view_obj.nested_parent_object
if nested_append_arg:
nested_id = getattr(nested_parent_object, nested_append_arg,
None)
else:
nested_id = None
if callable(manager_name):
nested_manager = manager_name(nested_parent_object)
else:
if hasattr(nested_parent_object, manager_name):
nested_manager = getattr(nested_parent_object,
manager_name)
else:
view_manager_function_name = f'get_manager_{manager_name}'
nested_manager_func = getattr(view_obj,
view_manager_function_name)
nested_manager = nested_manager_func(nested_parent_object)
NestedView.__name__ = self.view.__name__
NestedView.master_view = view_obj
NestedView.lookup_field = nested_append_arg
NestedView.lookup_url_kwarg = nested_request_arg
NestedView.nested_detail = detail
NestedView.nested_allow_append = view_obj.nested_allow_append
NestedView.nested_append_arg = nested_append_arg
NestedView.nested_request_arg = nested_request_arg
NestedView.nested_parent_object = nested_parent_object
NestedView.nested_id = nested_id
NestedView.nested_manager = nested_manager
getattr(view_obj, 'nested_allow_check',
lambda *args, **kwargs: None)()
return nested_view_function(view_obj, NestedView, request, *args,
**kwargs)
nested_view_function_wrapper.__name__ = name
nested_view_function_wrapper.__doc__ = self.view.__doc__
nested_view_function_wrapper._nested_view = self.view # type: ignore
return name, nested_view_function_wrapper
def __init__(self, toolpath_object, **kwargs):
# type: (Dict[Text, Any], **Any) -> None
"""
kwargs:
metadata: tool document metadata
requirements: inherited requirements
hints: inherited hints
loader: schema_salad.ref_resolver.Loader used to load tool document
avsc_names: CWL Avro schema object used to validate document
strict: flag to determine strict validation (fail on unrecognized fields)
"""
self.metadata = kwargs.get("metadata", {}) # type: Dict[Text,Any]
self.names = None # type: avro.schema.Names
global SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY # pylint: disable=global-statement
if SCHEMA_FILE is None:
get_schema("v1.0")
SCHEMA_ANY = cast(Dict[Text, Any],
SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/salad#Any"])
SCHEMA_FILE = cast(Dict[Text, Any],
SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/cwl#File"])
SCHEMA_DIR = cast(Dict[Text, Any],
SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/cwl#Directory"])
names = schema_salad.schema.make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY],
schema_salad.ref_resolver.Loader({}))[0]
if isinstance(names, avro.schema.SchemaParseException):
raise names
else:
self.names = names
self.tool = toolpath_object
self.requirements = kwargs.get("requirements", []) + self.tool.get("requirements", [])
self.hints = kwargs.get("hints", []) + self.tool.get("hints", [])
self.formatgraph = None # type: Graph
if "loader" in kwargs:
self.formatgraph = kwargs["loader"].graph
self.doc_loader = kwargs["loader"]
self.doc_schema = kwargs["avsc_names"]
checkRequirements(self.tool, supportedProcessRequirements)
self.validate_hints(kwargs["avsc_names"], self.tool.get("hints", []),
strict=kwargs.get("strict"))
self.schemaDefs = {} # type: Dict[Text,Dict[Text, Any]]
sd, _ = self.get_requirement("SchemaDefRequirement")
if sd:
sdtypes = sd["types"]
av = schema_salad.schema.make_valid_avro(sdtypes, {t["name"]: t for t in avroize_type(sdtypes)}, set())
for i in av:
self.schemaDefs[i["name"]] = i
avro.schema.make_avsc_object(av, self.names)
# Build record schema from inputs
self.inputs_record_schema = {
"name": "input_record_schema", "type": "record",
"fields": []} # type: Dict[Text, Any]
self.outputs_record_schema = {
"name": "outputs_record_schema", "type": "record",
"fields": []} # type: Dict[Text, Any]
for key in ("inputs", "outputs"):
for i in self.tool[key]:
c = copy.copy(i)
c["name"] = shortname(c["id"])
del c["id"]
if "type" not in c:
raise validate.ValidationException(u"Missing `type` in parameter `%s`" % c["name"])
if "default" in c and "null" not in aslist(c["type"]):
c["type"] = ["null"] + aslist(c["type"])
else:
c["type"] = c["type"]
c["type"] = avroize_type(c["type"], c["name"])
if key == "inputs":
self.inputs_record_schema["fields"].append(c)
elif key == "outputs":
self.outputs_record_schema["fields"].append(c)
try:
self.inputs_record_schema = cast(Dict[unicode, Any], schema_salad.schema.make_valid_avro(self.inputs_record_schema, {}, set()))
avro.schema.make_avsc_object(self.inputs_record_schema, self.names)
except avro.schema.SchemaParseException as e:
raise validate.ValidationException(u"Got error `%s` while processing inputs of %s:\n%s" %
(Text(e), self.tool["id"],
json.dumps(self.inputs_record_schema, indent=4)))
try:
self.outputs_record_schema = cast(Dict[unicode, Any], schema_salad.schema.make_valid_avro(self.outputs_record_schema, {}, set()))
avro.schema.make_avsc_object(self.outputs_record_schema, self.names)
except avro.schema.SchemaParseException as e:
raise validate.ValidationException(u"Got error `%s` while processing outputs of %s:\n%s" %
(Text(e), self.tool["id"],
json.dumps(self.outputs_record_schema, indent=4)))
def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move"):
# type: (List[Text], MutableMapping[Text, Text], bool, Text) -> None
scr, _ = get_feature(self, "ShellCommandRequirement")
shouldquote = None # type: Callable[[Any], Any]
if scr:
shouldquote = lambda x: False
else:
shouldquote = needs_shell_quoting_re.search
_logger.info(
u"[job %s] %s$ %s%s%s%s", self.name, self.outdir,
" \\\n ".join([
shellescape.quote(Text(arg))
if shouldquote(Text(arg)) else Text(arg)
for arg in (runtime + self.command_line)
]), u' < %s' % self.stdin if self.stdin else '', u' > %s' %
os.path.join(self.outdir, self.stdout) if self.stdout else '',
u' 2> %s' %
os.path.join(self.outdir, self.stderr) if self.stderr else '')
outputs = {} # type: Dict[Text,Text]
try:
stdin_path = None
if self.stdin:
stdin_path = self.pathmapper.reversemap(self.stdin)[1]
stderr_path = None
if self.stderr:
abserr = os.path.join(self.outdir, self.stderr)
dnerr = os.path.dirname(abserr)
if dnerr and not os.path.exists(dnerr):
os.makedirs(dnerr)
stderr_path = abserr
stdout_path = None
if self.stdout:
absout = os.path.join(self.outdir, self.stdout)
dn = os.path.dirname(absout)
if dn and not os.path.exists(dn):
os.makedirs(dn)
stdout_path = absout
commands = [Text(x) for x in (runtime + self.command_line)]
job_script_contents = None # type: Text
builder = getattr(self, "builder", None) # type: Builder
if builder is not None:
job_script_contents = builder.build_job_script(commands)
rcode = _job_popen(
commands,
stdin_path=stdin_path,
stdout_path=stdout_path,
stderr_path=stderr_path,
env=env,
cwd=self.outdir,
job_script_contents=job_script_contents,
)
if self.successCodes and rcode in self.successCodes:
processStatus = "success"
elif self.temporaryFailCodes and rcode in self.temporaryFailCodes:
processStatus = "temporaryFail"
elif self.permanentFailCodes and rcode in self.permanentFailCodes:
processStatus = "permanentFail"
elif rcode == 0:
processStatus = "success"
else:
processStatus = "permanentFail"
if self.generatefiles["listing"]:
relink_initialworkdir(self.generatemapper,
self.outdir,
self.builder.outdir,
inplace_update=self.inplace_update)
outputs = self.collect_outputs(self.outdir)
outputs = bytes2str_in_dicts(outputs) # type: ignore
except OSError as e:
if e.errno == 2:
if runtime:
_logger.error(u"'%s' not found", runtime[0])
else:
_logger.error(u"'%s' not found", self.command_line[0])
else:
_logger.exception("Exception while running job")
processStatus = "permanentFail"
except WorkflowException as e:
_logger.error(u"[job %s] Job error:\n%s" % (self.name, e))
processStatus = "permanentFail"
except Exception as e:
_logger.exception("Exception while running job")
processStatus = "permanentFail"
if processStatus != "success":
_logger.warning(u"[job %s] completed %s", self.name, processStatus)
else:
_logger.info(u"[job %s] completed %s", self.name, processStatus)
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(u"[job %s] %s", self.name,
json.dumps(outputs, indent=4))
self.output_callback(outputs, processStatus)
if self.stagedir and os.path.exists(self.stagedir):
_logger.debug(u"[job %s] Removing input staging directory %s",
self.name, self.stagedir)
shutil.rmtree(self.stagedir, True)
if rm_tmpdir:
_logger.debug(u"[job %s] Removing temporary directory %s",
self.name, self.tmpdir)
shutil.rmtree(self.tmpdir, True)
def get_cuda_version(config): # type: (ConfigTree) -> (Text, Text)
# we assume os.environ already updated the config['agent.cuda_version'] & config['agent.cudnn_version']
cuda_version = config['agent.cuda_version']
cudnn_version = config['agent.cudnn_version']
if cuda_version and cudnn_version:
return normalize_cuda_version(
cuda_version), normalize_cuda_version(cudnn_version)
if not cuda_version and is_windows_platform():
try:
cuda_vers = [
int(k.replace('CUDA_PATH_V', '').replace('_', ''))
for k in os.environ.keys() if k.startswith('CUDA_PATH_V')
]
cuda_vers = max(cuda_vers)
if cuda_vers > 40:
cuda_version = cuda_vers
except:
pass
if not cuda_version:
try:
try:
nvcc = 'nvcc.exe' if is_windows_platform() else 'nvcc'
if is_windows_platform() and 'CUDA_PATH' in os.environ:
nvcc = os.path.join(os.environ['CUDA_PATH'], nvcc)
output = Argv(nvcc, '--version').get_output()
except OSError:
raise CudaNotFound('nvcc not found')
match = re.search(r'release (.{3})', output).group(1)
cuda_version = Text(int(float(match) * 10))
except:
pass
if not cuda_version:
try:
try:
output = Argv('nvidia-smi', ).get_output()
except OSError:
raise CudaNotFound('nvcc not found')
match = re.search(r'CUDA Version: ([0-9]+).([0-9]+)', output)
match = match.group(1) + '.' + match.group(2)
cuda_version = Text(int(float(match) * 10))
except:
pass
if not cudnn_version:
try:
cuda_lib = which('nvcc')
if is_windows_platform:
cudnn_h = path.sep.join(
cuda_lib.split(path.sep)[:-2] + ['include', 'cudnn.h'])
else:
cudnn_h = path.join(
path.sep,
*(cuda_lib.split(path.sep)[:-2] +
['include', 'cudnn.h']))
cudnn_major, cudnn_minor = None, None
try:
include_file = open(cudnn_h)
except OSError:
raise CudaNotFound('Could not read cudnn.h')
with include_file:
for line in include_file:
if 'CUDNN_MAJOR' in line:
cudnn_major = line.split()[-1]
if 'CUDNN_MINOR' in line:
cudnn_minor = line.split()[-1]
if cudnn_major and cudnn_minor:
break
cudnn_version = cudnn_major + (cudnn_minor or '0')
except:
pass
return (normalize_cuda_version(cuda_version or 0),
normalize_cuda_version(cudnn_version or 0))
def send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open("var/test_messages.json", "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + Text(random.randint(1, 3))
saved_data['subject'] = message.subject
# Spoofing time not supported with threading
if options['threads'] != 1:
message.pub_date = timezone_now()
else:
# Distrubutes 80% of messages starting from 5 days ago, over a period
# of 3 days. Then, distributes remaining messages over past 24 hours.
spoofed_date = timezone_now() - timezone_timedelta(days=5)
if (num_messages < tot_messages * 0.8):
# Maximum of 3 days ahead, convert to minutes
time_ahead = 3 * 24 * 60
time_ahead //= int(tot_messages * 0.8)
else:
time_ahead = 24 * 60
time_ahead //= int(tot_messages * 0.2)
spoofed_minute = random.randint(time_ahead * num_messages,
time_ahead * (num_messages + 1))
spoofed_date += timezone_timedelta(minutes=spoofed_minute)
message.pub_date = spoofed_date
# We disable USING_RABBITMQ here, so that deferred work is
# executed in do_send_message_messages, rather than being
# queued. This is important, because otherwise, if run-dev.py
# wasn't running when populate_db was run, a developer can end
# up with queued events that reference objects from a previous
# life of the database, which naturally throws exceptions.
settings.USING_RABBITMQ = False
do_send_messages([{'message': message}])
settings.USING_RABBITMQ = True
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
return tot_messages
def get_image(
docker_requirement, # type: Dict[Text, Text]
pull_image, # type: bool
force_pull=False, # type: bool
tmp_outdir_prefix=DEFAULT_TMP_PREFIX # type: Text
): # type: (...) -> bool
"""
Retrieve the relevant Docker container image.
Returns True upon success
"""
found = False
if "dockerImageId" not in docker_requirement \
and "dockerPull" in docker_requirement:
docker_requirement["dockerImageId"] = docker_requirement[
"dockerPull"]
with _IMAGES_LOCK:
if docker_requirement["dockerImageId"] in _IMAGES:
return True
for line in subprocess.check_output(
["docker", "images", "--no-trunc",
"--all"]).decode('utf-8').splitlines():
try:
match = re.match(r"^([^ ]+)\s+([^ ]+)\s+([^ ]+)", line)
split = docker_requirement["dockerImageId"].split(":")
if len(split) == 1:
split.append("latest")
elif len(split) == 2:
# if split[1] doesn't match valid tag names, it is a part of repository
if not re.match(r'[\w][\w.-]{0,127}', split[1]):
split[0] = split[0] + ":" + split[1]
split[1] = "latest"
elif len(split) == 3:
if re.match(r'[\w][\w.-]{0,127}', split[2]):
split[0] = split[0] + ":" + split[1]
split[1] = split[2]
del split[2]
# check for repository:tag match or image id match
if (match and
((split[0] == match.group(1)
and split[1] == match.group(2)) or
docker_requirement["dockerImageId"] == match.group(3))):
found = True
break
except ValueError:
pass
if (force_pull or not found) and pull_image:
cmd = [] # type: List[Text]
if "dockerPull" in docker_requirement:
cmd = ["docker", "pull", str(docker_requirement["dockerPull"])]
_logger.info(Text(cmd))
subprocess.check_call(cmd, stdout=sys.stderr)
found = True
elif "dockerFile" in docker_requirement:
dockerfile_dir = str(
tempfile.mkdtemp(prefix=tmp_outdir_prefix))
with open(os.path.join(dockerfile_dir, "Dockerfile"),
"wb") as dfile:
dfile.write(
docker_requirement["dockerFile"].encode('utf-8'))
cmd = [
"docker", "build",
"--tag=%s" % str(docker_requirement["dockerImageId"]),
dockerfile_dir
]
_logger.info(Text(cmd))
subprocess.check_call(cmd, stdout=sys.stderr)
found = True
elif "dockerLoad" in docker_requirement:
cmd = ["docker", "load"]
_logger.info(Text(cmd))
if os.path.exists(docker_requirement["dockerLoad"]):
_logger.info(u"Loading docker image from %s",
docker_requirement["dockerLoad"])
with open(docker_requirement["dockerLoad"], "rb") as dload:
loadproc = subprocess.Popen(cmd,
stdin=dload,
stdout=sys.stderr)
else:
loadproc = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=sys.stderr)
assert loadproc.stdin is not None
_logger.info(u"Sending GET request to %s",
docker_requirement["dockerLoad"])
req = requests.get(docker_requirement["dockerLoad"],
stream=True)
size = 0
for chunk in req.iter_content(1024 * 1024):
size += len(chunk)
_logger.info("\r%i bytes", size)
loadproc.stdin.write(chunk)
loadproc.stdin.close()
rcode = loadproc.wait()
if rcode != 0:
raise WorkflowException(
"Docker load returned non-zero exit status %i" %
(rcode))
found = True
elif "dockerImport" in docker_requirement:
cmd = [
"docker", "import",
str(docker_requirement["dockerImport"]),
str(docker_requirement["dockerImageId"])
]
_logger.info(Text(cmd))
subprocess.check_call(cmd, stdout=sys.stderr)
found = True
if found:
with _IMAGES_LOCK:
_IMAGES.add(docker_requirement["dockerImageId"])
return found
def __new__(cls, val):
# type: (Text) -> Any
assert "@" in val, "Email is not valid"
return Text.__new__(cls, val) # type: ignore
