def cpu_image(node_id: NodeID) -> ImageParams:
image = Image(
name="simcore/services/comp/pytest/cpu_image",
tag="1.5.5",
node_requirements=NodeRequirements(
CPU=1, RAM=parse_obj_as(ByteSize, "128 MiB"), GPU=None, MPI=None
),
) # type: ignore
return ImageParams(
image=image,
expected_annotations={
"resources": {
"CPU": 1.0,
"RAM": 128 * 1024 * 1024,
}
},
fake_tasks={node_id: image},
)
async def get_volume_inventory(self) -> List[VolumeDeviceModel]:
cmd = "ceph-volume inventory --format=json"
stdout, stderr, rc = await self.call(cmd)
if rc != 0:
raise CephadmError(stderr)
try:
devs = json.loads(stdout)
logger.debug(json.dumps(devs, indent=2))
except json.decoder.JSONDecodeError as e:
raise CephadmError("format error while obtaining inventory") from e
inventory = parse_obj_as(List[VolumeDeviceModel], devs)
for d in inventory:
if not d.human_readable_type:
if d.sys_api.rotational:
d.human_readable_type = "hdd"
else:
d.human_readable_type = "ssd"
return inventory
def fake_task_output_data(
fake_io_schema: Dict[str, Dict[str, str]],
fake_io_data: Dict[str, Any],
faker: Faker,
) -> TaskOutputData:
converted_data = {
key: {
"url": faker.url(),
"file_mapping": next(iter(fake_io_schema[key]["fileToKeyMap"]))
if "fileToKeyMap" in fake_io_schema[key]
else None,
}
if fake_io_schema[key]["type"] == "data:*/*"
else value
for key, value in fake_io_data.items()
}
data = parse_obj_as(TaskOutputData, converted_data)
assert data
return data
def test_parsing_model_naming():
with pytest.raises(ValidationError) as exc_info:
parse_obj_as(int, 'a')
assert str(exc_info.value).split(
'\n')[0] == '1 validation error for ParsingModel[int]'
with pytest.raises(ValidationError) as exc_info:
parse_obj_as(int, 'a', type_name='ParsingModel')
assert str(
exc_info.value).split('\n')[0] == '1 validation error for ParsingModel'
with pytest.raises(ValidationError) as exc_info:
parse_obj_as(int, 'a', type_name=lambda type_: type_.__name__)
assert str(exc_info.value).split('\n')[0] == '1 validation error for int'
def get_authorization(self, fsname: str,
clientid: Optional[str]) -> CephFSAuthorizationModel:
if not clientid:
clientid = "default"
cmd = {
"prefix": "auth get",
"entity": f"client.{fsname}-{clientid}",
"format": "json"
}
try:
res = self.mon.call(cmd)
except CephCommandError as e:
if e.rc == errno.ENOENT:
raise CephFSNoAuthorizationError(e.message)
raise CephFSError(str(e)) from e
lst = parse_obj_as(List[CephFSAuthorizationModel], res)
if len(lst) == 0:
raise CephFSNoAuthorizationError()
return lst[0]
async def factory() -> DaskClient:
client = await DaskClient.create(
app=minimal_app,
settings=minimal_app.state.settings.DASK_SCHEDULER,
endpoint=parse_obj_as(AnyUrl, dask_spec_local_cluster.scheduler_address),
authentication=NoAuthentication(),
)
assert client
assert client.app == minimal_app
assert client.settings == minimal_app.state.settings.DASK_SCHEDULER
assert not client._subscribed_tasks
assert client.dask_subsystem.client
assert not client.dask_subsystem.gateway
assert not client.dask_subsystem.gateway_cluster
scheduler_infos = client.dask_subsystem.client.scheduler_info() # type: ignore
print(
f"--> Connected to scheduler via client {client=} to scheduler {scheduler_infos=}"
)
created_clients.append(client)
return client
class Settings(BaseSettings):
API_V1_STR: str = "/api/v1"
SERVER_NAME: str
SERVER_HOST: AnyHttpUrl
# BACKEND_CORS_ORIGINS is a JSON-formatted list of origins
# e.g: '["http://localhost", "http://*****:*****@validator("BACKEND_CORS_ORIGINS", pre=True)
def assemble_cors_origins(
cls, v: Union[str, List[str]]) -> Union[List[str], str]:
if isinstance(v, str) and not v.startswith("["):
return [i.strip() for i in v.split(",")]
elif isinstance(v, (list, str)):
return v
raise ValueError(v)
PROJECT_NAME: str
SENTRY_DSN: Optional[HttpUrl] = None
WIKIFACTORY_API_BASE_URL: AnyHttpUrl = parse_obj_as(
AnyHttpUrl, "https://wikifactory.com")
@validator("WIKIFACTORY_API_BASE_URL", pre=True)
def wikifactory_base_url(cls, v: Union[str, AnyHttpUrl]) -> AnyHttpUrl:
if isinstance(v, str):
return parse_obj_as(AnyHttpUrl, v)
elif isinstance(v, AnyHttpUrl):
return v
raise ValueError(v)
WIKIFACTORY_API_HOST: Optional[str] = None
@validator("WIKIFACTORY_API_HOST")
def wikifactory_host_from_url(cls, v: Optional[str],
values: Dict[str, Any]) -> str:
if isinstance(v, str):
return v
api_base_url = values.get("WIKIFACTORY_API_BASE_URL")
if api_base_url:
if api_base_url.port:
return f"{api_base_url.host}:{api_base_url.port}"
return api_base_url.host
raise ValueError(api_base_url)
EXPORTER_GIT_MAIL: Optional[str] = None
EXPORTER_GIT_USER: Optional[str] = None
@validator("SENTRY_DSN", pre=True)
def sentry_dsn_can_be_blank(cls, v: str) -> Optional[str]:
if not v:
return None
return v
POSTGRES_SERVER: str
POSTGRES_USER: str
POSTGRES_PASSWORD: str
POSTGRES_DB: str
PYTEST_POSTGRES_SERVER: Optional[str]
PYTEST_POSTGRES_USER: Optional[str]
PYTEST_POSTGRES_PASSWORD: Optional[str]
PYTEST_POSTGRES_DB: Optional[str]
SQLALCHEMY_DATABASE_URI: Optional[PostgresDsn] = None
@validator("SQLALCHEMY_DATABASE_URI", pre=True)
def assemble_db_connection(cls, v: Optional[str],
values: Dict[str, Any]) -> Any:
if isinstance(v, str):
return v
if "pytest" in sys.modules:
return PostgresDsn.build(
scheme="postgresql",
user=values.get("PYTEST_POSTGRES_USER"),
password=values.get("PYTEST_POSTGRES_PASSWORD"),
host=values.get("PYTEST_POSTGRES_SERVER"),
path=f"/{values.get('PYTEST_POSTGRES_DB') or ''}",
)
return PostgresDsn.build(
scheme="postgresql",
user=values.get("POSTGRES_USER"),
password=values.get("POSTGRES_PASSWORD"),
host=values.get("POSTGRES_SERVER"),
path=f"/{values.get('POSTGRES_DB') or ''}",
)
# TODO - define validations
JOBS_BASE_PATH: str
@validator("JOBS_BASE_PATH", pre=True)
def ensure_download_path(cls, v: str) -> str:
if "pytest" in sys.modules:
current_dir = os.path.dirname(os.path.realpath(__file__))
return os.path.normpath(
os.path.join(current_dir, "../tests/test_files"))
# if directory can't be written, raise ValueError(v)
return v
BROKER_URL: Optional[AnyUrl]
class Config:
case_sensitive = True
def main() -> None:
"""Analyze mixed initiative schemas."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--input-dir",
type=Path,
required=True,
help="Input schema directory path.")
args = parser.parse_args()
if not args.input_dir.is_dir():
raise IOError(
f"Schema directory {args.input_dir} does not exist or is not a directory"
)
yaml_files = sorted(args.input_dir.glob("*.yaml"))
yaml_schemas = []
for yaml_file in yaml_files:
with yaml_file.open() as file:
yaml_data = yaml.safe_load(file)
yaml_schemas.extend(yaml_data)
schemas: List[Schema] = parse_obj_as(List[Schema], yaml_schemas)
for schema in schemas:
author = get_author(schema)
schema.private_data["author"] = author
schema.private_data["tracking"] = parse_obj_as(
List[LogItem], schema.private_data["tracking"])
for log_item in schema.private_data["tracking"]:
log_item.author = author
wrapper = textwrap.TextWrapper(width=80, subsequent_indent=" " * 4)
print("+++ General +++")
log_items = list(
itertools.chain.from_iterable(schema.private_data["tracking"]
for schema in schemas))
log_types = sorted(set(log_item.type for log_item in log_items))
print(wrapper.fill(f"Log types: {log_types}"))
authors = sorted(set(schema.private_data["author"] for schema in schemas))
print(f"Authors: {authors}")
log_type_counts = dict(
Counter(log_item.type for log_item in log_items).most_common())
print("Log type counts:")
for log_type, counts in log_type_counts.items():
print(f"\t{log_type}: {counts}")
log_type_counts_per_user = {
log_type: dict(
Counter(i.author for i in log_items
if i.type == log_type).most_common())
for log_type in log_types
}
print("Log type counts per user:"******"\t{log_type}: {counts_per_user}")
bad_reorders = sum(
log_item.type == "reorder_event"
and log_item.data["old_index"] != log_item.data["new_index"]
for log_item in log_items)
print(
f"Percent actual re-orderings: {bad_reorders / log_type_counts['reorder_event']:.0%} "
f"({bad_reorders}/{log_type_counts['reorder_event']})")
print("\n+++ Total step entering time +++")
times_taken = []
for schema in schemas:
start_time = schema.private_data["tracking"][0].date
end_time = schema.private_data["tracking"][-1].date
times_taken.append(end_time - start_time)
seconds_taken = [t.total_seconds() for t in times_taken]
print(
f"Minimum total step entering time: {timedelta(seconds=round(min(seconds_taken)))}"
)
print(
f"Maximum total step entering time: {timedelta(seconds=round(max(seconds_taken)))}"
)
print(
f"Mean total step entering time: {timedelta(seconds=round(statistics.mean(seconds_taken)))}"
)
print(f"Median total step entering time: "
f"{timedelta(seconds=round(statistics.median(seconds_taken)))}")
print("\n+++ Schema lengths +++")
script_lengths = sorted(len(schema.steps) for schema in schemas)
print(f"Minimum schema length: {min(script_lengths)}")
print(f"Maximum schema length: {max(script_lengths)}")
print(f"Mean schema length: {statistics.mean(script_lengths):.1f}")
print(f"Median schema length: {statistics.median(script_lengths):.1f}")
print("\n+++ Suggestion usage +++")
total_suggestions_given = 0
total_events_added = 0
total_suggestions_used_exact = 0
total_suggestions_used_ignore_case = 0
total_suggestions_used_rough = 0
for schema in schemas:
last_suggestion_selected: Optional[str] = None
for log_item in schema.private_data["tracking"]:
if log_item.type == "add_event":
total_events_added += 1
if last_suggestion_selected is not None:
suggested = last_suggestion_selected
added = log_item.data
if suggested == added:
total_suggestions_used_exact += 1
if suggested.lower() == added.lower():
total_suggestions_used_ignore_case += 1
if SequenceMatcher(None, suggested.lower(),
added.lower()).ratio() > 0.8:
total_suggestions_used_rough += 1
elif log_item.type == "gpt2_suggestion_output":
total_suggestions_given += 1
last_suggestion_selected = None
elif log_item.type == "gpt2_suggestion_select":
last_suggestion_selected = log_item.data
print(f"Total suggestion sets given: {total_suggestions_given}")
print(f"Total events added: {total_events_added}")
print(
f"Total suggestions used (exact match): {total_suggestions_used_exact} "
f"({total_suggestions_used_exact/total_events_added:.1%})")
print(f"Total suggestions used (exact match, ignoring case): "
f"{total_suggestions_used_ignore_case} "
f"({total_suggestions_used_ignore_case/total_events_added:.1%})")
print(
f"Total suggestions used (rough match): {total_suggestions_used_rough} "
f"({total_suggestions_used_rough/total_events_added:.1%})")
# Although timing is recorded in the logs, this is to get benchmarks with the improved server
print("\n+++ Suggestion generation time +++")
times = []
for log_item in log_items:
if log_item.type != "gpt2_suggestion_input":
continue
gpt2_input = log_item.data["input"]
request_url = "https://dev.example.org/api/get_gpt2_suggestions"
start = time.time()
request_response = requests.get(request_url,
params=gpt2_input,
timeout=30)
end = time.time()
times.append(end - start)
# Abort script if GPT-2 server does not return suggestions properly
if request_response.status_code != HTTPStatus.OK:
raise RuntimeError(
f"GPT-2 server returned status code {request_response.status_code}"
)
if "suggestions" not in request_response.json():
raise RuntimeError("GPT-2 server returned malformed output")
print(f"Total suggestion generation time: {sum(times):.1f} s")
print(f"Minimum suggestion generation time: {min(times):.1f} s")
print(f"Maximum suggestion generation time: {max(times):.1f} s")
print(f"Mean suggestion generation time: {statistics.mean(times):.1f} s")
print(
f"Median suggestion generation time: {statistics.median(times):.1f} s")
def host_ls(self) -> List[OrchHostListModel]:
cmd = {"prefix": "orch host ls"}
res = self.call(cmd)
return parse_obj_as(List[OrchHostListModel], res)
def get_animes() -> List[Anime]:
with client:
r = client.get(url=f"{TWIST_URL}{ANIME_ENDPOINT}")
r.raise_for_status()
return parse_obj_as(List[Anime], r.json())
def calculate_indicators(
market_data: List[InputMarketDataSchema],
) -> List[IndicatorSchema]:
"""calculate_indicator:
Transform streams of market data into an list of indicators calculated from that data.
Returns two indicators, one for buy and one for sell/close position.
"""
no_op = IndicatorSchema(label="None", indicator=[])
if not all(map(lambda x: check_is_good_for_signal(x.ticks), market_data)):
return [no_op, no_op]
min_ask_bid_ratio = 0.0035
disregard_bars_count: int = 10
rolling_indicator_window: str = "180s"
primary_data = parse_obj_as(List[PricepointSchema], market_data[0].ticks)
secondary_data = parse_obj_as(List[PricepointSchema], market_data[1].ticks)
# Convert Pydantic lists to dataframes
primary_dataframe = pd.DataFrame([x.dict() for x in primary_data])
primary_dataframe["Time"] = pd.to_datetime(primary_dataframe.time, unit="s")
primary_dataframe.set_index("Time", inplace=True)
primary_dataframe.sort_index(inplace=True)
secondary_dataframe = pd.DataFrame([x.dict() for x in secondary_data])
secondary_dataframe["Time"] = pd.to_datetime(secondary_dataframe.time, unit="s")
secondary_dataframe.set_index("Time", inplace=True)
secondary_dataframe.sort_index(inplace=True)
primary_without_latest = primary_dataframe["price"].shift(disregard_bars_count)
secondary_without_latest = secondary_dataframe["price"].shift(disregard_bars_count)
coefficients = secondary_without_latest / primary_without_latest
pseudo_spread_secondary = secondary_dataframe["price"] * min_ask_bid_ratio
# Calculate inter-market slip difference and get indicating list
arbitrage_difference = (
primary_dataframe["price"] * coefficients
) - secondary_dataframe["price"]
arbitrage_indicator = (arbitrage_difference / pseudo_spread_secondary).dropna()
# Disregard noise values (x < 1.5)
arbitrage_indicator.loc[arbitrage_indicator < 1.5] = 0
rolling_buy_indicator: pd.Series = arbitrage_indicator.rolling(
rolling_indicator_window
).sum()
sell_difference = (primary_dataframe["price"] * coefficients) - secondary_dataframe[
"price"
]
sell_indicator = (sell_difference / pseudo_spread_secondary).dropna()
sell_indicator.loc[sell_indicator > -0.8] = 0
rolling_sell_indicator: pd.Series = sell_indicator.rolling(
rolling_indicator_window
).sum()
buy_indicator = IndicatorSchema(
label=IndicatorsEnum.buy_probability_line,
indicator=[
dict(timestamp=x, value=y) for x, y in rolling_buy_indicator.items()
],
)
sell_indicator = IndicatorSchema(
label=IndicatorsEnum.sell_probability_line,
indicator=[
dict(timestamp=x, value=y) for x, y in rolling_sell_indicator.items()
],
)
return [buy_indicator, sell_indicator]
def get_pools_stats(self) -> List[CephOSDPoolStatsModel]:
cmd: Dict[str, str] = {"prefix": "osd pool stats", "format": "json"}
results: Dict[str, Any] = self.call(cmd)
return parse_obj_as(List[CephOSDPoolStatsModel], results)
from os import path
import sys
BASE_DIR = path.dirname(path.dirname(path.abspath(__file__)))
sys.path.append(BASE_DIR)
from textwrap import dedent
from pydantic.tools import parse_obj_as
from pepperbot.command import as_command, pattern, with_command
from pepperbot.main import *
from pepperbot.models.sender import Sender
class TestModel(BaseModel):
order: Text
face: Face
class Config:
arbitrary_types_allowed = True
# items = parse_obj_as(List[Face], [Text("1234")])
items = parse_obj_as(TestModel, [Text("1234")])
debug(items)
import os
from pydantic import EmailStr, HttpUrl
from pydantic.tools import parse_obj_as
from settings.adapters import RESTAdapterSettings
from settings.grabbers import FeedlyGrabberSettings
from settings.notification_channels import \
SendPulseNotificationChannelSettings
zzr_settings = RESTAdapterSettings(
client_id=os.environ["ZZR_CLIENT_ID"],
client_secret=os.environ["ZZR_CLIENT_SECRET"],
token_url=parse_obj_as(HttpUrl, os.environ["ZZR_TOKEN_URL"]),
base_url=parse_obj_as(HttpUrl, os.environ["ZZR_BASE_URL"]),
authorization_url=parse_obj_as(HttpUrl,
os.environ["ZZR_AUTHORIZATION_URL"]),
)
feedly_settings = FeedlyGrabberSettings(
refresh_token=str(os.environ.get("FEEDLY_REFRESH_TOKEN")),
returned_posts_count=int(os.environ["FEEDLY_RETURNED_POSTS_COUNT"]),
)
emails = [
EmailStr(email)
for email in os.environ["SENDPULSE_NOTIFICATION_EMAILS"].split(",")
]
sendpulse_settings = SendPulseNotificationChannelSettings(
emails=emails,
# '11id': '123',
# 'signup_ts': '2019-06-01 12:22',
# 'friends': [1, 2, '3'],
# '表头': 'head',
# }
# try:
# user = User(**external_data)
# print(user.dict())
# except ValidationError as e:
# print(e.errors())
# # print(e.json())
external_data_list = [{
'id': '123',
'name': 'a',
'signup_ts': '2019-06-01 12:22',
'friends': [1, 2, '3'],
'表头': 'head',
}, {
'id': '123',
'name': 'b',
'signup_ts': '2019-06-01 12:22',
'friends': [1, 2, '3'],
'表头': 'head',
}]
try:
user_list = parse_obj_as(List[User], external_data_list)
print([dict(user) for user in user_list])
except ValidationError as e:
print(e.errors())
async def execute(self, redis_list, writer, state, auth, packer):
try:
args = []
if len(redis_list) != self.num_args:
writer.write(b"-NUM_ARG_MISMATCH\r\n")
await writer.drain()
return True
pos = 0
for (arg_name, arg_type) in self.signature:
if arg_type == Auth:
args.append(auth)
continue
elif arg_type == AuthRequired:
if not auth.value:
await write_permission_denied(writer)
return True
args.append(AuthRequired(auth.value))
continue
elif arg_type == ConnState:
args.append(state)
continue
redis_value = redis_list[pos]
pos += 1
try:
raw_value = msgpack.unpackb(redis_value)
except msgpack.UnpackException as e:
loc = json.dumps(["command", arg_name])
msg = json.dumps("invalid msgpack")
writer.write(
f"-INVALID_MSGPACK {loc} {msg}\r\n".encode("utf8"))
await writer.drain()
return True
try:
obj = parse_obj_as(arg_type, raw_value)
except ValidationError as e:
err = e.errors()[0]
loc = json.dumps(("command", arg_name) + err["loc"][1:])
msg = json.dumps(err["msg"])
t = json.dumps(err["type"])
writer.write(
f"-VALIDATION_ERROR {loc} {msg} {t}\r\n".encode(
"utf8"))
await writer.drain()
return True
args.append(obj)
result = await self.handler(*args)
to_send = pack_msgpack(packer, result)
writer.write(b"$%d\r\n%b\r\n" % (len(to_send), to_send))
await writer.drain()
return False
except Exception as e:
msg = json.dumps(str(e))
writer.write(f"-UNEXPECTED_ERROR {msg}\r\n".encode("utf8"))
await writer.drain()
raise e
def test_object_list(self, test_object_dict: Dict[str, Any]) -> None:
model = parse_obj_as(list[ObjectModel], [test_object_dict])
assert len(model) == 1
assert model[0].dict(by_alias=True) == test_object_dict
import sys
from typing import Any, List, Mapping, Tuple
from pydantic.tools import parse_obj_as
from common import MethodAbs, get_metrics
with open(sys.argv[1], "r") as f:
dump1: Mapping[str, List[Tuple[str, Any]]] = json.load(f)
with open(sys.argv[2], "r") as f:
dump2: Mapping[str, List[Tuple[str, Any]]] = json.load(f)
for m in set(dump1.keys()).intersection(set(dump2.keys())):
m_printed = False
abs1 = dump1[m]
abs2 = dump2[m]
parsed1 = parse_obj_as(MethodAbs, abs1)
parsed2 = parse_obj_as(MethodAbs, abs2)
metrics1 = get_metrics(parsed1)
metrics2 = get_metrics(parsed2)
# assert len(metrics1) == len(metrics2)
if len(metrics1) != len(metrics2):
continue
if metrics1 != metrics2:
for s1, s2 in zip(metrics1, metrics2):
unit = s1["unit"]
# assert s1["unit"] == s2["unit"]
if s1["unit"] != s2["unit"]:
continue
for k in s1["metrics"].keys():
m1 = s1["metrics"][k]
m2 = s2["metrics"][k]
def test_parse_obj(obj, type_, parsed):
assert parse_obj_as(type_, obj) == parsed
import json
import sys
from typing import Any, List, Mapping, Tuple
from pydantic.tools import parse_obj_as
from common import MethodAbs, get_metrics
with open(sys.argv[1], "r") as f:
dump: Mapping[str, List[Tuple[str, Any]]] = json.load(f)
for method, abs_json in dump.items():
abstractions = parse_obj_as(MethodAbs, abs_json)
metrics = get_metrics(abstractions)
print("=====" * 20)
print("===== Method: " + method)
print(abstractions.body)
for stmt, details in abstractions.abstractions:
print(" > " + stmt)
print(" - Local:")
for k, v in details.localNodeMap.items():
print(f" - {k}: {v}")
print(" - Global:")
for k, v in details.globalNodeMap.items():
print(f" - {k}: {v}")
print(" - Node handlers:")
for nodeID, event, methods in details.nodeHandlerMap:
print(f" - {nodeID}, {event}: {methods}")
print(" - Dialog handlers:")
for nodeID, event, methods in details.dialogHandlerMap:
print(f" - {nodeID}, {event}: {methods}")
print()
def test_parse_mapping_as():
inputs = {'1': '2'}
assert parse_obj_as(Dict[int, int], inputs) == {1: 2}
def ubuntu_task(request: FixtureRequest,
ftp_server: List[URL]) -> ServiceExampleParam:
"""Creates a console task in an ubuntu distro that checks for the expected files and error in case they are missing"""
integration_version = version.Version(request.param)
print("Using service integration:", integration_version)
# defines the inputs of the task
input_data = TaskInputData.parse_obj({
"input_1": 23,
"input_23": "a string input",
"the_input_43": 15.0,
"the_bool_input_54": False,
**{
f"some_file_input_{index+1}": FileUrl(url=f"{file}")
for index, file in enumerate(ftp_server)
},
**{
f"some_file_input_with_mapping{index+1}": FileUrl(url=f"{file}",
file_mapping=f"{index+1}/some_file_input")
for index, file in enumerate(ftp_server)
},
})
# check in the console that the expected files are present in the expected INPUT folder (set as ${INPUT_FOLDER} in the service)
file_names = [file.path for file in ftp_server]
list_of_commands = [
"echo User: $(id $(whoami))",
"echo Inputs:",
"ls -tlah -R ${INPUT_FOLDER}",
"echo Outputs:",
"ls -tlah -R ${OUTPUT_FOLDER}",
"echo Logs:",
"ls -tlah -R ${LOG_FOLDER}",
]
list_of_commands += [
f"(test -f ${{INPUT_FOLDER}}/{file} || (echo ${{INPUT_FOLDER}}/{file} does not exists && exit 1))"
for file in file_names
] + [f"echo $(cat ${{INPUT_FOLDER}}/{file})" for file in file_names]
input_json_file_name = ("inputs.json"
if integration_version > LEGACY_INTEGRATION_VERSION
else "input.json")
list_of_commands += [
f"(test -f ${{INPUT_FOLDER}}/{input_json_file_name} || (echo ${{INPUT_FOLDER}}/{input_json_file_name} file does not exists && exit 1))",
f"echo $(cat ${{INPUT_FOLDER}}/{input_json_file_name})",
f"sleep {randint(1,4)}",
]
# defines the expected outputs
jsonable_outputs = {
"pytest_string": "is quite an amazing feat",
"pytest_integer": 432,
"pytest_float": 3.2,
"pytest_bool": False,
}
output_file_url = next(iter(ftp_server)).with_path("output_file")
expected_output_keys = TaskOutputDataSchema.parse_obj({
**{k: {
"required": True
}
for k in jsonable_outputs.keys()},
**{
"pytest_file": {
"required": True,
"mapping": "a_outputfile",
"url": f"{output_file_url}",
},
"pytest_file_with_mapping": {
"required": True,
"mapping": "subfolder/a_outputfile",
"url": f"{output_file_url}",
},
},
})
expected_output_data = TaskOutputData.parse_obj({
**jsonable_outputs,
**{
"pytest_file": {
"url": f"{output_file_url}",
"file_mapping": "a_outputfile",
},
"pytest_file_with_mapping": {
"url": f"{output_file_url}",
"file_mapping": "subfolder/a_outputfile",
},
},
})
jsonized_outputs = json.dumps(jsonable_outputs).replace('"', '\\"')
output_json_file_name = ("outputs.json" if
integration_version > LEGACY_INTEGRATION_VERSION
else "output.json")
# check for the log file if legacy version
list_of_commands += [
"echo $(ls -tlah ${LOG_FOLDER})",
f"(test {'!' if integration_version > LEGACY_INTEGRATION_VERSION else ''} -f ${{LOG_FOLDER}}/{LEGACY_SERVICE_LOG_FILE_NAME} || (echo ${{LOG_FOLDER}}/{LEGACY_SERVICE_LOG_FILE_NAME} file does {'' if integration_version > LEGACY_INTEGRATION_VERSION else 'not'} exists && exit 1))",
]
if integration_version == LEGACY_INTEGRATION_VERSION:
list_of_commands = [
f"{c} >> ${{LOG_FOLDER}}/{LEGACY_SERVICE_LOG_FILE_NAME}"
for c in list_of_commands
]
# set the final command to generate the output file(s) (files and json output)
list_of_commands += [
f"echo {jsonized_outputs} > ${{OUTPUT_FOLDER}}/{output_json_file_name}",
"echo 'some data for the output file' > ${OUTPUT_FOLDER}/a_outputfile",
"mkdir -p ${OUTPUT_FOLDER}/subfolder",
"echo 'some data for the output file' > ${OUTPUT_FOLDER}/subfolder/a_outputfile",
]
log_file_url = parse_obj_as(
AnyUrl, f"{next(iter(ftp_server)).with_path('log.dat')}")
return ServiceExampleParam(
docker_basic_auth=DockerBasicAuth(server_address="docker.io",
username="******",
password=""),
#
# NOTE: we use sleeper because it defines a user
# that can write in outputs and the
# sidecar can remove the outputs dirs
#
service_key="itisfoundation/sleeper",
service_version="2.1.2",
command=[
"/bin/bash",
"-c",
" && ".join(list_of_commands),
],
input_data=input_data,
output_data_keys=expected_output_keys,
log_file_url=log_file_url,
expected_output_data=expected_output_data,
expected_logs=[
'{"input_1": 23, "input_23": "a string input", "the_input_43": 15.0, "the_bool_input_54": false}',
"This is the file contents of 'file_1'",
"This is the file contents of 'file_2'",
"This is the file contents of 'file_3'",
],
integration_version=integration_version,
)
def devices_ls(self) -> List[OrchDevicesPerHostModel]:
cmd = {"prefix": "orch device ls"}
res = self.call(cmd)
return parse_obj_as(List[OrchDevicesPerHostModel], res)
def _get_buckets(self) -> List[BucketModel]:
response = self._ensured_client.list_buckets()
return parse_obj_as(List[BucketModel], response.get("Buckets"))
def wikifactory_base_url(cls, v: Union[str, AnyHttpUrl]) -> AnyHttpUrl:
if isinstance(v, str):
return parse_obj_as(AnyHttpUrl, v)
elif isinstance(v, AnyHttpUrl):
return v
raise ValueError(v)
def get_bucket_objects(self, bucket: str) -> List[ObjectModel]:
response = self._ensured_client.list_objects(Bucket=bucket)
logger.debug("List objects response:\n%s", response)
return parse_obj_as(List[ObjectModel], response.get("Contents"))
def __validate_cls_namespace__(name: str,
namespace: Dict) -> None: # noqa C901
"""Validate the class name space in place"""
annotations = resolve_annotations(namespace.get("__annotations__", {}),
namespace.get("__module__"))
config = validate_config(namespace.get("Config", BaseODMConfig), name)
odm_fields: Dict[str, ODMBaseField] = {}
references: List[str] = []
bson_serialized_fields: Set[str] = set()
mutable_fields: Set[str] = set()
# Make sure all fields are defined with type annotation
for field_name, value in namespace.items():
if (should_touch_field(value=value) and not is_dunder(field_name)
and field_name not in annotations):
raise TypeError(
f"field {field_name} is defined without type annotation")
# Validate fields types and substitute bson fields
for (field_name, field_type) in annotations.items():
if not is_dunder(field_name) and should_touch_field(
type_=field_type):
substituted_type = validate_type(field_type)
# Handle BSON serialized fields after substitution to allow some
# builtin substitution
bson_serialization_method = getattr(substituted_type,
"__bson__", None)
if bson_serialization_method is not None:
bson_serialized_fields.add(field_name)
annotations[field_name] = substituted_type
# Validate fields
for (field_name, field_type) in annotations.items():
value = namespace.get(field_name, Undefined)
if is_dunder(field_name) or not should_touch_field(
value, field_type):
continue # pragma: no cover
# https://github.com/nedbat/coveragepy/issues/198
if isinstance(value, PDFieldInfo):
raise TypeError(
"please use odmantic.Field instead of pydantic.Field")
if is_type_mutable(field_type):
mutable_fields.add(field_name)
if lenient_issubclass(field_type, EmbeddedModel):
if isinstance(value, ODMFieldInfo):
namespace[field_name] = value.pydantic_field_info
key_name = (value.key_name
if value.key_name is not None else field_name)
primary_field = value.primary_field
else:
key_name = field_name
primary_field = False
odm_fields[field_name] = ODMEmbedded(
primary_field=primary_field,
model=field_type,
key_name=key_name,
model_config=config,
)
elif lenient_issubclass(field_type, Model):
if not isinstance(value, ODMReferenceInfo):
raise TypeError(
f"cannot define a reference {field_name} (in {name}) without"
" a Reference assigned to it")
key_name = value.key_name if value.key_name is not None else field_name
raise_on_invalid_key_name(key_name)
odm_fields[field_name] = ODMReference(model=field_type,
key_name=key_name,
model_config=config)
references.append(field_name)
del namespace[
field_name] # Remove default ODMReferenceInfo value
else:
if isinstance(value, ODMFieldInfo):
key_name = (value.key_name
if value.key_name is not None else field_name)
raise_on_invalid_key_name(key_name)
odm_fields[field_name] = ODMField(
primary_field=value.primary_field,
key_name=key_name,
model_config=config,
)
namespace[field_name] = value.pydantic_field_info
elif value is Undefined:
odm_fields[field_name] = ODMField(primary_field=False,
key_name=field_name,
model_config=config)
else:
try:
parse_obj_as(field_type, value)
except ValidationError:
raise TypeError(
f"Unhandled field definition {name}: {repr(field_type)}"
f" = {repr(value)}")
odm_fields[field_name] = ODMField(primary_field=False,
key_name=field_name,
model_config=config)
duplicate_key = find_duplicate_key(odm_fields.values())
if duplicate_key is not None:
raise TypeError(f"Duplicated key_name: {duplicate_key} in {name}")
# NOTE: Duplicate key detection make sur that at most one primary key is
# defined
namespace["__annotations__"] = annotations
namespace["__odm_fields__"] = odm_fields
namespace["__references__"] = tuple(references)
namespace["__bson_serialized_fields__"] = frozenset(
bson_serialized_fields)
namespace["__mutable_fields__"] = frozenset(mutable_fields)
namespace["Config"] = config
def get_settings_override() -> Settings:
return Settings(redis_dsn=parse_obj_as(RedisDsn, 'redis://redis:6379/2'))
def return_fake_input_value(*args, **kwargs):
for value, value_type in zip(fake_inputs.values(), fake_io_schema.values()):
if value_type["type"] == "data:*/*":
yield parse_obj_as(AnyUrl, faker.url())
else:
yield value
def fmt_url(url_fmt: str, **kwargs: Union[int, str]) -> HttpUrl:
url: HttpUrl = parse_obj_as(HttpUrl, url_fmt.format(**kwargs))
return url
