def attr_to_json_dict(attr_obj: attr.Attribute) -> Dict[str, Any]:
"""Converts an attr-defined object to a JSON dict. The resulting dict should
be unstructured using |attr_from_json_dict| below, which uses the __module__
and __classname__ fields to reconstruct the object using the same Converter
used to unstructure it here."""
converter = cattr.Converter()
converter.register_unstructure_hook(
datetime.datetime,
lambda d: datetime_to_serializable(d))
attr_dict = cattr.unstructure(attr_obj)
attr_dict['__classname__'] = attr_obj.__class__.__name__
attr_dict['__module__'] = attr_obj.__module__
return attr_dict
def test_call_landingzone_create(capsys, requests_mock):
# Query project information.
project = factories.ProjectFactory()
i_args = {
"sodar_url": SODAR_SERVER_URL,
"sodar_api_token": SODAR_API_TOKEN,
"project_uuid": project.sodar_uuid,
}
i_tpl = "%(sodar_url)ssamplesheets/api/investigation/retrieve/%(project_uuid)s"
investigation = factories.InvestigationFactory()
requests_mock.register_uri(
"GET",
i_tpl % i_args,
headers={"Authorization": "Token %s" % i_args["sodar_api_token"]},
json=cattr.unstructure(investigation),
)
# Creation of landing zone.
l_tpl = "%(sodar_url)slandingzones/api/create/%(project_uuid)s"
lz_obj = factories.LandingZoneFactory(project=i_args["project_uuid"])
requests_mock.register_uri(
"POST",
l_tpl % i_args,
headers={"Authorization": "Token %s" % i_args["sodar_api_token"]},
json=cattr.unstructure(lz_obj),
)
main([
"--sodar-server-url",
SODAR_SERVER_URL,
"--sodar-api-token",
SODAR_API_TOKEN,
"landingzone",
"create",
project.sodar_uuid,
])
captured = capsys.readouterr()
assert json.dumps(cattr.unstructure(lz_obj)) + "\n" == captured.out
def on_export(self):
user_project: UserProject = app_settings.load_current_project()
current_project_folder = os.path.dirname(user_project.location)
file_location, _ = self.main_window.save_file(
"Export Environments",
current_project_folder,
file_filter="Environment Files (*.envs.json)",
)
if file_location:
envs: List[
Environment
] = app_settings.app_data_reader.get_environments_from_db()
envs_json = cattr.unstructure(envs)
Path(file_location).write_text(json.dumps(envs_json))
def wrapper(self, context, *args, **kwargs):
self.log.debug("Lineage called with inlets: %s, outlets: %s",
self.inlets, self.outlets)
ret_val = func(self, context, *args, **kwargs)
outlets = [unstructure(_to_dataset(x, f"{self.dag_id}.{self.task_id}"))
for x in self.outlets]
inlets = [unstructure(_to_dataset(x, None))
for x in self.inlets]
if self.outlets:
self.xcom_push(context,
key=PIPELINE_OUTLETS,
value=outlets,
execution_date=context['ti'].execution_date)
if self.inlets:
self.xcom_push(context,
key=PIPELINE_INLETS,
value=inlets,
execution_date=context['ti'].execution_date)
return ret_val
def _handle(self, params, request, *_args, **_kwargs):
"""Handle the GA4GH Beacon query.
NB: the remote ``Site`` object is stored in request.user. The permission logic ensures that only active
sites can query.
"""
allele_req = cattr.structure(dict(params.items()), BeaconAlleleRequest)
remote_site = request.user
# TODO: perform one large query only
project_pks = [p.pk for p in remote_site.get_all_projects()]
result = (
select(["*"])
.select_from(SmallVariant.sa)
.where(
and_(
SmallVariant.sa.case_id.in_(
select([Case.sa.id])
.select_from(Case.sa)
.where(Case.sa.project_id.in_(project_pks))
),
SmallVariant.sa.release == allele_req.assemblyId,
SmallVariant.sa.chromosome == allele_req.referenceName,
SmallVariant.sa.start == allele_req.start,
SmallVariant.sa.reference == allele_req.referenceBases,
SmallVariant.sa.alternative == allele_req.alternateBases,
)
)
)
sum_hom_alt = 0
sum_het_alt = 0
sum_hemi_alt = 0
for row in SQLALCHEMY_ENGINE.execute(result):
sum_hom_alt += row.num_hom_alt
sum_het_alt += row.num_het
sum_hemi_alt += row.num_hemi_alt
total_alleles = sum_hom_alt * 2 + sum_het_alt + sum_hemi_alt
site = Site.objects.get(role=Site.LOCAL)
if site.state != Site.ENABLED:
return (
Response({"detail": "The site is not enabled!"}, status=400, reason="invalid site"),
)
result = BeaconAlleleResponse(
beaconId=site.identifier,
apiVersion=API_VERSION,
exists=(total_alleles > 0),
alleleRequest=allele_req,
)
return Response(cattr.unstructure(result))
def build_experiment_config(
defaults: Optional[Union[dict, ExperimentConfig]]) -> ExperimentConfig:
if defaults is None:
defaults = {}
elif isinstance(defaults, ExperimentConfig):
defaults = cattr.unstructure(defaults)
args = {"uuid": defaults.get("uuid", uuid.uuid4())}
args["slug"] = click.prompt("Experiment slug",
type=str,
default=defaults.get("slug", None))
return cattr.structure(args, ExperimentConfig)
def update(*, sodar_url, sodar_api_token, project_uuid, project):
"""Update project information."""
while sodar_url.endswith("/"):
sodar_url = sodar_url[:-1]
url_tpl = "%(sodar_url)s/project/api/update/%(project_uuid)s"
url = url_tpl % {"sodar_url": sodar_url, "project_uuid": project_uuid}
logger.debug("HTTP PATCH request to %s", url)
headers = {"Authorization": "Token %s" % sodar_api_token}
data = cattr.unstructure(project)
data.pop("sodar_uuid")
r = requests.patch(url, headers=headers, data=data)
r.raise_for_status()
return cattr.structure(r.json(), models.Project)
def run():
s = Scraper()
p = Parser()
# get the draft group of the main slate
gcd = p.getcontests(s.getcontests(sport='NFL'))
draft_group_id, game_set_key = gcd.find_main_slate()
# download the player data
# from main slate draftables
ddoc = p.draftables(s.draftables(draft_group_id))
salaries = ddoc.player_salaries()
print(
pd.DataFrame([cattr.unstructure(s) for s in salaries
]).drop_duplicates(subset=['player_id', 'player_dk_id']))
def test_samplesheet_retrieve(requests_mock):
args = {
"sodar_url": "https://sodar.example.com/",
"project_uuid": "46f4d0d7-b446-4a04-99c4-53cbffe952a3",
"sodar_api_token": "token",
}
tpl = "%(sodar_url)ssamplesheets/api/investigation/retrieve/%(project_uuid)s"
expected = factories.InvestigationFactory()
requests_mock.register_uri(
"GET",
tpl % args,
headers={"Authorization": "Token %s" % args["sodar_api_token"]},
json=cattr.unstructure(expected),
)
result = samplesheet.retrieve(**args)
assert expected == result
def test_no_owner(self):
""" It raises an Attribute Error if the event has no owner. """
app = Celery(task_always_eager=True)
app.conf.update(CELERY_ALWAYS_EAGER=True)
event_ = Event(Exchange(), 'test_no_owner')
@attr.s(auto_attribs=True)
class Test2:
data: int
event = event_
task = event_.task.delay(instance=cattr.unstructure(Test2(1)))
with self.assertRaises(AttributeError):
task.get()
def __get__(self, instance: Message, owner: Type[Message]):
"""
Set up the owner or initialize the task.
Unfortunately the instance knows about its owner only
from when the __get__ is called first.
"""
if owner is not None:
self.owner = owner
elif instance:
self.owner = instance.__class__
if instance:
# returns a partial that can be called whenever you want.
return partial(self.task.delay,
instance=cattr.unstructure(instance))
return self
def archive(self, content):
"""
Archive an object
"""
if hasattr(content, 'name'):
print("Archiving {0}".format(content.name))
if content.name not in self._bloom_filter:
self.archive_file.write(
json.dumps(
cattr.unstructure(content), indent=4, sort_keys=True) +
"\n")
self._bloom_filter.add(content.name)
return
def test_landingzone_retrieve(requests_mock):
args = {
"sodar_url": "https://sodar.example.com/",
"landingzone_uuid": "46f4d0d7-b446-4a04-99c4-53cbffe952a3",
"sodar_api_token": "token",
}
tpl = "%(sodar_url)slandingzones/api/retrieve/%(landingzone_uuid)s"
expected = factories.LandingZoneFactory()
requests_mock.register_uri(
"GET",
tpl % args,
headers={"Authorization": "Token %s" % args["sodar_api_token"]},
json=cattr.unstructure(expected),
)
result = landingzone.retrieve(**args)
assert expected == result
def format_config(config, format="toml", pretty=True):
config = cattr.unstructure(config)
if format == "toml":
config = toml.dumps(config)
if pretty:
config = highlight(config, TOMLLexer(), Terminal256Formatter())
elif format == "json":
if pretty:
config = highlight(json.dumps(config, indent=2), JsonLexer(),
Terminal256Formatter())
else:
config = json.dumps(config)
else:
raise DBHooksError(f"Unrecognized format {format}!")
return config
def __init__(self, obj_name, texture_props):
self.texture_props = texture_props
self.obj_name = obj_name
if not self.texture_props:
print(
"\033[1;30;40m-\033[0m\033[1;31;40m Error \033[0m\033[1;30;40m- \033[0m\n"
)
raise ValueError(
" self.texture_type should have an argument type.\n")
with open("blocks.json", "w") as block_json_file:
json_schema = Dict[str, Block]
block_data = {"dnb:{}".format(self.obj_name): self.texture_props}
json.dump(cattr.unstructure(block_data), block_json_file, indent=4)
def test_success(self):
url = reverse("beaconsite:beacon-api-query")
url += "?" + "&".join(
"%s=%s" % (k, v)
for k, v in cattr.unstructure(self.beacon_allele_request).items())
# The user does not matter, the key does.
all_users = [
self.superuser,
self.owner_as.user,
self.delegate_as.user,
self.contributor_as.user,
self.guest_as.user,
self.user_no_roles,
self.anonymous,
]
self.assert_response_api(url, all_users, 200)
def send_robot_whos(self, robotident: RobotIdentifier,
whos: List[WarehouseOrder]) -> None:
"""
Send warehouse order from SAP EWM to the robot.
Returns True on success and False on fail.
"""
# Send orders to robot
for who in whos:
self.ordercontroller.send_who_to_robot(robotident,
unstructure(who))
# Create success log message
whos_who = [entry.who for entry in whos]
_LOGGER.info(
'Warehouse orders "%s" sent to robot "%s" in warehouse "%s"',
whos_who, robotident.rsrc, robotident.lgnum)
def put(self, scope: str, principal: str, permission: AclPermission):
"""Create or overwrite the ACL associated
with the given principal (user or group) on the specified scope point.
Args:
scope (str): The name of the scope to apply permissions to. This field is required.
principal (str): The principal to which the permission is applied. This field is required.
permission (AclPermission): The permission level applied to the principal. This field is required.
Returns:
bool: True
"""
endpoint = '/secrets/acls/put'
data = {'scope': scope, 'principal': principal,
'permission': unstructure(permission)}
res = self._post(endpoint, data)
return self._safe_handle(res, True)
def confirm_warehousetask(self,
wht: WarehouseTask,
enforce_first_conf: bool = False) -> None:
"""Confirm warehouse task."""
confirmations = []
clear_progress = False
if wht.vlpla or enforce_first_conf:
confirmation = ConfirmWarehouseTask(
lgnum=wht.lgnum,
tanum=wht.tanum,
rsrc=self.rsrc,
who=wht.who,
confirmationnumber=ConfirmWarehouseTask.FIRST_CONF,
confirmationtype=ConfirmWarehouseTask.CONF_SUCCESS)
if enforce_first_conf:
_LOGGER.info('First confirmation enforced - confirming')
else:
_LOGGER.info('Source bin reached - confirming')
confirmations.append(confirmation)
elif wht.nlpla:
confirmation = ConfirmWarehouseTask(
lgnum=wht.lgnum,
tanum=wht.tanum,
rsrc=self.rsrc,
who=wht.who,
confirmationnumber=ConfirmWarehouseTask.SECOND_CONF,
confirmationtype=ConfirmWarehouseTask.CONF_SUCCESS)
_LOGGER.info('Target bin reached - confirming')
confirmations.append(confirmation)
clear_progress = True
for confirmation in confirmations:
dtype = create_robcoewmtype_str(confirmation)
success = self.send_wht_confirmation(dtype,
unstructure(confirmation),
clear_progress)
if success is True:
_LOGGER.info(
'Confirmation message for warehouse task "%s" sent to order manager',
wht.tanum)
else:
_LOGGER.error(
'Error sending confirmation message for warehouse task "%s" - Try again',
wht.tanum)
raise ConnectionError
def player_salaries(
self,
players: List[PlayerDocument] = None
) -> List[PlayerSalaryDocument]:
"""Converts PlayerDocument to PlayerSalaryDocument
Args:
players (List[PlayerDocument]): default None
Returns:
List[PlayerSalaryDocument]
"""
l = self.draftables if not players else players
return [
cattr.structure(cattr.unstructure(o), PlayerSalaryDocument)
for o in l
]
def structure(d: Mapping, t: type) -> Any:
"""
Helper method to structure a TrainerSettings class. Meant to be registered with
cattr.register_structure_hook() and called with cattr.structure().
"""
if not isinstance(d, Mapping):
raise TrainerConfigError(f"Unsupported config {d} for {t.__name__}.")
d_copy: Dict[str, Any] = {}
# Check if a default_settings was specified. If so, used those as the default
# rather than an empty dict.
if TrainerSettings.default_override is not None:
d_copy.update(cattr.unstructure(TrainerSettings.default_override))
deep_update_dict(d_copy, d)
if "framework" in d_copy:
logger.warning("Framework option was deprecated but was specified")
d_copy.pop("framework", None)
for key, val in d_copy.items():
if attr.has(type(val)):
# Don't convert already-converted attrs classes.
continue
if key == "hyperparameters":
if "trainer_type" not in d_copy:
raise TrainerConfigError(
"Hyperparameters were specified but no trainer_type was given."
)
else:
d_copy[key] = check_hyperparam_schedules(
val, d_copy["trainer_type"]
)
d_copy[key] = strict_to_cls(
d_copy[key], TrainerType(d_copy["trainer_type"]).to_settings()
)
elif key == "max_steps":
d_copy[key] = int(float(val))
# In some legacy configs, max steps was specified as a float
else:
d_copy[key] = check_and_structure(key, val, t)
return t(**d_copy)
def get(self, request, *args, **kwargs):
local_site = Site.objects.get(role=Site.LOCAL)
remote_site = get_object_or_404(Site, sodar_uuid=kwargs.get("site"))
allele_req = cattr.structure(request.GET, BeaconAlleleRequest)
entrypoint_url = "%s/query" % remote_site.entrypoint_url
r = requests.get(
entrypoint_url,
params=cattr.unstructure(allele_req),
headers={"X-Beacon-User": request.user.username},
auth=HTTPSignatureAuth(
algorithm=local_site.key_algo,
key=str(local_site.private_key).encode("utf-8"),
key_id=local_site.identifier,
headers=["date", "x-beacon-user"],
),
)
return JsonResponse(data=r.json(),
status=r.status_code,
reason=r.reason)
def _serialize_operator_extra_links(cls, operator_extra_links):
"""
Serialize Operator Links. Store the import path of the OperatorLink and the arguments
passed to it. Example ``[{'airflow.gcp.operators.bigquery.BigQueryConsoleLink': {}}]``
:param operator_extra_links: Operator Link
:return: Serialized Operator Link
"""
serialize_operator_extra_links = []
for operator_extra_link in operator_extra_links:
op_link_arguments = cattr.unstructure(operator_extra_link)
if not isinstance(op_link_arguments, dict):
op_link_arguments = {}
serialize_operator_extra_links.append({
"{}.{}".format(operator_extra_link.__class__.__module__, operator_extra_link.__class__.__name__):
op_link_arguments
})
return serialize_operator_extra_links
def run(config, toml_config, args, _parser, _subparser, file=None):
"""Run landingzone retrieve command."""
config = ProjectCreateConfig.create(args, config, toml_config)
logger.info("Configuration: %s", config)
logger.info("Creating project")
project = api.Project(
sodar_uuid=None,
title=args.title,
type=args.type,
parent=args.parent_uuid,
description=args.description,
readme=args.readme,
)
result = api.project.create(
sodar_url=config.project_config.global_config.sodar_server_url,
sodar_api_token=config.project_config.global_config.sodar_api_token,
project=project,
)
print(json.dumps(cattr.unstructure(result)), file=(file or sys.stdout))
async def register(self, recipe: "Recipe", dep_book: DependencyBook):
paramhash = paramhash_recipe(recipe)
await self.db.execute(
"""
INSERT INTO dishcache
(recipe_kind, paramhash, dep_book, ts)
VALUES (?, ?, ?, ?)
ON CONFLICT (recipe_kind, paramhash)
DO UPDATE SET
dep_book = excluded.dep_book,
ts = excluded.ts
""",
(
recipe_name_getter(recipe.__class__),
paramhash,
canonicaljson.encode_canonical_json(cattr.unstructure(dep_book)),
self.time,
),
)
await self.db.commit()
def update_ewm_resource(self, rsrc: str,
config_spec: RobotConfigurationSpec) -> None:
"""Update an existing EWM resource."""
# check if robot exists
robots_key = '{}.{}'.format(config_spec.lgnum, rsrc)
if self.existing_robots.get(robots_key) is None:
_LOGGER.error(
'Resource "%s" does not exist in EWM warehouse "%s" yet', rsrc,
config_spec.lgnum)
return
if self.existing_robots.get(robots_key) == config_spec:
_LOGGER.debug('Resource "%s" did not change', rsrc)
return
try:
self.ewmrobot.change_robot(config_spec.lgnum,
rsrc,
rsrctype=config_spec.rsrctype,
rsrcgrp=config_spec.rsrcgrp)
except RobotNotFoundError:
_LOGGER.error(
'Robot resource %s not found in EWM warehouse %s - recreating on next run',
rsrc, config_spec.lgnum)
self.existing_robots.pop(robots_key, None)
except (InternalError, InternalServerError, ForeignLockError) as err:
_LOGGER.error(
'Updating Robot resource %s in EWM warehouse %s failed: %s - trying again',
rsrc, config_spec.lgnum, err)
except ODataAPIException as err:
_LOGGER.error('%s: restoring previous version of CR %s', err,
rsrc.lower())
self.robot_config.update_cr_spec(
rsrc.lower(),
unstructure(self.existing_robots.get(robots_key)))
else:
self.existing_robots[robots_key] = config_spec
_LOGGER.info(
'EWM robot resource %s in warehouse %s updated to resource type %s and resource '
'group %s', rsrc, config_spec.lgnum, config_spec.rsrctype,
config_spec.rsrcgrp)
def _serialize_operator_extra_links(cls, operator_extra_links: Iterable[BaseOperatorLink]):
"""
Serialize Operator Links. Store the import path of the OperatorLink and the arguments
passed to it. Example
``[{'airflow.providers.google.cloud.operators.bigquery.BigQueryConsoleLink': {}}]``
:param operator_extra_links: Operator Link
:return: Serialized Operator Link
"""
serialize_operator_extra_links = []
for operator_extra_link in operator_extra_links:
op_link_arguments = cattr.unstructure(operator_extra_link)
if not isinstance(op_link_arguments, dict):
op_link_arguments = {}
module_path = (
f"{operator_extra_link.__class__.__module__}.{operator_extra_link.__class__.__name__}"
)
serialize_operator_extra_links.append({module_path: op_link_arguments})
return serialize_operator_extra_links
def execute(self) -> typing.Optional[int]:
"""Execute the landing zone moving."""
res = self.check_args(self.args)
if res: # pragma: nocover
return res
logger.info("Starting cubi-tk sodar landing-zone-move")
logger.info(" args: %s", self.args)
landing_zone = api.landing_zones.move(
sodar_url=self.args.sodar_url,
sodar_api_token=self.args.sodar_api_token,
landing_zone_uuid=self.args.landing_zone_uuid,
)
values = cattr.unstructure(landing_zone)
if self.args.format_string:
print(self.args.format_string.replace(r"\t", "\t") % values)
else:
print(json.dumps(values))
return 0
def send_robot_whos(self, robotident: RobotIdentifier, whos: List[WarehouseOrder]) -> bool:
"""
Send warehouse order from SAP EWM to the robot.
Returns True on success and False on fail.
"""
# Send orders to robot
for who in whos:
dtype = create_robcoewmtype_str(who)
success = self.send_who_to_robot(robotident, dtype, unstructure(who))
if success is False:
raise ConnectionError
# Create success log message
whos_who = [entry.who for entry in whos]
_LOGGER.info(
'Warehouse orders "%s" sent to robot "%s" in warehouse "%s"',
whos_who, robotident.rsrc, robotident.lgnum)
return True
def update_render_env(self, render_env):
"""Modify the user rendering environment configuration.
Args:
render_env (dict): Rendering environment configuration.
e.g.:
{
'cgId': "2000",
'cgName': 'Maya',
'cgVersion': '2018',
'renderLayerType': 0,
'editName': 'tests',
'renderSystem': '1',
'pluginIds': 2703,
}.
"""
if isinstance(render_env, dict):
render_env = fields.Env(**render_env)
data = unstructure(render_env)
self._connect.post(constants.UPDATE_RENDER_ENV, data)
