def _dump_sign(self, data: typing.Dict):
l = list(data.keys())
l.sort()
s = ''
for i in l:
s += i+'='+str(data[i])+'&'
s = s[:-1]
return hashlib.md5(s.encode('utf-8')).hexdigest()
def storeProductIndex(self, _product_dict: typing.Dict):
for product, v in _product_dict.items():
# for each product, compute index of product
tmp_data_list = []
for instrument in v['InstrumentList']:
self.instrument_day_data_cur.execute(
"SELECT openprice, highprice, lowprice, closeprice, "
"volume, openinterest FROM {} WHERE tradingday='{}'".format(
instrument, v['TradingDay']
)
)
values = self.instrument_day_data_cur.fetchone()
tmp_data_list.append(dict(zip(
INDEX_TABLE_KEYS[1:], values
)))
openprice_arr = self._get_arr(tmp_data_list, 'OpenPrice')
highprice_arr = self._get_arr(tmp_data_list, 'HighPrice')
lowprice_arr = self._get_arr(tmp_data_list, 'LowPrice')
closeprice_arr = self._get_arr(tmp_data_list, 'ClosePrice')
volume_arr = self._get_arr(tmp_data_list, 'Volume')
openinterest_arr = self._get_arr(tmp_data_list, 'OpenInterest')
total_openinterest = openinterest_arr.sum()
if total_openinterest == 0:
index_dict = {
'TradingDay': v['TradingDay'],
'OpenPrice': openprice_arr.mean(),
'HighPrice': highprice_arr.mean(),
'LowPrice': lowprice_arr.mean(),
'ClosePrice': closeprice_arr.mean(),
'Volume': int(volume_arr.sum()),
'OpenInterest': total_openinterest,
}
else:
tmp_rate = openinterest_arr / float(total_openinterest)
index_dict = {
'TradingDay': v['TradingDay'],
'OpenPrice': np.sum(tmp_rate * openprice_arr),
'HighPrice': np.sum(tmp_rate * highprice_arr),
'LowPrice': np.sum(tmp_rate * lowprice_arr),
'ClosePrice': np.sum(tmp_rate * closeprice_arr),
'Volume': int(volume_arr.sum()),
'OpenInterest': total_openinterest
}
# true store part
try:
self._store_product_index(product, index_dict)
except psycopg2.DatabaseError as e:
self.product_index_con.rollback()
if e.pgcode == '42P01':
logging.warning(e)
self._create_product_index_table(product)
self._store_product_index(product, index_dict)
else:
logging.error(e)
sys.exit(1)
def _check_for_invalid_ids(cls, multi_ids_dict: typing.Dict, entity_type: str):
check_result = CheckResult(check_name=CHECK_NAMES.check_valid_ids, error_message=[])
if not multi_ids_dict:
check_result.result = RESULT.FAILURE
check_result.error_message.append("No ids found.")
for k, values in multi_ids_dict.items():
wrong_ids = [id for id in values if not cls._is_id_valid(id)]
if wrong_ids:
check_result.error_message.append("Invalid " + str(k) + "(s) for " + str(entity_type) + ": " + str(wrong_ids))
check_result.result = RESULT.FAILURE
return check_result
def storeDominantIndex(self, _product_dict: typing.Dict):
for product, v in _product_dict.items():
# for each product, compute index of dominant
dominant = v['Dominant']
self.instrument_day_data_cur.execute( # get the dominant data
"SELECT openprice, highprice, lowprice, closeprice, "
"volume, openinterest FROM {} WHERE tradingday='{}'".format(
dominant, v['TradingDay']
)
)
values = self.instrument_day_data_cur.fetchone()
cur_data = dict(zip(INDEX_TABLE_KEYS[1:], values))
self.instrument_day_data_cur.execute( # get the closeprice of last data
"SELECT closeprice FROM {} WHERE tradingday<'{}' "
"ORDER BY tradingday DESC LIMIT 1".format(
dominant, v['TradingDay']
)
)
values = self.instrument_day_data_cur.fetchone()
if values is not None:
return_rate = cur_data['ClosePrice'] / values[0]
else:
return_rate = 1.0
last_index_price = self._get_last_dominant_index(
product, v['TradingDay']
)
if last_index_price is None:
last_index_price = 1000.0
new_index_price = last_index_price * return_rate
price_scale = new_index_price / cur_data['ClosePrice']
new_openprice = cur_data['OpenPrice'] * price_scale
new_highprice = cur_data['HighPrice'] * price_scale
new_lowprice = cur_data['LowPrice'] * price_scale
self.dominant_index_cur.execute(
"INSERT INTO {} VALUES "
"(%s, %s, %s, %s, %s, %s, %s) "
"ON CONFLICT (TradingDay) DO UPDATE SET "
"OpenPrice = EXCLUDED.OpenPrice,"
"HighPrice = EXCLUDED.HighPrice,"
"LowPrice = EXCLUDED.LowPrice,"
"ClosePrice = EXCLUDED.ClosePrice,"
"Volume = EXCLUDED.Volume,"
"OpenInterest = EXCLUDED.OpenInterest"
"".format(product),
[
v['TradingDay'],
new_openprice, new_highprice,
new_lowprice, new_index_price,
cur_data['Volume'], cur_data['OpenInterest']
]
)
self.dominant_index_con.commit()
def storeInstrumentDayData(self, _data_dict: typing.Dict):
for instrument, v in _data_dict.items():
try:
self._store_instrument_day_data(instrument, v)
except psycopg2.DatabaseError as e:
self.instrument_day_data_con.rollback()
if e.pgcode == '42P01':
logging.warning(e)
self._create_instrument_day_data_table(instrument)
self._store_instrument_day_data(instrument, v)
else:
logging.error(e)
sys.exit(1)
def init_validator(hints: typing.Dict, parent: typing.Optional[Validator]=None) -> Validator:
"""
Returns a new validator instance from a given dictionary of type hints
"""
validator = Validator(parent)
for name, hint in hints.items():
if hint is None:
hint = type(None)
root_parser = get_parser(None, hint, validator)
syntax_tree = visit(root_parser)
validator.roots[name] = syntax_tree
return validator
def __init__(self, config: typing.Dict):
"""Initialize a text input."""
self._config = config
self.editable = True
self._current_value = config.get(CONF_INITIAL)
def lat_lon_to_solr(coreMetadata: typing.Dict, latitude: typing.SupportsFloat,
longitude: typing.SupportsFloat):
coreMetadata.update(
shapely_to_solr(shapely.geometry.Point(longitude, latitude)))
def __init__(self):
self.dealias: Dict[str, ID] = {}
self.aliases: Dict(ID, List[str]) = {}
def map_processing_unit(document: typing.Dict) -> ProcessingUnit:
id = document.get('id')
clock_rate = document['clock-rate']
cores = document['cores']
return ProcessingUnit(id=id, clock_rate=clock_rate, cores=cores)
def map_rack(document: typing.Dict) -> Rack:
id = document.get('id')
capacity = document.get('capacity')
units = list(map(map_rack_unit, document['units']))
return Rack(capacity=capacity, units=units, id=id)
def values_view(self):
return Dict.values(self)
def transform(self, x: t.Dict) -> xr.Dataset:
# Basic preparation
prefix = self.prefix
sensor_dims = []
sensor_dim_metadata = {}
for y in self.sensor_dims:
if isinstance(y, str):
sensor_dims.append(y)
sensor_dim_metadata[y] = {}
else:
sensor_dims.append(y[0])
sensor_dim_metadata[y[0]] = y[1]
spectral_dims = []
spectral_dim_metadata = {}
for y in _spectral_dims():
if isinstance(y, str):
spectral_dims.append(y)
spectral_dim_metadata[y] = {}
else:
spectral_dims.append(y[0])
spectral_dim_metadata[y[0]] = y[1]
sensor_datasets = []
regex = re.compile(
r"\_".join(
[prefix]
+ [rf"{sensor_dim}(?P<{sensor_dim}>\d*)" for sensor_dim in sensor_dims]
)
)
# Loop on spectral indexes
for spectral_index in x.keys():
# Loop on sensors
for sensor_id, sensor_data in x[spectral_index]["values"].items():
# Collect data
ds = sensor_data.copy(deep=False)
spp = x[spectral_index]["spp"][sensor_id]
# Set spectral coordinates
spectral_coords = {
spectral_dim: [spectral_coord]
for spectral_dim, spectral_coord in zip(
spectral_dims, always_iterable(spectral_index)
)
}
# Detect sensor coordinates
match = regex.match(sensor_id)
if match is None:
raise RuntimeError(
"could not detect requested sensor dimensions in "
f"sensor ID '{sensor_id}' using regex '{regex.pattern}'; "
"this could be due to incorrect values or order of "
"'sensor_dims'"
)
sensor_coords = {
f"{sensor_dim}_index": [int(match.group(sensor_dim))]
for sensor_dim in sensor_dims
}
# Add spp dimension even though sample count split did not
# produce any extra sensor
if "spp" not in sensor_dims:
sensor_coords["spp_index"] = [0]
# Add spectral and sensor dimensions to img array
all_coords = {**spectral_coords, **sensor_coords}
ds["img"] = ds.img.expand_dims(dim=all_coords)
# Package spp in a data array
all_dims = list(all_coords.keys())
ds["spp"] = (all_dims, np.reshape(spp, [1 for _ in all_dims]))
sensor_datasets.append(ds)
# Combine all the data
with xr.set_options(keep_attrs=True):
result = xr.merge(sensor_datasets)
# Drop "channel" dimension when using a mono variant
if eradiate.mode().has_flags(ModeFlags.MTS_MONO):
result = result.squeeze("channel", drop=True)
# Apply metadata to new dimensions
for sensor_dim in sensor_dims:
result[f"{sensor_dim}_index"].attrs = sensor_dim_metadata[sensor_dim]
if "spp" not in sensor_dims:
result["spp_index"].attrs = {
"standard_name": "spp_index",
"long_name": "SPP index",
}
for spectral_dim in spectral_dims:
result[spectral_dim].attrs = spectral_dim_metadata[spectral_dim]
# Apply metadata to data variables
if isinstance(self.var, str):
var = self.var
var_metadata = {}
else:
var = self.var[0]
var_metadata = self.var[1]
result = result.rename({"img": var})
result[var].attrs.update(var_metadata)
result["spp"].attrs = {
"standard_name": "sample_count",
"long_name": "sample count",
}
return result
def resampleProbabilisticValues(self, Dict=None):
'''
Normal Distribution Sampling:
If (key + "_stdDev") exists and the value being returned is a scalar or Vector value, returns a scalar or vector sampled from a normal distribution
Where the mean of the normal distribution is taken to be the (original) value of 'key' (moved to 'key_mean' when this function first runs) and the standard deviation of the distribution is the value of 'key_stdDev'
For a vector value, a vector of standard deviations is expected
For repeatable sampling, set the value "MonteCarlo.randomSeed" in the file loaded by this class
'''
if Dict is None:
Dict = self.dict
if not self.disableDistributionSampling:
keys = list(
Dict.keys()
) # Get a list of keys at the beginning to avoid issues from the number of keys changing during iterations
for key in keys:
### Sample any probabilistic values from normal distribution ###
stdDevKey = key + "_stdDev"
if stdDevKey in Dict:
logLine = None
meanKey = key + "_mean"
try:
meanString = Dict[meanKey]
except KeyError:
# Take the value of the variable as the mean if a _mean value is not provided
meanString = Dict[key]
Dict[meanKey] = meanString
# Try parsing scalar values
try:
mu = float(meanString)
sigma = float(Dict[stdDevKey])
sampledValue = self.rng.gauss(mu, sigma)
Dict[key] = str(sampledValue)
logLine = "Sampling scalar parameter: {}, value: {:1.3f}".format(
key, sampledValue)
except ValueError:
# Try parsing vector value
try:
muVec = Vector(meanString)
sigmaVec = Vector(Dict[stdDevKey])
sampledVec = Vector(*[
self.rng.gauss(mu, sigma)
for mu, sigma in zip(muVec, sigmaVec)
])
Dict[key] = str(sampledVec)
logLine = "Sampling vector parameter: {}, value: ({:1.3f})".format(
key, sampledVec)
except ValueError:
# ValueError throws if either conversion to Vector fails
# Note that monte carlo / probabilistic variables can only be scalars or vectors
print(
"ERROR: Unable to parse probabilistic value: {} for key {} (or {} for key {}). Note that probabilistic values must be either scalars or vectors of length 3."
.format(meanString, meanKey,
self.getValue(stdDevKey), stdDevKey))
raise
### Logging ###
if logLine != None:
if self.monteCarloLogger != None:
self.monteCarloLogger.log(logLine)
elif not self.silent:
print(logLine)
def persistent(cls,
computing_table: CTableABC,
namespace,
name,
schema=None,
part_of_data=None,
engine=None,
engine_address=None,
store_type=None,
token: typing.Dict = None) -> StorageTableMeta:
if engine:
if engine != StorageEngine.PATH and engine not in Relationship.Computing.get(
computing_table.engine, {}).get(EngineType.STORAGE,
{}).get("support", []):
raise Exception(
f"storage engine {engine} not supported with computing engine {computing_table.engine}"
)
else:
engine = Relationship.Computing.get(computing_table.engine,
{}).get(
EngineType.STORAGE,
{}).get("default", None)
if not engine:
raise Exception(
f"can not found {computing_table.engine} default storage engine"
)
if engine_address is None:
# find engine address from service_conf.yaml
engine_address = engine_utils.get_engines_config_from_conf().get(
EngineType.STORAGE, {}).get(engine, {})
address_dict = engine_address.copy()
partitions = computing_table.partitions
if engine == StorageEngine.STANDALONE:
address_dict.update({"name": name, "namespace": namespace})
store_type = StandaloneStoreType.ROLLPAIR_LMDB if store_type is None else store_type
elif engine == StorageEngine.EGGROLL:
address_dict.update({"name": name, "namespace": namespace})
store_type = EggRollStoreType.ROLLPAIR_LMDB if store_type is None else store_type
elif engine == StorageEngine.HIVE:
address_dict.update({"database": namespace, "name": f"{name}"})
store_type = HiveStoreType.DEFAULT if store_type is None else store_type
elif engine == StorageEngine.LINKIS_HIVE:
address_dict.update({
"database": None,
"name": f"{namespace}_{name}",
"username": token.get("username", "")
})
store_type = LinkisHiveStoreType.DEFAULT if store_type is None else store_type
elif engine == StorageEngine.HDFS:
if not address_dict.get("path"):
address_dict.update({
"path":
default_output_fs_path(
name=name,
namespace=namespace,
prefix=address_dict.get("path_prefix"))
})
store_type = HDFSStoreType.DISK if store_type is None else store_type
elif engine == StorageEngine.LOCALFS:
if not address_dict.get("path"):
address_dict.update({
"path":
default_output_fs_path(
name=name,
namespace=namespace,
storage_engine=StorageEngine.LOCALFS)
})
store_type = LocalFSStoreType.DISK if store_type is None else store_type
elif engine == StorageEngine.PATH:
store_type = PathStoreType.PICTURE if store_type is None else store_type
else:
raise RuntimeError(f"{engine} storage is not supported")
address = StorageTableMeta.create_address(storage_engine=engine,
address_dict=address_dict)
schema = schema if schema else {}
computing_table.save(address,
schema=schema,
partitions=partitions,
store_type=store_type)
table_count = computing_table.count()
table_meta = StorageTableMeta(name=name, namespace=namespace, new=True)
table_meta.address = address
table_meta.partitions = computing_table.partitions
table_meta.engine = engine
table_meta.store_type = store_type
table_meta.schema = schema
table_meta.part_of_data = part_of_data if part_of_data else {}
table_meta.count = table_count
table_meta.write_access_time = current_timestamp()
table_meta.origin = StorageTableOrigin.OUTPUT
table_meta.create()
return table_meta
def get(self, key):
return Dict.get(self, key)
def v(self):
return List(*Dict.values(self))
def k(self):
return List(*Dict.keys(self))
def entities(self) -> Dict(str, Entity): """Returns a mapping of entity names to Entity instances.""" return self._entities
def __init__(self):
self.bones: Dict(str, 'Animation.Bone') = {}
def get_cs_dimensions(api_config: typing.Dict) -> ConfigurationSpace:
"""
Help routine to setup ConfigurationSpace search space in constructor.
Take api_config as argument so this can be static.
Parameters
----------
api_config: Dict
api dictionary to construct
Returns
-------
cs: ConfigurationSpace
ConfigurationSpace that contains the same hyperparameter as api_config
"""
# TODO 2 options to transform the real and int hyperaparameters in different scales
# option 1: similar to example_submission.skopt.optimizer, merge 'logit' into 'log' and 'bilog' into 'linear'
# option 2: use the api bayesmark.space.space to warp and unwarp the samples
cs = ConfigurationSpace()
param_list = sorted(api_config.keys())
hp_list = []
for param_name in param_list:
param_config = api_config[param_name]
param_type = param_config["type"]
param_space = param_config.get("space", None)
param_values = param_config.get("values", None)
param_range = param_config.get("range", None)
if param_type == "cat":
assert param_space is None
assert param_range is None
hp = CategoricalHyperparameter(name=param_name,
choices=param_values)
elif param_type == "bool":
assert param_space is None
assert param_values is None
assert param_range is None
hp = CategoricalHyperparameter(name=param_name,
choices=[True, False])
elif param_type == "ordinal":
# appear in example_submission.skopt.optimizer but not in README
assert param_space is None
assert param_range is None
hp = OrdinalHyperparameter(name=param_name,
sequence=param_values)
elif param_type in ("int", "real"):
if param_values is not None:
# TODO: decide whether we treat these parameters as discrete values
# or step function (example see example_submission.skopt.optimizer, line 71-77)
# sort the values to store them in OrdinalHyperparameter
param_values_sorted = np.sort(param_values)
hp = OrdinalHyperparameter(name=param_name,
sequence=param_values_sorted)
else:
log = True if param_space == "log" else False
if param_type == "int":
hp = UniformIntegerHyperparameter(
name=param_name,
lower=param_range[0],
upper=param_range[-1],
log=log)
else:
hp = UniformFloatHyperparameter(name=param_name,
lower=param_range[0],
upper=param_range[-1],
log=log)
else:
assert False, "type %s not handled in API" % param_type
hp_list.append(hp)
cs.add_hyperparameters(hp_list)
return cs
def __init__(self, name: str, device_type: SystemType):
self.name = name
self.type = device_type
self.bandwidth = 0
self.ping = 0
self.benchmarks: Dict(value, Scenario_result) = {}
def map_machine(document: typing.Dict) -> PhysicalMachine:
id = document.get('id')
state = MachineState[document.get('state', 'RUNNING')]
cpus = list(map(map_processing_unit, document.get('cpus', [])))
gpus = list(map(map_processing_unit, document.get('gpus', [])))
return PhysicalMachine(id=id, state=state, cpus=cpus, gpus=gpus)
def __new__(mcs, name: typing.AnyStr, bases: typing.Tuple,
class_dict: typing.Dict):
class_dict_copy = class_dict.copy()
_id = Field(
verbose_name='Instance ID',
data_type=str,
required=True,
default=None,
)
_ts = Field(
verbose_name='Timestamp',
data_type=int,
required=True,
default=lambda: time.time(),
)
class_dict_copy.update({
'_id': _id,
'_ts': _ts,
})
# init all field instances
for k, v in class_dict_copy.items():
if isinstance(v, Field):
v.name = k
v.fullname = f'{name.lower()}.{k}'
v.internal_name = f'field__{k}'
meta_data: typing.Dict = {}
meta_class = class_dict_copy.get('Meta')
if meta_class:
assert isinstance(meta_class, type)
meta_class_dict = meta_class.__dict__
for mk, mv in meta_class_dict.items():
if not mk.startswith('__'):
meta_data.update({mk: mv})
class_dict_copy.update({'meta_data': meta_data})
def generate_id(self):
return 'test'
class_dict_copy.update({'generate_id': generate_id})
def initialize_instance(self, init_data: typing.Dict):
cls_dict: typing.Dict = self.__class__.__dict__
for sk, sv in cls_dict.items():
if isinstance(sv, Field):
if sk in init_data:
sk_v = init_data[sk]
data_type: typing.Any = sv.data_type
multi: bool = sv.multi
if multi:
assert isinstance(sk_v, list)
if isinstance(data_type, BlueprintMeta):
v_deserialized = []
for d in sk_v:
# init every blueprint instance
deserialized_instance = data_type(**d)
# who is the parent of this blueprint ?
deserialized_instance.parent = self
v_deserialized.append(
deserialized_instance)
else:
# Note: create new instance of list
v_deserialized = [item for item in sk_v]
else:
assert not isinstance(sk_v, list)
if isinstance(data_type, BlueprintMeta):
v_deserialized = data_type(**sk_v)
v_deserialized.parent = self
else:
v_deserialized = sk_v
# set attr value (through descriptor)
setattr(self, sk, v_deserialized)
else:
# user not provide value for the field, use default value to initialize the field if possible
# default value for multi field should be []
if sv.multi and sv.default is None:
sv.default = []
# if data type of field is Blueprint, create a new blueprint instance as default value
if sv.data_type and isinstance(sv.data_type,
BlueprintMeta):
if sv.default and isinstance(
sv.default, Blueprint):
sv.default = copy.copy(sv.default)
# will get default value if any,
# and set instance field value to default (force check and clean, create new instance if needed)
sk_v = getattr(self, sk)
# after initialize, value of every field should be in valid state (pass descriptor's check)
# check_and_clean_if_possible
setattr(self, sk, sk_v)
# generate id if needed
if self.is_new:
assert getattr(self, self.ID_NAME) is None
id_template = self.meta_data.get('id_template')
if id_template is None:
raise BlueprintTypeException(
f'cannot generate id for new created blueprint '
f'because id_template not specified in Meta')
context_render: typing.Dict = {
key: getattr(self, key)
for key in cls_dict if isinstance(cls_dict[key], Field)
}
new_id = id_template.format(**context_render)
setattr(self, self.ID_NAME, new_id)
# check required value
for sk, sv in cls_dict.items():
if isinstance(sv, Field):
sk_v = getattr(self, sk, None)
if sk_v is None and sv.required and sv.default is None:
raise BlueprintTypeException(
f'{sv.fullname} is required '
f'but no value provided and no default value set')
def init(self, **kwargs):
self.kwargs = kwargs # used to copy a new blueprint instance
self.parent = None
self.id_context = {}
self.is_new = False
if self.ID_NAME not in kwargs:
self.is_new = True
self.initialize_instance(kwargs)
def serialize(self, selected_fields=None):
if selected_fields is None:
selected_fields = []
else:
selected_fields = list(selected_fields)
cls_dict: typing.Dict = self.__class__.__dict__
serialized: typing.Dict = {}
if not self.should_serialize():
return serialized
else:
for sk, sv in cls_dict.items():
if isinstance(sv, Field):
if selected_fields and sv.name not in selected_fields:
continue
sk_v = getattr(self, sk)
# serialize each field according to sv.data_type and sv.multi
data_type: typing.Any = sv.data_type
multi: bool = sv.multi
if multi:
# should serialize each item in the value
assert isinstance(sk_v, list)
if isinstance(data_type, BlueprintMeta):
serialized.update(
{sk: [item.serialize() for item in sk_v]})
else:
# just create a new list with the same content
serialized.update(
{sk: [item for item in sk_v]})
else:
assert not isinstance(sk_v, list)
if isinstance(data_type, BlueprintMeta):
if sk_v.should_serialize():
serialized.update({sk: sk_v.serialize()})
else:
serialized.update({sk: sk_v})
return serialized
def should_serialize(self):
return True
class_dict_copy.update({
'ID_NAME': '_id',
'TS_NAME': '_ts',
'__init__': init,
'initialize_instance': initialize_instance,
'serialize': serialize,
'should_serialize': should_serialize,
})
cls = type.__new__(mcs, name, bases, class_dict_copy)
return cls
def from_dict(cls, raw: typing.Dict):
try:
report = cls(raw["algorithm_id"], raw.get("extra_parameters"))
except KeyError:
raise
return report
def error_checker(data: typing.Dict) -> None:
if data.get("user_id") or data.get("channel_id"):
raise InvalidSnowflake(
data.get("user_id")[0] if data.get("user_id") != None else data.
get("channel_id")[0])
elif data.get("code"):
if data.get("code") == 50001:
raise MissingAccess(data.get("message"))
else:
raise InvalidSnowflake(data.get("message"))
elif data.get("avatar"):
if isinstance(data.get("avatar"), list):
raise AvatarError(data.get("avatar")[0])
def map_storage(
fgraph: FunctionGraph,
order: typing.Iterable[Apply],
input_storage: typing.Optional[typing.List],
output_storage: typing.Optional[typing.List],
storage_map: typing.Dict = None,
) -> typing.Tuple[typing.List, typing.List, typing.Dict]:
"""Ensure there is storage (a length-1 list) for inputs, outputs, and interior nodes.
:param fgraph: The current fgraph. This function uses the inputs and outputs attributes.
:param order: an iterable over Apply instances (in program running order)
:param input_storage: None or existing input storage (see below)
:param output_storage: None or existing output storage (see below)
:rtype: 3-tuple
:returns: (list of storage for inputs, list of storage for outputs, and the `storage_map`)
Parameters
----------
fgraph
The current fgraph. This function uses the inputs and outputs
attributes.
order
An iterable over Apply instances (in program running order).
input_storage
None or existing input storage (see below).
output_storage
None or existing output storage (see below).
Returns
-------
3-tuple
List of storage for inputs, list of storage for outputs, and
the `storage_map`.
Extended summary
----------------
This function iterates over the nodes in `order` and ensures that for every
input and output `Variable`, there is a unique storage container. This is
returned as a dictionary Variable -> storage called the `storage_map`.
This function also returns `input_storage`, which is a list of storages
corresponding to fgraph.inputs.
This function also returns `output_storage`, which is a list of storages
corresponding to fgraph.outputs.
"""
# each Apply argument's data is stored in a list of length 1 (these lists act like pointers)
if storage_map is None:
storage_map = {}
# input_storage is a list of data-containers for the inputs.
if input_storage is None:
input_storage = [[None] for input in fgraph.inputs]
else:
assert len(fgraph.inputs) == len(input_storage)
# add input storage into storage_map
for r, storage in zip(fgraph.inputs, input_storage):
if r in storage_map:
assert storage_map[r] is storage, (
"Given input_storage conflicts "
"with storage in given storage_"
"map. Given input_storage: ",
storage,
"Storage in storage_ma" "p: ",
storage_map[r],
)
else:
storage_map[r] = storage
# for orphan in fgraph.orphans:
# if not isinstance(orphan, Constant):
# raise TypeError("Cannot link a graph with non-constant orphans.", orphan)
# storage_map[orphan] = [orphan.data]
# allocate output storage
if output_storage is not None:
assert len(fgraph.outputs) == len(output_storage)
for r, storage in zip(fgraph.outputs, output_storage):
if r in storage_map:
assert storage_map[r] is storage, (
"Given output_storage confl"
"icts with storage in given"
" storage_map. Given output"
"_storage: ",
storage,
"Sto" "rage in storage_map: ",
storage_map[r],
)
else:
storage_map[r] = storage
# allocate storage for intermediate computation
for node in order:
for r in node.inputs:
if r not in storage_map:
assert isinstance(r, Constant)
storage_map[r] = [r.data]
for r in node.outputs:
storage_map.setdefault(r, [None])
for r in fgraph.outputs:
if isinstance(r, Constant):
storage_map.setdefault(r, [r.data])
# extract output storage
if output_storage is None:
output_storage = [storage_map[r] for r in fgraph.outputs]
return input_storage, output_storage, storage_map
def write_templated_file_to_path(path: Path,
lookup_table: typing.Dict) -> None:
logger.debug(
f"Writing templated file {get_templates_path(path.name)} to {path}")
path.write_text(get_template_file(lookup_table.get(path.name)))
def log_from_changes(changes: ty.Dict) -> str:
halfway = {title: '\n'.join(f' - {item}' for item in content)
for title, content in changes.items()}
res = '\n\n'.join(f'### {title}\n{content}' for title, content in halfway.items())
return res
def null_to_string(d: typing.Dict) -> None:
for k in d.keys():
if isinstance(d[k], dict):
null_to_string(d[k])
elif d[k] is None:
d[k] = ""
def get_distribution_info(
self,
csw_distribution: etree.Element,
api_record: typing.Dict,
harvestable_resource: models.HarvestableResource,
) -> resourcedescriptor.RecordDistribution:
online_elements = csw_distribution.xpath(
".//gmd:transferOptions//gmd:onLine",
namespaces=csw_distribution.nsmap)
link = None
wms = None
wfs = None
wcs = None
legend = None
geojson = None
original = None
original_format_values = (
"original dataset format",
"hosted document format",
)
for online_el in online_elements:
protocol = get_xpath_value(online_el, ".//gmd:protocol").lower()
linkage = get_xpath_value(online_el, ".//gmd:linkage")
description = (get_xpath_value(online_el, ".//gmd:description")
or "").lower()
if "link" in protocol:
link = linkage
elif "ogc:wms" in protocol:
wms = linkage
elif "ogc:wfs" in protocol:
wfs = linkage
elif "ogc:wcs" in protocol:
wcs = linkage
elif "legend" in description.lower():
legend = linkage
elif "geojson" in description.lower():
geojson = linkage
elif api_record.get("doc_file") is not None:
# NOTE: for resources of type document, the GeoNode API returns a
# relative URL which can be used directly, as opposed to its CSW API,
# which returns a generic download URL
document_url: str = api_record.get("doc_file")
if document_url.startswith("/"):
original = f"{self.remote_url}{document_url}"
else:
original = document_url
else:
for original_value in original_format_values:
if original_value in description.lower():
original = linkage
break
return resourcedescriptor.RecordDistribution(
link_url=link,
wms_url=wms,
wfs_url=wfs,
wcs_url=wcs,
thumbnail_url=self._retrieve_thumbnail_url(api_record,
harvestable_resource),
legend_url=legend,
geojson_url=geojson,
original_format_url=original,
)
def parse_query(query_str: str, res: typing.Dict = None) -> typing.Dict:
"""
Basically takes a query string (like the ones in the examples of commands for the search__offers function) and
processes it into a dict of URL parameters to be sent to the server.
:param str query_str:
:param Dict res:
:return Dict:
"""
if res is None: res = {}
if type(query_str) == list:
query_str = " ".join(query_str)
query_str = query_str.strip()
opts = re.findall(
"([a-zA-Z0-9_]+)( *[=><!]+| +(?:[lg]te?|nin|neq|eq|not ?eq|not ?in|in) )?( *)(\[[^\]]+\]|[^ ]+)?( *)",
query_str)
# res = {}
op_names = {
">=": "gte",
">": "gt",
"gt": "gt",
"gte": "gte",
"<=": "lte",
"<": "lt",
"lt": "lt",
"lte": "lte",
"!=": "neq",
"==": "eq",
"=": "eq",
"eq": "eq",
"neq": "neq",
"noteq": "neq",
"not eq": "neq",
"notin": "notin",
"not in": "notin",
"nin": "notin",
"in": "in",
}
field_alias = {
"cuda_vers": "cuda_max_good",
"display_active": "gpu_display_active",
"reliability": "reliability2",
"dlperf_usd": "dlperf_per_dphtotal",
"dph": "dph_total",
"flops_usd": "flops_per_dphtotal",
}
field_multiplier = {
"cpu_ram": 1000,
"gpu_ram": 1000,
"duration": 1.0 / (24.0 * 60.0 * 60.0),
}
fields = {
"compute_cap",
"cpu_cores",
"cpu_cores_effective",
"cpu_ram",
"cuda_max_good",
"driver_version",
"disk_bw",
"disk_space",
"dlperf",
"dlperf_per_dphtotal",
"dph_total",
"duration",
"external",
"flops_per_dphtotal",
"gpu_display_active",
# "gpu_ram_free_min",
"gpu_mem_bw",
"gpu_name",
"gpu_ram",
"has_avx",
"host_id",
"id",
"inet_down",
"inet_down_cost",
"inet_up",
"inet_up_cost",
"min_bid",
"mobo_name",
"num_gpus",
"pci_gen",
"pcie_bw",
"reliability2",
"rentable",
"rented",
"storage_cost",
"total_flops",
"verified"
}
joined = "".join("".join(x) for x in opts)
if joined != query_str:
raise ValueError(
"Unconsumed text. Did you forget to quote your query? " +
repr(joined) + " != " + repr(query_str))
for field, op, _, value, _ in opts:
value = value.strip(",[]")
v = res.setdefault(field, {})
op = op.strip()
op_name = op_names.get(op)
if field in field_alias:
field = field_alias[field]
if not field in fields:
print(
"Warning: Unrecognized field: {}, see list of recognized fields."
.format(field),
file=sys.stderr)
if not op_name:
raise ValueError(
"Unknown operator. Did you forget to quote your query? " +
repr(op).strip("u"))
if op_name in ["in", "notin"]:
value = [x.strip() for x in value.split(",") if x.strip()]
if not value:
raise ValueError(
"Value cannot be blank. Did you forget to quote your query? " +
repr((field, op, value)))
if not field:
raise ValueError(
"Field cannot be blank. Did you forget to quote your query? " +
repr((field, op, value)))
if value in ["?", "*", "any"]:
if op_name != "eq":
raise ValueError("Wildcard only makes sense with equals.")
if field in v:
del v[field]
if field in res:
del res[field]
continue
if field in field_multiplier:
value = str(float(value) * field_multiplier[field])
v[op_name] = value
res[field] = v
return res
def from_dict(cls, model_dict: typing.Dict) -> typing.Type['Model']:
"""Creates a model from a dictionary.
Parameters
----------
model_dict :
Dictionary containing the model.
"""
model = cls()
# iterate over items
for name, attribute in list(model_dict.items()):
# we determine if we the item is known by the model by looking for
# a setter with same name.
if hasattr(model, f'set_{name}'):
# get the set function
set = getattr(model, f'set_{name}')
# we retrieve the actual class from the signature
for label, item in attribute.items():
item_cls = set.__func__.__annotations__['item']
is_typed = hasattr(item_cls, "_glotaran_model_attribute_typed")
if isinstance(item, dict):
if is_typed:
if 'type' not in item:
raise Exception(f"Missing type for attribute '{name}'")
item_type = item['type']
if item_type not in item_cls._glotaran_model_attribute_types:
raise Exception(f"Unknown type '{item_type}' "
f"for attribute '{name}'")
item_cls = \
item_cls._glotaran_model_attribute_types[item_type]
item['label'] = label
set(label, item_cls.from_dict(item))
elif isinstance(item, list):
if is_typed:
if len(item) < 2 and len(item) != 1:
raise Exception(f"Missing type for attribute '{name}'")
item_type = item[1] if len(item) != 1 and \
hasattr(item_cls, 'label') else item[0]
if item_type not in item_cls._glotaran_model_attribute_types:
raise Exception(f"Unknown type '{item_type}' "
f"for attribute '{name}'")
item_cls = \
item_cls._glotaran_model_attribute_types[item_type]
item = [label] + item
set(label, item_cls.from_list(item))
del model_dict[name]
elif hasattr(model, f'add_{name}'):
# get the set function
add = getattr(model, f'add_{name}')
# we retrieve the actual class from the signature
for item in attribute:
item_cls = add.__func__.__annotations__['item']
is_typed = hasattr(item_cls, "_glotaran_model_attribute_typed")
if isinstance(item, dict):
if is_typed:
if 'type' not in item:
raise Exception(f"Missing type for attribute '{name}'")
item_type = item['type']
if item_type not in item_cls._glotaran_model_attribute_types:
raise Exception(f"Unknown type '{item_type}' "
f"for attribute '{name}'")
item_cls = \
item_cls._glotaran_model_attribute_types[item_type]
add(item_cls.from_dict(item))
elif isinstance(item, list):
if is_typed:
if len(item) < 2 and len(item) != 1:
raise Exception(f"Missing type for attribute '{name}'")
item_type = item[1] if len(item) != 1 and \
hasattr(item_cls, 'label') else item[0]
if item_type not in item_cls._glotaran_model_attribute_types:
raise Exception(f"Unknown type '{item_type}' "
f"for attribute '{name}'")
item_cls = \
item_cls._glotaran_model_attribute_types[item_type]
add(item_cls.from_list(item))
del model_dict[name]
return model
def from_dict(cls, init_dict: tg.Dict) -> 'MetaData':
meta_dict = init_dict['meta_dict']
init_args = init_dict.get('init_args', [])
init_kwargs = init_dict.get('init_kwargs', {})
return cls(meta_dict, *init_args, **init_kwargs)
def synthesize_library(
library: typing.Dict,
github_token: str,
extra_args: typing.List[str],
base_log_path: pathlib.Path,
runner: Runner = _execute,
) -> typing.Dict:
"""Run autosynth on a single library.
Arguments:
library {dict} - Library configuration
"""
logger.info(f"Synthesizing {library['name']}.")
command = [sys.executable, "-m", "autosynth.synth"]
env = os.environ
env["GITHUB_TOKEN"] = github_token
library_args = [
"--repository",
library["repository"],
"--synth-path",
library.get("synth-path", ""),
"--branch-suffix",
library.get("branch-suffix", ""),
"--pr-title",
library.get("pr-title", ""),
"--base-log-dir",
str(base_log_path),
]
if library.get("metadata-path"):
library_args.extend(["--metadata-path", library.get("metadata-path")])
if library.get("deprecated-execution", False):
library_args.append("--deprecated-execution")
log_file_dir = (pathlib.Path(base_log_path) / pathlib.Path(
library.get("synth-path", "") or library["repository"]).name)
log_file_path = log_file_dir / "sponge_log.log"
# run autosynth in a separate process
(returncode, output) = runner(
command + library_args + library.get("args", []) + extra_args,
env,
log_file_path,
)
error = returncode not in (0, synth.EXIT_CODE_SKIPPED)
skipped = returncode == synth.EXIT_CODE_SKIPPED
# Leave a sponge_log.xml side-by-side with sponge_log.log, and sponge
# will understand they're for the same task and render them accordingly.
results = [{
"name": library["name"],
"error": error,
"output": "See the test log.",
"skipped": skipped,
}]
make_report(library["name"], results, log_file_dir)
if error:
logger.error(f"Synthesis failed for {library['name']}")
return {
"name": library["name"],
"output": output.decode("utf-8"),
"error": error,
"skipped": skipped,
}
def matches(small: typing.Dict, group: typing.Dict):
for key in small.keys():
if key not in group or small[key] != group[key]:
return False
return True
from typing import List,Dict
import sqlite3
Book = Dict(str, Union(str,int))
def func_that_has_no_return() -> None: # We add the type that is returned so that if this fct is assigned to a var it wil give a warning
print('There is no return in this function')
def funct_that_returns_a_list_of_dict () -> List[Dict]: # List and Dict exists in the Typing package
return [{'a':1, 'b':2}]
def function_with_param (name: str, age : int) -> int:
if age > 100
return 1
else:
return 0
def db_connection() -> sqlite3.Connection: # we must return a Connection class type variable
return sqlite3.connect()
var1 = func_that_has_no_return()
print(var1)
print (funct_that_returns_a_list_of_dict())
print(function_with_param(4,5)) # the first argument has a yellow backgroun, wth a warning also, than its a int and not a str
connection1_var = db_connection()
def keys_view(self):
return Dict.keys(self)
def get_source(self, source_config: typing.Dict):
cls = self._sources[source_config['name']]
return cls(source_config.get('authentication_key', ""))
def map_datacenter(document: typing.Dict) -> Datacenter:
id = document.get('id')
rooms = list(map(map_room, document['rooms']))
return Datacenter(rooms=rooms, id=id)
def stringify_dict(inp: typing.Dict):
return {str(k): str(v) for k, v in inp.items()}
def map_room(document: typing.Dict) -> Room:
id = document.get('id')
type = RoomType[document.get('type', 'SERVER')]
objects = list(map(map_room_object, document.get('objects', [])))
return Room(objects=objects, type=type, id=id)
