def __on_pairing_event(self, action, pairing): key = (pairing.user_unique_id, pairing.player_id) if action == "added": for notifier in _.filter_(self.__notifiers, lambda n: n.user_unique_id == pairing.user_unique_id): notifier.set_player_ids(database.get_user_paired_player_ids(pairing.user_unique_id)) elif action == "removed": for notifier in _.filter_(self.__notifiers, lambda n: n.user_unique_id == pairing.user_unique_id): notifier.set_player_ids(database.get_user_paired_player_ids(pairing.user_unique_id))
def add_relationship_mutations(field_type, names_only=False):
mutations = ''
mutation_names = []
def f1(field):
i = 0
while i < len(field.ast_node.directives):
return field.ast_node.directives[i].name.value == 'relation'
relationship_fields = filter_(field_type.fields, f1)
for field in relationship_fields:
relation_directive = (
filter_(field.ast_node.directives, lambda d: d.name.value == 'relation')[0])
rel_type = filter_(relation_directive.arguments, lambda a: a.name.value == 'name')[0]
rel_direction = filter_(relation_directive.arguments, lambda a: a.name.value == 'direction')[0]
if rel_direction.value.value in ['out', 'OUT']:
from_type = field_type
to_type = inner_type(field.type)
else:
from_type = inner_type(field.type)
to_type = field_type
from_pk = primary_key(from_type)
to_pk = primary_key(to_type)
# FIXME: could add relationship properties here
mutations += (f'Add{from_type.name}{to_type.name}'
f'({low_first_letter(from_type.name + from_pk.ast_node.name.value)}: {inner_type(from_pk.type).name}!, '
f'{low_first_letter(to_type.name + to_pk.ast_node.name.value)}: {inner_type(to_pk.type).name}!): '
f'{from_type.name} @MutationMeta(relationship: "{rel_type.value.value}", from: "{from_type.name}", to: "{to_type.name}")')
mutation_names.append(f'Add{from_type.name}{to_type.name}')
if names_only:
return mutation_names
else:
return mutations
def all(cls, *args):
try:
dem_args = list(args)
if some(dem_args, lambda result: isinstance(result, Error)):
return head(
filter_(dem_args,
lambda result: isinstance(result, Error)))
if every(dem_args, lambda result: isinstance(result, Ok) == False):
return Error(
Exception('Some items passed in were not a Result.'))
return Ok(map_(dem_args, lambda result: result.getValue()))
except Exception as e:
return Error(e)
def check_all():
'''Check all spec files, all specs.'''
spec_files = ps.filter_(os.listdir(SPEC_DIR), lambda f: f.endswith('.json') and not f.startswith('_'))
for spec_file in spec_files:
spec_dict = util.read(f'{SPEC_DIR}/{spec_file}')
for spec_name, spec in spec_dict.items():
try:
spec['name'] = spec_name
spec['git_SHA'] = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode().strip()
check(spec)
except Exception as e:
logger.exception(f'spec_file {spec_file} fails spec check')
raise e
logger.info(f'Checked all specs from: {ps.join(spec_files, ",")}')
return True
def for_each_facility(self, data, combination, previous_cycle_data=None):
result = NOT_REPORTING
values = values_for_records(self.fields, data.c_records)
number_of_consumption_record_blanks = len(
pydash.filter_(values, lambda v: v is None))
if data.c_count == 0 and data.a_count == 0 and data.p_count == 0:
return result
if data.c_count < 25 or data.a_count < 22 or data.p_count < 7:
result = NO
elif number_of_consumption_record_blanks > 2:
result = NO
else:
result = YES
return result
def find_registry_tags(registry_url, repo_name, src_tag, cred):
url = "https://{}/api/repositories/{}/tags".format(registry_url, repo_name)
payload = ""
headers = {
'Accept': "application/json",
'Content-Type': "application/json",
'Authorization': "Basic YWRtaW46SGFyYm9yMTIzNDU="
}
if cred:
response = get(url,
data=payload,
headers=headers,
auth=HTTPBasicAuth(cred['username'], cred['password']))
else:
response = get(url, data=payload, headers=headers)
# if response.status_code != 200:
# raise requests.ConnectTimeout
response = response.json()
replicated_tags = []
digest = _.filter_(response, {"name": src_tag})
if not digest:
return [src_tag]
replicated_tags = _.filter_(response, {"digest": digest[0]['digest']})
return _.map_(replicated_tags, 'name')
def cypher_query(context,
resolve_info,
first=-1,
offset=0,
_id=None,
**kwargs):
types_ident = type_identifiers(resolve_info.return_type)
type_name = types_ident.get('type_name')
variable_name = types_ident.get('variable_name')
schema_type = resolve_info.schema.get_type(type_name)
filtered_field_nodes = filter_(
resolve_info.field_nodes,
lambda n: n.name.value == resolve_info.field_name)
# FIXME: how to handle multiple field_node matches
selections = extract_selections(
filtered_field_nodes[0].selection_set.selections,
resolve_info.fragments)
# if len(selections) == 0:
# # FIXME: why aren't the selections found in the filteredFieldNode?
# selections = extract_selections(resolve_info.operation.selection_set.selections, resolve_info.fragments)
# FIXME: support IN for multiple values -> WHERE
arg_string = re.sub(r"\"([^(\")]+)\":", "\\1:", json.dumps(kwargs))
id_where_predicate = f'WHERE ID({variable_name})={_id} ' if _id is not None else ''
outer_skip_limit = f'SKIP {offset}{" LIMIT " + str(first) if first > -1 else ""}'
cyp_dir = cypher_directive(resolve_info.schema.query_type,
resolve_info.field_name)
if cyp_dir:
custom_cypher = cyp_dir.get('statement')
query = (
f'WITH apoc.cypher.runFirstColumn("{custom_cypher}", {arg_string}, true) AS x '
f'UNWIND x AS {variable_name} RETURN {variable_name} '
f'{{{build_cypher_selection("", selections, variable_name, schema_type, resolve_info)}}} '
f'AS {variable_name} {outer_skip_limit}')
else:
# No @cypher directive on QueryType
query = f'MATCH ({variable_name}:{type_name} {arg_string}) {id_where_predicate}'
query += (
f'RETURN {variable_name} '
f'{{{build_cypher_selection("", selections, variable_name, schema_type, resolve_info)}}}'
f' AS {variable_name} {outer_skip_limit}')
return query
def export_settings(conn_id: int,
payload: s.ExportIn,
db: Session = Depends(get_db)):
conn = db.query(models.Connection).get(conn_id)
result = {}
if "expressions" in payload.options:
result["expressions"] = [
e.dict() for e in redact.get_expressions(connection=conn)
]
if "policies" in payload.options:
policies = redact.get_policies(connection=conn)
columns = redact.get_columns(connection=conn)
result["policies"] = []
for p in policies:
policy = p.dict()
policy["columns"] = [
c.dict() for c in pydash.filter_(
columns,
lambda x: x.object_owner == p.object_owner and x.
object_name == p.object_name,
)
]
result["policies"].append(policy)
if "categories" in payload.options:
result["categories"] = [
CategoryOut.from_orm(c).dict()
for c in db.query(models.Category).filter(
models.Connection.id == conn_id).all()
]
if "plans" in payload.options:
result["plans"] = [
PlanOut.from_orm(p).dict() for p in db.query(models.Plan).filter(
models.Connection.id == conn_id).all()
]
if "rules" in payload.options:
result["rules"] = [
RuleOut.from_orm(r).dict() for r in db.query(models.Rule).filter(
models.Connection.id == conn_id).all()
]
with tempfile.NamedTemporaryFile(prefix=conn.name,
suffix="_export.json",
mode="w",
delete=False) as handler:
handler.write(json.dumps(jsonable_encoder(result), indent=4))
name = handler.name
return FileResponse(name)
def check_all():
'''Check all spec files, all specs.'''
spec_files = ps.filter_(os.listdir(SPEC_DIR), lambda f: f.endswith('.json') and not f.startswith('_'))
for spec_file in spec_files:
spec_dict = util.read(f'{SPEC_DIR}/{spec_file}')
for spec_name, spec in spec_dict.items():
# fill-in info at runtime
spec['name'] = spec_name
spec = extend_meta_spec(spec)
try:
check(spec)
except Exception as e:
logger.exception(f'spec_file {spec_file} fails spec check')
raise e
logger.info(f'Checked all specs from: {ps.join(spec_files, ",")}')
return True
async def download_maps(self):
try:
map_list = await self._api_config.get_maps()
def cb(m):
return pydash.get(m, 'site')
map_list = pydash.filter_(map_list, cb)
if len(map_list) is 0:
print('there are no maps on site configuration')
return False
except Exception as err:
print('failed to download maps')
return False
return True
async def test_command(self, ctx: Context):
"""
Count how many messages you have in this channel.
"""
placeholder = await self.bot.say('Fetching messages..')
messages = [m async for m in self.bot.logs_from(ctx.message.channel)]
my_messages = pydash.filter_(
messages,
lambda m: m.author == ctx.message.author
)
return await self.bot.edit_message(
placeholder,
f'You have {len(my_messages)} messages.'
)
async def turn_on_electrodes(self, payload, params):
# Get the three object from device-model
scicad = MicropedeAsync(APPNAME, port=self.port, loop=self.loop)
three_object = await scicad.get_state('device-model', 'three-object')
active_electrodes = payload
def active_filter(obj):
return _.includes(active_electrodes, obj["id"])
active_objects = _.filter_(three_object, active_filter)
channels = _.map_(_.map_(active_objects, "channel"), int)
max_channels = self.control_board.number_of_channels
channel_states = np.zeros(max_channels, dtype=int)
channel_states[channels] = 1
self.control_board.set_state_of_channels(channel_states)
print(self.control_board.state_of_channels)
await self.update_board_info()
def check_all():
'''Check all spec files, all specs.'''
spec_files = ps.filter_(
os.listdir(SPEC_DIR),
lambda f: f.endswith('.json') and not f.startswith('_'))
for spec_file in spec_files:
spec_dict = util.read(f'{SPEC_DIR}/{spec_file}')
for spec_name, spec in spec_dict.items():
try:
spec['name'] = spec_name
spec['git_SHA'] = subprocess.check_output(
['git', 'rev-parse', 'HEAD']).decode().strip()
check(spec)
except Exception as e:
logger.exception(f'spec_file {spec_file} fails spec check')
raise e
logger.info(f'Checked all specs from: {ps.join(spec_files, ",")}')
return True
def find_adjacent_cells(origin: Coord,
cells: List[Coord],
only_orthogonal: bool = False) -> List[Coord]:
"""
Находит соседние клетки от origin среди cells
:param only_orthogonal:
:param origin:
:param cells:
:return:
"""
chosen_nearness = orthogonal if only_orthogonal else adjacent_square
diffs = py_.map_(cells, lambda other: other - origin)
adjacent = py_.filter_(
py_.zip_(cells, diffs),
lambda pair: pair[1] in chosen_nearness,
)
return py_.map_(adjacent, 0)
def filter_list(self, by: str):
"""Filter a dict or list
Args:
by (str): Required. What to filter by. Defaults to None.
Raises:
TypeError: If state is not a list
Returns:
Chepy: The Chepy object.
Examples:
>>> Chepy('[{"a": 1}, {"b": 2}, {"a": 1, "b": 3}]').str_list_to_list().filter_list("b").o
[{"b": 2}, {"a": 1, "b": 3}]
"""
assert isinstance(self.state, list), StateNotList()
self.state = pydash.filter_(self.state, by)
return self
def get_yield_strength_to_composed_data():
conn = db.get_connection()
cur = conn.cursor()
main_query_select = 'SELECT main_info_id, sortament, sigma_t FROM rloveshhenko$mydbtest.mechanical_properties WHERE sigma_t != " " and main_info_id in (SELECT id FROM mydbtest.main_info WHERE classification like "%Сталь%");'
ids_query_select = 'SELECT distinct main_info_id FROM rloveshhenko$mydbtest.mechanical_properties WHERE sigma_t != " " and main_info_id in (SELECT id FROM mydbtest.main_info WHERE classification like "%Сталь%");'
cur.execute(main_query_select)
data = cur.fetchall()
cur.execute(ids_query_select)
ids = cur.fetchall()
sigmas = map_(
ids, lambda item: {
'id':
item['main_info_id'],
'sigmas':
map_(
filter_(
data, lambda it: it['main_info_id'] == item[
'main_info_id']), lambda x: get(x, 'sigma_t'))
})
new_sigmas = map_(
sigmas, lambda item: {
'id':
item['id'],
'sigma':
format(
reduce_(
item['sigmas'], lambda total, x: float(total) + float(
x) / len(item['sigmas']), 0), '.2f')
})
for item in new_sigmas:
cur.execute(
"UPDATE rloveshhenko$mydbtest.composed_data SET sigma_t = %s WHERE id = %s",
(item['sigma'], item['id']))
conn.commit()
cur.close()
return True
def chooseSong(song_data):
"""
chooses a song to tweet and updates recently choosen songs
"""
recent = song_data['recentSongIds']
songs = song_data['songs']
# filter out recently choosen songs and randomly choose a song
filtered_song_ids = pydash.filter_(songs.keys(), lambda x: x not in recent)
song_id = pydash.sample(filtered_song_ids)
# get chosen song and increment play count
song = songs[song_id]
song['playCount'] += 1
# pop least recently choosen song and push new one
if len(recent) == 7:
pydash.shift(recent)
pydash.push(recent, song_id)
return song
def get_containers(self, name=None, database=None):
docker = self.app.docker
if name:
if not len(re.findall(r'^anydb_', name)):
if database and not len(
re.findall(r'^anydb_' + database + '_', name)):
name = 'anydb_' + database + name
else:
name = 'anydb_' + name
else:
if database:
name = 'anydb_' + database + '_'
else:
name = 'anydb_'
def filter_(value):
if len(re.findall(r'^' + name, value.name)):
return True
return False
return _.filter_(docker.containers.list(all=True), filter_)
def get_markets(self, main_market_filter=None):
"""
Gets all the Bittrex markets and filters them based on the main market filter
:param main_market_filter: Main market to filter on (ex: BTC, ETH, USDT)
:type main_market_filter: str
:return: All Bittrex markets (with filter applied, if any)
:rtype : list
"""
markets = self.Bittrex.get_markets()
if not markets["success"]:
logger.error("Failed to fetch Bittrex markets")
exit()
markets = markets["result"]
if main_market_filter is not None:
market_check = main_market_filter + "-"
markets = py_.filter_(markets, lambda market: market_check in market["MarketName"])
markets = py_.map_(markets, lambda market: market["MarketName"])
return markets
def filter_by(self, predicate: Any = None):
"""Filter a dict or list
Args:
predicate (Any, optional): What to filter by. Defaults to None.
Raises:
TypeError: If state is not a list or dict
Returns:
Chepy: The Chepy object.
Examples:
>>> Chepy('[{"a": 1}, {"b": 2}, {"a": 1, "b": 3}]').str_list_to_list().filter_by("b").o
[{"b": 2}, {"a": 1, "b": 3}]
"""
if isinstance(self.state, (list, dict)):
self.state = pydash.filter_(self.state, predicate)
return self
else: # pragma: no cover
raise TypeError("State is not a list")
def get_markets(self, main_market_filter=None):
"""
Gets all the Bittrex markets and filters them based on the main market filter
:param main_market_filter: Main market to filter on (ex: BTC, ETH, USDT)
:type main_market_filter: str
:return: All Bittrex markets (with filter applied, if any)
:rtype : list
"""
markets = self.Bittrex.get_markets()
if not markets["success"]:
logger.error("Failed to fetch Bittrex markets")
exit()
markets = markets["result"]
if main_market_filter is not None:
market_check = main_market_filter + "-"
markets = py_.filter_(
markets, lambda market: market_check in market["MarketName"])
markets = py_.map_(markets, lambda market: market["MarketName"])
return markets
def get_markets(main_market_filter=None):
"""
Gets all the Bittrex markets and filters them based on the main market filter
:param main_market_filter: Main market to filter on (ex: BTC, ETH, USDT)
:type main_market_filter: str
:return: All Bittrex markets (with filter applied, if any)
:rtype : list
"""
markets = Bittrex.get_markets()
if not markets['success']:
logger.error('Failed to fetch Bittrex markets')
exit()
markets = markets['result']
if main_market_filter is not None:
market_check = main_market_filter + '-'
markets = py_.filter_(markets, lambda market: market_check in market['MarketName'])
markets = py_.map_(markets, lambda market: market['MarketName'])
return markets
def get_countries_data_by_currency(self, currency_name, continent_name=False):
# Parameters validation.
if (not currency_name):
raise NameError(
'You must provide a currency name in order to use the get_countries_data_by_currency() method.')
# Setting up conditions.
conditions = {'Currency Name': currency_name.strip()}
# Checking if a continent name has been provided.
if continent_name:
# Sanitizing input fields.
continent_name = self._sanitize_continent_name(continent_name)
# Trimming and assigning value to the conditions object.
conditions['Continent Name'] = continent_name.strip()
# Filtering GWOD and return.
countries = pydash.filter_(self._GWOD, conditions)
countries = pydash.sort_by(countries, ['Country Name'])
return countries
def convert_csv(path):
ap = []
result = ""
with open(path) as csvfile:
dialect = csv.Sniffer().sniff(csvfile.read(4096))
dialect.doublequote = True
csvfile.seek(0)
reader = csv.reader(csvfile, dialect)
header = False
for row in reader:
if not header:
header = row
else:
item = {}
for i in range(0, len(row)):
item[header[i]] = row[i]
ap.append(item)
domains = pydash.without(pydash.uniq(pydash.map_(ap, 'EA-Domain')), '', None)
codelists = pydash.filter_(ap, {'EA-Type': 'ENUMERATION'})
domains = list(set(domains) - set(pydash.map_(codelists.copy(), 'EA-Name')))
domains.sort()
final_domains = []
final_datypes = []
classes = pydash.filter_(ap, {'EA-Type': 'CLASS'}) + pydash.filter_(ap, {'EA-Type': 'DATATYPE'})
datatypes = pydash.map_(pydash.filter_(ap, {'EA-Type': 'DATATYPE'}), 'EA-Name')
classes_only = pydash.map_(pydash.filter_(ap, {'EA-Type': 'CLASS'}), 'EA-Name')
attributes = pydash.filter_(ap, {'EA-Type': 'attribute'}) + pydash.filter_(ap, {'EA-Type': 'connector'})
attributes = pydash.sort_by(attributes, 'EA-Domain')
# for enumeration in codelists:
# attributes = pydash.remove(attributes, {'EA-Domain': enumeration})
title = os.path.splitext(os.path.basename(path))[0]
package = pydash.find(ap, {'EA-Type': 'Package'})
if len(domains) > 0:
for domain in domains:
klassen = pydash.filter_(classes, {'EA-Name': domain})
if 0 < len(klassen) <= 1:
klasse = pydash.find(classes, {'EA-Name': domain})
if klasse['EA-Type'] == 'DATATYPE':
result += "\n[%s]\n" % domain
final_datypes.append(domain)
else:
result += "\n[%s]\n" % domain
final_domains.append(domain)
if klasse is not None:
result += 'ap-definition-nl=%s\n' % klasse['ap-definition-nl']
result += 'ap-usagenote-nl=%s\n' % klasse['ap-usageNote-nl']
result += 'namespace=%s\n' % klasse['namespace']
result += 'localname=%s\n' % klasse['localname']
domain_attributes = pydash.filter_(attributes, {'EA-Domain': domain})
domain_attribute_names = pydash.without(pydash.uniq(pydash.map_(domain_attributes, 'EA-Name')), '', None) #localname
result += 'attributes=%s\n' % ','.join(domain_attribute_names)
for attr_name in domain_attribute_names:
result += "\n[%s:%s]\n" % (domain, attr_name)
attr = pydash.find(domain_attributes, {'EA-Name': attr_name})
if attr['range'] == "http://www.w3.org/2004/02/skos/core#Concept":
ap_codelist = pydash.find(codelists, {'EA-Name': attr['EA-Range']})
if not ap_codelist is None:
attr['ap-codelist'] = ap_codelist['ap-codelist']
for key in attr:
result += '%s=%s\n' % (key, attr[key].replace('&', '&'))
elif len(klassen) > 1:
for klasse in klassen:
if klasse['ap-label-nl'] == "":
klasse['ap-label-nl'] = domain
if klasse['EA-Type'] == 'DATATYPE':
result += "\n[%s]\n" % klasse['ap-label-nl']
final_datypes.append(klasse['ap-label-nl'])
else:
result += "\n[%s]\n" % klasse['ap-label-nl']
final_domains.append(klasse['ap-label-nl'])
if klasse is not None:
result += 'ap-definition-nl=%s\n' % klasse['ap-definition-nl']
result += 'ap-usagenote-nl=%s\n' % klasse['ap-usageNote-nl']
result += 'namespace=%s\n' % klasse['namespace']
result += 'localname=%s\n' % klasse['localname']
domain_attributes = pydash.filter_(attributes,
{'EA-Domain-GUID': klasse['EA-GUID']})
domain_attribute_names = pydash.without(pydash.uniq(
pydash.map_(domain_attributes, 'localname')), '', None)
result += 'attributes=%s\n' % ','.join(
domain_attribute_names)
for attr_name in domain_attribute_names:
result += "\n[%s:%s]\n" % (klasse['ap-label-nl'], attr_name)
attr = pydash.find(domain_attributes,
{'localname': attr_name})
if attr[
'range'] == "http://www.w3.org/2004/02/skos/core#Concept":
ap_codelist = pydash.find(codelists, {
'EA-Name': attr['EA-Range']})
if not ap_codelist is None:
attr['ap-codelist'] = ap_codelist[
'ap-codelist']
for key in attr:
result += '%s=%s\n' % (key, attr[key])
result += "\n[overview]\n"
final_domains = list(set(final_domains))
final_domains.sort()
result += 'entities=%s\n' % ','.join(final_domains)
result += 'dtypes=%s\n' % ','.join(final_datypes)
if package is not None:
result += 'package=%s\n' % package['EA-Name'].replace('OSLO-', '')
result += 'title=%s\n' % title
return [result, package['EA-Name'].replace('OSLO-', '')]
def cypher_mutation(context,
resolve_info,
first=-1,
offset=0,
_id=None,
**kwargs):
# FIXME: lots of duplication here with cypherQuery, extract into util module
types_ident = type_identifiers(resolve_info.return_type)
type_name = types_ident.get('type_name')
variable_name = types_ident.get('variable_name')
schema_type = resolve_info.schema.get_type(type_name)
filtered_field_nodes = filter_(
resolve_info.field_nodes,
lambda n: n.name.value == resolve_info.field_name)
# FIXME: how to handle multiple field_node matches
selections = extract_selections(
filtered_field_nodes[0].selection_set.selections,
resolve_info.fragments)
# FIXME: support IN for multiple values -> WHERE
arg_string = re.sub(r"\"([^(\")]+)\":", "\\1:", json.dumps(kwargs))
id_where_predicate = f'WHERE ID({variable_name})={_id} ' if _id is not None else ''
outer_skip_limit = f'SKIP {offset}{" LIMIT " + str(first) if first > -1 else ""}'
cyp_dir = cypher_directive(resolve_info.schema.mutation_type,
resolve_info.field_name)
if cyp_dir:
custom_cypher = cyp_dir.get('statement')
query = (
f'CALL apoc.cypher.doIt("{custom_cypher}", {arg_string}) YIELD value '
f'WITH apoc.map.values(value, [keys(value)[0]])[0] AS {variable_name} '
f'RETURN {variable_name} {{{build_cypher_selection("", selections, variable_name, schema_type, resolve_info)}}} '
f'AS {variable_name} {outer_skip_limit}')
# No @cypher directive on MutationType
elif resolve_info.field_name.startswith(
'create') or resolve_info.field_name.startswith('Create'):
# Create node
# TODO: handle for create relationship
# TODO: update / delete
# TODO: augment schema
query = (
f'CREATE ({variable_name}:{type_name}) SET {variable_name} = $params RETURN {variable_name} '
f'{{{build_cypher_selection("", selections, variable_name, schema_type, resolve_info)}}} '
f'AS {variable_name}')
elif resolve_info.field_name.startswith(
'add') or resolve_info.field_name.startswith('Add'):
mutation_meta = mutation_meta_directive(
resolve_info.schema.mutation_type, resolve_info.field_name)
relation_name = mutation_meta.get('relationship')
from_type = mutation_meta.get('from')
from_var = low_first_letter(from_type)
to_type = mutation_meta.get('to')
to_var = low_first_letter(to_type)
from_param = resolve_info.schema.mutation_type.fields[
resolve_info.
field_name].ast_node.arguments[0].name.value[len(from_var):]
to_param = resolve_info.schema.mutation_type.fields[
resolve_info.
field_name].ast_node.arguments[1].name.value[len(to_var):]
query = (
f'MATCH ({from_var}:{from_type} {{{from_param}: ${from_param}}}) '
f'MATCH ({to_var}:{to_type} {{{to_param}: ${to_param}}}) '
f'CREATE ({from_var})-[:{relation_name}]->({to_var}) '
f'RETURN {from_var} '
f'{{{build_cypher_selection("", selections, variable_name, schema_type, resolve_info)}}} '
f'AS {from_var}')
else:
raise Exception('Mutation does not follow naming conventions')
return query
# -*- coding: utf-8 -*-
from copy import deepcopy
import pydash as _
from .fixtures import parametrize
pydash_methods = _.filter_(dir(_), lambda m: callable(getattr(_, m, None)))
def test_chaining_methods():
chain = _.chain([])
for method in dir(_):
if not callable(method):
continue
chained = getattr(chain, method)
assert chained.method is getattr(_, method)
@parametrize('value,methods', [
([1, 2, 3, 4], [('without', (2, 3)),
('reject', (lambda x: x > 1,))])
])
def test_chaining(value, methods):
expected = deepcopy(value)
actual = _.chain(deepcopy(value))
def multi_file_handler(self, msg):
'''子进程处理单个文件的写入
'''
try:
# 上传进程内部加载sqlalchemy时需要控制每个进程的数据库连接池大小
os.environ['SQLALCHEMY_POOL_SIZE'] = '1'
from run import app as inner_app
from app import aios_redis
with inner_app.app_context():
self.aios_redis = aios_redis
file_key = msg.get('file_key')
dir_path = msg.get('dir_path')
file_name = msg.get('file_name')
curr_chunk = msg.get('curr_chunk')
total_chunks = msg.get('total_chunks')
tenant_id = msg.get('tenant_id')
user_id = msg.get('user_id')
cache_expired_time = msg.get('cache_expired_time')
wait_lock = f'plus_uploader:lock:{file_key}'
while not self.aios_redis.setnx(wait_lock,
f'lock.{curr_chunk}'):
# self.aios_print(file_key, curr_chunk, 'task waiting...')
time.sleep(0.001)
else:
self.lock = f'{wait_lock} by {curr_chunk}'
self.aios_print(file_key, curr_chunk,
f'current lock: lock.{curr_chunk}')
# 锁独占状态
# wait_lock超时时间 = 已存在但未合并的分片个数 * 单个分片预估的失效时间
parts = len(
_.filter_(
os.listdir(dir_path),
lambda x: '-' not in x and '.deleted' not in x))
count = max(parts, 1)
self.aios_print(f'文件存放目录{dir_path}, 实时已存在的分片数: {parts}')
self.aios_redis.expire(wait_lock,
cache_expired_time * count)
# self.aios_print(f'修正的lock超时时间, {cache_expired_time * count}')
# 更新任务状态
args = {
'task_id': file_key,
'tenant_id': tenant_id,
'created_by': user_id,
'updated_by': user_id,
'batch': Utils.get_batch(dir_path),
'status': TASK_STATUS_NONE,
'chunks': '',
'size': 0,
'link': {
'host': Utils.get_host(dir_path)
}
}
taskdao = TaskDAO()
# task_json = taskdao.get_task_from_cache(tenant_id, file_key)
task_json = taskdao.get_task(tenant_id,
file_key,
json=True)
if task_json is None:
task_json = taskdao.add(args)
if task_json['status'] != TASK_STATUS_MERGED:
task_json['status'] = TASK_STATUS_MERGING
# taskdao.update(task_json)
TaskModel.bulk_update_mappings([task_json])
# 合并分片
# 合并后的完整文件路径
merged_file = os.path.join(dir_path, file_name)
merge_process = self.partation_merge(
dir_path, file_key, total_chunks)
merge_process = _.map_(
merge_process,
lambda x: x.replace(f'{file_key}.', ''))
except_complete_name = f'1-{total_chunks}' if total_chunks > 1 else '1'
# 检测是否完整 ['1-701'] except_complete_name 1-701
self.aios_print('检测是否完整', merge_process,
'except_complete_name',
except_complete_name)
if except_complete_name in merge_process:
for f in os.listdir(dir_path):
self.aios_print('比较文件名', f,
except_complete_name)
if f.startswith(f'{file_key}.') and (
not f.endswith(except_complete_name)):
Utils.try_remove(os.path.join(dir_path, f))
self.aios_print('删除残留异常文件', f)
# 修改合并后的文件名称
final_merged_file = os.path.join(
dir_path, f'{file_key}.{except_complete_name}')
shutil.move(final_merged_file, merged_file)
self.aios_print('文件改名', final_merged_file, '>>',
merged_file)
# 记录任务状态
task_json['chunks'] = ','.join(
[str(i + 1) for i in range(total_chunks)])
task_json['status'] = TASK_STATUS_MERGED
task_json['size'] = os.path.getsize(merged_file)
task_json['link'] = {
'host': Utils.get_host(dir_path)
}
# 合并完成
self.aios_print(f'{curr_chunk}.合并完成')
else:
# ['1', '3-5', '10']
covert_process = []
for section in merge_process:
if '-' in section:
[left, right] = section.split('-')
covert_process.extend([
str(i)
for i in range(int(left),
int(right) + 1)
])
else:
covert_process.append(section)
task_json['chunks'] = ','.join(covert_process)
# 不完整
self.aios_print(f'{curr_chunk}.不完整,继续等待')
# 保存到缓存
# taskdao.update(task_json)
self.aios_print(f'更新状态, {task_json}')
TaskModel.bulk_update_mappings([task_json])
else:
# 已经合并完成后,不需要再次合并,直接退出
self.aios_print(f'{curr_chunk}.已经合并完成后, 不需要再次合并')
# 释放锁
self.aios_redis.delete(wait_lock)
self.aios_print(f'结束.{curr_chunk}/{total_chunks}')
# 清理被标记的可删除分片
for file in os.listdir(dir_path):
if file.endswith('.deleted'):
self.aios_print('清理被标记的可删除分片', file)
Utils.try_remove(os.path.join(dir_path, file))
return {'file_key': file_key, 'curr_chunk': curr_chunk}, None
except Exception as err:
import traceback
traceback.print_exc()
print('multi_file_handler', err)
return None, {
'file_key': msg['file_key'],
'curr_chunk': msg['curr_chunk']
}
def convert_contributor_csv(path, voc):
items = []
result = ""
with open(path) as csvfile:
dialect = csv.Sniffer().sniff(csvfile.read(4096))
csvfile.seek(0)
reader = csv.reader(csvfile, dialect)
header = False
for row in reader:
if not header:
header = row
else:
item = {}
for i in range(0, len(row)):
item[header[i]] = row[i]
items.append(item)
contributors = pydash.filter_(items, {voc: 'C'})
editors = pydash.filter_(items, {voc: 'E'})
authors = pydash.filter_(items, {voc: 'A'})
contributor_emails = pydash.map_(contributors.copy(), 'E-mail')
editor_emails = pydash.map_(editors.copy(), 'E-mail')
author_emails = pydash.map_(authors.copy(), 'E-mail')
if len(items) > 0:
result += "\n[overview_contributors]\n"
result += 'voc=%s\n' % voc.lower()
result += 'issued=%s\n' % time.strftime("%Y-%m-%d")
if len(contributor_emails) > 0:
result += 'contributors=%s\n' % ','.join(contributor_emails)
if len(editor_emails) > 0:
result += 'editors=%s\n' % ','.join(editor_emails)
if len(author_emails) > 0:
result += 'authors=%s\n' % ','.join(author_emails)
for contributor in contributor_emails:
result += "\n[contributor:%s]\n" % contributor
bijdrager = pydash.find(contributors, {'E-mail': contributor})
if bijdrager is not None:
result += 'naam=%s, %s\n' % (bijdrager['Naam'],
bijdrager['Voornaam'])
result += 'email=%s\n' % bijdrager['E-mail']
result += 'organisatie=%s\n' % bijdrager['Affiliatie']
result += 'website=%s\n' % bijdrager['Website']
for author in author_emails:
result += "\n[author:%s]\n" % author
bijdrager = pydash.find(authors, {'E-mail': author})
if bijdrager is not None:
result += 'naam=%s, %s\n' % (bijdrager['Naam'],
bijdrager['Voornaam'])
result += 'email=%s\n' % bijdrager['E-mail']
result += 'organisatie=%s\n' % bijdrager['Affiliatie']
result += 'website=%s\n' % bijdrager['Website']
for editor in editor_emails:
result += "\n[editor:%s]\n" % editor
bijdrager = pydash.find(editors, {'E-mail': editor})
if bijdrager is not None:
result += 'naam=%s, %s\n' % (bijdrager['Naam'],
bijdrager['Voornaam'])
result += 'email=%s\n' % bijdrager['E-mail']
result += 'organisatie=%s\n' % bijdrager['Affiliatie']
result += 'website=%s\n' % bijdrager['Website']
return result
def test_dash_suffixed_method_aliases():
methods = _.filter_(pydash_methods, lambda m: m.endswith('_'))
assert methods
for method in methods:
assert getattr(_._, method[:-1]) is getattr(_, method)
def first_id_field(field_type):
fields = filter_(list(field_type.fields.keys()), lambda t: inner_type(field_type.fields[t].type).name == 'ID')
if len(fields) > 0:
return field_type.fields[fields[0]]
else:
return None
def get_combination(combinations, name):
return pydash.filter_(combinations, lambda x: x[NAME] == name)[0]
def first_non_null_field(field_type):
fields = filter_(list(field_type.fields.keys()), lambda t: type(field_type.fields[t]).__name__ == 'GraphQLNonNull')
if len(fields) > 0:
return field_type.fields[fields[0]]
else:
return None
# print("Note : Default is [1] : Get All Repositories\n")
choice = None
isPrivate = None
while True:
try:
choice = int(input("Enter Your Choice : "))
isPrivate = repo_select(choice)
if isPrivate == "Invalid Selection":
continue
except ValueError:
print("Invalid Selection! Try again.")
print(isPrivate)
continue
else:
break
loop_request(init_api_url)
if isPrivate != "All":
print(json.dumps(_.filter_(repo_data, {"private": isPrivate}), indent=4))
write_data_to_file('outdata.json',
(_.filter_(repo_data, {"private": isPrivate})))
else:
print(json.dumps(repo_data, indent=4))
write_data_to_file('outdata.json', repo_data)
def test_dash_suffixed_method_aliases():
methods = _.filter_(pydash_methods, lambda m: m.endswith('_'))
assert methods
for method in methods:
assert getattr(_._, method[:-1]) is getattr(_, method)
# -*- coding: utf-8 -*-
from copy import deepcopy
import pydash as _
from .fixtures import parametrize
pydash_methods = _.filter_(dir(_), lambda m: callable(getattr(_, m, None)))
def test_chaining_methods():
chain = _.chain([])
for method in dir(_):
if not callable(method):
continue
chained = getattr(chain, method)
assert chained.method is getattr(_, method)
@parametrize('value,methods',
[([1, 2, 3, 4], [('without', (2, 3)),
('reject', (lambda x: x > 1, ))])])
def test_chaining(value, methods):
expected = deepcopy(value)
actual = _.chain(deepcopy(value))
for method, args in methods:
expected = getattr(_, method)(expected, *args)
def test_filter_(case, expected):
assert _.filter_(*case) == expected
