def get_vbb_data(centre):
global stations
global station_types
g = Graph()
with open('nodes.ndjson') as f:
dataSta = ndjson.load(f)
# convert to and from objects
textSta = ndjson.dumps(dataSta)
dataSta = ndjson.loads(textSta)
for i in dataSta:
#tupel = str(i['metadata']['x'])+","+str(i['metadata']['y'])
x = float(i['metadata']['longitude'])
y = float(i['metadata']['latitude'])
idSt = str(i['id'])
g.add_node(idSt)
stations[idSt] = (x, y)
# g.add_node(tupel)
with open('edges.ndjson') as f:
dataDist = ndjson.load(f)
# convert to and from objects
textDist = ndjson.dumps(dataDist)
dataDist = ndjson.loads(textDist)
for i in dataDist:
stationA = str(i['source'])
stationB = str(i['target'])
distance = int(i['metadata']['time'])
line = i['metadata']['line']
if line.startswith('RB') or line.startswith('RB'):
station_types[stationA] = 1
station_types[stationB] = 1
elif line.startswith('U') or line.startswith('S'):
if stationA in station_types:
if station_types[stationA] > 1:
station_types[stationA] = 2
else:
station_types[stationA] = 2
if stationB in station_types:
if station_types[stationB] > 1:
station_types[stationB] = 2
else:
station_types[stationB] = 2
else:
if stationA in station_types:
if station_types[stationA] > 2:
station_types[stationA] = 3
else:
station_types[stationA] = 3
if stationB in station_types:
if station_types[stationB] > 2:
station_types[stationB] = 3
else:
station_types[stationB] = 3
g.add_edge(stationA, stationB, distance)
return dijsktra(g, centre) # Station name of Dabendorf node: 900000245024
def single_data_handler(self, item):
topic, value = item
_data = ndjson.dumps([
value,
])
# print(_data)
self.send(topic, _data)
async def run_app(
loop: asyncio.AbstractEventLoop,
payload: List[dict],
entity: str,
mock_url: str,
) -> None:
asyncio.set_event_loop(loop)
with aioresponses() as mocked:
body = ndjson.dumps(payload)
mocked.get(mock_url, status=200, body=body)
test_app = init_app(
loop=loop,
settings=settings,
command_line_args=argparse.Namespace(verbose=False))
if entity == "patients":
task = loop.create_task(test_app.resolve_patients())
elif entity == "encounters":
task = loop.create_task(test_app.resolve_encounters())
elif entity == "procedures":
task = loop.create_task(test_app.resolve_procedures())
elif entity == "observations":
task = loop.create_task(test_app.resolve_observations())
else:
raise ValueError("unknown entity")
await asyncio.sleep(SLEEP_PERIOD)
task.cancel()
await asyncio.gather(task, return_exceptions=True)
def multi_data_handler(self, item):
topic, value = item
if not self.cache.get(topic):
self.cache[topic] = []
self.cache[topic].append(value)
if len(self.cache[topic]) >= 1:
_data = ndjson.dumps(self.cache[topic])
# print(_data)
self.send(topic, _data)
self.cache[topic] = []
def create_from_objects(cls,
client,
project_id: str,
name: str,
predictions: Iterable[Dict],
validate=True) -> 'BulkImportRequest':
"""
Creates a `BulkImportRequest` from an iterable of dictionaries.
Conforms to JSON predictions format, e.g.:
``{
"uuid": "9fd9a92e-2560-4e77-81d4-b2e955800092",
"schemaId": "ckappz7d700gn0zbocmqkwd9i",
"dataRow": {
"id": "ck1s02fqxm8fi0757f0e6qtdc"
},
"bbox": {
"top": 48,
"left": 58,
"height": 865,
"width": 1512
}
}``
Args:
client (Client): a Labelbox client
project_id (str): id of project for which predictions will be imported
name (str): name of BulkImportRequest
predictions (Iterable[dict]): iterable of dictionaries representing predictions
validate (bool): a flag indicating if there should be a validation
if `predictions` is valid ndjson
Returns:
BulkImportRequest object
"""
if validate:
_validate_ndjson(predictions, client.get_project(project_id))
data_str = ndjson.dumps(predictions)
if not data_str:
raise ValueError('annotations cannot be empty')
data = data_str.encode('utf-8')
file_name = _make_file_name(project_id, name)
request_data = _make_request_data(project_id, name, len(data_str),
file_name)
file_data = (file_name, data, NDJSON_MIME_TYPE)
response_data = _send_create_file_command(client,
request_data=request_data,
file_name=file_name,
file_data=file_data)
return cls(client, response_data["createBulkImportRequest"])
def json_to_gcs(path, json_object, bucket_name):
blob = storage.Blob(
name=path,
bucket=storage_client.get_bucket(bucket_name),
)
blob.upload_from_string(
# dataflow needs newline-delimited json, so use ndjson
data=ndjson.dumps(json_object),
content_type='application/json',
client=storage_client,
)
logging.info('Successfully uploaded blob %r to bucket %r.', path,
bucket_name)
def json_to_gcs(path, json_object_list, bucket_name):
"""
take list of dicts in memory and upload to GCS as newline JSON
"""
blob = storage.Blob(
name=path,
bucket=storage_client.get_bucket(bucket_name),
)
blob.upload_from_string(
# dataflow needs newline-delimited json, so use ndjson
data=ndjson.dumps(json_object_list),
content_type='application/json',
client=storage_client,
)
logging.info('Successfully uploaded blob %r to bucket %r.', path,
bucket_name)
print('Successfully uploaded blob {} to bucket {}'.format(
path, bucket_name))
def test_wait_till_done(rectangle_inference, configured_project):
name = str(uuid.uuid4())
url = configured_project.client.upload_data(content=ndjson.dumps(
[rectangle_inference]),
sign=True)
bulk_import_request = configured_project.upload_annotations(
name=name, annotations=url, validate=False)
assert len(bulk_import_request.inputs) == 1
bulk_import_request.wait_until_done()
assert bulk_import_request.state == BulkImportRequestState.FINISHED
# Check that the status files are being returned as expected
assert len(bulk_import_request.errors) == 0
assert len(bulk_import_request.inputs) == 1
assert bulk_import_request.inputs[0]['uuid'] == rectangle_inference['uuid']
assert len(bulk_import_request.statuses) == 1
assert bulk_import_request.statuses[0]['status'] == 'SUCCESS'
assert bulk_import_request.statuses[0]['uuid'] == rectangle_inference[
'uuid']
def main():
parser = argparse.ArgumentParser()
parser.add_argument('seq_fn')
parser.add_argument('--format', default='fasta')
parser.add_argument('--sleep', default='1', type=int,
help="Time to wait between queries (sec)")
parser.add_argument('--max_per_query', default=10, type=int)
args = parser.parse_args()
gen = cast_to_seq(Bio.SeqIO.parse(args.seq_fn, args.format))
for batch in chunk(gen, args.max_per_query):
data = query_interpro(batch)
# print output as NDjson
print(ndjson.dumps(data))
time.sleep(args.sleep)
return
def search_ndjson(request, database_name, collection_name,
skip=0, limit=getattr(settings, 'MONGO_LIMIT', 200),
sort=None, return_keys=(),
query={}):
result = prepare_search_results(
request,
database_name=database_name,
collection_name=collection_name,
skip=skip,
sort=sort,
limit=limit,
return_keys=return_keys,
query=query)
if int(result['code']) == 200:
return HttpResponse(ndjson.dumps(result["results"]),
status=int(result['code']),
content_type="application/x-ndjson")
else:
response = json.dumps(result, indent=4)
return HttpResponse(response, status=int(result['code']),
content_type="application/json")
def get(self, current_user):
'''
Takes a list of organizations and exports all the Event Rules for the supplied organizations
as NDJSON, if no organizations are provided, just dump the rules for the users current
organization.
'''
args = export_rule_parser.parse_args()
event_rules = EventRule.search()
if args.organizations:
print(args.organizations)
else:
event_rules = event_rules.filter('term', organization=current_user.organization)
event_rules = event_rules.scan()
output = ndjson.dumps([marshal(e, mod_event_rule_list) for e in event_rules])
resp = Response(output,headers={'Content-Type': 'application/x-ndjson', 'Content-disposition':'attachment; filename=event_rules.ndjson'})
return resp
def __assert_file_content(url: str):
response = requests.get(url)
assert response.text == ndjson.dumps(PREDICTIONS)
def assert_file_content(url: str, predictions):
response = requests.get(url)
assert response.text == ndjson.dumps(predictions)
def print_ndjson(self):
print(ndjson.dumps({k: v} for k, v in self.data_as_json().items()))
def msearch(self, json_queries, *args, **kwargs):
r = self.session.get(self.host + "/_msearch",
headers={"Content-Type": "application/x-ndjson"},
data=ndjson.dumps(json_queries) + "\n")
return r.json()
def test_dumps(text):
objects = _create_list(text)
assert ndjson.dumps(objects, sort_keys=True) == text
def export_to_ndjson(records: List[CallRecord], filepath: str) -> None:
ndjson_records = ndjson.dumps([r.dict() for r in records])
with open(filepath, "w") as export_file:
export_file.write(ndjson_records)
