def test_fd(self):
with open('foo', 'w') as f:
with jsonstreams.Stream(jsonstreams.Type.object, fd=f) as s:
s.write('foo', 'bar')
with open('foo', 'r') as f:
assert f.read() == '{"foo": "bar"}'
def main():
args = parse_args()
gh = get_github3_client()
json_fout = jsonstreams.Stream(jsonstreams.Type.array, args.json,
indent=4) if args.json else None
for org in args.orgs:
full_query = 'org:{} {}'.format(org, args.query)
if args.verbose:
print('searching with query {}'.format(full_query))
sleep_if_rate_limited(gh, verbose=args.verbose)
print("{0:<16}{1:<32}{2:<64}".format('org', 'repo', 'file path'))
for result in gh.search_code(full_query):
print("{0:<16}{1.repository.name:<32}{1.path:<64}".format(
org, result))
if json_fout:
json_fout.write(result.to_json())
if json_fout:
# must explicitly close -- that's what outputs the closing delimiter
json_fout.close()
def test_write_two(self):
with jsonstreams.Stream(jsonstreams.Type.object, filename='foo') as s:
s.write('foo', 'bar')
s.write('bar', 'foo')
with open('foo', 'r') as f:
assert f.read() == '{"foo": "bar", "bar": "foo"}'
def test_context_manager_sub(self):
with jsonstreams.Stream(jsonstreams.Type.object, filename='foo') as s:
with s.subarray('foo') as a:
with a.subarray() as b:
with b.subobject() as c:
with c.subobject('bar') as _:
pass
def main():
args = sys.argv[1:]
if len(args) % 2:
sys.exit("Expected an even number of arguments")
with jsonstreams.Stream(jsonstreams.Type.object,
filename="/dev/stdout") as s:
for k, v in more_itertools.sliced(args, 2):
if k.endswith("@!") or k.endswith("!@"):
# This currently isn't memory-efficient at all. :(
if v == '-':
s.write(k[0:-2], json.load(sys.stdin))
else:
with open(v, "r") as f:
with flocked(f):
s.write(k[0:-2], json.load(f))
elif k.endswith("@"):
# Waiting on https://github.com/dcbaker/jsonstreams/issues/30
# for a way to do this in bounded memory.
if v == '-':
s.write(k[0:-1], sys.stdin.read())
else:
with open(v, "r") as f:
with flocked(f):
s.write(k[0:-1], f.read())
elif k.endswith("!"):
s.write(k[0:-1], json.loads(v))
else:
s.write(k, v)
def test_basic(self):
s = jsonstreams.Stream(jsonstreams.Type.object, filename='foo')
s.write('foo', 'bar')
s.close()
with open('foo', 'r') as f:
assert f.read() == '{"foo": "bar"}'
def test_subobject(self):
with jsonstreams.Stream(jsonstreams.Type.object, filename='foo') as s:
s.write('foo', 'bar')
with s.subobject('bar') as b:
b.write('1', 'foo')
with open('foo', 'r') as f:
assert f.read() == '{"foo": "bar", "bar": {"1": "foo"}}'
def test_basic(self):
with jsonstreams.Stream(
jsonstreams.Type.array, filename='foo') as s:
s.iterwrite(range(5))
with open('foo', 'r') as f:
actual = json.load(f)
assert actual == list(range(5))
def test_subarray(self):
with jsonstreams.Stream(jsonstreams.Type.array, filename='foo') as s:
s.write('foo')
with s.subarray() as b:
b.write(1)
b.write(2)
with open('foo', 'r') as f:
assert f.read() == '["foo", [1, 2]]'
def main(users_list_path: str):
screen_name = "nijisanji_app"
list_name = "list1"
cursor = Cursor(api.list_members,
slug=list_name,
owner_screen_name=screen_name)
with jsonstreams.Stream(jsonstreams.Type.array, users_list_path) as s:
for user in cursor.items():
s.write(user._json)
def test_basic(self):
with jsonstreams.Stream(
jsonstreams.Type.object, filename='foo') as s:
s.iterwrite(
((str(s), k) for s in range(5) for k in range(5)))
with open('foo', 'r') as f:
actual = json.load(f)
assert actual == {str(s): k for s in range(5) for k in range(5)}
def test_sub(self):
s = jsonstreams.Stream(jsonstreams.Type.object, filename='foo')
a = s.subarray('foo')
b = a.subarray()
c = b.subobject()
d = c.subobject('bar')
d.close()
c.close()
b.close()
a.close()
s.close()
def test_mixed(self):
with jsonstreams.Stream(
jsonstreams.Type.object, filename='foo') as s:
s.iterwrite(
((str(s), k) for s in range(5) for k in range(5)))
s.write("6", 'a')
with open('foo', 'r') as f:
actual = json.load(f)
expected = {str(s): k for s in range(5) for k in range(5)}
expected['6'] = 'a'
assert actual == expected
def test_encoder_indent(self):
with jsonstreams.Stream(jsonstreams.Type.object, filename='foo',
indent=4) as s:
s.write('oink', {'bar': {'b': 0}})
with open('foo', 'r') as f:
actual = f.read()
assert actual == textwrap.dedent("""\
{
"oink": {
"bar": {
"b": 0
}
}
}""")
def __init__(self,
filename: str,
format: str = 'json',
compression: Optional[str] = None,
**kwargs: Any):
super().__init__()
self.filename = filename
if compression:
self.compression = compression
else:
self.compression = None
self.FH = jsonstreams.Stream(jsonstreams.Type.object,
filename=filename,
pretty=True,
indent=4)
self.NH = None
self.EH = None
def enumerate_tables(self):
'''Start enumerating various details including table names, number of columns and rows, and column names'''
enum_logger.info("Enumerating table names")
total_tables = self.DB['info']['table_count']
if not self.debug:
print(
clr.yellow(
"\n'{}' tables found, Enumerating columns ...\n".format(
clr.red(total_tables))))
prog_bar = trange(int(total_tables))
table_dict = {}
with jsonstreams.Stream(jsonstreams.Type.object,
fd=self.get_fd(),
pretty=True,
indent=4) as s:
'''Using jsonstreams to write the enumerated information to a json file as it is retreived from the target_url through SQL injection. This is done so that in the event of a crash or connection reset/timeout, whatever was retreived is not lost.'''
s.write('date', self.DB['date'])
s.write('params', self.DB['params'])
s.write('info', self.DB['info'])
with s.subobject('tables') as t:
for i in prog_bar:
live_enum_pl = self.sqli.replace(
self.payload_delimiter, self.enum_tables_pl +
self.limits.replace("~num~", str(i)))
self.data[self.vuln_field] = live_enum_pl
table = self.send_sqli()
prog_bar.set_description("Table '{}': '{}'".format(
clr.red(str(i + 1)), clr.red(table)))
enum_logger.info("Enumerating table '{}': '{}'".format(
i + 1, table))
table_dict[table] = {}
table_dict[table]['row_count'] = self.get_num_rows(table)
table_dict[table]['col_count'] = self.get_num_cols(table)
table_dict[table]['cols'] = self.get_cols(
table_dict[table]['col_count'], table)
t.write(table, table_dict[table])
return table_dict
def write_results(codebase, output_file, pretty=False, **kwargs):
"""
Write headers, files, and other attributes from `codebase` to `output_file`
Enable JSON indentation if `pretty` is True
"""
# Set indentation for JSON output if `pretty` is True
# We use a separate dict for jsonstream kwargs since we are passing
# this function's kwargs as arguments to OutputPlugin.get_files()
if pretty:
jsonstreams_kwargs = dict(indent=2, pretty=True)
else:
jsonstreams_kwargs = dict(indent=None, pretty=False)
# If `output_file` is a path string, open the file at path `output_file` and use it as `output_file`
close_fd = False
if isinstance(output_file, str):
output_file = open(output_file, 'w')
close_fd = True
# Begin wri'w' JSON to `output_file`
with jsonstreams.Stream(
jsonstreams.Type.OBJECT,
fd=output_file,
close_fd=close_fd,
**jsonstreams_kwargs
) as s:
# Write headers
codebase.add_files_count_to_current_header()
codebase_headers = codebase.get_headers()
s.write('headers', codebase_headers)
# Write attributes
if codebase.attributes:
for attribute_key, attribute_value in codebase.attributes.to_dict().items():
s.write(attribute_key, attribute_value)
# Write files
codebase_files = OutputPlugin.get_files(codebase, **kwargs)
# OutputPlugin.get_files() returns a generator, not JSON-serializable
codebase_files = list(codebase_files)
s.write('files', codebase_files)
def __init__(
self,
owner,
filename: str,
format: str = "json",
compression: Optional[str] = None,
**kwargs: Any,
):
super().__init__(owner)
self.filename = filename
if compression:
self.compression = compression
else:
self.compression = None
self.FH = jsonstreams.Stream(jsonstreams.Type.OBJECT,
filename=filename,
pretty=True,
indent=4)
self.NH = None
self.EH = None
def download_and_write_streaming(
outdir,
filename_prefix,
compress,
generator_func,
generator_func_args,
item_id_dict_key,
addl_info_dict_key=None,
):
if compress:
outfile = gzip.open(f'{outdir}/{filename_prefix}.json.gz', 'wt')
else:
outfile = open(f'{outdir}/{filename_prefix}.json', 'w')
item_infos = set()
with jsonstreams.Stream(jsonstreams.Type.array,
fd=outfile,
encoder=StrDefaultEncoder) as s:
for item in generator_func(*generator_func_args):
if isinstance(item, list):
for i in item:
s.write(i)
if not addl_info_dict_key:
item_infos.add(_get_item_by_key(i, item_id_dict_key))
else:
item_infos.add((
_get_item_by_key(i, item_id_dict_key),
_get_item_by_key(i, addl_info_dict_key),
))
else:
s.write(item)
if not addl_info_dict_key:
item_infos.add(_get_item_by_key(item, item_id_dict_key))
else:
item_infos.add((
_get_item_by_key(item, item_id_dict_key),
_get_item_by_key(item, addl_info_dict_key),
))
outfile.close()
return item_infos
def serializeCycle(config: Config):
"""Serialize and deserialize a configuration"""
from io import StringIO
import jsonstreams
from experimaestro.core.objects import ConfigInformation
import json
import experimaestro.taskglobals as taskglobals
taskglobals.Env.instance().wspath = Path("/tmp-xpm1234")
stringOut = StringIO()
serialized: Set[int] = set()
with jsonstreams.Stream(jsonstreams.Type.ARRAY,
fd=stringOut,
close_fd=False) as out:
config.__xpm__._outputjson_inner(out, None, serialized)
objects = json.loads(stringOut.getvalue())
return ConfigInformation.fromParameters(objects)
def test_pretty(self):
with jsonstreams.Stream(jsonstreams.Type.array, filename='foo',
indent=4, pretty=True) as s:
s.write({'bar': {"b": 0}})
s.write({'fob': {"f": 0}})
with open('foo', 'r') as f:
actual = f.read()
assert actual == textwrap.dedent("""\
[
{
"bar": {
"b": 0
}
},
{
"fob": {
"f": 0
}
}
]""")
def __init__(self, path, log_level='TRACE', rpa=False, generator='Robot'):
if not jsonstreams:
raise DataError('Using the JSON output format requires the '
'jsonstreams module to be installed. Typically '
'you can install it by running '
'`pip install jsonstreams`.')
self._log_message_is_logged = IsLogged(log_level)
self._error_message_is_logged = IsLogged('WARN')
self._path = path
# Setup the JSON data to store before writing the file
self._data = {
'rpa': rpa is not None,
'generator': get_full_version(generator),
'generated': get_timestamp()
}
self._errors = []
# Setup stacks
self._item_stack = list()
# We need to keep track of the active suite, test, and body item
self._suite = None
self._body = None
self._body_item = None
self._test = None
self._errors_element = None
# We need to be able to track the type of item being processed
# at any moment
self._item_type = None
self._root = jsonstreams.Stream(jsonstreams.Type.object,
filename=self._path)
self._root.write('rpa', rpa is not None)
self._root.write('generator', get_full_version(generator))
self._root.write('generated', get_timestamp())
async def fetch(file_name, rewrite):
if not Path(file_name).exists() or rewrite:
async with httpx.AsyncClient() as client:
with jsonstreams.Stream(jsonstreams.Type.object,
filename=file_name,
indent=2,
pretty=True) as file_stream:
with file_stream.subarray("cards") as cards:
for stack, url in STACKS.items():
console.print(f"Fetching {stack}")
response = await client.get(url)
pages = int(get_pages(response.text))
for page in range(1, pages + 1):
page_url = f"{url}?page={page}"
response = await client.get(page_url)
page_cards = parse_page(response.text, stack)
cards.iterwrite(page_cards)
else:
message = (
f"File [bold magenta]{file_name}[/bold magenta] already exists!\n\n"
"If you want to rewrite, use [bold yellow]--rewrite[/bold yellow] option.\n"
"You can also specify different file name with [bold yellow]--file [FILENAME][/bold yellow] option."
)
console.print(message)
for i in range(len(file_names_train)):
print('file name:', file_names_train[i])
# read image using PIL:
I = Image.open(os.path.join(data_path, file_names_train[i]))
# convert to numpy array:
I = np.asarray(I)
preds_train[file_names_train[i]] = detect_red_light_mf(I,
det_filters,
is_weak=is_weak)
with jsonstreams.Stream(jsonstreams.Type.object,
fd=open(
os.path.join(
preds_path,
'preds_weak_train_backup.json'), 'a+'),
indent=2,
pretty=True) as s:
s.write(file_names_train[i], preds_train[file_names_train[i]])
# save preds (overwrites any previous predictions!)
with open(os.path.join(preds_path, 'preds_weak_train.json'), 'w') as f:
json.dump(preds_train, f)
if done_tweaking:
'''
Make predictions on the test set.
'''
preds_test = {}
for i in range(len(file_names_test)):
# WebCam capture dimensions, a cv2 method
cap.set(3, 640)
cap.set(4, 480)
# Video Codec settings. See fourcc.org for more info
fourcc = cv.VideoWriter_fourcc(*'mp4v')
# Set write path destination for video output, Walabot and timestamp
video_out = cv.VideoWriter(
'../data/raw/rbg_' + parse_results.datasession + '.mp4', fourcc, 20.0,
(640, 480))
time_stamp_out = jsonstreams.Stream(jsonstreams.Type.object,
filename='../data/raw/framedata_' +
parse_results.datasession + '.json')
wala_out = jsonstreams.Stream(jsonstreams.Type.object,
filename='../data/raw/wala_' +
parse_results.datasession + '.json')
# frame_count used to incrimente every loop, matches frame captures
frame_count = 1
"""
ax1 = plt.subplot(1, 1, 1)
ret, frame = cap.read()
first_image = cv.cvtColor(frame, cv.COLOR_BGR2RGB)
im1 = ax1.imshow(first_image)
"""
parameter_messages_stream.write(
zz)
files.add(zz)
return files
def extract_messages(filename, target_stream):
for message in pe_messages(filename):
event_id, value = message
target_stream.write("%s" % event_id, value)
files = process_sources('sources.json')
with jsonstreams.Stream(jsonstreams.Type.OBJECT,
filename='messages.json',
pretty=True,
indent=2) as sources_stream:
files_count = len(files)
for idx, source_file in enumerate(files):
with sources_stream.subobject(source_file) as source_stream:
mui_file = path.join(path.dirname(source_file), 'en-US',
path.basename(source_file) + '.mui')
for file in [mui_file, source_file]:
print("[%s/%s] %s" % (idx, files_count, file))
if path.exists(file):
for message in pe_messages(file):
event_id, value = message
source_stream.write("%s" % event_id, value)
break
def finalize(self, metadata=None):
""" End json serialization and cleanup
This method is called after all of tests are written, it closes any
containers that are still open and closes the file
"""
tests_dir = os.path.join(self._dest, 'tests')
file_list = sorted(
(f for f in os.listdir(tests_dir) if f.endswith('.json')),
key=lambda p: int(os.path.splitext(p)[0]))
# If jsonstreams is not present then build a complete tree of all of
# the data and write it with json.dump
if not _STREAMS:
# Create a dictionary that is full of data to be written to a
# single file
data = collections.OrderedDict()
# Load the metadata and put it into a dictionary
with open(os.path.join(self._dest, 'metadata.json'), 'r') as f:
data.update(json.load(f))
# If there is more metadata add it the dictionary
if metadata:
data.update(metadata)
# Add the tests to the dictionary
data['tests'] = collections.OrderedDict()
for test in file_list:
test = os.path.join(tests_dir, test)
if os.path.isfile(test):
# Try to open the json snippets. If we fail to open a test
# then throw the whole thing out. This gives us atomic
# writes, the writing worked and is valid or it didn't
# work.
try:
with open(test, 'r') as f:
data['tests'].update(json.load(f))
except ValueError:
pass
assert data['tests']
data = results.TestrunResult.from_dict(data)
# write out the combined file. Use the compression writer from the
# FileBackend
with self._write_final(os.path.join(self._dest, 'results.json')) as f:
json.dump(data, f, default=piglit_encoder, indent=INDENT)
# Otherwise use jsonstreams to write the final dictionary. This uses an
# external library, but is slightly faster and uses considerably less
# memory that building a complete tree.
else:
encoder = functools.partial(json.JSONEncoder, default=piglit_encoder)
with self._write_final(os.path.join(self._dest, 'results.json')) as f:
with jsonstreams.Stream(jsonstreams.Type.object, fd=f, indent=4,
encoder=encoder, pretty=True) as s:
s.write('__type__', 'TestrunResult')
with open(os.path.join(self._dest, 'metadata.json'),
'r') as n:
s.iterwrite(six.iteritems(json.load(n, object_pairs_hook=collections.OrderedDict)))
if metadata:
s.iterwrite(six.iteritems(metadata))
with s.subobject('tests') as t:
for test in file_list:
test = os.path.join(tests_dir, test)
if os.path.isfile(test):
try:
with open(test, 'r') as f:
a = json.load(f)
except ValueError:
continue
t.iterwrite(six.iteritems(a))
# Delete the temporary files
os.unlink(os.path.join(self._dest, 'metadata.json'))
shutil.rmtree(os.path.join(self._dest, 'tests'))
d['x_l_hip'] = x
d['y_l_hip'] = y
elif (l == 12):
d['x_l_knee'] = x
d['y_l_knee'] = y
elif (l == 13):
d['x_l_foot'] = x
d['y_l_foot'] = y
except:
pass
# data[k]['Person ' + str(i)].append(d)
h['Person ' + str(i)].append(d)
if (i == m):
os.chdir('/content/drive/My Drive/Pose-Estimation/json')
with jsonstreams.Stream(jsonstreams.Type.object,
filename='foo' + str(k)) as s:
s.write(str(k), h)
# print(data)
i = i + 1
k = k + 1
logger.debug('show+')
frame_end = time.time()
output_file.write(image)
print(round(frame_end - frame_start))
# cv2.imshow('tf-pose-estimation result', image)
cv2.waitKey(1)
logger.debug('finished+')
def process_sources(outfile):
files = set()
with jsonstreams.Stream(jsonstreams.Type.OBJECT,
filename=outfile,
pretty=True,
indent=2) as sources_stream:
with ConnectRegistry(None, HKEY_LOCAL_MACHINE) as hive:
with OpenKey(hive, r"SYSTEM\CurrentControlSet\Services\EventLog",
0, KEY_READ) as event_log_key:
for log in subkeys(event_log_key):
with sources_stream.subobject(log) as log_stream:
with OpenKey(event_log_key, log) as log_key:
for source in subkeys(log_key):
with log_stream.subobject(
source) as source_stream:
with OpenKey(log_key,
source) as source_key:
event_message_file = value(
source_key, "EventMessageFile")
category_message_file = value(
source_key, "CategoryMessageFile")
category_count = value(
source_key, "CategoryCount")
parameter_message_file = value(
source_key, "ParameterMessageFile")
if category_count:
source_stream.write(
'category_count',
category_count)
if event_message_file:
with source_stream.subarray(
'event_messages'
) as event_messages_stream:
for zz in event_message_file.split(
';'):
zz = winreg.ExpandEnvironmentStrings(
zz)
event_messages_stream.write(
zz)
files.add(zz)
if category_message_file:
with source_stream.subarray(
'category_messages'
) as category_messages_stream:
for zz in category_message_file.split(
';'):
zz = winreg.ExpandEnvironmentStrings(
zz)
category_messages_stream.write(
zz)
files.add(zz)
if parameter_message_file:
with source_stream.subarray(
'parameter_messages'
) as parameter_messages_stream:
for zz in parameter_message_file.split(
';'):
zz = winreg.ExpandEnvironmentStrings(
zz)
parameter_messages_stream.write(
zz)
files.add(zz)
return files
msg = "\t\tOutput will appear on the console every {} minute".format(
args.duration)
if args.duration > 1:
msg += "s"
print(msg, '\n\n')
# convert the provided arguments from minutes to seconds
delay = args.delay * 60
duration = args.duration * 60
outfile = None
iterations = round(duration / delay) or 1
if args.output_file:
with jsonstreams.Stream(jsonstreams.Type.object,
filename=args.output_file) as outstream:
display_usage(
client,
iterations=iterations,
delay=delay,
alert_threshold=args.alert_threshold,
outstream=outstream,
)
else:
display_usage(
client,
iterations=iterations,
delay=delay,
alert_threshold=args.alert_threshold,
)
