def test_fd(self): with open('foo', 'w') as f: with jsonstreams.Stream(jsonstreams.Type.object, fd=f) as s: s.write('foo', 'bar') with open('foo', 'r') as f: assert f.read() == '{"foo": "bar"}'
def main(): args = parse_args() gh = get_github3_client() json_fout = jsonstreams.Stream(jsonstreams.Type.array, args.json, indent=4) if args.json else None for org in args.orgs: full_query = 'org:{} {}'.format(org, args.query) if args.verbose: print('searching with query {}'.format(full_query)) sleep_if_rate_limited(gh, verbose=args.verbose) print("{0:<16}{1:<32}{2:<64}".format('org', 'repo', 'file path')) for result in gh.search_code(full_query): print("{0:<16}{1.repository.name:<32}{1.path:<64}".format( org, result)) if json_fout: json_fout.write(result.to_json()) if json_fout: # must explicitly close -- that's what outputs the closing delimiter json_fout.close()
def test_write_two(self): with jsonstreams.Stream(jsonstreams.Type.object, filename='foo') as s: s.write('foo', 'bar') s.write('bar', 'foo') with open('foo', 'r') as f: assert f.read() == '{"foo": "bar", "bar": "foo"}'
def test_context_manager_sub(self): with jsonstreams.Stream(jsonstreams.Type.object, filename='foo') as s: with s.subarray('foo') as a: with a.subarray() as b: with b.subobject() as c: with c.subobject('bar') as _: pass
def main(): args = sys.argv[1:] if len(args) % 2: sys.exit("Expected an even number of arguments") with jsonstreams.Stream(jsonstreams.Type.object, filename="/dev/stdout") as s: for k, v in more_itertools.sliced(args, 2): if k.endswith("@!") or k.endswith("!@"): # This currently isn't memory-efficient at all. :( if v == '-': s.write(k[0:-2], json.load(sys.stdin)) else: with open(v, "r") as f: with flocked(f): s.write(k[0:-2], json.load(f)) elif k.endswith("@"): # Waiting on https://github.com/dcbaker/jsonstreams/issues/30 # for a way to do this in bounded memory. if v == '-': s.write(k[0:-1], sys.stdin.read()) else: with open(v, "r") as f: with flocked(f): s.write(k[0:-1], f.read()) elif k.endswith("!"): s.write(k[0:-1], json.loads(v)) else: s.write(k, v)
def test_basic(self): s = jsonstreams.Stream(jsonstreams.Type.object, filename='foo') s.write('foo', 'bar') s.close() with open('foo', 'r') as f: assert f.read() == '{"foo": "bar"}'
def test_subobject(self): with jsonstreams.Stream(jsonstreams.Type.object, filename='foo') as s: s.write('foo', 'bar') with s.subobject('bar') as b: b.write('1', 'foo') with open('foo', 'r') as f: assert f.read() == '{"foo": "bar", "bar": {"1": "foo"}}'
def test_basic(self): with jsonstreams.Stream( jsonstreams.Type.array, filename='foo') as s: s.iterwrite(range(5)) with open('foo', 'r') as f: actual = json.load(f) assert actual == list(range(5))
def test_subarray(self): with jsonstreams.Stream(jsonstreams.Type.array, filename='foo') as s: s.write('foo') with s.subarray() as b: b.write(1) b.write(2) with open('foo', 'r') as f: assert f.read() == '["foo", [1, 2]]'
def main(users_list_path: str): screen_name = "nijisanji_app" list_name = "list1" cursor = Cursor(api.list_members, slug=list_name, owner_screen_name=screen_name) with jsonstreams.Stream(jsonstreams.Type.array, users_list_path) as s: for user in cursor.items(): s.write(user._json)
def test_basic(self): with jsonstreams.Stream( jsonstreams.Type.object, filename='foo') as s: s.iterwrite( ((str(s), k) for s in range(5) for k in range(5))) with open('foo', 'r') as f: actual = json.load(f) assert actual == {str(s): k for s in range(5) for k in range(5)}
def test_sub(self): s = jsonstreams.Stream(jsonstreams.Type.object, filename='foo') a = s.subarray('foo') b = a.subarray() c = b.subobject() d = c.subobject('bar') d.close() c.close() b.close() a.close() s.close()
def test_mixed(self): with jsonstreams.Stream( jsonstreams.Type.object, filename='foo') as s: s.iterwrite( ((str(s), k) for s in range(5) for k in range(5))) s.write("6", 'a') with open('foo', 'r') as f: actual = json.load(f) expected = {str(s): k for s in range(5) for k in range(5)} expected['6'] = 'a' assert actual == expected
def test_encoder_indent(self): with jsonstreams.Stream(jsonstreams.Type.object, filename='foo', indent=4) as s: s.write('oink', {'bar': {'b': 0}}) with open('foo', 'r') as f: actual = f.read() assert actual == textwrap.dedent("""\ { "oink": { "bar": { "b": 0 } } }""")
def __init__(self, filename: str, format: str = 'json', compression: Optional[str] = None, **kwargs: Any): super().__init__() self.filename = filename if compression: self.compression = compression else: self.compression = None self.FH = jsonstreams.Stream(jsonstreams.Type.object, filename=filename, pretty=True, indent=4) self.NH = None self.EH = None
def enumerate_tables(self): '''Start enumerating various details including table names, number of columns and rows, and column names''' enum_logger.info("Enumerating table names") total_tables = self.DB['info']['table_count'] if not self.debug: print( clr.yellow( "\n'{}' tables found, Enumerating columns ...\n".format( clr.red(total_tables)))) prog_bar = trange(int(total_tables)) table_dict = {} with jsonstreams.Stream(jsonstreams.Type.object, fd=self.get_fd(), pretty=True, indent=4) as s: '''Using jsonstreams to write the enumerated information to a json file as it is retreived from the target_url through SQL injection. This is done so that in the event of a crash or connection reset/timeout, whatever was retreived is not lost.''' s.write('date', self.DB['date']) s.write('params', self.DB['params']) s.write('info', self.DB['info']) with s.subobject('tables') as t: for i in prog_bar: live_enum_pl = self.sqli.replace( self.payload_delimiter, self.enum_tables_pl + self.limits.replace("~num~", str(i))) self.data[self.vuln_field] = live_enum_pl table = self.send_sqli() prog_bar.set_description("Table '{}': '{}'".format( clr.red(str(i + 1)), clr.red(table))) enum_logger.info("Enumerating table '{}': '{}'".format( i + 1, table)) table_dict[table] = {} table_dict[table]['row_count'] = self.get_num_rows(table) table_dict[table]['col_count'] = self.get_num_cols(table) table_dict[table]['cols'] = self.get_cols( table_dict[table]['col_count'], table) t.write(table, table_dict[table]) return table_dict
def write_results(codebase, output_file, pretty=False, **kwargs): """ Write headers, files, and other attributes from `codebase` to `output_file` Enable JSON indentation if `pretty` is True """ # Set indentation for JSON output if `pretty` is True # We use a separate dict for jsonstream kwargs since we are passing # this function's kwargs as arguments to OutputPlugin.get_files() if pretty: jsonstreams_kwargs = dict(indent=2, pretty=True) else: jsonstreams_kwargs = dict(indent=None, pretty=False) # If `output_file` is a path string, open the file at path `output_file` and use it as `output_file` close_fd = False if isinstance(output_file, str): output_file = open(output_file, 'w') close_fd = True # Begin wri'w' JSON to `output_file` with jsonstreams.Stream( jsonstreams.Type.OBJECT, fd=output_file, close_fd=close_fd, **jsonstreams_kwargs ) as s: # Write headers codebase.add_files_count_to_current_header() codebase_headers = codebase.get_headers() s.write('headers', codebase_headers) # Write attributes if codebase.attributes: for attribute_key, attribute_value in codebase.attributes.to_dict().items(): s.write(attribute_key, attribute_value) # Write files codebase_files = OutputPlugin.get_files(codebase, **kwargs) # OutputPlugin.get_files() returns a generator, not JSON-serializable codebase_files = list(codebase_files) s.write('files', codebase_files)
def __init__( self, owner, filename: str, format: str = "json", compression: Optional[str] = None, **kwargs: Any, ): super().__init__(owner) self.filename = filename if compression: self.compression = compression else: self.compression = None self.FH = jsonstreams.Stream(jsonstreams.Type.OBJECT, filename=filename, pretty=True, indent=4) self.NH = None self.EH = None
def download_and_write_streaming( outdir, filename_prefix, compress, generator_func, generator_func_args, item_id_dict_key, addl_info_dict_key=None, ): if compress: outfile = gzip.open(f'{outdir}/{filename_prefix}.json.gz', 'wt') else: outfile = open(f'{outdir}/{filename_prefix}.json', 'w') item_infos = set() with jsonstreams.Stream(jsonstreams.Type.array, fd=outfile, encoder=StrDefaultEncoder) as s: for item in generator_func(*generator_func_args): if isinstance(item, list): for i in item: s.write(i) if not addl_info_dict_key: item_infos.add(_get_item_by_key(i, item_id_dict_key)) else: item_infos.add(( _get_item_by_key(i, item_id_dict_key), _get_item_by_key(i, addl_info_dict_key), )) else: s.write(item) if not addl_info_dict_key: item_infos.add(_get_item_by_key(item, item_id_dict_key)) else: item_infos.add(( _get_item_by_key(item, item_id_dict_key), _get_item_by_key(item, addl_info_dict_key), )) outfile.close() return item_infos
def serializeCycle(config: Config): """Serialize and deserialize a configuration""" from io import StringIO import jsonstreams from experimaestro.core.objects import ConfigInformation import json import experimaestro.taskglobals as taskglobals taskglobals.Env.instance().wspath = Path("/tmp-xpm1234") stringOut = StringIO() serialized: Set[int] = set() with jsonstreams.Stream(jsonstreams.Type.ARRAY, fd=stringOut, close_fd=False) as out: config.__xpm__._outputjson_inner(out, None, serialized) objects = json.loads(stringOut.getvalue()) return ConfigInformation.fromParameters(objects)
def test_pretty(self): with jsonstreams.Stream(jsonstreams.Type.array, filename='foo', indent=4, pretty=True) as s: s.write({'bar': {"b": 0}}) s.write({'fob': {"f": 0}}) with open('foo', 'r') as f: actual = f.read() assert actual == textwrap.dedent("""\ [ { "bar": { "b": 0 } }, { "fob": { "f": 0 } } ]""")
def __init__(self, path, log_level='TRACE', rpa=False, generator='Robot'): if not jsonstreams: raise DataError('Using the JSON output format requires the ' 'jsonstreams module to be installed. Typically ' 'you can install it by running ' '`pip install jsonstreams`.') self._log_message_is_logged = IsLogged(log_level) self._error_message_is_logged = IsLogged('WARN') self._path = path # Setup the JSON data to store before writing the file self._data = { 'rpa': rpa is not None, 'generator': get_full_version(generator), 'generated': get_timestamp() } self._errors = [] # Setup stacks self._item_stack = list() # We need to keep track of the active suite, test, and body item self._suite = None self._body = None self._body_item = None self._test = None self._errors_element = None # We need to be able to track the type of item being processed # at any moment self._item_type = None self._root = jsonstreams.Stream(jsonstreams.Type.object, filename=self._path) self._root.write('rpa', rpa is not None) self._root.write('generator', get_full_version(generator)) self._root.write('generated', get_timestamp())
async def fetch(file_name, rewrite): if not Path(file_name).exists() or rewrite: async with httpx.AsyncClient() as client: with jsonstreams.Stream(jsonstreams.Type.object, filename=file_name, indent=2, pretty=True) as file_stream: with file_stream.subarray("cards") as cards: for stack, url in STACKS.items(): console.print(f"Fetching {stack}") response = await client.get(url) pages = int(get_pages(response.text)) for page in range(1, pages + 1): page_url = f"{url}?page={page}" response = await client.get(page_url) page_cards = parse_page(response.text, stack) cards.iterwrite(page_cards) else: message = ( f"File [bold magenta]{file_name}[/bold magenta] already exists!\n\n" "If you want to rewrite, use [bold yellow]--rewrite[/bold yellow] option.\n" "You can also specify different file name with [bold yellow]--file [FILENAME][/bold yellow] option." ) console.print(message)
for i in range(len(file_names_train)): print('file name:', file_names_train[i]) # read image using PIL: I = Image.open(os.path.join(data_path, file_names_train[i])) # convert to numpy array: I = np.asarray(I) preds_train[file_names_train[i]] = detect_red_light_mf(I, det_filters, is_weak=is_weak) with jsonstreams.Stream(jsonstreams.Type.object, fd=open( os.path.join( preds_path, 'preds_weak_train_backup.json'), 'a+'), indent=2, pretty=True) as s: s.write(file_names_train[i], preds_train[file_names_train[i]]) # save preds (overwrites any previous predictions!) with open(os.path.join(preds_path, 'preds_weak_train.json'), 'w') as f: json.dump(preds_train, f) if done_tweaking: ''' Make predictions on the test set. ''' preds_test = {} for i in range(len(file_names_test)):
# WebCam capture dimensions, a cv2 method cap.set(3, 640) cap.set(4, 480) # Video Codec settings. See fourcc.org for more info fourcc = cv.VideoWriter_fourcc(*'mp4v') # Set write path destination for video output, Walabot and timestamp video_out = cv.VideoWriter( '../data/raw/rbg_' + parse_results.datasession + '.mp4', fourcc, 20.0, (640, 480)) time_stamp_out = jsonstreams.Stream(jsonstreams.Type.object, filename='../data/raw/framedata_' + parse_results.datasession + '.json') wala_out = jsonstreams.Stream(jsonstreams.Type.object, filename='../data/raw/wala_' + parse_results.datasession + '.json') # frame_count used to incrimente every loop, matches frame captures frame_count = 1 """ ax1 = plt.subplot(1, 1, 1) ret, frame = cap.read() first_image = cv.cvtColor(frame, cv.COLOR_BGR2RGB) im1 = ax1.imshow(first_image) """
parameter_messages_stream.write( zz) files.add(zz) return files def extract_messages(filename, target_stream): for message in pe_messages(filename): event_id, value = message target_stream.write("%s" % event_id, value) files = process_sources('sources.json') with jsonstreams.Stream(jsonstreams.Type.OBJECT, filename='messages.json', pretty=True, indent=2) as sources_stream: files_count = len(files) for idx, source_file in enumerate(files): with sources_stream.subobject(source_file) as source_stream: mui_file = path.join(path.dirname(source_file), 'en-US', path.basename(source_file) + '.mui') for file in [mui_file, source_file]: print("[%s/%s] %s" % (idx, files_count, file)) if path.exists(file): for message in pe_messages(file): event_id, value = message source_stream.write("%s" % event_id, value) break
def finalize(self, metadata=None): """ End json serialization and cleanup This method is called after all of tests are written, it closes any containers that are still open and closes the file """ tests_dir = os.path.join(self._dest, 'tests') file_list = sorted( (f for f in os.listdir(tests_dir) if f.endswith('.json')), key=lambda p: int(os.path.splitext(p)[0])) # If jsonstreams is not present then build a complete tree of all of # the data and write it with json.dump if not _STREAMS: # Create a dictionary that is full of data to be written to a # single file data = collections.OrderedDict() # Load the metadata and put it into a dictionary with open(os.path.join(self._dest, 'metadata.json'), 'r') as f: data.update(json.load(f)) # If there is more metadata add it the dictionary if metadata: data.update(metadata) # Add the tests to the dictionary data['tests'] = collections.OrderedDict() for test in file_list: test = os.path.join(tests_dir, test) if os.path.isfile(test): # Try to open the json snippets. If we fail to open a test # then throw the whole thing out. This gives us atomic # writes, the writing worked and is valid or it didn't # work. try: with open(test, 'r') as f: data['tests'].update(json.load(f)) except ValueError: pass assert data['tests'] data = results.TestrunResult.from_dict(data) # write out the combined file. Use the compression writer from the # FileBackend with self._write_final(os.path.join(self._dest, 'results.json')) as f: json.dump(data, f, default=piglit_encoder, indent=INDENT) # Otherwise use jsonstreams to write the final dictionary. This uses an # external library, but is slightly faster and uses considerably less # memory that building a complete tree. else: encoder = functools.partial(json.JSONEncoder, default=piglit_encoder) with self._write_final(os.path.join(self._dest, 'results.json')) as f: with jsonstreams.Stream(jsonstreams.Type.object, fd=f, indent=4, encoder=encoder, pretty=True) as s: s.write('__type__', 'TestrunResult') with open(os.path.join(self._dest, 'metadata.json'), 'r') as n: s.iterwrite(six.iteritems(json.load(n, object_pairs_hook=collections.OrderedDict))) if metadata: s.iterwrite(six.iteritems(metadata)) with s.subobject('tests') as t: for test in file_list: test = os.path.join(tests_dir, test) if os.path.isfile(test): try: with open(test, 'r') as f: a = json.load(f) except ValueError: continue t.iterwrite(six.iteritems(a)) # Delete the temporary files os.unlink(os.path.join(self._dest, 'metadata.json')) shutil.rmtree(os.path.join(self._dest, 'tests'))
d['x_l_hip'] = x d['y_l_hip'] = y elif (l == 12): d['x_l_knee'] = x d['y_l_knee'] = y elif (l == 13): d['x_l_foot'] = x d['y_l_foot'] = y except: pass # data[k]['Person ' + str(i)].append(d) h['Person ' + str(i)].append(d) if (i == m): os.chdir('/content/drive/My Drive/Pose-Estimation/json') with jsonstreams.Stream(jsonstreams.Type.object, filename='foo' + str(k)) as s: s.write(str(k), h) # print(data) i = i + 1 k = k + 1 logger.debug('show+') frame_end = time.time() output_file.write(image) print(round(frame_end - frame_start)) # cv2.imshow('tf-pose-estimation result', image) cv2.waitKey(1) logger.debug('finished+')
def process_sources(outfile): files = set() with jsonstreams.Stream(jsonstreams.Type.OBJECT, filename=outfile, pretty=True, indent=2) as sources_stream: with ConnectRegistry(None, HKEY_LOCAL_MACHINE) as hive: with OpenKey(hive, r"SYSTEM\CurrentControlSet\Services\EventLog", 0, KEY_READ) as event_log_key: for log in subkeys(event_log_key): with sources_stream.subobject(log) as log_stream: with OpenKey(event_log_key, log) as log_key: for source in subkeys(log_key): with log_stream.subobject( source) as source_stream: with OpenKey(log_key, source) as source_key: event_message_file = value( source_key, "EventMessageFile") category_message_file = value( source_key, "CategoryMessageFile") category_count = value( source_key, "CategoryCount") parameter_message_file = value( source_key, "ParameterMessageFile") if category_count: source_stream.write( 'category_count', category_count) if event_message_file: with source_stream.subarray( 'event_messages' ) as event_messages_stream: for zz in event_message_file.split( ';'): zz = winreg.ExpandEnvironmentStrings( zz) event_messages_stream.write( zz) files.add(zz) if category_message_file: with source_stream.subarray( 'category_messages' ) as category_messages_stream: for zz in category_message_file.split( ';'): zz = winreg.ExpandEnvironmentStrings( zz) category_messages_stream.write( zz) files.add(zz) if parameter_message_file: with source_stream.subarray( 'parameter_messages' ) as parameter_messages_stream: for zz in parameter_message_file.split( ';'): zz = winreg.ExpandEnvironmentStrings( zz) parameter_messages_stream.write( zz) files.add(zz) return files
msg = "\t\tOutput will appear on the console every {} minute".format( args.duration) if args.duration > 1: msg += "s" print(msg, '\n\n') # convert the provided arguments from minutes to seconds delay = args.delay * 60 duration = args.duration * 60 outfile = None iterations = round(duration / delay) or 1 if args.output_file: with jsonstreams.Stream(jsonstreams.Type.object, filename=args.output_file) as outstream: display_usage( client, iterations=iterations, delay=delay, alert_threshold=args.alert_threshold, outstream=outstream, ) else: display_usage( client, iterations=iterations, delay=delay, alert_threshold=args.alert_threshold, )