def formdata(self, handler, instance): request = handler.request id = int(request.REQUEST.get('id', -1)) if request.method == "POST": formdata = load_json(request.POST.get('data', '{}')) extra = load_json(request.POST.get('extra', '{}')) formname = extra.get('name', 'new form') spokes = ",".join(extra.get('spokes', [])) if id == -1: pf = PropertyForm(conf=instance, name=formname, form=dump_json(formdata), types=spokes) pf.save() id = pf.id else: pf = PropertyForm.objects.get(pk=id) pf.name = formname pf.form = dump_json(formdata) pf.types = spokes pf.save() if id != -1: pf = PropertyForm.objects.get(pk=id) return dict(form=load_json(pf.form), extra=dict(name=pf.name, id=pf.id, spokes=pf.types.split(","))) return dict(form=[], extra={'name':'', 'spokes':[]})
def backup(self, full = False): transfer = self.transfer(newshare = self) #print(repr(self.newbackup.predecessor)) #print(repr(self.reference)) hostconfig = self.host.config info = dict( failed = False, name = self.name, path = self.path, mountpoint = self.mountpoint, ) with hostconfig.setenv(self.env): self.pre_command(fruitbak = self.fruitbak, host = self.host, backup = self.newbackup, newshare = self) info['startTime'] = time_ns() with self.hardhat_maker: transfer.transfer() info['endTime'] = time_ns() self.post_command(fruitbak = self.fruitbak, host = self.host, backup = self.newbackup, newshare = self) with open('info.json', 'w', opener = self.sharedir_fd.opener) as fp: dump_json(info, fp) return info
def do_context_diff(original, new, dump_as_json=False): if dump_as_json: return ''.join(context_diff(dump_json(original, indent=2).splitlines(True), dump_json(new, indent=2).splitlines(True), 'Original', 'New')) return ''.join(context_diff(original.splitlines(True), new.splitlines(True), 'Original', 'New'))
def write(pl, fn): "Write a playlist dictionary to file." assert valid_playlist(pl) if not type(fn) == file: plf = codecs.open(fn, 'wb', encoding="utf-8") else: writer = codecs.getwriter("utf8") plf = writer(fn) try: dump_json(pl, plf) finally: plf.close()
def backup(self): backupdir = self.backupdir backupdir_fd = self.backupdir_fd flock(backupdir_fd, LOCK_EX|LOCK_NB) def onerror(exc): raise exc for root, dirs, files, root_fd in fwalk(dir_fd = backupdir_fd, topdown = False, onerror = onerror): for name in files: unlink(name, dir_fd = root_fd) for name in dirs: rmdir(name, dir_fd = root_fd) env = self.env config = self.config shares_info = {} info = dict(level = self.level, failed = False, shares = shares_info) with config.setenv(env): self.pre_command(fruitbak = self.fruitbak, host = self.host, newbackup = self) info['startTime'] = time_ns() for share_config in self.shares: combined_config = config.copy() combined_config.update(share_config) share = NewShare(config = combined_config, newbackup = self) shares_info[share.name] = share.backup() self.agent.sync() info['endTime'] = time_ns() self.post_command(fruitbak = self.fruitbak, host = self.host, newbackup = self) with open('info.json', 'w', opener = backupdir_fd.opener) as fp: dump_json(info, fp) hostdir_fd = self.host.hostdir_fd self.hashes_fp.close() Hashset.sortfile('hashes', self.fruitbak.hash_size, dir_fd = backupdir_fd) rename('new', str(self.index), src_dir_fd = hostdir_fd, dst_dir_fd = hostdir_fd) return info
def verify_node(node): node_string = dump_json(node) variables = None secrets = None if upload_variables: variables_in_conf = regex_findall(r'\$ENV\((\S*?)\)', node_string) # Find env vars variables: dict = load_json( open(git_cloned_dir + sync_root + 'node/' + var_file_path).read()) for var in variables_in_conf: # Verify they exist in git repo if var not in variables: logging.error( f'Missing env var {var} in variables file {var_file_path}') if upload_secrets: secrets_in_conf = regex_findall(r'\$SECRET\((\S*?)\)', node_string) # Find secrets vault = Vaulter(vault_url, vault_git_token, vault_mounting_point) # Create keyvault object secrets: dict = vault.get_secrets( secrets_in_conf) # Get the secrets from keyvault if vault.verify() is False: # Verify all secrets exist. logging.error( f'These secrets do not exist in the vault {vault.get_missing_secrets()}' ) return variables, secrets
def to_geos(json_dict_polygon): """Return GEOSGeometry polygon from polygon dictionary""" json_dict_polygon = json_dict_polygon['geometry'] geo = GEOSGeometry(dump_json(json_dict_polygon)) geo.set_srid(original_project) geo.transform(desired_projection) return geo
def check_and_replace_orchestrator_systems(): for old_filename in os.listdir(sesam_checkout_dir + "/unpacked/systems/"): with open( os.path.join(sesam_checkout_dir + "/unpacked/systems/", old_filename), 'r') as f: # open in readonly mode old_file = load_json(f.read()) try: old_file["metadata"]["orchestrator"]["original_configuration"] for new_filename in os.listdir(git_cloned_dir + "/sesam-node/systems/"): with open( os.path.join( git_cloned_dir + "/sesam-node/systems/", new_filename), 'r') as g: # open in readonly mode new_file = load_json(g.read()) if old_file["metadata"]["orchestrator"][ "original_configuration"] == new_file: logging.info( "The system %s is restored to orchestrator mode" % new_file["_id"]) with open( os.path.join(payload_dir + "/systems/", new_filename), 'w') as h: h.write(dump_json(old_file)) except KeyError: None
def json_stringify(data): """ :type data: object :rtype: str """ return dump_json(data, sort_keys=True, separators=(',', ':')).encode("utf-8") if data else None
def generate_config(self): Sample = [{ "sass": { "source": "sass/style.scss", "output": "style.min.css", "output_style": "compressed", "source_comments": False } }, { "js": { "include": ["js/fo.js", "js/bar.js"], "output": "script.min.js" } }] # Save String = dump_json(Sample, indent="\t") Filepath = path.join(self.BasePath, "eleran.json") # Print echo_click(" * Creating config file to", Filepath) # Save FileWrite = open(Filepath, "w") FileWrite.write(String) FileWrite.close()
async def publish(self, topic, payload, retain=False): payload = ( dump_json(payload) if isinstance(payload, dict) else str(payload) ) _LOGGER.debug(f"Publishing on {topic}: {payload}") await self.mqtt.publish(topic, payload.encode("utf-8"), retain=retain) _LOGGER.debug(f"Published on {topic}: {payload}")
def query_matchingrule(self, mr_name, json=False): """Returns a single matching rule instance that matches the mr_name. Returns None if the matching rule doesn't exist. @param mr_name - The name of the matching rule you want to query. return MatchingRule or None <ldap.schema.models.MatchingRule instance> """ matchingRules = self.get_matchingrules() matchingRule = [ mr for mr in matchingRules if mr_name.lower() in list(map(str.lower, mr.names)) ] if len(matchingRule) != 1: # This is an error. if json: raise ValueError('Could not find matchingrule: ' + objectclassname) else: return None matchingRule = matchingRule[0] if json: result = {'type': 'schema', 'mr': vars(matchingRule)} return dump_json(result) else: return matchingRule
def list_all(inst, basedn, log, args): log = log.getChild('list_all') schema = Schema(inst) json = False if args is not None and args.json: json = True objectclass_elems = schema.get_objectclasses(json=json) attributetype_elems = schema.get_attributetypes(json=json) matchingrule_elems = schema.get_matchingrules(json=json) if json: print( dump_json( { 'type': 'schema', 'objectclasses': objectclass_elems, 'attributetypes': attributetype_elems, 'matchingrules': matchingrule_elems }, indent=4)) else: separator_line = "".join(["-" for _ in range(50)]) print("Objectclasses:\n", separator_line) for oc in objectclass_elems: print(oc) print("AttributeTypes:\n", separator_line) for at in attributetype_elems: print(at) print("MathingRules:\n", separator_line) for mr in matchingrule_elems: print(mr)
def query_objectclass(self, objectclassname, json=False): """Returns a single ObjectClass instance that matches objectclassname. Returns None if the objectClass doesn't exist. @param objectclassname - The name of the objectClass you want to query. return ObjectClass or None ex. query_objectclass('account') <ldap.schema.models.ObjectClass instance> """ objectclasses = self.get_objectclasses() objectclass = [ oc for oc in objectclasses if objectclassname.lower() in list(map(str.lower, oc.names)) ] if len(objectclass) != 1: # This is an error. if json: raise ValueError('Could not find objectcass: ' + objectclassname) else: return None objectclass = objectclass[0] if json: result = {'type': 'schema', 'oc': vars(objectclass)} return dump_json(result) else: return objectclass
def properties_data_handler(handler, request, action): """ return relevant form data """ ## return first form that matches. TODO: combine forms into groups ## combine with data # import pdb; pdb.set_trace() spoke = handler.spoke() if request.method == "POST": data = load_json(request.POST.get('data')) for (id, formdata) in data.iteritems(): form_obj_storage, _ = PropertyFormData.objects.get_or_create(form_id=id, content=spoke.instance) form_obj_storage.properties = dump_json(formdata) form_obj_storage.save() spokename = handler.spoke().name() forms = [] for pf in PropertyForm.objects.all(): types = pf.types.split(",") if spokename in types: forms.append(dict(id=pf.id, name=pf.name, form=load_json(pf.form))) data = {} for pfd in spoke.instance.properties.all(): data[pfd.form_id] = load_json(pfd.properties) return dict(forms=forms, data=data)
def state(self): state = self.instrument.state if self.is_lock: return (STATE_UNLOCK, STATE_LOCK)[state] elif self.is_switch: return (STATE_OFF, STATE_ON)[state] elif self.is_opening: return (STATE_CLOSE, STATE_OPEN)[state] elif self.is_safety: return (STATE_SAFE, STATE_UNSAFE)[state] elif self.is_binary_sensor: return (STATE_OFF, STATE_ON)[state] elif self.is_position: lat, lon = state key = self.config.get(CONF_OWNTRACKS_KEY) res = dict(_type='location', tid='volvo', t='p', lat=lat, lon=lon, acc=1, tst=int(time())) return (dict(_type='encrypted', data=owntracks_encrypt(dump_json(res), key)) if key else res) else: return state
def query_attributetype(self, attributetypename, json=False): """Returns a tuple of the AttributeType, and what objectclasses may or must take this attributeType. Returns None if attributetype doesn't exist. @param attributetypename - The name of the attributeType you want to query return (AttributeType, Must, May) or None ex. query_attributetype('uid') ( <ldap.schema.models.AttributeType instance>, [<ldap.schema.models.ObjectClass instance>, ...], [<ldap.schema.models.ObjectClass instance>, ...] ) """ # First, get the attribute that matches name. We need to consider # alternate names. There is no way to search this, so we have to # filter our set of all attribute types. objectclasses = self.get_objectclasses() attributetypes = self.get_attributetypes() attributetypename = attributetypename.lower() attributetype = [at for at in attributetypes if attributetypename.lower() in list(map(str.lower, at.names))] if len(attributetype) != 1: # This is an error. if json: raise ValueError('Could not find attribute: ' + attributetypename) else: return None attributetype = attributetype[0] # Get the primary name of this attribute attributetypename = attributetype.names[0] # Build a set if they have may. may = [oc for oc in objectclasses if attributetypename.lower() in list(map(str.lower, oc.may))] # Build a set if they have must. must = [oc for oc in objectclasses if attributetypename.lower() in list(map(str.lower, oc.must))] if json: # convert Objectclass class to dict, then sort each list may = [vars(oc) for oc in may] must = [vars(oc) for oc in must] # Add normalized 'name' for sorting for oc in may: oc['name'] = oc['names'][0].lower() for oc in must: oc['name'] = oc['names'][0].lower() may = sorted(may, key=itemgetter('name')) must = sorted(must, key=itemgetter('name')) result = {'type': 'schema', 'at': vars(attributetype), 'may': may, 'must': must} return dump_json(result) else: return (attributetype, must, may)
def data(request): redis = get_redis_connection('cirrus') data_id = request.GET.get('uuid', '') df = _get_data(data_id) key = KEY_DIGEST_PREFIX + data_id header = load_json(redis.hget(key, 'header')) new_limits = request.GET.get('new_limits', None) if new_limits: name, limits_lower, limits_upper = new_limits.split('|') else: name = None for row in header: if row['name'] == name: row['limits'] = [limits_lower, limits_upper] redis.hset(key, 'header', dump_json(header)) if row['limits'][0] != '': if row['type'].startswith('float'): lim = float(row['limits'][0]) elif row['type'].startswith('int'): lim = int(row['limits'][0]) df = df[df[row['name']] >= lim] elif row['limits'][1] != '': if row['type'].startswith('float'): lim = float(row['limits'][1]) elif row['type'].startswith('int'): lim = int(row['limits'][1]) df = df[df[row['name']] <= lim] print (df) orders = request.GET.get('order', '') orders = [row for row in orders.split(',') if row] if orders: sort_func = request.GET.get('sort', 'quicksort') ascending = [ row[0] != '-' for row in orders ] by = [ row[1:] if row[0] == '-' else row for row in orders ] df.sort_values(by=by, ascending=ascending, inplace=True, kind=sort_func) columns = {} for row in df.columns: if str(df[row].dtype).startswith('float'): columns[row] = [(i if isfinite(i) else None) for i in df[row]] else: columns[row] = [i for i in df[row]] return { 'index': [row for row in df.index], 'data': columns, 'order': orders }
def get_cli(): print("Fetching") sleep(0.5) data = m.get_new_lines() res = make_response(dump_json({"message": data}), 200) print("fetched", data) return res
def publish(self, topic, payload, retain=False): payload = dump_json(payload) if isinstance(payload, dict) else payload _LOGGER.debug(f'Publishing on {topic}: {payload}') res, mid = self.client.publish(topic, payload, retain=retain) if res == MQTT_ERR_SUCCESS: Entity.pending[mid] = (topic, payload) else: _LOGGER.warning('Failure to publish on %s', topic)
def list_attributetypes(inst, basedn, log, args): log = log.getChild('list_attributetypes') schema = Schema(inst) if args is not None and args.json: print(dump_json(schema.get_attributetypes(json=True), indent=4)) else: for attributetype in schema.get_attributetypes(): print(attributetype)
def write_report(world): path = 'output/report.json' if not os.path.exists(path): os.mkdir('output') with open(path, 'w') as f: pass with open(path, 'w') as f: f.write(dump_json(world.env.data))
def list_objectclasses(inst, basedn, log, args): log = log.getChild('list_objectclasses') schema = Schema(inst) if args is not None and args.json: print(dump_json(schema.get_objectclasses(json=True), indent=4)) else: for oc in schema.get_objectclasses(): print(oc)
def json_stringify(data): """ :type data: object :rtype: str """ # TODO remove this unused function return dump_json(data, sort_keys=True, separators=(',', ':')).encode("utf-8") if data else None
async def publish(self, topic, payload, retain=False): payload = (dump_json(payload) if isinstance(payload, dict) else str(payload)) _LOGGER.debug("Publishing on %s: %s", topic, payload) await self.client.publish(topic, payload.encode("utf-8"), retain=retain) _LOGGER.debug("Published on %s: %s", topic, payload)
def list_matchingrules(inst, basedn, log, args): log = log.getChild('list_matchingrules') schema = Schema(inst) if args is not None and args.json: print(dump_json(schema.get_matchingrules(json=True), indent=4)) else: for mr in schema.get_matchingrules(): print(mr)
def publish(self, mqtt, topic, payload, retain=False): payload = dump_json(payload) if isinstance(payload, dict) else payload _LOGGER.debug(f'Publishing on {topic}: {payload}') res, mid = mqtt.publish(topic, payload, retain=retain) if res == paho.MQTT_ERR_SUCCESS: Entity.subscriptions[mid] = (topic, payload) else: _LOGGER.warning('Failure to publish on %s', topic)
def write_report(world): report_directory = 'report/' with open(report_directory + 'report.json', 'w') as f: f.write(dump_json(world.env.data)) with open(report_directory + 'propagation-time.csv', 'w') as f: f.write('connection, count, sum, average\n') for connection in world.env.data['block_propagation']: propagation_values = world.env.data['block_propagation'][connection] if len(propagation_values) > 0: sum = 0 for i in propagation_values: sum = + propagation_values[i] avg = sum / len(propagation_values) f.write(connection + ', ' + str(len(propagation_values)) + ', ' + str(sum) + ', ' + str(avg) + '\n') # f.write(dump_json(world.env.data['block_propagation'])) # TODO: remove putting data in the memory with open(report_directory + 'verification-time.csv', 'w') as f: f.write(dump_json(world.env.data['block_verification'])) db = DBConnection() verification = db.getAllBlock_verification() with open(report_directory + 'verification-time-db.csv', 'w') as f: for ver in verification: f.write(str(ver)) vf_node = {} with open(report_directory + 'block-verification-time.csv', 'w') as f: # f.write(str(world.report_verification_time())) writer = csv.writer(f) for block_vf in world.report_verification_time(): split = block_vf.split(':') writer.writerow([split[1]]) if split[0] in vf_node: value = vf_node[split[0]] split_value = value.split(',') split_value[0] = str(int(split_value[0]) + 1) split_value[1] = str(float(split[1]) + float(split_value[1])) split_value[2] = str(float(split_value[1]) / float(split_value[0])) vf_node[split[0]] = split_value[0] + ', ' + split_value[1] + ', ' + split_value[2] else: vf_node[split[0]] = '1, ' + split[1] + ', ' + split[1] with open(report_directory + 'node-verification-time-average.csv', 'w') as f: f.write('node, count, sum, average\n') for block_vf in vf_node: f.write(block_vf + ', ' + vf_node[block_vf] + '\n')
def get_syntaxes(inst, basedn, log, args): log = log.getChild('get_syntaxes') schema = Schema(inst) result = schema.get_attr_syntaxes(json=args.json) if args.json: print(dump_json(result, indent=4)) else: for id, name in result.items(): print("%s (%s)", name, id)
async def hello(uri): print(uri) async with websockets.connect(uri) as websocket: # Subscribe KBar data = { 'action': 'subscribe', 'subscribe': 'kbar', 'kbar': '5min', 'pair': 'eth_btc' } data = dump_json(data) await websocket.send(data) sleep(0.1) # Subscribe Trade data = {'action': 'subscribe', 'subscribe': 'trade', 'pair': 'eth_btc'} data = dump_json(data) await websocket.send(data) sleep(0.1) # Subscribe Depth data = { 'action': 'subscribe', 'subscribe': 'depth', 'depth': 10, 'pair': 'eth_btc' } data = dump_json(data) await websocket.send(data) sleep(0.1) # Subscribe Tick data = {'action': 'subscribe', 'subscribe': 'tick', 'pair': 'eth_btc'} data = dump_json(data) await websocket.send(data) async for message in websocket: message = load_json(message) if 'ping' in message: data = {'action': 'pong', 'pong': message['ping']} await websocket.send(dump_json(data)) else: print(message)
def query_objectclass(inst, basedn, log, args): log = log.getChild('query_objectclass') schema = Schema(inst) # Need the query type oc = _get_arg(args.name, msg="Enter objectclass to query") result = schema.query_objectclass(oc, json=args.json) if args.json: print(dump_json(result, indent=4)) else: print(result)
def query_matchingrule(inst, basedn, log, args): log = log.getChild('query_matchingrule') schema = Schema(inst) # Need the query type attr = _get_arg(args.name, msg="Enter attribute to query") result = schema.query_matchingrule(attr, json=args.json) if args.json: print(dump_json(result, indent=4)) else: print(result)
def format_json(w_file, cursor, single=False, **kwargs): entries = [] resource = { "meta": {}, "data": entries, "jsonapi": { "version": "1.0" }, } if cursor.label: resource['meta']['type'] = cursor.label for row in cursor: entries.append(row) if single and len(entries) < 1: raise errors.UnknownResource() elif single and len(entries) > 0: resource['data'] = entries[0] dump_json(resource, w_file, cls=ModelEncoder)
def __call__(self): year, month = self.yearMonth fn = b'{0}-{1}-posts-{2:4d}{3:02d}.json'.format( self.siteInfo.id, self.groupInfo.id, year, month) self.request.response.setHeader(b'Content-Disposition', b'inline; filename=' + fn) self.request.response.setHeader(b'Content-type', b'application/json') posts = self.query.posts_for_month(self.siteInfo.id, self.groupInfo.id, year, month) retval = dump_json(posts) return retval
def state(self): key = self.config.get('owntracks_key') res = dict(_type='location', tid='volvo', t='p', lat=self.vehicle.position['latitude'], lon=self.vehicle.position['longitude'], acc=1, tst=int(time())) return (dict(_type='encrypted', data=owntracks_encrypt(dump_json(res), key)) if key else res)
async def _http(self, callback, method, url, params=None, json=None, headers=None, files=None): data = None if files: data = files elif json: data = dump_json(json) data = files if files else data async with self.request(method, url, params=params, headers=headers, data=data) as response: body = await response.text(encoding='utf-8') return callback(Response(response.status, response.headers, body, method, url))
def main(): parser = argparse.ArgumentParser() parser.add_argument("--root", required=True, help="Root path of Turbulenz Engine") parser.add_argument("--assets-path", required=True, action="append", help="Path to root of source assets") parser.add_argument("--build-path", default=path_join("_build", "assets"), help="Path for intermediate build files") parser.add_argument("--install-path", default="staticmax", help="Path to install output assets into") parser.add_argument("--verbose", action="store_true") parser.add_argument("--imagemagick-convert", help="Path to ImageMagick convert executable (enables TGA support)") args = parser.parse_args(argv[1:]) assets_paths = [normpath(p) for p in args.assets_path] base_build_path = normpath(args.build_path) build_paths = [ path_join(base_build_path, "textures"), path_join(base_build_path, "models"), path_join(base_build_path, "sounds"), path_join(base_build_path, "materials"), path_join(base_build_path, "shaders"), path_join(base_build_path, "fonts"), path_join(base_build_path, "videos"), ] create_dir(base_build_path) for path in build_paths: create_dir(path) create_dir(args.install_path) tools = Tools(args, base_build_path) with open("deps.yaml", "rt") as f: asset_build_info = load_yaml(f.read()) if asset_build_info: asset_build_info = [AssetInfo(asset_info) for asset_info in asset_build_info] else: asset_build_info = [] try: with open(path_join(base_build_path, "sourcehashes.json"), "rt") as f: source_list = SourceList(load_json(f.read()), assets_paths) except IOError: if args.verbose: print "No source hash file" source_list = SourceList({}, assets_paths) try: assets_rebuilt = 0 for asset_info in asset_build_info: rebuild = build_asset(asset_info, source_list, tools, base_build_path, args.verbose) if rebuild: assets_rebuilt += 1 except CalledProcessError as e: error("Tool failed - %s" % str(e)) return 1 except IOError as e: error(str(e)) with open(path_join(base_build_path, "sourcehashes.json"), "wt") as f: f.write(dump_json(source_list.get_hashes())) print "Installing assets and building mapping table..." mapping = install(asset_build_info, args.install_path) with open("mapping_table.json", "wt") as f: f.write(dump_json({"urnmapping": mapping})) remove_old_build_files(asset_build_info, build_paths) print "%d assets rebuilt" % assets_rebuilt print "Assets build complete"
def post(self, url, json): return Response(self._client.post(url, data=dump_json(json), content_type='application/json'))
def render(self, controller, data, request, response): response.type = 'application/json' return dump_json(data)
def main(): parser = optparse.OptionParser( usage="Usage: %prog [options] course_code") parser.add_option('-v', '--verbose', dest="verbose", default=False, action="store_true", help="Print lots of output to stdout" ) parser.add_option('--canvasid', dest='canvasid', help="Canvas id for the course (or use lms api)" ) parser.add_option('--dir', dest='dumpdir', default='dump', help="Directory to read dumps from" ) parser.add_option('--nop', dest='nop', default=False, action='store_true', help="Only show canvas course for course round." ) options, args = parser.parse_args() if options.canvasid and len(args) != 1: parser.error("Exactly one course_code is required when giving canvas id") elif len(args) == 0: parser.error("At least one course_code is required") for course_code in args: course_id = options.canvasid or find_canvas_id(course_code) if not course_id: print("Canvas course id not given or found") exit(1) dumpdir = options.dumpdir if options.verbose: print("Upload to %s (canvas #%s) from %s" % ( course_code, course_id, dumpdir)) if options.nop: continue course_code = course_code[:6] with open('%s/%s/pages.json' % (dumpdir, course_code)) as json: dumpdata = parse_json(json) uploaded_files = {} for data in dumpdata: if options.verbose: print("Should upload", data) # Use the Canvas API to insert the page #POST /api/v1/courses/:course_id/pages # wiki_page[title] # wiki_page[body] # wiki_page[published] html = BeautifulSoup(open("%s/%s/pages/%s.html" % (dumpdir, course_code, data['slug'])), "html.parser") for link in html.findAll(href=True): linkdata = next(filter(lambda i: i['url'] == link['href'], data['links']), None) if linkdata and linkdata.get('category') == 'file': canvas_url = uploaded_files.get(link['href']) if not canvas_url: canvas_url = create_file(course_id, '%s/%s/pages/%s' % (dumpdir, course_code, linkdata['url']), basename(linkdata['url'])) print("Uploaded %s to %s for link" % (link['href'], canvas_url)) uploaded_files[link['href']] = canvas_url else: print("%s is allready at %s" % (link['href'], canvas_url)) link['href'] = canvas_url linkdata['url'] = canvas_url for img in html.findAll('img'): imgdata = next(filter(lambda i: i['url'] == img.get('src'), data['links']), {}) if imgdata.get('category') == 'file': canvas_url = uploaded_files.get(img['src']) if not canvas_url: canvas_url = create_file(course_id, '%s/%s/pages/%s' % (dumpdir, course_code, imgdata['url']), basename(imgdata['url'])) print("Uploaded %s to %s for img" % (img['src'], canvas_url)) uploaded_files[img['src']] = canvas_url else: print("%s is allready at %s" % (img['src'], canvas_url)) img['src'] = canvas_url imgdata['url'] = canvas_url for tex in html.findAll('span', attrs={'role': 'formula', 'data-language': 'tex'}): img = html.new_tag('img') img['src'] = '/equation_images/' + urlquote(tex.text) img['alt'] = tex.text img['class'] = tex.get('class') tex.replace_with(img) if options.verbose: print("Modified formula %s to: %s" % (tex, img)) url = baseUrl + '%s/pages' % (course_id) print("Should post page to", url) payload={ 'wiki_page[title]': data['title'], 'wiki_page[published]': False, 'wiki_page[body]': str(html) } if options.verbose: print(payload) r = requests.post(url, headers = header, data=payload) if r.status_code == requests.codes.ok: page_response=r.json() if options.verbose: print("result of post creating page: %s" % page_response) print("Uploaded page to %s" % page_response['html_url']) data['url'] = page_response['html_url'] else: print("Failed to upload page %s: %s" % (data['title'], r)) dumpname = '%s/%s/zzz-import-%s-%s.json' % ( dumpdir, course_code, course_code, datetime.now().strftime('%Y%m%d-%H%M%S')) with open(dumpname, 'w') as json: dump_json(dumpdata, json, indent=4) result = create_file(course_id, dumpname, basename(dumpname)) print('Uploaded final result to %s' % result)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--root', required=True, help="Root path of Turbulenz Engine") parser.add_argument('--assets-path', required=True, action='append', help="Path to root of source assets") parser.add_argument('--build-path', default=path_join('_build', 'assets'), help="Path for intermediate build files") parser.add_argument('--install-path', default='staticmax', help="Path to install output assets into") parser.add_argument('--verbose', action='store_true') parser.add_argument('--imagemagick-convert', help="Path to ImageMagick convert executable (enables TGA support)") args = parser.parse_args(argv[1:]) assets_paths = [ normpath(p) for p in args.assets_path ] base_build_path = normpath(args.build_path) build_paths = [ path_join(base_build_path, 'textures'), path_join(base_build_path, 'models'), path_join(base_build_path, 'sounds'), path_join(base_build_path, 'materials'), path_join(base_build_path, 'shaders'), path_join(base_build_path, 'fonts'), path_join(base_build_path, 'videos'), ] create_dir(base_build_path) for path in build_paths: create_dir(path) create_dir(args.install_path) tools = Tools(args, base_build_path) with open('deps.yaml', 'rt') as f: asset_build_info = load_yaml(f.read()) if asset_build_info: asset_build_info = [AssetInfo(asset_info) for asset_info in asset_build_info] else: asset_build_info = [] try: with open(path_join(base_build_path, 'sourcehashes.json'), 'rt') as f: source_list = SourceList(load_json(f.read()), assets_paths) except IOError: if args.verbose: print 'No source hash file' source_list = SourceList({}, assets_paths) try: assets_rebuilt = 0 for asset_info in asset_build_info: rebuild = build_asset(asset_info, source_list, tools, base_build_path, args.verbose) if rebuild: assets_rebuilt += 1 except CalledProcessError as e: error('Tool failed - %s' % str(e)) return 1 except IOError as e: error(str(e)) with open(path_join(base_build_path, 'sourcehashes.json'), 'wt') as f: f.write(dump_json(source_list.get_hashes())) print 'Installing assets and building mapping table...' mapping = install(asset_build_info, args.install_path) with open('mapping_table.json', 'wt') as f: f.write(dump_json({ 'urnmapping': mapping })) remove_old_build_files(asset_build_info, build_paths) print '%d assets rebuilt' % assets_rebuilt print 'Assets build complete'
def main(): parser = argparse.ArgumentParser() parser.add_argument('--root', required=True, help="Root path of Turbulenz Engine") parser.add_argument('--assets-path', required=True, action='append', help="Path to root of source assets") parser.add_argument('--build-path', default=path_join('_build', 'assets'), help="Path for intermediate build files") parser.add_argument('--install-path', default='staticmax', help="Path to install output assets into") parser.add_argument('--verbose', action='store_true') parser.add_argument('--imagemagick-convert', help="Path to ImageMagick convert executable (enables TGA support)") try: default_num_threads = multiprocessing.cpu_count() except NotImplementedError: default_num_threads = 1 parser.add_argument('-j', '--num-threads', help="Specify how many threads to use for building", default=default_num_threads, type=int) args = parser.parse_args(argv[1:]) assets_paths = [ normpath(p) for p in args.assets_path ] base_build_path = normpath(args.build_path) create_dir(base_build_path) create_dir(args.install_path) tools = Tools(args, base_build_path) with open('deps.yaml', 'r') as f: asset_build_info = load_yaml(f.read()) if asset_build_info: asset_build_info = [AssetInfo(asset_info) for asset_info in asset_build_info] else: asset_build_info = [] try: with open(path_join(base_build_path, 'sourcehashes.json'), 'r') as f: source_list = SourceList(load_json(f.read()), assets_paths) except IOError: if args.verbose: print 'No source hash file' source_list = SourceList({}, assets_paths) # Ensure all sources are in the source list so that the threads aren't writing to the list for a in asset_build_info: source_list.get_source(a.path) class AssetBuildThread(Thread): def __init__(self, asset_list, asset_list_mutex): Thread.__init__(self) self.asset_list = asset_list self.mutex = asset_list_mutex self.assets_rebuilt = 0 self.exit = False self.error = None def run(self): while True: if self.exit: return 0 self.mutex.acquire(True) try: # Try and pull the head off the list and if all it's dependencies are already built then # build it. This could iterate down the remaining list in case the head isn't buildable but # things later in the list are asset_info = self.asset_list[0] deps = [ source_list.get_source(path) for path in asset_info.deps if path != asset_info.path ] if any([not d.built for d in deps]): self.mutex.release() sleep(0.01) continue self.asset_list.pop(0) self.mutex.release() except IndexError: self.mutex.release() return 0 try: rebuild = build_asset(asset_info, source_list, tools, base_build_path, args.verbose) except CalledProcessError as e: self.error = '%s - Tool failed - %s' % (asset_info.path, str(e)) return 1 except IOError as e: self.error = str(e) return 1 if rebuild: self.assets_rebuilt += 1 num_threads = args.num_threads # Sort assets by dependencies assets_to_build = [] while len(assets_to_build) != len(asset_build_info): num_assets_sorted = len(assets_to_build) for asset in asset_build_info: if asset in assets_to_build: continue for dep in asset.deps: if dep != asset.path and dep not in [ a.path for a in assets_to_build ]: break else: assets_to_build.append(asset) if num_assets_sorted == len(assets_to_build): assets_left = [ a for a in asset_build_info if a not in assets_to_build ] error('Detected cyclic dependencies between assets within - \n%s' % '\n'.join([ a.path for a in assets_left ])) return 1 # Create and start threads to build the assets in the sorted dependency list asset_threads = [] asset_list_mutex = Lock() for t in xrange(num_threads): asset_threads.append(AssetBuildThread(assets_to_build, asset_list_mutex)) for t in xrange(num_threads): asset_threads[t].start() while any(a.isAlive() for a in asset_threads): for t in xrange(num_threads): asset_threads[t].join(0.1) if not asset_threads[t].isAlive() and asset_threads[t].error: # One thread has an error ask all the others to finish asap for o in xrange(num_threads): asset_threads[o].exit = True # Update the stats on number of assets rebuilt assets_rebuilt = 0 for t in xrange(num_threads): assets_rebuilt += asset_threads[t].assets_rebuilt # Dump the state of the build for partial rebuilds with open(path_join(base_build_path, 'sourcehashes.json'), 'w') as f: f.write(dump_json(source_list.get_hashes())) # Check if any build threads failed and if so exit with an error for t in xrange(num_threads): if asset_threads[t].error: error(asset_threads[t].error) return 1 # Dump the mapping table for the built assets print 'Installing assets and building mapping table...' mapping = install(asset_build_info, args.install_path) with open('mapping_table.json', 'w') as f: f.write(dump_json({ 'urnmapping': mapping })) # Cleanup any built files no longer referenced by the new mapping table remove_old_build_files(asset_build_info, base_build_path) print '%d assets rebuilt' % assets_rebuilt print 'Assets build complete'
def get(self, q): q = urllib2.unquote(q) self.write(dump_json(list(P.q_search.search(query=q))))
def get(self, q): q = urllib2.unquote(q) self.write(dump_json(list(P.a_search.search(answer=q))))
def get(self, q): q = urllib2.unquote(q) queries = list(P.q_search.search(query=q)) answers = list(P.a_search.search(answer=q)) self.write(dump_json(queries + answers))