def _FindPsOutputFormat(cls, cmd, args): """Return our best guess the formating of the "ps" output.""" output_format = [] for arg in args: # If the "ps" arg contains a comma, it's probably an output format defn. if "," in arg: output_format.extend(arg.split(",")) if not output_format: # Assume a default format for the "-f" style formating. output_format = ["user", "pid", "ppid", "pcpu", "not_implemented", "tty", "not_implemented", "cmd"] # Do some sanity checking for the cmd/cmdline if present. for option in ["cmd", "command", "args"]: if option in output_format: if output_format.count(option) > 1: logging.warn("Multiple commandline outputs expected in '%s %s' " "output. Skipping parsing.", cmd, " ".join(args)) return [] if output_format[-1] != option: logging.warn("'ps's output has the commandline not as the last " "column. We can't safely parse output of '%s %s'." "Skipping parsing.", cmd, " ".join(args)) return [] # If we made it here, we should be able to parse the output and we have a # good idea of it's format. return output_format
def read(self, request): logging.info("__INIT__read[VappVm]") res = vapp_vm_pb2.ReadVappVmResult() res.present = False org_resource = self.client.get_org() org = Org(self.client, resource=org_resource) try: vdc_resource = org.get_vdc(request.target_vdc) vdc = VDC( self.client, name=request.target_vdc, resource=vdc_resource) vapp_resource = vdc.get_vapp(request.target_vapp) vapp = VApp( self.client, name=request.target_vapp, resource=vapp_resource) read_vapp_vm_resp = vapp.get_vm(request.target_vm_name) vm = VM(client=self.client, href=None, resource=read_vapp_vm_resp) res.present = True except Exception as e: errmsg = '__ERROR_read[VappVm] failed for VappVm {0}. __ErrorMessage__ {1}' logging.warn(errmsg.format(request.target_vm_name, str(e))) return res logging.info("__DONE__read[VappVm]") return res
def post_config_change(self, method): route = CsRoute() if method == "add": route.add_table(self.dev) route.add_route(self.dev, str(self.address["network"])) elif method == "delete": logging.warn("delete route not implemented") self.fw_router() self.fw_vpcrouter() # On deletion nw_type will no longer be known if self.get_type() in ["guest"] and self.config.is_vpc(): CsDevice(self.dev, self.config).configure_rp() logging.error( "Not able to setup source-nat for a regular router yet") dns = CsDnsmasq(self) dns.add_firewall_rules() app = CsApache(self) app.setup() cmdline = self.config.cmdline() # If redundant then this is dealt with by the master backup functions if self.get_type() in ["guest"] and not cmdline.is_redundant(): pwdsvc = CsPasswdSvc(self.address['public_ip']).start() if self.get_type() == "public" and self.config.is_vpc(): if self.address["source_nat"]: vpccidr = cmdline.get_vpccidr() self.fw.append( ["filter", "", "-A FORWARD -s %s ! -d %s -j ACCEPT" % (vpccidr, vpccidr)]) self.fw.append( ["nat", "", "-A POSTROUTING -j SNAT -o %s --to-source %s" % (self.dev, self.address['public_ip'])])
def _ota_chunk_data(self, data, data_cb): # spec https://shadowsocks.org/en/spec/one-time-auth.html unchunk_data = b'' while len(data) > 0: if self._ota_len == 0: # get DATA.LEN + HMAC-SHA1 length = ONETIMEAUTH_CHUNK_BYTES - len(self._ota_buff_head) self._ota_buff_head += data[:length] data = data[length:] if len(self._ota_buff_head) < ONETIMEAUTH_CHUNK_BYTES: # wait more data return data_len = self._ota_buff_head[:ONETIMEAUTH_CHUNK_DATA_LEN] self._ota_len = struct.unpack('>H', data_len)[0] length = min(self._ota_len, len(data)) self._ota_buff_data += data[:length] data = data[length:] if len(self._ota_buff_data) == self._ota_len: # get a chunk data _hash = self._ota_buff_head[ONETIMEAUTH_CHUNK_DATA_LEN:] _data = self._ota_buff_data index = struct.pack('>I', self._ota_chunk_idx) key = self._encryptor.decipher_iv + index if onetimeauth_verify(_hash, _data, key) is False: logging.warn('one time auth fail, drop chunk !') else: unchunk_data += _data self._ota_chunk_idx += 1 self._ota_buff_head = b'' self._ota_buff_data = b'' self._ota_len = 0 data_cb(unchunk_data) return
def create_from_vapp(self, request): logging.info("__INIT__create[VappVm] source_catalog_name[%s]", request.source_vapp) res = vapp_vm_pb2.CreateVappVmResult() res.created = False source_vapp_resource = self.get_vapp_resource( request.target_vdc, vapp_name=request.source_vapp) target_vapp_resource = self.get_vapp_resource( request.target_vdc, vapp_name=request.target_vapp) specs = [{ 'vapp': source_vapp_resource, 'source_vm_name': request.source_vm_name, 'target_vm_name': request.target_vm_name, 'hostname': request.hostname, 'password': request.password, 'password_auto': request.password_auto, 'password_reset': request.password_reset, 'cust_script': request.cust_script, 'network': request.network, # 'storage_profile': request.storage_profile }] try: vapp = VApp(self.client, resource=target_vapp_resource) create_vapp_vm_resp = vapp.add_vms( specs, power_on=request.power_on, all_eulas_accepted=request.all_eulas_accepted) task_monitor = self.client.get_task_monitor() task = task_monitor.wait_for_status( task=create_vapp_vm_resp, timeout=60, poll_frequency=2, fail_on_statuses=None, expected_target_statuses=[ TaskStatus.SUCCESS, TaskStatus.ABORTED, TaskStatus.ERROR, TaskStatus.CANCELED ], callback=None) st = task.get('status') if st != TaskStatus.SUCCESS.value: raise errors.VappVmCreateError( etree.tostring(task, pretty_print=True)) message = 'status : {0} '.format(st) logging.info(message) res.created = True except Exception as e: errmsg = '''__ERROR_create[VappVm] failed for vm {0}. __ErrorMessage__ {1}''' logging.warn(errmsg.format(request.target_vm_name, str(e))) self.context.set_code(grpc.StatusCode.INVALID_ARGUMENT) self.context.set_details(errmsg) return res logging.info("__DONE__create[VappVm]") return res
def __init__(self, **names_vals): for name, val in names_vals.iteritems(): if name.startswith(('_', 'is_valid_')): raise NameError('The parameter name %s is not acceptable' % name) try: convert = self.__class__.params[name] except KeyError: logging.warn('The parameter %r is unknown, ignoring' % name) continue try: value = convert(val) except: raise ValueError('Could not convert to %s: %s=%s' % (convert.__name__, name, val)) setattr(self, name, value) valids = sorted(getattr(self, valid) for valid in dir(self.__class__) if valid.startswith('is_valid_')) for is_valid in valids: if not is_valid(): dump = '\n'.join('%s=%s' % (n, v) for n, v in sorted(self.__dict__.items())) raise ValueError(is_valid.__doc__ + 'Got:\n' + dump)
def gather(suffix, options): url = options.get("url") if url is None: logging.warn("A --url is required. (Can be a local path.)") exit(1) # remote URL if url.startswith("http:") or url.startswith("https:"): # Though it's saved in cache/, it will be downloaded every time. remote_path = os.path.join(utils.cache_dir(), "url.csv") try: response = requests.get(url) utils.write(response.text, remote_path) except: logging.error("Remote URL not downloaded successfully.") print(utils.format_last_exception()) exit(1) # local path else: remote_path = url for domain in utils.load_domains(remote_path): yield domain
def _stage_final_image(self): try: makedirs(self.__ensure_isodir() + "/LiveOS") self._resparse() if not self.skip_minimize: create_image_minimizer(self.__isodir + "/LiveOS/osmin.img", self._image, self.compress_type) if self.skip_compression: shutil.move(self._image, self.__isodir + "/LiveOS/ext3fs.img") if os.stat(self.__isodir + "/LiveOS/ext3fs.img").st_size >= 4*1024*1024*1024: self._isofstype = "udf" logging.warn("Switching to UDF due to size of LiveOS/ext3fs.img") else: makedirs(os.path.join(os.path.dirname(self._image), "LiveOS")) shutil.move(self._image, os.path.join(os.path.dirname(self._image), "LiveOS", "ext3fs.img")) mksquashfs(os.path.dirname(self._image), self.__isodir + "/LiveOS/squashfs.img", self.compress_type) if os.stat(self.__isodir + "/LiveOS/squashfs.img").st_size >= 4*1024*1024*1024: self._isofstype = "udf" logging.warn("Switching to UDF due to size of LiveOS/squashfs.img") self.__create_iso(self.__isodir) finally: shutil.rmtree(self.__isodir, ignore_errors = True) self.__isodir = None
def _launchSlaves(self): """ Launch a group of worker processes (self._slaves), the queue (self._workQueue) that will be used to send them chunks of work, and the queue that will be used to receive back the results (self._resultsQueue). Additionally, launch the result collector process. """ availableCpus = multiprocessing.cpu_count() logging.info("Available CPUs: %d" % (availableCpus,)) logging.info("Requested workers: %d" % (options.numWorkers,)) logging.info("Parallel Mode: %s" % ("Threaded" if options.threaded else "Process",)) if (options.numWorkers > availableCpus): logging.warn("More workers requested (%d) than CPUs available (%d);" " may result in suboptimal performance." % (options.numWorkers, availableCpus)) self._initQueues() WorkerType, ResultCollectorType = self._algorithm.slaveFactories(options.threaded) self._slaves = [] for i in xrange(options.numWorkers): p = WorkerType(self._workQueue, self._resultsQueue, self._algorithmConfiguration) self._slaves.append(p) p.start() logging.info("Launched compute slaves.") rcp = ResultCollectorType(self._resultsQueue, self._algorithmConfiguration) rcp.start() self._slaves.append(rcp) logging.info("Launched collector slave.")
def search(self, search_query): """ Search in the store for tiddlers that match search_query. This is intentionally simple, slow and broken to encourage overriding. """ bag_filenames = self._bag_filenames() query = search_query.lower() for bagname in bag_filenames: bagname = urllib.unquote(bagname).decode('utf-8') tiddler_dir = self._tiddlers_dir(bagname) tiddler_files = self._files_in_dir(tiddler_dir) for tiddler_name in tiddler_files: tiddler = Tiddler( title=urllib.unquote(tiddler_name).decode('utf-8'), bag=bagname) try: revision_id = self.list_tiddler_revisions(tiddler)[0] if query in tiddler.title.lower(): yield tiddler continue tiddler_file = codecs.open( self._tiddler_full_filename(tiddler, revision_id), encoding='utf-8') for line in tiddler_file: if query in line.lower(): yield tiddler break except (OSError, NoTiddlerError), exc: logging.warn('malformed tiddler during search: %s:%s', bagname, tiddler_name)
def dataReadsBlocks2(self, addrThreshold = 16): lastAddr = 0 lastInstruction = None blocks = [] currentBlock = [] for i in self: datas = [] for data in i.data: if data.mode == data.modeRead: datas.append(data) if len(datas) == 1: data = datas[0] if abs(lastAddr-data.addr) <= addrThreshold: if len(currentBlock) == 0 and lastInstruction != None: currentBlock.append(lastInstruction) currentBlock.append(i) elif len(currentBlock) == 0: blocks.append([lastInstruction]) else: blocks.append(currentBlock) currentBlock = [] lastAddr = data.addr lastInstruction = i if len(datas) == 2: pass # todo if len(datas) > 2: logging.warn('ouch len(datas) > 2') return blocks
def handle(self, input, addr): """ handle an incoming udp packet. """ if cfg['udpseed']: data = "" for i in range(len(input)/16): try: data += crypt.decrypt(input[i*16:i*16+16]) except Exception as ex: logging.warn("udp - can't decrypt: %s" % str(ex)) data = input break else: data = input if cfg['udpstrip']: data = strippedtxt(data) # check if udp is enabled and source ip is in udpallow list if cfg['udp'] and (addr[0] in cfg['udpallow'] or _inmask(addr[0])): # get printto and passwd data header = re.search('(\S+) (\S+) (.*)', data) if header: # check password if header.group(1) == cfg['udppassword']: printto = header.group(2) # is the nick/channel # check if printto is in allowednicks if cfg['udpalloednicks'] and not printto in cfg['udpallowednicks']: logging.warn("udp - udp denied %s" % printto ) return logging.debug('udp - ' + str(addr[0]) + " - udp allowed") text = header.group(3) # is the text self.say(printto, text) else: logging.warn("udp - can't match udppasswd from " + str(addr)) else: logging.warn("udp - can't match udp from " + str(addr[0])) else: logging.warn('udp - denied udp from ' + str(addr[0]))
def OpenClient(client_id=None): """Opens the client, getting potential approval tokens. Args: client_id: The client id the approval should be revoked for. Returns: tuple containing (client, token) objects or (None, None) on if no appropriate aproval tokens were found. """ token = access_control.ACLToken(username="******") try: token = ApprovalFind(client_id, token=token) except access_control.UnauthorizedAccess as e: logging.warn("No authorization found for access to client: %s", e) try: # Try and open with the token we managed to retrieve or the default. client = aff4.FACTORY.Open(rdfvalue.RDFURN(client_id), mode="r", token=token) return client, token except access_control.UnauthorizedAccess: logging.warning("Unable to find a valid reason for client %s. You may need " "to request approval.", client_id) return None, None
def get(self, section, key, **kwargs): section = str(section).lower() key = str(key).lower() d = self.defaults # first check environment variables option = self._get_env_var_option(section, key) if option: return option # ...then the config file if self.has_option(section, key): return expand_env_var( ConfigParser.get(self, section, key, **kwargs)) # ...then commands option = self._get_cmd_option(section, key) if option: return option # ...then the defaults if section in d and key in d[section]: return expand_env_var(d[section][key]) else: logging.warn("section/key [{section}/{key}] not found " "in config".format(**locals())) raise AirflowConfigException( "section/key [{section}/{key}] not found " "in config".format(**locals()))
def _listen(self): """ listen for udp messages .. /msg via bot""" if not cfg['udp']: return fleet = getfleet() for botname in cfg['udpbots']: if not fleet.byname(botname): logging.info("udp - can't find %s bot" % botname) try: fleet.startok.wait(5) logging.warn('udp listening on %s %s' % (cfg['udphost'], cfg['udpport'])) self.sock.bind((cfg['udphost'], cfg['udpport'])) self.stop = 0 except IOError: handle_exception() self.sock = None self.stop = 1 return # loop on listening udp socket while not self.stop: try: input, addr = self.sock.recvfrom(64000) except socket.timeout: continue except Exception as ex: try: (errno, errstr) = ex except ValueError: errno = 0 ; errstr = str(ex) if errno == 4: logging.warn("udp - %s - %s" % (self.name, str(ex))) ; break if errno == 35: continue else: handle_exception() ; break if self.stop: break self.queue.put((input, addr)) logging.info('udp - shutting down main loop')
def refresh(): 'Load or refresh the library.' objs={} for root,dirs,files in os.walk(libDir): for f in files: ext=os.path.splitext(f)[-1].lower() ff=os.path.join(root,f) assert ff.startswith(libDir+'/') libPath=ff[len(libDir)+1:] if ext=='xls': raise NotImplementedError('Reading library objects from XLS not yet implemented.') # load multiple objects from the XLS file for key,val in loadFromXLS(ff): objs[tuple((libPath+'/'+key).split('/'))]=val else: try: obj=woo._monkey.io.Object_load(None,ff) objs[tuple(libPath.split('/'))]=obj except: logging.warn('Loading library object from %s failed (skipped):\n\n'%ff) import traceback traceback.print_exc() print logging.warn('--------------------------------------------------------') global libObjs libObjs=objs
def runCmd(cmd): logging.debug("Running %s", cmd) rv=os.system(cmd) if rv != 0: ce=CmdError(cmd, rv) logging.warn("Command failed: %s", ce) raise ce
def inhibit(self, *a): """inhibit a gluster filesystem Mount glusterfs over a temporary mountpoint, change into the mount, and lazy unmount the filesystem. """ mounted = False try: po = Popen(self.make_mount_argv(*a), **self.mountkw) self.handle_mounter(po) po.terminate_geterr() d = self.mntpt mounted = True logging.debug("auxiliary glusterfs mount in place") os.chdir(d) self.umount_l(d).terminate_geterr() mounted = False finally: try: if mounted: self.umount_l(d).terminate_geterr(fail_on_err=False) self.cleanup_mntpt() except: logging.warn("stale mount possibly left behind on " + d) logging.debug("auxiliary glusterfs mount prepared")
def measure_timeslice_used(tree, device, timevals): """Measures the actual timeslice that was charged to the group. This is done because we dont charge the first seek to the group and so the service_time and actual charged time can diverge outside the allowed margin of error for some pathological workloads, masking any fair scheduling errors. """ for container in tree: found_data = False for line in container['blkio_cgroup'].get_attr('timeslice_used'): parts = line.split() if parts[0] == device: timevals[container['name']] = int(parts[-1]) found_data = True if not found_data: timevals[container['name']] = 0 logging.warn('No data for container %s.' % container['name']) for line in container['blkio_cgroup'].get_attr('unaccounted_time'): parts = line.split() if parts[0] == device: timevals[container['name']] -= int(parts[-1]) # Recurse to nested containers. measure_timeslice_used(container['nest'], device, timevals)
def update(self): args = list(map(self.getValue, self.__fields__)) args.append(self.getValue(self.__primary_key__)) rows = yield from execute(self.__update__, args) if rows != 1: logging.warn('failed to update by primary key: affected rows: %s' % rows)
def get_compressor(ctools=COMPRESSING_TOOLS): global UPTO am_dir_pattern = "/usr/share/automake-*" am_files_pattern = am_dir_pattern + "/am/*.am" if len(glob.glob(am_dir_pattern)) == 0: # automake is not installed. UPTO = STEP_PRECONFIGURE logging.warn("Automake not found. The process will go up to the step: " + UPTO) return ctools[-1] # fallback to the default (gzip). for ct in ctools: # NOTE: bzip2 tries compressing input from stdin even it is invoked # with --version option. So give null input (/dev/null) to it. c_exists = 0 == subprocess.check_call( "%(command)s --version > /dev/null 2> /dev/null < /dev/null" % ct, shell=True ) if c_exists: am_support_c = 0 == subprocess.check_call( "grep -q -e '^dist-%s:' %s 2> /dev/null" % (ct.command, am_files_pattern), shell=True ) if am_support_c: return ct # gzip must exist at least and not reached here: raise RuntimeError("No compressor found! Aborting...")
def _request(sock, headers, payload, bufsize=8192): request_data = 'POST /_gh/ HTTP/1.1\r\n' request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items() if k not in skip_headers) request_data += '\r\n' if isinstance(payload, bytes): sock.send(request_data.encode() + payload) elif hasattr(payload, 'read'): sock.send(request_data) while True: data = payload.read(bufsize) if not data: break sock.send(data) else: raise TypeError('_request(payload) must be a string or buffer, not %r' % type(payload)) response = httplib.HTTPResponse(sock, buffering=True) try: orig_timeout = sock.gettimeout() sock.settimeout(90) response.begin() sock.settimeout(orig_timeout) except httplib.BadStatusLine as e: logging.warn("_request bad status line:%r", e) response = None except Exception as e: logging.exception("_request:%r", e) return response
def setup(self): exp = self.exp data_obj = exp.create_data_object('in_data', 'tomo') data_obj.backing_file = \ h5py.File(self.exp.meta_data.get_meta_data("data_file"), 'r') data_obj.data = data_obj.backing_file[self.parameters['data_path']] self.__set_dark_and_flat(data_obj) if self.parameters['3d_to_4d']: if not self.parameters['angles']: raise Exception('Angles are required in the loader.') self.__setup_4d(data_obj) n_angles = self.__set_rotation_angles(data_obj) shape = self.__setup_3d_to_4d(data_obj, n_angles) else: if len(data_obj.data.shape) is 3: shape = self.__setup_3d(data_obj) else: shape = self.__setup_4d(data_obj) self.__set_rotation_angles(data_obj) try: control = data_obj.backing_file['entry1/tomo_entry/control/data'] data_obj.meta_data.set_meta_data("control", control[...]) except: logging.warn("No Control information available") self.__check_angles data_obj.set_original_shape(shape) self.set_data_reduction_params(data_obj)
def regjob(e, xte, pb): if pb.wait(): logging.debug("synced " + e) self.sendmark_regular(e, xte) return True else: logging.warn("failed to sync " + e)
def import_key(stream): """Run C{gpg --import} with this stream as stdin.""" errors = tempfile.TemporaryFile() child = _run_gpg(['--quiet', '--import', '--batch'], stdin = stream, stderr = errors) status = child.wait() errors.seek(0) error_messages = errors.read().strip() errors.close() if error_messages: import codecs decoder = codecs.lookup('utf-8') error_messages = decoder.decode(error_messages, errors = 'replace')[0] if status != 0: if error_messages: raise SafeException(_("Errors from 'gpg --import':\n%s") % error_messages) else: raise SafeException(_("Non-zero exit code %d from 'gpg --import'") % status) elif error_messages: warn(_("Warnings from 'gpg --import':\n%s") % error_messages)
def _sweep_timeout(self): # tornado's timeout memory management is more flexible than we need # we just need a sorted last_activity queue and it's faster than heapq # in fact we can do O(1) insertion/remove so we invent our own if self._timeouts: logging.log(shell.VERBOSE_LEVEL, 'sweeping timeouts') now = time.time() length = len(self._timeouts) pos = self._timeout_offset while pos < length: handler = self._timeouts[pos] if handler: if now - handler.last_activity < self._timeout: break else: if handler.remote_address: logging.warn('timed out: %s:%d' % handler.remote_address) else: logging.warn('timed out') handler.destroy() self._timeouts[pos] = None # free memory pos += 1 else: pos += 1 if pos > TIMEOUTS_CLEAN_SIZE and pos > length >> 1: # clean up the timeout queue when it gets larger than half # of the queue self._timeouts = self._timeouts[pos:] for key in self._handler_to_timeouts: self._handler_to_timeouts[key] -= pos pos = 0 self._timeout_offset = pos
def handle_event(self, sock, fd, event): # handle events and dispatch to handlers if sock: logging.log(shell.VERBOSE_LEVEL, 'fd %d %s', fd, eventloop.EVENT_NAMES.get(event, event)) if sock == self._server_socket: if event & eventloop.POLL_ERR: # TODO raise Exception('server_socket error') try: logging.debug('accept') conn = self._server_socket.accept() TCPRelayHandler(self, self._fd_to_handlers, self._eventloop, conn[0], self._config, self._dns_resolver, self._is_local) except (OSError, IOError) as e: error_no = eventloop.errno_from_exception(e) if error_no in (errno.EAGAIN, errno.EINPROGRESS, errno.EWOULDBLOCK): return else: shell.print_exception(e) if self._config['verbose']: traceback.print_exc() else: if sock: handler = self._fd_to_handlers.get(fd, None) if handler: handler.handle_event(sock, event) else: logging.warn('poll removed fd')
def request(headers={}, payload=None): max_retry = 3 for i in range(max_retry): ssl_sock = None try: ssl_sock = https_manager.create_ssl_connection() if not ssl_sock: logging.debug('create_ssl_connection fail') continue if ssl_sock.host == '': ssl_sock.appid = appid_manager.get_appid() if not ssl_sock.appid: raise GAE_Exception(1, "no appid can use") headers['Host'] = ssl_sock.appid + ".appspot.com" ssl_sock.host = headers['Host'] else: headers['Host'] = ssl_sock.host response = _request(ssl_sock, headers, payload) if not response: continue response.ssl_sock = ssl_sock return response except Exception as e: logging.warn('request failed:%s', e) if ssl_sock: ssl_sock.close() raise GAE_Exception(2, "try max times")
def handle_event(self, sock, event): # handle all events in this handler and dispatch them to methods if self._stage == STAGE_DESTROYED: logging.debug('ignore handle_event: destroyed') return # order is important if sock == self._remote_sock: if event & eventloop.POLL_ERR: self._on_remote_error() if self._stage == STAGE_DESTROYED: return if event & (eventloop.POLL_IN | eventloop.POLL_HUP): self._on_remote_read() if self._stage == STAGE_DESTROYED: return if event & eventloop.POLL_OUT: self._on_remote_write() elif sock == self._local_sock: if event & eventloop.POLL_ERR: self._on_local_error() if self._stage == STAGE_DESTROYED: return if event & (eventloop.POLL_IN | eventloop.POLL_HUP): self._on_local_read() if self._stage == STAGE_DESTROYED: return if event & eventloop.POLL_OUT: self._on_local_write() else: logging.warn('unknown socket')
def save(self): args = list(map(self.getValueOrDefault, self.__fields__)) args.append(self.getValueOrDefault(self.__primary_key__)) rows = yield from execute(self.__insert__, args) if rows != 1: logging.warn('failed to insert record: affected rows: %s' % rows)
async def __call__(self, request): kw = None # 定义kw,用于保存request中参数 if self._has_named_kw_arg or self._has_var_kw_arg: # 若视图函数有命名关键词或关键词参数 if request.method == 'POST': # 根据request参数中的content_type使用不同解析方法: if request.content_type == None: # 如果content_type不存在,返回400错误 return web.HTTPBadRequest(text='Missing Content_Type.') ct = request.content_type.lower() # 小写,便于检查 if ct.startwith('application/json'): # json格式数据 params = await request.json() # 仅解析body字段的json数据 if not isinstance(params, dict): # request.json()返回dict对象 return web.HTTPBadRequest(text='JSON body must be object.') kw = params # form表单请求的编码形式 elif ct.startwith('application/x-www-form-urlencoded') or ct.startswith('multipart/form-data'): params = await request.post() # 返回post的内容中解析后的数据。dict-like对象。 kw = dict(**params) # 组成dict,统一kw格式 else: return web.HTTPBadRequest(text='Unsupported Content-Type: %s' % request.content_type) if request.method == 'GET': qs = request.query_string # 返回URL查询语句,?后的键值。string形式。 if qs: kw = dict() ''''' 解析url中?后面的键值对的内容 qs = 'first=f,s&second=s' parse.parse_qs(qs, True).items() >>> dict([('first', ['f,s']), ('second', ['s'])]) ''' for k, v in parse.parse_qs(qs, True).items(): # 返回查询变量和值的映射,dict对象。True表示不忽略空格。 kw[k] = v[0] if kw is None: # 若request中无参数 # request.match_info返回dict对象。可变路由中的可变字段{variable}为参数名,传入request请求的path为值 # 若存在可变路由:/a/{name}/c,可匹配path为:/a/jack/c的request # 则reqwuest.match_info返回{name = jack} kw = dict(**request.match_info) else: # request有参数 if self._has_named_kw_arg and (not self._has_var_kw_arg): # 若视图函数只有命名关键词参数没有关键词参数 copy = dict() # 只保留命名关键词参数 for name in self._named_kw_arg: if name in kw: copy[name] = kw[name] kw = copy # kw中只存在命名关键词参数 # 将request.match_info中的参数传入kw for k, v in request.match_info.items(): # 检查kw中的参数是否和match_info中的重复 if k in kw: logging.warn('Duplicate arg name in named arg and kw args: %s' % k) kw[k] = v if self._has_request_arg: # 视图函数存在request参数 kw['request'] = request if self._required_kw_args: # 视图函数存在无默认值的命名关键词参数 for name in self._required_kw_args: if not name in kw: # 若未传入必须参数值,报错。 return web.HTTPBadRequest('Missing argument: %s' % name) # 至此,kw为视图函数fn真正能调用的参数 # request请求中的参数,终于传递给了视图函数 logging.info('call with args: %s' % str(kw)) #try: r = await self._func(**kw) return r
def main(**kwargs): """ Entry point for dx-build-app(let). Don't call this function as a subroutine in your program! It is liable to sys.exit your program when it detects certain error conditions, so you can't recover from those as you could if it raised exceptions. Instead, call dx_build_app.build_and_upload_locally which provides the real implementation for dx-build-app(let) but is easier to use in your program. """ if len(sys.argv) > 0: if sys.argv[0].endswith('dx-build-app'): logging.warn('Warning: dx-build-app has been replaced with "dx build --create-app". Please update your scripts.') elif sys.argv[0].endswith('dx-build-applet'): logging.warn('Warning: dx-build-applet has been replaced with "dx build". Please update your scripts.') if len(kwargs) == 0: args = parser.parse_args() else: args = parser.parse_args(**kwargs) if dxpy.AUTH_HELPER is None and not args.dry_run: parser.error('Authentication required to build an executable on the platform; please run "dx login" first') if args.src_dir is None: args.src_dir = os.getcwd() if USING_PYTHON2: args.src_dir = args.src_dir.decode(sys.getfilesystemencoding()) if args.mode == "app" and args.destination != '.': parser.error("--destination cannot be used when creating an app (only an applet)") if args.dx_toolkit_autodep in ['beta', 'unstable']: logging.warn('The --dx-toolkit-beta-autodep and --dx-toolkit-unstable-autodep flags have no effect and will be removed at some date in the future.') if args.overwrite and args.archive: parser.error("Options -f/--overwrite and -a/--archive cannot be specified together") if args.run is not None and args.dry_run: parser.error("Options --dry-run and --run cannot be specified together") if args.run and args.remote and args.mode == 'app': parser.error("Options --remote, --app, and --run cannot all be specified together. Try removing --run and then separately invoking dx run.") executable_id = _build_app(args, json.loads(args.extra_args) if args.extra_args else {}) if args.run is not None: if executable_id is None: raise AssertionError('Expected executable_id to be set here') try: subprocess.check_call(['dx', 'run', executable_id, '--priority', 'high'] + args.run) except subprocess.CalledProcessError as e: sys.exit(e.returncode) except: err_exit() return
def _verify_app_source_dir_impl(src_dir, temp_dir, mode, enforce=True): """Performs syntax and lint checks on the app source. Precondition: the dxapp.json file exists and can be parsed. """ _lint(os.path.join(src_dir, "dxapp.json"), mode) # Check that the entry point file parses as the type it is going to # be interpreted as. The extension is irrelevant. manifest = json.load(open(os.path.join(src_dir, "dxapp.json"))) if "runSpec" in manifest: if "interpreter" not in manifest['runSpec']: raise dxpy.app_builder.AppBuilderException('runSpec.interpreter field was not present') if manifest['runSpec']['interpreter'] in ["python2.7", "bash"]: if "file" in manifest['runSpec']: entry_point_file = os.path.abspath(os.path.join(src_dir, manifest['runSpec']['file'])) try: _check_file_syntax(entry_point_file, temp_dir, override_lang=manifest['runSpec']['interpreter'], enforce=enforce) except IOError as e: raise dxpy.app_builder.AppBuilderException( 'Could not open runSpec.file=%r. The problem was: %s' % (entry_point_file, e)) except DXSyntaxError: raise dxpy.app_builder.AppBuilderException('Entry point file %s has syntax errors, see above for details. Rerun with --no-check-syntax to proceed anyway.' % (entry_point_file,)) elif "code" in manifest['runSpec']: try: _check_syntax(manifest['runSpec']['code'], manifest['runSpec']['interpreter'], temp_dir, enforce=enforce) except DXSyntaxError: raise dxpy.app_builder.AppBuilderException('Code in runSpec.code has syntax errors, see above for details. Rerun with --no-check-syntax to proceed anyway.') if 'execDepends' in manifest['runSpec']: if not isinstance(manifest['runSpec']['execDepends'], list): raise dxpy.app_builder.AppBuilderException('Expected runSpec.execDepends to be an array. Rerun with --no-check-syntax to proceed anyway.') if not all(isinstance(dep, dict) for dep in manifest['runSpec']['execDepends']): raise dxpy.app_builder.AppBuilderException('Expected runSpec.execDepends to be an array of hashes. Rerun with --no-check-syntax to proceed anyway.') if any(dep.get('package_manager', 'apt') != 'apt' for dep in manifest['runSpec']['execDepends']): if not isinstance(manifest.get('access'), dict) or 'network' not in manifest['access']: msg = '\n'.join(['runSpec.execDepends specifies non-APT dependencies, but no network access spec is given.', 'Add {"access": {"network": ["*"]}} to allow dependencies to install.', 'See https://wiki.dnanexus.com/Developer-Tutorials/Request-Additional-App-Resources#Network-Access.', 'Rerun with --no-check-syntax to proceed anyway.']) raise dxpy.app_builder.AppBuilderException(msg) if 'authorizedUsers' in manifest: if not isinstance(manifest['authorizedUsers'], list) or isinstance(manifest['authorizedUsers'], basestring): raise dxpy.app_builder.AppBuilderException('Expected authorizedUsers to be a list of strings') for thing in manifest['authorizedUsers']: if thing != 'PUBLIC' and (not isinstance(thing, basestring) or not re.match("^(org-|user-)", thing)): raise dxpy.app_builder.AppBuilderException('authorizedUsers field contains an entry which is not either the string "PUBLIC" or a user or org ID') # Check all other files that are going to be in the resources tree. # For these we detect the language based on the filename extension. # Obviously this check can have false positives, since the app can # execute (or not execute!) all these files in whatever way it # wishes, e.g. it could use Python != 2.7 or some non-bash shell. # Consequently errors here are non-fatal. files_with_problems = [] for dirpath, dirnames, filenames in os.walk(os.path.abspath(os.path.join(src_dir, "resources"))): for filename in filenames: # On Mac OS, the resource fork for "FILE.EXT" gets tarred up # as a file named "._FILE.EXT". To a naive check this # appears to be a file of the same extension. Therefore, we # exclude these from syntax checking since they are likely # to not parse as whatever language they appear to be. if not filename.startswith("._"): try: _check_file_syntax(os.path.join(dirpath, filename), temp_dir, enforce=True) except IOError as e: raise dxpy.app_builder.AppBuilderException( 'Could not open file in resources directory %r. The problem was: %s' % (os.path.join(dirpath, filename), e) ) except DXSyntaxError: # Suppresses errors from _check_file_syntax so we # only print a nice error message files_with_problems.append(os.path.join(dirpath, filename)) if files_with_problems: # Make a message of the form: # "/path/to/my/app.py" # OR "/path/to/my/app.py and 3 other files" files_str = files_with_problems[0] if len(files_with_problems) == 1 else (files_with_problems[0] + " and " + str(len(files_with_problems) - 1) + " other file" + ("s" if len(files_with_problems) > 2 else "")) logging.warn('%s contained syntax errors, see above for details' % (files_str,))
def gen_audio_features(item, config): """Generate audio features and transformations Args: item (Dict): dictionary containing the attributes to encode. config (Dict): configuration dictionary. Returns: (bool): keep this sample or not. mel (ndarray): mel matrix in np.float32. energy (ndarray): energy audio profile. f0 (ndarray): fundamental frequency. item (Dict): dictionary containing the updated attributes. """ # get info from sample. audio = item["audio"] utt_id = item["utt_id"] rate = item["rate"] # check audio properties assert len(audio.shape) == 1, f"{utt_id} seems to be multi-channel signal." assert np.abs( audio).max() <= 1.0, f"{utt_id} is different from 16 bit PCM." # check sample rate if rate != config["sampling_rate"]: audio = librosa.resample(audio, rate, config["sampling_rate"]) logging.info( f"{utt_id} sampling rate is {rate}, not {config['sampling_rate']}, we resample it." ) # trim silence if config["trim_silence"]: if "trim_mfa" in config and config["trim_mfa"]: _, item["text_ids"], audio = ph_based_trim( config, utt_id, item["text_ids"], item["raw_text"], audio, config["hop_size"], ) if ( audio.__len__() < 1 ): # very short files can get trimmed fully if mfa didnt extract any tokens LibriTTS maybe take only longer files? logging.warning( f"File have only silence or MFA didnt extract any token {utt_id}" ) return False, None, None, None, item else: audio, _ = librosa.effects.trim( audio, top_db=config["trim_threshold_in_db"], frame_length=config["trim_frame_size"], hop_length=config["trim_hop_size"], ) # resample audio if necessary if "sampling_rate_for_feats" in config: audio = librosa.resample(audio, rate, config["sampling_rate_for_feats"]) sampling_rate = config["sampling_rate_for_feats"] assert ( config["hop_size"] * config["sampling_rate_for_feats"] % rate == 0 ), "'hop_size' must be 'int' value. Please check if 'sampling_rate_for_feats' is correct." hop_size = config["hop_size"] * config[ "sampling_rate_for_feats"] // rate else: sampling_rate = config["sampling_rate"] hop_size = config["hop_size"] # get spectrogram D = librosa.stft( audio, n_fft=config["fft_size"], hop_length=hop_size, win_length=config["win_length"], window=config["window"], pad_mode="reflect", ) S, _ = librosa.magphase(D) # (#bins, #frames) # get mel basis fmin = 0 if config["fmin"] is None else config["fmin"] fmax = sampling_rate // 2 if config["fmax"] is None else config["fmax"] mel_basis = librosa.filters.mel( sr=sampling_rate, n_fft=config["fft_size"], n_mels=config["num_mels"], fmin=fmin, fmax=fmax, ) mel = np.log10(np.maximum(np.dot(mel_basis, S), 1e-10)).T # (#frames, #bins) # check audio and feature length audio = np.pad(audio, (0, config["fft_size"]), mode="edge") audio = audio[:len(mel) * hop_size] assert len(mel) * hop_size == len(audio) # extract raw pitch _f0, t = pw.dio( audio.astype(np.double), fs=sampling_rate, f0_ceil=fmax, frame_period=1000 * hop_size / sampling_rate, ) f0 = pw.stonemask(audio.astype(np.double), _f0, t, sampling_rate) if len(f0) >= len(mel): f0 = f0[:len(mel)] else: f0 = np.pad(f0, (0, len(mel) - len(f0))) # extract energy energy = np.sqrt(np.sum(S**2, axis=0)) assert len(mel) == len(f0) == len(energy) # remove outlier f0/energy f0 = remove_outlier(f0) energy = remove_outlier(energy) # apply global gain if config["global_gain_scale"] > 0.0: audio *= config["global_gain_scale"] if np.abs(audio).max() >= 1.0: logging.warn( f"{utt_id} causes clipping. It is better to reconsider global gain scale value." ) item["audio"] = audio item["mel"] = mel item["f0"] = f0 item["energy"] = energy return True, mel, energy, f0, item
def error(bot, update, error): logging.warn('Update:"%s" caused Error:"%s"' % (update, error))
from __future__ import (absolute_import,) import os import logging import re from setuptools import setup, find_packages readme_dir = os.path.dirname(__file__) readme_path = os.path.join(readme_dir, 'README.md') try: with open(readme_path, 'r') as f: readme_markdown = f.read() except: logging.warn("Failed to load %s" % readme_path) readme_markdown = "" try: import pypandoc readme_restructured = pypandoc.convert(readme_markdown, to='rst', format='md') except: readme_restructured = readme_markdown logging.warn("Conversion of long_description from MD to RST failed") pass with open('isovar/__init__.py', 'r') as f: version = re.search( r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]', f.read(),
def testCaseOutputV2(self): if vobject is None: logging.warn( "QTI v1 to v2 migration tests skipped: vobject required") return self.cp.manifest.root.SetID('outputv2') dPath = os.path.join(self.dataPath, 'input') fList = [] for f in os.listdir(dPath): if self.cp.IgnoreFile(f): continue stem, ext = os.path.splitext(f) if ext.lower() == '.xml': fList.append(f) fList.sort() for f in fList: doc = QTIDocument( baseURI=str(uri.URI.from_path(os.path.join(dPath, f)))) doc.Read() doc.MigrateV2(self.cp) # Having migrated everything in the input folder, we now check our CP # against the output cp2 = imscp.ContentPackage(os.path.join(self.dataPath, 'outputv2')) # To do.... # Compare the manifests # Compare each file fList1 = self.cp.fileTable.keys() fList1.sort() fList2 = cp2.fileTable.keys() fList2.sort() if fList1 != fList2: diagnosis = [] for f in fList1: if f not in fList2: diagnosis.append("Extra file found: %s" % f) for f in fList2: if f not in fList1: diagnosis.append("Missing file: %s" % f) self.fail("File lists:\n %s" % string.join(diagnosis, '\n ')) logging.debug(str(self.cp.manifest)) logging.debug(str(cp2.manifest)) output = self.cp.manifest.DiffString(cp2.manifest) self.assertTrue(self.cp.manifest.root == cp2.manifest.root, "Manifests differ:\n%s" % output) checkFiles = {} for r in cp2.manifest.root.Resources.Resource: # Check the entry-point of each resource f = r.GetEntryPoint() if f: fPath = f.PackagePath(cp2) qtiDoc = qtiv2.core.QTIDocument(baseURI=str( uri.URI.from_virtual_path(self.cp.dPath.join(fPath)))) qtiDoc.Read() # print str(qtiDoc) qtiDoc2 = qtiv2.core.QTIDocument(baseURI=str( uri.URI.from_virtual_path(cp2.dPath.join(fPath)))) qtiDoc2.Read() # print str(qtiDoc2) output = qtiDoc.DiffString(qtiDoc2) result = (qtiDoc.root == qtiDoc2.root) if not result and output is None: # This should not happen self.PrintPrettyWeird(qtiDoc.root, qtiDoc2.root) self.assertTrue( qtiDoc.root == qtiDoc2.root, "QTI Files differ at %s (actual output shown first)\n%s" % (fPath, output)) for f in r.File: if f.href is None or f.href.is_absolute(): continue fPath = f.PackagePath(cp2) fAbsPath = self.cp.dPath.join(fPath) fAbsPath2 = cp2.dPath.join(fPath) baseURI = str(uri.URI.from_virtual_path(fAbsPath)) baseURI2 = str(uri.URI.from_virtual_path(fAbsPath2)) if fAbsPath.splitext()[1].lower() == '.xml': # Two xml files, compare with simple XMLElement doc = xml.Document(baseURI=baseURI) doc.Read() doc2 = xml.Document(baseURI=baseURI2) doc2.Read() output = doc.DiffString(doc2) result = (doc.root == doc2.root) if not result and output is None: # This should not happen self.PrintPrettyWeird(doc.root, doc2.root) self.assertTrue( doc.root == doc2.root, "XML Files differ at %s (actual output shown first)\n%s" % (fPath, output)) else: # Binary compare the two files. f = fAbsPath.open('rb') f2 = fAbsPath2.open('rb') while True: fData = f.read(1024) fData2 = f2.read(1024) self.assertTrue(fData == fData2, "Binary files don't match: %s" % fPath) if not fData: break
async def remove(self): args = [self.getValue(self.__primary_key__)] rows = await execute(self.__delete__, args) if rows != 1: logging.warn('failed to remove by primary key: affected rows: %s' % rows)
def post(self): logging.warn("error response") self.set_status(500)
def do_generate(self, lane): feeds = [] annotator = self.app.manager.annotator(lane) if isinstance(lane, Lane) and lane.parent: languages = lane.language_key lane_name = lane.name else: languages = None lane_name = None library = lane.library url = self.app.manager.cdn_url_for( "feed", languages=languages, lane_name=lane_name, library_short_name=library.short_name) default_order = library.default_facet(Facets.ORDER_FACET_GROUP_NAME) allowed_orders = library.enabled_facets(Facets.ORDER_FACET_GROUP_NAME) chosen_orders = self.orders or [default_order] default_availability = library.default_facet( Facets.AVAILABILITY_FACET_GROUP_NAME) allowed_availabilities = library.enabled_facets( Facets.AVAILABILITY_FACET_GROUP_NAME) chosen_availabilities = self.availabilities or [default_availability] default_collection = library.default_facet( Facets.COLLECTION_FACET_GROUP_NAME) allowed_collections = library.enabled_facets( Facets.COLLECTION_FACET_GROUP_NAME) chosen_collections = self.collections or [default_collection] for order in chosen_orders: if order not in allowed_orders: logging.warn("Ignoring unsupported ordering %s" % order) continue for availability in chosen_availabilities: if availability not in allowed_availabilities: logging.warn("Ignoring unsupported availability %s" % availability) continue for collection in chosen_collections: if collection not in allowed_collections: logging.warn("Ignoring unsupported collection %s" % collection) continue pagination = Pagination.default() facets = Facets(library=library, collection=collection, availability=availability, order=order, order_ascending=True) title = lane.display_name for pagenum in range(0, self.pages): yield AcquisitionFeed.page(self._db, title, url, lane, annotator, facets=facets, pagination=pagination, force_refresh=True) pagination = pagination.next_page
def get_source_models(oqparam, gsim_lt, source_model_lt, in_memory=True): """ Build all the source models generated by the logic tree. :param oqparam: an :class:`openquake.commonlib.oqvalidation.OqParam` instance :param gsim_lt: a :class:`openquake.commonlib.logictree.GsimLogicTree` instance :param source_model_lt: a :class:`openquake.commonlib.logictree.SourceModelLogicTree` instance :param in_memory: if True, keep in memory the sources, else just collect the TRTs :returns: an iterator over :class:`openquake.commonlib.logictree.SourceModel` tuples """ converter = sourceconverter.SourceConverter( oqparam.investigation_time, oqparam.rupture_mesh_spacing, oqparam.complex_fault_mesh_spacing, oqparam.width_of_mfd_bin, oqparam.area_source_discretization) psr = nrml.SourceModelParser(converter) # consider only the effective realizations smlt_dir = os.path.dirname(source_model_lt.filename) for sm in source_model_lt.gen_source_models(gsim_lt): src_groups = [] for name in sm.names.split(): fname = os.path.abspath(os.path.join(smlt_dir, name)) if in_memory: apply_unc = source_model_lt.make_apply_uncertainties(sm.path) logging.info('Reading %s', fname) src_groups.extend(psr.parse_src_groups(fname, apply_unc)) else: # just collect the TRT models smodel = nrml.read(fname).sourceModel if smodel[0].tag.endswith('sourceGroup'): # NRML 0.5 format for sg_node in smodel: sg = sourceconverter.SourceGroup( sg_node['tectonicRegion']) sg.sources = sg_node.nodes src_groups.append(sg) else: # NRML 0.4 format: smodel is a list of source nodes src_groups.extend( sourceconverter.SourceGroup.collect(smodel)) num_sources = sum(len(sg.sources) for sg in src_groups) sm.src_groups = src_groups trts = [mod.trt for mod in src_groups] source_model_lt.tectonic_region_types.update(trts) logging.info( 'Processed source model %d with %d potential gsim path(s) and %d ' 'sources', sm.ordinal + 1, sm.num_gsim_paths, num_sources) gsim_file = oqparam.inputs.get('gsim_logic_tree') if gsim_file: # check TRTs for src_group in src_groups: if src_group.trt not in gsim_lt.values: raise ValueError( "Found in %r a tectonic region type %r inconsistent " "with the ones in %r" % (sm, src_group.trt, gsim_file)) yield sm # check investigation_time psr.check_nonparametric_sources(oqparam.investigation_time) # log if some source file is being used more than once dupl = 0 for fname, hits in psr.fname_hits.items(): if hits > 1: logging.info('%s has been considered %d times', fname, hits) if not psr.changed_sources: dupl += hits if dupl and not oqparam.optimize_same_id_sources: logging.warn('You are doing redundant calculations: please make sure ' 'that different sources have different IDs and set ' 'optimize_same_id_sources=true in your .ini file')
async def save(self): args = list(map(self.getValueOrDefault, self.__fields__)) args.append(self.getValueOrDefault(self.__primary_key__)) rows = await execute(self.__insert__, args) if rows != 1: logging.warn('failed to insert record: affected rows: %s' % rows)
def corner(xs, bins=20, range=None, weights=None, color="k", smooth=None, smooth1d=None, labels=None, label_kwargs=None, show_titles=False, title_fmt=".2f", title_kwargs=None, truths=None, truth_color="#4682b4", scale_hist=False, quantiles=None, verbose=False, fig=None, max_n_ticks=5, top_ticks=False, use_math_text=False, hist_kwargs=None, **hist2d_kwargs): """ Make a *sick* corner plot showing the projections of a data set in a multi-dimensional space. kwargs are passed to hist2d() or used for `matplotlib` styling. Parameters ---------- xs : array_like (nsamples, ndim) The samples. This should be a 1- or 2-dimensional array. For a 1-D array this results in a simple histogram. For a 2-D array, the zeroth axis is the list of samples and the next axis are the dimensions of the space. bins : int or array_like (ndim,) (optional) The number of bins to use in histograms, either as a fixed value for all dimensions, or as a list of integers for each dimension. weights : array_like (nsamples,) The weight of each sample. If `None` (default), samples are given equal weight. color : str (optional) A ``matplotlib`` style color for all histograms. smooth, smooth1d : float (optional) The standard deviation for Gaussian kernel passed to `scipy.ndimage.gaussian_filter` to smooth the 2-D and 1-D histograms respectively. If `None` (default), no smoothing is applied. labels : iterable (ndim,) (optional) A list of names for the dimensions. If a ``xs`` is a ``pandas.DataFrame``, labels will default to column names. label_kwargs : dict (optional) Any extra keyword arguments to send to the `set_xlabel` and `set_ylabel` methods. show_titles : bool (optional) Displays a title above each 1-D histogram showing the 0.5 quantile with the upper and lower errors supplied by the quantiles argument. title_fmt : string (optional) The format string for the quantiles given in titles. If you explicitly set ``show_titles=True`` and ``title_fmt=None``, the labels will be shown as the titles. (default: ``.2f``) title_kwargs : dict (optional) Any extra keyword arguments to send to the `set_title` command. range : iterable (ndim,) (optional) A list where each element is either a length 2 tuple containing lower and upper bounds or a float in range (0., 1.) giving the fraction of samples to include in bounds, e.g., [(0.,10.), (1.,5), 0.999, etc.]. If a fraction, the bounds are chosen to be equal-tailed. truths : iterable (ndim,) (optional) A list of reference values to indicate on the plots. Individual values can be omitted by using ``None``. truth_color : str (optional) A ``matplotlib`` style color for the ``truths`` makers. scale_hist : bool (optional) Should the 1-D histograms be scaled in such a way that the zero line is visible? quantiles : iterable (optional) A list of fractional quantiles to show on the 1-D histograms as vertical dashed lines. verbose : bool (optional) If true, print the values of the computed quantiles. plot_contours : bool (optional) Draw contours for dense regions of the plot. use_math_text : bool (optional) If true, then axis tick labels for very large or small exponents will be displayed as powers of 10 rather than using `e`. max_n_ticks: int (optional) Maximum number of ticks to try to use top_ticks : bool (optional) If true, label the top ticks of each axis fig : matplotlib.Figure (optional) Overplot onto the provided figure object. hist_kwargs : dict (optional) Any extra keyword arguments to send to the 1-D histogram plots. **hist2d_kwargs : (optional) Any remaining keyword arguments are sent to `corner.hist2d` to generate the 2-D histogram plots. """ if quantiles is None: quantiles = [] if title_kwargs is None: title_kwargs = dict() if label_kwargs is None: label_kwargs = dict() # Try filling in labels from pandas.DataFrame columns. if labels is None: try: labels = xs.columns except AttributeError: pass # Deal with 1D sample lists. xs = np.atleast_1d(xs) if len(xs.shape) == 1: xs = np.atleast_2d(xs) else: assert len(xs.shape) == 2, "The input sample array must be 1- or 2-D." xs = xs.T assert xs.shape[0] <= xs.shape[1], "I don't believe that you want more " \ "dimensions than samples!" # Parse the weight array. if weights is not None: weights = np.asarray(weights) if weights.ndim != 1: raise ValueError("Weights must be 1-D") if xs.shape[1] != weights.shape[0]: raise ValueError("Lengths of weights must match number of samples") # Parse the parameter ranges. if range is None: if "extents" in hist2d_kwargs: logging.warn("Deprecated keyword argument 'extents'. " "Use 'range' instead.") range = hist2d_kwargs.pop("extents") else: range = [[x.min(), x.max()] for x in xs] # Check for parameters that never change. m = np.array([e[0] == e[1] for e in range], dtype=bool) if np.any(m): raise ValueError( ("It looks like the parameter(s) in " "column(s) {0} have no dynamic range. " "Please provide a `range` argument.").format(", ".join( map("{0}".format, np.arange(len(m))[m])))) else: # If any of the extents are percentiles, convert them to ranges. # Also make sure it's a normal list. range = list(range) for i, _ in enumerate(range): try: emin, emax = range[i] except TypeError: q = [0.5 - 0.5 * range[i], 0.5 + 0.5 * range[i]] range[i] = quantile(xs[i], q, weights=weights) if len(range) != xs.shape[0]: raise ValueError("Dimension mismatch between samples and range") # Parse the bin specifications. try: bins = [float(bins) for _ in range] except TypeError: if len(bins) != len(range): raise ValueError("Dimension mismatch between bins and range") # Some magic numbers for pretty axis layout. K = len(xs) factor = 2.0 # size of one side of one panel lbdim = 0.5 * factor # size of left/bottom margin trdim = 0.2 * factor # size of top/right margin whspace = 0.05 # w/hspace size plotdim = factor * K + factor * (K - 1.) * whspace dim = lbdim + plotdim + trdim # Create a new figure if one wasn't provided. if fig is None: fig, axes = pl.subplots(K, K, figsize=(dim, dim)) else: try: axes = np.array(fig.axes).reshape((K, K)) except: raise ValueError("Provided figure has {0} axes, but data has " "dimensions K={1}".format(len(fig.axes), K)) # Format the figure. lb = lbdim / dim tr = (lbdim + plotdim) / dim fig.subplots_adjust(left=lb, bottom=lb, right=tr, top=tr, wspace=whspace, hspace=whspace) # Set up the default histogram keywords. if hist_kwargs is None: hist_kwargs = dict() hist_kwargs["color"] = hist_kwargs.get("color", color) for i, x in enumerate(xs): # Deal with masked arrays. if hasattr(x, "compressed"): x = x.compressed() if np.shape(xs)[0] == 1: ax = axes else: ax = axes[i, i] # Plot the histograms. if gaussian_filter is None: raise ImportError("Please install scipy for smoothing") n, b = np.histogram(x, bins=bins[i], weights=weights, range=range[i]) if smooth1d is not None: n = gaussian_filter(n, smooth1d) x0 = 0.5 * (b[:-1] + b[1:]) y0 = n.copy() ax.plot(x0, y0, **hist_kwargs) if truths is not None and truths[i] is not None: ax.axvline(truths[i], color=truth_color, lw=2) # Plot quantiles if wanted. if len(quantiles) > 0: qvalues = quantile(x, quantiles, weights=weights) # for q in qvalues: # ax.axvline(q, ls="dashed", color=color) if verbose: print("Quantiles:") print([item for item in zip(quantiles, qvalues)]) if show_titles: title = None if title_fmt is not None: # Compute the quantiles for the title. This might redo # unneeded computation but who cares. q_16, q_50, q_84 = quantile(x, [0.16, 0.5, 0.84], weights=weights) q_m, q_p = q_50 - q_16, q_84 - q_50 # # Format the quantile display. # fmt = "{{0:{0}}}".format(title_fmt).format # title = r"${{{0}}}_{{-{1}}}^{{+{2}}}$" # title = title.format(fmt(q_50), fmt(q_m), fmt(q_p)) # format the quantile display max_dq = max(q_m, q_p) expo = lambda x: np.floor(np.log10(np.abs(x))).astype(int) q_50_str = str(round(q_50, -expo(max_dq))) q_m_str = str(round(q_m, -expo(q_m) + 1)) q_p_str = str(round(q_p, -expo(q_p) + 1)) title = r"$" + q_50_str + "_{-" + q_m_str + "}^{+" + q_p_str + "}$" # # just to check the format # print(str(q_50)+", "+str(q_50_str)) # print(str(q_m)+", "+str(q_m_str)) # print(str(q_p)+", "+str(q_p_str)) # print("\n") # Add in the column name if it's given. if labels is not None: title = "{0} = {1}".format(labels[i], title) elif labels is not None: title = "{0}".format(labels[i]) if title is not None: ax.set_title(title, **title_kwargs) # fill in the confidence levels q_2_5, q_16, q_84, q_97_5 = quantile(x, [0.025, 0.16, 0.84, 0.975], weights=weights) # find closest x bin to quantiles q_2_5 = x0[x0 >= q_2_5][0] q_16 = x0[x0 >= q_16][0] q_84 = x0[x0 <= q_84][-1] q_97_5 = x0[x0 <= q_97_5][-1] # fill below the 1d histogram ax.fill_between(x0, y0, color=color, alpha=0.8, where=(x0 >= q_16) * (x0 <= q_84)) ax.fill_between(x0, y0, color=color, alpha=0.4, where=(x0 >= q_2_5) * (x0 <= q_16) + (x0 >= q_84) * (x0 <= q_97_5)) # Set up the axes. ax.set_xlim(range[i]) if scale_hist: maxn = np.max(n) ax.set_ylim(-0.1 * maxn, 1.1 * maxn) else: ax.set_ylim(0, 1.1 * np.max(n)) ax.set_yticklabels([]) ax.xaxis.set_major_locator(MaxNLocator(max_n_ticks, prune="lower")) if i < K - 1: if top_ticks: ax.xaxis.set_ticks_position("top") [l.set_rotation(45) for l in ax.get_xticklabels()] [l.set_size(12) for l in ax.get_xticklabels()] else: ax.set_xticklabels([]) else: [l.set_rotation(45) for l in ax.get_xticklabels()] [l.set_size(12) for l in ax.get_xticklabels()] if labels is not None: ax.set_xlabel(labels[i], **label_kwargs) ax.xaxis.set_label_coords(0.5, -0.3) # use MathText for axes ticks ax.xaxis.set_major_formatter( ScalarFormatter(useMathText=use_math_text)) for j, y in enumerate(xs): if np.shape(xs)[0] == 1: ax = axes else: ax = axes[i, j] if j > i: ax.set_frame_on(False) ax.set_xticks([]) ax.set_yticks([]) continue elif j == i: continue # Deal with masked arrays. if hasattr(y, "compressed"): y = y.compressed() # plot 2d histogram hist2d(y, x, ax=ax, range=[range[j], range[i]], weights=weights, color=color, smooth=smooth, bins=[bins[j], bins[i]], **hist2d_kwargs) if truths is not None: if truths[i] is not None and truths[j] is not None: ax.plot(truths[j], truths[i], "s", color=truth_color) # if truths[j] is not None: # ax.axvline(truths[j], color=truth_color, lw=2) # if truths[i] is not None: # ax.axhline(truths[i], color=truth_color, lw=2) ax.xaxis.set_major_locator(MaxNLocator(max_n_ticks, prune="lower")) ax.yaxis.set_major_locator(MaxNLocator(max_n_ticks, prune="lower")) if i < K - 1: ax.set_xticklabels([]) else: [l.set_rotation(45) for l in ax.get_xticklabels()] [l.set_size(12) for l in ax.get_xticklabels()] if labels is not None: ax.set_xlabel(labels[j], **label_kwargs) ax.xaxis.set_label_coords(0.5, -0.3) # use MathText for axes ticks ax.xaxis.set_major_formatter( ScalarFormatter(useMathText=use_math_text)) if j > 0: ax.set_yticklabels([]) else: [l.set_rotation(45) for l in ax.get_yticklabels()] [l.set_size(12) for l in ax.get_yticklabels()] if labels is not None: ax.set_ylabel(labels[i], **label_kwargs) ax.yaxis.set_label_coords(-0.3, 0.5) # use MathText for axes ticks ax.yaxis.set_major_formatter( ScalarFormatter(useMathText=use_math_text)) return fig
def delayed_response(): logging.warn("delayed response") self.finish()
def copy_element(val, src_dt, tgt_dt, ctx): logging.debug("copy_element, val: " + str(val) + " val type: " + str(type(val)) + "src_dt: " + dump_dtype(src_dt) + " tgt_dt: " + dump_dtype(tgt_dt)) fin = ctx["fin"] fout = ctx["fout"] out = None if len(src_dt) > 0: out_fields = [] i = 0 for name in src_dt.fields: field_src_dt = src_dt.fields[name][0] field_tgt_dt = tgt_dt.fields[name][0] field_val = val[i] i += 1 out_field = copy_element(field_val, field_src_dt, field_tgt_dt, ctx) out_fields.append(out_field) out = tuple(out_fields) elif src_dt.metadata and 'ref' in src_dt.metadata: if not tgt_dt.metadata or 'ref' not in tgt_dt.metadata: raise TypeError("Expected tgt dtype to be ref, but got: {}".format(tgt_dt)) ref = tgt_dt.metadata['ref'] if is_reference(ref): # initialize out to null ref if is_h5py(ctx['fout']): out = h5py.Reference() # null h5py ref else: out = '' # h5pyd refs are strings if ref: try: fin_obj = fin[val] except AttributeError as ae: msg = "Unable able to get obj for ref value: {}".format(ae) logging.error(msg) print(msg) return None # TBD - for hsget, the name property is not getting set h5path = fin_obj.name if not h5path: msg = "No path found for ref object" logging.warn(msg) if ctx["verbose"]: print(msg) else: fout_obj = fout[h5path] if is_h5py(ctx['fout']): out = fout_obj.ref else: out = str(fout_obj.ref) # convert to string for JSON serialization elif is_regionreference(ref): out = "tbd" else: raise TypeError("Unexpected ref type: {}".format(type(ref))) elif src_dt.metadata and 'vlen' in src_dt.metadata: logging.debug("copy_elment, got vlen element, dt: {}".format(src_dt.metadata["vlen"])) if not isinstance(val, np.ndarray): raise TypeError("Expecting ndarray or vlen element, but got: {}".format(type(val))) if not tgt_dt.metadata or 'vlen' not in tgt_dt.metadata: raise TypeError("Expected tgt dtype to be vlen, but got: {}".format(tgt_dt)) src_vlen_dt = src_dt.metadata["vlen"] tgt_vlen_dt = tgt_dt.metadata["vlen"] if has_reference(src_vlen_dt): if len(val.shape) == 0: # scalar array e = val[()] v = copy_element(e, src_vlen_dt, tgt_vlen_dt, ctx) out = np.array(v, dtype=tgt_dt) else: out = np.zeros(val.shape, dtype=tgt_dt) for i in range(len(out)): e = val[i] out[i] = copy_element(e, src_vlen_dt, tgt_vlen_dt, ctx) else: # can just directly copy the array out = np.zeros(val.shape, dtype=tgt_dt) out[...] = val[...] else: out = val # can just copy as is return out
def hist2d(x, y, bins=20, range=None, weights=None, levels=None, smooth=None, ax=None, color=None, plot_datapoints=True, plot_density=True, plot_contours=True, no_fill_contours=False, fill_contours=False, contour_kwargs=None, contourf_kwargs=None, data_kwargs=None, **kwargs): """ Plot a 2-D histogram of samples. Parameters ---------- x, y : array_like (nsamples,) The samples. levels : array_like The contour levels to draw. ax : matplotlib.Axes (optional) A axes instance on which to add the 2-D histogram. plot_datapoints : bool (optional) Draw the individual data points. plot_density : bool (optional) Draw the density colormap. plot_contours : bool (optional) Draw the contours. no_fill_contours : bool (optional) Add no filling at all to the contours (unlike setting ``fill_contours=False``, which still adds a white fill at the densest points). fill_contours : bool (optional) Fill the contours. contour_kwargs : dict (optional) Any additional keyword arguments to pass to the `contour` method. contourf_kwargs : dict (optional) Any additional keyword arguments to pass to the `contourf` method. data_kwargs : dict (optional) Any additional keyword arguments to pass to the `plot` method when adding the individual data points. """ if ax is None: ax = pl.gca() # Set the default range based on the data range if not provided. if range is None: if "extent" in kwargs: logging.warn("Deprecated keyword argument 'extent'. " "Use 'range' instead.") range = kwargs["extent"] else: range = [[x.min(), x.max()], [y.min(), y.max()]] # Set up the default plotting arguments. if color is None: color = "k" # Choose the default "sigma" contour levels. if levels is None: levels = 1.0 - np.exp(-0.5 * np.arange(0.5, 2.1, 0.5)**2) # This is the color map for the density plot, over-plotted to indicate the # density of the points near the center. density_cmap = LinearSegmentedColormap.from_list("density_cmap", [color, (1, 1, 1, 0)]) # This color map is used to hide the points at the high density areas. white_cmap = LinearSegmentedColormap.from_list("white_cmap", [(1, 1, 1), (1, 1, 1)], N=2) # This "color map" is the list of colors for the contour levels if the # contours are filled. rgba_color = colorConverter.to_rgba(color) contour_cmap = [list(rgba_color) for l in levels] + [rgba_color] for i, l in enumerate(levels): contour_cmap[i][-1] *= float(i) / (len(levels) + 1) # We'll make the 2D histogram to directly estimate the density. try: # MANUWARNING: removed the stupid range argument # H, X, Y = np.histogram2d(x.flatten(), y.flatten(), bins=bins, # range=range, weights=weights) H, X, Y = np.histogram2d(x.flatten(), y.flatten(), bins=bins, weights=weights) except ValueError: raise ValueError("It looks like at least one of your sample columns " "have no dynamic range. You could try using the " "'range' argument.") if smooth is not None: if gaussian_filter is None: raise ImportError("Please install scipy for smoothing") H = gaussian_filter(H, smooth) # Compute the density levels. Hflat = H.flatten() inds = np.argsort(Hflat)[::-1] Hflat = Hflat[inds] sm = np.cumsum(Hflat) sm /= sm[-1] V = np.empty(len(levels)) for i, v0 in enumerate(levels): try: V[i] = Hflat[sm <= v0][-1] except: V[i] = Hflat[0] V.sort() m = np.diff(V) == 0 if np.any(m): logging.warning("Too few points to create valid contours") while np.any(m): V[np.where(m)[0][0]] *= 1.0 - 1e-4 m = np.diff(V) == 0 V.sort() # Compute the bin centers. X1, Y1 = 0.5 * (X[1:] + X[:-1]), 0.5 * (Y[1:] + Y[:-1]) # Extend the array for the sake of the contours at the plot edges. H2 = H.min() + np.zeros((H.shape[0] + 4, H.shape[1] + 4)) H2[2:-2, 2:-2] = H H2[2:-2, 1] = H[:, 0] H2[2:-2, -2] = H[:, -1] H2[1, 2:-2] = H[0] H2[-2, 2:-2] = H[-1] H2[1, 1] = H[0, 0] H2[1, -2] = H[0, -1] H2[-2, 1] = H[-1, 0] H2[-2, -2] = H[-1, -1] X2 = np.concatenate([ X1[0] + np.array([-2, -1]) * np.diff(X1[:2]), X1, X1[-1] + np.array([1, 2]) * np.diff(X1[-2:]), ]) Y2 = np.concatenate([ Y1[0] + np.array([-2, -1]) * np.diff(Y1[:2]), Y1, Y1[-1] + np.array([1, 2]) * np.diff(Y1[-2:]), ]) if plot_datapoints: if data_kwargs is None: data_kwargs = dict() data_kwargs["color"] = data_kwargs.get("color", color) data_kwargs["ms"] = data_kwargs.get("ms", 2.0) data_kwargs["mec"] = data_kwargs.get("mec", "none") data_kwargs["alpha"] = data_kwargs.get("alpha", 0.1) ax.plot(x, y, "o", zorder=-1, rasterized=True, **data_kwargs) # Plot the base fill to hide the densest data points. if (plot_contours or plot_density) and not no_fill_contours: ax.contourf(X2, Y2, H2.T, [V.min(), H.max()], cmap=white_cmap, antialiased=False) # if plot_contours and fill_contours: # if contourf_kwargs is None: # contourf_kwargs = dict() # contourf_kwargs["colors"] = contourf_kwargs.get("colors", contour_cmap) # contourf_kwargs["antialiased"] = contourf_kwargs.get("antialiased", # False) # ax.contourf(X2, Y2, H2.T, np.concatenate([[0], V, [H.max()*(1+1e-4)]]), # **contourf_kwargs) # # # Plot the density map. This can't be plotted at the same time as the # # contour fills. # elif plot_density: # ax.pcolor(X, Y, H.max() - H.T, cmap=density_cmap) # if plot_contours and fill_contours: # if contourf_kwargs is None: # contourf_kwargs = dict() # contourf_kwargs["colors"] = contourf_kwargs.get("colors", contour_cmap) # contourf_kwargs["antialiased"] = contourf_kwargs.get("antialiased", # False) ax.contourf(X2, Y2, H2.T, [V[-1], H.max() * (1 + 1e-4)], antialiased=False, colors=color, alpha=0.8) ax.contourf(X2, Y2, H2.T, [V[-2], V[-1]], antialiased=False, colors=color, alpha=0.4) # Plot the contour edge colors. if plot_contours: if contour_kwargs is None: contour_kwargs = dict() contour_kwargs["colors"] = contour_kwargs.get("colors", color) ax.contour(X2, Y2, H2.T, V, **contour_kwargs) ax.set_xlim(range[0]) ax.set_ylim(range[1])
os.system(cmd) def start_cluster(): logging.info("start cluster") servers = [('127.0.0.1', port) for port in PORTS] half = len(servers) / 2 command.start_cluster_on_multi(servers[0:half]) time.sleep(5) for i in range(half): command.replicate("127.0.0.1", PORTS[i], "127.0.0.1", PORTS[i + half]) if os.path.exists(cluster): start_servers() logging.warn("cluster dir has already exists") sys.exit(0) else: start_servers() start_cluster() logging.info("done")
def handle(self, *args, **options): username = options['user'] groupname = options['group'] rules_status = options['status'] rules_source = options['source'] rules_category = options['category'] folder_override = options['folder_as'] try: user = User.objects.get(username=username) except ObjectDoesNotExist: user = None logging.error('[!] Specified user does not exist') try: group = Group.objects.get(name=groupname) except ObjectDoesNotExist: group = None logging.error('[!] Specified group does not exist') if user and group: for yara_master in options['rule_masters']: if os.path.isfile(yara_master): # Change working directory to account for relative imports os.chdir(os.path.dirname(os.path.abspath(yara_master))) with open(yara_master, 'r') as file_object: master_file_contents = file_object.read() include_pattern = r'\ninclude \"([^\"]+)\"' import_files = re.findall(include_pattern, master_file_contents) for file_path in import_files: with open(file_path, 'rb') as raw_content: content_results = parse_rule_submission(raw_content) # Inspect the results from content submission parsed_rules = content_results['parsed_rules'] parsing_error = content_results['parser_error'] # Identify any parsing errors that occur if parsing_error: message = '[!] Parsing Error: {}'.format(parsing_error) logging.error(message) else: # Save successfully parsed rules if folder_override == 'source': rules_source = str(Path(file_path).parent.name) save_results = YaraRule.objects.process_parsed_rules(parsed_rules, rules_source, rules_category, user, group, status=rules_status, force_source=True) elif folder_override == 'category': rules_category = str(Path(file_path).parent.name) save_results = YaraRule.objects.process_parsed_rules(parsed_rules, rules_source, rules_category, user, group, status=rules_status, force_category=True) else: save_results = YaraRule.objects.process_parsed_rules(parsed_rules, rules_source, rules_category, user, group, status=rules_status) upload_count = save_results['rule_upload_count'] collision_count = save_results['rule_collision_count'] message = '[+] Successfully uploaded {} rules and prevented {} rule collisions from {}'.format(upload_count, collision_count, file_path) logging.info(message) for error in save_results['errors']: logging.error(error) for warning in save_results['warnings']: logging.warn(warning)
def test_param(self): u'''参数测试''' logging.warn(u'参数测试。。。') # 发请求调用,接收到的参数,全为字符串类型 response = self.fetch("/test_param/555/run/?a=1&b=2&c=3&b=4&c=3") result = response.body self.assertTrue(isinstance(result, basestring), u'返回结果的类型不对!') # 网页调用,返回字符串结果 result = json.loads(result) self.assertTrue(isinstance(result, dict), u'返回结果的类型不对!') self.assertEqual(result.get('num'), '555', u'返回结果的参数值不对!') # 接收url地址参数 self.assertEqual(result.get('service'), 'run', u'返回结果的参数值不对!') self.assertEqual(result.get('a'), '1', u'返回结果的参数值不对!') # 接收到的参数,全为字符串类型 self.assertEqual(result.get('b'), ['2', '4'], u'返回结果的参数值不对!') # 多个同名参数 self.assertEqual(result.get('c'), '3', u'返回结果的参数值不对!') # c 参数重复写了,要求会过滤掉 self.assertTrue(result.get('self') != 'None', u'首参数有内容!') # 首参数是请求类 # 发请求调用,请求参数不足时 response = self.fetch("/test_param/555/run/?a=1") result = response.body self.assertTrue(isinstance(result, basestring), u'返回结果的类型不对!') # 网页调用,返回字符串结果 result = json.loads(result) self.assertTrue(isinstance(result, dict), u'返回结果的类型不对!') self.assertEqual(result, { 'result': 1, 'reason': u'请求参数错误' }, u'返回结果的参数值不对!') # 返回参数错误提示 # 内部程序直接调用,接收到的参数跟调用传参一样,且首参数为空 result = param_fun(555, 'service', a=1, b=2, c=3) self.assertTrue(isinstance(result, dict), u'返回结果的类型不对!') # 内部程序直接调用,返回原始结果 self.assertEqual(result.get('num'), 555, u'返回结果的参数值不对!') # 参数是传来的原始参数 self.assertEqual(result.get('service'), 'service', u'返回结果的参数值不对!') self.assertEqual(result.get('a'), 1, u'返回结果的参数值不对!') self.assertEqual(result.get('b'), 2, u'返回结果的参数值不对!') self.assertEqual(result.get('c'), 3, u'返回结果的参数值不对!') self.assertEqual(result.get('self'), 'None', u'返回结果的参数值不对!') # 内部程序直接调用,参数错误时 result = param_fun(555, 'service') self.assertTrue(isinstance(result, dict), u'返回结果的类型不对!') # 内部程序直接调用,返回原始结果 self.assertEqual(result, { 'result': 1, 'reason': u'请求参数错误' }, u'返回结果的参数值不对!') # 返回参数错误提示 # 发请求调用,参数中带有非法字符时,返回参数错误 response = self.fetch("/test_param/555/run/?a=1&c=3&b=4%27ss&c=3") result = response.body self.assertTrue(isinstance(result, basestring), u'返回结果的类型不对!') # 网页调用,返回字符串结果 result = json.loads(result) self.assertTrue(isinstance(result, dict), u'返回结果的类型不对!') self.assertEqual(result, { 'result': 1, 'reason': u'请求参数错误' }, u'返回结果的参数值不对!') # 返回参数错误提示 # 发请求调用,参数中带有非法字符时,返回参数错误 response = self.fetch("/test_param/555/run/?a=1&b=2&c=3&b=s'ss&c=3") result = response.body self.assertTrue(isinstance(result, basestring), u'返回结果的类型不对!') # 网页调用,返回字符串结果 result = json.loads(result) self.assertTrue(isinstance(result, dict), u'返回结果的类型不对!') self.assertEqual(result, { 'result': 1, 'reason': u'请求参数错误' }, u'返回结果的参数值不对!') # 返回参数错误提示
#!/usr/bin/python3 # # Example of using joblog.py. # import joblog import logging import random logging.info("example in Python") total = random.randint(1000000, 2000000) for i in range(0, total): if ((i + 1) % 100000) == 0 or i + 1 == total: logging.info("%i/%i (%.1f%%) processed", i + 1, total, 100.0 * (i + 1) / total) logging.warn("all done") try: x = 5 / 0 except Exception: logging.exception("intentional exception")
def upload_schedule(): req = flask.request user = view_helpers.get_current_user() schedule_data = util.json_loads(req.form.get('schedule_data')) processed_items = schedule_data['processed_items'] failed_items = schedule_data['failed_items'] term_name = schedule_data['term_name'] term_id = m.Term.id_from_name(term_name) # FIXME TODO(david): Save these in models and display on schedule #failed_items = schedule_data['failed_items'] rmclogger.log_event( rmclogger.LOG_CATEGORY_API, rmclogger.LOG_EVENT_SCHEDULE, { 'schedule_data': schedule_data, 'term_id': term_id, 'user_id': user.id, }, ) now = datetime.now() user.last_good_schedule_paste = req.form.get('schedule_text') user.last_good_schedule_paste_date = now user.save() # Remove existing schedule items for the user for the given term for usi in m.UserScheduleItem.objects(user_id=user.id, term_id=term_id): usi.delete() for item in processed_items: try: # Create this UserScheduleItem first_name, last_name = m.Professor.guess_names(item['prof_name']) prof_id = m.Professor.get_id_from_name( first_name=first_name, last_name=last_name, ) if first_name and last_name: if not m.Professor.objects.with_id(prof_id): m.Professor( id=prof_id, first_name=first_name, last_name=last_name, ).save() usi = m.UserScheduleItem( user_id=user.id, class_num=item['class_num'], building=item['building'], room=item.get('room'), section_type=item['section_type'].upper(), section_num=item['section_num'], start_date=datetime.utcfromtimestamp(item['start_date']), end_date=datetime.utcfromtimestamp(item['end_date']), course_id=item['course_id'], prof_id=prof_id, term_id=term_id, ) try: usi.save() except me.NotUniqueError as ex: # Likely the case where the user pastes in two or more valid # schedules into the same input box logging.info( 'Duplicate error on UserScheduleItem .save(): %s' % (ex)) # Add this item to the user's course history # FIXME(Sandy): See if we can get program_year_id from Quest # Or just increment their last one user.add_course(usi.course_id, usi.term_id) except KeyError: logging.error("Invalid item in uploaded schedule: %s" % (item)) # Add courses that failed to fully parse, probably due to unavailable times for course_id in set(failed_items): fsi = m.FailedScheduleItem( user_id=user.id, course_id=course_id, parsed_date=now, ) try: fsi.save() except me.NotUniqueError as ex: # This should never happen since we're iterating over a set logging.warn('WTF this should never happen.') logging.warn('Duplicate error FailedScheduleItem.save(): %s' % ex) user.add_course(course_id, term_id) user.schedules_imported += 1 user.save() schedule_screenshot.update_screenshot_async(user) rmclogger.log_event(rmclogger.LOG_CATEGORY_SCHEDULE, rmclogger.LOG_EVENT_UPLOAD, user.id) return ''
def remove(self): args = [self.getValue(self.__primary_key__)] rows = yield from execute(self.__delete__, args) if rows != 1: logging.warn('failed to delete record: affected rows: %s' % rows)
import jinja2, logging, os from google.appengine.api import app_identity PROJECT_DIR = os.path.abspath(os.path.dirname(__file__)) env = jinja2.Environment( #extensions=, autoescape=True, loader=jinja2.FileSystemLoader( os.path.join(PROJECT_DIR, "templates")), cache_size=-1, auto_reload=False, trim_blocks=True) try: env.globals['app_id'] = app_identity.get_application_id() except Exception as e: logging.warn(e, exc_info=True) env.globals['app_id'] = None
def model_run(queue,func):##需要传入两个参数,queue为manage中管理的queue,func为需要提交的线程任务,需要带参数 '''主函数''' avg_time=model_test(queue=queue,func=func)##平均一个线程花费的时间 logging.info(u'from function test get avg_time=%s'%avg_time) print u'from function test get avg_time=%s'%avg_time count=0 start_time=time.time() multiple=g_multiple while True: try: logging.info(u'max wait 100 second try to get element from queue') # d=song_info_queue.get(timeout=100) d=queue.get(timeout=100) logging.info(u'get %s from queue'%d) # msi=Multi_Song_Info(song_ids=d['ids'],refer=d['refer']) # pool.submit(msi.insert_song) pool.submit(func,d) logging.info(u'%s start new threading, get song ids=%s,threading name is:%s,pid is:%s'%(sys._getframe().f_code.co_name,d['ids'],threading.current_thread().name,os.getpid())) count+=1 if count>=max_pool*multiple: sleeptime=multiple*avg_time info=u'generate %s threading,so sleep %s second! current active threading num=%s'%(count,sleeptime,threading.active_count()) print info logging.debug(info) time.sleep(sleeptime) count=0 end_time=time.time() if end_time - start_time>=60:##每过10十分做一次检测,确保数据库连接数最好介于30到150之间,保证程序稳定运行 mysql=Cloud_Music_MySQL() Threads_connected=mysql.show_Threads_connected() Threads_connected=int(Threads_connected) mysql.close_connect() if Threads_connected<=30: ##说明程序运行效率不高,可以适当提高multiple,或是降低avg_time avg_time=model_test(queue=queue,func=func) multiple=multiple+1 info=u'程序闲置过多,current Threads_connected=%s,重设avg_time=%s,multiple=%s'%(Threads_connected,avg_time,multiple) print info logging.info(info) elif Threads_connected>=150: ##说明程序负荷过重,可以适当降低multiple,或是提高avg_time avg_time=model_test(queue=queue,func=func) multiple=max(multiple-1,2)##multiple最小为2 info=u'程序负荷过重,current Threads_connected=%s,重设avg_time=%s,multiple=%s'%(Threads_connected,avg_time,multiple) print info logging.info(info) else: info=u'程序运行良好,current Threads_connected=%s,保持avg_time=%s,multiple=%s'%(Threads_connected,avg_time,multiple) print info logging.info(info) start_time=time.time()##重设start_time except Exception,e: if str(e): e=str(e).decode('utf-8','ignore') else:##queue raise error e , str(e)为空 e=u'queue empty' logging.warn(u'function %s raise error cause by %s,traceback info is:%s '%(sys._getframe().f_code.co_name,e,traceback.format_exc())) print u'error info is:%s'%e if 'many connections' in e:##最好使用joinablequeue,##经过600秒一次的性能检测,很难抛出too many connections 异常了 print u'current too many connections,sleep 3 second wait runing connections close' # song_info_queue.put(d) queue.put(d) print u'catch too many connections error ,so put d=%s back into queue'%d logging.info(u'catch too many connections error ,so put d=%s back into queue'%d) ##发生异常在于数据库操作,d的值可以获取到,所以把他重新放回queue中,所以不需要joinablequeue了 mysql=Cloud_Music_MySQL() Threads_connected=mysql.show_Threads_connected() while Threads_connected>=100: info=u'current Threads_connected is:%s,also too much,so sleep 3 second!'%Threads_connected print info logging.debug(info) time.sleep(3) Threads_connected=mysql.show_Threads_connected() mysql.close_connect() continue elif 'empty' in e: print u'empty queue,break loop!' print u'wait 20 second ensure runing threading done' time.sleep(20) break else: info=u'unexcept error,here is traceback info:%s'%(traceback.format_exc()) print info logging.error(info) # song_info_queue.put(d) queue.put(d) print u'catch unexcept error ,so put d=%s back into queue'%d break
def handle_default(self, params): logging.warn("got %s" % (params, ))
def crash(): """For testing error logging""" logging.warn("Crashing because you want me to (hit /crash)") raise Exception("OH NOES we've crashed!!!!!!!!!! /crash was hit")
def command_upload(args): "Implements the upload subcommand" d = utils.docker_client(args) image = get_container_image(args, d) oauth2_instance = oauth2.build_oauth2(args) auth = oauth2_instance.build_authorizer() # TODO: use transloadit's signatures for upload signing. # authorization = authorize_upload(args, auth) # Generate a random uuid for upload. upload_id = uuid.uuid4().hex transloadit_host = idle_transloadit_server(args) upload_url = 'https://%(host)s/assemblies/%(id)s' % { 'host': transloadit_host, 'id': upload_id, } if args.upload_to_requestbin is not None: upload_url = 'http://requestb.in/%s' % args.upload_to_requestbin if not args.quiet > 0: sys.stdout.write( 'About to upload to server:\n\t%(transloadit_host)s\n' 'with upload id:\n\t%(upload_id)s\nStatus API:\n' '\t%(upload_url)s\nUploading...' % { 'transloadit_host': transloadit_host, 'upload_id': upload_id, 'upload_url': upload_url, }) sys.stdout.flush() p = multiprocessing.Process(target=upload, args=(args, upload_url, image)) p.daemon = True # Auto-kill when the main process exits. p.start() time.sleep(20) # Yield control to the child process to kick off upload. upload_information = None while p.is_alive(): upload_information = poll_transloadit(args, upload_url) if upload_information is not None: logging.warn( 'Upload information retrieved before upload completed??! %s', upload_information) break time.sleep(10) # 10 seconds p.join(1) # Join to clean up zombie. # TODO: make time waiting for transloadit to finish processing configurable for i in xrange(300): upload_information = poll_transloadit(args, upload_url) if upload_information is not None: break time.sleep(5) if upload_information is None: logging.error( 'Upload did not complete within expected time limits. Upload ' 'URL: %s', upload_url) return 1 # Register the grader with Coursera to initiate the image cleaning process logging.debug('Grader upload info is: %s', upload_information) # Rebuild an authorizer to ensure it's fresh and not expired auth = oauth2_instance.build_authorizer() grader_id = register_grader(auth, args, bucket=upload_information[0], key=upload_information[1]) return update_assignments(auth, grader_id, args)
def parse_csv(infile, player_num): """Parse csv. fields: ------- 0 - division 1 - opponent 2 - date 3 - youtube 4 - yards 5 - description 6 - player number 7 - Offense or Defense 8 - start time hr:min:sec 9 - duration 10 - url link1 11 - url link2 """ rv_list = [] with open(infile, 'r') as fd: rd_list = csv.reader(fd, delimiter=',') base_len = 0 for i,row in enumerate(rd_list): if i == 0: # header base_len = len(row) continue if len(row) != base_len: continue # valid row logging.debug('%d: %s', i, ','.join(row)) if player_num != row[6]: continue # build description text txt = '#' + row[6] if row[7].lower().startswith('o'): txt += ':offense' elif row[7].lower().startswith('s'): txt += ':steams' else: txt += ':defense' if row[5] != '': txt += ":%s" % row[5] if row[8] == '' or row[2] == '': logging.warn('Skipping %s line ... date/time missing', row) continue time_idx = time.mktime( time.strptime("%s %s" % (csvdate2filedate(row[2]), row[8]), "%Y-%m-%d %H:%M:%S")) movie = find_file(row[0], row[1], row[2]) logging.debug("time_idx:%d movie:%s", time_idx, movie) if movie is None: logging.warn('Unable to find movie for division:%s vs ' '%s on %s', row[0], row[1], row[2]) break # movie, descrip, time, duration rv_list.append([time_idx, movie, txt, row[8], row[9]]) return rv_list
def __init__(self, filename, file=None, path=None): self.materials = {} self.meshes = {} # Name mapping self.mesh_list = [] # Also includes anonymous meshes if file is None: file = open(filename, 'r') if path is None: path = os.path.dirname(filename) self.path = path mesh = None group = None material = None vertices = [[0., 0., 0.]] normals = [[0., 0., 0.]] tex_coords = [[0., 0.]] for line in file: if line.startswith('#'): continue values = line.split() if not values: continue if values[0] == 'v': vertices.append(map(float, values[1:4])) elif values[0] == 'vn': normals.append(map(float, values[1:4])) elif values[0] == 'vt': tex_coords.append(map(float, values[1:3])) elif values[0] == 'mtllib': self.load_material_library(values[1]) elif values[0] in ('usemtl', 'usemat'): material = self.materials.get(values[1], None) if material is None: logging.warn('Unknown material: %s' % values[1]) if mesh is not None: group = MaterialGroup(material) mesh.groups.append(group) elif values[0] == 'o': mesh = Mesh(values[1]) self.meshes[mesh.name] = mesh self.mesh_list.append(mesh) group = None elif values[0] == 'f': if mesh is None: mesh = Mesh('') self.mesh_list.append(mesh) if material is None: # FIXME material = Material("<unknown>") if group is None: group = MaterialGroup(material) mesh.groups.append(group) # For fan triangulation, remember first and latest vertices n1 = None nlast = None t1 = None tlast = None v1 = None vlast = None #points = [] for i, v in enumerate(values[1:]): v_index, t_index, n_index = \ (map(int, [j or 0 for j in v.split('/')]) + [0, 0])[:3] if v_index < 0: v_index += len(vertices) - 1 if t_index < 0: t_index += len(tex_coords) - 1 if n_index < 0: n_index += len(normals) - 1 #vertex = tex_coords[t_index] + \ # normals[n_index] + \ # vertices[v_index] group.normals += normals[n_index] group.tex_coords += tex_coords[t_index] group.vertices += vertices[v_index] if i >= 3: # Triangulate group.normals += n1 + nlast group.tex_coords += t1 + tlast group.vertices += v1 + vlast if i == 0: n1 = normals[n_index] t1 = tex_coords[t_index] v1 = vertices[v_index] nlast = normals[n_index] tlast = tex_coords[t_index] vlast = vertices[v_index]
def handle_unknown(self, params): logging.warn("Unknown message type %d: %s" % (params['#msgid'], repr(params['#msg'])))