def setup_weight(self, weight_mode="num_wins"): """ Use Window information to setup weight. :returns: """ logger.info("*" * 15) logger.info("Start weighting...") if self.config.weight_data: # first calculate azimuth and distance for each data pair self.prepare_for_weighting() # then calculate azimuth weighting for idx, window in enumerate(self.window): if weight_mode.lower() == "num_files": # weighted by the number of files in each azimuth bin self.setup_weight_for_location(window, self.naz_files, self.naz_files_all) else: # weighted by the number of windows in each azimuth bin self.setup_weight_for_location(window, self.naz_wins, self.naz_wins_all) if self.config.normalize_category: self.setup_weight_for_category(window) # normalization of data weights self.normalize_weight() # prepare the weight array self.weight_array = np.zeros([self.data_container.nwins]) _idx = 0 for window in self.window: for win_idx in range(window.num_wins): self.weight_array[_idx] = window.weight[win_idx] _idx += 1
def fetch_election(year): ''' Fetch all papers related to an election campaign; year should be one of: 2016, 2012, 2008, 2004, 1960 ''' year_html = get_html(base_url + '/' + year + '_election.php') if year == '2008': # fix weird issue in Fred Thompson's entry year_html = year_html.replace( 'Status: withdrew on <span class="docdate">', 'Status: <span class="docdate">withdrew on ') soup = BeautifulSoup(year_html) container = soup.find('td', class_='doctext').find_parent('table') for td in container.find_all('td', class_='doctext'): paragraphs = td.find_all('p') if len(paragraphs) > 0: info_paragraph, links_paragraph = paragraphs candidate = _get_candidate_info(info_paragraph) for category, category_url in _iter_candidate_categories( links_paragraph): logger.info('Fetching papers from category "%s"', category) category_soup = get_soup(category_url) category_pids = _get_pids(category_soup) for pid in category_pids: paper = fetch(pid) if candidate['name'] != paper['author']: logger.warn( 'candidate name "%s" does not match paper author "%s" (%s)', candidate['name'], paper['author'], pid) paper['category'] = category yield paper
def current_version(binary, version_modpath): """ Summary: Returns current binary package version if locally installed, master branch __version__ if the binary being built is not installed locally Args: :root (str): path to the project root directory :binary (str): Name of main project exectuable :version_modpath (str): path to __version__ module Returns: current version number of the project, TYPE: str """ pkgmgr = 'apt' pkgmgr_bkup = 'apt-cache' if which(binary): if which(pkgmgr): cmd = pkgmgr + ' show ' + binary + ' 2>/dev/null | grep Version | head -n1' elif which(pkgmgr_bkup): cmd = pkgmgr_bkup + ' policy ' + binary + ' 2>/dev/null | grep Installed' try: installed_version = subprocess.getoutput(cmd).split(':')[1].strip() return greater_version(installed_version, __version__) except Exception: logger.info( '%s: Build binary %s not installed, comparing current branch version to master branch version' % (inspect.stack()[0][3], binary)) return greater_version(masterbranch_version(version_modpath), __version__)
def grid_search_energy(self): logger.info('Energy Search...') m00_s = self.config.m00_s m00_e = self.config.m00_e dm00 = self.config.dm00 m00_array = np.arange(m00_s, m00_e+dm00, dm00) nm00 = m00_array.shape[0] misfit = np.zeros(nm00) for i in range(nm00): m00 = m00_array[i] dlnA_array, cc_amp_array, kai_array = \ self.calculate_misfit_for_m00(m00) if self.config.energy_misfit_function == 'energy': misfit[i] = \ np.sum((0.25 * dlnA_array ** 2 + 0.0 * cc_amp_array ** 2) * self.weight_array) else: misfit[i] = np.sum(kai_array * self.weight_array) # find minimum min_idx = misfit.argmin() m00_best = m00_array[min_idx] logger.info("best m00: %6.3f" % m00_best) self.m00_best = m00_best self.m00_misfit = misfit self.m00_array = m00_array
def ospackages(pkg_list): """Summary Install OS Package Prerequisites Returns: Success | Failure, TYPE: bool """ try: for pkg in pkg_list: if is_installed(pkg): logger.info(f'{pkg} binary is already installed - skip') continue elif which('yum'): cmd = 'sudo yum install ' + pkg + ' 2>/dev/null' print(subprocess.getoutput(cmd)) elif which('dnf'): cmd = 'sudo dnf install ' + pkg + ' 2>/dev/null' print(subprocess.getoutput(cmd)) else: logger.warning( '%s: Dependent OS binaries not installed - package manager not identified' % inspect.stack()[0][3]) except OSError as e: logger.exception('{}: Problem installing os package {}'.format( inspect.stack()[0][3], pkg)) return False return True
def merge_graph_2_paraphrases(node_dict: Dict[str, Node]) -> Dict[str, Node]: single_node_token_words = [] is_graph_changed = True while is_graph_changed is True: is_graph_changed = False if len(node_dict) % 50 == 0: logger.info(f"- merging turn with {len(node_dict)} nodes") for curr_node_id, curr_node_obj in node_dict.items(): curr_token_word = curr_node_obj.token_word if curr_token_word in single_node_token_words: continue node_id_list_with_curr_token_word = _get_node_id_list_with_token_word( curr_token_word, curr_node_id, node_dict) if len(node_id_list_with_curr_token_word) < 1: single_node_token_words.append(curr_token_word) else: is_graph_changed = _changing_match( curr_node_obj, node_id_list_with_curr_token_word, node_dict, "incoming") if is_graph_changed is True: break is_graph_changed = _changing_match( curr_node_obj, node_id_list_with_curr_token_word, node_dict, "outgoing") if is_graph_changed is True: break if is_graph_changed: break return node_dict
def fetch_speeches(): for author, title, date, href in _iter_speeches(): speech_url = base_url + href speech_html = requests.get(speech_url).text # Lincoln's "Cooper Union Address" has some issues :( speech_html = speech_html.replace( '<div id="_mcePaste" style="position: absolute; left: -10000px; top: 120px; width: 1px; height: 1px; overflow-x: hidden; overflow-y: hidden;">', '<p>') soup = BeautifulSoup(speech_html) # Herbert Hoover's "Campaign speech in Indianapolis, Indiana" has even worse issues :( if author == 'Herbert Hoover' and title == 'Campaign speech in Indianapolis, Indiana.': logger.info("Fixing Hoover's Indianapolis speech") transcript_p = soup.find(id='description').next_sibling.extract() soup.find(id='transcript').append(transcript_p) transcript = soup.find(id='transcript') # two of the speeches have missing transcripts :( if transcript: paragraphs = [ paragraph.strip() for paragraph in _iter_paragraphs(transcript) if not paragraph.isspace() ] # replace + space with just the space text = '\n'.join(paragraphs).replace(u'\xA0 ', ' ') timestamp = date.date().isoformat() if date else None yield { 'author': author, 'title': title, 'timestamp': timestamp, 'text': text, 'source': speech_url, }
def calculate_variance(self): """ Calculate variance reduction based on old and new source solution :return: """ npar = self.config.npar dm = self.new_cmt_par[0:npar] - self.cmt_par[0:npar] var_all = 0.0 var_all_new = 0.0 self.stats_before = {} self.stats_after = {} for _idx, window in enumerate(self.window): obsd = window.datalist['obsd'] synt = window.datalist['synt'] dt = obsd.stats.delta self.compute_new_syn(window.datalist, dm) new_synt = window.datalist['new_synt'] # calculate old variance [v1, d1, nshift1, cc1, dlnA1, cc_amp_value1] = \ self.calculate_var_one_trace(obsd, synt, window.win_time) # calculate new variance [v2, d2, nshift2, cc2, dlnA2, cc_amp_value2] = \ self.calculate_var_one_trace(obsd, new_synt, window.win_time) var_all += np.sum(0.5 * v1 * window.weight * obsd.stats.delta) var_all_new += np.sum(0.5 * v2 * window.weight * obsd.stats.delta) # prepare stats tag = window.tag['obsd'] if tag not in self.stats_before.keys(): self.stats_before[tag] = [] if tag not in self.stats_after.keys(): self.stats_after[tag] = [] for _i in range(window.num_wins): self.stats_before[tag].append( [nshift1[_i]*dt, cc1[_i], dlnA1[_i], cc_amp_value1[_i], v1[_i]/d1[_i]]) self.stats_after[tag].append( [nshift2[_i]*dt, cc2[_i], dlnA2[_i], cc_amp_value2[_i], v2[_i]/d2[_i]]) for tag in self.stats_before.keys(): self.stats_before[tag] = np.array(self.stats_before[tag]) self.stats_after[tag] = np.array(self.stats_after[tag]) logger.info( "Total Variance Reduced from %e to %e ===== %f %%" % (var_all, var_all_new, (var_all - var_all_new) / var_all * 100)) self.var_all = var_all self.var_all_new = var_all_new self.var_reduction = (var_all - var_all_new) / var_all
def add_measurements_from_sac(self, flexwinfile, tag="untaged", initial_weight=1.0, load_mode="obsolute_time"): """ Add measurments(window and data) from the given flexwinfile and the data format should be sac :param flexwinfile: :return: """ load_mode = load_mode.lower() if load_mode not in ["obsolute_time", "relative_time"]: raise ValueError("load_winfile mode incorrect: 1)obsolute_time" "2)relative_time") t1 = time.time() win_list = self.load_winfile(flexwinfile, initial_weight=initial_weight) for win_obj in win_list: self.load_data_from_sac(win_obj, tag=tag, mode=load_mode) self.window += win_list # count the total number of files and windows nfiles = len(win_list) self.nfiles += nfiles nwins = 0 for window in win_list: nwins += window.win_time.shape[0] self.nwins += nwins t2 = time.time() logger.info("="*10 + " Measurements Loading " + "="*10) logger.info("Data loaded in sac format: %s" % flexwinfile) logger.info("Elapsed time: %5.2f s" % (t2-t1)) logger.info("Number of files and window added: [%d, %d]" % (nfiles, nwins))
def wrapper(*args, **kwargs): global base_url kwargs['url'] = f"{base_url}{kwargs['url']}" logger.info( '{type_query}: {url}\t\tparams: {params}\t\tjson={json}\t\theaders: {headers}' .format( type_query=func.__name__.upper(), url=f"{kwargs['url']}", params=kwargs['query_params'] if 'query_params' in kwargs else None, json=kwargs['json_data'] if 'json_data' in kwargs else None, headers=kwargs['headers'] if 'headers' in kwargs else None)) response = func(*args, **kwargs) logger.info('Response: {resp}'.format(resp=response.json())) return response
def setup_matrix(self): """ Calculate A and b for all windows :return: """ logger.info("*" * 15) logger.info("Set up inversion matrix") for window in self.window: # loop over pair of data dsyn = self.calculate_dsyn(window.datalist) for win_idx in range(window.num_wins): # loop over each window # here, A and b are exact measurements # and no weightings are applied [A1, b1] = self.compute_A_b(window, win_idx, dsyn) self.A1_all.append(A1) self.b1_all.append(b1)
def grid_search_origin_time(self): logger.info("Origin time search...") self.calculate_tshift() t00_s = self.config.t00_s t00_e = self.config.t00_e dt00 = self.config.dt00_over_dt * \ self.window[0].datalist['obsd'].stats.delta logger.info("Grid search dt00: %6.3f" % dt00) t00_array = np.arange(t00_s, t00_e+dt00, dt00) nt00 = t00_array.shape[0] misfit = np.zeros(nt00) for i in range(nt00): t00 = t00_array[i] misfit[i] = self.calculate_tshift_misfit(t00) # find minimum min_idx = misfit.argmin() t00_best = t00_array[min_idx] logger.info("minimum t00: %6.3f" % t00_best) self.t00_best = t00_best self.t00_misfit = misfit self.t00_array = t00_array
def main(): args = get_arguments(sys.argv) config_fn = args.config config = get_config(config_fn) input_fn = args.input_filename end_points = args.end_points ep = " with additional start/end points" if end_points else "" logger.info(f"Reading sentences from '{input_fn}'{ep}.") output_dir = args.output_dir if args.output_dir else os.path.dirname( os.path.realpath(input_fn)) os.makedirs(output_dir, exist_ok=True) ep = "_we" if end_points else "" output_fn = os.path.join( output_dir, os.path.splitext(os.path.basename(input_fn))[0] + ep + "_fsa.graphml") nodes, edges = build_graphml_automaton(input_fn, config, end_points) logger.info( f"Automaton graph with {len(nodes)} nodes and {len(edges)} edges built." ) write_graphml(nodes, edges, output_fn) logger.info(f"See output in '{output_fn}'.")
def provision(self, create_or_deploy): try: self.setup_keypair() except LibcloudError as e: logger.warn('{cls}: {msg}'.format(cls=e.__class__.__name__, msg=e.message)) if 'ex_securitygroup' in self.node_specs and self.provider_dict['provider']['name'].startswith('EC2'): print self.node_specs['ex_securitygroup'] if create_or_deploy == 'deploy': with open(self.provider_dict['ssh']['public_key_path'], mode='rt') as f: public_ssh_key = f.read() self.node_specs.update({'deploy': SSHKeyDeployment(public_ssh_key)}) self.node_name = self.strategy.get_node_name() try: self.node = getattr( self, '{0}_node'.format(create_or_deploy) )(name=self.node_name, **self.node_specs) except NotImplementedError as e: if create_or_deploy != 'deploy': raise e error_message = 'deploy_node not implemented for this driver' if e.message != error_message: raise logger.info('{error_message}, so running `create_node` instead.'.format( error_message=error_message.replace('deploy_node', '`deploy_node`') )) self.node = self.create_node(name=self.node_name, **self.node_specs) # logger.info('SoftLayer billing is giving error, will remove condition once resolved.') except LibcloudError as e: logger.warn('{cls}: {msg}'.format(cls=e.__class__.__name__, msg=e.message)) except Exception as e: if e.message.startswith('InvalidGroup.NotFound'): print 'InvalidGroup.NotFound' exit(1) else: raise e
def add_measurements_from_asdf(self, flexwinfile, asdf_file_dict, obsd_tag=None, synt_tag=None, external_stationfile=None, initial_weight=1.0, winfile_format="txt"): """ Add measurments(window and data) from the given flexwinfile and the data format should be asdf. Usually, you can leave the obsd_tag=None and synt_tag=None unless if you have multiple tags in asdf file. :param flexwinfile: :param asdf_file_dict: :return: """ t1 = time.time() # load window information win_list = self.load_winfile(flexwinfile, initial_weight=initial_weight, file_format=winfile_format) # load in the asdf data asdf_dataset = self.check_and_load_asdf_file(asdf_file_dict) if external_stationfile is not None: station_info = \ self.load_station_from_text(external_stationfile) else: station_info = None # load data for each window for win_obj in win_list: self.load_data_from_asdf( win_obj, asdf_dataset, obsd_tag=obsd_tag, synt_tag=synt_tag, station_dict=station_info) self.window += win_list # count the total number of files and windows nfiles = len(win_list) self.nfiles += nfiles nwins = 0 for window in win_list: nwins += window.win_time.shape[0] self.nwins += nwins t2 = time.time() logger.info("="*10 + " Measurements Loading " + "="*10) logger.info("Data loaded in asdf format: %s" % flexwinfile) logger.info("Elapsed time: %5.2f s" % (t2-t1)) logger.info("Number of files and window added: [%d, %d]" % (nfiles, nwins))
def attempt_provision(self, create_or_deploy='create', prefer_provider=None, prefer_image=None): if ping_port() is not True: raise EnvironmentError('etcd server not up') if prefer_provider: self.strategy.strategy['provider']['options'] = (next( ifilter( lambda obj: obj.keys()[0] == prefer_provider, self.strategy.strategy['provider']['options'] ) ),) ''' # Prefer syntax self.strategy.strategy['provider']['options'].insert( 0, self.strategy.strategy['provider']['options'].pop( next( ifilter( lambda (idx, obj): obj.keys()[0] == prefer_provider, enumerate(self.strategy.strategy['provider']['options']) ) )[0] ) ) ''' for i in xrange(len(self.strategy.strategy['provider']['options'])): # Threshold logger.info('Attempting to create node "{node_name}" on: {provider}'.format( node_name=self.strategy.get_node_name(), provider=self.provider_dict['provider']['name'] )) self.provision(create_or_deploy) if self.node: save_node_info(self.node_name, node_to_dict(self.node), marshall=json) return self.node self.restrategise() raise LibcloudError('Failed to provision node')
def invert_solver(self, A, b, print_mode=False): """ Solver part. Hession matrix A and misfit vector b will be reconstructed here based on different constraints. :param A: basic Hessian matrix :param b: basic misfit vector :param print_mode: if True, then print out log information; if False, then no log information :return: """ npar = self.config.npar old_par = self.cmt_par[0:npar] / self.config.scale_par[0:npar] # scale the A and b matrix max_row = np.amax(abs(A), axis=1) for i in range(len(b)): A[i, :] /= max_row[i] b[i] /= max_row[i] # setup inversion schema if self.config.double_couple: linear_inversion = False na = npar + 2 elif self.config.zero_trace: linear_inversion = True na = npar + 1 else: linear_inversion = True na = npar # add damping trace = np.matrix.trace(A) damp_matrix = np.zeros([npar, npar]) np.fill_diagonal(damp_matrix, trace * self.config.lamda_damping) A = A + damp_matrix if print_mode: logger.info("Condition number of new A: %10.2f" % np.linalg.cond(A)) if linear_inversion: if print_mode: logger.info("Linear Inversion...") new_par = self.linear_solver(old_par, A, b, npar, na) else: if print_mode: logger.info("Nonlinear Inversion...") new_par = self.nonlinear_solver(old_par, A, b, npar, na) new_cmt_par = np.copy(self.cmt_par) new_cmt_par[0:npar] = new_par[0:npar] * self.config.scale_par[0:npar] return new_cmt_par
def main(): args = get_arguments(sys.argv) config_fn = args.config config = get_config(config_fn) input_fn = args.input_filename end_points = args.end_points ep = " with additional start/end points" if end_points else "" logger.info(f"Reading sentences from '{input_fn}'{ep}.") output_dir = args.output_dir if args.output_dir else os.path.dirname( os.path.realpath(input_fn)) os.makedirs(output_dir, exist_ok=True) ep = "_we" if end_points else "" output_fn = os.path.join( output_dir, os.path.splitext(os.path.basename(input_fn))[0] + ep + "_prp.graphml") tokenized_sentences = read_sentences(input_fn, config, end_points) initial_node_dict = build_initial_graph(tokenized_sentences) # make the paraphrases ! logger.info(f"Initial graph with {len(initial_node_dict)} nodes built.") if len(initial_node_dict) > 1000: very = "" if len(initial_node_dict) > 2000: very = "very " logger.warning( f"! Generation of the paraphrase graphs with this initial size could be {very}slow." ) node_dict_paraphrases = merge_graph_2_paraphrases(initial_node_dict) logger.info( f"Paraphrase graph with {len(node_dict_paraphrases)} nodes built.") write_graphml(node_dict_paraphrases, config, output_fn) logger.info(f"See output in '{output_fn}'.")
def run_command_with_output(command_description, command): process_output = "" logger.info("Starting process: " + command_description) logger.info("Running command: " + command) with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, universal_newlines=True, shell=True) as process: for line in process.stdout: process_output += line errors = os.linesep.join(process.stderr.readlines()) if process.returncode != 0: logger.error(command_description + " failed!" + os.linesep + errors) raise subprocess.CalledProcessError(process.returncode, process.args) else: logger.info(command_description + " completed successfully") return process_output
def prepare_for_weighting(self): """ Prepare necessary information for weighting, e.x., calculating azimuth, distance and energty of a window. Also, based on the tags, sort window into different categories. :return: """ for window in self.window: # calculate energy #window.win_energy(mode=self.config.norm_mode) # calculate location window.get_location_info(self.cmtsource) self.naz_files, self.naz_wins = self.calculate_azimuth_bin() # add all category together # if not weighted by category, then use total number self.naz_files_all = np.zeros(const.NREGIONS) self.naz_wins_all = np.zeros(const.NREGIONS) for key in self.naz_files.keys(): self.naz_files_all += self.naz_files[key] self.naz_wins_all += self.naz_wins[key] logger.info("Category: %s" % key) logger.info("Azimuth file bin: [%s]" % (', '.join(map(str, self.naz_files[key])))) logger.info("Azimuth win bin: [%s]" % (', '.join(map(str, self.naz_wins[key])))) # stat different category bin_category = {} for window in self.window: tag = window.tag['obsd'] if tag in bin_category.keys(): bin_category[tag] += window.num_wins else: bin_category[tag] = window.num_wins self.bin_category = bin_category
def builddir_structure(param_dict, version): """ Summary. - Updates paths in binary exectuable - Updates Args: :root (str): full path to root directory of the git project :builddir (str): name of current build directory which we need to populate Vars: :lib_path (str): src path to library modules in project root :builddir_path (str): dst path to root of the current build directory (/<path>/nlines-1.X.X dir) Returns: Success | Failure, TYPE: bool """ root = git_root() project_dirname = root.split('/')[-1] core_dir = root + '/' + 'core' config_dir = root + '/' + 'config' build_root = TMPDIR # files binary = param_dict['Executable'] control_file = param_dict['ControlFile']['Name'] compfile = param_dict['BashCompletion'] builddir = param_dict['ControlFile']['BuildDirName'] # full paths builddir_path = build_root + '/' + builddir deb_src = root + '/packaging/deb' debian_dir = 'DEBIAN' debian_path = deb_src + '/' + debian_dir binary_path = builddir_path + '/usr/local/bin' lib_path = builddir_path + '/usr/local/lib/' + PROJECT comp_src = root + '/' + 'bash' comp_dst = builddir_path + '/etc/bash_completion.d' arrow = yl + Colors.BOLD + '-->' + rst try: stdout_message( f'Assembling build directory artifacts in {bn + builddir + rst}') # create build directory if os.path.exists(builddir_path): rmtree(builddir_path) os.makedirs(builddir_path) stdout_message( message='Created builddir_path: {}'.format(yl + builddir_path + rst), prefix='OK') if not os.path.exists(builddir_path + '/' + debian_dir): copytree(debian_path, builddir_path + '/' + debian_dir) # status msg _src_path = '../' + project_dirname + debian_path.split( project_dirname)[1] _dst_path = builddir_path + '/' + debian_dir stdout_message(message='Copied: {} {} {}'.format( lk + _src_path + rst, arrow, lk + _dst_path + rst), prefix='OK') if not os.path.exists(binary_path): os.makedirs(binary_path) _dst_path = binary_path stdout_message(message='Created: {}'.format(lk + _dst_path + rst), prefix='OK') if not os.path.exists(binary_path + '/' + PROJECT): binary_src = PROJECT_ROOT + '/' + binary binary_dst = binary_path + '/' + binary copyfile(binary_src, binary_dst) # status msg _src_path = '../' + project_dirname + '/' + os.path.split( binary_src)[1] _dst_path = '../' + project_dirname + '/' + os.path.split( binary_dst)[1] stdout_message(message='Copied:\t{} {} {}'.format( lk + _src_path + rst, arrow, lk + _dst_path + rst), prefix='OK') if not os.path.exists(lib_path): os.makedirs(lib_path) # create library dir in builddir # status msg branching _dst_path = '../' + project_dirname + lib_path.split( project_dirname)[1] if os.path.exists(lib_path): stdout_message(message='Created:\t{}'.format(lk + _dst_path + rst), prefix='OK') else: stdout_message( message='Failed to create:\t{}'.format(lk + _dst_path + rst), prefix='FAIL') for libfile in os.listdir(core_dir): if os.path.exists(lib_path + '/' + libfile): stdout_message( f'{libfile} target exists - skip adding to builddir') if libfile.endswith('.log'): # log file, do not place in build logger.info(f'{libfile} is log file - skip adding to builddir') else: # place lib files in build lib_src = core_dir + '/' + libfile lib_dst = lib_path + '/' + libfile copyfile(lib_src, lib_dst) # status msg _src_path = '../' + project_dirname + lib_src.split( project_dirname)[1] _dst_path = '../' + project_dirname + lib_dst.split( project_dirname)[1] stdout_message(message='Copied:\t{} {} {}'.format( lk + _src_path + rst, arrow, lk + _dst_path + rst), prefix='OK') for confile in os.listdir(config_dir): if not os.path.exists(lib_path + '/config'): os.makedirs(lib_path + '/config') # create config dir in builddir _src = config_dir + '/' + confile _dst = lib_path + '/config/' + confile copyfile(_src, _dst) # status msg _src_path = '../' + project_dirname + _src.split( project_dirname)[1] _dst_path = '../' + project_dirname + _dst.split( project_dirname)[1] stdout_message(message='Copied:\t{} {} {}'.format( lk + _src_path + rst, arrow, lk + _dst_path + rst), prefix='OK') if not os.path.exists(comp_dst): # create path os.makedirs(comp_dst) _dst_path = '../' + project_dirname + comp_dst.split( project_dirname)[1] stdout_message(message='Created:\t{}'.format(lk + _dst_path + rst), prefix='OK') # copy for artifact in list( filter(lambda x: x.endswith('.bash'), os.listdir(comp_src))): copyfile(comp_src + '/' + artifact, comp_dst + '/' + artifact) except OSError as e: logger.exception('{}: Problem creating dirs on local fs'.format( inspect.stack()[0][3])) return False return True
def validate_imported_contigs(assembly_properties_file, config_file): config = get_args_from_private_config_file(config_file) config.update( get_args_from_assembly_properties_file(assembly_properties_file)) # We need to rename the keys because string interpolation won't work if there is a dot character in them config["assembly_report_path"] = config[ "parameters.assemblyReportUrl"].split("file:")[-1] config["assembly_md5"] = hashlib.md5( config["parameters.assemblyName"].encode("utf-8")).hexdigest() config["taxonomy_accession"] = config["parameters.taxonomyAccession"] config[ "contig_chr_mismatch_table"] = "dbsnp_ensembl_species.dbsnp_species_with_contig_chromosome_start_mismatch" config[ "contig_chr_match_table"] = "dbsnp_ensembl_species.dbsnp_species_with_contig_chromosome_start_match" final_formatting_genbank_accessions_cmd = r"cut -d$'\t' -f5 | sort | uniq | paste -s - | sed s/$'\t'/\",\"/g | sed s/$/\"/g | sed s/^/\"/g" get_contigs_start_mismatch_cmd = "psql -A -t -h {metahost} -U {metauser} -d {metadb} -c " \ "\"select distinct contig_name from {contig_chr_mismatch_table} " \ "where table_name like '%{assembly_md5}%'" \ " and schema_name like '%{taxonomy_accession}%'\" -P pager=off " \ "| grep {assembly_report_path} -f - |".format(**config) + \ final_formatting_genbank_accessions_cmd get_contigs_start_match_cmd = "psql -A -t -h {metahost} -U {metauser} -d {metadb} -c " \ "\"select distinct contig_name from {contig_chr_match_table} " \ "where table_name like '%{assembly_md5}%'" \ " and schema_name like '%{taxonomy_accession}%'\" -P pager=off " \ "| grep {assembly_report_path} -f - | grep -v assembled-molecule |".format(**config) \ + final_formatting_genbank_accessions_cmd mongo_run_command_template = "mongo --quiet --host {0} --port {1} --username {2} " \ "--password {3} --authenticationDatabase=admin " \ "{4} --eval 'db.{5}.findOne({{\"{6}\": \"{7}\", " \ "\"contig\": {{$in: [{8}]}}}})'" mismatch_contig_set = run_command_with_output( "Get contigs with start mismatch against chromosome:", get_contigs_start_mismatch_cmd).strip() match_contig_set = run_command_with_output( "Get contigs with start match against chromosome:", get_contigs_start_match_cmd).strip() collections_to_check = { "dbsnpSubmittedVariantEntity": "seq", "dbsnpClusteredVariantEntity": "asm" } if mismatch_contig_set != '""': for collection, asm_col in collections_to_check.items(): mongo_run_command = mongo_run_command_template.format( config["mongo_host"], config["mongo_port"], config["mongo_user"], config["mongo_password"], config["mongo_acc_db"], collection, asm_col, config["parameters.assemblyAccession"], mismatch_contig_set) mongo_run_command_output = run_command_with_output( "Check if mismatched contigs from above " + "are present in " + collection + " for the assembly", mongo_run_command) logger.info("Mongo command output:" + os.linesep + mongo_run_command_output) else: logger.info("No mismatch contig set available!") if match_contig_set != '""': for collection, asm_col in collections_to_check.items(): mongo_run_command = mongo_run_command_template.format( config["mongo_host"], config["mongo_port"], config["mongo_user"], config["mongo_password"], config["mongo_acc_db"], collection, asm_col, config["parameters.assemblyAccession"], match_contig_set) mongo_run_command_output = run_command_with_output( "Check if matched contigs from above " + "are present in " + collection + " for the assembly", mongo_run_command) logger.info("Mongo command output:" + os.linesep + mongo_run_command_output) else: logger.info("No matched non-chromosome contig set available!")
def print_summary(self): """ Print function of configuration :return: """ npar = self.npar logger.info("="*10 + " Config Summary " + "="*10) logger.info("Number of Inversion Par: %d" % npar) logger.info(" Par: [%s]" % (', '.join(self.par_name[0:npar]))) logger.info(" Delta: [%s]" % ( ', '.join(map(str, self.dcmt_par[0:npar]*self.scale_par[0:npar])))) logger.info("Weighting scheme") if self.weight_data: if self.weight_function == default_weight_function: logger.info(" Weighting data ===> " "Using Default weighting function") else: logger.info(" Weighting data ===> " "Using user-defined weighting function") else: logger.info(" No weighting applied") logger.info("Inversion Scheme") if self.double_couple: logger.info(" invert for double-couple source ===> " "Non-linear Inversion") elif self.zero_trace: logger.info(" invert for zero-trace source ===> " "Linear Inversion") else: logger.info(" No constraints applied ===> Linear Inversion ") logger.info(" inversion dampling lambda: %f" % self.lamda_damping)
def github(): """ Entry point for github webhook testkey""" logger.info("newnewnewcodenewcode") sha,signature = request.headers.get('X-Hub-Signature').split('=') logger.info(f'signature:{signature}'+ str(request.data)) secret = str.encode(current_app.config.get('GITHUB_SECRET')) hashhex = hmac.new(secret, request.data, digestmod='sha1').hexdigest() logger.info(f'hashhex:{hashhex}') if hmac.compare_digest(hashhex, signature): try: if 'GIT_DIR' in os.environ: del os.environ['GIT_DIR'] logger.info('hash对比正确' + current_app.config.get('REPO_PATH')) logger.info("开始拉仓库") # repo = Repo(current_app.config.get('REPO_PATH')) repo = Repo("/usr/local/python3/graduate998/backStageNew") # repo.git.pull() #--- origin = repo.remotes.origin origin.pull() #--- # 获取默认版本库 origin # remote = repo.remote() # 从远程版本库拉取分支 # remote.pull() logger.info("pull操作完成了啊!") # commit = request.json['after'][0:6] # logger.info('Repository updated with commit {}'.format(commit)) except: logger.info(traceback.format_exc()) return jsonify({"error": str(traceback.format_exc())}),500 return jsonify({}),200
def print_summary(self): """ Print summary of data container :return: """ nfiles_r = 0 nfiles_t = 0 nfiles_z = 0 nwins_r = 0 nwins_t = 0 nwins_z = 0 for window in self.window: if window.component[2:3] == "R": nfiles_r += 1 nwins_r += window.num_wins elif window.component[2:3] == "T": nfiles_t += 1 nwins_t += window.num_wins elif window.component[2:3] == "Z": nfiles_z += 1 nwins_z += window.num_wins else: raise ValueError( "Unrecognized compoent in windows: %s.%s.%s" % (window.station, window.network, window.component)) logger.info("="*10 + " Data Summary " + "="*10) logger.info("Number of Deriv synt: %d" % len(self.par_list)) logger.info(" Par: [%s]" % (', '.join(self.par_list))) logger.info("Number of data pairs: %d" % self.nfiles) logger.info(" [Z, R, T] = [%d, %d, %d]" % (nfiles_z, nfiles_r, nfiles_t)) logger.info("Number of windows: %d" % self.nwins) logger.info(" [Z, R, T] = [%d, %d, %d]" % (nwins_z, nwins_r, nwins_t)) logger.info("Loading takes %6.2f seconds" % self.elapse_time)
def print_cmtsource_summary(cmt): """ Print CMTSolution source summary :return: """ logger.info("=" * 10 + " Event Summary " + "=" * 10) logger.info("Event name: %s" % cmt.eventname) logger.info(" Latitude and longitude: %.2f, %.2f" % ( cmt.latitude, cmt.longitude)) logger.info(" Depth: %.1f km" % (cmt.depth_in_m / 1000.0)) logger.info(" Region tag: %s" % cmt.region_tag) logger.info(" Trace: %.3e" % ( (cmt.m_rr + cmt.m_tt + cmt.m_pp) / cmt.M0)) logger.info(" Moment Magnitude: %.2f" % cmt.moment_magnitude)
def inversion_result_table(self): """ Print out the inversion table :return: """ title = "*" * 20 + " Inversion Result Table(%d npar) " % \ self.config.npar + "*" * 20 logger.info(title) if not self.config.bootstrap: logger.info("PAR Old_CMT New_CMT") logger.info("Mrr: %15.6e %15.6e" % ( self.cmtsource.m_rr, self.new_cmtsource.m_rr)) logger.info("Mtt: %15.6e %15.6e" % ( self.cmtsource.m_tt, self.new_cmtsource.m_tt)) logger.info("Mpp: %15.6e %15.6e" % ( self.cmtsource.m_pp, self.new_cmtsource.m_pp)) logger.info("Mrt: %15.6e %15.6e" % ( self.cmtsource.m_rt, self.new_cmtsource.m_rt)) logger.info("Mrp: %15.6e %15.6e" % ( self.cmtsource.m_rp, self.new_cmtsource.m_rp)) logger.info("Mtp: %15.6e %15.6e" % ( self.cmtsource.m_tp, self.new_cmtsource.m_tp)) logger.info( "dep: %15.3f %15.3f" % ( self.cmtsource.depth_in_m / 1000.0, self.new_cmtsource.depth_in_m / 1000.0)) logger.info("lon: %15.3f %15.3f" % ( self.cmtsource.longitude, self.new_cmtsource.longitude)) logger.info("lat: %15.3f %15.3f" % ( self.cmtsource.latitude, self.new_cmtsource.latitude)) logger.info("ctm: %15.3f %15.3f" % ( self.cmtsource.time_shift, self.new_cmtsource.time_shift)) logger.info("hdr: %15.3f %15.3f" % ( self.cmtsource.half_duration, self.new_cmtsource.half_duration)) else: logger.info("PAR Old_CMT New_CMT " "Bootstrap_Mean Bootstrap_STD STD/Mean") logger.info( "Mrr: %15.6e %15.6e %15.6e %15.6e %10.2f%%" % ( self.cmtsource.m_rr, self.new_cmtsource.m_rr, self.par_mean[0], self.par_std[0], self.std_over_mean[0] * 100)) logger.info( "Mtt: %15.6e %15.6e %15.6e %15.6e %10.2f%%" % ( self.cmtsource.m_tt, self.new_cmtsource.m_tt, self.par_mean[1], self.par_std[1], self.std_over_mean[1] * 100)) logger.info( "Mpp: %15.6e %15.6e %15.6e %15.6e %10.2f%%" % ( self.cmtsource.m_pp, self.new_cmtsource.m_pp, self.par_mean[2], self.par_std[2], self.std_over_mean[2] * 100)) logger.info( "Mrt: %15.6e %15.6e %15.6e %15.6e %10.2f%%" % ( self.cmtsource.m_rt, self.new_cmtsource.m_rt, self.par_mean[3], self.par_std[3], self.std_over_mean[3] * 100)) logger.info( "Mrp: %15.6e %15.6e %15.6e %15.6e %10.2f%%" % ( self.cmtsource.m_rp, self.new_cmtsource.m_rp, self.par_mean[4], self.par_std[4], self.std_over_mean[4] * 100)) logger.info( "Mtp: %15.6e %15.6e %15.6e %15.6e %10.2f%%" % ( self.cmtsource.m_tp, self.new_cmtsource.m_tp, self.par_mean[5], self.par_std[5], self.std_over_mean[5] * 100)) logger.info("dep: %15.3f %15.3f %15.3f %15.3f %10.2f%%" % ( self.cmtsource.depth_in_m / 1000.0, self.new_cmtsource.depth_in_m / 1000.0, self.par_mean[6], self.par_std[6], self.std_over_mean[6] * 100)) logger.info("lon: %15.3f %15.3f %15.3f %15.3f %10.2f%%" % ( self.cmtsource.longitude, self.new_cmtsource.longitude, self.par_mean[7], self.par_std[7], self.std_over_mean[7] * 100)) logger.info("lat: %15.3f %15.3f %15.3f %15.3f %10.2f%%" % ( self.cmtsource.latitude, self.new_cmtsource.latitude, self.par_mean[8], self.par_std[8], self.std_over_mean[8] * 100)) logger.info("ctm: %15.3f %15.3f %15.3f %15.3f %10.2f%%" % ( self.cmtsource.time_shift, self.new_cmtsource.time_shift, self.par_mean[9], self.par_std[9], self.std_over_mean[9] * 100)) logger.info("hdr: %15.3f %15.3f %15.3f %15.3f %10.2f%%" % ( self.cmtsource.half_duration, self.new_cmtsource.half_duration, self.par_mean[10], self.par_std[10], self.std_over_mean[10] * 100))
def stats_energy(self): logger.info('Energy Search...') self.calculate_misfit_for_m00(1.00)
def one(*ignore): compute = Compute(args.strategy) logger.info(compute.attempt_provision('create', prefer_provider=args.provider, prefer_image=args.image))
def _build_parser(): parser = ArgumentParser(description='Create compute nodes') parser.add_argument('-s', '--strategy', help='strategy file [strategy.sample.json]', default=config_join('strategy.sample.json')) parser.add_argument('-n', '--number_of_nodes', help='number of nodes to create [1]', default=1, type=int) parser.add_argument('--provider', help='Try this provider first') parser.add_argument('--image', help='Try this image first') return parser if __name__ == '__main__': args = _build_parser().parse_args() def one(*ignore): compute = Compute(args.strategy) logger.info(compute.attempt_provision('create', prefer_provider=args.provider, prefer_image=args.image)) logger.info(('Provisioning {} node'.format(args.number_of_nodes) + 's' if args.number_of_nodes > 1 else '')) if args.number_of_nodes == 1: one() else: p = Pool(args.number_of_nodes) p.map(one, xrange(args.number_of_nodes))
def print_inversion_summary(self): """ Print out the inversion summary :return: """ logger.info("*" * 20) logger.info("Invert cmt parameters(%d par)" % self.config.npar) logger.info("Old CMT par: [%s]" % ( ', '.join(map(str, self.cmt_par)))) logger.info("dm: [%s]" % ( ', '.join(map(str, self.new_cmt_par - self.cmt_par)))) logger.info("New CMT par: [%s]" % ( ', '.join(map(str, self.new_cmt_par)))) logger.info("Trace: %e" % (np.sum(self.new_cmt_par[0:3]))) logger.info("Energy change(scalar moment): %5.2f%%" % ( (self.new_cmtsource.M0 - self.cmtsource.M0) / self.cmtsource.M0 * 100.0)) self.inversion_result_table()
def test_second(): global supported_commands try: tests_file_path = 'tests_descriptions.json' logger.info( f'---------------------- run test {tests_file_path} ----------------------' ) saved_values = {} test_data = {} with open(tests_file_path, 'r', encoding='utf-8') as file: test_data = json.load(file) for query in test_data: if query['commands_before_running'] != {}: logger.info( f"commands_before_running: {query['commands_before_running']}" ) for command in query['commands_before_running']: logger.info( f"run: {query['commands_before_running']}{query['commands_before_running'][command]}" ) supported_commands[command]['func']( **(query['commands_before_running'][command])) if 'saved_before' in query['params'].keys(): for type_param, array_name_params in query['params'][ 'saved_before'].items(): for name in array_name_params: query['params'][type_param][name] = saved_values[name] if 'saved_before' in query['custom_headers'].keys(): for name_header in query['custom_headers']['saved_before']: query['custom_headers']['headers'][ name_header] = saved_values[name_header] # print(f"params: {query['params']}") # print(f"custom_headers: {query['custom_headers']}") url = str(query['url']).format(**query['params']['path']) response = http_query.http_query_by_type[str( query['method']).lower()]( url=url, headers=query['custom_headers']['headers'], query_params=query['params']['query'], json_data=query['request_body_json'], ) # checks.check_mimetype(mimetype=response.content_type) checks.check_mimetype(mimetype=response.headers['content-type']) checks.check_http_code(code=response.status_code) response_data_json = response.json() checks.check_structure_successful_response( response=response_data_json) for scan_area, pattern in query['checks'].items(): if pattern is not None and pattern != {} and pattern != []: area = str(scan_area).replace('in_', '') ok, res_comparison = compare(pattern, response_data_json[area]) assert ok is True, f'A result that does not match the pattern was found.\nResult of comparing the url [{url}] response:\n{res_comparison}' for param_for_save in query['saves_from_body']: saved_values[param_for_save] = response_data_json['body'][ param_for_save] # show_dialog() return True except AssertionError as ex: logger.error(ex) print(f'[Error]{ex}') raise MyValueError(f'Тест не пройден: {ex}')
def stats_tshift(self): logger.info("Origin time search...") self.calculate_tshift()
def invert_cmt(self): """ ensemble all measurements together to form Matrix A and vector b to solve the A * (dm) = b A is the Hessian Matrix and b is the misfit :return: """ logger.info("*"*15) logger.info("CMT Inversion") logger.info("*"*15) # ensemble A and b A = util.sum_matrix(self.weight_array, self.A1_all) b = util.sum_matrix(self.weight_array, self.b1_all) logger.info("Inversion Matrix A is as follows:") logger.info("\n%s" % ('\n'.join(map(self._float_array_to_str, A)))) logger.info("Condition number of A: %10.2f" % (np.linalg.cond(A))) logger.info("RHS vector b is as follows:") logger.info("[%s]" % (self._float_array_to_str(b))) # source inversion self.new_cmt_par = self.invert_solver(A, b, print_mode=True) self.convert_new_cmt_par()
def destroy(config_filename, restrict_provider_to=None): with open(config_filename, 'rt') as f: config_contents = f.read() config_dict = loads(replace_variables(config_contents)) del config_contents providers = tuple(obj for obj in config_dict['provider']['options'] if obj['provider']['name'] in restrict_provider_to or obj['provider']['name'] == restrict_provider_to) if restrict_provider_to \ else tuple(obj for obj in config_dict['provider']['options']) client = (lambda etcd_server_location: Client( protocol=etcd_server_location.scheme, host=etcd_server_location.hostname, port=etcd_server_location.port ))(urlparse(config_dict['etcd_server'])) provider2conf_and_driver = dict( imap(lambda provider_dict: (provider_dict['provider']['name'], namedtuple('_', 'conf driver_cls')( provider_dict, (lambda provider_cls: provider_cls( region=provider_dict['provider']['region'], **provider_dict['auth'] ))(get_driver( getattr(Provider, provider_dict['provider']['name']) if hasattr(Provider, provider_dict['provider']['name']) else itemgetter(1)(next(ifilter( lambda (prov_name, value): value == provider_dict['provider'][ 'name'].lower(), imap(lambda prov_name: (prov_name, getattr(Provider, prov_name)), dir(Provider)) ))) )))), providers) ) # Map nodes to their provider, including ones outside etcd provider2nodes = { provider: tuple( namedtuple('_', 'uuid node')(node.uuid, node) for node in driver.driver_cls.list_nodes(*((driver.conf['create_with']['ex_cloud_service_name'],) if driver.conf['provider']['name'] == 'AZURE' else tuple() )) if driver.driver_cls.NODE_STATE_MAP and node.state in ( driver.driver_cls.NODE_STATE_MAP.get( 'running', next((node.state for k, v in driver.driver_cls.NODE_STATE_MAP.iteritems() if 'running' in v), None) ), driver.driver_cls.NODE_STATE_MAP.get('active') ) or not driver.driver_cls.NODE_STATE_MAP and node.state in ('running',) ) for provider, driver in provider2conf_and_driver.iteritems()} uuid2key = {loads(client.get(key).value)['uuid']: key for key in flatten(etcd_ls(client)) if (lambda v: isinstance(v, basestring) and v.startswith('{'))(client.get(key).value)} # TODO: Only call `client.get` once per `key` ^ # Filter to just ones inside etcd; then deprovision and delete from etcd logger.info('Dropped: {}'.format( { provider: tuple(imap(lambda n: rm_prov_etcd(client, n.node), nodes)) for provider, nodes in provider2nodes.iteritems() for node in nodes if node.uuid in uuid2key } )) # Delete all empty etcd directories. for i in xrange(20): # TODO: walk the tree rather than hackily rerun it_consumes( logger.info('rmdir {directory}'.format(directory=directory, res=client.delete(directory, dir=True))) for directory in flatten(etcd_empty_dirs(client)) ) return client
def print_summary(self): """ Print function of configuration :return: """ logger.info("="*10 + " Config Summary " + "="*10) logger.info("Origin time inversion: %s" % self.origin_time_inversion) logger.info("Energy inversion: %s" % self.energy_inversion) logger.info("Weighting scheme") if self.weight_data: if self.weight_function == default_weight_function: logger.info(" Weighting data ===> Using Default " "weighting function") else: logger.info(" Weighting data ===> Using user-defined" "weighting function") else: logger.info(" No weighting applied") if self.origin_time_inversion: logger.info("Time start, grid ratio and end: [%6.3f %6.3f]" % (self.t00_s, self.t00_e)) if self.energy_inversion: logger.info("Energy start, grid, and end : [%6.3f %6.3f %6.3f]" % (self.m00_s, self.dm00, self.m00_e))
def get_extracts(self): logger.info('extracts: {}'.format(self.extracts)) onlydirs = [ f for f in listdir(self.extracts) if isdir(join(self.extracts, f)) ] return onlydirs