def update_queues_conf_file(filename, queues_info): # read specific slurm.conf if not os.path.isfile(filename): logger.error("%s is not exists", filename) return -1 partitions_conf = get_partition_conf_from_queues(queues_info) logger.info("partition conf will be %s", partitions_conf) partition_begin = False buf = StringIO.StringIO() with open(filename, 'r') as f: for line in f: if line.startswith("PartitionName=controller ") and \ partition_begin is False: partition_begin = True buf.write(line) elif partition_begin: if not line.startswith("PartitionName="): partition_begin = False # put Slurm Partition conf here buf.write(partitions_conf) buf.write("\n") else: # skip current Partition conf logger.info("current partition conf %s is skipped", line) else: buf.write(line) content = buf.getvalue() buf.close() write_file(filename, content) return 0
def get_run_activitys(self): ''' 获取运行的Activitie列表 :return:个数,接口列表 ''' startnumber = None endnumber = None actlen = None actlist = [] try: with open(self.monkeylog) as f_w: result = f_w.read() if re.findall("Total activities", result) and re.findall("How many Events Dropped", result): with open(self.monkeylog) as f_w: for number,line in enumerate(f_w.readlines()): if re.findall("Total activities",line): actlen = line.split("Total activities")[1].replace("\n","").strip() startnumber = number + 1 elif re.findall("How many Events Dropped",line): endnumber = number act_result = linecache.getlines(self.monkeylog)[startnumber:endnumber] for act in act_result: actlist.append(str(act).split("-")[1].replace("\n","").strip()) else: logger.info("{}文件中未查询到activity列表".format(self.monkeylog)) common.write_file(run_activity_path, actlist) except Exception as e: logger.error('获取activity列表异常:{}'.format(e)) finally: return actlen, actlist
def channel_playlist(): channels, _code = _fetch_channels() if not channels: return _code, -1, -1 channels_sorted = sorted(channels.values(), key=lambda _channel: _channel.weight) if cfg.channel_group == 1: group = c.default_group_name else: group = cfg.channel_group_name if cfg.my_script == 1: streamer = c.pipe + os.path.join(cfg.playlist_path, cfg.my_script_name) else: streamer = c.pipe + os.path.join(cfg.playlist_path, cfg.playlist_streamer) playlist_src = '#EXTM3U\n' playlist_dst = '#EXTM3U\n' _num = 0 _err = 0 for channel in channels_sorted: try: _log("Adding: " + c.to_string(channel.name)) playlist_src += '#EXTINF:-1, %s\n%s\n' % (c.to_string(channel.name), c.to_string(channel.url())) playlist_dst += c.build_channel_lines(channel, cfg.channel_logo ,_logo_path_file(channel.name), streamer, group, cfg.playlist_type, cfg.channel_epg_name, cfg.channel_epg_id, cfg.channel_group) _num += 1 except ChannelIsNotBroadcastingError: _err += 1 _log("... Not broadcasting. Skipped.") except AuthenticationError: return c.authent_error, 0, 0 except TooManyDevicesError: return c.toomany_error, 0, 0 c.write_file(playlist_src, os.path.join(cfg.playlist_path, cfg.playlist_src), _log) c.write_file(playlist_dst, os.path.join(cfg.playlist_path, cfg.playlist_dst), _log) return 'OK', _num, _err
def collect_html(args): """ Collect html data locally, so you don't have to keep hitting the url. :param args: args for collect html :type args: Namespacd """ url_list = args.url_list output_dir = args.output_dir print(url_list) # do some checks try: assert os.path.exists(url_list), 'url_list must exist' assert os.path.exists(output_dir), 'output_dir must exist' except AssertionError as err: logger.error('Failed check: {}'.format(err)) return urls = common.read_file(url_list) for url in urls: logger.debug(url) html = spy_tools.collect_html(url) out = url.split('/') output = os.path.join(output_dir, out[-1] + '.html') common.write_file(html, output)
def main(): #argparser = argparse.ArgumentParser(description="Compares two csv files and produces a result file.") #argparser.add_argument("files", metavar="F", type=open, nargs="+", help="files to process") #args = argparser.parse_args() #print(args) print(sys.argv) if len(sys.argv) < 5: print("Usage: main.py <first_file> <second_file> <output_file> <second_file_format> [-e]") print(" Use a -e parameter to print errors during parsing.") sys.exit(1) second_file_parser = get_parser(sys.argv[4]) print("The '{0}' parser has been selected for the second file by format '{1}'". format(second_file_parser.type, sys.argv[4])) print_errors = len(sys.argv) > 5 and sys.argv[5] == '-e' first_list = common.parse_file(sys.argv[1], rigla.Parser(), print_errors) second_list = common.parse_file(sys.argv[2], second_file_parser, print_errors) list_of_compared_results = comparer.Comparer(first_list, second_list).compare() common.write_file(sys.argv[3], list_of_compared_results.not_in_first_list, list_of_compared_results.not_in_second_list, list_of_compared_results.in_both_lists)
def create_env_script(self): common.logit("\nCreating VO frontend env script.") data = """#!/bin/bash source %(condor_location)s/condor.sh """ % { "condor_location" : self.condor_location(),} common.write_file("w",0644,self.env_script(),data) common.logit("VO frontend env script created: %s" % self.env_script() )
def get_ca_cert(fb, gid): # Make a Facebook query call to fetch data results = fb.fql.query('SELECT data FROM SecureGridNet.ipopdata \ WHERE _id IN (SELECT obj_id FROM SecureGridNet.certificate \ WHERE gid = ' + gid + ')') # Get the last result, there should be only one for result in results: data = result['data'] # Return if certificate is not found if(data == ''): return -1 # Certificate is base64 encoded, so it needs to be decoded cert = common.fb_decode(data) # Save certificate to floppy image common.write_file('/mnt/fd/cacert.pem', cert) # Set certificate directory rdir = "/etc/racoon/certs" # Prepare ca key for racoon os.system("cp -f /mnt/fd/cacert.pem " + rdir + "/.") os.system("ln -sf " + rdir + "/cacert.pem " + rdir + "/`openssl x509 -noout \ -hash -in " + rdir + "/cacert.pem`.0") return 0
def __create_condor_config__(self): """ This first updates the primary condor_config with either: a. the gwms condor_config file if a tarball install b. the config.d directory containing the gwms config files Then it creates the individual condor config files. """ #if len(self.colocated_services) > 0: # return # we've already updated this common.logit("... updating: %s" % self.condor_config()) common.logit(" to point to GWMS config files directory") cfg_data = """ ######################################################## # Using local configuration file directory below ######################################################## LOCAL_CONFIG_FILE = LOCAL_CONFIG_DIR = %s """ % (self.local_config_dir()) common.write_file("a",0644,self.condor_config(),cfg_data,SILENT=False) stdout = glideinwms.lib.subprocessSupport.iexe_cmd("tail -5 %s" % self.condor_config()) common.logit(stdout) common.logit("\nCreating GWMS condor_config files in:") common.logit("%s" % self.local_config_dir()) common.make_directory(self.local_config_dir(),self.username(),0755) types = self.condor_config_data.keys() types.sort() for type in types: filename = "%s/%s.config" % (self.local_config_dir(),type) common.logit(" %s" % os.path.basename(filename)) common.write_file("w",0644,filename,self.condor_config_data[type],SILENT=True) self.__create_secondary_schedd_dirs__()
def check_cert(fb): # Get object id from file os.system obj_id = common.read_file('fb_req_obj_id.txt') # Check Facebook for certificate results = fb.fql.query('SELECT id, data FROM SecureGridNet.ipopdata \ WHERE _id IN (SELECT obj_id FROM SecureGridNet.certificate \ WHERE gid = ' + obj_id + ')') # Get result from Facebook database id = '' data = '' for result in results: id = result['id'] data = result['data'] # Return if certificate is not found if(data == ''): return -1 # Check to see if the certificate is for the current user if(id != obj_id): return -1 # Decode data to cert cert = common.fb_decode(data) # Write certificate to file common.write_file('/etc/racoon/certs/host-cert.pem', cert) return 0
def plot_vehicle_status(): fig, axs = plt.subplots(3, figsize=(8, 4)) # fig.set_size_inches(18.5, 10.5) plt.subplots_adjust(hspace=0.5, top=0.95, bottom=0.05) ax = axs[1] ax2 = axs[0] # ax = fig.gca() # ax2 = ax.twiny() ax.set_yticks([]) ax2.set_yticks([]) ax.set_xlim([0, 140]) ax2.set_xlim([0, 7500]) ax.set_xlabel("Speed (KM/H)") ax2.set_xlabel("RPM") # plt.show() cur_spd_rect = ax.barh(0.05, 1, color = 'cyan', height = 0.25, edgecolor='none') # set_spd_rect = ax.barh(0.25, 1, color = 'b', height = 0.05, edgecolor='none') rpm_rect = ax2.barh(0.6, 1, color = 'tomato', height = 0.25, edgecolor='none') axamp = axs[2] samp = Slider(axamp, 'Set Spd', 0.0, 140.0, valinit=0) gear_label = plt.text(70, 3.3, " " ,ha='center', va='center', size='x-large', weight='bold') while(1): set_spd = int(samp.val) write_file("set_speed.txt", set_spd) cur_spd_rect.patches[0].set_width(read_file('cur_speed.txt', float)) rpm_rect.patches[0].set_width(read_file('rpm.txt', float)) gear_label.set_text("Gear %d" % read_file('gear.txt', int)) plt.pause(0.2)
def channel_playlist(): global _channel_group_, _channel_groupname_, _myscript_, _myscript_name_, _channel_logo_, \ _playlist_type_, _channel_epgname_, _channel_epgid_, _ffmpeg_, _no_error_ channels, _code = _fetch_channels() if not channels: return _code, -1, -1 channels_sorted = sorted(channels.values(), key=lambda _channel: _channel.weight) if _channel_group_ == 1: group = c.default_group_name else: group = _channel_groupname_ if _myscript_ == 1: streamer = c.pipe + os.path.join(_playlist_path_, _myscript_name_) else: streamer = c.pipe + os.path.join(_playlist_path_, _playlist_streamer_) playlist_src = '#EXTM3U\n' playlist_dst = '#EXTM3U\n' _num = 0 _err = 0 for channel in channels_sorted: try: log_not("Adding: " + channel.name) playlist_src += '#EXTINF:-1, %s\n%s\n' % (c.to_string( channel.name), c.to_string(channel.url())) playlist_dst += c.build_channel_lines( channel, _channel_logo_, _logo_path_file(channel.name), streamer, group, _playlist_type_, _channel_epgname_, _channel_epgid_, _channel_group_) _num += 1 except ChannelIsNotBroadcastingError: log_not("... Not broadcasting. Skipped.") _err += 1 except AuthenticationError: return _authent_error_, 0, 0 except TooManyDevicesError: return _toomany_error_, 0, 0 except NoPlaylistUrlsError: log_not("... No playlist URL provided. Skipped.") _err += 1 c.write_file(playlist_src, xbmc.translatePath(os.path.join(_profile_, _playlist_src_)), _log_dbg) c.write_file(playlist_dst, xbmc.translatePath(os.path.join(_profile_, _playlist_dst_)), _log_dbg) if _playlist_type_ == 3: c.write_streamer( xbmc.translatePath(os.path.join(_profile_, _playlist_streamer_)), xbmc.translatePath(os.path.join(_profile_, _playlist_src_)), _ffmpeg_, _log_dbg) set_setting('last_time', time.strftime('%Y-%m-%d %H:%M')) set_setting('last_downloaded', c.to_string(_num)) set_setting('last_skipped', c.to_string(_err)) return _no_error_, _num, _err
def get_new_config_entries(self): """This method is intended to retrieve new configuration file entry element after the initial installation is complete. It will create a file containing the selected entry points that can be merged into the existing Factory configuration file. """ self.get_config_entries_data() filename = "%s/new_entries.%s" % (self.config_dir(),common.time_suffix()) common.write_file("w",0644,filename,self.config_entries_data())
def create_ipop_config(path, gid) : data = '' data += '<IpopConfig>' data += '<IpopNamespace>' + gid + '</IpopNamespace>' data += '<VirtualNetworkDevice>tapipop</VirtualNetworkDevice>' data += '<EnableMulticast>false</EnableMulticast>' data += '</IpopConfig>' common.write_file(path, data) return 0
def create_env_script(self): common.logit("\nCreating VO frontend env script.") data = """#!/bin/bash . %(condor_location)s/condor.sh export PYTHONPATH=$PYTHONPATH:%(install_location)s/.. """ % { "condor_location" : self.condor_location(), "install_location" : self.glideinwms_location(),} common.write_file("w",0644,self.env_script(),data) common.logit("VO frontend env script created: %s" % self.env_script() )
def main(): config = sys.argv[1] data = read_files(config).split("\n") remove_list = list() for line in data: remove_list = check_hdfs_dirs(line) for item in remove_list: write_file(output_file, item) remove_hdfs_dirs(item)
def create_cert(): # Checks to see if path exists if(not os.path.exists('demoCA/cacert.pem')): os.system('/usr/lib/ssl/misc/CA.pl -newca') # Create blank signed request file common.write_file('signed_cert_ids.txt','') return 0
def applyChanges(self): """ Redefined from *EditionWidget* class. """ self.prev_state = _text2unicode(self.editor.toPlainText()) try: write_file(self.file_name, self.prev_state) except IOError: message = translate("AsterStudy", "Cannot write file.") Q.QMessageBox.critical(self, "AsterStudy", message)
def modify_and_push(self): test_file = os.path.join("php","test.php") common.write_file(os.path.join(self.app_name, test_file), "<?php phpinfo(); ?>") cmd = "cd %s && git add %s && git commit -m 'test' -a && git push" % (self.app_name, test_file), (status, output) = common.cmd_get_status_output(cmd) if (status == 0): return 0 else: self.error(output) return 1
def accept(self): """ Redefined from *Dialog* class. """ text = _text2unicode(self.editor.toPlainText()) try: write_file(self.file_name, text) except IOError: message = translate("AsterStudy", "Cannot write file.") Q.QMessageBox.critical(self, "AsterStudy", message) super(TextFileDialog, self).accept()
def get_new_config_group(self): """This method is intended to create a new group element after the initial installation is complete. It will create a file containing the group and job selection/matchin criteria. This can then be manually merged into the existing frontend configuration file. """ filename = "%(config_dir)s/%(group)s.%(time)s" % \ { "config_dir" : self.config_dir(), "group" : self.group_name(), "time" : common.time_suffix(),} common.write_file("w",0644,filename,self.get_match_criteria())
def download_file(reqs_ses, url, filepath): res = common.fetch( requests_session=reqs_ses, url=url, method='get', expect_status=200, ) common.write_file( # Store page to disk file_path=filepath, data=res.content) logging.debug('Saved {0} to {1}'.format(url, filepath)) return
def fb_store_req(fb, cert_req): # Prepare data for Facebook data = common.fb_encode(cert_req) # Put data on Facebook obj_id = common.fb_put_data(fb, fb.uid, data, 'request') # Write object_id to file common.write_file('fb_req_obj_id.txt', str(obj_id)) return 0
def create_env_script(self): """This creates an "env" type script that must be used before starting the factory. """ common.logit("Creating environment script...") data = """#!/bin/bash export X509_CERT_DIR=%(x509_cert_dir)s . %(condor_location)s/condor.sh """ % { "x509_cert_dir" : self.wms.x509_cert_dir(), "condor_location" : self.wms.condor_location(),} common.write_file("w",0644,self.env_script(),data) common.logit("%s\n" % data)
def __create_initd_script__(self): if self.client_only_install == True: common.logit("... client only install. No startup initd script required.") return if self.install_type() == "rpm": common.logit("... This is an 'rpm' install. An initd script already exists.") return common.logit("") common.logit("Creating startup /etc/init.d script") common.logit(" %s" % self.initd_script()) data = self.__initd_script__() common.write_file("w",0755,self.initd_script(),data,SILENT=True)
def train_data(args): """ Train the model. :param args: args for trin_data :type args: Namespace """ # pull args out training_dir = args.training_dir train_file = args.train_file eval_file = args.eval_file epochs = args.epochs continue_training = args.keep_training checkpoint_dir = args.checkpoint_dir #truncate = args.truncate vocab = args.vocab # do some checks try: if continue_training: assert checkpoint_dir is not None, 'to continue training you must give checkpoint_dir' assert os.path.exists(checkpoint_dir), 'checkpoint_dir must exist' assert epochs > 0, 'epochs must be positive' assert os.path.exists(train_file), "train_file must exist" assert os.path.exists(training_dir), "training_dir must exist" assert os.path.exists(eval_file), "eval_file must exist" assert os.path.exists(vocab), "vocab file must exist" #assert truncate > 0, 'truncate must be positive' except AssertionError as err: logger.error("Failed check: {}".format(err)) return # get train and eval data train_data, train_labels = process_data.process_data(train_file, vocab) test_data, test_labels = process_data.process_data(eval_file, vocab) # make the checkpoint directory if not continue_training and checkpoint_dir is None: checkpoint_dir = common.grab_next_session(training_dir) # train the model history, model_summary = rnn.train_and_validate(train_data, train_labels, test_data, test_labels, epochs, checkpoint_dir, continue_training) # plot the data if not continue_training: common.write_file(model_summary, checkpoint_dir + "/MODEL_SUMMARY") common.plot_graphs_val(history, 'categorical_accuracy', checkpoint_dir) common.plot_graphs_val(history, 'loss', checkpoint_dir)
def create_env_script(self): """This creates an "env" type script that must be used before starting the factory. """ common.logit("Creating environment script...") data = """#!/bin/bash export X509_CERT_DIR=%(x509_cert_dir)s source %(condor_location)s/condor.sh """ % { "x509_cert_dir" : self.wms.x509_cert_dir(), "condor_location" : self.wms.condor_location(),} if self.use_vofrontend_proxy() == "n": data += "export X509_USER_PROXY=%s" % self.x509_proxy() common.write_file("w",0644,self.env_script(),data) common.logit("%s\n" % data)
def create_word2vec_model(): mecab = MeCab.Tagger('-Owakati') nouns = [] articles = {} # text from NHK NEWS WEB urls = [ 'http://www3.nhk.or.jp/rss/news/cat' + str(i) + '.xml' for i in range(8) ] nhk = '' for url in urls: html = urllib.request.urlopen(url) soup = BeautifulSoup(html, 'html.parser') items = soup.find_all('item') for i in items: item_soup = BeautifulSoup(str(i), 'html.parser') text = remove_tag(item_soup.title, 'title') + ' ' + remove_tag( item_soup.description, 'description') nhk += text articles[remove_tag(item_soup.title, 'title')] = { 'url': remove_tag(item_soup.link, 'link'), 'nouns': list(set(extract_nouns(text))) } nhk_nouns = extract_nouns(nhk) write_json_file(list(set(nhk_nouns)), 'nhk-nouns') write_json_file(articles, 'nhk-articles') nouns.extend(nhk_nouns) # text from twitter user time-line tweets = api.user_timeline(count=200) twitter = '' for tweet in tweets: tweet = process_tweet(tweet) if tweet: twitter += tweet + '\n' twitter_nouns = extract_nouns(twitter) nouns.extend(twitter_nouns) write_json_file(list(set(twitter_nouns)), 'twitter-nouns') write_json_file(list(set(nouns)), 'nouns') write_file(mecab.parse(nhk + twitter), 'wakati.txt') data = word2vec.Text8Corpus('wakati.txt') model = word2vec.Word2Vec(data, size=200, min_count=0) return model
def parse_formula_sheet(sh, output_file): all_fun = begin for row in range(2, sh.nrows): skill_id_str = get_str_from_sheet(sh, row, 0) skill_name_str = get_str_from_sheet(sh, row, 1) skill_formula_str = get_str_from_sheet(sh, row, 2) fun_str = parse_function(skill_formula_str, u"技能名:" + skill_name_str, u"ser_%s" % skill_id_str, dicVar, srcSkillFormla) all_fun += fun_str # src_file = path + "/" + skill_id_str + ".c" # write_src(output_file, begin, end, fun_str, "utf-8") all_fun += end write_file(output_file, all_fun)
def sign_cert_req(req): # Write req to file fname = 'tmp_cert_req.pem' fname_cert = 'newcert.pem' common.write_file(fname, req) # Sign the certificate os.system('openssl ca -batch -policy policy_anything -key secret \ -in ' + fname + ' -out ' + fname_cert) # Read certificate signed_cert = common.read_file(fname_cert) return signed_cert
def __create_condor_mapfile__(self,users): """ Creates the condor mapfile for GSI authentication""" if self.client_only_install == True: common.logit( "... No Condor mapfile file needed. Client only install") return mapfile_entries = self.__condor_mapfile_entries__(users) filename = self.condor_mapfile() common.logit("... creating Condor mapfile") common.logit(" %s" % filename) common.make_directory(os.path.dirname(filename),pwd.getpwuid(os.getuid())[0],0755) mapfile_entries += """GSI (.*) anonymous FS (.*) \\1 """ common.write_file("w",0644,filename,mapfile_entries,SILENT=True) common.logit("\nCondor mapfile entries:") common.logit("%s" % mapfile_entries)
def save_followed_file(req_ses, followed_list_path): logging.info('Saving list of followed blogs') followed_list_path = os.path.join('dl', 'followed.opml') # Get followed list file # www.tumblr.com/following.opml logging.debug('Loading followed list') followed_res = common.fetch( requests_session=req_ses, url='https://www.tumblr.com/following.opml', method='get', ) common.write_file( # Save to file file_path=followed_list_path, data=followed_res.content) logging.info('Saved list of followed blogs') return
def main(): template_data = common.read_file(ALL_TESTS_TEMPLATE_FILE) template = Template(template_data) test_html_paths = [] for file_name in common.get_files_with_suffix( common.GENERATED_TEST_BASE_PATH, '_test.html'): test_html_paths.append(" '%s/%s'," % (GRPC_WEB_BASE_DIR, file_name)) # Example output paths: # 'packages/grpc-web/generated/test_htmls/javascript__net__grpc__web__grpcwebclientbase_test.html', # 'packages/grpc-web/generated/test_htmls/javascript__net__grpc__web__grpcwebstreamparser_test.html', test_html_paths_str = "\n".join(test_html_paths) # Writes the generated output to the all_tests.js file. common.write_file(GENERATED_ALL_TESTS_JS_PATH, template.substitute(test_html_paths=test_html_paths_str))
def create_values(hostip, port, section, time_key, db): dest_dir = "%s/%s" % (base_dir, section) create_current_dir(dest_dir) check_key = time_key out_dict = get_redis_info(hostip, port) write_info_disk(out_dict, dest_dir) lasttime = return_redis_keys(hostip, port, db, check_key) timeout_files = os.path.join(dest_dir, 'sync_timeout') if not lasttime or diff_time(lasttime) > 600: write_file(timeout_files, lasttime or '1') else: if os.path.isfile(timeout_files): os.remove(timeout_files) if int(out_dict['connected_clients']) > 1000: redis_connect_high(hostip, port, dest_dir)
def main(): dirs = sys.argv[1] expired_values = ['20170227', '20170226', '20170225', '20170224', '20170223'] expired_dirs = get_expired_list(dirs, expired_values) # 计算出的过期日志写入文件 expired_log = "expired.log" for dirs in expired_dirs: write_file(expired_log, dirs) # mv 过期目录到另一个目录 suffix = "/data/temp/backup/hdfs_backup" target = "/data/backups/expired" for dirs in expired_dirs: target_dirs = dirs.replace(suffix, target)[:-9] mv_cmd = "mv %s %s" % (dirs, target_dirs) print(mv_cmd) # run_cmd(mv_cmd) print "all expired files remove"
def _gen_test_html(js_file_path: str, template: Template): """Generates a Closure test wrapper HTML and saves it to the filesystem.""" # Generates the test_file_name so that: # ../../javascript/net/grpc/web/grpcwebclientbase_test.js # will now be named: # javascript__net__grpc__web__grpcwebclientbase_test.html test_file_name = js_file_path while test_file_name.startswith('../'): test_file_name = test_file_name[3:] test_file_name = test_file_name.replace('/', '__') test_file_name = os.path.splitext(test_file_name)[0] + '.html' # Generates the test HTML using the package name of the test file package_name = _extract_closure_package(js_file_path) generated_html = template.substitute(package=package_name) # Writes the test HTML files common.write_file(common.GENERATED_TEST_BASE_PATH + test_file_name, generated_html)
def update_cmpnodes_res_conf(filename): """ update compute node resource in slurm configuration :param filename: :return: """ # read slurm.conf if not os.path.isfile(filename): logger.error("%s is not exists", filename) return -1 cluster_info = get_cluster_info() # 1. get compute node resource ctl_resource = "" # the first line in resource.info cmp_resource = "" # the last line in resource.info with open(RESOURCE_INFO_FILE, "r") as info: lines = info.readlines() for line in lines: if line: if not ctl_resource: ctl_resource = line else: cmp_resource = line # get modified compute node node_list = get_slurm_cmpnode_list(cluster_info) buf = StringIO.StringIO() with open(filename, 'r') as f: for line in f: if line.startswith("NodeName=node"): # put Slurm Partition conf here line = NODE_RESOURCE_FMT.format(node_list, cmp_resource) buf.write("\n") buf.write(line) content = buf.getvalue() buf.close() write_file(filename, content) return 0
def start(init_url): urls = [] # init_url = input('specify an area: ') url_pattern = re.search(r'-g\d+-', init_url) url_pos = url_pattern.start() + len(url_pattern.group(0)) page_number = find_max_page(common.load_soup_online(init_url)) print('locations in {} pages.'.format(page_number)) for idx_page in range(page_number): page_url = init_url if idx_page == 0 else ''.join( [init_url[:url_pos], 'oa', str(idx_page * 20), '-', init_url[url_pos:]]) print('[page {}] {}'.format(idx_page + 1, page_url)) soup = common.load_soup_online(page_url) for hotel in soup.findAll('div', class_='geo_name'): a = hotel.find('a') a = common.TA_ROOT + a['href'][1:] urls.append(a) urls = list(set(urls)) print('{} locations found.'.format(len(urls))) common.write_file('locations', ';'.join(urls))
def run_client_tasks(fb, gid, task): # Task 1: Get CA certificate from Facebook if(task == '1' or task == '0') : print 'Getting CA certificate from Facebook' if(get_ca_cert(fb, gid) == -1): print 'CA certificate now found for this group, terminating' return print 'CA certificate retreived successfully' # Task 2: Make certificate request if(task == '2' or task == '0'): print 'Making certificate request' make_cert_request(fb) print 'Request made successfully' # Loop task 3 until certificate is found while(True): # Task 3: Check Facebook for certificate if(task == '3' or task == '0'): print 'Checking Facebook for certificate' if(check_cert(fb) == 0): print 'Certificate retreived successfully' break else: print 'Certificate not retreived' # Wait for 60 seconds before checking Facebook again sleep(60) # Task 4: Configure IPOP and IPSec if(task == '4' or task == '0'): print 'Configuring virtual private network' configure_ipop(gid) configure_ipsec() common.write_file('/mnt/fd/fb_done', '') print '<h2>Virtual private network configured successfully<h2/>' return 0
def main(): api_key = 'd33a7f5305f6ff7dd4a603f9049f87f3' secret = '2b7b550dea45ccea6ada02b0fafb38c5' # Create facebook object fb = facebook.Facebook(api_key, secret) # Get session information fb.session_key = sys.argv[1] fb.uid = sys.argv[2] gid = sys.argv[3] node_type = sys.argv[4] # Store type to fileos.system common.write_file('type', node_type) # Run CA tasks run_ca_tasks(fb, gid, '0') return
def update(self): common.logit("\n--- Privilege Separation is in effect ---\nThe following directories/files are being created to support this.") #-- some validation on the condor install --- if not os.path.isdir(self.condor_location): common.logerr("The CONDOR_LOCATION specified does not exist: %s" % self.condor_location) #--- check for Condor switchboard --- if not os.path.isfile(self.switchboard_bin): common.logerr("Privilege separation binary (%s) does not exist. Do you have the right version of Condor?" % self.switchboard_bin) if os.stat(self.switchboard_bin)[stat.ST_UID] != 0: common.logerr("Privilege separation binary (%s) must be owned by root!" % self.switchboard_bin) #-- create the config file --- common.logit("... creating condor config file: %s" % (self.config_file)) if not os.path.isdir(os.path.dirname(self.config_file)): os.mkdir(os.path.dirname(self.config_file)) common.write_file("w",0644,self.config_file,self.config_data()) #-- setuid on swtichboard --- common.logit("... changing permissions on %s to %s" % (self.switchboard_bin,"04755")) os.chmod(self.switchboard_bin,04755) #-- create factory directories --- #-- factory dirs done in Factory install -- # self.factory.create_factory_dirs(self.factory.username(),0755) self.create_factory_client_dirs('root',0755) common.logit("--- End of updates for Privilege Separation.--- ")
def parse_formula_sheet(sh, output_file): print(dicVar) all_fun = begin for row in range(3, sh.nrows): skill_id_str = get_str_from_sheet(sh, row, 0) #skill_name_str = get_str_from_sheet(sh, row, 1) skill_formula_str = get_str_from_sheet(sh, row, 1) print(skill_id_str, skill_formula_str) #fun_str = parse_function(skill_formula_str, u"技能ID:"+skill_id_str, u"param_%s" % skill_id_str, dicVar, srcSkillFormla) #print( fun_str ); expSrc = parse_expr_right(skill_formula_str, dicVar, 0, {}) fun_str = srcSkillFormla % (skill_id_str, skill_id_str, skill_formula_str, expSrc[1]) print(fun_str) all_fun += fun_str write_src(output_file, begin, end, all_fun, "utf-8") all_fun += end write_file(output_file, all_fun)
def find_blog_name_thorough(req_ses, blog_url): # TODO logging.debug( 'Using slower, more thorough name-finding on {0!r}'.format(blog_url)) # Extract domain # 'http://nsfw.kevinsano.com' # 'nsfw.kevinsano.com' domain_search = re.search(r'(?:https?://)?([^\\/]+\.\w+)/?', blog_url) if domain_search: domain = domain_search.group(1) logging.debug('domain={0!r}'.format(domain)) else: logging.error('Could not identify domain! Failing.') return None # Genreate archive page URL blog_rss_url = 'http://{0}/rss'.format(domain) logging.debug('blog_rss_url={0!r}'.format(blog_rss_url)) rss_path = os.path.join('debug', 'run_grab_site.find_blog_name_thorough.rss.rss') # Load archive page rss_res = common.fetch( requests_session=req_ses, url=blog_rss_url, method='get', ) common.write_file( # Save to file for debugging file_path=rss_path, data=rss_res.content) # Extract blog name from page # '<generator>Tumblr (3.0; @nsfwkevinsano)</generator>' # 'nsfwkevinsano' name_search = re.search('<generator>[^<]{0,25}@([^)<]+)\)</generator>', rss_res.content) if name_search: blog_name = name_search.group(1) logging.debug('blog_name={0!r}'.format(blog_name)) return blog_name
def set_text_section(self): """ Pull the .text section out. """ if os.path.exists(GDB_OUTPUT): os.remove(GDB_OUTPUT) sections_gdb = os.path.join(GDB_SCRIPT_DIR, 'sections.gdb') if not os.path.exists(sections_gdb): common.write_file(SECTIONS_GDB, sections_gdb) cmd = ['gdb', '--batch', '--command={}'.format(GDB_TEXT_SEC), self.elf_file] gdb = subprocess.Popen(cmd, stdout=PIPE) gdb.wait() gdb.stdout.close() gdb_output = common.read_file(GDB_OUTPUT) for line in gdb_output: if '.text' in line: line = line.split() self.__text_start__ = line[0] # start addr self.__text_end__ = line[2] # end addr break
def load_archive(req_ses, blog_name): base_url = 'http://{0}.tumblr.com/archive'.format(blog_name) all_history = '' # Load first block first_response = common.fetch( requests_session=req_ses, url=base_url, method='get', ) common.write_file(file_path=os.path.join('debug', 'first_response.html'), data=first_response.content) all_history += first_response.content # Find last post date last_post_date = find_last_post_date(html=first_response.content) # Load subsequent history while True: # Load next block scroll_url = 'http://{0}.tumblr.com/archive?before_time={1}'.format( blog_name, last_post_date) scroll_response = common.fetch( requests_session=req_ses, url=base_url, method='get', ) common.write_file(file_path=os.path.join('debug', 'scroll_response.html'), data=scroll_response.content) all_history += scroll_response.content # Find last post date last_post_date = find_last_post_date(html=scroll_response.content) # Stop if no more posts if not last_post_date: break # Store combined page common.write_file(file_path=os.path.join('debug', 'all_history.html'), data=all_history) return all_history
def configure_ipop(gid): # Create files based on user input dhcpdata = common.read_file('configs/dhcpdata.conf') dhcpdata = dhcpdata.replace('UFGrid00', gid) # Files are saved to floppy to maintain compatibility with appliance common.write_file('/mnt/fd/dhcpdata.conf', dhcpdata) common.write_file('/mnt/fd/ipop_ns', gid) common.write_file('/usr/local/ipop/var/ipop_ns', gid) # Create ipop config file create_ipop_config('/usr/local/ipop/var/ipop.config',gid) # Define the type of server created by user os.system('cp type /mnt/fd/type') # Update the node config file os.system('cp /mnt/fd/node.config /usr/local/ipop/var/node.config') # Add racoon to ipop restart os.system('echo \'/etc/init.d/racoon restart\' >> /etc/init.d/ipop.sh') return 0
return dict_out def merge_dicts(dict_list_id_sorted): output = [] pre_dict_item = dict_list_id_sorted[0] same_id_term = [dict_per_ad(pre_dict_item)] for dict_item in dict_list_id_sorted[1:]: if dict_item["uid"] == pre_dict_item["uid"]: same_id_term.append(dict_per_ad(dict_item)) else: output.append({"uid": pre_dict_item["uid"], "name": pre_dict_item["name"], "former_names": pre_dict_item["former_names"], "each_term": same_id_term}) same_id_term = [dict_per_ad(dict_item)] pre_dict_item = dict_item output.append({"uid": pre_dict_item["uid"], "name": pre_dict_item["name"], "former_names": pre_dict_item["former_names"], "each_term": same_id_term}) return output objs = json.load(open('../data/npl_ly.json')) for npl_legislator in objs: common.normalize_name(npl_legislator) if not npl_legislator.get('elected_party'): npl_legislator['elected_party'] = npl_legislator['party'] dump_data = json.dumps(objs, sort_keys=True, ensure_ascii=False) common.write_file(dump_data, '../data/npl_ly.json') dump_data = json.dumps(objs, sort_keys=True, indent=4, ensure_ascii=False) common.write_file(dump_data, '../data/pretty_format/npl_ly.json') merged_npl = merge_dicts(sorted(objs, key=lambda d: [d["uid"], d['ad']])) dump_data = json.dumps(merged_npl, sort_keys=True, indent=4, ensure_ascii=False) common.write_file(dump_data, '../data/pretty_format/npl_ly(same_id_in_one_dict).json') dump_data = json.dumps(merged_npl, sort_keys=True) common.write_file(dump_data, '../data/npl_ly(same_id_in_one_dict).json')
def write_info_disk(data_d, dest_dir): for key in data_d.keys(): filename = os.path.join(dest_dir, key) write_file(filename, str(data_d[key]).strip())
def run(): """ create trafodion user, bashrc, setup passwordless SSH """ dbcfgs = json.loads(dbcfgs_json) distro = dbcfgs['distro'] if 'CDH' in distro: hadoop_type = 'cloudera' elif 'HDP' in distro: hadoop_type = 'hortonworks' elif 'APACHE' in distro: hadoop_type = 'apache' home_dir = get_default_home() # customize trafodion home dir if dbcfgs.has_key('home_dir') and dbcfgs['home_dir']: home_dir = dbcfgs['home_dir'] traf_user = dbcfgs['traf_user'] traf_user_dir = '%s/%s' % (home_dir, traf_user) traf_dirname = dbcfgs['traf_dirname'] traf_home = '%s/%s' % (traf_user_dir, traf_dirname) hbase_xml_file = dbcfgs['hbase_xml_file'] auth_key_file = '%s/.ssh/authorized_keys' % traf_user_dir ssh_cfg_file = '%s/.ssh/config' % traf_user_dir ulimits_file = '/etc/security/limits.d/%s.conf' % traf_user # create trafodion user and group if cmd_output('getent passwd %s' % traf_user): # trafodion user exists, set actual trafodion group traf_group = cmd_output('id -ng %s' % traf_user) else: # default trafodion group traf_group = traf_user if not cmd_output('getent group %s' % traf_group): run_cmd('groupadd %s > /dev/null 2>&1' % traf_group) traf_pwd = dbcfgs['traf_pwd'] run_cmd('useradd --shell /bin/bash -m %s -g %s --home %s --password "$(openssl passwd %s)"' % (traf_user, traf_group, traf_user_dir, traf_pwd)) # hbase group is generally either hbase or hadoop, depending on distro if cmd_output('getent group hbase'): cmd_output('/usr/sbin/usermod -a -G hbase %s' % traf_user) if cmd_output('getent group hadoop'): cmd_output('/usr/sbin/usermod -a -G hadoop %s' % traf_user) if cmd_output('getent group hive'): cmd_output('/usr/sbin/usermod -a -G hive %s' % traf_user) if not os.path.exists(traf_user_dir): run_cmd('mkdir -p %s' % traf_user_dir) run_cmd('chmod 700 %s' % traf_user_dir) # set ssh key run_cmd_as_user(traf_user, 'echo -e "y" | ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa') # the key is generated in copy_file script running on the installer node run_cmd('cp %s{,.pub} %s/.ssh/' % (SSHKEY_FILE, traf_user_dir)) run_cmd_as_user(traf_user, 'cat ~/.ssh/id_rsa.pub > %s' % auth_key_file) run_cmd('chmod 644 %s' % auth_key_file) ssh_cfg = 'StrictHostKeyChecking=no\nNoHostAuthenticationForLocalhost=yes\n' with open(ssh_cfg_file, 'w') as f: f.write(ssh_cfg) run_cmd('chmod 600 %s' % ssh_cfg_file) run_cmd('chown -R %s:%s %s/.ssh/' % (traf_user, traf_group, traf_user_dir)) hb = ParseXML(hbase_xml_file) zk_nodes = hb.get_property('hbase.zookeeper.quorum') zk_port = hb.get_property('hbase.zookeeper.property.clientPort') # set trafodion_config nodes = dbcfgs['node_list'].split(',') trafodion_config = """ export TRAF_HOME="%s" export TRAF_VAR=$TRAF_HOME/tmp export MY_SQROOT=$TRAF_HOME # for compatibility export JAVA_HOME="%s" export node_count="%s" export HADOOP_TYPE="%s" export ENABLE_HA="%s" export ZOOKEEPER_NODES="%s" export ZOOKEEPER_PORT="%s" export SECURE_HADOOP="%s" export CLUSTERNAME="%s" """ % (traf_home, dbcfgs['java_home'], str(len(nodes)), hadoop_type, dbcfgs['enable_ha'], zk_nodes, zk_port, dbcfgs['secure_hadoop'], socket.gethostname()) # save additonal configs for elastic trafodion_config += """ export hbase_xml_file="%s" export hbase_lib_path="%s" export traf_user="******" export traf_version="%s" export dcs_cnt_per_node="%s" """ % (dbcfgs['hbase_xml_file'], dbcfgs['hbase_lib_path'], dbcfgs['traf_user'], dbcfgs['traf_version'], dbcfgs['dcs_cnt_per_node']) run_cmd('mkdir -p %s' % TRAF_CFG_DIR) write_file(TRAF_CFG_FILE, trafodion_config) if 'APACHE' in distro: extra_config = """ export HADOOP_PREFIX=%s export HBASE_HOME=%s export HIVE_HOME=%s export PATH=$PATH:$HADOOP_PREFIX/bin:$HADOOP_PREFIX/sbin:$HBASE_HOME/bin """ % (dbcfgs['hadoop_home'], dbcfgs['hbase_home'], dbcfgs['hive_home']) append_file(TRAFODION_CFG_FILE, extra_config) # set permission run_cmd('chown -R %s:%s %s*' % (traf_user, traf_group, TRAF_CFG_DIR)) # set ulimits for trafodion user ulimits_config = ''' # Trafodion settings %s soft core unlimited %s hard core unlimited %s soft memlock unlimited %s hard memlock unlimited %s soft nofile 32768 %s hard nofile 65536 %s soft nproc 100000 %s hard nproc 100000 %s soft nofile 8192 %s hard nofile 65535 ''' % ((traf_user,) * 10) write_file(ulimits_file, ulimits_config) # change permission for hsperfdata if os.path.exists(TRAF_HSPERFDATA_FILE): run_cmd('chown -R %s:%s %s' % (traf_user, traf_group, TRAF_HSPERFDATA_FILE)) # clean up unused key file at the last step run_cmd('rm -rf %s{,.pub}' % SSHKEY_FILE) print 'Setup trafodion user successfully!'
#!/usr/bin/python # -*- coding: utf-8 -* import json import common objs = json.load(open('../data/ly_info.json')) for ly_legislator in objs: common.normalize_name(ly_legislator) dump_data = json.dumps(objs) common.write_file(dump_data, '../data/ly_info.json') dump_data = json.dumps(objs, sort_keys=True, indent=4, ensure_ascii=False) common.write_file(dump_data, '../data/pretty_format/ly_info.json') empty_term_start = [(legislator["ad"], legislator["name"], legislator["links"]["ly"]) for legislator in objs if not legislator.has_key("term_start")] dump_data = json.dumps(empty_term_start, sort_keys=True, indent=4, ensure_ascii=False) common.write_file(dump_data, '../log/term_start_empty_on_lygovtw.json')
if compare[key] != base[key]: f.write('key, %s, (ly.gov.tw), %s, (npl), %s, uid, %s, ad, %s, name, %s, links, %s\n' % (key, compare[key], base[key], base["uid"], base["ad"], base["name"], compare["links"]["ly"])) else: f.write('can not find key: %s\n' % key) ly_dict_list = json.load(open('../data/%d/ly_info.json' % args.ad)) npl_dict_list = json.load(open('../data/%d/npl_ly.json' % args.ad)) for source in [ly_dict_list, npl_dict_list]: for legislator in source: common.normalize_name(legislator) for npl_legislator in npl_dict_list: names_list = [npl_legislator["name"]] for name in npl_legislator.get("former_names", []): names_list.append(name) ly_legislator = find_legislator_from_ly_info(names_list, npl_legislator, ly_dict_list) if ly_legislator: term = complement(ly_legislator, npl_legislator) # --> cross check data conflict f = codecs.open('../log/conflict.txt','w', encoding='utf-8') for ly_legislator in ly_dict_list: npl_legislator = find_legislator_from_npl(ly_legislator, npl_dict_list) if npl_legislator: conflict(ly_legislator, npl_legislator, f) f.close() # <-- end dump_data = json.dumps(npl_dict_list, sort_keys=True, ensure_ascii=False) common.write_file(dump_data, '../data/%d/merged.json' % args.ad) dump_data = json.dumps(npl_dict_list, sort_keys=True, indent=4, ensure_ascii=False) common.write_file(dump_data, '../data/pretty_format/%d/merged.json' % args.ad)
def create_config(self): config_xml = self.config_data() common.logit("\nCreating configuration file: %s" % self.config_file()) common.make_directory(self.config_dir(),self.username(),0755) common.write_file("w",0644,self.config_file(),config_xml)
def parse_sailor_sheet(sh, dicVar, output_path): startRow = get_int_from_sheet(sh, 3, 1) - 1 headRow = startRow - 1 print("startRow = %d" % startRow) print("headRow = %d" % headRow) #表添加了新的列,这个变量要改 #endCol = 27 endCol = sh.ncols for row in range(startRow, sh.nrows): baseSrcLst = [] fightValueSrcLst = [] sailorTmpSrcLst = [] sailorBaseSrcLst = [] sailorTrainSrcLst = [] upstarUpgradeSailorLst = [] upgradeSailorLst = [] sailorInvestLst = [] attrLst = [] reviseAttrLst = [] commonLst = [] sailorId = get_str_from_sheet(sh, row, 0) for col in range(0, endCol): head = get_str_from_sheet(sh, headRow, col) if (len(head) == 0): continue data = get_str_from_sheet(sh, row, col) if (len(data) == 0): continue #print( "[%s]=%s"%(head, data) ) if (head == u"formula"): note = get_str_from_sheet(sh, headRow - 2, col) # TODO:打断公式取左值,看看是那种类型的变量 eqCharIdx = data.find(u"=") lrSplit = re.compile(u'[==]{1}') if eqCharIdx != -1: lvalue = data[0:eqCharIdx] lvalue = lvalue.strip() src = parse_function(data, note, "", dicVar, "\t// %s%s%s") print(src) if dicVar[lvalue]["var_type"] == u"战斗数值": fightValueSrcLst.append(src) elif dicVar[lvalue]["var_type"] == u"水手临时数值": sailorTmpSrcLst.append(src) elif dicVar[lvalue]["var_type"] == u"船只基础数值": baseSrcLst.append(src) elif dicVar[lvalue]["var_type"] == u"水手基础属性值": sailorBaseSrcLst.append(src) elif dicVar[lvalue]["var_type"] == u"水手洗练属性值": sailorTrainSrcLst.append(src) elif dicVar[lvalue]["var_type"] == u"水手升星属性值": upstarUpgradeSailorLst.append(src) elif dicVar[lvalue]["var_type"] == u"水手升级属性值": upgradeSailorLst.append(src) elif dicVar[lvalue]["var_type"] == u"水手投资获得经验值": sailorInvestLst.append(src) elif dicVar[lvalue]["var_type"] == u"公用": commonLst.append(src) continue tmp = headExp.match(head) if tmp == None: continue var_name = tmp.group("var_name") var_type = tmp.group("var_type") if var_type == "int": var = int(data) srcAttr = attrSrc % (var_name, PythonData2Lpc(var)) attrLst.append(srcAttr) if var_type == "float": var = float(data) srcAttr = attrSrc % (var_name, PythonData2Lpc(var)) attrLst.append(srcAttr) if var_type == "string": var = data srcAttr = attrSrc % (var_name, PythonData2Lpc(var)) attrLst.append(srcAttr) if var_type == "array": var = [] tmpLst = data.split(",") for tmp in tmpLst: var.append(int(tmp)) srcAttr = attrSrc % (var_name, PythonData2Lpc(var)) attrLst.append(srcAttr) if var_type == "special": var = [] tmpLst = data.split(";") for st in tmpLst: tm = st.split(",") tmp1 = [] for tmp in tm: tmp1.append(int(tmp)) var.append(tmp1) srcAttr = attrSrc % (var_name, PythonData2Lpc(var)) attrLst.append(srcAttr) if var_type == "intermediate": #reviseVar = int(data)/100.0 reviseVar = int(data) reviseAttr = attrReviseSrc % (var_name, PythonData2Lpc(reviseVar)) reviseAttrLst.append(reviseAttr) if var_type == "dir": var = [] tmpLst = data.split(";") for st in tmpLst: tm = st.split(",") tmp1 = [] for tmp in tm: tmp1.append(tmp) var.append(tmp1) srcAttr = attrSrc % (var_name, PythonData2Lpc(var)) attrLst.append(srcAttr) if var_type == "dir_map": var = [] tmpLst = data.split(";") for st in tmpLst: tm = st.split(",") tmp1 = [] for tmp in tm: tmp1.append(int(tmp)) var.append(tmp1) srcAttr = attrSrc % (var_name, PythonData2Lpc(var)) attrLst.append(srcAttr) if var_type == "map": var = parse_map(data) srcAttr = attrSrc % (var_name, PythonData2Lpc(var)) attrLst.append(srcAttr) if sailorId != "": filename = "%s/%s.c" % (output_path, sailorId) filecontent = srcSailorTemplete % ( u"\n".join(attrLst), u"\n".join(reviseAttrLst), u"\n".join(sailorInvestLst), u"\n".join(commonLst), u"\n".join(sailorBaseSrcLst), u"\n".join(sailorTrainSrcLst), u"\n".join(upstarUpgradeSailorLst), u"\n".join(upgradeSailorLst), u"\n".join(sailorTmpSrcLst), u"\n".join(baseSrcLst), u"\n".join(fightValueSrcLst)) print(filecontent) try: src_data = open(filename, "rb").read() if len(src_data) == 0: src_data = begin + "\n\n" + end except IOError: src_data = begin + "\n\n" + end tmp = p.sub(begin + "\n" + filecontent + "\n" + end, src_data) write_file(filename, tmp)
f.write('key, %s, (ly.gov.tw), %s, (npl), %s, uid, %s, ad, %s, name, %s, links, %s\n' % (key, compare[key], base[key], base["uid"], base["ad"], base["name"], compare["links"]["ly"])) else: f.write('can not find key: %s\n' % key) ly_dict_list = json.load(open('../data/ly_info.json')) npl_dict_list = json.load(open('../data/npl_ly(same_id_in_one_dict).json')) origin_npl_dict_list = json.load(open('../data/npl_ly.json')) for npl_legislator in npl_dict_list: names_list = [npl_legislator["name"]] for name in npl_legislator.get("former_names", []): names_list.append(name) for term in npl_legislator["each_term"]: # ly.gov.tw didn't have legislators information of ad = 1 if term["ad"] != 1: ly_legislator = find_legislator_from_ly_info(names_list, term, ly_dict_list) if ly_legislator: term = complement(ly_legislator, term) # --> cross check data conflict f = codecs.open('../log/conflict.txt','w', encoding='utf-8') for ly_legislator in ly_dict_list: npl_legislator = find_legislator_from_npl(ly_legislator, origin_npl_dict_list) if npl_legislator: conflict(ly_legislator, npl_legislator, f) f.close() # <-- end dump_data = json.dumps(npl_dict_list, sort_keys=True, ensure_ascii=False) common.write_file(dump_data, '../data/merged.json') dump_data = json.dumps(npl_dict_list, sort_keys=True, indent=4, ensure_ascii=False) common.write_file(dump_data, '../data/pretty_format/merged.json')
_log('Please check config.py') exit() _log('Config OK') device_id = _get_id(c.id_file) if not (device_id or cfg.device_id): first_device_id = c.device_id() second_device_id = c.device_id() if first_device_id == second_device_id: cfg.device_id = first_device_id else: _device_id_ = c.random_hex16() _log('New Device Id: %s' % cfg.device_id) else: if device_id: cfg.device_id = device_id c.write_file(cfg.device_id, c.id_file, _log) if cfg.stream_quality == 'PC': _quality_ = 'PC' else: _quality_ = 'MOBILE' _o2tvgo_ = O2TVGO(cfg.device_id, cfg.username, cfg.password, _quality_, _log) if cfg.playlist_type == 3: c.write_streamer(os.path.join(cfg.playlist_path, cfg.playlist_streamer), os.path.join(cfg.playlist_path, cfg.playlist_src), cfg.ffmpeg_command, _log) code, num, err = channel_playlist()
def run(): dbcfgs = json.loads(dbcfgs_json) if not os.path.exists(dbcfgs['hbase_lib_path']): err('Cannot find HBase lib folder') if not os.path.exists(dbcfgs['java_home']): err('Cannot find Java, please set the JAVA_HOME on the new nodes to: %s' % dbcfgs['java_home']) home_dir = get_default_home() if dbcfgs.has_key('home_dir'): home_dir = dbcfgs['home_dir'] traf_user = dbcfgs['traf_user'] traf_home = dbcfgs['traf_home'] traf_user_dir = '%s/%s' % (home_dir, traf_user) traf_ver = dbcfgs['traf_version'] # scratch_locs = dbcfgs['scratch_locs'].split(',') SUDOER_CFG = """ ## Allow trafodion id to run commands needed for backup and restore %%%s ALL =(hbase) NOPASSWD: %s/bin/hbase" """ % (traf_user, DEF_HBASE_HOME) ### add trafodion user ### # create trafodion user and group if cmd_output('getent passwd %s' % traf_user): print 'user [%s] exists' % traf_user # trafodion user exists, set actual trafodion group traf_group = cmd_output('id -ng %s' % traf_user) else: # default trafodion group traf_group = traf_user if not cmd_output('getent group %s' % traf_group): run_cmd('groupadd %s' % traf_group) traf_shadow = dbcfgs['traf_shadow'] print 'Adding user [%s]' % traf_user run_cmd( 'useradd --shell /bin/bash -m %s -g %s --home %s --password "%s"' % (traf_user, traf_group, traf_user_dir, traf_shadow)) print 'Added user [%s]' % traf_user if not os.path.exists(traf_user_dir): run_cmd('mkdir -p %s' % traf_user_dir) run_cmd('chmod 700 %s' % traf_user_dir) ### untar the copied trafoion binaries ### TRAF_PKG_FILE = '/tmp/traf_bin.tar.gz' run_cmd('mkdir -p %s' % traf_home) run_cmd('mkdir -p /etc/trafodion') run_cmd('tar xf %s -C %s' % (TRAF_PKG_FILE, traf_home)) run_cmd('mv -f /tmp/trafodion_config %s' % TRAF_CFG_FILE) run_cmd('cp -rf /tmp/.ssh %s/..' % traf_home) run_cmd('mv -f /tmp/hbase-trx-* %s' % dbcfgs['hbase_lib_path']) run_cmd('mv -f /tmp/trafodion-utility-* %s' % dbcfgs['hbase_lib_path']) ### copy trafodion bashrc ### bashrc_template = '%s/sysinstall/home/trafodion/.bashrc' % traf_home bashrc_file = '%s/%s/.bashrc' % (home_dir, traf_user) # backup orig bashrc if os.path.exists(bashrc_file): run_cmd('cp -f %s %s.bak' % (bashrc_file, bashrc_file)) run_cmd('cp -f %s %s' % (bashrc_template, bashrc_file)) # set permission run_cmd('chmod 700 %s/../.ssh' % traf_home) cmd_output('chmod 600 %s/../.ssh/{id_rsa,config,authorized_keys}' % traf_home) run_cmd('chmod 777 %s' % TRAF_CFG_FILE) run_cmd('chown -R %s:%s /etc/trafodion' % (traf_user, traf_group)) run_cmd('chmod +r %s/{hbase-trx-*,trafodion-utility-*}' % dbcfgs['hbase_lib_path']) run_cmd('chown -R %s:%s %s' % (traf_user, traf_group, traf_user_dir)) ### modify CLUSTERNAME ### mod_file(TRAF_CFG_FILE, {'CLUSTERNAME=.*': 'CLUSTERNAME=%s' % socket.gethostname()}) ### kernel settings ### run_cmd('echo "kernel.pid_max=65535" >> /etc/sysctl.conf') run_cmd('echo "kernel.msgmnb=65536" >> /etc/sysctl.conf') run_cmd('echo "kernel.msgmax=65536" >> /etc/sysctl.conf') run_cmd('/sbin/sysctl -p /etc/sysctl.conf 2>&1 > /dev/null') ### copy init script ### init_script = '%s/sysinstall/etc/init.d/trafodion' % traf_home if os.path.exists(init_script): run_cmd('cp -rf %s /etc/init.d/' % init_script) run_cmd('chkconfig --add trafodion') run_cmd('chkconfig --level 06 trafodion on') ### create and set permission for scratch file dir ### # for loc in scratch_locs: # # don't set permission for HOME folder # if not os.path.exists(loc): # run_cmd('mkdir -p %s' % loc) # if home_dir not in loc: # run_cmd('chmod 777 %s' % loc) if dbcfgs['enable_ha'] == 'true': # set trafodion sudoer file for specific cmds SUDOER_CFG += """ ## Trafodion Floating IP commands Cmnd_Alias IP = /sbin/ip Cmnd_Alias ARP = /sbin/arping ## Allow Trafodion id to run commands needed to configure floating IP %%%s ALL = NOPASSWD: IP, ARP """ % traf_user ### write trafodion sudoer file ### with open(TRAF_SUDOER_FILE, 'w') as f: f.write(SUDOER_CFG) # set ulimits for trafodion user ulimits_file = '/etc/security/limits.d/%s.conf' % traf_user ulimits_config = ''' # Trafodion settings %s soft core unlimited %s hard core unlimited %s soft memlock unlimited %s hard memlock unlimited %s soft nofile 32768 %s hard nofile 65536 %s soft nproc 100000 %s hard nproc 100000 ''' % ((traf_user, ) * 8) write_file(ulimits_file, ulimits_config)
def create_config(self,config_xml): common.logit("\nCreating configuration files") common.logit(" %s" % self.config_file()) common.make_directory(self.config_dir(),self.username(),0755) common.write_file("w",0644,self.config_file(),config_xml,SILENT=True)
def write_file_by_dict(filename, dict_values): for key in dict_values.keys(): msg = "%s#&#%s" % (key, dict_values[key]) write_file(filename, msg) return True
#!/usr/bin/python # -*- coding: utf-8 -* import json import common objs = json.load(open('../data/ly_info.json')) for ly_legislator in objs: common.normalize_name(ly_legislator) dump_data = json.dumps(objs, sort_keys=True, ensure_ascii=False) common.write_file(dump_data, '../data/ly_info.json') dump_data = json.dumps(objs, sort_keys=True, indent=4, ensure_ascii=False) common.write_file(dump_data, '../data/pretty_format/ly_info.json') empty_term_start = [(legislator["ad"], legislator["name"], legislator["links"]["ly"]) for legislator in objs if not legislator.has_key("term_start")] dump_data = json.dumps(empty_term_start, sort_keys=True, indent=4, ensure_ascii=False) common.write_file(dump_data, '../log/term_start_empty_on_lygovtw.json')
url = a['href'] desc = li.find('li', {"class": "font-balck"}).text # time = li.find('li', {"class": "publish"}).text h3 = li.findAll('h3') name = h3[0].text.strip() price = h3[1].text url = common_url + url if rental not in name: continue print url print name print desc print price urls.append(name + ' | ' + desc + ' | ' + price + ' | ' + url) return urls def get_house_list(): house_list = [] rentals = get_community_list() for rental in rentals: house_list = get_woaiwojia_url(house_list, rental) return house_list if __name__ == '__main__': urls = get_house_list() write_file(urls, 'woaiwojia.txt') get_woaiwojia_url([], '') print 'over'
a = h3.findAll('a')[-1] url = a['href'] name = a.text data_addr = h3.find('b')['data-addr'].replace('\n', '') data_addr = str(data_addr) data_addr = data_addr.split(" ")[0] price = box.find('p', {"class": "p1"}) price = price.find('span', {"class": "en"}).text if rental not in data_addr: continue print url print data_addr print name print price urls.append(name + ' | ' + price + ' | ' + data_addr + ' | ' + common_url + url) return urls def get_house_list(): house_list = [] rentals = get_community_list() for rental in rentals: house_list = get_ziru_url(house_list, rental) return house_list if __name__ == '__main__': urls = get_house_list() write_file(urls, 'ziru.txt') print 'over'