def loadConfig(self): """ Loads plugin config Uses json to store config because i dont like yml """ #default config self.pluginCfg = {"localChat":True, "msgRadius":150, "msgDelay":60, "msgCost":100, "noEcoMsg":u"&4[RCchat] Eco doesnt work.Install Vault.", "noMoneyMsg":u"&9[RCchat]&f Not enough money. You need %s $", "noEnoughTime":u"&9[RCchat]&f You write to global chat too fast. Please wait %s sec. or write '*message' to use chargeable message for global chat", "globalPrefix":"&9[G]&f", "localPrefix":"&9[L]&f", "permissionsGroups":{"prem":30, "vip":40} } name = self.description.getName() if path.exists("plugins/" + name) and path.isdir("plugins/" + name): if path.exists("plugins/" + name + "/config.json"): f = ioopen("plugins/" + name + "/config.json", "r", encoding="utf-8") self.pluginCfg = json.load(f) f.close() else: f = ioopen("plugins/" + name + "/config.json", "w", encoding="utf-8") f.write(unicode(json.dumps(self.pluginCfg, indent=4, ensure_ascii=False))) f.close() else: os.mkdir("plugins/" + name) f = ioopen("plugins/" + name + "/config.json", "w", encoding="utf-8") f.write(unicode(json.dumps(self.pluginCfg, indent=4, ensure_ascii=False))) f.close()
def loadConfig(self): """ Loads plugin config Uses json to store config because i dont like yml """ #default config self.pluginCfg = { "localChat": True, "msgRadius": 150, "msgDelay": 60, "msgCost": 100, "noEcoMsg": u"&4[RCchat] Eco doesnt work.Install Vault.", "noMoneyMsg": u"&9[RCchat]&f Not enough money. You need %s $", "noEnoughTime": u"&9[RCchat]&f You write to global chat too fast. Please wait %s sec. or write '*message' to use chargeable message for global chat", "globalPrefix": "&9[G]&f", "localPrefix": "&9[L]&f", "permissionsGroups": { "prem": 30, "vip": 40 } } name = self.description.getName() if path.exists("plugins/" + name) and path.isdir("plugins/" + name): if path.exists("plugins/" + name + "/config.json"): f = ioopen("plugins/" + name + "/config.json", "r", encoding="utf-8") self.pluginCfg = json.load(f) f.close() else: f = ioopen("plugins/" + name + "/config.json", "w", encoding="utf-8") f.write( unicode( json.dumps(self.pluginCfg, indent=4, ensure_ascii=False))) f.close() else: os.mkdir("plugins/" + name) f = ioopen("plugins/" + name + "/config.json", "w", encoding="utf-8") f.write( unicode( json.dumps(self.pluginCfg, indent=4, ensure_ascii=False))) f.close()
def from_file(self, filename): """ Reads in a prior from a file specification Parameters ---------- filename: str Name of the file to be read in Notes ----- Lines beginning with '#' or empty lines will be ignored. Priors can be loaded from: bilby.core.prior as, e.g., foo = Uniform(minimum=0, maximum=1) floats, e.g., foo = 1 bilby.gw.prior as, e.g., foo = bilby.gw.prior.AlignedSpin() other external modules, e.g., foo = my.module.CustomPrior(...) """ comments = ['#', '\n'] prior = dict() with ioopen(filename, 'r', encoding='unicode_escape') as f: for line in f: if line[0] in comments: continue line.replace(' ', '') elements = line.split('=') key = elements[0].replace(' ', '') val = '='.join(elements[1:]).strip() prior[key] = val self.from_dictionary(prior)
def header_parser(input_log_file): """ The function parsing the header of csv file and return the dictionary of field as key and index of field as value :type input_log_file: string :rtype fields: dictionary """ fields = {} with ioopen(input_log_file, encoding='utf-8') as csvfile: line_1 = csvfile.readline() field_index = line_1.split(',') for i in field_index: if 'ip' in i: fields[ip_index] = field_index.index(i) if 'date' in i: fields[date_index] = field_index.index(i) if 'time' in i: fields[time_index] = field_index.index(i) if 'cik' in i: fields[cik_index] = field_index.index(i) if 'accession' in i: fields[accession_index] = field_index.index(i) if 'extention' in i: fields[extention_index] = field_index.index(i) return fields
def _parse(cfg_file, cfg_section, c2ctemplate_style=False): """ Parses the defined YAML file and returns the defined section as dictionary. Args: cfg_file (str): The YAML file to be parsed. cfg_section (str): The section to be returned. Returns: dict: The parsed section as dictionary. """ if cfg_file is None: raise ConfigurationError( 'Missing configuration parameter "pyramid_oereb.cfg.file" or ' '"pyramid_oereb.cfg.c2ctemplate.file".' ) if cfg_section is None: raise ConfigurationError('Missing configuration parameter "pyramid_oereb.cfg.section".') try: if c2ctemplate_style: import c2c.template content = c2c.template.get_config(cfg_file) else: with ioopen(cfg_file, encoding='utf-8') as f: content = yaml.safe_load(f.read()) except IOError as e: e.strerror = '{0}{1} \'{2}\', Current working directory is {3}'.format( e.strerror, e.args[1], e.filename, os.getcwd()) raise cfg = content.get(cfg_section) if cfg is None: raise ConfigurationError('YAML file contains no section "{0}"'.format(cfg_section)) return cfg
def renderJSON(script_path, time_file_path): # 使用io.open打开命令文件 with ioopen(script_path, encoding='utf-8', errors='replace', newline='\r\n') as scriptf: # 打开时间记录文件 with open(time_file_path) as timef: timing = getTiming(timef) ret = {} with closing(scriptf): scriptf.readline() # ignore first header line from script file offset = 0 for t in timing: dt = scriptf.read(t[1]) offset += t[0] ret[str(offset / float(1000))] = dt.decode('utf-8', 'replace') return dumps(ret)
def find_headers_and_url_save(self): ret = False try: x = self.find_headers_and_url() if len(x) > 0: with ioopen(self.file_out, "w", encoding="utf-8") as output_file: output_file.write( unicode(dumps(x, ensure_ascii=False, indent=4))) #with open(self.file_out, "w", encoding="utf-8") as output_file: # dump(x, output_file, ensure_ascii=False, indent=4) log_string(self.uuid, "Extracted headers and URLs", "Green") ret = True except Exception as e: log_string(self.uuid, "find_headers_and_url_save Failed {}".format(e), "Red") return ret
def get_log_json(self): """ 返回日志内容,用于回放使用 """ # 使用io.open打开命令文件 with ioopen(self.log_file_path + '.log', encoding='utf-8', errors='replace', newline='\r\n') as scriptf: # 打开时间记录文件 with open(self.log_file_path + '.time') as timef: timing = self._getTiming(timef) ret = {} with closing(scriptf): offset = 0 for t in timing: dt = scriptf.read(t[1]) offset += t[0] ret[str(offset / float(1000))] = dt.decode('utf-8', 'replace') return json.dumps(ret)
def parse_file(self, uuid, box): ret = False good = False try: self.uuid = uuid self.file_in = path.join(box["output"], self.uuid, box["chrome_ouput"]) self.file_out = path.join(box["output"], self.uuid, box["chrome_ouput_parsed"]) with ioopen(self.file_in, encoding="utf-8") as data_file: _buffer = data_file.read() try: log_string(uuid, "Parsing logs without trimming", "Green") self.data = loads(_buffer) good = True except: log_string(uuid, "Parsing logs without trimming failed", "Red") try: if not good: log_string(uuid, "Parsing logs with trimming", "Green") self.data = loads(_buffer[:-2] + "]}") good = True except: log_string(uuid, "Parsing logs with trimming failed", "Red") if good: self.log_event_types = self.data["constants"][ "logEventTypes"] self.reversed_log_event_types = dict( map(reversed, self.log_event_types.items())) self.log_source_types = self.data["constants"][ "logSourceType"] self.reversed_log_source_types = dict( map(reversed, self.log_source_types.items())) log_string(uuid, "Log file has been parsed", "Green") ret = True except Exception as e: log_string(uuid, "parse_file Failed {}".format(e), "Red") return ret
def get_log_json(self): """ 返回日志内容,用于回放使用 """ # 使用io.open打开命令文件 with ioopen(self.log_file_path + '.log', encoding='utf-8', errors='replace', newline='\r\n') as scriptf: # 打开时间记录文件 with open(self.log_file_path + '.time') as timef: timing = self._getTiming(timef) ret = {} with closing(scriptf): offset = 0 for t in timing: dt = scriptf.read(t[1]) offset += t[0] ret[str(offset / float(1000))] = dt.decode( 'utf-8', 'replace') return json.dumps(ret)
def run_analysis(): #take the sys arguments as the input and output file path and names input_dir = sys.argv[1] output_top_10_occupations_file = sys.argv[2] output_top_10_states_file = sys.argv[3] #find the file in input directory and pass to the input_file file_list = os.listdir(input_dir) input_file = input_dir + file_list[0] #create the variables of map data structures holding the key of occupations and working_states # and values of counts respectfully Certified_Total = 0.0 occupations = {} working_states = {} # read the csvfile line by line and parsing each line of data into # fields, then put the SOC(occupation names) and working states fields # into occupations and working_states map with ioopen(input_file, encoding='utf-8') as csvfile: lines = csvfile.readlines() field_index = lines[0].split(';') for i in field_index: if 'STATUS' in i: status_index = field_index.index(i) print(status_index) if 'SOC_NAME' in i: SOC_index = field_index.index(i) print(SOC_index) if 'WORKLOC1_STATE' in i or 'WORKSITE_STATE' in i: states_index = field_index.index(i) print(states_index) rest_lines = lines[1:] for line in rest_lines: row = line.split(";") status = row[status_index].replace('"', '') occupation = row[SOC_index].replace('"', '') state = row[states_index].replace('"', '') if status == 'CERTIFIED': Certified_Total += 1 if occupation in occupations and state in working_states: occupations[occupation] += 1 working_states[state] += 1 elif occupation in occupations and state not in working_states: occupations[occupation] += 1 working_states[state] = 1 elif occupation not in occupations and state in working_states: occupations[occupation] = 1 working_states[state] += 1 else: occupations[occupation] = 1 working_states[state] = 1 csvfile.close() # sort the occupations and working_states by values(total counts of the key) # and store the sorted result as list of tuples sorted_Occupations = sorted(sorted(occupations.items(), key=lambda x: x[0]), key=lambda x: x[1], reverse=True) sorted_States = sorted(sorted(working_states.items(), key=lambda x: x[0]), key=lambda x: x[1], reverse=True) # create the top10 occupations output file and write the top 10 occupations # from the sorted list results to it occupations_file = open(output_top_10_occupations_file, "w") header = 'TOP_OCCUPATIONS;NUMBER_CERTIFIED_APPLICATIONS;PERCENTAGE' + '\n' occupations_file.write(header) for i in sorted_Occupations[0:10]: output_str = str(i[0]) + ';' + str(i[1]) + ';' + str( round((i[1] / Certified_Total) * 100, 1)) + '%' + '\n' occupations_file.write(output_str) print( str(i[0]) + ';' + str(i[1]) + ';' + str(round((i[1] / Certified_Total) * 100, 1)) + '%') occupations_file.close() # create the top10 states output file and write the top 10 states info # from the sorted list restults states_file = open(output_top_10_states_file, "w") header = 'TOP_STATES;NUMBER_CERTIFIED_APPLICATIONS;PERCENTAGE' + '\n' states_file.write(header) for j in sorted_States[0:10]: output_state = str(j[0]) + ';' + str(j[1]) + ';' + str( round((j[1] / Certified_Total) * 100, 1)) + '%' + '\n' states_file.write(output_state) print( str(j[0]) + ';' + str(j[1]) + ';' + str(round((j[1] / Certified_Total) * 100, 1)) + '%') states_file.close()
def session_producer(input_log_file, input_inactivity_file, output_file_uri): """ The function that take the input log file and input inactive_interval file location to caculate the time spends and file requested per user session and write the result into output file location :type input_log_file: string :type input_inactivity_file: string :type output_file_uri: string :rtype : """ # read the csvfile first line and parsing header info for each of fields fields_idx = header_parser(input_log_file) #open the output file for writing output output_file = open(output_file_uri, 'w') #read the idle_interval parameter from input_inactivity_file with open(input_inactivity_file, 'r') as interval: inactive_interval = int(interval.read()[0]) # read the csvfile line by line and parsing each line of data into # a session object with ioopen(input_log_file, encoding='utf-8') as csvfile: lines = csvfile.readlines() rest_lines = lines[1:] user_sessions = {} curr_time = None for line in rest_lines: fields = line.split(',') ip = fields[fields_idx[ip_index]] date = fields[fields_idx[date_index]] time = fields[fields_idx[time_index]] cik = fields[fields_idx[cik_index]] timestamp = datetime.strptime((date + '' + time).strip(' '), '%Y-%m-%d%H:%M:%S') #first check all the seesions in the dictionary once time reflected,detact the end #session and pop it to ouput file if curr_time is None or timestamp > curr_time: curr_time = timestamp print curr_time for user in user_sessions.values(): if user.idle_interval(curr_time) > inactive_interval: u_ip = user.user_ip end_session = user_sessions.pop(u_ip) line = ','.join([ end_session.user_ip, datetime.strftime(end_session.start_tm, '%Y-%m-%d %H:%M:%S'), datetime.strftime(end_session.latest_active_tm, '%Y-%m-%d %H:%M:%S'), str(end_session.session_duration()), str(end_session.doc_num) ]) output_file.write(line + '\n') #processing the curr line and check with user_sessions for insert or update if ip in user_sessions.keys(): user_sessions[ip].update_active_time(timestamp) user_sessions[ip].update_doc_num(1) else: new_session = userSession(ip, timestamp, timestamp, 1) print(new_session.user_ip) user_sessions[ip] = new_session #append the rest of the remaining sessoins into the output file once reach the last line line = ','.join([ i.user_ip, datetime.strftime(i.start_tm, '%Y-%m-%d %H:%M:%S'), datetime.strftime(i.latest_active_tm, '%Y-%m-%d %H:%M:%S'), str(i.session_duration()), str(i.doc_num) ]) output_file.write(line + '\n') csvfile.close() output_file.close()
def from_file(self, filename): """ Reads in a prior from a file specification Parameters ---------- filename: str Name of the file to be read in Notes ----- Lines beginning with '#' or empty lines will be ignored. Priors can be loaded from: bilby.core.prior as, e.g., foo = Uniform(minimum=0, maximum=1) floats, e.g., foo = 1 bilby.gw.prior as, e.g., foo = bilby.gw.prior.AlignedSpin() other external modules, e.g., foo = my.module.CustomPrior(...) """ comments = ['#', '\n'] prior = dict() mvgdict = dict(inf=np.inf) # evaluate inf as np.inf with ioopen(filename, 'r', encoding='unicode_escape') as f: for line in f: if line[0] in comments: continue line.replace(' ', '') elements = line.split('=') key = elements[0].replace(' ', '') val = '='.join(elements[1:]).strip() cls = val.split('(')[0] args = '('.join(val.split('(')[1:])[:-1] try: prior[key] = DeltaFunction(peak=float(cls)) logger.debug( "{} converted to DeltaFunction prior".format(key)) continue except ValueError: pass if "." in cls: module = '.'.join(cls.split('.')[:-1]) cls = cls.split('.')[-1] else: module = __name__.replace( '.' + os.path.basename(__file__).replace('.py', ''), '') cls = getattr(import_module(module), cls, cls) if key.lower() in ["conversion_function", "condition_func"]: setattr(self, key, cls) elif (cls.__name__ in [ 'MultivariateGaussianDist', 'MultivariateNormalDist' ]): if key not in mvgdict: mvgdict[key] = eval(val, None, mvgdict) elif (cls.__name__ in ['MultivariateGaussian', 'MultivariateNormal']): prior[key] = eval(val, None, mvgdict) else: try: prior[key] = cls.from_repr(args) except TypeError as e: raise TypeError( "Unable to parse dictionary file {}, bad line: {} " "= {}. Error message {}".format( filename, key, val, e)) self.update(prior)