def compute_hiperspaces(self): if not len(self.points) > 0: logger.error('No points to compute hull!') raise Exception('No points to compute hull!') # The heuristic caracteristic when searching to connect # different clusters does that it might fail # so we redirect the stdout to avoid such error # being visible to user stderr_fd = sys.stderr.fileno() with open('/tmp/qhull-output.log', 'w') as f, stderr_redirected(f): points = list(self.points) logger.info('Searching for hull in dimension %s based on %s points', len(points[0]),len(points)) output = qconvex('n',points) if len(output) == 1: logger.debug('Could not get Hull. Joggle input?') try: dim, facets_nbr, facets = self.__parse_hs_output(output) except IncorrectOutput: logger.error('Could not get hull') raise CannotGetHull() logger.info('Found hull in dimension %s of %s facets', dim,facets_nbr) self.dim = dim self.facets = facets return self.dim
def save(self): """Dump all data we can collect to tmp directory""" data_dir_timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') data_dir_prefix = 'satellite-sanity-save-%s' % data_dir_timestamp data_dir_base = tempfile.mkdtemp(suffix='', prefix='%s-' % data_dir_prefix, dir=D_TMP) data_dir = os.path.join(data_dir_base, 'satellite-sanity') os.makedirs(data_dir) logger.debug("Saving to directory %s" % data_dir) for key in self.config['commands'].keys(): data_file = os.path.join(data_dir, key) fd = open(data_file, 'w') try: for row in self[key]: fd.write("%s\n" % row) except DataNotAvailable: logger.warn("Failed when obtaining %s" % key) fd.close() if self[key] is not None: data_file_lines = len(self[key]) else: data_file_lines = -1 logger.debug("Saved %s lines to %s" % (data_file_lines, data_file)) data_tarxz = "%s.tar.xz" % data_dir_base command = ['tar', '-cJf', data_tarxz, '-C', data_dir_base, 'satellite-sanity'] logger.debug("Running %s" % command) process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() assert len(stderr) == 0, "Compress failed with '%s' when running '%s'" % (stderr, command) logger.info("Saved to %s" % data_tarxz) return data_tarxz
def main(no_confirm=True): src_dump = get_src_dump() download_list = get_file_list_for_download() if len(download_list) == 0: logging.info("No newer file found. Abort now.") sys.exit(0) doc = src_dump.find_one({'_id': 'ucsc'}) if not os.path.exists(DATA_FOLDER): os.makedirs(DATA_FOLDER) logfile = os.path.join(DATA_FOLDER, 'ucsc_dump.log') setup_logfile(logfile) # mark the download starts doc = {'_id': 'ucsc', 'timestamp': timestamp, 'data_folder': DATA_FOLDER, 'lastmodified': latest_lastmodified, 'logfile': logfile, 'status': 'downloading'} src_dump.save(doc) t0 = time.time() download(download_list, no_confirm) # mark the download finished successfully _updates = { 'status': 'success', 'time': timesofar(t0), 'pending_to_upload': True # a flag to trigger data uploading } src_dump.update({'_id': 'ucsc'}, {'$set': _updates})
def closest_points(self,clusters): rel = {} closest_tuple = (None,None) max_all = 0 seen = {} for key,cluster in clusters.items(): for point in cluster: # Which cluster contains this point seen[point] = key corr_list = rel.setdefault(point,[]) for idx,corr in enumerate(self[point,:]): corr = abs(corr) corr_list.append(corr) if idx in cluster: continue if max_all < corr: closest = idx max_all = corr closest_tuple = (point,idx) p0,p1 = closest_tuple if p0 and p1: for pnbr,p in enumerate([p0,p1]): if p in seen: logger.info('Closests point p%d is from cluster %s', pnbr, seen[p]) else: logger.info('Closests point p%d was disconected',pnbr) cor0 = np.array(rel.get(p0,self[p0,:])) cor1 = np.array(rel.get(p1,self[p1,:])) else: cor0 = cor1 = None return p0,p1,cor0,cor1
def gather_csv_info(): """ Reads the input files one by one and stores the data in dictionaries containing lists of lists. """ global args global w_dict, w_info global p_dict, p_headers_list, p_no_sum w_dict, p_dict = {}, {} p_headers_list = [] p_no_sum = 0 for input_file in wimport_lib.find_csv_filenames(args.input_dir): # get webinar and participants info w_info = wimport_lib.get_webinar_info(input_file, DETAILS_MARK) if w_info: w_id = wimport_lib.get_parameter('Webinar ID', w_info[0], w_info[1]) p_info = wimport_lib.get_participants_info(input_file, w_id, DETAILS_MARK) p_len = len(p_info[1]) p_no_sum += p_len logger.info("Reading {} \n\t --> {} participants.".format(input_file, p_len)) # store info for later writing to files and database if w_id not in w_dict: w_dict[w_id] = [w_info[1]] else: w_dict[w_id] += [w_info[1]] if w_id not in p_dict: p_dict[w_id] = p_info[1] else: p_dict[w_id] += p_info[1] p_headers_list += [p_info[0]]
def wrapper(*args, **kwargs): logger.info("Running %s", func.__name__) start = time.time() result = func(*args, **kwargs) end = time.time() logger.info("Execution time: %s", end - start) return result
def send(self, cmd, payload): logger.info("send %s, payload:%s", cmd, str(payload)) req = { "cmd":cmd, "payload":payload } self._send(req)
def resume(): logger.info('(' + get_addr(request) + ')') db.counter.update_one({'counter': 'resume'}, {'$inc': {'value': 1}}) with codecs.open(app.static_folder + '/files/md/resume.md', 'r', encoding='utf8') as f: text = f.read() res = m.html(text) return render_template('blank.html', html=res, title='Resume')
def insert_beer(beer_list) : for beer in beer_list : # print(beer) # print("\n")d if beer.get("name") is not None : string_name = unicodedata.normalize('NFKD', beer.get("name")).encode('ascii','ignore') else : continue if beer.get("description") is not None : string_desc = unicodedata.normalize('NFKD', beer.get("description")).encode('ascii','ignore') else : continue if beer.get("labels") is not None : beer_image = beer.get("labels").get("large") else : continue if beer.get('isOrganic') is None or beer.get("abv") is None or beer.get("ibu") is None or beer.get("breweryId") is None or beer.get("styleId") is None or beer_image is None: continue b = Beer(name=string_name, description=string_desc, is_organic=beer.get('isOrganic'), abv=beer.get("abv"), ibu=beer.get("ibu"), scrape_brew_id=beer.get("breweryId"), scrape_style_id=beer.get("styleId"), image=beer_image) db.session.add(b) db.session.commit() if beer.get("name") is not None: logger.info("beer: " + beer.get("name") + " populated") db.session.commit()
def package_delete(self, data_package): """ Delete a package by package_title return True if the operation success False otherwise """ msg = '' package_title = data_package[u'title'] package_identifier = data_package[u'identifier'] resp = self.package_search( prop='identifier', value=package_identifier ) if not resp: msg = 'Package delete: \'%s\' NOT FOUND.' % package_title logger.error(msg) return False, msg package_id = resp[0][u'id'] try: resp = self.__conn.action.package_delete(id=package_id) except ckanapi.NotFound: msg = 'Package delete: \'%s\' NOT FOUND.' % package_title logger.info(msg) return False, msg except Exception, error: msg = 'Package delete: ERROR to execute the command for %s.: %s' % ( package_title, error ) logger.error(msg) return False, msg
def upload_file(local_path, remote_name, bucket=STORAGE_BUCKET): """ Upload a file to Google Storage :param local_path: The local path to the file to upload :param remote_name: The name of the file in the google cloud storage :param bucket: The bucket on google cloud storage you want to upload the file to :return: True if uploaded, False otherwise """ try: service = discovery.build('storage', 'v1', credentials=get_storage_credentials()) logger.info("Uploading %s to google cloud" % local_path) req = service.objects().insert( bucket=bucket, name=remote_name, # predefinedAcl="publicRead", Uncomment this line if you want your files to be accessible to anyone media_body=local_path) req.execute() uploaded = check_if_file_exists(remote_name) if uploaded is True: logger.info("Upload complete!") return True else: return False except Exception as e: logger.debug("Unable to upload file %s to google cloud: %s" % (local_path, e)) return False
def slice_segments(source_files, segments, output_dir=VIDEO_OUTPUT_DIR): tmp_files = [] for i, seg in enumerate(segments): tmp_files.append(slice_segment(i, seg, source_files, output_dir)) if (i + 1) % 20 == 0: logger.info('Produced segment %d / %d', i + 1, len(segments)) return tmp_files
def build_ffmpeg(self, version, enable=[], disable=[]): """build ffmpeg libraries""" saved = os.getcwd() os.chdir(os.path.join(self.build_dir_, "ffmpeg-%s" % version)) command( "sed -i -e 's|SDL_CONFIG=\"${cross_prefix}sdl-config\"|" "SDL_CONFIG=\"%s/bin/sdl-config\"|' ./configure" % self.install_dir_) shutil.copy2("configure", "configure.orig") os.environ['PKG_CONFIG_PATH'] = "%s/lib/pkgconfig" % self.install_dir_ cmd = ("./configure --prefix=%s --extra-cflags=\"-I%s/include\" " "--extra-ldflags=\"-L%s/lib\" --enable-libfaac --enable-libmp3lame " "--enable-libx264 --enable-libzvbi --enable-libass --enable-gpl " "--enable-pthreads --enable-nonfree" % ( self.install_dir_, self.install_dir_, self.install_dir_)) if enable: for opt in enable: cmd += ' --enable-' + opt if disable: for opt in disable: cmd += ' --disable-' + opt logger.info(cmd) command(cmd) shutil.move("configure.orig", "configure") command(["make", "-j%d" % self.cpu_count_]) command(["make", "install"]) os.chdir(saved)
def _cut_segment(i, seg, source_files, output_dir): tmp_file = os.path.join(output_dir, 'segment_%s_%f_%f.mp4' % ( seg.epnum, seg.start, seg.duration)) if os.path.exists(tmp_file): logger.info('Cached segment: %d - %s', i, tmp_file) return tmp_file if seg.epnum not in source_files: raise ValueError('Process fail at ' + str(i) + ': no such epnum ' + seg.epnum) source_file = source_files[seg.epnum] args = [ config['avconv'], '-ss', str(seg.start), '-i', source_file, '-t', str(seg.duration), '-vf', 'scale=' + config['resolution'], '-vcodec', config['vcodec'], '-b:v', config['bitrate'], '-r', config['fps'], '-an', tmp_file, ] p = shell.execute(*args) if p.returncode != 0: raise shell.ShellError(args, 'process fail at %d : %s' % (i, p.stderr)) logger.info('Generated segment: %d - %s', i, tmp_file) return tmp_file
def generate_key(func_name, args, dict_args_original, skip_args): args_concat = [v for key, v in sorted(dict_args_original.iteritems()) if key not in skip_args] # Get serialized arguments (function name, or string of v if is not reference checked in ugly way args_serialized = \ '_'.join([ v.__name__ if hasattr(v, '__call__') else (str(v) if len(str(v)) < 200 else hashlib.md5(str(v)).hexdigest()) for v in args_concat if hasattr(v, '__call__') or hasattr(v, "__init__") or str(v).find("0x") == -1]) logger.info("Serialized args to " + args_serialized) key = func_name + "_" + args_serialized full_key = func_name + "(" + "".join( [str(k) + "=" + (str(v) if len(str(v)) < 200 else hashlib.md5(str(v)).hexdigest()) for k, v in sorted(dict_args_original.iteritems()) if key not in skip_args]) if len(key) > 400: key = key[0:400] return key, full_key
def insert_last_month_intrest(): INSERT_STR="INSERT INTO sc_sta_mlm (user_id,intrest,month) " \ "SELECT * FROM view_get_last_month_intrest" logger.info("============开始更新每月客户利润贡献===========") INSERT_UPDATE_TRAN(INSERT_STR) logger.info("============每月客户利润贡献更新完毕===========") return None
def generate_clinvar_lib(data_folder): sys.path.insert(0,data_folder) orig_path = os.getcwd() try: os.chdir(data_folder) logging.info("Generate XM parser") ret = os.system('''generateDS.py -f -o "clinvar_tmp.py" -s "clinvarsubs.py" clinvar_public.xsd''') if ret != 0: logging.error("Unable to generate parser, return code: %s" % ret) raise try: py = open("clinvar_tmp.py").read() # convert py2 to py3 (though they claim it support both versions) py = py.replace("from StringIO import StringIO","from io import StringIO") fout = open("clinvar.py","w") fout.write(py) fout.close() os.unlink("clinvar_tmp.py") # can we import it ? import clinvar logging.info("Found generated clinvar module: %s" % clinvar) except Exception as e: logging.error("Cannot convert to py3: %s" % e) raise finally: os.chdir(orig_path)
def OnInsert(self, table, columns, values): logger.info('OnInsert: columns=%s, values=%s' % (columns, values)) if columns[0] == 'id' and values[0] == self.id: self.average_sale = self.db_handle.GetAverageSale(self.id) logger.info('Updating average sale to %s' % self.average_sale) if not self.is_alive() and self.average_sale != -1: self.run()
def insert_rep(loan_apply_id,tran_no,last_repayment_day,crnt_pr,arfn_pr,crnt_int,arfn_int): try: f_arfn_pr=float(arfn_pr)#已还本金 f_arfn_int=float(arfn_int)#已还利息 f_crnt_pr=float(crnt_pr)#应还本金 f_crnt_int=float(crnt_int)#应还利息 if f_arfn_int<f_crnt_int or f_arfn_pr<f_crnt_pr: if DAO_overdue.get_is_overdue(loan_apply_id,tran_no): status=2 elif f_arfn_pr==0 and f_arfn_int==0: status=0 else: status=1 else: status=3 total_repayment=f_arfn_pr+f_arfn_int logger.info("插入还款编号-"+str(loan_apply_id)+",期数-"+str(tran_no)+"") REP_INSERT_STR="INSERT INTO sc_repayment \ (loan_apply_id,repayment_installments,re_principal,re_interest, \ clear_date,total_repayment,status) \ VALUES \ (%s,%s,%s,%s,%s,%s,%s)"%(loan_apply_id,tran_no,arfn_pr,arfn_int,last_repayment_day,total_repayment,status) INSERT_UPDATE_TRAN(REP_INSERT_STR) except: logger.exception('exception') return None
def update_rep(loan_apply_id,tran_no,last_repayment_day,crnt_pr,arfn_pr,crnt_int,arfn_int,id): f_arfn_pr=float(arfn_pr)#已还本金 f_arfn_int=float(arfn_int)#已还利息 f_crnt_pr=float(crnt_pr)#应还本金 f_crnt_int=float(crnt_int)#应还利息 if f_arfn_int<f_crnt_int or f_arfn_pr<f_crnt_pr: if DAO_overdue.get_is_overdue(loan_apply_id,tran_no): status=2 elif f_arfn_pr==0 and f_arfn_int==0: status=0 else: status=1 else: status=3 total_repayment=f_arfn_pr+f_arfn_int logger.info("更新贷款编号-"+str(loan_apply_id)+",期数-"+str(tran_no)+"") REP_UPDATE_STR="UPDATE sc_repayment SET " \ "loan_apply_id=%s,repayment_installments=%s,re_principal=%f,re_interest=%f," \ "clear_date=%d,total_repayment=%f,status=%d " \ "WHERE id=%s"%(loan_apply_id,tran_no,arfn_pr,arfn_int,last_repayment_day,total_repayment,status,id) INSERT_UPDATE_TRAN(REP_UPDATE_STR) return None
def save(self, filename=None): pnml = etree.Element('pnml') net = etree.SubElement(pnml, 'net', id=self.id, type="http://www.pnml.org/version-2009/grammar/ptnet") net_name = etree.SubElement(net, 'name') net_name_text = etree.SubElement(net_name, 'text') net_name_text.text = self.name page = etree.SubElement(net, 'page', id='page') for tr_id, tr in self.transitions.items(): transition = etree.SubElement(page, 'transition', id=tr_id) transition_name = etree.SubElement(transition, 'name') transition_name_text = etree.SubElement(transition_name, 'text') transition_name_text.text = tr.label for pl_id, pl in self.places.items(): place = etree.SubElement(page, 'place', id=pl_id) place_name = etree.SubElement(place, 'name') place_name_text = etree.SubElement(place_name, 'text') place_name_text.text = pl.label place_init_mark = etree.SubElement(place, 'initialMarking') place_init_mark_text = etree.SubElement(place_init_mark, 'text') place_init_mark_text.text = str(pl.marking) for arc in self.arcs: arc_node = etree.SubElement(page, 'arc', id=arc.id, source=arc.source.id, target=arc.destination.id) if arc.value > 1: arc_value = etree.SubElement(arc_node, 'value') arc_value_text = etree.SubElement(arc_value, 'text') arc_value_text.text = str(arc.value) tree = etree.ElementTree(element=pnml) tree.write(filename or self.filename, encoding="utf-8", xml_declaration=True, method="xml", pretty_print=True) logger.info('Generated the PNML %s', self.filename)
def log_building_start(self): if self.merge_logging: #setup logging logfile = 'databuild_{}_{}.log'.format('genedoc' + '_' + self._build_config['name'], time.strftime('%Y%m%d')) logfile = os.path.join(self.log_folder, logfile) setup_logfile(logfile) src_build = getattr(self, 'src_build', None) if src_build: #src_build.update({'_id': self._build_config['_id']}, {"$unset": {"build": ""}}) d = {'status': 'building', 'started_at': datetime.now(), 'logfile': logfile, 'target_backend': self.target.name} if self.target.name == 'mongodb': d['target'] = self.target.target_collection.name elif self.target.name == 'es': d['target'] = self.target.target_esidxer.ES_INDEX_NAME logging.info(pformat(d)) src_build.update({'_id': self._build_config['_id']}, {"$push": {'build': d}}) _cfg = src_build.find_one({'_id': self._build_config['_id']}) if len(_cfg['build']) > self.max_build_status: #remove the first build status record src_build.update({'_id': self._build_config['_id']}, {"$pop": {'build': -1}})
def compute_hiperspaces(self): # La característica heurística al buscar conexiones entre # diferentes clusters hace que pueda fallar # por lo que redirigimos la salida para ser silenciosos # en esos casos if not len(self.points) > 0: logger.error('No points to compute hull!') raise Exception('No points to compute hull!') stderr_fd = sys.stderr.fileno() with open('/tmp/qhull-output.log', 'w') as f, stderr_redirected(f): points = list(self.points) logger.info('Searching for hull in dimension %s based on %s points', len(points[0]),len(points)) output = qconvex('n',points) if len(output) == 1: logger.debug('Could not get Hull. Joggle input?') try: dim, facets_nbr, facets = self.__parse_hs_output(output) except IncorrectOutput: logger.warning('Could not get hull') raise CannotGetHull() logger.info('Found hull in dimension %s of %s facets', dim,len(facets)) self.dim = dim self.facets = facets if self.verbose: print "Computed MCH with ",facets_nbr," halfspaces" print 'This are them:\n' for facet in self.facets:print facet return self.dim
def run(self): UDPServer(self.ifname, self.ip, self.test).start() TCPClient(self.host, self.ip, self.send_err, self.file_path, self.test).start() logger.info('Emulator started on %s' % self.ip) if self.test: time.sleep(60) stop()
def run(self): logger.info('Running.') while True: if not self.Scrape(): logger.info('Stopping.') break util.Sleep(self.scrape_every)
def get_word_vector(self, word): try: vector = self.m[word] logger.info('Got vector for word: %s', word) except KeyError: logger.info('No vector for word: %s', word) vector = get_random_vector(distribution=vector_distribution) return vector
def __init__(self, user_agent=None): self.__address = ckan_config['ckan_address'] self.__apikey = ckan_config['ckan_apikey'] self.__user_agent = user_agent self.__conn = ckanapi.RemoteCKAN(self.__address, self.__apikey, self.__user_agent) logger.info('Connected to %s' % self.__address)
def process_messages(self): """ Process all the messages from the queue and stop after """ logger.info('START to process messages in \'%s\'', self.queue_name) self.rabbit.open_connection() print 'work in progress' self.rabbit.close_connection() logger.info('DONE processing messages in \'%s\'', self.queue_name)
def main(*args, **kwargs): usage = """ Usage: ./pnml_comparator_script.py <.ini config filename> Config file options: %s\n NOTE: Do you have the needed environment variables? - XES : Path to .xes file with traces (for running PacH) - PNML : Path to .pnml file with Petri net model (for simplifying PROM models) - NXES : Path to .xes file with negative traces - PETRI : Path where simplified .pnml files should be moved to after script ends - STATS : Path where statistic files should be moved to after script ends IMPORTANT: NO PATH MUST END IN '/' (it is added automatically) """%(config_options) if not check_argv(sys.argv, minimum=1, maximum=4): print usage ret = -1 else: ret = 0 try: config_file = sys.argv[1] if not config_file.endswith('.ini'): print config_file, ' does not end in .ini. It should...' raise Exception('Filename has wrong extension') if not isfile(config_file): raise Exception("No such file") if '--debug' in sys.argv: pdb.set_trace() for filename, arguments in parse_config(config_file): comparator = ComparatorPnml(filename, **arguments) complexity = comparator.compare() logger.info('%s complexity -> %s',filename,complexity) comparator.generate_pnml() comparator.generate_outputs() comparator.check_hull() pnml_folder,out_folder = parse_config_output(config_file) pwd = os.getcwd() for basename in os.listdir(pwd): if basename.endswith('.pnml'): pnml_file = os.path.join(pwd, basename) if os.path.isfile(pnml_file): shutil.copy2(pnml_file, pnml_folder) os.remove(pnml_file) elif basename.endswith('.out'): out_file = os.path.join(pwd, basename) if os.path.isfile(out_file): shutil.copy2(out_file, out_folder) os.remove(out_file) except Exception, err: ret = 1 if hasattr(err, 'message'): print 'Error: ', err.message else: print 'Error: ', err logger.error('Error: %s' % err, exc_info=True) raise err return ret
def _load_ensembl2entrez_li(self): ensembl2entrez_li = loadobj(("ensembl_gene__2entrezgene_list.pyobj", self.src), mode='gridfs') #filter out those deprecated entrez gene ids logging.info(len(ensembl2entrez_li)) ensembl2entrez_li = [(ensembl_id, self._entrez_geneid_d[int(entrez_id)]) for (ensembl_id, entrez_id) in ensembl2entrez_li if int(entrez_id) in self._entrez_geneid_d] logging.info(len(ensembl2entrez_li)) ensembl2entrez = list2dict(ensembl2entrez_li, 0) self._idmapping_d_cache['ensembl_gene'] = ensembl2entrez
def download_magic(): """ Download raw .jpgs of magic the gathering cards """ parser = argparse.ArgumentParser(prog='MTG Download') parser.add_argument('--aws', action='store_true') parser.add_argument('--windows', action='store_true') parser.add_argument('--skipdata', action='store_true') parser.add_argument('--skipresults', action='store_true') parser.add_argument('--dryrun', action='store_true') args = parser.parse_args() if args.aws: # General commands sync_base = 'aws s3 sync ' dryrun_arg = ' --dryrun' results_sync = '{} {}'.format(Config.CLOUD_RESULTS, Config.RESULTS_DIR) data_sync = '{} {}'.format(Config.CLOUD_DATA, Config.DATA_DIR) include_flags = " --exclude '*' --include 'card_classifier/*'" if args.windows: include_flags = re.sub("'", "", include_flags) if not args.skipdata: logger.info('Downloading Data from AWS') cc_sync = sync_base + data_sync + include_flags cc_sync += dryrun_arg if args.dryrun else '' logger.info(cc_sync) os.system(cc_sync) logger.info('Unzip image archives.') for archive in tqdm( ['cropped.zip', 'curated.zip', 'mtg_images.zip']): full_archive = os.path.join(Config.DATA_DIR, 'card_classifier', archive) if os.path.exists(full_archive): shutil.unpack_archive(full_archive, re.sub('.zip', '', full_archive), 'zip') os.remove(full_archive) if not args.skipresults: logger.info('Downloading Results from AWS') cc_sync = sync_base + results_sync + include_flags cc_sync += dryrun_arg if args.dryrun else '' logger.info(cc_sync) os.system(cc_sync) else: logger.info('Downloading Metadata') metadata = get_mtg_metadata() logger.info('Wrangling Metadata') metadata = wrangle_mtg_metadata(metadata) logger.info('Downloading Image files') df_failed = _download_magic(metadata) logger.info('Failed to download {} images.'.format(df_failed.shape[0])) logger.info('Saving Metadata') metadata.to_csv(os.path.join(Config.DATA_DIR, 'mtg_images', 'metadata.csv'), index=False) logger.info('Saving Failed Images info.') df_failed.to_csv(os.path.join(Config.DATA_DIR, 'mtg_images', 'failed_images.csv'), index=False)
async def on_ready(self): logger.info("User is ready")
def diagnose(self): """ Print diagnostics for the fit model """ if self.summary is None: raise ValueError('Fit a model first.') logger.info('Printing Results.') # Get trues y = self.fit_transform(self.etl())['y'] preds = self.summary[self.summary['labels'].str.contains('y_hat')]['mean'].values # Random Intercepts df_random_effects = self.summary[self.summary['labels'].str.startswith('a[')]. \ sort_values('mean', ascending=False).reset_index(drop=True) df_random_effects['labels'] = df_random_effects['labels'].map(self.random_effect_inv) # Coefficients df_coefs = self.summary[self.summary['labels'].str.contains('^b[0-9]', regex=True)]. \ assign(labels=self.features). \ sort_values('mean', ascending=False). \ reset_index(drop=True) # Globals df_globals = self.summary[self.summary['labels'].isin(['mu_a', 'sigma_a', 'sigma_y'])].reset_index(drop=True) if self.response_distributions[self.response] == 'bernoulli_logit': df_globals = df_globals[df_globals['labels'] != 'sigma_y'].reset_index(drop=True) with PdfPages(os.path.join(self.results_dir, 'diagnostics_{}.pdf'.format(self.version))) as pdf: # Bar graph of random effects for top 10, bottom 10, big10 teams plt.figure(figsize=(8, 8)) df_top10 = df_random_effects.sort_values('mean', ascending=False).head(10).reset_index(drop=True) plt.bar(df_top10['labels'], df_top10['mean']) plt.errorbar(x=df_top10.index, y=df_top10['mean'], yerr=df_top10['sd'], fmt='none', ecolor='black') plt.grid(True) plt.xticks(rotation=90) plt.title('Top Ten Teams') plt.tight_layout() pdf.savefig() plt.close() # Bottom 10 plt.figure(figsize=(8, 8)) df_bot10 = df_random_effects.sort_values('mean', ascending=False).tail(10).reset_index(drop=True) plt.bar(df_bot10['labels'], df_bot10['mean']) plt.errorbar(x=df_bot10.index, y=df_bot10['mean'], yerr=df_bot10['sd'], fmt='none', ecolor='black') plt.grid(True) plt.title('Bottom Ten Teams') plt.xticks(rotation=90) plt.tight_layout() pdf.savefig() plt.close() if self.random_effect in ['team', 'opponent']: # nfc north plt.figure(figsize=(8, 8)) df_nfcn = df_random_effects[df_random_effects['labels'].isin(['CHI', 'GNB', 'DET', 'MIN'])].\ sort_values('mean', ascending=False).reset_index(drop=True) plt.bar(df_nfcn['labels'], df_nfcn['mean']) plt.errorbar(x=df_nfcn.index, y=df_nfcn['mean'], yerr=df_nfcn['sd'], fmt='none', ecolor='black') plt.grid(True) plt.hlines(xmax=max(df_nfcn.index) + 1, xmin=min(df_nfcn.index) - 1, y=0, linestyles='dashed') plt.title('NFC North Teams') plt.xticks(rotation=90) plt.tight_layout() pdf.savefig() plt.close() # Coefficients plt.figure(figsize=(8, 8)) plt.bar(df_coefs['labels'], df_coefs['mean']) plt.errorbar(x=df_coefs.index, y=df_coefs['mean'], yerr=df_coefs['sd'], fmt='none', ecolor='black') plt.grid(True) plt.hlines(xmax=max(df_coefs.index) + 1, xmin=min(df_coefs.index) - 1, y=0, linestyles='dashed') plt.title('Coefficients') plt.xticks(rotation=90) plt.tight_layout() pdf.savefig() plt.close() # Globals plt.figure(figsize=(8, 8)) plt.bar(df_globals['labels'], df_globals['mean']) plt.errorbar(x=df_globals.index, y=df_globals['mean'], yerr=df_globals['sd'], fmt='none', ecolor='black') plt.grid(True) plt.hlines(xmax=max(df_globals.index) + 1, xmin=min(df_globals.index) - 1, y=0, linestyles='dashed') plt.title('Globals') plt.xticks(rotation=90) plt.tight_layout() pdf.savefig() plt.close() if self.response_distributions[self.response] == 'bernoulli_logit': logger.info('Extra diagnostics for {}'.format(self.response)) # For Binaries, plot a ROC curve, histogram of predictions by class fpr, tpr, th = roc_curve(y, preds) score = auc(fpr, tpr) plt.figure(figsize=(8, 8)) plt.plot(fpr, tpr, label='AUC: {a:0.3f}'.format(a=score)) plt.plot([0, 1], [0, 1], color='black', linestyle='dashed') plt.grid(True) plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.legend() plt.tight_layout() pdf.savefig() plt.close() # Precision / Recall plt.figure(figsize=(8, 8)) plt.plot(th, fpr, label='False Positive Rate') plt.plot(th, tpr, label='True Positive Rate') plt.title('Precisions / Recall') plt.xlabel('Cutoff') plt.ylabel('Rate') plt.grid(True) plt.legend() plt.tight_layout() pdf.savefig() plt.close() # Histograms bins = np.linspace(min(preds) * 0.99, max(preds) * 1.01, 20) plt.figure(figsize=(8, 8)) plt.hist(preds[y == 1], alpha=0.5, bins=bins, color='darkorange', density=True, label='Positive') plt.hist(preds[y == 0], alpha=0.5, bins=bins, density=True, label='Negative') plt.grid(True) plt.legend() plt.xlabel('Probability') plt.ylabel('Density') plt.tight_layout() pdf.savefig() plt.close() elif self.response_distributions[self.response] == 'linear': logger.info('Extra Diagnostics for {}'.format(self.response)) # For continuous, plot a distribution of residuals with r-squared and MSE residuals = y - preds mse = np.sum(residuals ** 2) plt.figure(figsize=(8, 8)) plt.hist(residuals, label='MSE: {m:0.3f}'.format(m=mse), bins=20) plt.legend() plt.xlabel('Residuals') plt.ylabel('Counts') plt.grid(True) plt.tight_layout() pdf.savefig() plt.close()
import argparse import shlex import subprocess from config import get_accounts, logger, LOG_LEVEL from vjudge.main import VJudge parser = argparse.ArgumentParser() parser.add_argument('-b', required=False, dest='address', default='localhost:5000', help='address to bind') args = parser.parse_args() p = subprocess.Popen( shlex.split( f"gunicorn -w 2 -k gevent --logger-class config.GLogger --log-level {LOG_LEVEL} " f"-b '{args.address}' manage:app")) try: normal_accounts, contest_accounts = get_accounts() vjudge = VJudge(normal_accounts=normal_accounts, contest_accounts=contest_accounts) vjudge.start() except KeyboardInterrupt: logger.info('VJudge exiting') finally: p.terminate() p.wait()
t = t.cuda() for _ in range(1): c = t.clone() dist.all_reduce(c, dist.ReduceOp.SUM) t.set_(c) logger.info("{} value: {}".format(rank, t)) def init_processes(rank, size, fn, backend='nccl'): """ Initialize the distributed environment. """ os.environ['MASTER_ADDR'] = '127.0.0.1' os.environ['MASTER_PORT'] = '29500' dist.init_process_group(backend, rank=rank, world_size=size) fn(rank, size) def start(size): processes = [] for rank in range(size): p = Process(target=init_processes, args=(rank, size, run)) p.start() processes.append(p) for p in processes: p.join() if __name__ == "__main__": set_global_variables() args = get_args() logger.info("args: {}".format(args)) start(args.size)
def process_phone_sms_verify_pc(self): """ # 手机短信验证码验证 :param kwargs: :return: 成功返回 True, 失败返回 False 提示 为了调试方便 调整为FALSE """ try: logger.info("手机短信验证处理中") try: tel_code = self.driver.find_element_by_css_selector( self.get_key_words(7, "css", 1)) if tel_code: sub_button = self.driver.find_element_by_css_selector( 'button[id="checkpointSecondaryButton"]') self.click(sub_button) except: pass try: tel_button = self.driver.find_elements_by_css_selector( 'i[class^="img sp_"]') if not tel_button: return False, -1 except: pass self.click(tel_button[3]) tel_stutas = self.driver.find_elements_by_css_selector( 'a[role="menuitemcheckbox"]') self.click(tel_stutas[45]) self.sleep() send_tel = self.driver.find_element_by_css_selector( 'input[type="tel"]') self.send_keys(send_tel, "16500000000") self.sleep() submit_button = self.driver.find_element_by_css_selector( 'button[id="checkpointSubmitButton"]') self.click(submit_button) # 提交失败 try: submit_error = WebDriverWait(self.driver, 6).until( EC.presence_of_element_located( (By.CSS_SELECTOR, 'input[data-xui-error-position="above"]'))) if submit_error: logger.error("请填写正确的手机号码") return False, 7 except: pass # 短信验证码 self.sleep() tel_code = self.driver.find_element_by_css_selector( 'input[name="p_c"]') self.send_keys(tel_code, "414141") self.sleep() submit_button = self.driver.find_element_by_css_selector( 'button[id="checkpointSubmitButton"]') self.click(submit_button) except Exception as e: logger.exception("处理手机短信验证处理异常, e={}".format(e)) return False, 7 logger.info("处理手机短信验证处理完成") return False, 7
try: side1_detector_action, side1_fighter_action = agent1.get_action( side1_obs_dict, step_cnt) # todo 更改我方动作 # for i in range(len(side1_fighter_action)): # if args.agent1 == 'fix_rule': # # 规则 # side1_fighter_action[i]['r_fre_point'] = i + 1 # else: # # model # # side1_fighter_action[i][1] = i+1 # side1_fighter_action[i][1] = random.randint(1, 10) # # side1_fighter_action[i][2] = 11 logger.info('side1 actions:') for i, act in enumerate(side1_fighter_action): logger.info('fight_{}:{}'.format(i + 1, act)) except: env.set_surrender(0) # reward o_detector_reward, o_fighter_reward, o_game_reward, e_detector_reward, e_fighter_reward, e_game_reward = env.get_reward( ) agent1_crash_list.append(round_cnt) print('Side 1 crashed!') side1_obs_raw, side2_obs_raw = env.get_obs_raw() side1_detector_obs_raw_list = side1_obs_raw[ 'detector_obs_list'] side1_fighter_obs_raw_list = side1_obs_raw['fighter_obs_list'] side1_joint_obs_raw_dict = side1_obs_raw['joint_obs_dict'] side2_detector_obs_raw_list = side2_obs_raw[
def monitor_vm(self, vm, all_vms): """ Main function of the monitor """ try: vm_pct_free_memory = float(vm.free_memory) / float( vm.total_memory) * 100.0 if vm.id not in self.vm_data: self.vm_data[vm.id] = VMMonitorData(vm.id) if self.vm_data[vm.id].mem_diff is None: self.vm_data[vm.id].mem_diff = vm.real_memory - vm.total_memory vmid_msg = "VMID " + str(vm.id) + ": " vm.host = self.get_host_info(vm.host.id) logger.info(vmid_msg + "Real Memory: " + str(vm.real_memory)) logger.info(vmid_msg + "Total Memory: " + str(vm.total_memory)) logger.info(vmid_msg + "Free Memory: %d (%.2f%%)" % (vm.free_memory, vm_pct_free_memory)) mem_over_ratio = Config.MEM_OVER if vm.mem_over_ratio: mem_over_ratio = vm.mem_over_ratio if vm_pct_free_memory < (mem_over_ratio - Config.MEM_MARGIN ) or vm_pct_free_memory > ( mem_over_ratio + Config.MEM_MARGIN): now = time.time() logger.debug( vmid_msg + "VM %s has %.2f%% of free memory, change the memory size" % (vm.id, vm_pct_free_memory)) if self.vm_data[vm.id].last_set_mem is not None: logger.debug(vmid_msg + "Last memory change was %s secs ago." % (now - self.vm_data[vm.id].last_set_mem)) else: self.vm_data[vm.id].original_mem = vm.allocated_memory logger.debug( vmid_msg + "The memory of this VM has been never modified. Store the initial memory : " + str(self.vm_data[vm.id].original_mem)) self.vm_data[vm.id].last_set_mem = now if (now - self.vm_data[vm.id].last_set_mem) < Config.COOLDOWN: logger.debug( vmid_msg + "It is in cooldown period. No changing the memory.") else: used_mem = vm.total_memory - vm.free_memory min_free_memory = Config.MIN_FREE_MEMORY # check if the VM has defined a specific MIN_FREE_MEMORY value if vm.min_free_mem: min_free_memory = vm.min_free_mem # it not free memory use exponential backoff idea if vm.free_memory <= min_free_memory: logger.debug(vmid_msg + "No free memory in the VM!") if self.vm_data[vm.id].no_free_memory_count > 1: # if this is the third time with no free memory use the original size logger.debug( vmid_msg + "Increase the mem to the original size.") new_mem = self.vm_data[vm.id].original_mem self.vm_data[vm.id].no_free_memory_count = 0 else: logger.debug( vmid_msg + "Increase the mem with 50% of the original.") new_mem = int(used_mem + (self.vm_data[vm.id].original_mem - used_mem) * 0.5) self.vm_data[vm.id].no_free_memory_count += 1 else: divider = 1.0 - (mem_over_ratio / 100.0) logger.debug(vmid_msg + "The used memory %d is divided by %.2f" % (int(used_mem), divider)) new_mem = int(used_mem / divider) # Check for minimum memory if new_mem < Config.MEM_MIN: new_mem = Config.MEM_MIN # add diff to new_mem value and to total_memory to make it real_memory (vm.real_memory has delays between updates) new_mem += self.vm_data[vm.id].mem_diff vm.total_memory += self.vm_data[vm.id].mem_diff # We never set more memory that the initial amount if new_mem > self.vm_data[vm.id].original_mem: new_mem = self.vm_data[vm.id].original_mem if abs(int(vm.total_memory) - new_mem) < Config.MEM_DIFF_TO_CHANGE: logger.debug( vmid_msg + "Not changing the memory. Too small difference.") else: logger.debug(vmid_msg + "Changing the memory from %d to %d" % (vm.total_memory, new_mem)) if new_mem > vm.total_memory: # If we increase the memory we must check if the host has enough free space if not self.host_has_memory_free( vm.host, new_mem - vm.total_memory): # The host has not enough free memory. Let's try to migrate a VM. logger.debug(vmid_msg + "The host " + vm.host.name + " has not enough free memory!") if Config.MIGRATION: logger.debug(vmid_msg + "Let's try to migrate a VM.") if vm.host.id in self.last_migration and ( now - self.last_migration[vm.host.id] ) < Config.MIGRATION_COOLDOWN: logger.debug( "The host %s is in migration cooldown period, let's wait.." % vm.host.name) else: if self.migrate_vm( vm.id, vm.host, all_vms): logger.debug( "A VM has been migrated from host %d. Store the timestamp." % vm.host.id) self.last_migration[ vm.host.id] = now else: logger.debug(vmid_msg + "Migration is disabled.") if Config.FORCE_INCREASE_MEMORY: logger.debug( vmid_msg + "But Force increase memory is activated. Changing memory." ) self.change_memory( vm.id, vm.host, new_mem) self.vm_data[vm.id].last_set_mem = now else: logger.debug(vmid_msg + "Not increase memory.") else: logger.debug(vmid_msg + "The host " + vm.host.name + " has enough free memory.") self.change_memory(vm.id, vm.host, new_mem) self.vm_data[vm.id].last_set_mem = now else: self.change_memory(vm.id, vm.host, new_mem) self.vm_data[vm.id].last_set_mem = now except: logger.exception("Error in monitor loop!")
def run_main(): logger.info("... Starting Raspberry Pi Power Monitor") logger.info("Press Ctrl-c to quit...") # The following empty dictionaries will hold the respective calculated values at the end of each polling cycle, which are then averaged prior to storing the value to the DB. solar_power_values = dict(power=[], pf=[], current=[]) home_load_values = dict(power=[], pf=[], current=[]) net_power_values = dict(power=[], current=[]) ct0_dict = dict(power=[], pf=[], current=[]) ct1_dict = dict(power=[], pf=[], current=[]) ct2_dict = dict(power=[], pf=[], current=[]) ct3_dict = dict(power=[], pf=[], current=[]) ct4_dict = dict(power=[], pf=[], current=[]) ct5_dict = dict(power=[], pf=[], current=[]) rms_voltages = [] i = 0 # Counter for aggregate function while True: try: board_voltage = get_board_voltage() samples = collect_data(2000) poll_time = samples['time'] ct0_samples = samples['ct0'] ct1_samples = samples['ct1'] ct2_samples = samples['ct2'] ct3_samples = samples['ct3'] ct4_samples = samples['ct4'] ct5_samples = samples['ct5'] v_samples = samples['voltage'] rebuilt_waves = rebuild_waves(samples, ct0_phasecal, ct1_phasecal, ct2_phasecal, ct3_phasecal, ct4_phasecal, ct5_phasecal) results = calculate_power(rebuilt_waves, board_voltage) # # RMS calculation for phase correction only - this is not needed after everything is tuned. The following code is used to compare the RMS power to the calculated real power. # # Ideally, you want the RMS power to equal the real power when you are measuring a purely resistive load. # rms_power_0 = round(results['ct0']['current'] * results['ct0']['voltage'], 2) # AKA apparent power # rms_power_1 = round(results['ct1']['current'] * results['ct1']['voltage'], 2) # AKA apparent power # rms_power_2 = round(results['ct2']['current'] * results['ct2']['voltage'], 2) # AKA apparent power # rms_power_3 = round(results['ct3']['current'] * results['ct3']['voltage'], 2) # AKA apparent power # rms_power_4 = round(results['ct4']['current'] * results['ct4']['voltage'], 2) # AKA apparent power # rms_power_5 = round(results['ct5']['current'] * results['ct5']['voltage'], 2) # AKA apparent power # Prepare values for database storage grid_0_power = results['ct0']['power'] # CT0 Real Power grid_1_power = results['ct1']['power'] # CT1 Real Power grid_2_power = results['ct2']['power'] # CT2 Real Power grid_3_power = results['ct3']['power'] # CT3 Real Power grid_4_power = results['ct4']['power'] # CT4 Real Power grid_5_power = results['ct5']['power'] # CT5 Real Power grid_0_current = results['ct0']['current'] # CT0 Current grid_1_current = results['ct1']['current'] # CT1 Current grid_2_current = results['ct2']['current'] # CT2 Current grid_3_current = results['ct3']['current'] # CT3 Current grid_4_current = results['ct4']['current'] # CT4 Current grid_5_current = results['ct5']['current'] # CT5 Current # If you are monitoring solar/generator inputs to your panel, specify which CT number(s) you are using, and uncomment the commented lines. solar_power = 0 solar_current = 0 solar_pf = 0 # solar_power = results['ct3']['power'] # solar_current = results['ct3']['current'] # solar_pf = results['ct3']['pf'] voltage = results['voltage'] # Set solar power and current to zero if the solar power is under 20W. if solar_power < 20: solar_power = 0 solar_current = 0 solar_pf = 0 # Determine if the system is net producing or net consuming right now by looking at the two panel mains. # Since the current measured is always positive, we need to add a negative sign to the amperage value if we're exporting power. if grid_0_power < 0: grid_0_current = grid_0_current * -1 if grid_1_power < 0: grid_1_current = grid_1_current * -1 if solar_power > 0: solar_current = solar_current * -1 # Unless your specific panel setup matches mine exactly, the following four lines will likely need to be re-written: home_consumption_power = grid_0_power + grid_1_power + grid_2_power + grid_3_power + grid_4_power + grid_5_power + solar_power net_power = home_consumption_power - solar_power home_consumption_current = grid_0_current + grid_1_current + grid_2_current + grid_3_current + grid_4_current + grid_5_current - solar_current net_current = grid_0_current + grid_1_current + grid_2_current + grid_3_current + grid_4_current + grid_5_current + solar_current if net_power < 0: current_status = "Producing" else: current_status = "Consuming" # Average 2 readings before sending to db if i < 2: solar_power_values['power'].append(solar_power) solar_power_values['current'].append(solar_current) solar_power_values['pf'].append(solar_pf) home_load_values['power'].append(home_consumption_power) home_load_values['current'].append(home_consumption_current) net_power_values['power'].append(net_power) net_power_values['current'].append(net_current) ct0_dict['power'].append(results['ct0']['power']) ct0_dict['current'].append(results['ct0']['current']) ct0_dict['pf'].append(results['ct0']['pf']) ct1_dict['power'].append(results['ct1']['power']) ct1_dict['current'].append(results['ct1']['current']) ct1_dict['pf'].append(results['ct1']['pf']) ct2_dict['power'].append(results['ct2']['power']) ct2_dict['current'].append(results['ct2']['current']) ct2_dict['pf'].append(results['ct2']['pf']) ct3_dict['power'].append(results['ct3']['power']) ct3_dict['current'].append(results['ct3']['current']) ct3_dict['pf'].append(results['ct3']['pf']) ct4_dict['power'].append(results['ct4']['power']) ct4_dict['current'].append(results['ct4']['current']) ct4_dict['pf'].append(results['ct4']['pf']) ct5_dict['power'].append(results['ct5']['power']) ct5_dict['current'].append(results['ct5']['current']) ct5_dict['pf'].append(results['ct5']['pf']) rms_voltages.append(voltage) i += 1 else: # Calculate the average, send the result to InfluxDB, and reset the dictionaries for the next 2 sets of data. infl.write_to_influx( solar_power_values, home_load_values, net_power_values, ct0_dict, ct1_dict, ct2_dict, ct3_dict, ct4_dict, ct5_dict, poll_time, i, rms_voltages, ) solar_power_values = dict(power=[], pf=[], current=[]) home_load_values = dict(power=[], pf=[], current=[]) net_power_values = dict(power=[], current=[]) ct0_dict = dict(power=[], pf=[], current=[]) ct1_dict = dict(power=[], pf=[], current=[]) ct2_dict = dict(power=[], pf=[], current=[]) ct3_dict = dict(power=[], pf=[], current=[]) ct4_dict = dict(power=[], pf=[], current=[]) ct5_dict = dict(power=[], pf=[], current=[]) rms_voltages = [] i = 0 if logger.handlers[0].level == 10: t = PrettyTable( ['', 'CT0', 'CT1', 'CT2', 'CT3', 'CT4', 'CT5']) t.add_row([ 'Watts', round(results['ct0']['power'], 3), round(results['ct1']['power'], 3), round(results['ct2']['power'], 3), round(results['ct3']['power'], 3), round(results['ct4']['power'], 3), round(results['ct5']['power'], 3) ]) t.add_row([ 'Current', round(results['ct0']['current'], 3), round(results['ct1']['current'], 3), round(results['ct2']['current'], 3), round(results['ct3']['current'], 3), round(results['ct4']['current'], 3), round(results['ct5']['current'], 3) ]) t.add_row([ 'P.F.', round(results['ct0']['pf'], 3), round(results['ct1']['pf'], 3), round(results['ct2']['pf'], 3), round(results['ct3']['pf'], 3), round(results['ct4']['pf'], 3), round(results['ct5']['pf'], 3) ]) t.add_row([ 'Voltage', round(results['voltage'], 3), '', '', '', '', '' ]) s = t.get_string() logger.debug('\n' + s) #sleep(0.1) except KeyboardInterrupt: infl.close_db() sys.exit()
def __init__(self, bot): logger.info('==================== Start application... ====================') self._bot = bot self._bot_current_elapsed_time = time.time() self.pairs_cache = {} # key - str that contains events ids, value - date when found os.environ['MOZ_HEADLESS'] = '1'
def verify_changes(self, changes): _timestamp = changes['timestamp'] target = GeneDocMongoDBBackend(self._target_col) if changes['add']: logging.info('Verifying "add"...') # _cnt = self._target_col.find({'_id': {'$in': changes['add']}}).count() _cnt = target.count_from_ids(changes['add']) if _cnt == len(changes['add']): logging.info('...{}=={}...OK'.format(_cnt, len(changes['add']))) else: logging.info('...{}!={}...ERROR!!!'.format( _cnt, len(changes['add']))) if changes['delete']: logging.info('Verifying "delete"...') # _cnt = self._target_col.find({'_id': {'$in': changes['delete']}}).count() _cnt = target.count_from_ids(changes['delete']) if _cnt == 0: logging.info('...{}==0...OK'.format(_cnt)) else: logging.info('...{}!=0...ERROR!!!'.format(_cnt)) logging.info("Verifying all docs have timestamp...") _cnt = self._target_col.find({'_timestamp': {'$exists': True}}).count() _cnt_all = self._target_col.count() if _cnt == _cnt_all: logging.info('{}=={}...OK'.format(_cnt, _cnt_all)) else: logging.info('ERROR!!!\n\t Should be "{}", but get "{}"'.format( _cnt_all, _cnt)) logging.info("Verifying all new docs have updated timestamp...") cur = self._target_col.find({'_timestamp': { '$gte': _timestamp }}, projection={}) _li1 = sorted(changes['add'] + [x['_id'] for x in changes['update']]) _li2 = sorted([x['_id'] for x in cur]) if _li1 == _li2: logging.info("{}=={}...OK".format(len(_li1), len(_li2))) else: logging.info('ERROR!!!\n\t Should be "{}", but get "{}"'.format( len(_li1), len(_li2)))
feat = pd.read_csv('/home/ubuntu/projects/kddcup2019track1/input/data_set_phase1/var_dist_min.csv') trn = trn.merge(feat, how='left', on='sid') tst = tst.merge(feat, how='left', on='sid') trn.drop(['sid', 'click_mode'], axis=1, inplace=True) tst.drop(['sid', 'click_mode'], axis=1, inplace=True) return trn, y, tst, sub if __name__ == "__main__": df = merge_raw_data() if not os.path.exists(config.plan_file): logger.info('extracting plans from JSON objects.') plans = extract_plans(df) plans.to_csv(config.plan_file, index=False) if not os.path.exists(config.pid_feature_file): logger.info('generating pid features.') feat = generate_pid_features(df) feat.to_csv(config.pid_feature_file, index=False) if not os.path.exists(config.od_feature_file): logger.info('generating od features.') feat = generate_od_features(df) feat.to_csv(config.od_feature_file, index=False) if not os.path.exists(config.od_cluster_feature_file): logger.info('generating od cluster features.')
def Scrape(self): logger.info('Scraping %s' % self.request.get_full_url()) feed = feedparser.parse(urllib2.urlopen(self.request)) for entry in feed['entries']: pass
def get_bert_embeddings(self, processed_tweets, batch_size=128): logger.info("creating bert encodings") encoded = np.array( self.bert.encode(processed_tweets, batch_size=batch_size)) logger.info("creating bert encodings complete") return encoded
def init(self): if self.mode == "TRAIN": filters = {"lang": "en"} else: filters = {} db = MongoHelper(MONGODB_HOST_PORT, db_name=MONGO_DB_NAME) logger.info("loading data") data = db.select(MONGODB_TWEETS_COLLECTION, filters, limit=self.data_len) logger.info(f"loaded data {len(data)} items") processed_tweets = [ d.get('processed') for d in data if d.get("processed") is not None and d.get("processed") != "" ] if self.shuffle: random.shuffle(processed_tweets) self.tweets = processed_tweets self.n_batches = int(np.ceil(len(processed_tweets) / self.batch_size)) concatenate_text = ' '.join(processed_tweets) all_words = list(concatenate_text.split()) word_count = {} for w in all_words: count = word_count.get(w, 0) count += 1 word_count[w] = count self.vocab = [ w for w, i in word_count.items() if i >= MIN_WORD_FREQUENCY ] + [OOV_SYMBOL] logger.info(f"Vocab size {len(self.vocab)}") #self.vocab = list(set(all_words)) label_encoder = LabelEncoder() logger.info("creating label encoding") label_encoder.fit(self.vocab) self.label_encoder = label_encoder logger.info("creating label encoding complete") transformed_labels = label_encoder.transform(self.vocab) for vocab, label in zip(self.vocab, transformed_labels): self.vocab_dict[vocab] = label #self.index_dd = np.array(list(map(lambda y: np.array(list(map(lambda x: self.vocab_dict[x], y.split()))), processed_tweets))) self.idx2token = {v: k for (k, v) in self.vocab_dict.items()} logger.info("encoding all data") self.bert = SentenceTransformer(self.bert_model_name) self.batch_generator = self.get_batch() logger.info("Initializing complete")
def getAll(self): r = self.wintest.run_cmd('dnscmd /zoneprint {zone}'.format(zone=self.zone)) res = str(r.std_out, encoding="utf-8") logger.info(res)
def train(train_loader, model, optimizer, epoch, logger, writer): model.train() # train mode (dropout and batchnorm is used) losses = AverageMeter() times = AverageMeter() start = time.time() # Batches for i, (data) in enumerate(train_loader): # Move to GPU, if available padded_input, padded_target, input_lengths = data padded_input = padded_input.to(device) padded_target = padded_target.to(device) input_lengths = input_lengths.to(device) # Forward prop. pred, gold = model(padded_input, input_lengths, padded_target) loss, n_correct = cal_performance(pred, gold, smoothing=args.label_smoothing) try: assert (not math.isnan(loss.item())) except AssertionError: print('n_correct: ' + str(n_correct)) print('data: ' + str(n_correct)) continue # Back prop. optimizer.zero_grad() loss.backward() # Clip gradients clip_gradient(optimizer.optimizer, grad_clip) # Update weights optimizer.step() # Keep track of metrics elapsed = time.time() - start start = time.time() losses.update(loss.item()) times.update(elapsed) # Print status if i % print_freq == 0: logger.info('Epoch: [{0}][{1}/{2}]\t' 'Batch time {time.val:.5f} ({time.avg:.5f})\t' 'Loss {loss.val:.5f} ({loss.avg:.5f})'.format( epoch, i, len(train_loader), time=times, loss=losses)) writer.add_scalar('step_num/train_loss', losses.avg, optimizer.step_num) writer.add_scalar('step_num/learning_rate', optimizer.lr, optimizer.step_num) return losses.avg
def get_action(self, obs_dict, step_cnt): """ get actions :param detector_obs_list: :param fighter_obs_list: :param joint_obs_dict: :param step_cnt: :return: """ detector_action = [] fighter_action = [] for y in range(self.fighter_num): tmp_course = obs_dict['fighter'][y]['course'] # (1, ) tmp_pos = obs_dict['fighter'][y]['pos'] # (2, ) tmp_r_visible_pos = obs_dict['fighter'][y][ 'r_visible_pos'] # (10, 2) tmp_l_missile = obs_dict['fighter'][y]['l_missile'] # rule use tmp_s_missile = obs_dict['fighter'][y]['s_missile'] # rule use tmp_j_visible_fp = obs_dict['fighter'][y][ 'j_visible_fp'] # rule use tmp_j_visible_dir = obs_dict['fighter'][y][ 'j_visible_dir'] # (10, 1) tmp_g_striking_pos = obs_dict['fighter'][y][ 'g_striking_pos'] # (10, 2) tmp_r_visible_dis = obs_dict['fighter'][y][ 'r_visible_dis'] # (10, 1) tmp_striking_id = obs_dict['fighter'][y]['striking_id'] # model obs change, 归一化 if step_cnt > STEP_BEFORE_TRAIN: course = tmp_course / 359. pos = tmp_pos / self.size_x l_missile = tmp_l_missile / 2. s_missile = tmp_s_missile / 4. r_visible_pos = tmp_r_visible_pos.reshape( 1, -1)[0] / self.size_x # (20,) j_visible_dir = tmp_j_visible_dir.reshape(1, -1)[0] / 359 # (10,) # g_striking_pos = tmp_g_striking_pos.reshape(1, -1)[0] / self.size_x # (20, ) # todo striking_id = tmp_striking_id.reshape(1, -1)[0] / 1 obs = np.concatenate( (course, pos, l_missile, s_missile, r_visible_pos, j_visible_dir, striking_id), axis=0) logger.debug('obs: {}'.format(obs)) true_action = np.array([0, 1, 0, 0], dtype=np.int32) if obs_dict['fighter'][y]['alive']: # rule policy true_action = fighter_rule(tmp_course, tmp_pos, tmp_l_missile, tmp_s_missile, tmp_r_visible_pos, tmp_r_visible_dis, tmp_j_visible_dir, tmp_j_visible_fp, tmp_striking_id, tmp_g_striking_pos, step_cnt) logger.debug('true action rule out: {}'.format(true_action)) # model policy if step_cnt > STEP_BEFORE_TRAIN and not any( [any(r_visible_pos >= 0), any(j_visible_dir >= 0)]): tmp_action = self.maddpg.select_action(y, obs) logger.debug('tmp action: {}'.format(tmp_action)) # 添加动作, 将动作转换为偏角 tmp_action_i = np.argmax(tmp_action) logger.debug('tmp action i: {}'.format(tmp_action_i)) true_action[0] = tmp_action_i logger.debug('course: {}'.format(true_action[0])) logger.info('true action: {}'.format(true_action)) fighter_action.append(true_action) fighter_action = np.array(fighter_action) return detector_action, fighter_action
def main(params): t1 = time.time() # environment env = gym.make("Pendulum-v0") # env = NormalizedActions(env) env.seed(1) # ou_noise = OUNoise(env.action_space) ou_noise = GaussianNoise(env.action_space) n_states = env.observation_space.shape[0] # (4, ) n_actions = env.action_space.shape[0] # 1 -2<x<2, changed: -1<x<1 logger.info("obs num: %d" % n_states) logger.info("act num: %d" % n_actions) logger.info("act high: %d" % env.action_space.high) logger.info("act low: %d" % env.action_space.low) RL = DDPG(dim_obs=n_states, dim_act=n_actions, actor_lr=params.actor_lr, critic_lr=params.critic_lr, gamma=params.gamma, capacity=params.memory_capacity, batch_size=params.batch_size, tau=params.tau, hidden_size=params.hidden_size) # execution total_rewards = [] moving_average_rewards = [] moving_average_reward = 0.0 a_loss_list = [] c_loss_list = [] total_cnt = 0 for i_episode in range(1, params.max_eps + 1): total_reward = 0.0 # 每回合所有智能体的总体奖励 # eps_r = 0. obs = env.reset() ou_noise.reset() for i_step in range(params.max_steps): # env.render() act = RL.select_action(obs) # logger.info("select act: {}".format(act)) action = np.squeeze(act) action = ou_noise.get_action(action, i_step) # logger.info("select action: {}".format(act)) # action = env.action_space.sample() next_obs, r, done, _ = env.step(action) total_reward += r # if done: r=0 RL.memory.push(obs, action, next_obs, r, done) obs = next_obs if total_cnt > params.batch_size: a_loss, c_loss = RL.learn() a_loss_list.append(a_loss) c_loss_list.append(c_loss) if done: break total_cnt += 1 moving_average_reward = total_reward if i_episode == 1 else moving_average_reward * 0.9 + total_reward * 0.1 moving_average_rewards.append(moving_average_reward) total_rewards.append(total_reward) logger.info('episode:{}, reward:{}, step:{}'.format( i_episode, total_reward, i_step + 1)) if not IS_TEST: label = '' or 'gaussian' # file_w(total_rewards, 'reward@{}@.txt'.format(label)) file_w(moving_average_rewards, 'moving_average_reward@{}@.txt'.format(label)) # file_w(a_loss_list, 'a_loss@{}@.txt'.format(label)) # file_w(c_loss_list, 'c_loss@{}@.txt'.format(label)) env.close() t2 = time.time() logger.info('**********train finish!**************, time:%f' % (t2 - t1))
def reset_training_data(self): """ reshuffle and reset the data""" logger.info("resetting training data") if self.shuffle: random.shuffle(self.tweets) self.batch_generator = self.get_batch()
def train_net(args): torch.manual_seed(7) np.random.seed(7) checkpoint = args.checkpoint start_epoch = 0 best_acc = float('-inf') writer = SummaryWriter() epochs_since_improvement = 0 # Initialize / load checkpoint if checkpoint is None: if args.network == 'r18': model = resnet18(args) elif args.network == 'r34': model = resnet34(args) elif args.network == 'r50': model = resnet50(args) elif args.network == 'r101': model = resnet101(args) elif args.network == 'r152': model = resnet152(args) else: raise TypeError('network {} is not supported.'.format( args.network)) if args.pretrained: model.load_state_dict(torch.load('insight-face-v3.pt')) model = nn.DataParallel(model) metric_fc = ArcMarginModel(args) metric_fc = nn.DataParallel(metric_fc) if args.optimizer == 'sgd': optimizer = torch.optim.SGD([{ 'params': model.parameters() }, { 'params': metric_fc.parameters() }], lr=args.lr, momentum=args.mom, nesterov=True, weight_decay=args.weight_decay) else: optimizer = torch.optim.Adam([{ 'params': model.parameters() }, { 'params': metric_fc.parameters() }], lr=args.lr, weight_decay=args.weight_decay) else: checkpoint = torch.load(checkpoint) start_epoch = checkpoint['epoch'] + 1 epochs_since_improvement = checkpoint['epochs_since_improvement'] model = checkpoint['model'] metric_fc = checkpoint['metric_fc'] optimizer = checkpoint['optimizer'] # Move to GPU, if available model = model.to(device) metric_fc = metric_fc.to(device) # Loss function if args.focal_loss: criterion = FocalLoss(gamma=args.gamma) else: criterion = nn.CrossEntropyLoss() # Custom dataloaders # train_dataset = ArcFaceDataset('train') # train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, # num_workers=num_workers) train_dataset = ArcFaceDatasetBatched('train', img_batch_size) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size // img_batch_size, shuffle=True, num_workers=num_workers, collate_fn=batched_collate_fn) scheduler = MultiStepLR(optimizer, milestones=[8, 16, 24, 32], gamma=0.1) # Epochs for epoch in range(start_epoch, args.end_epoch): lr = optimizer.param_groups[0]['lr'] logger.info('\nCurrent effective learning rate: {}\n'.format(lr)) # print('Step num: {}\n'.format(optimizer.step_num)) writer.add_scalar('model/learning_rate', lr, epoch) # One epoch's training train_loss, train_top1_accs = train(train_loader=train_loader, model=model, metric_fc=metric_fc, criterion=criterion, optimizer=optimizer, epoch=epoch) writer.add_scalar('model/train_loss', train_loss, epoch) writer.add_scalar('model/train_accuracy', train_top1_accs, epoch) scheduler.step(epoch) if args.eval_ds == "LFW": from lfw_eval import lfw_test # One epochs's validata accuracy, threshold = lfw_test(model) elif args.eval_ds == "Megaface": from megaface_eval import megaface_test accuracy = megaface_test(model) else: accuracy = -1 writer.add_scalar('model/evaluation_accuracy', accuracy, epoch) # Check if there was an improvement is_best = accuracy > best_acc best_acc = max(accuracy, best_acc) if not is_best: epochs_since_improvement += 1 logger.info("\nEpochs since last improvement: %d\n" % (epochs_since_improvement, )) else: epochs_since_improvement = 0 # Save checkpoint save_checkpoint(epoch, epochs_since_improvement, model, metric_fc, optimizer, best_acc, is_best, scheduler)
def parse_twitter(parser): """ Main Twitter posts parsing function. :param parser: Parser :param keywords: list :return: None """ start = time.time() print(start) keywords = parser.keywords.keywords browser = parser.browser available_time = 10 * 60 * 60 / len(keywords) # 30 minutes total available logger.info("Maximum available time per word: %s seconds" % available_time) yesterday = datetime.now() - timedelta(days=1) yesterday = yesterday.strftime(" %d").replace(" 0", "") logger.info("Starting parsing process. yesterday: %str_check", yesterday) send("Starting parsing process.") proxies_iterations = 0 for ind, keyword in enumerate(keywords): print(keyword) i = 0 loaded = False while not loaded: try: url = 'https://twitter.com/search?q=' + keyword + '%20lang%3Auk&f=live' browser.get(url) WebDriverWait(browser, 7).until( expected_conditions.presence_of_element_located( (By.CSS_SELECTOR, 'section[aria-labelledby="accessible-list-0"]'))) loaded = True except: logger.error("Error with proxy! Trying %s time" % i) i += 1 proxies_iterations += 1 if i % 20 == 0: logger.info("Refreshing proxy list") parser.browser = parser.browser_setup( iterator=proxies_iterations, update_proxies=True) elif i % 2 == 0: parser.browser = parser.browser_setup(proxies_iterations) if not loaded: continue main_start = time.time() print(main_start) all_tweets = set() finished = False last_len = 0 stop_parsing_count = 0 iterations = 0 while not finished and time.time() - main_start < available_time * ( ind + 1): finished = parse_tweets(browser, all_tweets, parser) if finished == 'error': logger.info("IP Address Error. Changing it...") send("IP Address Error. Changing it...") proxies_iterations += 1 parser.browser = parser.browser_setup(proxies_iterations) finished = True continue browser.execute_script( "window.scrollTo(0, document.body.scrollHeight)") print(iterations, stop_parsing_count, len(all_tweets), (time.time() - start) / 60, all_tweets) if len(all_tweets) == last_len: stop_parsing_count += 1 else: stop_parsing_count = 0 if stop_parsing_count == 100: logger.info("IP Address exhausted. Changing it...") send("IP Address exhausted. Changing it...") send("Parsed %s tweets for word %s" % (last_len, keyword)) proxies_iterations += 1 parser.browser = parser.browser_setup(proxies_iterations) finished = True continue iterations += 1 last_len = len(all_tweets) send("Parsed %s tweets for word %s. Taken %s minutes" % (len(all_tweets), keyword, (time.time() - main_start) / 60))
def curate_nfl(): save_dir = os.path.join(Config.DATA_DIR, 'sports_bettors', 'curated', 'nfl') if not os.path.exists(save_dir): os.makedirs(save_dir) # Remove spaces, dashes from feature labels def _clean_label(l): return re.sub('\.', '', re.sub(' ', '', re.sub('-', '', l))) curation = [] logger.info('Importing NFL Data to Pandas') for team in tqdm([ fn for fn in os.listdir( os.path.join(Config.DATA_DIR, 'sports_bettors', 'raw', 'nfl')) if '_raw' in fn ]): with open( os.path.join(Config.DATA_DIR, 'sports_bettors', 'raw', 'nfl', team)) as fp: team_data = json.load(fp) # Iterate through dates for team for date, game_data in team_data.items(): # Save date information of game curated = { 'year': pd.Timestamp(date).year, 'month': pd.Timestamp(date).month, 'day': pd.Timestamp(date).day, 'away_team': game_data['teams'][0], 'home_team': game_data['teams'][1] } # Wrangle features for home team home_features = [ 'home_' + _clean_label(feature) for feature in game_data['features'] ] home_vals = [val[0] for val in game_data['values']] curated.update( {label: val for label, val in zip(home_features, home_vals)}) # Wrangle features for away team away_features = [ 'away_' + _clean_label(feature) for feature in game_data['features'] ] away_vals = [val[1] for val in game_data['values']] curated.update( {label: val for label, val in zip(away_features, away_vals)}) # Wrangle home points score_labels = ['pts_Q1', 'pts_Q2', 'pts_Q3', 'pts_Q4', 'points'] home_score_labels = ['home_' + label for label in score_labels] home_scores = [ val for val in game_data['quarter_values'][0][2:6] + [game_data['quarter_values'][0][-1]] ] curated.update({ label: val for label, val in zip(home_score_labels, home_scores) }) # Wrangle away points away_score_labels = ['away_' + label for label in score_labels] away_scores = [ val for val in game_data['quarter_values'][1][2:6] + [game_data['quarter_values'][1][-1]] ] curated.update({ label: val for label, val in zip(away_score_labels, away_scores) }) # Gather curation.append(curated) df_curation = pd.DataFrame.from_records(curation).drop_duplicates() logger.info('Saving Curation with shape: {}'.format(df_curation.shape)) df_curation.to_csv(os.path.join(save_dir, 'df_stats.csv'), index=False) # Clean up fields def _dash_curate(stat: str, idx: int): stat = re.sub('--', '-', stat) try: return int(float( stat.split('-')[idx])) if len(stat) > 2 else np.nan except Exception as err: logger.info('{}: {}'.format(stat, err)) return np.nan for home_away in ['home', 'away']: # Clean up passing stats df_curation[home_away + '_passCompletions'] = df_curation[ home_away + '_CmpAttYdTDINT'].apply(lambda stat: _dash_curate(stat, 0)) df_curation[home_away + '_passAttempts'] = df_curation[ home_away + '_CmpAttYdTDINT'].apply(lambda stat: _dash_curate(stat, 1)) df_curation[home_away + '_passYards'] = df_curation[ home_away + '_CmpAttYdTDINT'].apply(lambda stat: _dash_curate(stat, 2)) df_curation[home_away + '_passTDs'] = df_curation[ home_away + '_CmpAttYdTDINT'].apply(lambda stat: _dash_curate(stat, 3)) df_curation[home_away + '_interceptions'] = df_curation[ home_away + '_CmpAttYdTDINT'].apply(lambda stat: _dash_curate(stat, 4)) df_curation = df_curation.drop(home_away + '_CmpAttYdTDINT', axis=1) # Clean up rushing stats df_curation[home_away + '_rushAttempts'] = df_curation[ home_away + '_RushYdsTDs'].apply(lambda stat: _dash_curate(stat, 0)) df_curation[home_away + '_rushYards'] = df_curation[ home_away + '_RushYdsTDs'].apply(lambda stat: _dash_curate(stat, 1)) df_curation[home_away + '_rushTDs'] = df_curation[home_away + '_RushYdsTDs'].apply( lambda stat: _dash_curate(stat, 2)) df_curation = df_curation.drop(home_away + '_RushYdsTDs', axis=1) # Fourth Down df_curation[home_away + '_FourthDownConv'] = df_curation[ home_away + '_FourthDownConv'].fillna('0-0') df_curation[home_away + '_fourthDownConversions'] = df_curation[ home_away + '_FourthDownConv'].apply(lambda stat: _dash_curate(stat, 0)) df_curation[home_away + '_fourthDownAttempts'] = df_curation[ home_away + '_FourthDownConv'].apply(lambda stat: _dash_curate(stat, 1)) df_curation = df_curation.drop(home_away + '_FourthDownConv', axis=1) # Fumbles Lost df_curation[home_away + '_fumbles'] = df_curation[ home_away + '_FumblesLost'].apply(lambda stat: _dash_curate(stat, 0)) df_curation[home_away + '_fumblesLost'] = df_curation[ home_away + '_FumblesLost'].apply(lambda stat: _dash_curate(stat, 1)) df_curation = df_curation.drop(home_away + '_FumblesLost', axis=1) # Clean up penalties df_curation[home_away + '_penalties'] = df_curation[ home_away + '_PenaltiesYards'].apply(lambda stat: _dash_curate(stat, 0)) df_curation[home_away + '_penaltyYards'] = df_curation[ home_away + '_PenaltiesYards'].apply(lambda stat: _dash_curate(stat, 1)) df_curation = df_curation.drop(home_away + '_PenaltiesYards', axis=1) # Clean up sacks df_curation[home_away + '_sacks'] = df_curation[home_away + '_SackedYards'].apply( lambda stat: _dash_curate(stat, 0)) df_curation[home_away + '_sackedYards'] = df_curation[ home_away + '_SackedYards'].apply(lambda stat: _dash_curate(stat, 1)) df_curation = df_curation.drop(home_away + '_SackedYards', axis=1) # Third Down df_curation[home_away + '_ThirdDownConv'] = df_curation[ home_away + '_ThirdDownConv'].fillna('0-0') df_curation[home_away + '_thirdDownConversions'] = df_curation[ home_away + '_ThirdDownConv'].apply(lambda stat: _dash_curate(stat, 0)) df_curation[home_away + '_thirdDownAttempts'] = df_curation[ home_away + '_ThirdDownConv'].apply(lambda stat: _dash_curate(stat, 1)) df_curation = df_curation.drop(home_away + '_ThirdDownConv', axis=1) # Possession Time df_curation[home_away + '_TimeofPossession'] = df_curation[ home_away + '_TimeofPossession'].fillna('00:00') df_curation[home_away + '_possessionTime'] = df_curation[ home_away + '_TimeofPossession'].apply( lambda stat: float(stat.split(':')[0]) + float( stat.split(':')[1]) / 60 if len(stat) > 4 else np.nan) df_curation = df_curation.drop(home_away + '_TimeofPossession', axis=1) # Convert back to NA df_curation[home_away + '_possessionTime'] = df_curation.apply( lambda row: row[home_away + '_possessionTime'] if row['year'] > 1983 else np.nan, axis=1) df_curation[home_away + '_fourthDownConversions'] = df_curation.apply( lambda row: row[home_away + '_fourthDownConversions'] if row['year'] > 1991 else np.nan, axis=1) df_curation[home_away + '_fourthDownAttempts'] = df_curation.apply( lambda row: row[home_away + '_fourthDownAttempts'] if row['year'] > 1991 else np.nan, axis=1) df_curation[home_away + '_thirdDownConversions'] = df_curation.apply( lambda row: row[home_away + '_thirdDownConversions'] if row['year'] > 1991 else np.nan, axis=1) df_curation[home_away + '_thirdDownAttempts'] = df_curation.apply( lambda row: row[home_away + '_thirdDownAttempts'] if row['year'] > 1991 else np.nan, axis=1) # Wrangle from home / away to team / opponent df_modeling = [] all_teams = set( list(df_curation['home_team']) + list(df_curation['away_team'])) for team in tqdm(all_teams): # Games where team is home df_home = df_curation[df_curation['home_team'] == team].copy() df_home['team'] = team df_home['is_home'] = 1 df_home['opponent'] = df_home['away_team'] df_home = df_home.drop(['home_team', 'away_team'], axis=1) df_home.columns = [ re.sub('away_', 'opp_', re.sub('home_', '', col)) for col in df_home.columns ] df_modeling.append(df_home) # Games where team is away df_away = df_curation[df_curation['away_team'] == team].copy() df_away['team'] = team df_away['is_home'] = 0 df_away['opponent'] = df_away['home_team'] df_away = df_away.drop(['home_team', 'away_team'], axis=1) df_away.columns = [ re.sub('home_', 'opp_', re.sub('away_', '', col)) for col in df_away.columns ] df_modeling.append(df_away) df_modeling = pd.concat(df_modeling, sort=True).drop_duplicates().reset_index(drop=True) # Matchup def _define_matchup(main_team, opponent): return '_vs_'.join(sorted([main_team, opponent])) df_modeling['matchup'] = df_modeling.apply( lambda row: _define_matchup(row['team'], row['opponent']), axis=1) logger.info('Save Curated data for {} games.'.format(df_modeling.shape[0])) df_modeling.to_csv(os.path.join(Config.DATA_DIR, 'sports_bettors', 'curated', 'nfl', 'df_curated.csv'), index=False)
if not connection_established: raise Exception( "Could not connect to InfluxDB. Check that the container is running!" ) run_main() else: # Program launched in one of the non-main modes. Increase logging level. logger.setLevel(logging.DEBUG) logger.handlers[0].setLevel(logging.DEBUG) if 'help' in MODE.lower() or '-h' in MODE.lower(): logger.info( "\nSee the project Wiki for more detailed usage instructions: https://github.com/David00/rpi-power-monitor/wiki" ) logger.info("""\nUsage: Start the program: python3 power-monitor.py Collect raw data and build an interactive plot: python3 power-monitor.py debug "chart title here" Use the previously collected data to tune phase correction: python3 power-monitor.py phase "chart title here" """) if MODE.lower() == 'debug': # This mode is intended to take a look at the raw CT sensor data. It will take 2000 samples from each CT sensor, plot them to a single chart, write the chart to an HTML file located in /var/www/html/, and then terminate. # It also stores the samples to a file located in ./data/samples/last-debug.pkl so that the sample data can be read when this program is started in 'phase' mode. samples = collect_data(2000) ct0_samples = samples['ct0']
def gen_plan_feas(data): n = data.shape[0] mode_list_feas = np.zeros((n, 12)) max_dist, min_dist, mean_dist, std_dist = np.zeros((n,)), np.zeros((n,)), np.zeros((n,)), np.zeros((n,)) max_price, min_price, mean_price, std_price = np.zeros((n,)), np.zeros((n,)), np.zeros((n,)), np.zeros((n,)) max_eta, min_eta, mean_eta, std_eta = np.zeros((n,)), np.zeros((n,)), np.zeros((n,)), np.zeros((n,)) min_dist_mode, max_dist_mode, min_price_mode, max_price_mode, min_eta_mode, max_eta_mode, first_mode = np.zeros( (n,)), np.zeros((n,)), np.zeros((n,)), np.zeros((n,)), np.zeros((n,)), np.zeros((n,)), np.zeros((n,)) mode_texts = [] for i, plan in tqdm(enumerate(data['plans'].values)): try: cur_plan_list = json.loads(plan) except: cur_plan_list = [] if len(cur_plan_list) == 0: mode_list_feas[i, 0] = 1 first_mode[i] = 0 max_dist[i] = -1 min_dist[i] = -1 mean_dist[i] = -1 std_dist[i] = -1 max_price[i] = -1 min_price[i] = -1 mean_price[i] = -1 std_price[i] = -1 max_eta[i] = -1 min_eta[i] = -1 mean_eta[i] = -1 std_eta[i] = -1 min_dist_mode[i] = -1 max_dist_mode[i] = -1 min_price_mode[i] = -1 max_price_mode[i] = -1 min_eta_mode[i] = -1 max_eta_mode[i] = -1 mode_texts.append('word_null') else: distance_list = [] price_list = [] eta_list = [] mode_list = [] for tmp_dit in cur_plan_list: distance_list.append(int(tmp_dit['distance'])) if tmp_dit['price'] == '': price_list.append(0) else: price_list.append(int(tmp_dit['price'])) eta_list.append(int(tmp_dit['eta'])) mode_list.append(int(tmp_dit['transport_mode'])) mode_texts.append( ' '.join(['word_{}'.format(mode) for mode in mode_list])) distance_list = np.array(distance_list) price_list = np.array(price_list) eta_list = np.array(eta_list) mode_list = np.array(mode_list, dtype='int') mode_list_feas[i, mode_list] = 1 distance_sort_idx = np.argsort(distance_list) price_sort_idx = np.argsort(price_list) eta_sort_idx = np.argsort(eta_list) max_dist[i] = distance_list[distance_sort_idx[-1]] min_dist[i] = distance_list[distance_sort_idx[0]] mean_dist[i] = np.mean(distance_list) std_dist[i] = np.std(distance_list) max_price[i] = price_list[price_sort_idx[-1]] min_price[i] = price_list[price_sort_idx[0]] mean_price[i] = np.mean(price_list) std_price[i] = np.std(price_list) max_eta[i] = eta_list[eta_sort_idx[-1]] min_eta[i] = eta_list[eta_sort_idx[0]] mean_eta[i] = np.mean(eta_list) std_eta[i] = np.std(eta_list) first_mode[i] = mode_list[0] max_dist_mode[i] = mode_list[distance_sort_idx[-1]] min_dist_mode[i] = mode_list[distance_sort_idx[0]] max_price_mode[i] = mode_list[price_sort_idx[-1]] min_price_mode[i] = mode_list[price_sort_idx[0]] max_eta_mode[i] = mode_list[eta_sort_idx[-1]] min_eta_mode[i] = mode_list[eta_sort_idx[0]] feature_data = pd.DataFrame(mode_list_feas) feature_data.columns = ['mode_feas_{}'.format(i) for i in range(12)] feature_data['max_dist'] = max_dist feature_data['min_dist'] = min_dist feature_data['mean_dist'] = mean_dist feature_data['std_dist'] = std_dist feature_data['max_price'] = max_price feature_data['min_price'] = min_price feature_data['mean_price'] = mean_price feature_data['std_price'] = std_price feature_data['max_eta'] = max_eta feature_data['min_eta'] = min_eta feature_data['mean_eta'] = mean_eta feature_data['std_eta'] = std_eta feature_data['max_dist_mode'] = max_dist_mode feature_data['min_dist_mode'] = min_dist_mode feature_data['max_price_mode'] = max_price_mode feature_data['min_price_mode'] = min_price_mode feature_data['max_eta_mode'] = max_eta_mode feature_data['min_eta_mode'] = min_eta_mode feature_data['first_mode'] = first_mode logger.info('mode tfidf...') tfidf_enc = TfidfVectorizer(ngram_range=(1, 2)) tfidf_vec = tfidf_enc.fit_transform(mode_texts) svd_enc = TruncatedSVD(n_components=10, n_iter=20, random_state=2019) mode_svd = svd_enc.fit_transform(tfidf_vec) mode_svd = pd.DataFrame(mode_svd) mode_svd.columns = ['svd_mode_{}'.format(i) for i in range(10)] data = pd.concat([data, feature_data, mode_svd], axis=1) data = data.drop(['plans'], axis=1) return data
def run_main(): logger.info("Press Ctrl-c to quit...") # The following empty dictionaries will hold the respective calculated values at the end of each polling cycle, which are then averaged prior to storing the value to the DB. solar_power_values = dict(power=[], pf=[], current=[]) home_load_values = dict(power=[], pf=[], current=[]) net_power_values = dict(power=[], current=[]) ct0_dict = dict(power=[], pf=[], current=[]) ct1_dict = dict(power=[], pf=[], current=[]) ct2_dict = dict(power=[], pf=[], current=[]) ct3_dict = dict(power=[], pf=[], current=[]) ct4_dict = dict(power=[], pf=[], current=[]) ct5_dict = dict(power=[], pf=[], current=[]) rms_voltage_values = [] i = 0 # Counter for aggregate function while True: try: board_voltage = get_board_voltage() samples = collect_data(2000) poll_time = samples['time'] ct0_samples = samples['ct0'] ct1_samples = samples['ct1'] ct2_samples = samples['ct2'] ct3_samples = samples['ct3'] ct4_samples = samples['ct4'] ct5_samples = samples['ct5'] v_samples = samples['voltage'] rebuilt_waves = rebuild_waves(samples, ct0_phasecal, ct1_phasecal, ct2_phasecal, ct3_phasecal, ct4_phasecal, ct5_phasecal) results = calculate_power(rebuilt_waves, board_voltage) # # RMS calculation for phase correction only - this is not needed after everything is tuned. The following code is used to compare the RMS power to the calculated real power. # # Ideally, you want the RMS power to equal the real power when you are measuring a purely resistive load. # rms_power_0 = round(results['ct0']['current'] * results['ct0']['voltage'], 2) # AKA apparent power # rms_power_1 = round(results['ct1']['current'] * results['ct1']['voltage'], 2) # AKA apparent power # rms_power_2 = round(results['ct2']['current'] * results['ct2']['voltage'], 2) # AKA apparent power # rms_power_3 = round(results['ct3']['current'] * results['ct3']['voltage'], 2) # AKA apparent power # rms_power_4 = round(results['ct4']['current'] * results['ct4']['voltage'], 2) # AKA apparent power # rms_power_5 = round(results['ct5']['current'] * results['ct5']['voltage'], 2) # AKA apparent power # phase_corrected_power_0 = results['ct0']['power'] # phase_corrected_power_1 = results['ct1']['power'] # phase_corrected_power_2 = results['ct2']['power'] # phase_corrected_power_3 = results['ct3']['power'] # phase_corrected_power_4 = results['ct4']['power'] # phase_corrected_power_5 = results['ct5']['power'] # # diff is the difference between the real_power (phase corrected) compared to the simple rms power calculation. # # This is used to calibrate for the "unknown" phase error in each CT. The phasecal value for each CT input should be adjusted so that diff comes as close to zero as possible. # diff_0 = phase_corrected_power_0 - rms_power_0 # diff_1 = phase_corrected_power_1 - rms_power_1 # diff_2 = phase_corrected_power_2 - rms_power_2 # diff_3 = phase_corrected_power_3 - rms_power_3 # diff_4 = phase_corrected_power_4 - rms_power_4 # diff_5 = phase_corrected_power_5 - rms_power_5 # Phase Corrected Results # logger.debug("\n") # logger.debug(f"CT0 Real Power: {round(results['ct0']['power'], 2):>10} W | Amps: {round(results['ct0']['current'], 2):<7} | RMS Power: {round(results['ct0']['current'] * results['ct0']['voltage'], 2):<6} W | PF: {round(results['ct0']['pf'], 5)}") # logger.debug(f"CT1 Real Power: {round(results['ct1']['power'], 2):>10} W | Amps: {round(results['ct1']['current'], 2):<7} | RMS Power: {round(results['ct1']['current'] * results['ct1']['voltage'], 2):<6} W | PF: {round(results['ct1']['pf'], 5)}") # logger.debug(f"CT2 Real Power: {round(results['ct2']['power'], 2):>10} W | Amps: {round(results['ct2']['current'], 2):<7} | RMS Power: {round(results['ct2']['current'] * results['ct2']['voltage'], 2):<6} W | PF: {round(results['ct2']['pf'], 5)}") # logger.debug(f"CT3 Real Power: {round(results['ct3']['power'], 2):>10} W | Amps: {round(results['ct3']['current'], 2):<7} | RMS Power: {round(results['ct3']['current'] * results['ct3']['voltage'], 2):<6} W | PF: {round(results['ct3']['pf'], 5)}") # logger.debug(f"CT4 Real Power: {round(results['ct4']['power'], 2):>10} W | Amps: {round(results['ct4']['current'], 2):<7} | RMS Power: {round(results['ct4']['current'] * results['ct4']['voltage'], 2):<6} W | PF: {round(results['ct4']['pf'], 5)}") # logger.debug(f"CT5 Real Power: {round(results['ct5']['power'], 2):>10} W | Amps: {round(results['ct5']['current'], 2):<7} | RMS Power: {round(results['ct5']['current'] * results['ct5']['voltage'], 2):<6} W | PF: {round(results['ct5']['pf'], 5)}") # logger.debug(f"Line Voltage: {round(results['voltage'], 2)} V") # Prepare values for database storage grid_0_power = results['ct0']['power'] # 200A Main (left) grid_1_power = results['ct1']['power'] # 200A Main (right) grid_2_power = results['ct2']['power'] # 100A Main (top) grid_4_power = results['ct4']['power'] # 100A Main (bottom) grid_5_power = results['ct5']['power'] # Unused grid_0_current = results['ct0']['current'] grid_1_current = results['ct1']['current'] grid_2_current = results['ct2']['current'] grid_4_current = results['ct4']['current'] grid_5_current = results['ct5']['current'] solar_power = results['ct3']['power'] solar_current = results['ct3']['current'] solar_pf = results['ct3']['pf'] # Set solar power and current to zero if the solar power is under 20W. if solar_power < 20: solar_power = 0 solar_current = 0 solar_pf = 0 # Determine if the system is net producing or net consuming right now by looking at the two panel mains. # Since the current measured is always positive, we need to add a negative sign to the amperage value if we're exporting power. if grid_0_power < 0: grid_0_current = grid_0_current * -1 if grid_1_power < 0: grid_1_current = grid_1_current * -1 if solar_power > 0: solar_current = solar_current * -1 # Unless your specific panel setup matches mine exactly, the following four lines will likely need to be re-written: home_consumption_power = grid_2_power + grid_4_power + grid_0_power + grid_1_power + solar_power net_power = home_consumption_power - solar_power home_consumption_current = grid_2_current + grid_4_current + grid_0_current + grid_1_current - solar_current net_current = grid_0_current + grid_1_current + grid_2_current + grid_4_current + solar_current if net_power < 0: current_status = "Producing" else: current_status = "Consuming" # Average 2 readings before sending to db if i < 2: solar_power_values['power'].append(solar_power) solar_power_values['current'].append(solar_current) solar_power_values['pf'].append(solar_pf) home_load_values['power'].append(home_consumption_power) home_load_values['current'].append(home_consumption_current) net_power_values['power'].append(net_power) net_power_values['current'].append(net_current) ct0_dict['power'].append(results['ct0']['power']) ct0_dict['current'].append(results['ct0']['current']) ct0_dict['pf'].append(results['ct0']['pf']) ct1_dict['power'].append(results['ct1']['power']) ct1_dict['current'].append(results['ct1']['current']) ct1_dict['pf'].append(results['ct1']['pf']) ct2_dict['power'].append(results['ct2']['power']) ct2_dict['current'].append(results['ct2']['current']) ct2_dict['pf'].append(results['ct2']['pf']) ct3_dict['power'].append(results['ct3']['power']) ct3_dict['current'].append(results['ct3']['current']) ct3_dict['pf'].append(results['ct3']['pf']) ct4_dict['power'].append(results['ct4']['power']) ct4_dict['current'].append(results['ct4']['current']) ct4_dict['pf'].append(results['ct4']['pf']) ct5_dict['power'].append(results['ct5']['power']) ct5_dict['current'].append(results['ct5']['current']) ct5_dict['pf'].append(results['ct5']['pf']) i += 1 else: # Calculate the average, send the result to InfluxDB, and reset the dictionaries for the next 2 sets of data. infl.write_to_influx(solar_power_values, home_load_values, net_power_values, ct0_dict, ct1_dict, ct2_dict, ct3_dict, ct4_dict, ct5_dict, poll_time, i) solar_power_values = dict(power=[], pf=[], current=[]) home_load_values = dict(power=[], pf=[], current=[]) net_power_values = dict(power=[], current=[]) ct0_dict = dict(power=[], pf=[], current=[]) ct1_dict = dict(power=[], pf=[], current=[]) ct2_dict = dict(power=[], pf=[], current=[]) ct3_dict = dict(power=[], pf=[], current=[]) ct4_dict = dict(power=[], pf=[], current=[]) ct5_dict = dict(power=[], pf=[], current=[]) i = 0 if logger.handlers[0].level == 10: t = PrettyTable( ['', 'CT0', 'CT1', 'CT2', 'CT3', 'CT4', 'CT5']) t.add_row([ 'Watts', round(results['ct0']['power'], 3), round(results['ct1']['power'], 3), round(results['ct2']['power'], 3), round(results['ct3']['power'], 3), round(results['ct4']['power'], 3), round(results['ct5']['power'], 3) ]) t.add_row([ 'Current', round(results['ct0']['current'], 3), round(results['ct1']['current'], 3), round(results['ct2']['current'], 3), round(results['ct3']['current'], 3), round(results['ct4']['current'], 3), round(results['ct5']['current'], 3) ]) t.add_row([ 'P.F.', round(results['ct0']['pf'], 3), round(results['ct1']['pf'], 3), round(results['ct2']['pf'], 3), round(results['ct3']['pf'], 3), round(results['ct4']['pf'], 3), round(results['ct5']['pf'], 3) ]) t.add_row([ 'Voltage', round(results['voltage'], 3), '', '', '', '', '' ]) s = t.get_string() logger.debug('\n' + s) #sleep(0.1) except KeyboardInterrupt: infl.close_db() sys.exit()
def update_from_temp_collections(config, no_confirm=False, use_parallel=False): t0 = time.time() sc = GeneDocSyncer(config) new_src_li = sc.get_new_source_list() if not new_src_li: logging.info("No new source collections need to update. Abort now.") return logging.info("Found {} new source collections need to update:".format( len(new_src_li))) logging.info("\n".join(['\t' + x for x in new_src_li])) if no_confirm or ask('Continue?') == 'Y': logfile = 'databuild_sync_{}_{}.log'.format(config, time.strftime('%Y%m%d')) logfile = os.path.join(LOG_FOLDER, logfile) setup_logfile(logfile) for src in new_src_li: t0 = time.time() logging.info("Current source collection: %s" % src) ts = _get_timestamp(src, as_str=True) logging.info("Calculating changes... ") changes = sc.get_changes(src, use_parallel=use_parallel) logging.info("Done") get_changes_stats(changes) if no_confirm or ask("Continue to save changes...") == 'Y': if config == 'genedoc_mygene': dumpfile = 'changes_{}.pyobj'.format(ts) else: dumpfile = 'changes_{}_allspecies.pyobj'.format(ts) dump(changes, dumpfile) dumpfile_key = 'genedoc_changes/' + dumpfile logging.info('Saving to S3: "{}"... '.format(dumpfile_key)) send_s3_file(dumpfile, dumpfile_key) logging.info('Done.') #os.remove(dumpfile) if no_confirm or ask("Continue to apply changes...") == 'Y': sc.apply_changes(changes) sc.verify_changes(changes) logging.info('=' * 20) logging.info("Finished. %s" % timesofar(t0))
from gevent import monkey monkey.patch_all() import os from config import logger from config import App from config import Db from config import Client from config import Jsonrpc from utils.sim_btc_utils import sim_btc_utils from utils.hc_utils import hc_utils from utils.usdt_utils import usdt_utils from utils.btm_utils import btm_utils import time logger.info('Start app...') app = App db = Db client = Client jsonrpc = Jsonrpc sim_btc_utils_all = ["btc", "ltc", "ub", "bch", "doge"] sim_btc_plugin = {} for value in sim_btc_utils_all: upper = value.upper() sim_btc_config = {} if app.config.has_key(upper + "_HOST") and app.config.has_key( upper + "_PORT") and app.config.has_key(upper + "_FEE"): sim_btc_config["host"] = app.config[upper + "_HOST"] sim_btc_config["port"] = app.config[upper + "_PORT"] sim_btc_config["collect_host"] = app.config[upper + "_COLLECT_HOST"] sim_btc_config["collect_port"] = app.config[upper + "_COLLECT_PORT"]
def SendPhoneEmail(phone, link, price, average_price): msg = MIMEText('<a href="%(link)s">%(link)s</a>' % link, 'html') msg['Subject'] = '%s ($%s, average $%s)' % (phone.ToString(), price, average_price) SendEmail(msg) logger.info('Email for %s sent.' % phone.ToString())
s.connect(('10.255.255.255', 1)) IP = s.getsockname()[0] except: IP = None finally: s.close() return IP if __name__ == '__main__': # Backup config.py file try: copyfile('config.py', 'config.py.backup') except FileNotFoundError: logger.info("Could not create a backup of config.py file.") if len(sys.argv) > 1: MODE = sys.argv[1] if MODE == 'debug' or MODE == 'phase': try: title = sys.argv[2] except IndexError: title = None # Create the data/samples directory: try: os.makedirs('data/samples/') except FileExistsError: pass else: MODE = None