def SameSvnNode(self, node): """ to check whether the repo infos(url, revesion/commit_id) of local directroy equals to node's Args: node : BrocNode object of module Returns: return Ture | False Raises: raise BrocTreeError when encounter some errors """ infos = RepoUtil.GetSvnUrlRevision(node.module.root_path, Log.Log()) if infos is None: raise BrocTreeError("parse module(%s) failed" % node.module.url) # check url if infos[0] != node.module.url: return False # check version if not node.module.revision: last_version = RepoUtil.GetSvnRevisionFromUrl( node.module.url, Log.Log()) if last_version and infos[1] == last_version: return True else: return False else: return True if infos[1] == node.module.revision else False
def DoCopy(self): """ if .a file has been built already before compile action, just copy it from code directory to output directory Returns: return True if copy success return False if fail to copy """ if len(self.tag_sources.V()): Log.Log().LevPrint( "ERROR", 'StaticLibrary(%s) can not copy because \ its source is not empty' % self.name) return False root, _ = os.path.splitext(self.name) frm = os.path.join(self.env.ModuleCVSPath(), 'lib', 'lib%s%s' % (root, '.a')) to = os.path.join("broc_out", self.env.ModuleCVSPath(), 'output/lib') cmd = "mkdir -p %(to)s && cp -Rp %(frm)s %(to)s" % (locals()) Log.Log().LevPrint('MSG', '[PreCopy] %s' % cmd) ret, msg = Function.RunCommand(cmd) if ret != 0: Log.Log().LevPrint("ERROR", "[ERROR] %s\n%s" % (cmd, msg)) return False else: return True
def Write(self, value): """Write the given value to the file @param value: (Unicode) String of text to write to disk @note: exceptions are allowed to be raised for the writing but """ # Check if a magic comment was added or changed tbuff = StringIO(value) enc = CheckMagicComment([tbuff.readline() for x in range(2)]) tbuff.close() del tbuff # Update encoding if necessary if enc is not None: self.encoding = enc # Open and write the file if self.DoOpen('wb'): Log("[ed_txt][info] Opened %s, writing as %s" % (self.path, self.encoding)) writer = codecs.getwriter(self.encoding)(self._handle) if self.HasBom(): Log("[ed_txt][info] Adding BOM back to text") value = self.bom + value writer.write(value) writer.close() Log("[ed_txt][info] %s was written successfully" % self.path) else: raise WriteError, self.last_err
def main(): if len(sys.argv) != 3 and len(sys.argv) != 5: Log("ERR. Incorrent number of args.") return if sys.argv[1] != "-f": Log("ERR. No '-f' param given.") return wp = -1 if len(sys.argv) > 3: if sys.argv[3] != "-n": Log("ERR. No -n param given") return wp = int(sys.argv[4]) f = open(sys.argv[2], "r") rep = json.load(f) # Log(json.dumps(rep, indent=2, sort_keys=True)) Log("Plotting generation " + str(rep["Reports"][wp]["Generation"])) for i, r in enumerate(rep["Reports"][wp]["Reports"]): x, y = ConstellationToXY(r["constellation"]) plt.subplot(331 + i) plt.plot(x, y, "*") plt.xlim([-2, 2]) plt.ylim([-2, 2]) plt.grid() plt.title("Individual " + str(i)) plt.show()
def Read(self): """Get the contents of the file as a string, automatically handling any decoding that may be needed. @return: unicode str """ if self.DoOpen('rb'): lines = [self._handle.readline() for x in range(2)] self._handle.seek(0) enc = None if len(lines): enc = CheckBom(lines[0]) if enc is None: self.bom = None enc = CheckMagicComment(lines) if enc: self._magic['comment'] = enc else: Log("[ed_txt][info] File Has %s BOM" % enc) self.bom = unicode(BOM.get(enc, None), enc) if enc is not None: self.encoding = enc try: reader = codecs.getreader(self.encoding)(self._handle) txt = reader.read() reader.close() except Exception, msg: Log("[ed_txt][err] Error while reading with %s" % self.encoding) Log("[ed_txt][err] %s" % msg) self.last_err = str(msg) self.Close() if self._magic['comment']: self._magic['bad'] = True enc, txt = FallbackReader(self.path) if enc is not None: self.encoding = enc else: raise UnicodeDecodeError, msg if self.bom is not None: Log("[ed_txt][info] Stripping %s BOM from text" % self.encoding) txt = txt.replace(self.bom, u'', 1) Log("[ed_txt][info] Decoded %s with %s" % (self.path, self.encoding)) return txt
def scratch(target_dir): """ create a BCLOUD template Args: target_dir : the directory of BROC file """ broc_file = os.path.join(target_dir, 'BROC') if os.path.exists(broc_file): Log.Log().LevPrint("ERROR", "BROC already existed in %s, couldn't create it" % target_dir) return -1 with open(broc_file, 'w') as f: f.write(BROC_TEMPLATE) Log.Log().LevPrint("MSG", 'create %s ...... ok!' % broc_file) return 0
def run(): options = ChromeOptions() options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") chrome = webdriver.Remote( command_executor='http://172.28.0.13:5555/wd/hub', desired_capabilities=DesiredCapabilities.CHROME, options=options) logger = Log() my_act = action(chrome) p_price = Process(target=collect_price, args=( my_act, 'EURUSD', logger, )) # p_prof_loss = Process(target=collect_prof_loss, args=(my_act, logger, )) my_act.login() p_price.start() time.sleep(5) # p_prof_loss.start() p_price.join() # p_prof_loss.join() """
def recluster_schedule(lock): logger = Log('re_cluster', 'logs/') schedule.every().day.at("00:05").do(job, lock, logger) while True: schedule.run_pending() logger.info('re_cluster_schedule processing...') time.sleep(50) # 50s检查一次,保证不会错过一分钟的时间
def IsModified(self): ''' to check whether object self changed Returns: -1 : the file that cacahe representing is missing 0 : not changed 1 : changed ''' if not os.path.exists(self.pathname): return -1 # check mtime modify_time = None try: modify_time = os.stat(self.pathname).st_mtime except BaseException: Log.Log().LevPrint('MSG', 'get %s modify_time failed' % self.pathname) self.modify_time = 0 return 1 if modify_time == self.modify_time: return 0 else: self.modify_time = modify_time ret = 0 # check hash _hash = Function.GetFileHash(self.pathname) if _hash != self.hash: self.hash = _hash ret = 1 # Log.Log().LevPrint('MSG', '%s content changed' % self.pathname) return ret
def order_transform(length=3600): start_time = time.time() dump = transform_data() logger = Log() while True: try: dump.init() for i in range(10000000): time_dist = int(time.time() - start_time) if time_dist % 55 > 52: dump.get_1s() dump.get_1m() if time_dist % 295 > 292: dump.get_5m() if time_dist % 1795 > 1792: dump.get_30m() if time_dist % 3595 > 2592: dump.get_1h() time.sleep(1) except Exception as e: logger.error(str(e)) print(str(e))
def Update(self): """ Update modify time and hash value of cache object """ # update modify time modify_time = None try: modify_time = os.stat(self.pathname).st_mtime except BaseException as err: Log.Log().LevPrint( "ERROR", "update cache(%s) failed: %s" % (self.pathname, err)) return if self.modify_time == modify_time: self.modified = False self.build = False return self.modify_time = modify_time # update hash _hash = Function.GetFileHash(self.pathname) # Log.Log().LevPrint("MSG", "update %s hash id(%s) %s --> %s" % (self.pathname, id(self), self.hash, _hash)) self.hash = _hash self.modified = False self.build = False
def Download(self, fname): """Download the file at the given path @param fname: string @return: (ftppath, temppath) """ if not self.IsActive(): raise FtpClientNotConnected, "FtpClient is not connected" name = None try: try: if not fname.startswith('.') and '.' in fname: pre, suf = fname.rsplit('.', 1) suf = u'.' + suf else: pre = fname suf = '' fid, name = tempfile.mkstemp(suf, pre, text=True) def GetFileData(data, fid=fid): """Write the downloaded data to disk""" os.write(fid, data + u"\n") self.retrlines('RETR ' + fname, GetFileData) except Exception, msg: self._ProcessException(msg) Log("[ftpedit][err] Download: %s" % msg) finally: os.close(fid) return (u"/".join([self._curdir, fname]), name)
def IsChanged(self, target): """" to check whether source file changed because need to check whether build option change, we have to take target as argument Args: target : the Souce.Source object Returns: if file is modified, return True if file is not modified, return False """ # to check source file if self.src_obj.IsChanged(None): Log.Log().LevPrint('MSG', "%s changed" % self.pathname) self.build_cmd = target.GetBuildCmd() self.build = True return True # to check build option if self.build_cmd != target.GetBuildCmd(): #Log.Log().LevPrint('INFO', "cache(%s, type:%s) build cmd changed" % (self.pathname, self.TYPE)) #Log.Log().LevPrint('MSG', "%s -- > %s" % (self.build_cmd, target.GetBuildCmd())) self.build_cmd = target.GetBuildCmd() self.build = True return True # to check obj file if BrocObject.IsChanged(self, target.InFile()): self.build_cmd = target.GetBuildCmd() self.build = True return True return False
def _download_broc(self, node): """ download BROC file from repository Args: node : the BrocNode object Returns: return abs path of BROC file if download success return None if download failed """ broc_path = None cmd = None # for svn # Log.Log().LevPrint("MSG", 'download BROC %s' % node.module.url) if node.module.repo_kind == BrocModule_pb2.Module.SVN: hash_value = Function.CalcHash(node.module.url) broc_url = os.path.join(node.module.url, 'BROC') broc_path = os.path.join(self._broc_dir, "%s_BROC" % hash_value) if node.module.revision: broc_url = "%s -r %s" % (broc_url, node.module.revision) cmd = "svn export %s %s" % (broc_url, broc_path) else: # for GIT cmd = "" broc_path = os.path.join(node.module.workspace, node.module.module_cvspath, 'BROC') broc_dir = os.path.dirname(broc_path) if not os.path.exists(broc_path): cmd += "git clone %s %s" \ % ("%s.git" % node.module.url, "%s" % broc_dir) if node.module.br_name and node.module.br_name != 'master': br_name = node.module.br_name cmd += " && cd %s && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \ % (broc_dir, br_name, br_name, br_name, br_name) elif node.module.tag_name: tag_name = node.module.tag_name cmd += " && cd %s && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \ % (broc_dir, tag_name, tag_name, tag_name, tag_name) if cmd: Log.Log().LevPrint("MSG", "Getting BROC(%s) ..." % cmd) ret, msg = Function.RunCommand(cmd) if ret != 0: Log.Log().LevPrint("ERROR", msg) return None return broc_path
def __init__(self, name='Spider', url_list_file='./urls', output_path='./output', interval=1, timeout=1, silent=False): ''' @name: string, 定向爬虫的名字 @url_list_file: string, 种子文件 @output_path: string, 输出文件路径 @interval: int, 爬取间隔 @timeout: int, 请求超时 @silent: bool, 是否为静默打印 ''' # 设置保存爬取页面的coolie值(非必要) cj = cookielib.LWPCookieJar() cookie_support = urllib2.HTTPCookieProcessor(cj) self.opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler) urllib2.install_opener(self.opener) # 设置请求头部(非必要) self.headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.89 Safari/537.36' } self.url_list_file = url_list_file self.output_path = output_path self.interval = interval self.timeout = timeout level = 'INFO' if silent else 'DEBUG' self.log = Log(name, level, 'spider.log', 'a') # 定义一个多线程共享队列,存放需要加载的url self.queue = Queue() # 存放非重复url集合 self.url_set = set() # 定义多线程访问锁 self.lock = Lock() # 匹配根域名 self.base_pattern = r'^(https?:\/\/[a-z0-9\-\.]+)[\/\?]?' if not os.path.exists(self.url_list_file): # 种子文件不存在 self.log.error('%s dose not exist. Exit program !' % (self.url_list_file)) os._exit(0) if not os.path.exists(self.output_path): # 输出文件目录不存在 self.log.info('Create new directory %s' % (self.output_path)) os.makedirs(self.output_path)
def __init__(self, browser, name='EURUSD'): self.browser = browser self.wait = WebDriverWait(browser, 300) self.last = '' self.path = os.path.dirname(os.path.abspath(__file__)) self.path += '/' self.name = name if not local_test_mode: self.logger = Log()
def DecodeString(string, encoding=None): """Decode the given string to Unicode using the provided encoding or the DEFAULT_ENCODING if None is provided. @param string: string to decode @keyword encoding: encoding to decode string with """ if not encoding: encoding = DEFAULT_ENCODING if isinstance(string, types.UnicodeType): try: rtxt = codecs.getdecoder(encoding)(string)[0] except Exception, msg: Log("[ed_txt][err] DecodeString with %s failed" % encoding) Log("[ed_txt][err] %s" % msg) rtxt = string return rtxt
def test_singleton(self): """ Test singleton interface """ logger = Log.Log() repo_domain = 'https://github.com' postfix = ['trunk', 'BRANCH', 'PD_BL'] root = PlanishUtil.CreateBrocModuleFromDir("..", repo_domain, postfix[1], postfix[2], logger) loader1 = Syntax.BrocLoader() loader2 = Syntax.BrocLoader() self.assertEqual(loader1.Id(), loader2.Id())
def GetFileList(self): """Get list of files at the given path @return: list of dict(isdir, name, size, date) """ if not self.IsActive(): raise FtpClientNotConnected, "FtpClient is not connected" try: code = self.retrlines('LIST', self._ProcessInput) except Exception, msg: Log("[ftpedit][err] GetFileList: %s" % msg) self._ProcessException(msg)
def Disconnect(self): """Disconnect from the site @return: bool """ try: if self._active: self.abort() self._active = False self.quit() except Exception, msg: self._ProcessException(msg) Log("[ftpedit][err] Disconnect: %s" % msg) return False
def test_UTMaster(self): """ test UTMaster """ queue = Queue.Queue() queue.put("ls -al") queue.put("whoami") queue.put("gcc --version") queue.put("gcc --watch out") queue.put("echo 'broc is great'") log = Log.Log() master = UTMaster.UTMaster(queue, log) master.Start() self.assertEqual(1, len(master.Errors()))
def test_load_broc(self): """ """ logger = Log.Log() repo_domain = 'https://github.com' postfix = ['trunk', 'BRANCH', 'PD_BL'] root = PlanishUtil.CreateBrocModuleFromDir("..", repo_domain, postfix[1], postfix[2], logger) node = BrocTree.BrocNode(root, None, True) Syntax.BrocLoader().SetRoot(node) Syntax.BrocLoader().LoadBROC() if Syntax.BrocLoader().LackBrocModules(): print(Syntax.BrocLoader().LackBrocModules())
def Login(self, user, password): """Login to the server @param user: username @param password: password @precondition: Connect has already been called @return: bool """ try: self.login(user, password) except ftplib.all_errors, msg: self._ProcessException(msg) Log("[ftplib][err] Login: %s" % msg) return False
def ChangeDir(self, path): """Change the current working directory and get the list of files @return: list """ if not self.IsActive(): raise FtpClientNotConnected, "FtpClient is not connected" try: self.cwd(path) self._curdir = self.pwd() except Exception, msg: self._lasterr = msg Log("[ftpedit][err] ChangeDir: %s" % msg)
def NewDir(self, dname): """Create a new directory relative to the current path @param dname: string """ if not self.IsActive(): raise FtpClientNotConnected, "FtpClient is not connected" try: self.mkd(dname) except Exception, msg: self._ProcessException(msg) Log("[ftpedit][err] NewDir: %s" % msg) return False
def Connect(self, user, password): """Connect to the site @param user: username @param password: password """ try: # First disconnect if there is an existing connection if self.IsActive(): Log("[ftpedit][warn] Doing disconnect on Connect") self.Disconnect() # Connect to the server self.connect(self._host, int(self._port)) self.login(user, password) self._lastlogin = (user, password) self.cwd(self._default) self._curdir = self.pwd() except Exception, msg: Log("[ftpedit][err] Connect: %s" % msg) self._ProcessException(msg) self._lastlogin = None return False
def NewFile(self, fname): """Create a new file relative to the current path @param fname: string """ if not self.IsActive(): raise FtpClientNotConnected, "FtpClient is not connected" try: buff = StringIO('') self.storlines('STOR ' + fname, buff) except Exception, msg: self._ProcessException(msg) Log("[ftpedit][err] Upload: %s" % msg) return False
def DeleteFile(self, fname): """Delete the given file @param fname: string @return: bool """ if not self.IsActive(): raise FtpClientNotConnected, "FtpClient is not connected" try: self.delete(fname) except Exception, msg: self._ProcessException(msg) Log("[ftpedit][err] DeleteFile: %s" % msg) return False
def Dump(self): """ save the infos of dependent module into file """ config_list = [] config_file = os.path.join(self._root.module.workspace, self._root.module.module_cvspath, ".BROC.ORIGIN.DEPS") self._dump(self._root, config_list, 0) try: with open(config_file, 'w') as f: f.write("".join(config_list)) except IOError as err: Log.Log().LevPrint('ERROR', 'save origin dependency failed(%s)' % err)
def test_singleton(self): """ Test singleton interface """ logger = Log.Log() repo_domain = 'https://github.com' postfix = ['trunk', 'BRANCH', 'PD_BL'] root = PlanishUtil.CreateBrocModuleFromDir("..", repo_domain, postfix[1], postfix[2], logger) tree = BrocTree.BrocTree() tree.SetRoot(root) tree1 = BrocTree.BrocTree() tree2 = BrocTree.BrocTree() self.assertEqual(tree.Id(), tree1.Id()) self.assertEqual(tree.Id(), tree2.Id())