def SameSvnNode(self, node):
"""
to check whether the repo infos(url, revesion/commit_id) of local directroy equals to node's
Args:
node : BrocNode object of module
Returns:
return Ture | False
Raises:
raise BrocTreeError when encounter some errors
"""
infos = RepoUtil.GetSvnUrlRevision(node.module.root_path,
Log.Log())
if infos is None:
raise BrocTreeError("parse module(%s) failed" %
node.module.url)
# check url
if infos[0] != node.module.url:
return False
# check version
if not node.module.revision:
last_version = RepoUtil.GetSvnRevisionFromUrl(
node.module.url, Log.Log())
if last_version and infos[1] == last_version:
return True
else:
return False
else:
return True if infos[1] == node.module.revision else False
def DoCopy(self):
"""
if .a file has been built already before compile action, just copy it from code directory to output directory
Returns:
return True if copy success
return False if fail to copy
"""
if len(self.tag_sources.V()):
Log.Log().LevPrint(
"ERROR", 'StaticLibrary(%s) can not copy because \
its source is not empty' % self.name)
return False
root, _ = os.path.splitext(self.name)
frm = os.path.join(self.env.ModuleCVSPath(), 'lib',
'lib%s%s' % (root, '.a'))
to = os.path.join("broc_out", self.env.ModuleCVSPath(), 'output/lib')
cmd = "mkdir -p %(to)s && cp -Rp %(frm)s %(to)s" % (locals())
Log.Log().LevPrint('MSG', '[PreCopy] %s' % cmd)
ret, msg = Function.RunCommand(cmd)
if ret != 0:
Log.Log().LevPrint("ERROR", "[ERROR] %s\n%s" % (cmd, msg))
return False
else:
return True
def Write(self, value):
"""Write the given value to the file
@param value: (Unicode) String of text to write to disk
@note: exceptions are allowed to be raised for the writing
but
"""
# Check if a magic comment was added or changed
tbuff = StringIO(value)
enc = CheckMagicComment([tbuff.readline() for x in range(2)])
tbuff.close()
del tbuff
# Update encoding if necessary
if enc is not None:
self.encoding = enc
# Open and write the file
if self.DoOpen('wb'):
Log("[ed_txt][info] Opened %s, writing as %s" %
(self.path, self.encoding))
writer = codecs.getwriter(self.encoding)(self._handle)
if self.HasBom():
Log("[ed_txt][info] Adding BOM back to text")
value = self.bom + value
writer.write(value)
writer.close()
Log("[ed_txt][info] %s was written successfully" % self.path)
else:
raise WriteError, self.last_err
def main():
if len(sys.argv) != 3 and len(sys.argv) != 5:
Log("ERR. Incorrent number of args.")
return
if sys.argv[1] != "-f":
Log("ERR. No '-f' param given.")
return
wp = -1
if len(sys.argv) > 3:
if sys.argv[3] != "-n":
Log("ERR. No -n param given")
return
wp = int(sys.argv[4])
f = open(sys.argv[2], "r")
rep = json.load(f)
# Log(json.dumps(rep, indent=2, sort_keys=True))
Log("Plotting generation " + str(rep["Reports"][wp]["Generation"]))
for i, r in enumerate(rep["Reports"][wp]["Reports"]):
x, y = ConstellationToXY(r["constellation"])
plt.subplot(331 + i)
plt.plot(x, y, "*")
plt.xlim([-2, 2])
plt.ylim([-2, 2])
plt.grid()
plt.title("Individual " + str(i))
plt.show()
def Read(self):
"""Get the contents of the file as a string, automatically handling
any decoding that may be needed.
@return: unicode str
"""
if self.DoOpen('rb'):
lines = [self._handle.readline() for x in range(2)]
self._handle.seek(0)
enc = None
if len(lines):
enc = CheckBom(lines[0])
if enc is None:
self.bom = None
enc = CheckMagicComment(lines)
if enc:
self._magic['comment'] = enc
else:
Log("[ed_txt][info] File Has %s BOM" % enc)
self.bom = unicode(BOM.get(enc, None), enc)
if enc is not None:
self.encoding = enc
try:
reader = codecs.getreader(self.encoding)(self._handle)
txt = reader.read()
reader.close()
except Exception, msg:
Log("[ed_txt][err] Error while reading with %s" %
self.encoding)
Log("[ed_txt][err] %s" % msg)
self.last_err = str(msg)
self.Close()
if self._magic['comment']:
self._magic['bad'] = True
enc, txt = FallbackReader(self.path)
if enc is not None:
self.encoding = enc
else:
raise UnicodeDecodeError, msg
if self.bom is not None:
Log("[ed_txt][info] Stripping %s BOM from text" %
self.encoding)
txt = txt.replace(self.bom, u'', 1)
Log("[ed_txt][info] Decoded %s with %s" %
(self.path, self.encoding))
return txt
def scratch(target_dir):
"""
create a BCLOUD template
Args:
target_dir : the directory of BROC file
"""
broc_file = os.path.join(target_dir, 'BROC')
if os.path.exists(broc_file):
Log.Log().LevPrint("ERROR", "BROC already existed in %s, couldn't create it" % target_dir)
return -1
with open(broc_file, 'w') as f:
f.write(BROC_TEMPLATE)
Log.Log().LevPrint("MSG", 'create %s ...... ok!' % broc_file)
return 0
def run():
options = ChromeOptions()
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
chrome = webdriver.Remote(
command_executor='http://172.28.0.13:5555/wd/hub',
desired_capabilities=DesiredCapabilities.CHROME,
options=options)
logger = Log()
my_act = action(chrome)
p_price = Process(target=collect_price, args=(
my_act,
'EURUSD',
logger,
))
# p_prof_loss = Process(target=collect_prof_loss, args=(my_act, logger, ))
my_act.login()
p_price.start()
time.sleep(5)
# p_prof_loss.start()
p_price.join()
# p_prof_loss.join()
"""
def recluster_schedule(lock):
logger = Log('re_cluster', 'logs/')
schedule.every().day.at("00:05").do(job, lock, logger)
while True:
schedule.run_pending()
logger.info('re_cluster_schedule processing...')
time.sleep(50) # 50s检查一次，保证不会错过一分钟的时间
def IsModified(self):
'''
to check whether object self changed
Returns:
-1 : the file that cacahe representing is missing
0 : not changed
1 : changed
'''
if not os.path.exists(self.pathname):
return -1
# check mtime
modify_time = None
try:
modify_time = os.stat(self.pathname).st_mtime
except BaseException:
Log.Log().LevPrint('MSG',
'get %s modify_time failed' % self.pathname)
self.modify_time = 0
return 1
if modify_time == self.modify_time:
return 0
else:
self.modify_time = modify_time
ret = 0
# check hash
_hash = Function.GetFileHash(self.pathname)
if _hash != self.hash:
self.hash = _hash
ret = 1
# Log.Log().LevPrint('MSG', '%s content changed' % self.pathname)
return ret
def order_transform(length=3600):
start_time = time.time()
dump = transform_data()
logger = Log()
while True:
try:
dump.init()
for i in range(10000000):
time_dist = int(time.time() - start_time)
if time_dist % 55 > 52:
dump.get_1s()
dump.get_1m()
if time_dist % 295 > 292:
dump.get_5m()
if time_dist % 1795 > 1792:
dump.get_30m()
if time_dist % 3595 > 2592:
dump.get_1h()
time.sleep(1)
except Exception as e:
logger.error(str(e))
print(str(e))
def Update(self):
"""
Update modify time and hash value of cache object
"""
# update modify time
modify_time = None
try:
modify_time = os.stat(self.pathname).st_mtime
except BaseException as err:
Log.Log().LevPrint(
"ERROR", "update cache(%s) failed: %s" % (self.pathname, err))
return
if self.modify_time == modify_time:
self.modified = False
self.build = False
return
self.modify_time = modify_time
# update hash
_hash = Function.GetFileHash(self.pathname)
# Log.Log().LevPrint("MSG", "update %s hash id(%s) %s --> %s" % (self.pathname, id(self), self.hash, _hash))
self.hash = _hash
self.modified = False
self.build = False
def Download(self, fname):
"""Download the file at the given path
@param fname: string
@return: (ftppath, temppath)
"""
if not self.IsActive():
raise FtpClientNotConnected, "FtpClient is not connected"
name = None
try:
try:
if not fname.startswith('.') and '.' in fname:
pre, suf = fname.rsplit('.', 1)
suf = u'.' + suf
else:
pre = fname
suf = ''
fid, name = tempfile.mkstemp(suf, pre, text=True)
def GetFileData(data, fid=fid):
"""Write the downloaded data to disk"""
os.write(fid, data + u"\n")
self.retrlines('RETR ' + fname, GetFileData)
except Exception, msg:
self._ProcessException(msg)
Log("[ftpedit][err] Download: %s" % msg)
finally:
os.close(fid)
return (u"/".join([self._curdir, fname]), name)
def IsChanged(self, target):
""""
to check whether source file changed
because need to check whether build option change, we have to take target as argument
Args:
target : the Souce.Source object
Returns:
if file is modified, return True
if file is not modified, return False
"""
# to check source file
if self.src_obj.IsChanged(None):
Log.Log().LevPrint('MSG', "%s changed" % self.pathname)
self.build_cmd = target.GetBuildCmd()
self.build = True
return True
# to check build option
if self.build_cmd != target.GetBuildCmd():
#Log.Log().LevPrint('INFO', "cache(%s, type:%s) build cmd changed" % (self.pathname, self.TYPE))
#Log.Log().LevPrint('MSG', "%s -- > %s" % (self.build_cmd, target.GetBuildCmd()))
self.build_cmd = target.GetBuildCmd()
self.build = True
return True
# to check obj file
if BrocObject.IsChanged(self, target.InFile()):
self.build_cmd = target.GetBuildCmd()
self.build = True
return True
return False
def _download_broc(self, node):
"""
download BROC file from repository
Args:
node : the BrocNode object
Returns:
return abs path of BROC file if download success
return None if download failed
"""
broc_path = None
cmd = None
# for svn
# Log.Log().LevPrint("MSG", 'download BROC %s' % node.module.url)
if node.module.repo_kind == BrocModule_pb2.Module.SVN:
hash_value = Function.CalcHash(node.module.url)
broc_url = os.path.join(node.module.url, 'BROC')
broc_path = os.path.join(self._broc_dir, "%s_BROC" % hash_value)
if node.module.revision:
broc_url = "%s -r %s" % (broc_url, node.module.revision)
cmd = "svn export %s %s" % (broc_url, broc_path)
else:
# for GIT
cmd = ""
broc_path = os.path.join(node.module.workspace, node.module.module_cvspath, 'BROC')
broc_dir = os.path.dirname(broc_path)
if not os.path.exists(broc_path):
cmd += "git clone %s %s" \
% ("%s.git" % node.module.url, "%s" % broc_dir)
if node.module.br_name and node.module.br_name != 'master':
br_name = node.module.br_name
cmd += " && cd %s && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \
% (broc_dir, br_name, br_name, br_name, br_name)
elif node.module.tag_name:
tag_name = node.module.tag_name
cmd += " && cd %s && (git checkout %s || (git fetch origin %s:%s && git checkout %s))" \
% (broc_dir, tag_name, tag_name, tag_name, tag_name)
if cmd:
Log.Log().LevPrint("MSG", "Getting BROC(%s) ..." % cmd)
ret, msg = Function.RunCommand(cmd)
if ret != 0:
Log.Log().LevPrint("ERROR", msg)
return None
return broc_path
def __init__(self,
name='Spider',
url_list_file='./urls',
output_path='./output',
interval=1,
timeout=1,
silent=False):
'''
@name: string, 定向爬虫的名字
@url_list_file: string, 种子文件
@output_path: string, 输出文件路径
@interval: int, 爬取间隔
@timeout: int, 请求超时
@silent: bool, 是否为静默打印
'''
# 设置保存爬取页面的coolie值（非必要）
cj = cookielib.LWPCookieJar()
cookie_support = urllib2.HTTPCookieProcessor(cj)
self.opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
urllib2.install_opener(self.opener)
# 设置请求头部（非必要）
self.headers = {
'Content-Type':
'application/x-www-form-urlencoded',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.89 Safari/537.36'
}
self.url_list_file = url_list_file
self.output_path = output_path
self.interval = interval
self.timeout = timeout
level = 'INFO' if silent else 'DEBUG'
self.log = Log(name, level, 'spider.log', 'a')
# 定义一个多线程共享队列，存放需要加载的url
self.queue = Queue()
# 存放非重复url集合
self.url_set = set()
# 定义多线程访问锁
self.lock = Lock()
# 匹配根域名
self.base_pattern = r'^(https?:\/\/[a-z0-9\-\.]+)[\/\?]?'
if not os.path.exists(self.url_list_file):
# 种子文件不存在
self.log.error('%s dose not exist. Exit program !' %
(self.url_list_file))
os._exit(0)
if not os.path.exists(self.output_path):
# 输出文件目录不存在
self.log.info('Create new directory %s' % (self.output_path))
os.makedirs(self.output_path)
def __init__(self, browser, name='EURUSD'):
self.browser = browser
self.wait = WebDriverWait(browser, 300)
self.last = ''
self.path = os.path.dirname(os.path.abspath(__file__))
self.path += '/'
self.name = name
if not local_test_mode:
self.logger = Log()
def DecodeString(string, encoding=None):
"""Decode the given string to Unicode using the provided
encoding or the DEFAULT_ENCODING if None is provided.
@param string: string to decode
@keyword encoding: encoding to decode string with
"""
if not encoding:
encoding = DEFAULT_ENCODING
if isinstance(string, types.UnicodeType):
try:
rtxt = codecs.getdecoder(encoding)(string)[0]
except Exception, msg:
Log("[ed_txt][err] DecodeString with %s failed" % encoding)
Log("[ed_txt][err] %s" % msg)
rtxt = string
return rtxt
def test_singleton(self):
"""
Test singleton interface
"""
logger = Log.Log()
repo_domain = 'https://github.com'
postfix = ['trunk', 'BRANCH', 'PD_BL']
root = PlanishUtil.CreateBrocModuleFromDir("..", repo_domain,
postfix[1], postfix[2],
logger)
loader1 = Syntax.BrocLoader()
loader2 = Syntax.BrocLoader()
self.assertEqual(loader1.Id(), loader2.Id())
def GetFileList(self):
"""Get list of files at the given path
@return: list of dict(isdir, name, size, date)
"""
if not self.IsActive():
raise FtpClientNotConnected, "FtpClient is not connected"
try:
code = self.retrlines('LIST', self._ProcessInput)
except Exception, msg:
Log("[ftpedit][err] GetFileList: %s" % msg)
self._ProcessException(msg)
def Disconnect(self):
"""Disconnect from the site
@return: bool
"""
try:
if self._active:
self.abort()
self._active = False
self.quit()
except Exception, msg:
self._ProcessException(msg)
Log("[ftpedit][err] Disconnect: %s" % msg)
return False
def test_UTMaster(self):
"""
test UTMaster
"""
queue = Queue.Queue()
queue.put("ls -al")
queue.put("whoami")
queue.put("gcc --version")
queue.put("gcc --watch out")
queue.put("echo 'broc is great'")
log = Log.Log()
master = UTMaster.UTMaster(queue, log)
master.Start()
self.assertEqual(1, len(master.Errors()))
def test_load_broc(self):
"""
"""
logger = Log.Log()
repo_domain = 'https://github.com'
postfix = ['trunk', 'BRANCH', 'PD_BL']
root = PlanishUtil.CreateBrocModuleFromDir("..", repo_domain,
postfix[1], postfix[2],
logger)
node = BrocTree.BrocNode(root, None, True)
Syntax.BrocLoader().SetRoot(node)
Syntax.BrocLoader().LoadBROC()
if Syntax.BrocLoader().LackBrocModules():
print(Syntax.BrocLoader().LackBrocModules())
def Login(self, user, password):
"""Login to the server
@param user: username
@param password: password
@precondition: Connect has already been called
@return: bool
"""
try:
self.login(user, password)
except ftplib.all_errors, msg:
self._ProcessException(msg)
Log("[ftplib][err] Login: %s" % msg)
return False
def ChangeDir(self, path):
"""Change the current working directory and get the list of files
@return: list
"""
if not self.IsActive():
raise FtpClientNotConnected, "FtpClient is not connected"
try:
self.cwd(path)
self._curdir = self.pwd()
except Exception, msg:
self._lasterr = msg
Log("[ftpedit][err] ChangeDir: %s" % msg)
def NewDir(self, dname):
"""Create a new directory relative to the current path
@param dname: string
"""
if not self.IsActive():
raise FtpClientNotConnected, "FtpClient is not connected"
try:
self.mkd(dname)
except Exception, msg:
self._ProcessException(msg)
Log("[ftpedit][err] NewDir: %s" % msg)
return False
def Connect(self, user, password):
"""Connect to the site
@param user: username
@param password: password
"""
try:
# First disconnect if there is an existing connection
if self.IsActive():
Log("[ftpedit][warn] Doing disconnect on Connect")
self.Disconnect()
# Connect to the server
self.connect(self._host, int(self._port))
self.login(user, password)
self._lastlogin = (user, password)
self.cwd(self._default)
self._curdir = self.pwd()
except Exception, msg:
Log("[ftpedit][err] Connect: %s" % msg)
self._ProcessException(msg)
self._lastlogin = None
return False
def NewFile(self, fname):
"""Create a new file relative to the current path
@param fname: string
"""
if not self.IsActive():
raise FtpClientNotConnected, "FtpClient is not connected"
try:
buff = StringIO('')
self.storlines('STOR ' + fname, buff)
except Exception, msg:
self._ProcessException(msg)
Log("[ftpedit][err] Upload: %s" % msg)
return False
def DeleteFile(self, fname):
"""Delete the given file
@param fname: string
@return: bool
"""
if not self.IsActive():
raise FtpClientNotConnected, "FtpClient is not connected"
try:
self.delete(fname)
except Exception, msg:
self._ProcessException(msg)
Log("[ftpedit][err] DeleteFile: %s" % msg)
return False
def Dump(self):
"""
save the infos of dependent module into file
"""
config_list = []
config_file = os.path.join(self._root.module.workspace,
self._root.module.module_cvspath,
".BROC.ORIGIN.DEPS")
self._dump(self._root, config_list, 0)
try:
with open(config_file, 'w') as f:
f.write("".join(config_list))
except IOError as err:
Log.Log().LevPrint('ERROR',
'save origin dependency failed(%s)' % err)
def test_singleton(self):
"""
Test singleton interface
"""
logger = Log.Log()
repo_domain = 'https://github.com'
postfix = ['trunk', 'BRANCH', 'PD_BL']
root = PlanishUtil.CreateBrocModuleFromDir("..", repo_domain,
postfix[1], postfix[2],
logger)
tree = BrocTree.BrocTree()
tree.SetRoot(root)
tree1 = BrocTree.BrocTree()
tree2 = BrocTree.BrocTree()
self.assertEqual(tree.Id(), tree1.Id())
self.assertEqual(tree.Id(), tree2.Id())
