def popLayer(self, layer):
Log.debug("View: Pop: %s" % layer.__class__.__name__)
if layer in self.incoming:
self.incoming.remove(layer)
if layer in self.layers and not layer in self.outgoing:
self.outgoing.append(layer)
def process(self):
try:
self._clear_result()
self._reduce_comments()
self._reduce_favers()
except Exception, ex:
Log.error(ex, "mapreduce")
def __init__(self, filename):
if(filename[0] != '/'):
Log.w(TAG, "Filename is not absolute, this may cause issues dispatching jobs.")
ffprobe = subprocess.Popen(["ffprobe","-v", "quiet", "-print_format", "json", "-show_format", "-show_streams",filename], stdout=subprocess.PIPE)
#Get everything from stdout once ffprobe exits, and
try:
ffprobe_string = ffprobe.communicate()[0]
self.ffprobe_dict=json.loads(ffprobe_string)
except ValueError:
Log.e(TAG, "File could not be read, are you sure it exists?")
ffmpeg_interlace = subprocess.Popen(["ffmpeg", "-filter:v", "idet", "-frames:v", "400", "-an", "-f", "null", "-", "-i", filename],stdout=subprocess.PIPE, stderr=subprocess.PIPE)
interlaced_details = ffmpeg_interlace.communicate()[1]
interlaced_lines = interlaced_details.split("\n")
num_progressive = 0
for line in interlaced_lines:
if line.find("idet") != -1 and line.find("Progressive") != -1:
#find the number of progressive frames in this line.
nframes = line.split("Progressive:")[1].split("Undetermined")[0]
num_progressive = num_progressive + int(nframes)
if num_progressive < 20:
self.is_interlaced = True
self.video_stream = self.parse_video(self.ffprobe_dict)
self.audio_streams = self.parse_audio(self.ffprobe_dict)
self.sub_streams = self.parse_subs(self.ffprobe_dict)
self.file_format = self.ffprobe_dict["format"]["format_name"]
self.duration = float(self.ffprobe_dict["format"]["duration"])
def keyName(value):
if value in CONTROL1:
name = "Controller 1"
control = CONTROL1
n = 0
elif value in CONTROL2:
name = "Controller 2"
control = CONTROL2
n = 1
elif value in CONTROL3:
name = "Controller 3"
control = CONTROL3
n = 2
else:
name = "Controller 4"
control = CONTROL4
n = 3
for j in range(20):
if value == control[j]:
if self.type[n] == 2:
return name + " " + drumkey4names[j]
elif self.type[n] == 3:
return name + " " + drumkey5names[j]
else:
return name + " " + guitarkeynames[j]
else:
Log.notice("Key value not found.")
return "Error"
def icon_crop(user_id, icon_path, coords):
response = {}
if not os.path.exists(icon_path):
response["status"] = False
response["message"] = "Not Found: %s" % icon_path
return response
image_path, ext = os.path.splitext(icon_path)
store_dir = _get_image_dir(ICON_PATH)
thumb_name = "u%s%s%s" % (user_id, str(int(time.time())), ext)
thumb_path = os.path.join(store_dir, thumb_name)
middle_name = "u%s%sb%s" % (user_id, str(int(time.time())), ext)
middle_path = os.path.join(store_dir, middle_name)
img = Image.open(icon_path)
left, top, width, height = tuple([int(i) for i in coords.split("|")])
box = (left, top, left+width, top+height)
img_thumb = img.crop(box)
big_size = (ICON_BIG_WIDTH, ICON_BIG_WIDTH)
img_thumb = img_thumb.resize(big_size, Image.ANTIALIAS)
img_thumb.save(middle_path, quality=150)
thumb_size = (ICON_WIDTH, ICON_WIDTH)
img_thumb = img_thumb.resize(thumb_size, Image.ANTIALIAS)
img_thumb.save(thumb_path, quality=150)
try:
os.remove(icon_path)
except Exception, ex:
Log.info(ex)
def post(self):
if not self.request.files:
self.render("account/icon.html", error=151)
return
files = self.request.files["icon"]
if len(files) == 0:
self.render("account/icon.html", error=151)
return
if files[0]["content_type"].split("/")[1] not in self.image_types:
self.render("account/icon.html", error=152)
return
image_type = files[0]["content_type"].split("/")[1]
"""TODO头像分片存储"""
ext = os.path.splitext(files[0]["filename"])[1]
filepath = "u_%s_%s%s" % (self.current_user["_id"], str(int(time.time())), ext)
file_dir = "%s/%s" % (self.settings["icon_dir"], filepath)
try:
writer = open(file_dir, "wb")
writer.write(files[0]["body"])
writer.flush()
writer.close()
except Exception, ex:
Log.error(ex)
self.render("account/icon.html", error=153)
return
def loadVideo(self, libraryName, songName):
vidSource = None
if self.songStage == 1:
songBackgroundVideoPath = os.path.join(libraryName, songName, "background.ogv")
if os.path.isfile(songBackgroundVideoPath):
vidSource = songBackgroundVideoPath
loop = False
else:
Log.warn("Video not found: %s" % songBackgroundVideoPath)
if vidSource is None:
vidSource = os.path.join(self.pathfull, "default.ogv")
loop = True
if not os.path.isfile(vidSource):
Log.warn("Video not found: %s" % vidSource)
Log.warn("Falling back to default stage mode.")
self.mode = 1 # Fallback
return
try: # Catches invalid video files or unsupported formats
Log.debug("Attempting to load video: %s" % vidSource)
self.vidPlayer = VideoLayer(self.engine, vidSource,
mute = True, loop = loop)
self.engine.view.pushLayer(self.vidPlayer)
except (IOError, VideoPlayerError):
self.mode = 1
Log.error("Failed to load song video (falling back to default stage mode):")
def __init__(self, engine, controlnum, samprate=44100):
Task.__init__(self)
self.engine = engine
self.controlnum = controlnum
devnum = self.engine.input.controls.micDevice[controlnum]
if devnum == -1:
devnum = None
self.devname = pa.get_default_input_device_info()['name']
else:
self.devname = pa.get_device_info_by_index(devnum)['name']
self.mic = pa.open(samprate, 1, pyaudio.paFloat32, input=True, input_device_index=devnum, start=False)
self.analyzer = pypitch.Analyzer(samprate)
self.mic_started = False
self.lastPeak = 0
self.detectTaps = True
self.tapStatus = False
self.tapThreshold = -self.engine.input.controls.micTapSensitivity[controlnum]
self.passthroughQueue = []
passthroughVolume = self.engine.input.controls.micPassthroughVolume[controlnum]
if passthroughVolume > 0.0:
Log.debug('Microphone: creating passthrough stream at %d%% volume' % round(passthroughVolume * 100))
self.passthroughStream = Audio.MicrophonePassthroughStream(engine, self)
self.passthroughStream.setVolume(passthroughVolume)
else:
Log.debug('Microphone: not creating passthrough stream')
self.passthroughStream = None
def work2JPG(filename, isPng = False):
filepath = FileHelper.realpath(filename)
filedir = FileHelper.dirname(filepath)
name = FileHelper.basename(filepath)
os.chdir(tpDir)
jpgCMD = """%s -quality 90 %s %s """ % (convertBin, filepath, filepath)
os.system(jpgCMD)
#return
tmpfilename = FileHelper.join(filedir, hashlib.md5(name.encode('utf-8')).hexdigest())
isSuccess = True
with open(tmpfilename, 'wb+') as tmpFile:
try:
tmpFile.write(b'MNG')
rgbname = filepath
FileHelper.writeWithSize(tmpFile, filepath)
except Exception:
Log.printDetailln ("error00 !!!", filename, "cannot convert.")
isSuccess = False
finally:
pass
if isSuccess:
FileHelper.remove(filepath)
FileHelper.rename(tmpfilename, filepath)
return 5
else:
return 2
def post(self):
user_id = self.get_argument("uid", None)
title = self.get_argument("title", "")
link = self.get_argument("link", "")
profile = self.get_argument("profile", True)
tags = self.get_argument("tags", "").split(" ")
description = self.get_argument("description", "")
image_path = self.get_argument("Filedata.path", None)
image_name = self.get_argument("Filedata.name", None)
image_md5 = self.get_argument("Filedata.md5", None)
image_size = self.get_argument("Filedata.size", None)
categories = self.get_argument("categories", None)
if not user_id: return
user_id = int(user_id)
if not image_path: return
name, ext = os.path.splitext(image_name)
response = upload_crop(image_path, ext)
if not response["status"]: return
source_path = response["source_path"].split("/upload/")[1]
thumb_path = response["thumb_path"].split("/upload/")[1]
middle_path = response["middle_path"].split("/upload/")[1]
width = response["width"]
height = response["height"]
entry = self.entry_dal.template()
entry["user_id"] = user_id
entry["user"] = self.entry_dal.dbref("users", user_id)
entry["title"] = title
entry["link"] = link
entry["profile"] = profile
entry["tags"] = tags
entry["description"] = description
entry["source"] = source_path
entry["thumb"] = thumb_path
entry["middle"] = middle_path
entry["height"] = height
entry["width"] = width
entry["md5"] = image_md5
entry["size"] = image_size
if categories:
entry["categories"] = [int(item.strip())
for item in categories.split(",")
if item.strip()]
if title:
title = title.encode("utf-8", "ignore")
keywords = [seg for seg in seg_txt_search(title) if len(seg) > 1]
if len(tags) and tags[0]:
keywords = keywords + tags
entry["_keywords"] = keywords
try:
self.entry_dal.save(entry)
self.user_dal.update_entries_count(user_id)
except Exception, ex:
Log.error(ex)
def getOptions(self, section, option):
"""
Read the preset options of a configuration key.
@param section: Section name
@param option: Option name
@return: Tuple of Key list and Values list
"""
try:
options = self.prototype[section][option].options.values()
keys = self.prototype[section][option].options.keys()
type = self.prototype[section][option].type
except KeyError:
Log.error("Config key %s.%s not defined while reading %s." % (section, option, self.fileName))
raise
optionList = []
if type != None:
for i in range(len(options)):
value = _convertValue(keys[i], type)
optionList.append(value)
return optionList, options
def print(self):
if self.empty():
return
Log.printDetailln("%s.print:" % (self.name))
while not self.empty():
task = self.get()
ret, *args = task.param
Log.printDetailln('\t', ret, *args)
def finishGame(self):
if not self.world:
Log.notice("GameEngine.finishGame called before World created.")
return
self.world.finishGame()
self.world = None
self.gameStarted = False
self.view.pushLayer(self.mainMenu)
def stop(self):
if self.mic_started:
if self.passthroughStream is not None:
Log.debug('Microphone: stopping passthrough stream')
self.passthroughStream.stop()
self.engine.removeTask(self)
self.mic.stop_stream()
self.mic_started = False
Log.debug('Microphone: stopped %s' % self.devname)
def start(self):
if not self.mic_started:
self.mic_started = True
self.mic.start_stream()
self.engine.addTask(self, synchronized=False)
Log.debug('Microphone: started %s' % self.devname)
if self.passthroughStream is not None:
Log.debug('Microphone: starting passthrough stream')
self.passthroughStream.play()
def remove(self):
""" Remove component with given name.
Returns True if the component successfully removed.
"""
Log.debug("Pubkey.remove(): removing")
self._remove_key()
super().remove()
return True
def process(self):
try:
self._clear_result()
self._reduce_entries()
self._reduce_likes()
self._reduce_followers()
self._reduce_friends()
except Exception, ex:
Log.error(ex, "mapreduce")
return
def drop_table(self):
""" Should remove the tables created by the class. Every child of
database which stores its own data should implement this function.
"""
Log.debug("Dropping Table %s" % self.__name__)
cursor = self.db_connection.cursor()
sql = "DROP TABLE "
sql += self.__name__
cursor.execute(sql)
self.db_connection.commit()
def __init__(self, name = None, target = GL_TEXTURE_2D, useMipmaps = True):
# Delete all pending textures
try:
func, args = cleanupQueue[0]
del cleanupQueue[0]
func(*args)
except IndexError:
pass
except Exception, e: #MFH - to catch "did you call glewInit?" crashes
Log.error("Texture.py texture deletion exception: %s" % e)
def loadLibrary(self):
Log.debug("Loading libraries in %s" % self.library)
self.loaded = False
self.tiersPresent = False
if self.splash:
Dialogs.changeLoadingSplashScreenText(self.engine, self.splash, _("Browsing Collection..."))
else:
self.splash = Dialogs.showLoadingSplashScreen(self.engine, _("Browsing Collection..."))
self.loadStartTime = time.time()
self.engine.resource.load(self, "libraries", lambda: Song.getAvailableLibraries(self.engine, self.library), onLoad = self.loadSongs, synch = True)
def getSoundObjectList(self, soundPath, soundPrefix, numSounds, soundExtension=".ogg"): # MFH
Log.debug("{0}1{2} - {0}{1}{2} found in {3}".format(soundPrefix, numSounds, soundExtension, soundPath))
sounds = []
for i in xrange(1, numSounds + 1):
filePath = os.path.join(soundPath, "%s%d%s" % (soundPrefix, i, soundExtension))
soundObject = Sound(self.resource.fileName(filePath))
sounds.append(soundObject)
return sounds
def setUp(self):
""" Establishes database connection
"""
Database.db_file = "test.db" # DB File for Testcase without config
Log.streamoutput()
# Log.set_verbosity(5)
# Instance objects
self.comp = Component("name", "/path/to/exe", 'hash')
self.connection = Database.db_connection
self.cursor = self.connection.cursor()
def _start(self):
# Some common tasks that need to be done when an operation starts
Log.log("Executing %s to %s" % (self.CommandType, self.DestIP))
self.DestName = IP.reverseDNS(self.DestIP)
self.DestAddress = IP.ip2int(self.DestIP)
# Current time
self.StartTime = str(datetime.utcnow()).strip("0")
self.LocalStartTime = str(datetime.now()).strip("0")
def showTutorial(self):
# evilynux - Make sure tutorial exists before launching
tutorialpath = self.engine.tutorialFolder
if not os.path.isdir(self.engine.resource.fileName(tutorialpath)):
Log.debug("No folder found: %s" % tutorialpath)
Dialogs.showMessage(self.engine, _("No tutorials found!"))
return
self.engine.startWorld(1, None, 0, 0, tutorial = True)
self.launchLayer(lambda: Lobby(self.engine))
def _merge(self, opponent):
from util.build_component import merge_components
import os
import tarfile
Log.info('About to merge components.')
merged_file = ""
with tarfile.open(self._component_file, 'r:gz') as one:
with tarfile.open(opponent._component_file, 'r:gz') as two:
merged_file = merge_components(one, two)
os.remove(opponent._component_file)
os.rename(merged_file, opponent._component_file)
raise MergedWarning('Merged components')
def add(cls, attr):
""" This method adds a new object to database and authorized_keys.
attr is expected to be a dictionary.
The attr are at least and have to be compatible with the create()
attributes:
* c_id
* f_Pubkey_pubkey
"""
Log.debug('Creating object with ssh access'
' and granting access for public key.')
return cls.create(attr)
def __init__(self, guitarScene, configFileName):
self.scene = guitarScene
self.engine = guitarScene.engine
self.config = LinedConfigParser()
self.backgroundLayers = []
self.foregroundLayers = []
self.textures = {}
self.reset()
self.wFull = None #MFH - needed for new stage background handling
self.hFull = None
# evilynux - imported myfingershurt stuff from GuitarScene
self.mode = self.engine.config.get("game", "stage_mode")
self.songStage = self.engine.config.get("game", "song_stage")
self.animatedFolder = self.engine.config.get("game", "animated_stage_folder")
# evilynux - imported myfingershurt stuff from GuitarScene w/ minor modifs
#MFH TODO - alter logic to accommodate separated animation and slideshow
# settings based on selected animated stage folder
animationMode = self.engine.config.get("game", "stage_animate")
slideShowMode = self.engine.config.get("game", "rotate_stages")
if self.animatedFolder == _("None"):
self.rotationMode = 0 #MFH: if no animated stage folders are available, disable rotation.
elif self.animatedFolder == "Normal":
self.rotationMode = slideShowMode
else:
self.rotationMode = animationMode
self.imgArr = [] #QQstarS:random
self.imgArrScaleFactors = [] #MFH - for precalculated scale factors
self.rotateDelay = self.engine.config.get("game", "stage_rotate_delay") #myfingershurt - user defined stage rotate delay
self.animateDelay = self.engine.config.get("game", "stage_animate_delay") #myfingershurt - user defined stage rotate delay
self.animation = False
self.indexCount = 0 #QQstarS:random time counter
self.arrNum = 0 #QQstarS:random the array num
self.arrDir = 1 #forwards
self.config.read(configFileName)
# evilynux - Improved stage error handling
self.themename = self.engine.data.themeLabel
self.path = os.path.join("themes",self.themename,"backgrounds")
self.pathfull = self.engine.getPath(self.path)
if not os.path.exists(self.pathfull): # evilynux
Log.warn("Stage folder does not exist: %s" % self.pathfull)
self.mode = 1 # Fallback to song-specific stage
self.loadLayers(configFileName)
def test_get(self):
from datetime import datetime
from datetime import timedelta
self.comp.create_table()
self.comp.save()
# Test get all
comps = Component.get()
self.assertIsNotNone(comps, "Could not retrieve Component")
self.assertEqual(comps[0], self.comp, "Could not deserialize data")
time_since = datetime.today() - timedelta(minutes=10)
comps = Component.get(time_since)
Log.debug("test_get(): comps: " + str(comps))
self.assertEqual(comps[0], self.comp, "Could not deserialize data")
def load(self, target = None, name = None, function = lambda: None, synch = False, onLoad = None, onCancel = None):
if self.logLoadings == 1:
Log.notice("Loading %s.%s %s" % (target.__class__.__name__, name, synch and "synchronously" or "asynchronously"))
l = Loader(target, name, function, self.resultQueue, self.loaderSemaphore, onLoad = onLoad, onCancel = onCancel)
if synch:
l.load()
return l.finish()
else:
self.loaders.append(l)
l.start()
return l
def parse_video(self, ffprobe_dict):
if ffprobe_dict == False:
return
foundVideo = False
video_stream = {}
for stream in ffprobe_dict["streams"]:
if stream["codec_type"] == "video":
if foundVideo:
Log.w(TAG, "File had more than one video stream. Using the first one. This is unsupported!")
foundVideo = True
video_stream = {"index": stream["index"], "width": stream["width"], "height": stream["height"], "codec": stream["codec_name"] }
return video_stream
class action:
def __init__(self, browser, name='EURUSD'):
self.browser = browser
self.wait = WebDriverWait(browser, 300)
self.last = ''
self.path = os.path.dirname(os.path.abspath(__file__))
self.path += '/'
self.name = name
if not local_test_mode:
self.logger = Log()
def extract(self, info):
pass
def load_url(self, url):
self.browser.get(url)
def save_source(self, fn='workspace/data/tmp.html'):
with open(fn, 'w') as f:
f.write(self.browser.page_source)
def save(self, obj, fn='workspace/data/print.txt'):
with open(fn, 'w') as f:
f.write(obj)
def login(self, account='account', verbose=False):
if local_test_mode:
pardir = os.path.join(self.path, os.pardir)
fn = pardir + '/data/' + account + '.txt'
else:
fn = 'workspace/data/' + account + '.txt'
with open(fn, 'r') as f:
lines = f.readlines()
user_index = [i for i in range(len(lines))]
index = random.choice(user_index)
user, pwd = lines[index].split(' ')
self.load_url('https://app.plus500.com/')
self.wait.until(EC.presence_of_element_located((By.ID, 'demoMode')))
demoMode = self.browser.find_element_by_id('demoMode')
demoMode.click()
self.wait.until(EC.presence_of_element_located((By.ID, 'newUserCancel')))
demoMode = self.browser.find_element_by_id('newUserCancel')
demoMode.click()
self.wait.until(EC.presence_of_element_located((By.ID, 'email')))
demoMode = self.browser.find_element_by_id('email')
demoMode.send_keys(user)
demoMode = self.browser.find_element_by_id('password')
time.sleep(1)
demoMode.send_keys(pwd)
if verbose:
print(user, pwd)
time.sleep(1)
try:
demoMode = self.browser.find_element_by_id('submitLogin')
# demoMode.screenshot('workspace/data/sub.png')
demoMode.click()
except StaleElementReferenceException:
time.sleep(5)
return
except NoSuchElementException:
time.sleep(5)
return
except WebDriverException:
time.sleep(5)
print('opps')
return
time.sleep(5)
self.wait.until(EC.presence_of_element_located((By.XPATH, '//span[@data-win-bind="textContent: AvailableBalance Converters.formatMoneyAmountFloor; winControl.innerHTML: AvailableBalanceExplained Converters.emptyConverter"]')))
def get_forex_page(self):
for _ in range(5):
try:
button = self.browser.find_element_by_xpath('//*[contains(text(), "Popular Pairs")]')
button.screenshot('workspace/data/popular_pairs.png')
button.click()
time.sleep(1)
return True
except NoSuchElementException:
self.save_source()
time.sleep(1)
except ElementClickInterceptedException:
self.save_source()
time.sleep(1)
return False
def get_crypto_page(self):
button = self.browser.find_element_by_xpath('//a[contains(text(), "Crypto Currencies")]')
button.screenshot('workspace/data/crypto_crurencies.png')
button.click()
time.sleep(1)
def collect_cc(self, name='Bitcoin', period=10, frequency=0.1, limit=1200, length=3600):
# collect crypto currencies data
root_p = '//span[@class="drag"]'
pair_p = root_p+'/following-sibling::span[1]'
parent_p = root_p+'/parent::div/following-sibling::'
change_p = parent_p+'div[@class="change"]'
buy_p = parent_p+'div[@class="buy"]'
sell_p = parent_p+'div[@class="sell"]'
pairs = self.browser.find_elements_by_xpath(pair_p)
changes = self.browser.find_elements_by_xpath(change_p)
sells = self.browser.find_elements_by_xpath(sell_p)
buys = self.browser.find_elements_by_xpath(buy_p)
start_time = time.time()
index = None
fn = 'workspace/data/'+name+'_new_record'
# extract data
while True:
time_dist = time.time()-start_time
if time_dist > period:
break
if index is None:
pairs = [p.text for p in pairs]
index = pairs.index(name)
data = [name, changes[index].text, \
sells[index].text, \
buys[index].text]
print(buys)
self._add_data(data, fn)
time.sleep(frequency)
def collect(self, pair='EURUSD', period=10, frequency=0.2, limit=1200, length=3600):
pair_ = pair[:3]+'/'+pair[3:]
success = self.get_forex_page()
if not success:
raise
start_time = time.time()
index = None
fn = 'workspace/data/'+pair+'_new_record'
# extract data
cnt = 0
pre_data = []
# elements
root_p = '//span[@class="drag"]'
pair_p = root_p+'/following-sibling::span[1]'
parent_p = root_p+'/parent::div/following-sibling::'
change_p = parent_p+'div[@class="change"]'
buy_p = parent_p+'div[@class="buy"]'
sell_p = parent_p+'div[@class="sell"]'
p_l = self.browser.find_element_by_xpath('//span[@data-win-bind="textContent: AvailableBalance Converters.formatMoneyAmountFloor; winControl.innerHTML: AvailableBalanceExplained Converters.emptyConverter"]')
# high_low_p = parent_p+'div[@class="high-low"]'
pairs = self.browser.find_elements_by_xpath(pair_p)
changes = self.browser.find_elements_by_xpath(change_p)
sells = self.browser.find_elements_by_xpath(sell_p)
buys = self.browser.find_elements_by_xpath(buy_p)
success = self.search_index(pair)
if not success:
raise
while True:
time_dist = time.time()-start_time
if time_dist > period:
break
if index is None:
pairs = [p.text for p in pairs]
index = pairs.index(pair_)
data = [pair_, changes[index].text, \
sells[index].text, \
buys[index].text]
cnt += 1
if cnt % 100 == 1:
try:
p_l.click()
except ElementClickInterceptedException as e:
print(e)
raise
if data == pre_data:
print('get the same data')
self.get_forex_page()
time.sleep(1)
pairs = self.browser.find_elements_by_xpath(pair_p)
changes = self.browser.find_elements_by_xpath(change_p)
sells = self.browser.find_elements_by_xpath(sell_p)
buys = self.browser.find_elements_by_xpath(buy_p)
print('search index')
success = self.search_index(pair)
if not success:
raise
pre_data = data
self._add_data(data, fn)
time.sleep(frequency)
def collect_prof_loss(self, period, freq):
for i in range(10):
try:
p_l = self.browser.find_element_by_xpath('//span[@data-win-bind="textContent: TotalNetProfitLoss Converters.formatMoneyAmount; winControl.innerHTML: TotalNetProfitLossExplained Converters.emptyConverter"]')
break
except NoSuchElementException:
self.save_source()
time.sleep(1)
start_time = time.time()
while True:
time_dist = time.time()-start_time
try:
p_l.click()
except ElementClickInterceptedException as e:
print(e)
raise
if time_dist > period:
break
self.logger.write_available('EURUSD', p_l.text)
time.sleep(freq)
def collect_available(self, period, freq):
self.save_source()
for i in range(10):
try:
p_l = self.browser.find_element_by_xpath('//span[@data-win-bind="textContent: AvailableBalance Converters.formatMoneyAmountFloor; winControl.innerHTML: AvailableBalanceExplained Converters.emptyConverter"]')
except NoSuchElementException as e:
self.save_source()
time.sleep(1)
continue
break
start_time = time.time()
while True:
time_dist = time.time() - start_time
if time_dist > period:
break
self.logger.write_available('EURUSD', p_l.text)
time.sleep(freq)
def name_map(self, name):
if name == 'EURUSD':
return 'EUR/USD'
else:
return name
def search_index(self, name='EURUSD'):
root_p = '//span[@class="drag"]'
pair_p = root_p+'/following-sibling::span[1]'
for _ in range(5):
try:
pairs = self.browser.find_elements_by_xpath(pair_p)
pairs = [p.text for p in pairs]
self.index = pairs.index(self.name_map(name))
return True
except Exception as e:
print(e)
return False
def get_opening_price(self, name='EURUSD'):
root_p = '//span[@class="drag"]'
parent_p = root_p+'/parent::div/following-sibling::'
change_p = parent_p+'div[@class="change"]'
sell_p = parent_p+'div[@class="sell"]'
changes = self.browser.find_elements_by_xpath(change_p)
sells = self.browser.find_elements_by_xpath(sell_p)
change = changes[self.index].text
change = float(change.strip('%'))/100
sell = float(sells[self.index].text)
opening_price = sell*(1+change)
return opening_price
def set_close_at_loss(self, limit=None):
try:
checkbox = self.browser.find_element_by_id('close-at-loss-checkbox')
checkbox.click()
time.sleep(0.1)
inputs = self.browser.find_element_by_id('close-at-loss-rate')
if limit is not None:
while True:
inputs.send_keys(Keys.CONTROL+'a')
time.sleep(0.2)
inputs.send_keys(Keys.DELETE)
time.sleep(0.2)
inputs.send_keys(limit)
time.sleep(0.2)
value = inputs.get_attribute('value')
if (value) == limit:
break
return True
except Exception as e:
print(e)
return False
def set_close_at_prof(self, limit=None):
try:
checkbox = self.browser.find_element_by_id('close-at-profit-checkbox')
checkbox.click()
time.sleep(0.1)
inputs = self.browser.find_element_by_id('close-at-profit-rate')
if limit is not None:
while True:
inputs.send_keys(Keys.CONTROL+'a')
time.sleep(0.2)
inputs.send_keys(Keys.DELETE)
time.sleep(0.2)
inputs.send_keys(limit)
time.sleep(0.2)
value = inputs.get_attribute('value')
if value == limit:
break
return True
except Exception:
return False
def buy(self, name='EURUSD', unit=1000, prof_lim=None, loss_lim=None):
print('buy function')
button = self.browser.find_elements_by_xpath('//button[@class="buySellButton" and contains(text(), "Buy")]')[self.index]
button.click()
self.wait.until(EC.presence_of_element_located((By.XPATH, '//button[@id="trade-button"]')))
try:
inputs = self.browser.find_element_by_xpath('//input[@id="amount-input"]')
while True:
inputs.send_keys(Keys.CONTROL+'a')
time.sleep(0.3)
inputs.send_keys(Keys.DELETE)
time.sleep(0.3)
inputs.send_keys(unit)
time.sleep(0.3)
value = inputs.get_attribute('value')
value = re.sub('\D', '', value)
if float(value) == float(unit):
break
if not self.set_close_at_loss(loss_lim):
return False
if not self.set_close_at_prof(prof_lim):
return False
button = self.browser.find_element_by_xpath('//button[@id="trade-button"]')
button.click()
except NoSuchElementException:
print('no such element in buy function')
return False
except StaleElementReferenceException:
print("can't attach the element")
return False
time.sleep(1)
if not local_test_mode:
self.logger.write_available('EURUSD', 'buy')
return True
def sell(self, unit=1000, prof_lim=None, loss_lim=None):
print('sell function')
button = self.browser.find_elements_by_xpath('//button[@class="buySellButton" and contains(text(), "Sell")]')[self.index]
button.click()
self.wait.until(EC.presence_of_element_located((By.XPATH, '//button[@id="trade-button"]')))
try:
inputs = self.browser.find_element_by_xpath('//input[@id="amount-input"]')
while True:
inputs.send_keys(Keys.CONTROL+'a')
time.sleep(0.2)
inputs.send_keys(Keys.DELETE)
time.sleep(0.2)
inputs.send_keys(unit)
time.sleep(0.2)
value = inputs.get_attribute('value')
value = re.sub('\D', '', value)
if float(value) == float(unit):
break
if not self.set_close_at_loss(loss_lim):
return False
if not self.set_close_at_prof(prof_lim):
return False
button = self.browser.find_element_by_xpath('//button[@id="trade-button"]')
button.click()
except NoSuchElementException:
print('no such element in sell fucntion')
return False
except StaleElementReferenceException:
print("can't attach the element")
return False
time.sleep(1)
if not local_test_mode:
self.logger.write_available('EURUSD', 'sell')
return True
def close(self):
print('close function')
for _ in range(20):
try:
buttons = self.browser.find_elements_by_xpath('//button[@class="close-position icon-times"]')
except Exception as e:
self.save_source()
print(e)
time.sleep(1)
continue
break
time.sleep(0.3)
for button in buttons:
try:
button.click()
except StaleElementReferenceException:
return False
time.sleep(0.2)
for _ in range(20):
try:
close_bt = self.browser.find_element_by_xpath('//button[@id="closePositionBtn"]')
except Exception as e:
print(e)
time.sleep(1)
close_bt.click()
time.sleep(0.2)
if not local_test_mode:
self.logger.write_available('EURUSD', 'close')
return True
def take_action(self, period, freq):
fn = 'workspace/data/decision'
start_time = time.time()
pre_action = None
while True:
time_dist = time.time()-start_time
if time_dist > period:
print('take action stop')
break
with open(fn, 'r') as f:
line = f.readline()
line = line.strip('\n')
line = line.split('|')
if self.last == line[0]:
time.sleep(freq)
continue
else:
self.last = line[0]
if len(line) < 2:
print(line)
action = line[1]
else:
action = line[1]
upper = line[2]
lower = line[3]
# print('pre action is:', pre_action)
if pre_action is None:
pre_action = 'wait'
time.sleep(freq)
continue
elif pre_action == 'sell fail':
action = 'sell'
elif pre_action == 'buy fail':
action = 'buy'
else:
if pre_action != action and action != 'wait':
print('take action: close')
success = self.close()
if not success:
pre_action = 'close fail'
else:
pre_action = action
else:
action = 'wait'
if action == 'sell':
print('take_action: sell')
success = self.sell(prof_lim=lower, loss_lim=upper)
if not success:
pre_action = 'sell fail'
elif action == 'buy':
print('take_action: buy')
success = self.buy(prof_lim=upper, loss_lim=lower)
if not success:
pre_action = 'buy fail'
time.sleep(freq)
def request_responser(self, request_fn='workspace/data/control/request', sleep_time=10):
# request format
# request: True/False
# open price: True/False
while True:
try:
with open(request_fn, 'r') as f:
request = json.load(f)
f.close()
if request['request'] is False:
time.sleep(sleep_time)
continue
reqs = request['order']
except Exception:
time.sleep(sleep_time)
continue
data = {}
for r in reqs:
if r == 'open price':
data[r] = self.get_opening_price(self.name)
if r == 'search index':
self.search_index(self.name)
while True:
success = IO.write_response(data)
if success:
success = IO.close_request()
if success:
break
time.sleep(1)
time.sleep(sleep_time)
def _add_data(self, data, fn):
# data: list of data
data = [datetime.now()] + data
data = ' '.join(str(v) for v in data)
cmd = 'echo "' + data + '" > ' + fn
subprocess.call(cmd, shell=True)
def __del__(self):
self.browser.quit()
# -*- coding: utf-8 -*-
import os
import json
import traceback
import inspect
import requests
from flask import Flask, request
from util import Util, Constants, Log, CodeReturn
log = Log('AccountingRoute')
util = Util()
constants = Constants()
codeReturn = CodeReturn()
class AccountingRoute:
def get_resume(self, request):
try:
header = request.headers
# Get Token from Header
token = str(header['token'])
# Get datas from PARAMS
data = json.loads(str(request.args.get('data')).replace("'", '"'))
companies_token = data['companies_token']
date = data['date']
except:
log.error(
inspect.getframeinfo(inspect.currentframe()).function,
def NewGeneration(reps):
scores = np.zeros(len(reps))
for i, rep in enumerate(reps):
scores[i] = ScoreReport(rep)
rankings = np.flipud(np.argsort(scores))
Log("rankings:" + str(rankings))
newpop = []
# Keep the top half of the individuals
nKeep = round(len(rankings) / 2 + 0.1)
nKeep = 2 # TODO: remove this, testing...
for i, r in enumerate(rankings):
if i > nKeep - 1:
break
# Deep copy parent constellation
constellation = []
for j, c in enumerate(reps[r]["constellation"]):
constellation.append({"I": c["I"], "Q": c["Q"]})
newpop.append(Individual(reps[r]["M"], reps[r]["spSym"],
constellation))
# Breed and mutate for the rest of the population
nNew = len(rankings) - nKeep
for i in range(0, nNew):
# Randomly choose mate1
mate1 = np.random.randint(0, nKeep)
# Randomly choose mate2, don't choose mate1 as mate2
mate2 = mate1
while mate2 == mate1:
mate2 = np.random.randint(0, nKeep)
# Decide what value of M to take
b1 = np.log2(reps[rankings[mate1]]["M"])
b2 = np.log2(reps[rankings[mate2]]["M"])
# average the two Ms, select closest value
b = round((b1 + b2) / 2)
# randomly add or subtract
if np.random.uniform() < M_MUTATION_CHANCE:
Log("---- Mutation of M from " + str(2**b), end="")
b += (2 * np.random.randint(0, 2)) - 1
Log(" to " + str(2**b) + " ----")
M = 2**b
# M must be at least 2
if M < 2:
M = 2
# Decide what value of SpSym to take
s = round((reps[rankings[mate1]]["spSym"] +
reps[rankings[mate1]]["spSym"]) / 2 +
np.random.normal(0, SPSYM_STDDEV))
if s < 2:
s = 2
# Decide what constellation to use
wParent = mate1
if np.random.uniform() > 0.5:
wParent = mate2
constellation = []
for j, c in enumerate(reps[rankings[wParent]]["constellation"]):
constellation.append({"I": c["I"], "Q": c["Q"]})
for j in range(0, len(constellation)):
c = constellation[j]
constellation[j]["I"] = c["I"] + np.random.normal(0, CPT_STDDEV)
constellation[j]["Q"] = c["Q"] + np.random.normal(0, CPT_STDDEV)
nPtsNeeded = M - len(constellation)
for j in range(0, nPtsNeeded + 1):
constellation.append(RandomConstellationPoint())
if len(constellation) > M:
constellation = constellation[0:M]
newpop.append(Individual(M, s, constellation))
return newpop
from flask import Flask
from rest import bp_camera, bp_search, bp_peer, bp_freq, bp_repo, bp_trace
from util import Mysql
app = Flask(__name__)
app.register_blueprint(bp_camera, url_prefix='/camera')
app.register_blueprint(bp_search, url_prefix='/search')
app.register_blueprint(bp_peer)
app.register_blueprint(bp_trace)
app.register_blueprint(bp_freq)
app.register_blueprint(bp_repo, url_prefix='/repository')
from util import Log
logger = Log('app', 'logs/')
config = configparser.ConfigParser()
config.read('./app.config')
# 设置数据库mode
conn = pymysql.connect(host=config.get('db', 'host'),
port=config.getint('db', 'port'),
user=config.get('db', 'user'),
password=config.get('db', 'password'),
connect_timeout=60)
cur = conn.cursor()
cur.execute("alter database {} character set utf8;".format(
config.get('db', 'db')))
cur.execute(
"SET GLOBAL sql_mode = 'STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';"
)
# -*- coding: utf-8 -*-
import mysql.connector
import datetime
import json
import traceback
import inspect
from dateutil.relativedelta import *
from datetime import date
from util import Util, Log, Constants, CodeReturn
from controller import Controller
from flask import Flask
from flask_mysqldb import MySQL
log = Log('UserDAO')
constants = Constants()
util = Util()
codeReturn = CodeReturn()
controller = Controller()
mySQL = MySQL()
app = Flask(__name__)
class UserDAO:
def __init__(self):
mySQL.init_app(app)
def insert_user(self, info):
try:
cursorMySQL = mySQL.connection.cursor()
def popAllLayers(self):
Log.debug("View: Pop all")
[self.popLayer(l) for l in list(self.layers)]
def train(global_step):
kl_targ = 0.025
def _kl(old_probs, new_probs):
kl = np.mean(
np.sum(old_probs *
(np.log(old_probs + 1e-10) - np.log(new_probs + 1e-10)),
axis=1))
return kl
global train_batch_size
global self_play_data
global player
global max_sim
global best_win_ratio
global train_step
global lr_multiplier
self_play()
if len(self_play_data) >= train_batch_size:
train_data = random.sample(self_play_data, k=train_batch_size)
state_batch = [d[0] for d in train_data]
action_prob_batch = [d[1].reshape([d[1].size]) for d in train_data]
winner_batch = np.array([d[2] for d in train_data])
winner_batch = winner_batch.reshape((winner_batch.size, 1))
nn = player.mcts.net
old_probs, old_v = nn.session.run(
[nn.action_prob, nn.value],
feed_dict={nn.state_input: state_batch})
for _ in range(5):
__, loss, lr = nn.session.run(
[nn.train_op, nn.loss, nn.learning_rate],
feed_dict={
nn.state_input:
state_batch,
nn.action_prob_input:
action_prob_batch,
nn.value_input:
winner_batch,
nn.train_step:
train_step + 1,
# nn.learning_rate: 1e-5
nn.learning_rate:
lr_multiplier * 0.005
})
train_step += 1
new_probs, new_v = nn.session.run(
[nn.action_prob, nn.value],
feed_dict={nn.state_input: state_batch})
kl = _kl(old_probs, new_probs)
print('lr_mul={:.5f} lr={:.5f} kl={:.5f} loss={:.5f}'.format(
lr_multiplier, float(lr), kl, loss))
Log.log(loss, end=', ', print_to_stdout=False)
if kl > kl_targ * 4:
break
Log.log('lr={:.7f}'.format(float(lr)), print_to_stdout=False)
# adjust learning rate
if kl > kl_targ * 2 and lr_multiplier > 0.1:
lr_multiplier /= 1.5
elif kl < kl_targ / 2 and lr_multiplier < 10:
lr_multiplier *= 1.5
Log.log('')
if global_step % evaluate_interval == 0:
win_ratio = evaluate(max_simulate_count=max_sim)
print('best win_ratio: {}'.format(best_win_ratio))
if win_ratio > best_win_ratio:
best_win_ratio = win_ratio
Log.log('new network')
player.mcts.net.save_network(global_step)
if win_ratio >= 0.9 and max_sim < 4000:
max_sim += 100
best_win_ratio = 0.0
Log.flush()
else:
Log.log(len(self_play_data))
# -*- coding: utf-8 -*-
import json
import requests
from flask import Flask, request
from util import Util, Constants, Log, CodeReturn
from controller import Controller
log = Log('AuthRoute')
util = Util()
constants = Constants()
controller = Controller()
codeReturn = CodeReturn()
class AuthRoute:
def login(self, request):
if (request.is_json):
try:
content = request.get_json()
#Get datas from JSON
user = str(content['user'])
password = str(content['pass'])
except:
return util.make_json(codeReturn.BAD_REQUEST_CODE, codeReturn.BAD_REQUEST_MSG, [])
headers = {
'Content-Type': 'application/json',
}
# -*- coding: utf-8 -*-
import os
import json
import asyncio
import traceback
import inspect
import requests
from flask import Flask, request
from flask import send_file, send_from_directory
from werkzeug.utils import secure_filename
from util import Util, Constants, Log, CodeReturn
from controller import Controller
log = Log('UploadRoute')
util = Util()
constants = Constants()
controller = Controller()
codeReturn = CodeReturn()
class UploadRoute:
def upload_launchs(self, request):
try:
content = request.get_json()
header = request.headers
#Get Token from Header
token = str(header['token'])
data = request.form.to_dict()
# -*- coding: utf-8 -*-
import json
import traceback
import inspect
import requests
from flask import Flask, request
from util import Util, Constants, Log, CodeReturn
from controller import Controller
log = Log('UserRoute')
util = Util()
constants = Constants()
controller = Controller()
codeReturn = CodeReturn()
class UserRoute:
def insert_user(self, request):
if (request.is_json):
content = request.get_json()
try:
#Get datas from JSON
data = json.loads(str(content['data']).replace("'", '"'))
info = data['info']
except:
return util.make_json(codeReturn.BAD_REQUEST_CODE,
codeReturn.BAD_REQUEST_MSG, [])
# -*- coding: utf-8 -*-
import jwt
import datetime
from util import Constants, Log, CodeReturn
from configuration import Configuration
log = Log('Controller')
constants = Constants()
configuration = Configuration()
codeReturn = CodeReturn()
class Controller:
def __init__(self):
pass
'''
HIDDEN CODE
'''
def DoTabSelected(self):
"""Performs updates that need to happen when this tab is selected"""
Log("[ed_editv][info] Tab has file: %s" % self.GetFileName())
self.PostPositionEvent()
def __init__(self, engine):
self.engine = engine
self.logClassInits = Config.get("game", "log_class_inits")
if self.logClassInits == 1:
Log.debug("MainMenu class init (MainMenu.py)...")
self.time = 0.0
self.nextLayer = None
self.visibility = 0.0
self.active = False
self.showStartupMessages = False
self.gfxVersionTag = Config.get("game", "gfx_version_tag")
self.chosenNeck = Config.get("game", "default_neck")
exists = 0
if engine.loadImgDrawing(
self, "ok", os.path.join("necks", self.chosenNeck + ".png")):
exists = 1
elif engine.loadImgDrawing(
self, "ok",
os.path.join("necks", "Neck_" + self.chosenNeck + ".png")):
exists = 1
#MFH - fallback logic now supports a couple valid default neck filenames
#MFH - check for Neck_1
if exists == 0:
if engine.loadImgDrawing(self, "ok",
os.path.join("necks", "Neck_1.png")):
Config.set("game", "default_neck", "1")
Log.warn(
"Default chosen neck not valid; fallback Neck_1.png forced."
)
exists = 1
#MFH - check for defaultneck
if exists == 0:
if engine.loadImgDrawing(self, "ok",
os.path.join("necks", "defaultneck.png")):
Log.warn(
"Default chosen neck not valid; fallback defaultneck.png forced."
)
Config.set("game", "default_neck", "defaultneck")
exists = 1
else:
Log.error(
"Default chosen neck not valid; fallbacks Neck_1.png and defaultneck.png also not valid!"
)
#Get theme
self.theme = self.engine.data.theme
self.themeCoOp = self.engine.data.themeCoOp
self.themename = self.engine.data.themeLabel
self.useSoloMenu = self.engine.theme.use_solo_submenu
allowMic = True
self.menux = self.engine.theme.menuPos[0]
self.menuy = self.engine.theme.menuPos[1]
self.rbmenu = self.engine.theme.menuRB
#MFH
self.main_menu_scale = self.engine.theme.main_menu_scaleVar
self.main_menu_vspacing = self.engine.theme.main_menu_vspacingVar
if not self.engine.loadImgDrawing(
self, "background",
os.path.join("themes", self.themename, "menu", "mainbg.png")):
self.background = None
self.engine.loadImgDrawing(
self, "BGText",
os.path.join("themes", self.themename, "menu", "maintext.png"))
self.engine.loadImgDrawing(
self, "optionsBG",
os.path.join("themes", self.themename, "menu", "optionsbg.png"))
self.engine.loadImgDrawing(
self, "optionsPanel",
os.path.join("themes", self.themename, "menu", "optionspanel.png"))
#racer: added version tag
if self.gfxVersionTag or self.engine.theme.versiontag == True:
if not self.engine.loadImgDrawing(
self, "version",
os.path.join("themes", self.themename, "menu",
"versiontag.png")):
if not self.engine.loadImgDrawing(
self, "version", "versiontag.png"
): #falls back on default versiontag.png in data\ folder
self.version = None
else:
self.version = None
#myfingershurt: random main menu music function, menu.ogg and menuXX.ogg (any filename with "menu" as the first 4 letters)
self.files = None
filepath = self.engine.getPath(
os.path.join("themes", self.themename, "sounds"))
if os.path.isdir(filepath):
self.files = []
allfiles = os.listdir(filepath)
for name in allfiles:
if os.path.splitext(name)[1] == ".ogg":
if string.find(name, "menu") > -1:
self.files.append(name)
if self.files:
i = random.randint(0, len(self.files) - 1)
filename = self.files[i]
sound = os.path.join("themes", self.themename, "sounds", filename)
self.menumusic = True
engine.menuMusic = True
self.song = Audio.Music(self.engine.resource.fileName(sound))
self.song.setVolume(self.engine.config.get("audio", "menu_volume"))
self.song.play(0) #no loop
else:
self.menumusic = False
self.opt_text_color = self.engine.theme.opt_text_colorVar
self.opt_selected_color = self.engine.theme.opt_selected_colorVar
trainingMenu = [
(_("Tutorials"), self.showTutorial),
(_("Practice"), lambda: self.newLocalGame(mode1p=1)),
]
self.opt_bkg_size = [float(i) for i in self.engine.theme.opt_bkg_size]
self.opt_text_color = self.engine.theme.hexToColor(
self.engine.theme.opt_text_colorVar)
self.opt_selected_color = self.engine.theme.hexToColor(
self.engine.theme.opt_selected_colorVar)
if self.BGText:
strCareer = ""
strQuickplay = ""
strSolo = ""
strMultiplayer = ""
strTraining = ""
strSettings = ""
strQuit = ""
else:
strCareer = "Career"
strQuickplay = "Quickplay"
strSolo = "Solo"
strMultiplayer = "Multiplayer"
strTraining = "Training"
strSettings = "Settings"
strQuit = "Quit"
multPlayerMenu = [
(_("Face-Off"),
lambda: self.newLocalGame(players=2, maxplayers=4)),
(_("Pro Face-Off"),
lambda: self.newLocalGame(players=2, mode2p=1, maxplayers=4)),
(_("Party Mode"), lambda: self.newLocalGame(mode2p=2)),
(_("FoFiX Co-Op"), lambda: self.newLocalGame(
players=2, mode2p=3, maxplayers=4, allowMic=allowMic)),
(_("RB Co-Op"), lambda: self.newLocalGame(
players=2, mode2p=4, maxplayers=4, allowMic=allowMic)),
(_("GH Co-Op"),
lambda: self.newLocalGame(players=2, mode2p=5, maxplayers=4)),
(_("GH Battle"),
lambda: self.newLocalGame(players=2, mode2p=6, allowDrum=False)
), #akedrou- so you can block drums
]
if not self.useSoloMenu:
mainMenu = [
(strCareer,
lambda: self.newLocalGame(mode1p=2, allowMic=allowMic)),
(strQuickplay, lambda: self.newLocalGame(allowMic=allowMic)),
((strMultiplayer, "multiplayer"), multPlayerMenu),
((strTraining, "training"), trainingMenu),
((strSettings, "settings"), self.settingsMenu),
(strQuit, self.quit),
]
else:
soloMenu = [
(_("Solo Tour"),
lambda: self.newLocalGame(mode1p=2, allowMic=allowMic)),
(_("Quickplay"), lambda: self.newLocalGame(allowMic=allowMic)),
]
mainMenu = [
((strSolo, "solo"), soloMenu),
((strMultiplayer, "multiplayer"), multPlayerMenu),
((strTraining, "training"), trainingMenu),
((strSettings, "settings"), self.settingsMenu),
(strQuit, self.quit),
]
w, h, = self.engine.view.geometry[2:4]
self.menu = Menu(self.engine,
mainMenu,
onClose=lambda: self.engine.view.popLayer(self),
pos=(self.menux, .75 - (.75 * self.menuy)))
engine.mainMenu = self #Points engine.mainMenu to the one and only MainMenu object instance
## whether the main menu has come into view at least once
self.shownOnce = False
# coding=utf-8
from _switch import Switch
import json
import sys
import util.Log as Log
###
# cronから実行するswitch.pyの入り口
###
Log.initLog()
#コマンドライン引数の取り出し。
# args[1] kadenId
# args[2] manipulateId
args = sys.argv
kadenId = args[1]
manipulateId = args[2]
print("cronToSwitch.py Start kadenId:" + kadenId + " manipulateId:" + manipulateId)
# switch.pyの実行。
swc = Switch()
swc.Switching({"kadenId":kadenId,"manipulateId":manipulateId})
print("cronToSwitch.py end ")
@jit
def outer_step(i, state, task_batch):
params = outer_get_params(state)
g, aux = grad(batch_maml_loss, has_aux=True)(params, task_batch)
return outer_opt_update(i, g, state), aux
# logging, plotting
win_loss_keys = ['loss_train', 'loss_test']
win_loss_eval_keys = ['loss_train_eval', 'loss_test_eval']
all_plot_keys = win_loss_keys + win_loss_eval_keys
if args.dataset == 'omniglot':
win_acc_keys = ['acc_train', 'acc_test']
win_acc_eval_keys = ['acc_train_eval', 'acc_test_eval']
all_plot_keys = all_plot_keys + win_acc_keys + win_acc_eval_keys
log = Log(keys=['update'] + all_plot_keys)
outer_state = outer_opt_init(params)
plotter = VisdomPlotter(viz)
if args.dataset == 'sinusoid':
task_fn = partial(sinusoid_task, n_support=args.n_support, n_query=args.n_query)
elif args.dataset == 'omniglot':
task_fn = partial(omniglot_task, split_dict=omniglot_splits['train'], n_way=args.n_way, n_support=args.n_support, n_query=args.n_query)
else:
raise ValueError
for i, task_batch in tqdm(enumerate(taskbatch(task_fn=task_fn,
batch_size=args.task_batch_size,
n_task=args.n_train_task)),
def Help(bin_name, subcommand=None):
"""
Show help imformation
Args:
bin_name : executable file name
subcommand : executable file subcommand. In broc, it's in [""]
Return:
0 : success
"""
if subcommand is None:
Log.colorprint("DEFAULT",
"Usage: %s <subcommand> [option] [args]" % (bin_name),
False)
Log.colorprint(
"DEFAULT",
"Type %s help <subcommand> for help on a specific subcommand" %
(bin_name), False)
Log.colorprint("DEFAULT", "Available subcommands:", False)
Log.colorprint("DEFAULT",
" build : Build the specified targets", False)
Log.colorprint(
"DEFAULT", " test : Build and runs the specified targets",
False)
Log.colorprint("DEFAULT",
" show-deps : Print the dependency graph", False)
Log.colorprint("DEFAULT", " clean : Remove output files",
False)
Log.colorprint("DEFAULT", " scratch : Create a BROC template",
False)
Log.colorprint("DEFAULT", " version : Display the version",
False)
Log.colorprint("DEFAULT",
" config : Display broc's config items", False)
Log.colorprint("DEFAULT", " help : Print the help commands",
False)
return 0
if subcommand == "build" or subcommand == "test":
if subcommand == "build":
Log.colorprint("DEFAULT", "build: Builds the specified targets",
False)
else:
Log.colorprint("DEFAULT",
"test: Builds and runs the specified targets",
False)
Log.colorprint("DEFAULT",
"Usage: %s %s [option] <path>" % (bin_name, subcommand),
False)
Log.colorprint("DEFAULT", "Valid options:", False)
Log.colorprint(
"DEFAULT",
"\t--mode=[release|debug] : Set build mode, default mode is debug",
False)
Log.colorprint("DEFAULT",
"\t--jobs=num\t\t: Set the number of build threads",
False)
Log.colorprint("DEFAULT",
"\t --all-log\t\t: Show all build log infomation")
return 0
if subcommand == "show-deps":
Log.colorprint("DEFAULT", "show-deps: Print the dependency graph",
False)
Log.colorprint("DEFAULT",
"Usage: %s %s <path>" % (bin_name, subcommand), False)
return 0
if subcommand == "clean":
Log.colorprint("DEFAULT", "clean: Remove output files", False)
Log.colorprint("DEFAULT", "Usage: %s clean" % (bin_name), False)
return 0
if subcommand == "scratch":
Log.colorprint("DEFAULT", "scratch: Create a BROC template", False)
Log.colorprint("DEFAULT", "Usage: %s scratch" % (bin_name), False)
return 0
if subcommand == "version":
Log.colorprint("DEFAULT", "version: Display the version", False)
Log.colorprint("DEFAULT", "Usage: %s version" % (bin_name), False)
return 0
if subcommand == "help":
Log.colorprint("DEFAULT", "help: Print the for commands", False)
Log.colorprint("DEFAULT", "Usage: %s help <subcommand>" % (bin_name),
False)
return 0
Log.colorprint("DEFAULT", "%s unknow commmand" % subcommand, False)
return 0
def log():
from util import Log
Log.info("Hello from test.py")
def renderNotes(self, visibility, song, pos):
if not song:
return
if not song.readyToGo:
return
self.bigMax = 0
# Update dynamic period
self.currentPeriod = self.neckSpeed
self.targetPeriod = self.neckSpeed
self.killPoints = False
w = self.boardWidth / self.strings
self.starNotesInView = False
self.openStarNotesInView = False
renderedNotes = reversed(self.getRequiredNotesForRender(song, pos))
for time, event in renderedNotes:
if isinstance(event, Tempo):
self.tempoBpm = event.bpm
if self.lastBpmChange > 0 and self.disableVBPM == True:
continue
if (pos - time > self.currentPeriod or
self.lastBpmChange < 0) and time > self.lastBpmChange:
self.baseBeat += (time -
self.lastBpmChange) / self.currentPeriod
self.targetBpm = event.bpm
self.lastBpmChange = time
self.neck.lastBpmChange = time
self.neck.baseBeat = self.baseBeat
continue
if not isinstance(event, Note):
continue
if (event.noteBpm == 0.0):
event.noteBpm = self.tempoBpm
if event.number == 0 and self.isDrum: #MFH - skip all open notes
continue
if self.coOpFailed:
if self.coOpRestart:
if time - self.coOpRescueTime < (self.currentPeriod *
self.beatsPerBoard * 2):
continue
elif self.coOpRescueTime + (self.currentPeriod *
self.beatsPerBoard * 2) < pos:
self.coOpFailed = False
self.coOpRestart = False
Log.debug("Turning off coOpFailed. Rescue successful.")
else:
continue #can't break. Tempo.
x = (self.strings / 2 - (event.number)) * w
c = self.fretColors[event.number]
if event.number == 4 and self.isDrum:
c = self.fretColors[
0] #myfingershurt: need to swap note 0 and note 4 colors for drums:
z = ((time - pos) / self.currentPeriod) / self.beatsPerUnit
z2 = ((time + event.length - pos) /
self.currentPeriod) / self.beatsPerUnit
if z > self.boardLength * .8:
f = (self.boardLength - z) / (self.boardLength * .2)
elif z < 0:
f = min(1, max(0, 1 + z2))
else:
f = 1.0
#volshebnyi - hide notes in BRE zone if BRE enabled
if self.freestyleEnabled:
if time > self.freestyleStart - self.freestyleOffset and time < self.freestyleStart + self.freestyleOffset + self.freestyleLength:
z = -2.0
if self.twoDnote == True and not self.useFretColors:
color = (1, 1, 1, 1 * visibility * f)
else:
color = (.1 + .8 * c[0], .1 + .8 * c[1], .1 + .8 * c[2],
1 * visibility * f)
if event.length > 120:
length = (event.length -
50) / self.currentPeriod / self.beatsPerUnit
else:
length = 0
tailOnly = False
spNote = False
#myfingershurt: user setting for starpower refill / replenish notes
if self.starPowerActive:
if self.spRefillMode == 0: #mode 0 = no starpower / overdrive refill notes
self.spEnabled = False
elif self.spRefillMode == 1 and self.theme != 2: #mode 1 = overdrive refill notes in RB themes only
self.spEnabled = False
elif self.spRefillMode == 2 and song.midiStyle != 1: #mode 2 = refill based on MIDI type
self.spEnabled = False
if event.star:
self.starNotesInView = True
if event.finalStar:
self.finalStarSeen = True
self.starNotesInView = True
if event.star and self.spEnabled:
spNote = True
if event.finalStar and self.spEnabled:
spNote = True
if event.played or event.hopod:
if event.flameCount < 1 and not self.starPowerGained:
if self.starPower < 50: #not enough starpower to activate yet, kill existing drumfills
for dfEvent in self.drumFillEvents:
dfEvent.happened = True
Log.debug("star power added")
if self.gameMode2p == 6 and not self.isDrum:
if self.battleSuddenDeath:
self.battleObjects = [
1
] + self.battleObjects[:2]
else:
self.battleObjects = [
self.battleObjectsEnabled[random.randint(
0,
len(self.battleObjectsEnabled) - 1)]
] + self.battleObjects[:2]
self.battleGetTime = pos
self.battleObjectGained = True
Log.debug("Battle Object Gained, Objects %s" %
str(self.battleObjects))
else:
if self.starPower < 100:
self.starPower += 25
if self.starPower > 100:
self.starPower = 100
self.overdriveFlashCount = 0 #MFH - this triggers the oFlash strings & timer
self.starPowerGained = True
if event.tappable < 2:
isTappable = False
else:
isTappable = True
if (event.played or event.hopod): #if the note is hit
continue
elif z < 0: #Notes past frets
#if none of the below they keep on going, it would be self.notedisappear == 1
if self.notedisappear == 0: #Notes disappear
continue
elif self.notedisappear == 2: #Notes turn red
color = (1, 0, 0, 1) #turn note red
if z + length < -1.0:
continue
if event.length <= 120:
length = None
sustain = False
if event.length > (1.4 * (60000.0 / event.noteBpm) / 4):
sustain = True
glPushMatrix()
glTranslatef(x, 0, z)
if shaders.turnon:
shaders.setVar("note_position",
(x, (1.0 - visibility)**(event.number + 1), z),
"notes")
if self.battleStatus[8]:
renderNote = random.randint(0, 2)
else:
renderNote = 0
if renderNote == 0:
self.renderNote(length,
sustain=sustain,
color=color,
tailOnly=tailOnly,
isTappable=isTappable,
string=event.lane,
fret=event.number,
spNote=spNote)
glPopMatrix()
#myfingershurt: end FOR loop / note rendering loop
if (not self.openStarNotesInView) and (
not self.starNotesInView) and self.finalStarSeen:
self.spEnabled = True
self.isStarPhrase = False
self.finalStarSeen = False
# -*- coding: utf-8 -*-
import mysql.connector
import datetime
import logging
import json
import traceback
import inspect
from dateutil.relativedelta import *
from datetime import datetime
from util import Util, Log, Constants, CodeReturn
from db import AccountDAO
from flask import Flask
from flask_mysqldb import MySQL
log = Log('LaunchDAO')
constants = Constants()
codeReturn = CodeReturn()
util = Util()
mySQL = MySQL()
accountDAO = AccountDAO()
app = Flask(__name__)
class LaunchDAO:
def __init__(self):
mySQL.init_app(app)
#Date format received: mm/yyyy
def list_launch(self, date, cod_account, companies_id, user_id):
# get current file dir
sup = os.path.dirname(os.path.realpath(__file__))
sup = os.path.dirname(sup)
if sup not in sys.path:
sys.path.append(sup)
from .error import *
from .param_tool import check_param_key, update_param, check_param_value, CAM_RE
from util import Face, mat_to_base64, base64_to_bytes, GrabJob
from util import Log
from util import Kafka
from util import Mysql
from util import WeedClient
logger = Log('camera', 'logs/')
config = configparser.ConfigParser()
config.read('./app.config')
db = Mysql(host=config.get('db', 'host'),
port=config.getint('db', 'port'),
user=config.get('db', 'user'),
password=config.get('db', 'password'),
db=config.get('db', 'db'),
charset=config.get('db', 'charset'))
db.set_logger(logger)
bp_camera = Blueprint('camera', __name__)
proc_pool = {}
def grab_proc(url, rate, camera_id):
'''
抓图处理进程
:param url:
:param rate:
:param camera_id:
:param logger:
:return:
'''
logger = Log('grab-proc' + str(os.getpid()), 'logs/')
logger.info('初始化seaweedfs')
master = WeedClient(config.get('weed', 'host'),
config.getint('weed', 'port'))
logger.info('初始化Kafka')
kafka = Kafka(bootstrap_servers=config.get('kafka', 'boot_servers'))
topic = config.get('camera', 'topic')
face_tool = Face(config.get('api', 'face_server'))
detect_count = 0 # 用于detect频次计数
frame_internal = track_internal * rate
trackable = False
# 启动抓图线程
q = queue.Queue(maxsize=100)
t = GrabJob(q, camera_id, url, rate,
Log('grab-proc' + str(os.getpid()) + '-thread', 'logs/'),
config)
t.start()
while True:
try:
img = q.get(timeout=20)
if detect_count % frame_internal == 0:
detect_count = 0
b64 = mat_to_base64(img)
t1 = time.time()
detect_result = face_tool.detect(b64)
logger.info('detect cost time: ',
round((time.time() - t1) * 1000), 'ms')
if detect_result['error_message'] != '601':
logger.warning('verifier detector error, error_message:',
detect_result['error_message'])
continue
tracker = cv2.MultiTracker_create()
latest_imgs = []
timestamp = round(time.time())
for face_num in range(detect_result['detect_nums']):
tmp = detect_result['detect'][face_num]
bbox = (tmp['left'], tmp['top'], tmp['width'],
tmp['height'])
tracker.add(cv2.TrackerKCF_create(), img, bbox)
face_b64 = face_tool.crop(bbox[0], bbox[1], bbox[2],
bbox[3], b64, True)
latest_img = {
'image_base64': face_b64,
'bbox': bbox,
'landmark':
detect_result['detect'][face_num]['landmark'],
'time': timestamp
}
# 增加人脸质量过滤
if tmp['sideFace'] == 0 and tmp[
'quality'] == 1 and tmp['score'] > 0.95:
latest_imgs.append(latest_img)
if len(latest_imgs) > 0:
trackable = True
else:
trackable = False
elif trackable:
# 开始追踪
ok, bboxs = tracker.update(img)
if ok and detect_count < frame_internal - 1:
if detect_count % 10 == 0:
logger.info('tracking..., detect_count = %d' %
detect_count)
detect_count += 1
continue
else:
# 取detect到的人脸
logger.info('tracking over! detect_count = %d' %
detect_count)
for latest in latest_imgs:
logger.info([camera_id], 'track person success!')
face_b64 = latest['image_base64']
# save img to seaweed fs
logger.info([camera_id],
'save grabbed detect_result to seaweed fs')
assign = master.assign()
logger.info([camera_id], 'assign result:', assign)
ret = master.upload(assign['url'], assign['fid'],
base64_to_bytes(face_b64),
assign['fid'] + '.jpg')
logger.info([camera_id], 'upload result:', ret)
# send to Kafka
url = 'http' + ':' + '//' + assign[
'url'] + '/' + assign['fid']
logger.info('[', camera_id, ']', 'img url:', url)
msg = json.dumps({
'url': url,
'time': latest['time'],
'camera_id': camera_id,
'landmark': latest['landmark']
})
logger.info([camera_id], 'send to kafka: ', msg)
kafka.send(topic, msg)
# 再次进入detect
detect_count = 0
trackable = False
logger.info('restart detection')
else:
if detect_count % 10 == 0:
logger.info('detect 0 detect_result, do not track',
'detect count= ', detect_count)
detect_count += 1
continue
except queue.Empty:
logger.error('grab queue empty error, exit')
break
detect_count += 1
logger.info('抓图进程终止')
def work_png(filename):
filepath = FileHelper.realpath(filename)
filedir = FileHelper.dirname(filepath)
os.chdir(tpDir)
isSaveTransFile = False
isPng = True
useGZIP = False
if isPng:
jpgCMD = """%s %s -background black %s """ % (convertBin, filepath, filepath.replace(".png", ".rgb.jpg"))
alphaCMD = """%s %s -alpha extract %s """ % (convertBin, filepath, filepath.replace(".png", ".alpha.jpg"))
try:
os.system(jpgCMD)
os.system(alphaCMD)
except Exception:
Log.printDetailln ("error33 !!!", filename, "cannot convert.")
return 2
finally:
pass
tmpfilename = filepath.replace(".png", ".tmp")
FileHelper.remove(tmpfilename)
isSuccess = True
with open(tmpfilename, 'wb+') as tmpFile:
try:
tmpFile.write(b'MNG')
rgbname = filepath.replace(".png", ".rgb.jpg")
FileHelper.writeWithSize(tmpFile, rgbname)
alphaname = filepath.replace(".png", ".alpha.jpg")
FileHelper.writeWithSize(tmpFile, alphaname)
if not isSaveTransFile:
FileHelper.remove(rgbname)
FileHelper.remove(alphaname)
except Exception:
Log.printError()
isSuccess = False
pass
finally:
pass
if isSuccess:
if useGZIP:
gzip_cmd = gzipBin + tmpfilename + " -n -f -9"
os.system(gzip_cmd)
FileHelper.remove(tmpfilename.replace(".tmp", ".png"))
FileHelper.rename(tmpfilename + ".gz", tmpfilename.replace(".tmp", ".png"))
return 3
else:
FileHelper.remove(tmpfilename.replace(".tmp", ".png"))
FileHelper.rename(tmpfilename, tmpfilename.replace(".tmp", ".png"))
return 5
else:
return 2
def main():
Log("Main program started.")
kb = threading.Thread(target=ThreadKB)
kb.start()
# constellation_bpsk = [
# np.array([1.0,0.0]), # 0
# np.array([-1.0,0.0]) # 1
# ]
# constellation_qpsk = [
# np.array([ 1.0, 1.0]), # 0
# np.array([-1.0, 1.0]), # 1
# np.array([-1.0, -1.0]), # 2
# np.array([ 1.0, -1.0]), # 3
# ]
# M = RandomM()
# nc = RandomConstellation(M)
# Log("randM=" + str(M))
# x,y = ConstellationToXY(nc)
# plt.plot(x,y,"*")
# plt.show()
nPop = 9
population = []
for i in range(0, nPop):
M = RandomM()
population.append(Individual(M, RandomSpSym(), RandomConstellation(M)))
nGen = 0
tReport = 15
tTurnOnTx = 5 # this depends on nSNR_averages
TxEnabled = False
global KILL
global kbQ
# Prepare the matplotlib figre window to be always up.
plt.figure()
plt.ion()
reports = []
fname = "reports_" + datetime.now().strftime("%b-%d-%H-%M-%S") + ".txt"
f = open(fname, "w")
fwrobj = {"Reports": []}
t = 0
while not KILL:
if not kbQ.empty():
act = kbQ.get()
for k in act.split(" "):
if k == "t":
for ind in population:
ind.TxToggle()
elif k == "r":
for ind in population:
ind.RxToggle()
for ind in population:
ind.Tick(t)
dT = 1 / Fs
t += dT
# DEBUG: is this needed? avoid overflow for various calculations
if t > 100 * Fs:
t = 0
# Here, a generation reproduces
if t > tReport:
# clear figure before overwriting
plt.clf()
Log("---- Generation " + str(nGen) + " ----")
reports = []
for i, ind in enumerate(population):
rep = ind.Report()
Log(str(i) + ": " + str(rep))
Log("Score=" + str(ScoreReport(rep)))
reports.append(rep)
x, y = ConstellationToXY(rep["constellation"])
plt.subplot(331 + i)
plt.plot(x, y, "*")
plt.title("Individual " + str(i))
plt.pause(0.01)
# for i,rep in enumerate(reports):
# f.write(json.dumps(rep))
# f.write("\n")
thisone = {
"Weights": SCORE_WEIGHTS,
"M Mutation": M_MUTATION_CHANCE,
"spSym Stddev": SPSYM_STDDEV,
"cpt Stddev": CPT_STDDEV,
"Generation": nGen,
"Reports": reports,
}
fwrobj["Reports"].append(thisone)
# Reset the simulation for the next generation
t = 0
TxEnabled = False
population = NewGeneration(reports)
nGen += 1
# Handle automation actions that run the GA
if TxEnabled == False:
if t >= tTurnOnTx:
Log("---- Enabled Transmitter ----")
TxEnabled = True
for ind in population:
ind.TxToggle()
f.write(json.dumps(fwrobj))
f.close()
kb.join()
Log("Main program ended.")
def train(train_loader, train_loader1, train_loader2, train_loader3, args, model, criterion, center_loss, optimizer, epoch, num_epochs):
since = time.time()
running_loss0 = AverageMeter()
running_loss1 = AverageMeter()
running_loss2 = AverageMeter()
running_loss3 = AverageMeter()
running_loss4 = AverageMeter()
running_loss5 = AverageMeter()
running_loss6 = AverageMeter()
running_loss = AverageMeter()
log = Log()
model.train()
for (i, (input, target)),(j, (input1, target1)),(k, (input2, target2)),(p, (input3, target3)) in zip(enumerate(train_loader),enumerate(train_loader1),enumerate(train_loader2),enumerate(train_loader3)):
input_var = Variable(input.cuda())
input_var1 = Variable(input1.cuda())
input_var2 = Variable(input2.cuda())
input_var3 = Variable(input3.cuda())
targets = torch.cat((target,target1,target2,target3),0)
targets = Variable(targets.cuda())
target_var = Variable(target.cuda())
target_var1 = Variable(target1.cuda())
target_var2 = Variable(target2.cuda())
target_var3 = Variable(target3.cuda())
outputs = model(input_var,input_var1,input_var2,input_var3)
size = int(outputs.size(0)/4)
img = outputs.narrow(0, 0, size)
vid = outputs.narrow(0, size, size)
aud = outputs.narrow(0, 2*size, size)
txt = outputs.narrow(0, 3*size, size)
loss0 = criterion(img, target_var)
loss1 = criterion(vid, target_var1)
loss2 = criterion(aud, target_var2)
loss3 = criterion(txt, target_var3)
loss4 = loss0 + loss1 + loss2 + loss3
loss5 = center_loss(outputs,targets)*0.001
if(args.loss_choose == 'r'):
loss6, _ = ranking_loss(targets, outputs, margin=1, margin2=0.5, squared=False)
loss6 = loss6 * 0.1
else:
loss6 = 0.0
loss = loss4 + loss5 + loss6
batchsize = input_var.size(0)
running_loss0.update(loss0.item(), batchsize)
running_loss1.update(loss1.item(), batchsize)
running_loss2.update(loss2.item(), batchsize)
running_loss3.update(loss3.item(), batchsize)
running_loss4.update(loss4.item(), batchsize)
running_loss5.update(loss5.item(), batchsize)
if(args.loss_choose == 'r'):
running_loss6.update(loss6.item(), batchsize)
running_loss.update(loss.item(), batchsize)
optimizer.zero_grad()
loss.backward()
for param in center_loss.parameters():
param.grad.data *= (1./0.001)
optimizer.step()
if (i % args.print_freq == 0):
print('-' * 20)
print('Epoch [{0}/{1}][{2}/{3}]'.format(epoch, num_epochs, i, len(train_loader)))
print('Image Loss: {loss.avg:.5f}'.format(loss=running_loss0))
print('Video Loss: {loss.avg:.5f}'.format(loss=running_loss1))
print('Audio Loss: {loss.avg:.5f}'.format(loss=running_loss2))
print('Text Loss: {loss.avg:.5f}'.format(loss=running_loss3))
print('AllMedia Loss: {loss.avg:.5f}'.format(loss=running_loss4))
print('Center Loss: {loss.avg:.5f}'.format(loss=running_loss5))
if(args.loss_choose == 'r'):
print('Ranking Loss: {loss.avg:.5f}'.format(loss=running_loss6))
print('All Loss: {loss.avg:.5f}'.format(loss=running_loss))
log.save_train_info(epoch, i, len(train_loader), running_loss)
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
class Spider(object):
''' Spider '''
def __init__(self,
name='Spider',
url_list_file='./urls',
output_path='./output',
interval=1,
timeout=1,
silent=False):
'''
@name: string, 定向爬虫的名字
@url_list_file: string, 种子文件
@output_path: string, 输出文件路径
@interval: int, 爬取间隔
@timeout: int, 请求超时
@silent: bool, 是否为静默打印
'''
# 设置保存爬取页面的coolie值（非必要）
cj = cookielib.LWPCookieJar()
cookie_support = urllib2.HTTPCookieProcessor(cj)
self.opener = urllib2.build_opener(cookie_support, urllib2.HTTPHandler)
urllib2.install_opener(self.opener)
# 设置请求头部（非必要）
self.headers = {
'Content-Type':
'application/x-www-form-urlencoded',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.89 Safari/537.36'
}
self.url_list_file = url_list_file
self.output_path = output_path
self.interval = interval
self.timeout = timeout
level = 'INFO' if silent else 'DEBUG'
self.log = Log(name, level, 'spider.log', 'a')
# 定义一个多线程共享队列，存放需要加载的url
self.queue = Queue()
# 存放非重复url集合
self.url_set = set()
# 定义多线程访问锁
self.lock = Lock()
# 匹配根域名
self.base_pattern = r'^(https?:\/\/[a-z0-9\-\.]+)[\/\?]?'
if not os.path.exists(self.url_list_file):
# 种子文件不存在
self.log.error('%s dose not exist. Exit program !' %
(self.url_list_file))
os._exit(0)
if not os.path.exists(self.output_path):
# 输出文件目录不存在
self.log.info('Create new directory %s' % (self.output_path))
os.makedirs(self.output_path)
'''
def __decodeHtml__(self, html):
"""
Decode Html
@html: string, 原生html内容
return: 返回解码后的html内容
"""
try:
encoding = chardet.detect(html)['encoding']
if encoding == 'GB2312':
encoding = 'GBK'
else:
encoding = 'utf-8'
return html.decode(encoding, 'ignore')
except Exception, e:
self.log.error("Decode error: %s.", e.message)
return None
'''
def __request__(self, url, threadID=-1, data=None):
'''
Request URL
@url: string, 指定抓取的url
@data: object, POST方法发送的数据
'''
try:
req = urllib2.Request(url, data, self.headers)
res = urllib2.urlopen(req, timeout=self.timeout).read()
self.log.debug('Thread-%d Get %s' % (threadID, url))
return res
except Exception, e:
self.log.error('Thread-%d in __requests__: %s %s' %
(threadID, e.message, url))
return None
# -*- coding: utf-8 -*-
import json
import traceback
import inspect
import requests
from flask import Flask, request
from util import Util, Constants, Log, CodeReturn
from controller import Controller
log = Log('DFCRoute')
util = Util()
constants = Constants()
controller = Controller()
codeReturn = CodeReturn()
class DFCRoute:
def get_dfc(self, request):
try:
header = request.headers
#Get Token from Header
token = str(header['token'])
#Get datas from PARAMS
data = json.loads(str(request.args.get('data')).replace("'", '"'))
companies_token = data['companies_token']
date = data['date']
except:
log.error(
# -*-coding:utf8;-*-
from loadAndroid import Load
from chatbot import Chatbot
from util import Voz
from util import Log
voz = Voz()
log = Log()
load = Load()
chatbor = Chatbot()
infos = load.carregarInformacoesIniciais()
modo_texto_ativo = False
aprender = True
def iniciar():
if not modo_texto_ativo:
voz.fale(load.iniciar(infos)) # Verificando nome e primeiro acesso
else:
load.iniciar(infos)
def receber_entrada():
if not modo_texto_ativo:
entrada_recebida = (voz.escute())
else:
entrada_recebida = input(str("%s: " % load.nomeUsu))
language = Config.load(Version.PROGRAM_UNIXSTYLE_NAME + ".ini").get(
"game", "language")
_ = dummyTranslator
if language:
try:
trFile = os.path.join(Version.dataPath(), "translations",
"%s.mo" % language.lower().replace(" ", "_"))
catalog = gettext.GNUTranslations(open(trFile, "rb"))
def translate(m):
return catalog.ugettext(m)
_ = translate
except Exception, x:
Log.warn("Unable to select language '%s': %s" % (language, x))
language = None
Config.set("game", "language", "")
# Define the config key again now that we have some options for it
langOptions = {"": "English"}
for lang in getAvailableLanguages():
langOptions[lang] = _(lang)
Config.define("game",
"language",
str,
"",
_("Language"),
langOptions,
tipText=_("Change the game language!"))
def fit(self, cnx, clients_count, products_count, from_ts, to_ts):
# type: (sqlite3.connect, int, int, long, long) -> None
"""
Inizializza il predittore recuperando i dati dal database, considerando solamente i dati nell'intervallo
temporale specificato (estremi inclusi).
:param cnx: connessione al database
:param clients_count: numero di clienti
:param products_count: numero di prodotti
:param from_ts: timestamp del giorno d'inizio dei dati di train
:param to_ts: timestamp del giorno finale dei dati di train (incluso)
:return:
"""
self.regressor_matrix = np.ndarray(shape=(clients_count,
products_count),
dtype=object)
days_count = (to_ts - from_ts + SECS_IN_DAY) / SECS_IN_DAY
# Calcola il numero medio di 1 per ogni cella della matrice
query = "SELECT count(*) " \
"FROM orders " \
"WHERE datetime >= %d AND datetime <= %d " % (from_ts, to_ts)
Log.d(TAG, query)
row = cnx.execute(query).fetchone()
total_cnt = row[0]
avg = float(total_cnt) / float(days_count)
avg = int(math.ceil(avg))
self.avg_ones = 1 if avg == 0 else avg
# df = self.__extract_day_group(dataset)
Log.d(TAG, "Fit dei regressori...")
for c in range(0, clients_count):
for p in range(0, products_count):
# Crea il DataFrame con anche le righe per i prodotti non ordinati
query = "SELECT datetime " \
"FROM orders " \
"WHERE datetime >= %d " \
"AND datetime <= %d " \
"AND client_id = %d " \
"AND product_id = %d " \
"ORDER BY datetime" % (from_ts, to_ts, c, p)
# ^ ORDER BY è fondamentale per effettuare la creazione in modo efficiente
Log.d(TAG, query)
cursor = cnx.execute(query)
next_row = cursor.fetchone()
df_rows = []
for ts in range(from_ts, to_ts + SECS_IN_DAY, SECS_IN_DAY):
ordered = 0
if next_row is not None and next_row[0] == ts:
ordered = 1
next_row = cursor.fetchone()
order_date = datetime.fromtimestamp(ts)
day_of_year = order_date.timetuple().tm_yday
year = order_date.timetuple().tm_year
df_rows.append({
'datetime': ts, # timestamp della data dell'ordine
'day_of_year': day_of_year,
'year': year,
SingleRegressorPredictor.TARGET_COL: ordered
})
df = pd.DataFrame(df_rows,
columns=SingleRegressorPredictor.TRAIN_COLS +
[SingleRegressorPredictor.TARGET_COL])
y = df[SingleRegressorPredictor.TARGET_COL].as_matrix()
X = df.drop([SingleRegressorPredictor.TARGET_COL],
axis=1).as_matrix()
clf = None
if self.regressor_name == 'SGD':
clf = linear_model.SGDRegressor()
elif self.regressor_name == 'SVR':
clf = SVR()
elif self.regressor_name == 'PAR':
clf = PassiveAggressiveRegressor()
self.regressor_matrix[c, p] = clf
self.regressor_matrix[c, p].fit(X, y)
return
def test_svn_FilterDepNodes(self):
"""
test Plansh._filter_dep_nodes
"""
#init
logger = Log.Log()
root_node = BrocModule_pb2.Module()
repo_domain = 'https://github.com'
postfix = ["trunk", "_BRANCH", "_PD_BL"]
planish = Planish.Planish(root_node, repo_domain, logger, postfix)
reserved = BrocModule_pb2.Module()
coming = BrocModule_pb2.Module()
first_node = BrocTree.BrocNode(reserved, root_node, False)
second_node = BrocTree.BrocNode(coming, root_node, False)
#dep_Level is 1 VS dep_level is 1
first_node.module.dep_level = 1
first_node.module.dep_level = 1
#tags VS BRANCH
first_node.module.br_kind = BrocModule_pb2.Module.TAG
second_node.module.br_kind = BrocModule_pb2.Module.BRANCH
first_node.module.tag_name = 'ub_1-1-1-1_PD_BL'
second_node.module.br_name = 'ub_1-0-0_BRANCH'
self.assertEqual(-1, planish._filter_dep_nodes(first_node, second_node))
#dep_level is 1 VS dep_level is 2 or more
first_node.module.dep_level = 1
second_node.module.dep_level = 2
#tags VS tags
first_node.module.br_kind = BrocModule_pb2.Module.TAG
second_node.module.br_kind = BrocModule_pb2.Module.TAG
#1-1-1-1 VS 2-2-2-2
first_node.module.tag_name = 'ub_1-1-1-1_PD_BL'
second_node.module.tag_name = 'ub_2-2-2-2_PD_BL'
self.assertEqual(0, planish._filter_dep_nodes(first_node, second_node))
#1-10-0-0 VS 1-9-0-0
first_node.module.tag_name = 'ub_1-10-0-0_PD_BL'
second_node.module.tag_name = 'ub_1-9-0-0_PD_BL'
self.assertEqual(0, planish._filter_dep_nodes(first_node, second_node))
#BRANCH VS BRANCH
first_node.module.br_kind = BrocModule_pb2.Module.BRANCH
second_node.module.br_kind = BrocModule_pb2.Module.BRANCH
#1-0-0_BRANCH VS 1-0-1_BRANCH
first_node.module.br_name = 'ub_1-0-0_BRANCH'
second_node.module.br_name = 'ub_1-0-1_BRANCH'
self.assertEqual(0, planish._filter_dep_nodes(first_node, second_node))
#1-0-0_BRANCH@12345 VS 1-0-0_BRANCH@12346
first_node.module.br_name = 'ub_1-0-0_BRANCH'
second_node.module.br_name = 'ub_1-0-0_BRANCH'
first_node.module.revision = '12345'
second_node.module.revision = '12346'
self.assertEqual(0, planish._filter_dep_nodes(first_node, second_node))
#1-0-0_BRANCH@12345 VS 1-0-0_BRANCH@234
second_node.module.revision = '234'
self.assertEqual(0, planish._filter_dep_nodes(first_node, second_node))
#tags VS BRANCH
first_node.module.br_kind = BrocModule_pb2.Module.TAG
second_node.module.br_kind = BrocModule_pb2.Module.BRANCH
first_node.module.tag_name = 'ub_1-1-1-1_PD_BL'
second_node.module.br_name = 'ub_1-0-0_BRANCH'
self.assertEqual(0, planish._filter_dep_nodes(first_node, second_node))
#dep_level is 2 or more VS dep_level is 2 or more
first_node.module.dep_level = 2
second_node.module.dep_level = 2
#tags VS tags
first_node.module.br_kind = BrocModule_pb2.Module.TAG
second_node.module.br_kind = BrocModule_pb2.Module.TAG
#1-1-1-1 VS 2-2-2-2
first_node.module.tag_name = 'ub_1-1-1-1_PD_BL'
second_node.module.tag_name = 'ub_2-2-2-2_PD_BL'
self.assertEqual(1, planish._filter_dep_nodes(first_node, second_node))
#1-10-0-0 VS 1-9-0-0
first_node.module.tag_name = 'ub_1-10-0-0_PD_BL'
second_node.module.tag_name = 'ub_1-9-0-0_PD_BL'
self.assertEqual(1, planish._filter_dep_nodes(first_node, second_node))
#BRANCH VS BRANCH
first_node.module.br_kind = BrocModule_pb2.Module.BRANCH
second_node.module.br_kind = BrocModule_pb2.Module.BRANCH
#1-0-0_BRANCH VS 1-0-1_BRANCH
first_node.module.br_name = 'ub_1-0-0_BRANCH'
second_node.module.br_name = 'ub_1-0-1_BRANCH'
self.assertEqual(-1, planish._filter_dep_nodes(first_node, second_node))
#1-0-0_BRANCH@12345 VS 1-0-0_BRANCH@12346
first_node.module.br_name = 'ub_1-0-0_BRANCH'
second_node.module.br_name = 'ub_1-0-0_BRANCH'
first_node.module.revision = '12345'
second_node.module.revision = '12346'
self.assertEqual(1, planish._filter_dep_nodes(first_node, second_node))
#1-0-0_BRANCH@12345 VS 1-0-0_BRANCH@234
second_node.module.revision = '234'
self.assertEqual(0, planish._filter_dep_nodes(first_node, second_node))
#tags VS BRANCH
first_node.module.br_kind = BrocModule_pb2.Module.TAG
second_node.module.br_kind = BrocModule_pb2.Module.BRANCH
first_node.module.tag_name = 'ub_1-1-1-1_PD_BL'
second_node.module.br_name = 'ub_1-0-0_BRANCH'
self.assertEqual(-1, planish._filter_dep_nodes(first_node, second_node))
