def add(request):
if request.method == 'GET':
return render_to_response('add.html')
else:
m=createMonitor(
log=getLogger(),
name=request.POST.get('name'),
interval=request.POST.get('interval'),
mailInterval=request.POST.get('mailInterval'),
smsInterval=request.POST.get('smsInterval'),
smsBegin=request.POST.get('smsBegin'),
smsEnd=request.POST.get('smsEnd'),
retryTimes=request.POST.get('retryTimes'),
retryInterval=request.POST.get('retryInterval'),
describe=request.POST.get('describe'),
url=request.POST.get('url'),
timeout=request.POST.get('timeout'),
contacts=request.POST.get('contacts'),
statusCode=request.POST.get('statusCode'),
status=False,
)
if m:
return HttpResponseRedirect('/index/')
else:
return HttpResponse('create failed')
def __init__(self, circle, fcp, totalsize=0):
BaseTask.__init__(self, circle)
self.circle = circle
self.fcp = fcp
self.totalsize = totalsize
# cache
self.fd_cache = LRU(512)
# failed
self.failed = {}
self.failcnt = 0
# debug
self.d = {"rank": "rank %s" % circle.rank}
self.logger = utils.getLogger(__name__)
# reduce
self.vsize = 0
assert len(circle.workq) == 0
if self.circle.rank == 0:
print("\nChecksum verification ...")
def initAPIExport (apiName, psName, apiType, newAPIFlag):
"""
Purpose: Method fo initializing the API export from Akana CM and PS (Production & Sandbox) export from Akana PM
Input: Name of the API and Physical Service, the type of API (INTERNAL/EXTERNAL) and whether this is the first time API is being exported (Y/N)
Returns: Dictionary object comprising of the relative path of the TAR file for API and PS
"""
global mylogger
mylogger = utils.getLogger ("APA-API-EXPORT")
api_tar_path = ""
if (apiName != ""):
api_tar_path = processCMExport (apiName, apiType, newAPIFlag)
ps_tar_path = ""
pssb_tar_path = ""
if (psName != ""):
# Export Prod PS
ps_tar_path = processPMExport (psName, apiType, newAPIFlag)
# Export Sandbox PS
pssb_tar_path = processPMExport (psName+"sb", apiType, newAPIFlag)
tarFileDict = { "api_tar_path" : api_tar_path, "ps_tar_path" : ps_tar_path , "pssb_tar_path" : pssb_tar_path }
return tarFileDict
def __init__(self, circle, fcp, total_chunks, totalsize=0,signature=False):
BaseTask.__init__(self, circle)
self.circle = circle
self.fcp = fcp
self.totalsize = totalsize
self.signature = signature
if self.signature:
self.bfsign = BFsignature(total_chunks)
# failed
self.failed = {}
self.failcnt = 0
# debug
self.d = {"rank": "rank %s" % circle.rank}
self.logger = utils.getLogger(__name__)
# reduce
self.vsize = 0
assert len(circle.workq) == 0
if self.circle.rank == 0:
print("\nChecksum verification ...")
def initSwaggerCreation(tarURL):
"""
Purpose: Initiate the process of the TAR file containing the Swagger specifications for an API. The TAR file is downloaded from a Nexus URL and stored in 'source' directory.
Since the Swagger files are embedded in a TAR-EAR-WAR structure the contents need to be extracted first and then modified
The target directory is the one in which the modified swagger files will be created
Input: URL of the TAR file containing the Swagger files
Returns: List containing relative path of the modified Swagger zip files
"""
global mylogger
mylogger = utils.getLogger("APA-SWG-PROC")
try:
mylogger.info("************************************")
mylogger.info("BEGIN -- initSwaggerCreation()")
if tarURL == "" or tarURL == None:
raise Exception("Swagger TAR file name not provided")
swaggerDir = swgConfig.get("SWG", "swaggerDir")
tmpSwaggerLocation = os.path.join(swaggerDir, swgConfig.get("SWG", "tmpSwaggerLocation"))
swaggerSourceLocation = os.path.join(swaggerDir, swgConfig.get("SWG", "swaggerSourceLocation"))
urllib.request.urlretrieve(tarURL, os.path.join(swaggerSourceLocation, "tarname.tar"))
sourcePathDict = extractSwaggerFromSource(swaggerSourceLocation, "tarname.tar")
swaggerZipFileList = processSwagger(tmpSwaggerLocation, sourcePathDict)
mylogger.info("END -- initSwaggerCreation()")
mylogger.info("************************************")
return swaggerZipFileList
except:
mylogger.error(traceback.format_exc())
sys.exit()
def treeherder_job_completed(self, th_id):
logger = utils.getLogger()
logger.debug('jobs.treeherder_job_completed: %s', th_id)
conn = self._conn()
self._execute_sql(conn, 'delete from treeherder where id=?', values=(th_id,))
self._commit_connection(conn)
self._close_connection(conn)
def stop(self):
"""Stops the pulse monitor listen thread."""
logger = utils.getLogger()
logger.debug('AutophonePulseMonitor stopping')
self._stopping.set()
self.listen_thread.join()
logger.debug('AutophonePulseMonitor stopped')
def hook_tk_errors():
"""TKinter catches and swallows callback errors.
we need to hook into that to log those seperately.
"""
import traceback
main_logger = utils.getLogger()
def tk_error(exc_type, exc_value, exc_tb):
"""Log TK errors."""
# The exception is caught inside the TK code.
# We don't care about that, so try and move the traceback up
# one level.
if exc_tb.tb_next:
exc_tb = exc_tb.tb_next
main_logger.error(
'TKinter callback exception occurred:\n{}',
''.join(
traceback.format_exception(
exc_type,
exc_value,
exc_tb,
)
),
)
# Don't shutdown - most of the time, the TK errors won't
# shutdown the application.
TK_ROOT.report_callback_exception = tk_error
def get_treeherder_privatebuild_info(self, project, job):
logger = utils.getLogger()
url = '%s/api/jobdetail/?repository=%s&job_id=%s' % (
self.treeherder_url, project, job['id'])
data = utils.get_remote_json(url)
logger.debug("get_treeherder_privatebuild_info: data: %s", data)
privatebuild_keys = set(['build_url', 'config_file', 'chunk',
'builder_type'])
info = {}
for detail in data['results']:
if detail['title'] in privatebuild_keys:
# the build_url property is too long to fit into "value"
if detail.get('url'):
value = detail['url']
else:
value = detail['value']
info[detail['title']] = value
# If we're missing a privatebuild key for some reason
# return None to avoid errors.
missing_keys = privatebuild_keys - set(info.keys())
if missing_keys:
logger.debug("get_treeherder_privatebuild_info: %s "
"missing keys: %s "
"job: %s", url, missing_keys, job)
return None
return info
def get_next_treeherder_job(self):
logger = utils.getLogger()
conn = self._conn()
job_cursor = self._execute_sql(
conn,
'select id,attempts,last_attempt,machine,project,job_collection from treeherder')
job_row = job_cursor.fetchone()
job_cursor.close()
if not job_row:
self._close_connection(conn)
return None
job = {'id': job_row[0],
'attempts': job_row[1],
'last_attempt': job_row[2],
'machine': job_row[3],
'project': job_row[4],
'job_collection': json.loads(job_row[5])}
job['attempts'] += 1
job['last_attempt'] = datetime.datetime.utcnow().isoformat()
self._execute_sql(
conn,
'update treeherder set attempts=?, last_attempt=? where id=?',
values=(job['attempts'], job['last_attempt'],
job['id']))
logger.debug('jobs.get_next_treeherder_job: %s', job)
self._commit_connection(conn)
self._close_connection(conn)
return job
def upload(self, path, destination):
try:
logger = utils.getLogger()
key = self.bucket.get_key(destination)
if not key:
logger.debug('Creating key: %s', destination)
key = self.bucket.new_key(destination)
ext = os.path.splitext(path)[-1]
if ext == '.log' or ext == '.txt':
key.set_metadata('Content-Type', 'text/plain')
with tempfile.NamedTemporaryFile('w+b', suffix=ext) as tf:
logger.debug('Compressing: %s', path)
with gzip.GzipFile(path, 'wb', fileobj=tf) as gz:
with open(path, 'rb') as f:
gz.writelines(f)
tf.flush()
tf.seek(0)
key.set_metadata('Content-Encoding', 'gzip')
logger.debug('Setting key contents from: %s', tf.name)
key.set_contents_from_file(tf)
url = key.generate_url(expires_in=0,
query_auth=False)
except boto.exception.S3ResponseError, e:
logger.exception(str(e))
raise S3Error('%s' % e)
def serve_forever(self):
logger = utils.getLogger()
while not self.shutdown_requested:
wait_seconds = 1 # avoid busy loop
job = self.jobs.get_next_treeherder_job()
if job:
tjc = TreeherderJobCollection()
for data in job['job_collection']:
tj = TreeherderJob(data)
tjc.add(tj)
if self.post_request(job['machine'], job['project'], tjc,
job['attempts'], job['last_attempt']):
self.jobs.treeherder_job_completed(job['id'])
wait_seconds = 0
else:
attempts = int(job['attempts'])
wait_seconds = min(self.retry_wait * attempts, 3600)
logger.debug('AutophoneTreeherder waiting for %d seconds after '
'failed attempt %d',
wait_seconds, attempts)
if wait_seconds > 0:
for i in range(wait_seconds):
if self.shutdown_requested:
break
time.sleep(1)
def __init__(self, worker_subprocess, options, jobs, s3_bucket=None,
mailer=None):
assert options, "options is required."
logger = utils.getLogger()
self.options = options
self.jobs = jobs
self.s3_bucket = s3_bucket
self.mailer = mailer
self.worker = worker_subprocess
self.shutdown_requested = False
logger.debug('AutophoneTreeherder')
self.url = self.options.treeherder_url
if not self.url:
logger.debug('AutophoneTreeherder: no treeherder url')
return
self.client_id = self.options.treeherder_client_id
self.secret = self.options.treeherder_secret
self.retry_wait = self.options.treeherder_retry_wait
self.client = TreeherderClient(server_url=self.url,
client_id=self.client_id,
secret=self.secret)
logger.debug('AutophoneTreeherder: %s', self)
def post_request(self, machine, project, job_collection, attempts, last_attempt):
logger = utils.getLogger()
logger.debug('AutophoneTreeherder.post_request: %s, attempt=%d, last=%s',
job_collection.__dict__, attempts, last_attempt)
try:
self.client.post_collection(project, job_collection)
return True
except Exception, e:
logger.exception('Error submitting request to Treeherder, attempt=%d, last=%s',
attempts, last_attempt)
if attempts > 1 and self.mailer:
if hasattr(e, 'response') and e.response:
response_json = json.dumps(e.response.json(),
indent=2, sort_keys=True)
else:
response_json = None
request_len = len(job_collection.to_json())
self.mailer.send(
'%s attempt %d Error submitting request to Treeherder' %
(utils.host(), attempts),
'Phone: %s\n'
'Exception: %s\n'
'Last attempt: %s\n'
'Request length: %d\n'
'Response: %s\n' % (
machine,
e,
last_attempt,
request_len,
response_json))
def test_completed(self, test_guid):
logger = utils.getLogger()
logger.debug('jobs.test_completed: %s', test_guid)
conn = self._conn()
self._execute_sql(conn, 'delete from tests where guid=?', values=(test_guid,))
self._commit_connection(conn)
self._close_connection(conn)
def modify(request):
if request.method == 'GET':
id=request.GET.get('id')
m=getMonitorById(id)
if m:
#sb=str(m.smsBegin)
#se=str(m.smsEnd)
return render_to_response('modify.html',{'monitor':m})
else:
return HttpResponse('id is not exisit')
else:
m=modifyMonitor(
oldName=request.POST.get('oldName'),
log=getLogger(),
name=request.POST.get('name'),
interval=request.POST.get('interval'),
mailInterval=request.POST.get('mailInterval'),
smsInterval=request.POST.get('smsInterval'),
smsBegin=request.POST.get('smsBegin'),
smsEnd=request.POST.get('smsEnd'),
retryTimes=request.POST.get('retryTimes'),
retryInterval=request.POST.get('retryInterval'),
describe=request.POST.get('describe'),
url=request.POST.get('url'),
timeout=request.POST.get('timeout'),
contacts=request.POST.get('contacts'),
statusCode=request.POST.get('statusCode'),
status=False,
)
if m:
return HttpResponseRedirect('/index/')
else:
return HttpResponse('modify failed')
def job_completed(self, job_id):
logger = utils.getLogger()
logger.debug('jobs.job_completed: %s', job_id)
conn = self._conn()
self._execute_sql(conn, 'delete from tests where jobid=?', values=(job_id,))
self._execute_sql(conn, 'delete from jobs where id=?', values=(job_id,))
self._commit_connection(conn)
self._close_connection(conn)
def handle(self):
try:
data = ''
while True:
data = self.connection.recv(4)
if len(data) < 4:
break
pickle_len = struct.unpack('>L', data)[0]
data = self.connection.recv(pickle_len)
while len(data) < pickle_len:
data = data + self.connection.recv(pickle_len - len(data))
record = logging.makeLogRecord(pickle.loads(data))
logger = utils.getLogger(record.name)
logger.handle(record)
except:
logger = utils.getLogger()
logger.exception("Error receiving log record: data: %s", data)
def delete_anr_traces(self, root=True):
"""Empty ANR traces.txt file."""
try:
self.adb.rm(TRACES, root=root)
self.adb.shell_output('echo > %s' % TRACES, root=root)
self.adb.chmod(TRACES, mask='666', root=root)
except ADBError, e:
logger = utils.getLogger()
logger.warning("Could not initialize ANR traces %s, %s", TRACES, e)
def __init__(self, cfgfile, subject_prefix=''):
self.cfgfile = cfgfile
self.subject_prefix = subject_prefix
self.from_address = None
self.mail_dest = None
self.mail_username = None
self.mail_password = None
self.mail_server = None
self.mail_port = None
self.mail_ssl = None
logger = utils.getLogger()
cfg = ConfigParser.ConfigParser()
if not cfg.read(self.cfgfile):
logger.info('No email configuration file found. No emails will be sent.')
return
try:
self.from_address = cfg.get('report', 'from')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
logger.error('No "from" option defined in "report" section '
'of file "%s".\n', self.cfgfile)
return
try:
self.mail_dest = [x.strip() for x in cfg.get('email', 'dest').split(',')]
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
logger.error('No "dest" option defined in "email" section '
'of file "%s".\n', self.cfgfile)
return
try:
self.mail_username = cfg.get('email', 'username')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
self.mail_username = None
try:
self.mail_password = cfg.get('email', 'password')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
self.mail_password = None
try:
self.mail_server = cfg.get('email', 'server')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
self.mail_server = 'smtp.mozilla.org'
try:
self.mail_port = cfg.getint('email', 'port')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
self.mail_port = 25
try:
self.mail_ssl = cfg.getboolean('email', 'ssl')
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
self.mail_ssl = False
def handle_message(self, data, message):
if self._stopping.is_set():
return
message.ack()
logger = utils.getLogger()
if (self.treeherder_url and 'action' in data and
'project' in data and 'job_id' in data):
self.handle_jobaction(data, message)
elif 'status' in data:
logger.debug('handle_message: data: %s, message: %s', data, message)
self.handle_taskcompleted(data, message)
def start(self):
"""Runs the `listen` method on a new thread."""
logger = utils.getLogger()
if self.listen_thread and self.listen_thread.is_alive():
logger.warning('AutophonePulseMonitor.start: listen thread already started')
return
logger.debug('AutophonePulseMonitor.start: listen thread starting')
self.listen_thread = threading.Thread(target=self.listen,
name='PulseMonitorThread')
self.listen_thread.daemon = True
self.listen_thread.start()
def rm(self, keys=[], keypattern=None, prefix=''):
assert isinstance(keys, list) or isinstance(keypattern, str)
if isinstance(keypattern, str):
keys = self.ls(prefix=prefix, keypattern=keypattern)
try:
for key in keys:
key.delete()
except boto.exception.S3ResponseError, e:
logger = utils.getLogger()
logger.exception(str(e))
raise S3Error('%s' % e)
def serve_forever(self):
import select
while not self.shutdown_requested:
try:
rlist, wlist, xlist = select.select(
[self.socket.fileno()],
[],
[],
self.timeout)
if rlist:
self.handle_request()
except:
logger = utils.getLogger()
logger.exception("Error selecting log record socket")
def new_treeherder_job(self, machine, project, job_collection):
logger = utils.getLogger()
logger.debug('jobs.new_treeherder_job: %s %s %s',
machine, project, job_collection.__dict__)
attempts = 0
now = datetime.datetime.utcnow().isoformat()
conn = self._conn()
job_cursor = self._execute_sql(
conn,
'insert into treeherder values (?, ?, ?, ?, ?, ?)',
values=(None, attempts, now, machine, project, job_collection.to_json()))
job_cursor.close()
self._commit_connection(conn)
self._close_connection(conn)
def report_sql_error(self, attempt, email_sent, sql, values):
logger = utils.getLogger()
message = '%s %s' % (sql, values)
logger.exception(message)
if attempt > self.SQL_MAX_RETRIES and not email_sent:
email_sent = True
email_subject = '%s jobs SQL Error' % utils.host()
email_body = (
'Attempt %d to execute %s failed.\n'
'%s' %
(attempt, message, traceback.format_exc()))
self.mailer.send(email_subject, email_body)
logger.info('Sent mail notification about jobs database sql error.')
time.sleep(self.SQL_RETRY_DELAY)
return email_sent
def cancel_test(self, test_guid, device=None):
logger = utils.getLogger()
logger.debug('jobs.cancel_test: test %s device %s',
test_guid, device)
if not device:
device = self.default_device
conn = self._conn()
# Get the jobid for this test.
job_ids = [
test_row[0]
for test_row in self._execute_sql(conn,
'select jobid from tests '
'where guid=?', values=(test_guid,))
]
if not job_ids:
logger.debug('jobs.cancel_test: test %s for device %s '
'already deleted', test_guid, device)
self._close_connection(conn)
return
job_id = job_ids[0]
# Delete the test.
self._execute_sql(
conn,
'delete from tests where guid=?',
values=(test_guid,))
# Get the number of remaining tests for this job and delete
# the job if there are no more remaining tests for the job.
test_cursor = self._execute_sql(
conn,
'select count(id) from tests '
'where jobid=?', values=(job_id,))
count = test_cursor.fetchone()[0]
test_cursor.close()
if count == 0:
logger.debug('jobs.cancel_test: delete job_id %s device %s',
job_id, device)
self._execute_sql(
conn,
'delete from jobs where id=?',
values=(job_id,))
self._commit_connection(conn)
self._close_connection(conn)
def hook_tk_errors():
"""TKinter catches and swallows callback errors.
we need to hook into that to log those seperately.
"""
import loadScreen
import traceback
main_logger = utils.getLogger()
def tk_error(exc_type, exc_value, exc_tb):
"""Log TK errors."""
# The exception is caught inside the TK code.
# We don't care about that, so try and move the traceback up
# one level.
import sys
if exc_tb.tb_next:
exc_tb = exc_tb.tb_next
main_logger.error(
'TKinter callback exception occurred:\n{}',
''.join(
traceback.format_exception(
exc_type,
exc_value,
exc_tb,
)
),
)
# Close loading screens if they're visible..
loadScreen.close_all()
# Release the grab, if it exists. Otherwise you can't see the error dialog.
TK_ROOT.grab_set_global()
TK_ROOT.grab_release()
messagebox.showerror(
title='BEE2 Error:',
message='{}: {!r}'.format(exc_type.__name__, exc_value),
parent=TK_ROOT,
)
# Since this isn't caught normally, it won't quit the application.
# Quit ourselves manually. to prevent TK just freezing.
TK_ROOT.quit()
sys.exit()
TK_ROOT.report_callback_exception = tk_error
def bucket(self):
if self._bucket:
return self._bucket
logger = utils.getLogger()
try:
conn = boto.s3.connection.S3Connection(self.access_key_id,
self.access_secret_key)
if not conn.lookup(self.bucket_name):
raise S3Error('bucket %s not found' % self.bucket_name)
if not self._bucket:
self._bucket = conn.get_bucket(self.bucket_name)
return self._bucket
except boto.exception.NoAuthHandlerFound:
logger.exception('Error accessing bucket')
raise S3Error('Authentication failed')
except boto.exception.S3ResponseError, e:
logger.exception('Error accessing bucket')
raise S3Error('%s' % e)
def get_java_exception(self):
"""Returns a summary of the first fatal Java exception found in
logcat output.
Example:
{
'reason': 'java-exception',
'signature': 'java.lang.NullPointerException at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833)'
}
"""
logger = utils.getLogger()
logre = re.compile(r".*\): \t?(.*)")
exception = None
logcat = self.adb.get_logcat()
for i, line in enumerate(logcat):
# Logs will be of form:
#
# 01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> REPORTING UNCAUGHT EXCEPTION FROM THREAD 9 ("GeckoBackgroundThread")
# 01-30 20:15:41.937 E/GeckoAppShell( 1703): java.lang.NullPointerException
# 01-30 20:15:41.937 E/GeckoAppShell( 1703): at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833)
# 01-30 20:15:41.937 E/GeckoAppShell( 1703): at android.os.Handler.handleCallback(Handler.java:587)
if "REPORTING UNCAUGHT EXCEPTION" in line or "FATAL EXCEPTION" in line:
# Strip away the date, time, logcat tag and pid from the next two lines and
# concatenate the remainder to form a concise summary of the exception.
if len(logcat) >= i + 3:
exception_type = ''
exception_location = ''
m = logre.search(logcat[i+1])
if m and m.group(1):
exception_type = m.group(1)
m = logre.search(logcat[i+2])
if m and m.group(1):
exception_location = m.group(1)
if exception_type:
exception = {'reason': 'java-exception',
'signature': "%s %s" % (
exception_type, exception_location)}
else:
logger.warning("Automation Error: check_for_java_exceptions: Logcat is truncated!")
break
return exception
from config import args
from sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import Mail
from ArticleParser import ArticleParser
from utils import getLogger
logger = getLogger("Feed Mail")
class FeedMail:
""""Sends Recommended feed as email to the user
Args:
user_email(str): Email of the user to which feed us to be sent
recommendations(dict): Recommendations of the corresponding user
Attributes:
from_email(str): Email of smartfeed.ai
key(str): SendGrid API's key
html_content(str): Consists the body of the email
subject(str): Consists the subject of the email
"""
def __init__(self, user_email, recommendations):
self.recommendations = recommendations
self.to_email = user_email
self.from_email = args["FROM_EMAIL"]
self.key = args["SEND_GRID_KEY"]
self.html_content = self._parseEmailContent(self.recommendations)
self.subject = self._getEmailSubject()
def __init__(self, region, interval):
self.region = region
self.interval = interval
self.logger = getLogger(region)
import conditions
import instanceLocs
import srctools
import vbsp
import vbsp_options
import utils
from conditions import (
make_result,
RES_EXHAUSTED,
GLOBAL_INSTANCES,
)
from srctools import Vec, Entity, Property
COND_MOD_NAME = 'Instance Generation'
LOGGER = utils.getLogger(__name__, 'cond.addInstance')
@make_result('addGlobal')
def res_add_global_inst(res: Property):
"""Add one instance in a location.
Options:
allow_multiple: Allow multiple copies of this instance. If 0, the
instance will not be added if it was already added.
name: The targetname of the instance. IF blank, the instance will
be given a name of the form 'inst_1234'.
file: The filename for the instance.
Angles: The orientation of the instance (defaults to '0 0 0').
Origin: The location of the instance (defaults to '0 0 -10000').
Fixup_style: The Fixup style for the instance. '0' (default) is
from tkinter import ttk
from weakref import WeakSet
from abc import abstractmethod
import contextlib
import multiprocessing
from loadScreen_daemon import run_screen as _splash_daemon
import utils
from typing import Set
# Keep a reference to all loading screens, so we can close them globally.
_ALL_SCREENS = WeakSet() # type: Set[BaseLoadScreen]
LOGGER = utils.getLogger(__name__)
def close_all():
"""Hide all loadscreen windows."""
for screen in _ALL_SCREENS:
screen.reset()
@contextlib.contextmanager
def surpress_screens():
"""A context manager to supress loadscreens while the body is active."""
active = []
for screen in _ALL_SCREENS:
if not screen.active:
continue
Dict,
List,
Tuple,
NamedTuple,
)
import comp_consts as consts
import srctools
import template_brush
import utils
import instanceLocs
from srctools import (Property, Vec_tuple, Vec, Entity, Output, Solid, Side)
COND_MOD_NAME = 'Main Conditions'
LOGGER = utils.getLogger(__name__, alias='cond.core')
# Stuff we get from VBSP in init()
GLOBAL_INSTANCES = set()
ALL_INST = set()
VMF = None # type: srctools.VMF
conditions = []
FLAG_LOOKUP = {}
RESULT_LOOKUP = {}
RESULT_SETUP = {}
# Used to dump a list of the flags, results, meta-conditions
ALL_FLAGS = []
ALL_RESULTS = []
ALL_META = []
# Wolfinch is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Wolfinch. If not, see <https://www.gnu.org/licenses/>.
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
from sklearn.preprocessing import MinMaxScaler
from utils import getLogger
log = getLogger ('MODEL')
log.setLevel(log.DEBUG)
class Model ():
EPOCH = 300
BATCH_SIZE = 64
def __init__(self, X_shape):
# init preprocessor/scaler as the number of feature layers and o/p layer
self.scX = []
for _ in range (X_shape[1]):
self.scX += [MinMaxScaler(feature_range = (0, 1))]
self.scY = MinMaxScaler(feature_range = (0, 1))
# Init Keras
self.regressor = Sequential()
mpl.rcParams["savefig.bbox"] = "tight"
mpl.rcParams["font.size"] = 10.0
mpl.rcParams["font.sans-serif"] = "Arial"
mpl.rcParams["savefig.format"] = "pdf"
import numpy as np
import pandas as pd
from joblib import Parallel, delayed
import brewer2mpl
#this
from utils import getLogger
#global settings
#logger
date = time.strftime(' %Y-%m-%d', time.localtime(time.time()))
logger = getLogger(fn=os.getcwd() + "/" + date.strip() + "_" +
os.path.basename(__file__) + ".log")
#data
CHROM_SIZE = "/home/caoy7/caoy7/Projects/0.Reference/2.mm10/1.fa/mm10.chrom.sizes"
STAR_INDEX = "/home/caoy7/caoy7/Projects/0.Reference/2.mm10/3.index/3.star/1.50bp"
def call_sys(cmds):
"""
Call systematic commands without return.
"""
for c in cmds:
logger.info(c)
try:
os.system(c)
except:
logger.error(c)
parser.add_argument("--dataset", type=str)
parser.add_argument("--action", type=str)
parser.add_argument("--max_words", type=int, default=50000)
parser.add_argument("--doc_count_threshold", type=int, default=1)
parser.add_argument("--lower_case", type=bool, default=False)
parser.add_argument("--include_unk", type=bool, default=False)
parser.add_argument("--vocab_freqs_file", type=str, default=None)
parser.add_argument("--summary_vocab_freqs_file", type=str, default=None)
parser.add_argument("--merged_summary_vocab_freqs_file",
type=str,
default=None)
parser.add_argument("--validation_rate", type=float, default=0)
parser.add_argument("--shuffle_onval", type=bool, default=True)
parser.add_argument("--no_need_start_tag", type=bool, default=True)
FLAGS = parser.parse_args()
logger = getLogger("DataGenerator")
def _load_words(filename, wordCount, max_words):
training_words = wordCount.transform([osp.join(FLAGS.data_dir, filename)],
max_words=max_words,
include_unk=FLAGS.include_unk)
n_samples_training_words = len(training_words)
min_seqlen_training_words = min(map(len, training_words))
max_seqlen_training_words = max(map(len, training_words))
logger.info(
"total number of words: %s; min_seqlen in words: %s; max_seqlen in words: %s"
% (n_samples_training_words, min_seqlen_training_words,
max_seqlen_training_words))
return training_words
import conditions
import instanceLocs
import srctools
import utils
import vbsp
import vbsp_options
import comp_consts as const
from conditions import (
make_result,
RES_EXHAUSTED,
)
from srctools import Property, Vec, Entity, Output
COND_MOD_NAME = None
LOGGER = utils.getLogger(__name__, alias='cond.resizeTrig')
@make_result('ResizeableTrigger')
def res_resizeable_trigger(res: Property):
"""Replace two markers with a trigger brush.
This is run once to affect all of an item.
Options:
* `markerInst`: <ITEM_ID:1,2> value referencing the marker instances, or a filename.
* `markerItem`: The item's ID
* `previewConf`: A item config which enables/disables the preview overlay.
* `previewinst`: An instance to place at the marker location in preview mode.
This should contain checkmarks to display the value when testing.
* `previewMat`: If set, the material to use for an overlay func_brush.
The brush will be parented to the trigger, so it vanishes once killed.
from conditions import (
make_result,
RES_EXHAUSTED,
INST_ANGLE,
)
from instanceLocs import resolve as resolve_inst
from srctools import Vec, Property
import conditions
import utils
import vbsp
LOGGER = utils.getLogger(__name__, alias='cond.catwalks')
CATWALK_TYPES = {
utils.CONN_TYPES.straight: 'straight_128',
utils.CONN_TYPES.corner: 'corner',
utils.CONN_TYPES.all: 'crossjunction',
utils.CONN_TYPES.side: 'end',
utils.CONN_TYPES.triple: 'tjunction',
utils.CONN_TYPES.none: 'NONE',
}
def place_catwalk_connections(instances, point_a, point_b):
"""Place catwalk sections to connect two straight points."""
diff = point_b - point_a
# The horizontal unit vector in the direction we are placing catwalks
direction = diff.copy()
direction.z = 0
distance = direction.len() - 128
#
# Wolfinch is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Wolfinch. If not, see <https://www.gnu.org/licenses/>.
import random
from . import eval_strategy
from utils import getLogger
# __name__ = "EA-OPS"
log = getLogger(__name__)
log.setLevel(log.CRITICAL)
#TODO: FIXME: lot of hacky code here to fix deap ind generator issue with strat dict
def selectOneMax(individual, res_dict):
log.debug(" individual: %s" % (individual))
fitnessVal = eval_strategy.eval_strategy_with_config(individual)
log.debug("fitnessVal: %d" % fitnessVal)
res_dict["res"] = fitnessVal,
def main():
global LOG
LOG = utils.getLogger("pack-tool")
try:
usage = "Usage: ./pack.py -t apk -m shared -a x86"
opts_parser = OptionParser(usage=usage)
opts_parser.add_option("-c",
"--cfg",
dest="pkgcfg",
help="specify the path of config json file")
opts_parser.add_option(
"-t",
"--type",
dest="pkgtype",
help="specify the pkg type, e.g. apk, cordova ...")
opts_parser.add_option(
"-m",
"--mode",
dest="pkgmode",
help=
"specify the apk mode, not for embeddingapi, e.g. shared, embedded"
)
opts_parser.add_option(
"-a",
"--arch",
dest="pkgarch",
help="specify the apk arch, not for embeddingapi, e.g. x86, arm")
opts_parser.add_option(
"-d",
"--dest",
dest="destdir",
help="specify the installation folder for packed package")
opts_parser.add_option(
"-s",
"--src",
dest="srcdir",
help="specify the path of pkg resource for packing")
opts_parser.add_option("--tools",
dest="pkgpacktools",
help="specify the parent folder of pack tools")
opts_parser.add_option(
"--notclean",
dest="bnotclean",
action="store_true",
help="disable the build root clean after the packing")
opts_parser.add_option("-v",
"--version",
dest="bversion",
action="store_true",
help="show this tool's version")
opts_parser.add_option("-u",
"--user",
dest="user",
help="specify the user in inst.py")
opts_parser.add_option(
"--sub-version",
dest="subversion",
help=
"specify the embeddingapi, cordova sub version, e.g. v1, v2, v3 ..."
)
opts_parser.add_option("--pkg-version",
dest="pkgversion",
help="specify the pkg version, e.g. 0.0.0.1")
opts_parser.add_option(
"--pack-type",
dest="packtype",
help="specify the pack type, e.g. gradle, maven")
opts_parser.add_option("--notdebug",
dest="bnotdebug",
action="store_true",
help="specify the packed pkg is not debug mode")
opts_parser.add_option("--resource-only",
dest="resourceonly",
action="store_true",
help="only restore resources to project root")
if len(sys.argv) == 1:
sys.argv.append("-h")
global BUILD_PARAMETERS
(BUILD_PARAMETERS, args) = opts_parser.parse_args()
except Exception as e:
LOG.error("Got wrong options: %s, exit ..." % e)
sys.exit(1)
if BUILD_PARAMETERS.bversion:
print "Version: %s" % TOOL_VERSION
sys.exit(0)
if not BUILD_PARAMETERS.srcdir:
BUILD_PARAMETERS.srcdir = os.getcwd()
BUILD_PARAMETERS.srcdir = os.path.expanduser(BUILD_PARAMETERS.srcdir)
if not BUILD_PARAMETERS.user:
BUILD_PARAMETERS.user = "******"
if not os.path.exists(
os.path.join(BUILD_PARAMETERS.srcdir, "..", "..", VERSION_FILE)):
if not os.path.exists(
os.path.join(BUILD_PARAMETERS.srcdir, "..", VERSION_FILE)):
if not os.path.exists(
os.path.join(BUILD_PARAMETERS.srcdir, VERSION_FILE)):
LOG.info(
"Not found pkg version file, try to use option --pkg-version"
)
pkg_version_file_path = None
else:
pkg_version_file_path = os.path.join(BUILD_PARAMETERS.srcdir,
VERSION_FILE)
else:
pkg_version_file_path = os.path.join(BUILD_PARAMETERS.srcdir, "..",
VERSION_FILE)
else:
pkg_version_file_path = os.path.join(BUILD_PARAMETERS.srcdir, "..",
"..", VERSION_FILE)
try:
pkg_main_version = 0
pkg_release_version = 1
if BUILD_PARAMETERS.pkgversion:
LOG.info("Using %s as pkg version " % BUILD_PARAMETERS.pkgversion)
pkg_main_version = BUILD_PARAMETERS.pkgversion
CROSSWALK_BRANCH = "master"
else:
if pkg_version_file_path is not None:
LOG.info("Using pkg version file: %s" % pkg_version_file_path)
with open(pkg_version_file_path, "rt") as pkg_version_file:
pkg_version_raw = pkg_version_file.read()
pkg_version_file.close()
pkg_version_json = json.loads(pkg_version_raw)
pkg_main_version = pkg_version_json["main-version"]
pkg_release_version = pkg_version_json["release-version"]
CROSSWALK_BRANCH = pkg_version_json["crosswalk-branch"]
except Exception as e:
LOG.error("Fail to read pkg version file: %s, exit ..." % e)
sys.exit(1)
CROSSWALK_VERSION = pkg_main_version
if not BUILD_PARAMETERS.pkgtype:
LOG.error("No pkg type provided, exit ...")
sys.exit(1)
elif not BUILD_PARAMETERS.pkgtype in PKG_TYPES:
LOG.error("Wrong pkg type, only support: %s, exit ..." % PKG_TYPES)
sys.exit(1)
if BUILD_PARAMETERS.resourceonly:
LOG.info("Starting copy resource only")
resource_only.copy_resource(BUILD_PARAMETERS.pkgtype)
sys.exit(0)
if BUILD_PARAMETERS.pkgtype == "apk" or \
BUILD_PARAMETERS.pkgtype == "apk-aio":
if not BUILD_PARAMETERS.pkgmode:
LOG.error("No pkg mode option provided, exit ...")
sys.exit(1)
elif not BUILD_PARAMETERS.pkgmode in PKG_MODES:
LOG.error(
"Wrong pkg mode option provided, only support:%s, exit ..." %
PKG_MODES)
sys.exit(1)
if not BUILD_PARAMETERS.pkgarch:
LOG.error("No pkg arch option provided, exit ...")
sys.exit(1)
elif not BUILD_PARAMETERS.pkgarch in PKG_ARCHS:
LOG.error(
"Wrong pkg arch option provided, only support:%s, exit ..." %
PKG_ARCHS)
sys.exit(1)
if BUILD_PARAMETERS.pkgtype == "apk-aio" or \
BUILD_PARAMETERS.pkgtype == "cordova-aio":
if not BUILD_PARAMETERS.destdir or not os.path.exists(
BUILD_PARAMETERS.destdir):
LOG.error("No all-in-one installation dest dir found, exit ...")
sys.exit(1)
elif not BUILD_PARAMETERS.destdir:
BUILD_PARAMETERS.destdir = BUILD_PARAMETERS.srcdir
BUILD_PARAMETERS.destdir = os.path.expanduser(BUILD_PARAMETERS.destdir)
if not BUILD_PARAMETERS.pkgpacktools:
BUILD_PARAMETERS.pkgpacktools = os.path.join(BUILD_PARAMETERS.srcdir,
"..", "..", "tools")
BUILD_PARAMETERS.pkgpacktools = os.path.expanduser(
BUILD_PARAMETERS.pkgpacktools)
config_json = None
if BUILD_PARAMETERS.pkgcfg:
config_json_file_path = BUILD_PARAMETERS.pkgcfg
else:
config_json_file_path = os.path.join(BUILD_PARAMETERS.srcdir,
"suite.json")
try:
LOG.info("Using config json file: %s" % config_json_file_path)
with open(config_json_file_path, "rt") as config_json_file:
config_raw = config_json_file.read()
config_json_file.close()
config_json = json.loads(config_raw)
except Exception as e:
LOG.error("Fail to read config json file: %s, exit ..." % e)
sys.exit(1)
global PKG_NAME
PKG_NAME = utils.safelyGetValue(config_json, "pkg-name")
if not PKG_NAME:
PKG_NAME = os.path.basename(BUILD_PARAMETERS.srcdir)
LOG.warning("Fail to read pkg name from json, "
"using src dir name as pkg name ...")
LOG.info("================= %s (%s-%s) ================" %
(PKG_NAME, pkg_main_version, pkg_release_version))
if not utils.safelyGetValue(config_json, "pkg-list"):
LOG.error("Fail to read pkg-list, exit ...")
sys.exit(1)
pkg_json = None
global parameters_type
parameters_type = None
if BUILD_PARAMETERS.pkgtype == "cordova" or BUILD_PARAMETERS.pkgtype == "cordova-aio":
if BUILD_PARAMETERS.pkgarch and not BUILD_PARAMETERS.pkgarch in PKG_ARCHS:
LOG.error("Wrong pkg-arch, only support: %s, exit ..." % PKG_ARCHS)
sys.exit(1)
if BUILD_PARAMETERS.pkgmode and not BUILD_PARAMETERS.pkgmode in PKG_MODES:
LOG.error("Wrong pkg-mode, only support: %s, exit ..." % PKG_MODES)
sys.exit(1)
if BUILD_PARAMETERS.packtype and not BUILD_PARAMETERS.packtype in CORDOVA_PACK_TYPES:
LOG.error("cordova packtype can only be npm, local")
sys.exit(1)
if BUILD_PARAMETERS.pkgtype == "embeddingapi":
if BUILD_PARAMETERS.packtype and not BUILD_PARAMETERS.packtype in PACK_TYPES:
LOG.error("embeddingapi packtype can only be gradle, maven or ant")
sys.exit(1)
if BUILD_PARAMETERS.subversion:
BUILD_PARAMETERS.pkgtype = BUILD_PARAMETERS.pkgtype + \
BUILD_PARAMETERS.subversion
all_pkg_string = "".join(config_json["pkg-list"].keys())
if parameters_type and parameters_type in all_pkg_string:
for i_pkg in config_json["pkg-list"].keys():
i_pkg_list = i_pkg.replace(" ", "").split(",")
if parameters_type in i_pkg_list:
pkg_json = config_json["pkg-list"][i_pkg]
break
elif BUILD_PARAMETERS.pkgtype in all_pkg_string:
for i_pkg in config_json["pkg-list"].keys():
i_pkg_list = i_pkg.replace(" ", "").split(",")
if BUILD_PARAMETERS.pkgtype in i_pkg_list:
pkg_json = config_json["pkg-list"][i_pkg]
break
if pkg_json == config_json['pkg-list'].get(
'apk') and BUILD_PARAMETERS.subversion is not None:
pkg_json = config_json["pkg-list"][BUILD_PARAMETERS.subversion]
if not pkg_json:
LOG.error("Fail to read pkg json, exit ...")
sys.exit(1)
if not prepareBuildRoot():
exitHandler(1)
if "pkg-blacklist" in config_json:
PKG_BLACK_LIST.extend(config_json["pkg-blacklist"])
try:
varshop.setValue("BUILD_PARAMETERS", BUILD_PARAMETERS)
varshop.setValue("BUILD_ROOT", BUILD_ROOT)
varshop.setValue("BUILD_ROOT_SRC", BUILD_ROOT_SRC)
varshop.setValue("BUILD_TIME", BUILD_TIME)
varshop.setValue("CROSSWALK_BRANCH", CROSSWALK_BRANCH)
varshop.setValue("CROSSWALK_VERSION", CROSSWALK_VERSION)
varshop.setValue("DEFAULT_CMD_TIMEOUT", DEFAULT_CMD_TIMEOUT)
varshop.setValue("PKG_MODES", PKG_MODES)
varshop.setValue("PKG_ARCHS", PKG_ARCHS)
except Exception as e:
LOG.error("Fail to set global vars: %s, exit ..." % e)
sys.exit(1)
if not buildPKG(pkg_json):
exitHandler(1)
LOG.info("+Building package ...")
if BUILD_PARAMETERS.pkgtype == "apk-aio" or \
BUILD_PARAMETERS.pkgtype == "cordova-aio":
pkg_file_list = os.listdir(os.path.join(BUILD_ROOT, "pkg"))
for i_file in pkg_file_list:
if not utils.doCopy(os.path.join(BUILD_ROOT, "pkg", i_file),
os.path.join(BUILD_PARAMETERS.destdir,
i_file)):
exitHandler(1)
elif BUILD_PARAMETERS.pkgtype == "embeddingapi" and BUILD_PARAMETERS.subversion:
pkg_file = os.path.join(
BUILD_PARAMETERS.destdir, "%s-%s-%s-%s.%s.zip" %
(PKG_NAME, pkg_main_version, pkg_release_version,
BUILD_PARAMETERS.subversion, BUILD_PARAMETERS.pkgtype))
LOG.info("pkg_file: %s" % pkg_file)
if not utils.zipDir(os.path.join(BUILD_ROOT, "pkg"), pkg_file):
exitHandler(1)
elif BUILD_PARAMETERS.pkgtype.startswith(
"embeddingapi") and BUILD_PARAMETERS.packtype:
pkg_file = os.path.join(
BUILD_PARAMETERS.destdir, "%s-%s-%s.%s-%s.zip" %
(PKG_NAME, pkg_main_version, pkg_release_version,
BUILD_PARAMETERS.pkgtype, BUILD_PARAMETERS.packtype))
if not utils.zipDir(os.path.join(BUILD_ROOT, "pkg"), pkg_file):
exitHandler(1)
else:
pkg_file = os.path.join(
BUILD_PARAMETERS.destdir, "%s-%s-%s.%s.zip" %
(PKG_NAME, pkg_main_version, pkg_release_version,
BUILD_PARAMETERS.pkgtype))
if not utils.zipDir(os.path.join(BUILD_ROOT, "pkg"), pkg_file):
exitHandler(1)
from decimal import getcontext
import random
# import logging
from utils import getLogger, get_product_config, load_config, get_config
import sims
import exchanges
from market import market_init, market_setup, get_market_list, \
feed_Q_process_msg, feed_deQ, get_market_by_product
import db
import stats
import ui
from ui import ui_conn_pipe
# mpl_logger = logging.getLogger('matplotlib')
# mpl_logger.setLevel(logging.WARNING)
log = getLogger('Wolfinch')
log.setLevel(log.INFO)
gRestart = False
# global Variables
MAIN_TICK_DELAY = 0.500 # 500 milli
def Wolfinch_init():
# seed random
random.seed()
# 1. Retrieve states back from Db
# db.init_order_db(Order)
# Custom imports
from src.warmup_scheduler import GradualWarmupScheduler
from utils import (
AverageMeter,
accuracy,
aggregrate_video_accuracy,
initialize_exp,
getLogger,
accuracy,
save_checkpoint,
load_model_parameters
)
from datasets.AVideoDataset import AVideoDataset
from model import load_model, Identity, get_video_dim
logger = getLogger()
# DICT with number of classes for each dataset
NUM_CLASSES = {
'hmdb51': 51,
'ucf101': 101,
}
# Create Finetune Model
class Finetune_Model(torch.nn.Module):
def __init__(
self,
base_arch,
num_ftrs=512,
import sys
from decimal import getcontext
import argparse
import os
import json
from threading import Lock
from flask import Flask, request, send_from_directory
from utils import getLogger
from . import db_events
g_markets_list = None
g_active_market = {} # {"CBPRO": "BTC-USD"}
log = getLogger('UI')
log.setLevel(log.INFO)
static_file_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)),
'../data/')
MARKET_STATS = "stats_market_%s_%s.json"
UI_CODES_FILE = "data/ui_codes.json"
UI_TRADE_SECRET = None
UI_PAGE_SECRET = None
g_mp_lock = None
def load_ui_codes():
global UI_TRADE_SECRET, UI_PAGE_SECRET
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors:
# Yang, Yunlong <*****@*****.**>
import os
import glob
import subprocess
import varshop
import utils
global LOG
LOG = utils.getLogger("build_extension")
def packExtension(build_json=None, app_src=None, app_dest=None, app_name=None):
BUILD_PARAMETERS = varshop.getValue("BUILD_PARAMETERS")
BUILD_ROOT = varshop.getValue("BUILD_ROOT")
BUILD_ROOT_SRC = varshop.getValue("BUILD_ROOT_SRC")
BUILD_TIME = varshop.getValue("BUILD_TIME")
DEFAULT_CMD_TIMEOUT = varshop.getValue("DEFAULT_CMD_TIMEOUT")
PKG_MODES = varshop.getValue("PKG_MODES")
PKG_ARCHS = varshop.getValue("PKG_ARCHS")
app_name = app_name.replace("-", "_")
if not os.path.exists(os.path.join(BUILD_ROOT, "crosswalk")):
if not utils.doCopy(
os.path.join(BUILD_PARAMETERS.pkgpacktools, "crosswalk"),
