def add(request): if request.method == 'GET': return render_to_response('add.html') else: m=createMonitor( log=getLogger(), name=request.POST.get('name'), interval=request.POST.get('interval'), mailInterval=request.POST.get('mailInterval'), smsInterval=request.POST.get('smsInterval'), smsBegin=request.POST.get('smsBegin'), smsEnd=request.POST.get('smsEnd'), retryTimes=request.POST.get('retryTimes'), retryInterval=request.POST.get('retryInterval'), describe=request.POST.get('describe'), url=request.POST.get('url'), timeout=request.POST.get('timeout'), contacts=request.POST.get('contacts'), statusCode=request.POST.get('statusCode'), status=False, ) if m: return HttpResponseRedirect('/index/') else: return HttpResponse('create failed')
def __init__(self, circle, fcp, totalsize=0): BaseTask.__init__(self, circle) self.circle = circle self.fcp = fcp self.totalsize = totalsize # cache self.fd_cache = LRU(512) # failed self.failed = {} self.failcnt = 0 # debug self.d = {"rank": "rank %s" % circle.rank} self.logger = utils.getLogger(__name__) # reduce self.vsize = 0 assert len(circle.workq) == 0 if self.circle.rank == 0: print("\nChecksum verification ...")
def initAPIExport (apiName, psName, apiType, newAPIFlag): """ Purpose: Method fo initializing the API export from Akana CM and PS (Production & Sandbox) export from Akana PM Input: Name of the API and Physical Service, the type of API (INTERNAL/EXTERNAL) and whether this is the first time API is being exported (Y/N) Returns: Dictionary object comprising of the relative path of the TAR file for API and PS """ global mylogger mylogger = utils.getLogger ("APA-API-EXPORT") api_tar_path = "" if (apiName != ""): api_tar_path = processCMExport (apiName, apiType, newAPIFlag) ps_tar_path = "" pssb_tar_path = "" if (psName != ""): # Export Prod PS ps_tar_path = processPMExport (psName, apiType, newAPIFlag) # Export Sandbox PS pssb_tar_path = processPMExport (psName+"sb", apiType, newAPIFlag) tarFileDict = { "api_tar_path" : api_tar_path, "ps_tar_path" : ps_tar_path , "pssb_tar_path" : pssb_tar_path } return tarFileDict
def __init__(self, circle, fcp, total_chunks, totalsize=0,signature=False): BaseTask.__init__(self, circle) self.circle = circle self.fcp = fcp self.totalsize = totalsize self.signature = signature if self.signature: self.bfsign = BFsignature(total_chunks) # failed self.failed = {} self.failcnt = 0 # debug self.d = {"rank": "rank %s" % circle.rank} self.logger = utils.getLogger(__name__) # reduce self.vsize = 0 assert len(circle.workq) == 0 if self.circle.rank == 0: print("\nChecksum verification ...")
def initSwaggerCreation(tarURL): """ Purpose: Initiate the process of the TAR file containing the Swagger specifications for an API. The TAR file is downloaded from a Nexus URL and stored in 'source' directory. Since the Swagger files are embedded in a TAR-EAR-WAR structure the contents need to be extracted first and then modified The target directory is the one in which the modified swagger files will be created Input: URL of the TAR file containing the Swagger files Returns: List containing relative path of the modified Swagger zip files """ global mylogger mylogger = utils.getLogger("APA-SWG-PROC") try: mylogger.info("************************************") mylogger.info("BEGIN -- initSwaggerCreation()") if tarURL == "" or tarURL == None: raise Exception("Swagger TAR file name not provided") swaggerDir = swgConfig.get("SWG", "swaggerDir") tmpSwaggerLocation = os.path.join(swaggerDir, swgConfig.get("SWG", "tmpSwaggerLocation")) swaggerSourceLocation = os.path.join(swaggerDir, swgConfig.get("SWG", "swaggerSourceLocation")) urllib.request.urlretrieve(tarURL, os.path.join(swaggerSourceLocation, "tarname.tar")) sourcePathDict = extractSwaggerFromSource(swaggerSourceLocation, "tarname.tar") swaggerZipFileList = processSwagger(tmpSwaggerLocation, sourcePathDict) mylogger.info("END -- initSwaggerCreation()") mylogger.info("************************************") return swaggerZipFileList except: mylogger.error(traceback.format_exc()) sys.exit()
def treeherder_job_completed(self, th_id): logger = utils.getLogger() logger.debug('jobs.treeherder_job_completed: %s', th_id) conn = self._conn() self._execute_sql(conn, 'delete from treeherder where id=?', values=(th_id,)) self._commit_connection(conn) self._close_connection(conn)
def stop(self): """Stops the pulse monitor listen thread.""" logger = utils.getLogger() logger.debug('AutophonePulseMonitor stopping') self._stopping.set() self.listen_thread.join() logger.debug('AutophonePulseMonitor stopped')
def hook_tk_errors(): """TKinter catches and swallows callback errors. we need to hook into that to log those seperately. """ import traceback main_logger = utils.getLogger() def tk_error(exc_type, exc_value, exc_tb): """Log TK errors.""" # The exception is caught inside the TK code. # We don't care about that, so try and move the traceback up # one level. if exc_tb.tb_next: exc_tb = exc_tb.tb_next main_logger.error( 'TKinter callback exception occurred:\n{}', ''.join( traceback.format_exception( exc_type, exc_value, exc_tb, ) ), ) # Don't shutdown - most of the time, the TK errors won't # shutdown the application. TK_ROOT.report_callback_exception = tk_error
def get_treeherder_privatebuild_info(self, project, job): logger = utils.getLogger() url = '%s/api/jobdetail/?repository=%s&job_id=%s' % ( self.treeherder_url, project, job['id']) data = utils.get_remote_json(url) logger.debug("get_treeherder_privatebuild_info: data: %s", data) privatebuild_keys = set(['build_url', 'config_file', 'chunk', 'builder_type']) info = {} for detail in data['results']: if detail['title'] in privatebuild_keys: # the build_url property is too long to fit into "value" if detail.get('url'): value = detail['url'] else: value = detail['value'] info[detail['title']] = value # If we're missing a privatebuild key for some reason # return None to avoid errors. missing_keys = privatebuild_keys - set(info.keys()) if missing_keys: logger.debug("get_treeherder_privatebuild_info: %s " "missing keys: %s " "job: %s", url, missing_keys, job) return None return info
def get_next_treeherder_job(self): logger = utils.getLogger() conn = self._conn() job_cursor = self._execute_sql( conn, 'select id,attempts,last_attempt,machine,project,job_collection from treeherder') job_row = job_cursor.fetchone() job_cursor.close() if not job_row: self._close_connection(conn) return None job = {'id': job_row[0], 'attempts': job_row[1], 'last_attempt': job_row[2], 'machine': job_row[3], 'project': job_row[4], 'job_collection': json.loads(job_row[5])} job['attempts'] += 1 job['last_attempt'] = datetime.datetime.utcnow().isoformat() self._execute_sql( conn, 'update treeherder set attempts=?, last_attempt=? where id=?', values=(job['attempts'], job['last_attempt'], job['id'])) logger.debug('jobs.get_next_treeherder_job: %s', job) self._commit_connection(conn) self._close_connection(conn) return job
def upload(self, path, destination): try: logger = utils.getLogger() key = self.bucket.get_key(destination) if not key: logger.debug('Creating key: %s', destination) key = self.bucket.new_key(destination) ext = os.path.splitext(path)[-1] if ext == '.log' or ext == '.txt': key.set_metadata('Content-Type', 'text/plain') with tempfile.NamedTemporaryFile('w+b', suffix=ext) as tf: logger.debug('Compressing: %s', path) with gzip.GzipFile(path, 'wb', fileobj=tf) as gz: with open(path, 'rb') as f: gz.writelines(f) tf.flush() tf.seek(0) key.set_metadata('Content-Encoding', 'gzip') logger.debug('Setting key contents from: %s', tf.name) key.set_contents_from_file(tf) url = key.generate_url(expires_in=0, query_auth=False) except boto.exception.S3ResponseError, e: logger.exception(str(e)) raise S3Error('%s' % e)
def serve_forever(self): logger = utils.getLogger() while not self.shutdown_requested: wait_seconds = 1 # avoid busy loop job = self.jobs.get_next_treeherder_job() if job: tjc = TreeherderJobCollection() for data in job['job_collection']: tj = TreeherderJob(data) tjc.add(tj) if self.post_request(job['machine'], job['project'], tjc, job['attempts'], job['last_attempt']): self.jobs.treeherder_job_completed(job['id']) wait_seconds = 0 else: attempts = int(job['attempts']) wait_seconds = min(self.retry_wait * attempts, 3600) logger.debug('AutophoneTreeherder waiting for %d seconds after ' 'failed attempt %d', wait_seconds, attempts) if wait_seconds > 0: for i in range(wait_seconds): if self.shutdown_requested: break time.sleep(1)
def __init__(self, worker_subprocess, options, jobs, s3_bucket=None, mailer=None): assert options, "options is required." logger = utils.getLogger() self.options = options self.jobs = jobs self.s3_bucket = s3_bucket self.mailer = mailer self.worker = worker_subprocess self.shutdown_requested = False logger.debug('AutophoneTreeherder') self.url = self.options.treeherder_url if not self.url: logger.debug('AutophoneTreeherder: no treeherder url') return self.client_id = self.options.treeherder_client_id self.secret = self.options.treeherder_secret self.retry_wait = self.options.treeherder_retry_wait self.client = TreeherderClient(server_url=self.url, client_id=self.client_id, secret=self.secret) logger.debug('AutophoneTreeherder: %s', self)
def post_request(self, machine, project, job_collection, attempts, last_attempt): logger = utils.getLogger() logger.debug('AutophoneTreeherder.post_request: %s, attempt=%d, last=%s', job_collection.__dict__, attempts, last_attempt) try: self.client.post_collection(project, job_collection) return True except Exception, e: logger.exception('Error submitting request to Treeherder, attempt=%d, last=%s', attempts, last_attempt) if attempts > 1 and self.mailer: if hasattr(e, 'response') and e.response: response_json = json.dumps(e.response.json(), indent=2, sort_keys=True) else: response_json = None request_len = len(job_collection.to_json()) self.mailer.send( '%s attempt %d Error submitting request to Treeherder' % (utils.host(), attempts), 'Phone: %s\n' 'Exception: %s\n' 'Last attempt: %s\n' 'Request length: %d\n' 'Response: %s\n' % ( machine, e, last_attempt, request_len, response_json))
def test_completed(self, test_guid): logger = utils.getLogger() logger.debug('jobs.test_completed: %s', test_guid) conn = self._conn() self._execute_sql(conn, 'delete from tests where guid=?', values=(test_guid,)) self._commit_connection(conn) self._close_connection(conn)
def modify(request): if request.method == 'GET': id=request.GET.get('id') m=getMonitorById(id) if m: #sb=str(m.smsBegin) #se=str(m.smsEnd) return render_to_response('modify.html',{'monitor':m}) else: return HttpResponse('id is not exisit') else: m=modifyMonitor( oldName=request.POST.get('oldName'), log=getLogger(), name=request.POST.get('name'), interval=request.POST.get('interval'), mailInterval=request.POST.get('mailInterval'), smsInterval=request.POST.get('smsInterval'), smsBegin=request.POST.get('smsBegin'), smsEnd=request.POST.get('smsEnd'), retryTimes=request.POST.get('retryTimes'), retryInterval=request.POST.get('retryInterval'), describe=request.POST.get('describe'), url=request.POST.get('url'), timeout=request.POST.get('timeout'), contacts=request.POST.get('contacts'), statusCode=request.POST.get('statusCode'), status=False, ) if m: return HttpResponseRedirect('/index/') else: return HttpResponse('modify failed')
def job_completed(self, job_id): logger = utils.getLogger() logger.debug('jobs.job_completed: %s', job_id) conn = self._conn() self._execute_sql(conn, 'delete from tests where jobid=?', values=(job_id,)) self._execute_sql(conn, 'delete from jobs where id=?', values=(job_id,)) self._commit_connection(conn) self._close_connection(conn)
def handle(self): try: data = '' while True: data = self.connection.recv(4) if len(data) < 4: break pickle_len = struct.unpack('>L', data)[0] data = self.connection.recv(pickle_len) while len(data) < pickle_len: data = data + self.connection.recv(pickle_len - len(data)) record = logging.makeLogRecord(pickle.loads(data)) logger = utils.getLogger(record.name) logger.handle(record) except: logger = utils.getLogger() logger.exception("Error receiving log record: data: %s", data)
def delete_anr_traces(self, root=True): """Empty ANR traces.txt file.""" try: self.adb.rm(TRACES, root=root) self.adb.shell_output('echo > %s' % TRACES, root=root) self.adb.chmod(TRACES, mask='666', root=root) except ADBError, e: logger = utils.getLogger() logger.warning("Could not initialize ANR traces %s, %s", TRACES, e)
def __init__(self, cfgfile, subject_prefix=''): self.cfgfile = cfgfile self.subject_prefix = subject_prefix self.from_address = None self.mail_dest = None self.mail_username = None self.mail_password = None self.mail_server = None self.mail_port = None self.mail_ssl = None logger = utils.getLogger() cfg = ConfigParser.ConfigParser() if not cfg.read(self.cfgfile): logger.info('No email configuration file found. No emails will be sent.') return try: self.from_address = cfg.get('report', 'from') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): logger.error('No "from" option defined in "report" section ' 'of file "%s".\n', self.cfgfile) return try: self.mail_dest = [x.strip() for x in cfg.get('email', 'dest').split(',')] except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): logger.error('No "dest" option defined in "email" section ' 'of file "%s".\n', self.cfgfile) return try: self.mail_username = cfg.get('email', 'username') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): self.mail_username = None try: self.mail_password = cfg.get('email', 'password') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): self.mail_password = None try: self.mail_server = cfg.get('email', 'server') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): self.mail_server = 'smtp.mozilla.org' try: self.mail_port = cfg.getint('email', 'port') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): self.mail_port = 25 try: self.mail_ssl = cfg.getboolean('email', 'ssl') except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): self.mail_ssl = False
def handle_message(self, data, message): if self._stopping.is_set(): return message.ack() logger = utils.getLogger() if (self.treeherder_url and 'action' in data and 'project' in data and 'job_id' in data): self.handle_jobaction(data, message) elif 'status' in data: logger.debug('handle_message: data: %s, message: %s', data, message) self.handle_taskcompleted(data, message)
def start(self): """Runs the `listen` method on a new thread.""" logger = utils.getLogger() if self.listen_thread and self.listen_thread.is_alive(): logger.warning('AutophonePulseMonitor.start: listen thread already started') return logger.debug('AutophonePulseMonitor.start: listen thread starting') self.listen_thread = threading.Thread(target=self.listen, name='PulseMonitorThread') self.listen_thread.daemon = True self.listen_thread.start()
def rm(self, keys=[], keypattern=None, prefix=''): assert isinstance(keys, list) or isinstance(keypattern, str) if isinstance(keypattern, str): keys = self.ls(prefix=prefix, keypattern=keypattern) try: for key in keys: key.delete() except boto.exception.S3ResponseError, e: logger = utils.getLogger() logger.exception(str(e)) raise S3Error('%s' % e)
def serve_forever(self): import select while not self.shutdown_requested: try: rlist, wlist, xlist = select.select( [self.socket.fileno()], [], [], self.timeout) if rlist: self.handle_request() except: logger = utils.getLogger() logger.exception("Error selecting log record socket")
def new_treeherder_job(self, machine, project, job_collection): logger = utils.getLogger() logger.debug('jobs.new_treeherder_job: %s %s %s', machine, project, job_collection.__dict__) attempts = 0 now = datetime.datetime.utcnow().isoformat() conn = self._conn() job_cursor = self._execute_sql( conn, 'insert into treeherder values (?, ?, ?, ?, ?, ?)', values=(None, attempts, now, machine, project, job_collection.to_json())) job_cursor.close() self._commit_connection(conn) self._close_connection(conn)
def report_sql_error(self, attempt, email_sent, sql, values): logger = utils.getLogger() message = '%s %s' % (sql, values) logger.exception(message) if attempt > self.SQL_MAX_RETRIES and not email_sent: email_sent = True email_subject = '%s jobs SQL Error' % utils.host() email_body = ( 'Attempt %d to execute %s failed.\n' '%s' % (attempt, message, traceback.format_exc())) self.mailer.send(email_subject, email_body) logger.info('Sent mail notification about jobs database sql error.') time.sleep(self.SQL_RETRY_DELAY) return email_sent
def cancel_test(self, test_guid, device=None): logger = utils.getLogger() logger.debug('jobs.cancel_test: test %s device %s', test_guid, device) if not device: device = self.default_device conn = self._conn() # Get the jobid for this test. job_ids = [ test_row[0] for test_row in self._execute_sql(conn, 'select jobid from tests ' 'where guid=?', values=(test_guid,)) ] if not job_ids: logger.debug('jobs.cancel_test: test %s for device %s ' 'already deleted', test_guid, device) self._close_connection(conn) return job_id = job_ids[0] # Delete the test. self._execute_sql( conn, 'delete from tests where guid=?', values=(test_guid,)) # Get the number of remaining tests for this job and delete # the job if there are no more remaining tests for the job. test_cursor = self._execute_sql( conn, 'select count(id) from tests ' 'where jobid=?', values=(job_id,)) count = test_cursor.fetchone()[0] test_cursor.close() if count == 0: logger.debug('jobs.cancel_test: delete job_id %s device %s', job_id, device) self._execute_sql( conn, 'delete from jobs where id=?', values=(job_id,)) self._commit_connection(conn) self._close_connection(conn)
def hook_tk_errors(): """TKinter catches and swallows callback errors. we need to hook into that to log those seperately. """ import loadScreen import traceback main_logger = utils.getLogger() def tk_error(exc_type, exc_value, exc_tb): """Log TK errors.""" # The exception is caught inside the TK code. # We don't care about that, so try and move the traceback up # one level. import sys if exc_tb.tb_next: exc_tb = exc_tb.tb_next main_logger.error( 'TKinter callback exception occurred:\n{}', ''.join( traceback.format_exception( exc_type, exc_value, exc_tb, ) ), ) # Close loading screens if they're visible.. loadScreen.close_all() # Release the grab, if it exists. Otherwise you can't see the error dialog. TK_ROOT.grab_set_global() TK_ROOT.grab_release() messagebox.showerror( title='BEE2 Error:', message='{}: {!r}'.format(exc_type.__name__, exc_value), parent=TK_ROOT, ) # Since this isn't caught normally, it won't quit the application. # Quit ourselves manually. to prevent TK just freezing. TK_ROOT.quit() sys.exit() TK_ROOT.report_callback_exception = tk_error
def bucket(self): if self._bucket: return self._bucket logger = utils.getLogger() try: conn = boto.s3.connection.S3Connection(self.access_key_id, self.access_secret_key) if not conn.lookup(self.bucket_name): raise S3Error('bucket %s not found' % self.bucket_name) if not self._bucket: self._bucket = conn.get_bucket(self.bucket_name) return self._bucket except boto.exception.NoAuthHandlerFound: logger.exception('Error accessing bucket') raise S3Error('Authentication failed') except boto.exception.S3ResponseError, e: logger.exception('Error accessing bucket') raise S3Error('%s' % e)
def get_java_exception(self): """Returns a summary of the first fatal Java exception found in logcat output. Example: { 'reason': 'java-exception', 'signature': 'java.lang.NullPointerException at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833)' } """ logger = utils.getLogger() logre = re.compile(r".*\): \t?(.*)") exception = None logcat = self.adb.get_logcat() for i, line in enumerate(logcat): # Logs will be of form: # # 01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> REPORTING UNCAUGHT EXCEPTION FROM THREAD 9 ("GeckoBackgroundThread") # 01-30 20:15:41.937 E/GeckoAppShell( 1703): java.lang.NullPointerException # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833) # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at android.os.Handler.handleCallback(Handler.java:587) if "REPORTING UNCAUGHT EXCEPTION" in line or "FATAL EXCEPTION" in line: # Strip away the date, time, logcat tag and pid from the next two lines and # concatenate the remainder to form a concise summary of the exception. if len(logcat) >= i + 3: exception_type = '' exception_location = '' m = logre.search(logcat[i+1]) if m and m.group(1): exception_type = m.group(1) m = logre.search(logcat[i+2]) if m and m.group(1): exception_location = m.group(1) if exception_type: exception = {'reason': 'java-exception', 'signature': "%s %s" % ( exception_type, exception_location)} else: logger.warning("Automation Error: check_for_java_exceptions: Logcat is truncated!") break return exception
from config import args from sendgrid import SendGridAPIClient from sendgrid.helpers.mail import Mail from ArticleParser import ArticleParser from utils import getLogger logger = getLogger("Feed Mail") class FeedMail: """"Sends Recommended feed as email to the user Args: user_email(str): Email of the user to which feed us to be sent recommendations(dict): Recommendations of the corresponding user Attributes: from_email(str): Email of smartfeed.ai key(str): SendGrid API's key html_content(str): Consists the body of the email subject(str): Consists the subject of the email """ def __init__(self, user_email, recommendations): self.recommendations = recommendations self.to_email = user_email self.from_email = args["FROM_EMAIL"] self.key = args["SEND_GRID_KEY"] self.html_content = self._parseEmailContent(self.recommendations) self.subject = self._getEmailSubject()
def __init__(self, region, interval): self.region = region self.interval = interval self.logger = getLogger(region)
import conditions import instanceLocs import srctools import vbsp import vbsp_options import utils from conditions import ( make_result, RES_EXHAUSTED, GLOBAL_INSTANCES, ) from srctools import Vec, Entity, Property COND_MOD_NAME = 'Instance Generation' LOGGER = utils.getLogger(__name__, 'cond.addInstance') @make_result('addGlobal') def res_add_global_inst(res: Property): """Add one instance in a location. Options: allow_multiple: Allow multiple copies of this instance. If 0, the instance will not be added if it was already added. name: The targetname of the instance. IF blank, the instance will be given a name of the form 'inst_1234'. file: The filename for the instance. Angles: The orientation of the instance (defaults to '0 0 0'). Origin: The location of the instance (defaults to '0 0 -10000'). Fixup_style: The Fixup style for the instance. '0' (default) is
from tkinter import ttk from weakref import WeakSet from abc import abstractmethod import contextlib import multiprocessing from loadScreen_daemon import run_screen as _splash_daemon import utils from typing import Set # Keep a reference to all loading screens, so we can close them globally. _ALL_SCREENS = WeakSet() # type: Set[BaseLoadScreen] LOGGER = utils.getLogger(__name__) def close_all(): """Hide all loadscreen windows.""" for screen in _ALL_SCREENS: screen.reset() @contextlib.contextmanager def surpress_screens(): """A context manager to supress loadscreens while the body is active.""" active = [] for screen in _ALL_SCREENS: if not screen.active: continue
Dict, List, Tuple, NamedTuple, ) import comp_consts as consts import srctools import template_brush import utils import instanceLocs from srctools import (Property, Vec_tuple, Vec, Entity, Output, Solid, Side) COND_MOD_NAME = 'Main Conditions' LOGGER = utils.getLogger(__name__, alias='cond.core') # Stuff we get from VBSP in init() GLOBAL_INSTANCES = set() ALL_INST = set() VMF = None # type: srctools.VMF conditions = [] FLAG_LOOKUP = {} RESULT_LOOKUP = {} RESULT_SETUP = {} # Used to dump a list of the flags, results, meta-conditions ALL_FLAGS = [] ALL_RESULTS = [] ALL_META = []
# Wolfinch is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Wolfinch. If not, see <https://www.gnu.org/licenses/>. from keras.models import Sequential from keras.layers import Dense from keras.layers import LSTM from keras.layers import Dropout from sklearn.preprocessing import MinMaxScaler from utils import getLogger log = getLogger ('MODEL') log.setLevel(log.DEBUG) class Model (): EPOCH = 300 BATCH_SIZE = 64 def __init__(self, X_shape): # init preprocessor/scaler as the number of feature layers and o/p layer self.scX = [] for _ in range (X_shape[1]): self.scX += [MinMaxScaler(feature_range = (0, 1))] self.scY = MinMaxScaler(feature_range = (0, 1)) # Init Keras self.regressor = Sequential()
mpl.rcParams["savefig.bbox"] = "tight" mpl.rcParams["font.size"] = 10.0 mpl.rcParams["font.sans-serif"] = "Arial" mpl.rcParams["savefig.format"] = "pdf" import numpy as np import pandas as pd from joblib import Parallel, delayed import brewer2mpl #this from utils import getLogger #global settings #logger date = time.strftime(' %Y-%m-%d', time.localtime(time.time())) logger = getLogger(fn=os.getcwd() + "/" + date.strip() + "_" + os.path.basename(__file__) + ".log") #data CHROM_SIZE = "/home/caoy7/caoy7/Projects/0.Reference/2.mm10/1.fa/mm10.chrom.sizes" STAR_INDEX = "/home/caoy7/caoy7/Projects/0.Reference/2.mm10/3.index/3.star/1.50bp" def call_sys(cmds): """ Call systematic commands without return. """ for c in cmds: logger.info(c) try: os.system(c) except: logger.error(c)
parser.add_argument("--dataset", type=str) parser.add_argument("--action", type=str) parser.add_argument("--max_words", type=int, default=50000) parser.add_argument("--doc_count_threshold", type=int, default=1) parser.add_argument("--lower_case", type=bool, default=False) parser.add_argument("--include_unk", type=bool, default=False) parser.add_argument("--vocab_freqs_file", type=str, default=None) parser.add_argument("--summary_vocab_freqs_file", type=str, default=None) parser.add_argument("--merged_summary_vocab_freqs_file", type=str, default=None) parser.add_argument("--validation_rate", type=float, default=0) parser.add_argument("--shuffle_onval", type=bool, default=True) parser.add_argument("--no_need_start_tag", type=bool, default=True) FLAGS = parser.parse_args() logger = getLogger("DataGenerator") def _load_words(filename, wordCount, max_words): training_words = wordCount.transform([osp.join(FLAGS.data_dir, filename)], max_words=max_words, include_unk=FLAGS.include_unk) n_samples_training_words = len(training_words) min_seqlen_training_words = min(map(len, training_words)) max_seqlen_training_words = max(map(len, training_words)) logger.info( "total number of words: %s; min_seqlen in words: %s; max_seqlen in words: %s" % (n_samples_training_words, min_seqlen_training_words, max_seqlen_training_words)) return training_words
import conditions import instanceLocs import srctools import utils import vbsp import vbsp_options import comp_consts as const from conditions import ( make_result, RES_EXHAUSTED, ) from srctools import Property, Vec, Entity, Output COND_MOD_NAME = None LOGGER = utils.getLogger(__name__, alias='cond.resizeTrig') @make_result('ResizeableTrigger') def res_resizeable_trigger(res: Property): """Replace two markers with a trigger brush. This is run once to affect all of an item. Options: * `markerInst`: <ITEM_ID:1,2> value referencing the marker instances, or a filename. * `markerItem`: The item's ID * `previewConf`: A item config which enables/disables the preview overlay. * `previewinst`: An instance to place at the marker location in preview mode. This should contain checkmarks to display the value when testing. * `previewMat`: If set, the material to use for an overlay func_brush. The brush will be parented to the trigger, so it vanishes once killed.
from conditions import ( make_result, RES_EXHAUSTED, INST_ANGLE, ) from instanceLocs import resolve as resolve_inst from srctools import Vec, Property import conditions import utils import vbsp LOGGER = utils.getLogger(__name__, alias='cond.catwalks') CATWALK_TYPES = { utils.CONN_TYPES.straight: 'straight_128', utils.CONN_TYPES.corner: 'corner', utils.CONN_TYPES.all: 'crossjunction', utils.CONN_TYPES.side: 'end', utils.CONN_TYPES.triple: 'tjunction', utils.CONN_TYPES.none: 'NONE', } def place_catwalk_connections(instances, point_a, point_b): """Place catwalk sections to connect two straight points.""" diff = point_b - point_a # The horizontal unit vector in the direction we are placing catwalks direction = diff.copy() direction.z = 0 distance = direction.len() - 128
# # Wolfinch is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Wolfinch. If not, see <https://www.gnu.org/licenses/>. import random from . import eval_strategy from utils import getLogger # __name__ = "EA-OPS" log = getLogger(__name__) log.setLevel(log.CRITICAL) #TODO: FIXME: lot of hacky code here to fix deap ind generator issue with strat dict def selectOneMax(individual, res_dict): log.debug(" individual: %s" % (individual)) fitnessVal = eval_strategy.eval_strategy_with_config(individual) log.debug("fitnessVal: %d" % fitnessVal) res_dict["res"] = fitnessVal,
def main(): global LOG LOG = utils.getLogger("pack-tool") try: usage = "Usage: ./pack.py -t apk -m shared -a x86" opts_parser = OptionParser(usage=usage) opts_parser.add_option("-c", "--cfg", dest="pkgcfg", help="specify the path of config json file") opts_parser.add_option( "-t", "--type", dest="pkgtype", help="specify the pkg type, e.g. apk, cordova ...") opts_parser.add_option( "-m", "--mode", dest="pkgmode", help= "specify the apk mode, not for embeddingapi, e.g. shared, embedded" ) opts_parser.add_option( "-a", "--arch", dest="pkgarch", help="specify the apk arch, not for embeddingapi, e.g. x86, arm") opts_parser.add_option( "-d", "--dest", dest="destdir", help="specify the installation folder for packed package") opts_parser.add_option( "-s", "--src", dest="srcdir", help="specify the path of pkg resource for packing") opts_parser.add_option("--tools", dest="pkgpacktools", help="specify the parent folder of pack tools") opts_parser.add_option( "--notclean", dest="bnotclean", action="store_true", help="disable the build root clean after the packing") opts_parser.add_option("-v", "--version", dest="bversion", action="store_true", help="show this tool's version") opts_parser.add_option("-u", "--user", dest="user", help="specify the user in inst.py") opts_parser.add_option( "--sub-version", dest="subversion", help= "specify the embeddingapi, cordova sub version, e.g. v1, v2, v3 ..." ) opts_parser.add_option("--pkg-version", dest="pkgversion", help="specify the pkg version, e.g. 0.0.0.1") opts_parser.add_option( "--pack-type", dest="packtype", help="specify the pack type, e.g. gradle, maven") opts_parser.add_option("--notdebug", dest="bnotdebug", action="store_true", help="specify the packed pkg is not debug mode") opts_parser.add_option("--resource-only", dest="resourceonly", action="store_true", help="only restore resources to project root") if len(sys.argv) == 1: sys.argv.append("-h") global BUILD_PARAMETERS (BUILD_PARAMETERS, args) = opts_parser.parse_args() except Exception as e: LOG.error("Got wrong options: %s, exit ..." % e) sys.exit(1) if BUILD_PARAMETERS.bversion: print "Version: %s" % TOOL_VERSION sys.exit(0) if not BUILD_PARAMETERS.srcdir: BUILD_PARAMETERS.srcdir = os.getcwd() BUILD_PARAMETERS.srcdir = os.path.expanduser(BUILD_PARAMETERS.srcdir) if not BUILD_PARAMETERS.user: BUILD_PARAMETERS.user = "******" if not os.path.exists( os.path.join(BUILD_PARAMETERS.srcdir, "..", "..", VERSION_FILE)): if not os.path.exists( os.path.join(BUILD_PARAMETERS.srcdir, "..", VERSION_FILE)): if not os.path.exists( os.path.join(BUILD_PARAMETERS.srcdir, VERSION_FILE)): LOG.info( "Not found pkg version file, try to use option --pkg-version" ) pkg_version_file_path = None else: pkg_version_file_path = os.path.join(BUILD_PARAMETERS.srcdir, VERSION_FILE) else: pkg_version_file_path = os.path.join(BUILD_PARAMETERS.srcdir, "..", VERSION_FILE) else: pkg_version_file_path = os.path.join(BUILD_PARAMETERS.srcdir, "..", "..", VERSION_FILE) try: pkg_main_version = 0 pkg_release_version = 1 if BUILD_PARAMETERS.pkgversion: LOG.info("Using %s as pkg version " % BUILD_PARAMETERS.pkgversion) pkg_main_version = BUILD_PARAMETERS.pkgversion CROSSWALK_BRANCH = "master" else: if pkg_version_file_path is not None: LOG.info("Using pkg version file: %s" % pkg_version_file_path) with open(pkg_version_file_path, "rt") as pkg_version_file: pkg_version_raw = pkg_version_file.read() pkg_version_file.close() pkg_version_json = json.loads(pkg_version_raw) pkg_main_version = pkg_version_json["main-version"] pkg_release_version = pkg_version_json["release-version"] CROSSWALK_BRANCH = pkg_version_json["crosswalk-branch"] except Exception as e: LOG.error("Fail to read pkg version file: %s, exit ..." % e) sys.exit(1) CROSSWALK_VERSION = pkg_main_version if not BUILD_PARAMETERS.pkgtype: LOG.error("No pkg type provided, exit ...") sys.exit(1) elif not BUILD_PARAMETERS.pkgtype in PKG_TYPES: LOG.error("Wrong pkg type, only support: %s, exit ..." % PKG_TYPES) sys.exit(1) if BUILD_PARAMETERS.resourceonly: LOG.info("Starting copy resource only") resource_only.copy_resource(BUILD_PARAMETERS.pkgtype) sys.exit(0) if BUILD_PARAMETERS.pkgtype == "apk" or \ BUILD_PARAMETERS.pkgtype == "apk-aio": if not BUILD_PARAMETERS.pkgmode: LOG.error("No pkg mode option provided, exit ...") sys.exit(1) elif not BUILD_PARAMETERS.pkgmode in PKG_MODES: LOG.error( "Wrong pkg mode option provided, only support:%s, exit ..." % PKG_MODES) sys.exit(1) if not BUILD_PARAMETERS.pkgarch: LOG.error("No pkg arch option provided, exit ...") sys.exit(1) elif not BUILD_PARAMETERS.pkgarch in PKG_ARCHS: LOG.error( "Wrong pkg arch option provided, only support:%s, exit ..." % PKG_ARCHS) sys.exit(1) if BUILD_PARAMETERS.pkgtype == "apk-aio" or \ BUILD_PARAMETERS.pkgtype == "cordova-aio": if not BUILD_PARAMETERS.destdir or not os.path.exists( BUILD_PARAMETERS.destdir): LOG.error("No all-in-one installation dest dir found, exit ...") sys.exit(1) elif not BUILD_PARAMETERS.destdir: BUILD_PARAMETERS.destdir = BUILD_PARAMETERS.srcdir BUILD_PARAMETERS.destdir = os.path.expanduser(BUILD_PARAMETERS.destdir) if not BUILD_PARAMETERS.pkgpacktools: BUILD_PARAMETERS.pkgpacktools = os.path.join(BUILD_PARAMETERS.srcdir, "..", "..", "tools") BUILD_PARAMETERS.pkgpacktools = os.path.expanduser( BUILD_PARAMETERS.pkgpacktools) config_json = None if BUILD_PARAMETERS.pkgcfg: config_json_file_path = BUILD_PARAMETERS.pkgcfg else: config_json_file_path = os.path.join(BUILD_PARAMETERS.srcdir, "suite.json") try: LOG.info("Using config json file: %s" % config_json_file_path) with open(config_json_file_path, "rt") as config_json_file: config_raw = config_json_file.read() config_json_file.close() config_json = json.loads(config_raw) except Exception as e: LOG.error("Fail to read config json file: %s, exit ..." % e) sys.exit(1) global PKG_NAME PKG_NAME = utils.safelyGetValue(config_json, "pkg-name") if not PKG_NAME: PKG_NAME = os.path.basename(BUILD_PARAMETERS.srcdir) LOG.warning("Fail to read pkg name from json, " "using src dir name as pkg name ...") LOG.info("================= %s (%s-%s) ================" % (PKG_NAME, pkg_main_version, pkg_release_version)) if not utils.safelyGetValue(config_json, "pkg-list"): LOG.error("Fail to read pkg-list, exit ...") sys.exit(1) pkg_json = None global parameters_type parameters_type = None if BUILD_PARAMETERS.pkgtype == "cordova" or BUILD_PARAMETERS.pkgtype == "cordova-aio": if BUILD_PARAMETERS.pkgarch and not BUILD_PARAMETERS.pkgarch in PKG_ARCHS: LOG.error("Wrong pkg-arch, only support: %s, exit ..." % PKG_ARCHS) sys.exit(1) if BUILD_PARAMETERS.pkgmode and not BUILD_PARAMETERS.pkgmode in PKG_MODES: LOG.error("Wrong pkg-mode, only support: %s, exit ..." % PKG_MODES) sys.exit(1) if BUILD_PARAMETERS.packtype and not BUILD_PARAMETERS.packtype in CORDOVA_PACK_TYPES: LOG.error("cordova packtype can only be npm, local") sys.exit(1) if BUILD_PARAMETERS.pkgtype == "embeddingapi": if BUILD_PARAMETERS.packtype and not BUILD_PARAMETERS.packtype in PACK_TYPES: LOG.error("embeddingapi packtype can only be gradle, maven or ant") sys.exit(1) if BUILD_PARAMETERS.subversion: BUILD_PARAMETERS.pkgtype = BUILD_PARAMETERS.pkgtype + \ BUILD_PARAMETERS.subversion all_pkg_string = "".join(config_json["pkg-list"].keys()) if parameters_type and parameters_type in all_pkg_string: for i_pkg in config_json["pkg-list"].keys(): i_pkg_list = i_pkg.replace(" ", "").split(",") if parameters_type in i_pkg_list: pkg_json = config_json["pkg-list"][i_pkg] break elif BUILD_PARAMETERS.pkgtype in all_pkg_string: for i_pkg in config_json["pkg-list"].keys(): i_pkg_list = i_pkg.replace(" ", "").split(",") if BUILD_PARAMETERS.pkgtype in i_pkg_list: pkg_json = config_json["pkg-list"][i_pkg] break if pkg_json == config_json['pkg-list'].get( 'apk') and BUILD_PARAMETERS.subversion is not None: pkg_json = config_json["pkg-list"][BUILD_PARAMETERS.subversion] if not pkg_json: LOG.error("Fail to read pkg json, exit ...") sys.exit(1) if not prepareBuildRoot(): exitHandler(1) if "pkg-blacklist" in config_json: PKG_BLACK_LIST.extend(config_json["pkg-blacklist"]) try: varshop.setValue("BUILD_PARAMETERS", BUILD_PARAMETERS) varshop.setValue("BUILD_ROOT", BUILD_ROOT) varshop.setValue("BUILD_ROOT_SRC", BUILD_ROOT_SRC) varshop.setValue("BUILD_TIME", BUILD_TIME) varshop.setValue("CROSSWALK_BRANCH", CROSSWALK_BRANCH) varshop.setValue("CROSSWALK_VERSION", CROSSWALK_VERSION) varshop.setValue("DEFAULT_CMD_TIMEOUT", DEFAULT_CMD_TIMEOUT) varshop.setValue("PKG_MODES", PKG_MODES) varshop.setValue("PKG_ARCHS", PKG_ARCHS) except Exception as e: LOG.error("Fail to set global vars: %s, exit ..." % e) sys.exit(1) if not buildPKG(pkg_json): exitHandler(1) LOG.info("+Building package ...") if BUILD_PARAMETERS.pkgtype == "apk-aio" or \ BUILD_PARAMETERS.pkgtype == "cordova-aio": pkg_file_list = os.listdir(os.path.join(BUILD_ROOT, "pkg")) for i_file in pkg_file_list: if not utils.doCopy(os.path.join(BUILD_ROOT, "pkg", i_file), os.path.join(BUILD_PARAMETERS.destdir, i_file)): exitHandler(1) elif BUILD_PARAMETERS.pkgtype == "embeddingapi" and BUILD_PARAMETERS.subversion: pkg_file = os.path.join( BUILD_PARAMETERS.destdir, "%s-%s-%s-%s.%s.zip" % (PKG_NAME, pkg_main_version, pkg_release_version, BUILD_PARAMETERS.subversion, BUILD_PARAMETERS.pkgtype)) LOG.info("pkg_file: %s" % pkg_file) if not utils.zipDir(os.path.join(BUILD_ROOT, "pkg"), pkg_file): exitHandler(1) elif BUILD_PARAMETERS.pkgtype.startswith( "embeddingapi") and BUILD_PARAMETERS.packtype: pkg_file = os.path.join( BUILD_PARAMETERS.destdir, "%s-%s-%s.%s-%s.zip" % (PKG_NAME, pkg_main_version, pkg_release_version, BUILD_PARAMETERS.pkgtype, BUILD_PARAMETERS.packtype)) if not utils.zipDir(os.path.join(BUILD_ROOT, "pkg"), pkg_file): exitHandler(1) else: pkg_file = os.path.join( BUILD_PARAMETERS.destdir, "%s-%s-%s.%s.zip" % (PKG_NAME, pkg_main_version, pkg_release_version, BUILD_PARAMETERS.pkgtype)) if not utils.zipDir(os.path.join(BUILD_ROOT, "pkg"), pkg_file): exitHandler(1)
from decimal import getcontext import random # import logging from utils import getLogger, get_product_config, load_config, get_config import sims import exchanges from market import market_init, market_setup, get_market_list, \ feed_Q_process_msg, feed_deQ, get_market_by_product import db import stats import ui from ui import ui_conn_pipe # mpl_logger = logging.getLogger('matplotlib') # mpl_logger.setLevel(logging.WARNING) log = getLogger('Wolfinch') log.setLevel(log.INFO) gRestart = False # global Variables MAIN_TICK_DELAY = 0.500 # 500 milli def Wolfinch_init(): # seed random random.seed() # 1. Retrieve states back from Db # db.init_order_db(Order)
# Custom imports from src.warmup_scheduler import GradualWarmupScheduler from utils import ( AverageMeter, accuracy, aggregrate_video_accuracy, initialize_exp, getLogger, accuracy, save_checkpoint, load_model_parameters ) from datasets.AVideoDataset import AVideoDataset from model import load_model, Identity, get_video_dim logger = getLogger() # DICT with number of classes for each dataset NUM_CLASSES = { 'hmdb51': 51, 'ucf101': 101, } # Create Finetune Model class Finetune_Model(torch.nn.Module): def __init__( self, base_arch, num_ftrs=512,
import sys from decimal import getcontext import argparse import os import json from threading import Lock from flask import Flask, request, send_from_directory from utils import getLogger from . import db_events g_markets_list = None g_active_market = {} # {"CBPRO": "BTC-USD"} log = getLogger('UI') log.setLevel(log.INFO) static_file_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '../data/') MARKET_STATS = "stats_market_%s_%s.json" UI_CODES_FILE = "data/ui_codes.json" UI_TRADE_SECRET = None UI_PAGE_SECRET = None g_mp_lock = None def load_ui_codes(): global UI_TRADE_SECRET, UI_PAGE_SECRET
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY # OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # Authors: # Yang, Yunlong <*****@*****.**> import os import glob import subprocess import varshop import utils global LOG LOG = utils.getLogger("build_extension") def packExtension(build_json=None, app_src=None, app_dest=None, app_name=None): BUILD_PARAMETERS = varshop.getValue("BUILD_PARAMETERS") BUILD_ROOT = varshop.getValue("BUILD_ROOT") BUILD_ROOT_SRC = varshop.getValue("BUILD_ROOT_SRC") BUILD_TIME = varshop.getValue("BUILD_TIME") DEFAULT_CMD_TIMEOUT = varshop.getValue("DEFAULT_CMD_TIMEOUT") PKG_MODES = varshop.getValue("PKG_MODES") PKG_ARCHS = varshop.getValue("PKG_ARCHS") app_name = app_name.replace("-", "_") if not os.path.exists(os.path.join(BUILD_ROOT, "crosswalk")): if not utils.doCopy( os.path.join(BUILD_PARAMETERS.pkgpacktools, "crosswalk"),