def FullyQualifiedTestSuites(exe, option_test_suite):
"""Return a fully qualified list
Args:
exe: if True, use the executable-based test runner.
option_test_suite: the test_suite specified as an option.
"""
# Assume the test suites are in out/Release.
test_suite_dir = os.path.abspath(os.path.join(constants.CHROME_DIR,
'out', 'Release'))
if option_test_suite:
all_test_suites = [option_test_suite]
else:
all_test_suites = _TEST_SUITES
if exe:
qualified_test_suites = [os.path.join(test_suite_dir, t)
for t in all_test_suites]
else:
# out/Release/$SUITE_apk/$SUITE-debug.apk
qualified_test_suites = [os.path.join(test_suite_dir,
t + '_apk',
t + '-debug.apk')
for t in all_test_suites]
for t, q in zip(all_test_suites, qualified_test_suites):
if not os.path.exists(q):
logging.critical('Test suite %s not found in %s.\n'
'Supported test suites:\n %s\n'
'Ensure it has been built.\n',
t, q, _TEST_SUITES)
return []
return qualified_test_suites
def change_column_attrs(self, model, mutation, field_name, new_attrs):
"""Returns the SQL for changing one or more column attributes.
This will generate all the statements needed for changing a set
of attributes for a column.
The resulting AlterTableSQLResult contains all the SQL needed
to apply these attributes.
"""
field = model._meta.get_field(field_name)
attrs_sql_result = AlterTableSQLResult(self, model)
for attr_name, attr_info in new_attrs.iteritems():
method_name = 'change_column_attr_%s' % attr_name
evolve_func = getattr(self, method_name)
try:
sql_result = evolve_func(model, mutation, field,
attr_info['old_value'],
attr_info['new_value'])
assert not sql_result or isinstance(sql_result, SQLResult)
except Exception, e:
logging.critical(
'Error running database evolver function %s: %s',
method_name, e,
exc_info=1)
raise
attrs_sql_result.add(sql_result)
def run(self, addr):
for protocol in self.__protocols:
protodef = SMBEXEC.KNOWN_PROTOCOLS[protocol]
port = protodef[1]
logging.info("Trying protocol %s..." % protocol)
logging.info("Creating service %s..." % self.__serviceName)
stringbinding = protodef[0] % addr
rpctransport = transport.DCERPCTransportFactory(stringbinding)
rpctransport.set_dport(port)
if hasattr(rpctransport,'preferred_dialect'):
rpctransport.preferred_dialect(SMB_DIALECT)
if hasattr(rpctransport, 'set_credentials'):
# This method exists only for selected protocol sequences.
rpctransport.set_credentials(self.__username, self.__password, self.__domain, self.__lmhash, self.__nthash, self.__aesKey)
rpctransport.set_kerberos(self.__doKerberos)
self.shell = None
try:
if self.__mode == 'SERVER':
serverThread = SMBServer()
serverThread.start()
self.shell = RemoteShell(self.__logger, self.__share, rpctransport, self.__mode, self.__serviceName, self.__noOutput)
self.shell.onecmd(self.__command)
self.shell.finish()
if self.__mode == 'SERVER':
serverThread.stop()
except (Exception, KeyboardInterrupt), e:
logging.critical(str(e))
if self.shell is not None:
self.shell.finish()
def generate(self, view, errors, error_format):
"""Generate a dictionary that stores all errors along with their
positions and descriptions. Needed to show these errors on the screen.
Args:
view (sublime.View): current view
errors (list): list of unparsed errors in format @error_format
error_format (str): either FORMAT_LIBCLANG or FORMAT_BINARY
"""
log.debug(" generating error regions for view %s", view.id())
# first clear old regions
if view.id() in self.err_regions:
log.debug(" removing old error regions")
del self.err_regions[view.id()]
# create an empty region dict for view id
self.err_regions[view.id()] = {}
if error_format == FORMAT_LIBCLANG:
# expect a tu_diagnostics instance
self.errors_from_tu_diag(view, errors)
elif error_format == FORMAT_BINARY:
# expect a list of strings for each line of cmd output
self.errors_from_clang_output(view, errors)
else:
logging.critical(
" error_format:'%s' should match '%s' or '%s'",
error_format, FORMAT_LIBCLANG, FORMAT_BINARY)
log.debug(" %s error regions ready", len(self.err_regions))
def get(self):
if users.get_current_user() is not None:
auth_uri = flow.step1_get_authorize_url()
logging.critical(str(auth_uri))
self.redirect(str(auth_uri))
else:
self.redirect(users.create_login_url("/login"))
def addWorker(self, worker, idleTime = 60, parameters = None):
"""
Adds a worker object and sets it running. Worker thread will sleep for
idleTime seconds between runs. Parameters, if present, are passed into
the worker thread's setup, algorithm and terminate methods
"""
# Check type of worker
if not isinstance(worker, BaseWorkerThread):
msg = "Attempting to add worker that does not inherit from "
msg += "BaseWorkerThread"
logging.critical(msg)
return
# Prepare the new worker thread
self.prepareWorker(worker, idleTime)
workerThread = threading.Thread(target = worker, args = (parameters,))
msg = "Created worker thread %s" % str(worker)
logging.info(msg)
# Increase the active thread count - note this must be done before
# starting the thread so the callback can decrease back in case of
# startup failure
self.lock.acquire()
self.activeThreadCount += 1
workerThread.name = "threadmanager-slave%s" % worker.slaveid
self.slavelist.append(workerThread.name)
self.lock.release()
# Actually start the thread
workerThread.start()
def post(self):
if 'X-AppEngine-TaskName' not in self.request.headers:
logging.critical('Detected an attempted XSRF attack. The header '
'"X-AppEngine-Taskname" was not set.')
self.response.set_status(403)
return
# TODO In AppScale we do not check for this XSRF attacks.
# We need some additional # auth like we do for taskqueue with a secret hash.
#
#in_prod = (
# not self.request.environ.get("SERVER_SOFTWARE").startswith("Devel"))
#if in_prod and self.request.environ.get("REMOTE_ADDR") != "0.1.0.2":
# logging.critical('Detected an attempted XSRF attack. This request did '
# 'not originate from Task Queue.')
# self.response.set_status(403)
# return
headers = ["%s:%s" % (k, v) for k, v in self.request.headers.items()
if k.lower().startswith("x-appengine-")]
try:
run(self.request.body)
except PermanentTaskFailure, e:
logging.exception("Permanent failure attempting to execute task")
def loadAndValidate( self, XMLFolder ):
"""
Load the source XML file and remove the header from the tree.
Also, extracts some useful elements from the header element.
"""
if BibleOrgSysGlobals.verbosityLevel > 2: print( _("Loading from {}…").format( XMLFolder ) )
self.XMLFolder = XMLFolder
XMLFilepath = os.path.join( XMLFolder, GreekStrongsFileConverter.databaseFilename )
try: self.XMLTree = ElementTree().parse( XMLFilepath )
except FileNotFoundError:
logging.critical( t("GreekStrongsFileConverter could not find database at {}").format( XMLFilepath ) )
raise FileNotFoundError
except ParseError as err:
logging.critical( exp("Loader parse error in xml file {}: {} {}").format( GreekStrongsFileConverter.databaseFilename, sys.exc_info()[0], err ) )
raise ParseError
if BibleOrgSysGlobals.debugFlag: assert len( self.XMLTree ) # Fail here if we didn't load anything at all
if self.XMLTree.tag == GreekStrongsFileConverter.treeTag:
for segment in self.XMLTree:
#print( segment.tag )
if segment.tag == "prologue":
pass
elif segment.tag == "entries":
self.validateEntries( segment )
else: logging.error( "ks24 Unprocessed {!r} element ({}) in entry".format( segment.tag, segment.text ) )
else: logging.error( "Expected to load {!r} but got {!r}".format( GreekStrongsFileConverter.treeTag, self.XMLTree.tag ) )
if self.XMLTree.tail is not None and self.XMLTree.tail.strip(): logging.error( "vs42 Unexpected {!r} tail data after {} element".format( self.XMLTree.tail, self.XMLTree.tag ) )
def checkCA():
#Check CA exists
keyFile = os.path.join(os.path.dirname(__file__), 'CA.key')
crtFile = os.path.join(os.path.dirname(__file__), 'CA.crt')
if not os.path.exists(keyFile):
if not OpenSSL:
logging.critical('CA.crt is not exist and OpenSSL is disabled, ABORT!')
sys.exit(-1)
key, crt = CertUtil.makeCA()
CertUtil.writeFile(keyFile, key)
CertUtil.writeFile(crtFile, crt)
[os.remove(os.path.join('certs', x)) for x in os.listdir('certs')]
#Check CA imported
cmd = {
'win32' : r'cd /d "%s" && certmgr.exe -add CA.crt -c -s -r localMachine Root >NUL' % os.path.dirname(__file__),
#'darwin' : r'sudo security add-trusted-cert -d �Cr trustRoot �Ck /Library/Keychains/System.keychain CA.crt',
}.get(sys.platform)
if cmd and os.system(cmd) != 0:
logging.warning('GoAgent install trusted root CA certificate failed, Please run goagent by administrator/root.')
if OpenSSL:
keyFile = os.path.join(os.path.dirname(__file__), 'CA.key')
crtFile = os.path.join(os.path.dirname(__file__), 'CA.crt')
cakey = CertUtil.readFile(keyFile)
cacrt = CertUtil.readFile(crtFile)
CertUtil.CA = (CertUtil.loadPEM(cakey, 0), CertUtil.loadPEM(cacrt, 2))
def main():
options = _parse_opts()
loglevel = logging.INFO
if options.verbose:
loglevel = logging.DEBUG
logging.basicConfig(
format='%(asctime)s [%(levelname)s] %(message)s',
datefmt='%d/%m/%Y %I:%M:%S %p',
level=loglevel
)
if options.outfmt == 'stix':
dump_report_to_stix(options)
elif options.outfmt == 'maec':
dump_report_to_maec(options)
elif options.outfmt == 'stix-ol':
dump_report_to_stix_ol(options)
elif options.outfmt == 'stix-il':
dump_report_to_stix_il(options)
else:
logging.critical('unhandled output format %s' % options.outfmt)
sys.exit(1)
sys.exit(0)
def upload_fastqc_counts(self, alignment_id):
if not alignment_id:
logging.critical("Could not upload fastqc_counts without an alignment id given")
return
self.get_alignment_counts(alignment_id)
total = 0
filtered = 0
for fastqc_file, fastqc_counts in self.fastqc_counts.items():
if not fastqc_counts:
log.error("Could not get counts from %s for uploading" % fastqc_file)
return
total += fastqc_counts["total"]
filtered += fastqc_counts["filtered"]
# FastQC's definition of total differs from ours
counts = {
"total": total + filtered,
"qc": filtered,
"pf": total
}
for count_name, count in counts.items():
self.upload_count(alignment_id, count_name, count)
def run(self):
# Here we write a mini config for the server
smbConfig = ConfigParser.ConfigParser()
smbConfig.add_section('global')
smbConfig.set('global','server_name','server_name')
smbConfig.set('global','server_os','UNIX')
smbConfig.set('global','server_domain','WORKGROUP')
smbConfig.set('global','log_file',SMBSERVER_DIR + '/smb.log')
smbConfig.set('global','credentials_file','')
# Let's add a dummy share
smbConfig.add_section(DUMMY_SHARE)
smbConfig.set(DUMMY_SHARE,'comment','')
smbConfig.set(DUMMY_SHARE,'read only','no')
smbConfig.set(DUMMY_SHARE,'share type','0')
smbConfig.set(DUMMY_SHARE,'path',SMBSERVER_DIR)
# IPC always needed
smbConfig.add_section('IPC$')
smbConfig.set('IPC$','comment','')
smbConfig.set('IPC$','read only','yes')
smbConfig.set('IPC$','share type','3')
smbConfig.set('IPC$','path')
self.smb = smbserver.SMBSERVER(('0.0.0.0',445), config_parser = smbConfig)
logging.info('Creating tmp directory')
try:
os.mkdir(SMBSERVER_DIR)
except Exception, e:
logging.critical(str(e))
pass
def __init__(self, cfg, file_version=None):
veris_logger.updateLogger(cfg)
logging.debug("Initializing CSVtoJSON object.")
if file_version is None:
file_version = cfg.get("file_version", None)
if file_version is None:
file_version = self.get_file_schema_version(cfg['input'])
if file_version is None:
logging.warning("Could not determine veris version of {0}. Please specify it as an argument to the class initialization, 'CSVtoJSON(cfg, file_version=<file version>)'".format(cfg['input']))
elif file_version != self.script_version:
logging.warning("File veris version {0} does not match script veris version {1}.".format(file_version, self.script_version))
cfg['file_version'] = file_version
if type(cfg["schemafile"]) == dict:
self.jschema = cfg["schemafile"]
else:
try:
self.jschema = self.openJSON(cfg["schemafile"])
except IOError:
logging.critical("ERROR: Schema file not found.")
raise
# exit(1)
self.sfields = self.parseSchema(self.jschema)
self.cfg = cfg
def __init__(self, parent=None):
"""Init class."""
super(Downloader, self).__init__(parent)
self.setWindowTitle(__doc__)
if not os.path.isfile(__file__) or not __source__:
return
if not os.access(__file__, os.W_OK):
error_msg = ("Destination file permission denied (not Writable)! "
"Try again to Update but as root or administrator.")
log.critical(error_msg)
QMessageBox.warning(self, __doc__.title(), error_msg)
return
self._time, self._date = time.time(), datetime.now().isoformat()[:-7]
self._url, self._dst = __source__, __file__
log.debug("Downloading from {} to {}.".format(self._url, self._dst))
if not self._url.lower().startswith("https:"):
log.warning("Unsecure Download over plain text without SSL.")
self.template = """<h3>Downloading</h3><hr><table>
<tr><td><b>From:</b></td> <td>{}</td>
<tr><td><b>To: </b></td> <td>{}</td> <tr>
<tr><td><b>Started:</b></td> <td>{}</td>
<tr><td><b>Actual:</b></td> <td>{}</td> <tr>
<tr><td><b>Elapsed:</b></td> <td>{}</td>
<tr><td><b>Remaining:</b></td> <td>{}</td> <tr>
<tr><td><b>Received:</b></td> <td>{} MegaBytes</td>
<tr><td><b>Total:</b></td> <td>{} MegaBytes</td> <tr>
<tr><td><b>Speed:</b></td> <td>{}</td>
<tr><td><b>Percent:</b></td> <td>{}%</td></table><hr>"""
self.manager = QNetworkAccessManager(self)
self.manager.finished.connect(self.save_downloaded_data)
self.manager.sslErrors.connect(self.download_failed)
self.progreso = self.manager.get(QNetworkRequest(QUrl(self._url)))
self.progreso.downloadProgress.connect(self.update_download_progress)
self.show()
self.exec_()
def __init__(self, h5parmFile, readonly = True, complevel = 5, complib='zlib'):
"""
Keyword arguments:
h5parmFile -- H5parm filename
readonly -- if True the table is open in readonly mode (default=True)
complevel -- compression level from 0 to 9 (default=5) when creating the file
complib -- library for compression: lzo, zlib, bzip2 (default=zlib)
"""
if os.path.isfile(h5parmFile):
if tables.is_pytables_file(h5parmFile) == None:
logging.critical('Wrong HDF5 format for '+h5parmFile+'.')
raise Exception('Wrong HDF5 format for '+h5parmFile+'.')
if readonly:
logging.debug('Reading from '+h5parmFile+'.')
self.H = tables.openFile(h5parmFile, 'r')
else:
logging.debug('Appending to '+h5parmFile+'.')
self.H = tables.openFile(h5parmFile, 'r+')
else:
if readonly:
raise Exception('Missing file '+h5parmFile+'.')
else:
logging.debug('Creating '+h5parmFile+'.')
# add a compression filter
f = tables.Filters(complevel=complevel, complib=complib)
self.H = tables.openFile(h5parmFile, filters=f, mode='w')
self.fileName = h5parmFile
def post(self):
if 'X-AppEngine-TaskName' not in self.request.headers:
logging.critical('Detected an attempted XSRF attack. The header '
'"X-AppEngine-Taskname" was not set.')
self.response.set_status(403)
return
in_prod = (
not self.request.environ.get("SERVER_SOFTWARE").startswith("Devel"))
if in_prod and self.request.environ.get("REMOTE_ADDR") != "0.1.0.2":
logging.critical('Detected an attempted XSRF attack. This request did '
'not originate from Task Queue.')
self.response.set_status(403)
return
headers = ["%s:%s" % (k, v) for k, v in self.request.headers.items()
if k.lower().startswith("x-appengine-")]
logging.info(", ".join(headers))
try:
run(self.request.body)
except PermanentTaskFailure, e:
logging.exception("Permanent failure attempting to execute task")
def TestLog():
logging.debug('debug message')
logging.info('info message')
logging.warning('warning message')
logging.error('error message')
logging.critical('critical message')
return
def check_scheduled_actions(self):
now = datetime.datetime.now()
# Re-schedule random tasks at midnight
if not hasattr(self, "last_random_schedule"):
self.last_random_schedule = self.bot.load("last_random_schedule")
if self.last_random_schedule is None or self.last_random_schedule.day != now.day:
self.bot.save("last_random_schedule", now)
self.last_random_schedule = now
self._clear_random_tasks()
for plugin_info, fn, function_name in self.bot.random_tasks:
meta = fn.will_fn_metadata
self.add_random_tasks(
plugin_info["full_module_name"],
plugin_info["name"],
function_name,
meta.start_hour,
meta.end_hour,
meta.day_of_week,
meta.num_times_per_day
)
try:
if not self.bot.load("scheduler_add_lock", False) or not self.bot.load("scheduler_lock", False):
self.bot.save("scheduler_lock", True)
self._run_applicable_actions_in_list(now,)
self._run_applicable_actions_in_list(now, periodic_list=True)
self.bot.save("scheduler_lock", False)
except:
logging.critical("Scheduler run blew up.\n\n%s\nContinuing...\n" % (traceback.format_exc(), ))
def save_meeting(self, meeting):
"""
Write meeting object to database. This means dereferencing all associated objects as DBrefs
"""
meeting_stored = self.get_object('meeting', 'originalId', meeting.originalId)
meeting_dict = meeting.dict()
# setting body
meeting_dict['body'] = DBRef(collection='body',id=self.body_uid)
# ensure that there is an originalId
if 'originalId' not in meeting_dict:
logging.critical("Fatal error: no originalId avaiable at url %s", meeting_dict.originalUrl)
# dereference items
meeting_dict = self.dereference_object(meeting_dict, 'organization')
meeting_dict = self.dereference_object(meeting_dict, 'agendaItem')
meeting_dict = self.dereference_object(meeting_dict, 'invitation', 'file')
meeting_dict = self.dereference_object(meeting_dict, 'resultsProtocol', 'file')
meeting_dict = self.dereference_object(meeting_dict, 'verbatimProtocol', 'file')
meeting_dict = self.dereference_object(meeting_dict, 'auxiliaryFile', 'file')
# create slug
# meeting_dict['slug'] = self.slugify(meeting_dict['identifier'])
# save data
return self.save_object(meeting_dict, meeting_stored, 'meeting')
def from_index(f, index, root=False):
"""Given an index file of the kind built by indexbuilder.py, and an
offset into that file, return the node that begins at that index.
"""
f.seek(index)
logging.debug("Sought to index %d" % (index))
size = struct.unpack('i', f.read(POINTER_SIZE))[0]
if size > 100:
logging.warning("That sure is a big node...")
remaining_bytes = size - POINTER_SIZE
if not root:
char = f.read(1)
terminal = f.read(1) == 't'
remaining_bytes -= 2
else:
char = None
# Wouldn't it be cool if the empty string was a word in the dictionary?
# I wonder what it would mean.
terminal = False
children = set([]) # Set of indices of child nodes
while remaining_bytes > 0:
try:
child_pointer = struct.unpack('i', f.read(POINTER_SIZE))
except struct.error:
logging.critical("Pointers got messed up. Have %d remaining bytes on char %s at index %d" \
% (remaining_bytes, char, index))
raise
remaining_bytes -= POINTER_SIZE
children.add(child_pointer[0])
assert remaining_bytes == 0, "Something went wrong. Didn't get a clean read."
return PrefixNode(char, f, terminal, children)
def _run_applicable_actions_in_list(self, now, periodic_list=False):
times_list = self.bot.get_times_list(periodic_list=periodic_list)
# Iterate through times_list first, before loading the full schedule_list into memory (big pickled stuff, etc)
a_task_needs_run = False
for task_hash, task_time in times_list.items():
if task_time < now:
a_task_needs_run = True
break
if a_task_needs_run:
sched_list = self.bot.get_schedule_list(periodic_list=periodic_list)
for item_hash, item in sched_list.items():
running_task = False
try:
if item["when"] < now:
running_task = True
self.run_action(item)
except:
logging.critical(
"Error running task %s. \n\n%s\n"
"Trying to delete it and recover...\n" % (item, traceback.format_exc())
)
if running_task:
try:
self.bot.remove_from_schedule(item["hash"], periodic_list=periodic_list)
except:
logging.critical(
"Unable to remove task. Leaving it in, you'll have to clean it out by hand."
"Sorry! \n\n%s\nContinuing...\n" % (traceback.format_exc(),))
def GetLogTimestamp(log_line):
"""Returns the timestamp of the given |log_line|."""
try:
return datetime.datetime.strptime(log_line[:18], '%m-%d %H:%M:%S.%f')
except (ValueError, IndexError):
logging.critical('Error reading timestamp from ' + log_line)
return None
def post(self):
if self.user:
response_info = {'visibility': self.user.display_type}
self.response.out.write(json.dumps(response_info))
else:
logging.critical('Unauthorized visibility check')
return
def distributions(self, records=None):
logging.info("Analysing %s ...", self.database)
try:
if records is not None and isinstance(records, pd.DataFrame):
max_mut = np.max(records['MUT'])
self.n_samples = records.shape[0]
else:
# load from file
max_mut, self.n_samples = io.get_max_mut(self.database)
lin = np.linspace(0, max_mut, min(self.n_samples / 15., 12))
sets = [(0, 0)] + zip(lin[:-1], lin[1:])
if len(sets) == 1:
# no correction needs to be applied
return None
out_muts = [self.intra_donor_distance(
records, i, j) for i, j in zip(sets, sets)]
except StandardError as msg:
logging.critical(msg)
out_muts = []
my_dict = dict()
for f, m in out_muts:
my_dict.setdefault(m, []).append(f)
return my_dict
def mutation_histogram(self, records, mut, filename):
"""Records is a pd.Dataframe."""
if os.path.exists(filename + '.npz'):
logging.critical(filename + '.npz esists.')
return filename
if records.shape[0] < self.min_seqs:
return ''
igs = [IgRecord(x.to_dict()) for _, x in records.iterrows()]
igsimilarity_learn = copy.deepcopy(self.igsimilarity)
igsimilarity_learn.correct = self.correction
igsimilarity_learn.rm_duplicates = True
if not self.correction:
igsimilarity_learn.tol = 1000
else:
igsimilarity_learn.correct_by = self.correction
sim_func = igsimilarity_learn.pairwise
logging.info("Computing %s", filename)
dnearest = parallel_distance.dnearest_intra_padding(
igs, sim_func, filt=lambda x: x > 0, func=max)
if not os.path.exists(filename.split('/')[0]):
os.makedirs(filename.split('/')[0])
np.savez(filename, X=dnearest, mut=mut)
# Plot distance distribution
title = "Similarities for {:.3f}-{:.3f}%" \
.format(np.min(records['MUT']), np.max(records['MUT']))
plot_hist(dnearest, self.bins, title, filename)
return filename
def pc_heartbeat(self):
for plc in self.plcs:
try:
plc.blink_pc_heartbeat()
except snap7.snap7exceptions.Snap7Exception:
logging.critical("Connection to PLC: {plc} lost. Trying to re-establish connection.".format(plc=plc))
plc.connect()
def sync_plcs_time_if_needed(self):
for plc in self.plcs:
try:
plc.sync_time_if_needed()
except snap7.snap7exceptions.Snap7Exception:
logging.critical("Connection to PLC: {plc} lost. Trying to re-establish connection.".format(plc=plc))
plc.connect()
def sendEmail(config, subject, body, to=""):
if config.getboolean('email', 'nomail'):
logging.info("Not sending email with subject '" + subject + '" but pretending we did.\n' + body)
return True
FROM = config.get('email', 'user')
PASS = config.get('email', 'pass')
if not to:
to = config.get('general', 'alertcontact')
# Prepare actual message
# Avoid gmail threading
subject = "[" + config.get('general', 'servername') + "] " + subject + " "
if config.getboolean('email', 'bustgmailthreading'):
subject += str(random.random())
message = """\From: %s\nTo: %s\nSubject: %s\n\n%s""" \
% (FROM, ", ".join(to), subject, body)
try:
server = smtplib.SMTP(config.get('email', 'smtpserver'), config.get('email', 'smtpport'))
server.ehlo()
server.starttls()
server.login(FROM, PASS)
server.sendmail(FROM, to, message)
server.close()
return True
except Exception as e:
logging.critical("Caught an exception trying to send an email:" + str(e))
return False
def read_config_file(cfg_file, sites, env):
""" Open and parse the config file, save the words in a list. """
# Open config file
logging.info("Reading configuration file: " + cfg_file)
config = ConfigParser.SafeConfigParser()
config.read(cfg_file)
sections = config.sections()
for section in sections:
site = Site()
if section == "General":
# Save name of directory to download files to
env.download_dir = config.get(section, "download_dir")
else:
# Save url to check
site.login_url = config.get(section, "login_url")
# Save url to check
site.feed_url = config.get(section, "rss_url")
# Save time interval (in seconds) for checking feeds
site.time_interval = config.getfloat(section, "interval") * 60.0
# Save list of words to look for
keys_str = str(config.get(section, "keys"))
site.keys = keys_str.split()
# Save username to site
site.username = config.get(section, "username")
# Save password to site
site.password = config.get(section, "password")
# Add to array of sites
sites.append(site)
# safety check
if len(sites) < 1:
logging.critical("Can't read config file")
exit(-1)
def StartMonitoringLogcat(self, clear=True, timeout=10, logfile=None,
filters=[]):
"""Starts monitoring the output of logcat, for use with WaitForLogMatch.
Args:
clear: If True the existing logcat output will be cleared, to avoiding
matching historical output lurking in the log.
timeout: How long WaitForLogMatch will wait for the given match
filters: A list of logcat filters to be used.
"""
if clear:
self.RunShellCommand('logcat -c')
args = ['logcat', '-v', 'threadtime']
if filters:
args.extend(filters)
else:
args.append('*:v')
# Spawn logcat and syncronize with it.
for _ in range(4):
self._logcat = pexpect.spawn('adb', args, timeout=timeout,
logfile=logfile)
self.RunShellCommand('log startup_sync')
if self._logcat.expect(['startup_sync', pexpect.EOF,
pexpect.TIMEOUT]) == 0:
break
self._logcat.close(force=True)
else:
logging.critical('Error reading from logcat: ' + str(self._logcat.match))
sys.exit(1)
from twython import Twython, TwythonError
import time
import logging
logging.debug('Debugging Information')
logging.info('Information')
logging.warning('Warnung:Datei %s nicht gefunden', ' server.conf')
logging.error('Fehler')
logging.critical('Kritischer Fehler!')
from auth import (consumer_key, consumer_secret, access_token,
access_token_secret)
twitter = Twython(consumer_key, consumer_secret, access_token,
access_token_secret)
def uhrzeit():
try:
#seconds = time.time()
#print ("Sekunden vergangen seit: ", seconds)
named_tuple = time.localtime()
time_string = time.strftime("%H:%M:%S -- %H.%m.%Y", named_tuple)
print time_string
except TwythonError as e:
print e
def timebot():
try:
print "BierbotTime Start"
PATH = os.path.dirname(os.path.abspath(__file__))
def get_evak_time(inifile, trajfile):
fps, N, traj = parse_file(trajfile)
etime = (max( traj[:, 1] ) - min(traj[:, 1])) / float(fps)
return etime
def call_test(num, suffix):
logging.info("copy master_ini_%s.xml"%suffix)
logging.info("copy geometry/geometry_test11_%s.xml"%suffix,)
copyfile("%s/master_ini_%s.xml"%(PATH, suffix), "%s/master_ini.xml"%PATH)
copyfile("%s/geometry/geometry_test11_%s.xml"%(PATH, suffix), "%s/geometry.xml"%PATH)
test = JPSRunTestDriver(num, argv0=sys.argv[0], testdir=sys.path[0], utestdir=utestdir, jpscore=sys.argv[1])
return test.run_test(testfunction=get_evak_time)
if __name__ == "__main__":
threashold = 0.001
result_a = call_test(111, "a")
result_b = call_test(112, "b")
diff = [a - b for a, b in zip(result_a, result_b)]
success = all(v <= threashold for v in diff)
if success:
logging.info("%s exits with SUCCESS" % (sys.argv[0]))
sys.exit(SUCCESS)
else:
logging.critical("%s exits with FAILURE" % (sys.argv[0]))
logging.debug("result_a {}".format(", ".join(map(str, result_a))))
logging.debug("result_b {}".format(", ".join(map(str, result_a))))
logging.debug("diff {} (threashold = {})".format(", ".join(map(str, diff)), threashold))
sys.exit(FAILURE)
def fetchTemp(self):
""" Retrieves temperature data from ESATAN output file. """
timestep = 0
self.data = {}
self.extrema = {}
self.components = []
self.time = []
lowerLim, upperLim = self.thresholds
logging.info(
"Ignoring values smaller than {} and higher than {}".format(
lowerLim, upperLim))
# This check might not be necessary
if not os.path.isfile(self.filePath):
logging.critical("Output file could not be found")
return
# Open ESATAN file
logging.info('Reading temperature data from ESATAN logfile {}'.format(
self.filePath))
with open(self.filePath) as logFile:
lines = logFile.readlines()
# Scan all lines
for i, line in enumerate(lines):
# Search for timestamp
if 'TIMEN' in line:
time = float(line.split()[2])
self.time.append(time)
# Search for temperature data paragraph
if '+MOVE' in line:
# This if is necessary because in some files the data following the +MOVE flag is not temperature data
if lines[i + 3].split()[2] == 'T':
for l in lines[i + 6:]: #skip subheader
# Break at end of paragraph
if not l.strip():
break
else:
lWords = l.split()
comp = lWords[1]
try:
temp = float(lWords[2])
except Exception:
logging.error(
"Temperature value seems to be faulty: {}".
format(lWords[2]))
break
## If this temperature doesn't pass a filter, set it to NaN
#if upperLim != None and temp > upperLim:
# logging.info("Filtered too high value {}".format(temp))
# break
# #temp = np.NaN
#if lowerLim != None and temp < lowerLim:
# logging.info("Filtered too low value {}".format(temp))
# break
# #temp = np.NaN
#if temp in self.ignoreValues:
# logging.info("Filtered ignored value")
# break
# #temp = np.NaN
# Create new keys if necessary
if comp not in self.data.keys():
self.components.append(comp)
self.data[comp] = {}
if comp not in self.extrema.keys():
self.extrema[comp] = {}
self.extrema[comp]['glob_max'] = (0.0, -9999)
self.extrema[comp]['glob_min'] = (0.0, 9999)
if time not in self.data[comp].keys():
self.data[comp][time] = {}
self.data[comp][time]['Tmax'] = -9999
self.data[comp][time]['Tmin'] = 9999
# See if this temperature is a global extremum
if temp > self.extrema[comp]['glob_max'][1]:
self.extrema[comp]['glob_max'] = (time, temp)
elif temp < self.extrema[comp]['glob_min'][1]:
self.extrema[comp]['glob_min'] = (time, temp)
# Determine extrema at current time
if temp > self.data[comp][time]['Tmax']:
self.data[comp][time]['Tmax'] = temp
elif temp < self.data[comp][time]['Tmin']:
self.data[comp][time]['Tmin'] = temp
# Check if for all times in array:times there are data points.
# If filters are activated, that is sometimes not the case.
# If there are no data points for a certain time, delete it
print "Cleansing time array"
print "Time array:\n", self.time
for comp in self.data.keys():
print "Data keys:\n", self.data[comp].keys()
for i, time in enumerate(self.time):
if time not in self.data[comp].keys():
print "Removing time", time
self.time.remove(time)
print self.time
import os
logging.info('Importing numpy')
import numpy as np
# Try to find UI file in temp folder created by exe. Works if UI file was included in the exe by tweaking the pyinstaller spec file
if hasattr(sys, '_MEIPASS'):
ui_path = os.path.join(sys._MEIPASS, "evaltan.ui")
elif hasattr(sys, '_MEIPASS2'):
ui_path = os.path.join(sys._MEIPASS2, "evatan.ui")
else:
ui_path = "evatan.ui"
try:
Ui_MainWindow, QMainWindow = loadUiType(ui_path)
except Exception:
logging.critical(
'GUI file could not be loaded from path {}!'.format(ui_path))
exit()
class ApplicationWindow(QMainWindow, Ui_MainWindow):
def __init__(self, ):
logging.info('Initiating main window')
super(ApplicationWindow, self).__init__()
self.fileLoaded = False
# Create application window
logging.info("Setting up UI")
self.setupUi(self)
self.setWindowTitle('ESATAN Data Evaluation')
def get_native_optimizer():
if os.environ.get('EMCC_FAST_COMPILER') == '0':
logging.critical(
'Non-fastcomp compiler is no longer available, please use fastcomp or an older version of emscripten'
)
sys.exit(1)
# Allow users to override the location of the optimizer executable by setting an environment variable EMSCRIPTEN_NATIVE_OPTIMIZER=/path/to/optimizer(.exe)
if os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER') and len(
os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER')) > 0:
logging.debug('env forcing native optimizer at ' +
os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER'))
return os.environ.get('EMSCRIPTEN_NATIVE_OPTIMIZER')
# Also, allow specifying the location of the optimizer in .emscripten configuration file under EMSCRIPTEN_NATIVE_OPTIMIZER='/path/to/optimizer'
if hasattr(shared, 'EMSCRIPTEN_NATIVE_OPTIMIZER') and len(
shared.EMSCRIPTEN_NATIVE_OPTIMIZER) > 0:
logging.debug('config forcing native optimizer at ' +
shared.EMSCRIPTEN_NATIVE_OPTIMIZER)
return shared.EMSCRIPTEN_NATIVE_OPTIMIZER
FAIL_MARKER = shared.Cache.get_path('optimizer.building_failed')
if os.path.exists(FAIL_MARKER):
shared.logging.debug(
'seeing that optimizer could not be built (run emcc --clear-cache or erase "optimizer.building_failed" in cache dir to retry)'
)
return None
def get_optimizer(name, args, handle_build_errors=None):
class NativeOptimizerCreationException(Exception):
pass
outs = []
errs = []
try:
def create_optimizer_cmake():
shared.logging.debug('building native optimizer via CMake: ' +
name)
output = shared.Cache.get_path(name)
shared.try_delete(output)
if NATIVE_OPTIMIZER == '1':
cmake_build_type = 'RelWithDebInfo'
elif NATIVE_OPTIMIZER == '2':
cmake_build_type = 'Release'
elif NATIVE_OPTIMIZER == 'g':
cmake_build_type = 'Debug'
build_path = shared.Cache.get_path('optimizer_build_' +
cmake_build_type)
shared.try_delete(os.path.join(build_path, 'CMakeCache.txt'))
log_output = None if DEBUG else subprocess.PIPE
if not os.path.exists(build_path):
os.mkdir(build_path)
if WINDOWS:
# Poor man's check for whether or not we should attempt 64 bit build
if os.environ.get('ProgramFiles(x86)'):
cmake_generators = [
'Visual Studio 12 Win64', 'Visual Studio 12',
'Visual Studio 11 Win64', 'Visual Studio 11',
'MinGW Makefiles', 'Unix Makefiles'
]
else:
cmake_generators = [
'Visual Studio 12', 'Visual Studio 11',
'MinGW Makefiles', 'Unix Makefiles'
]
else:
cmake_generators = ['Unix Makefiles']
for cmake_generator in cmake_generators:
proc = subprocess.Popen([
'cmake', '-G', cmake_generator,
'-DCMAKE_BUILD_TYPE=' + cmake_build_type,
shared.path_from_root('tools', 'optimizer')
],
cwd=build_path,
stdin=log_output,
stdout=log_output,
stderr=log_output)
proc.communicate()
make_env = os.environ.copy()
if proc.returncode == 0:
if 'Visual Studio' in cmake_generator:
ret = find_msbuild(
os.path.join(build_path,
'asmjs_optimizer.sln'), make_env)
make = [
ret[0], '/t:Build',
'/p:Configuration=' + cmake_build_type,
'/nologo', '/verbosity:minimal',
'asmjs_optimizer.sln'
]
make_env = ret[1]
elif 'MinGW' in cmake_generator:
make = ['mingw32-make']
else:
make = ['make']
proc = subprocess.Popen(make,
cwd=build_path,
stdin=log_output,
stdout=log_output,
stderr=log_output,
env=make_env)
proc.communicate()
if proc.returncode == 0:
if WINDOWS and 'Visual Studio' in cmake_generator:
shutil.copyfile(
os.path.join(build_path, cmake_build_type,
'optimizer.exe'), output)
else:
shutil.copyfile(
os.path.join(build_path, 'optimizer'),
output)
return output
else:
shared.try_delete(
os.path.join(build_path, 'CMakeCache.txt'))
# Proceed to next iteration of the loop to try next possible CMake generator.
raise NativeOptimizerCreationException()
def create_optimizer():
shared.logging.debug('building native optimizer: ' + name)
output = shared.Cache.get_path(name)
shared.try_delete(output)
for compiler in [
shared.CLANG, 'g++', 'clang++'
]: # try our clang first, otherwise hope for a system compiler in the path
shared.logging.debug(' using ' + compiler)
try:
out, err = subprocess.Popen(
[
compiler,
shared.path_from_root('tools', 'optimizer',
'parser.cpp'),
shared.path_from_root('tools', 'optimizer',
'simple_ast.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer-shared.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer-main.cpp'),
'-O3', '-std=c++11', '-fno-exceptions',
'-fno-rtti', '-o', output
] + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
outs.append(out)
errs.append(err)
except OSError:
if compiler == shared.CLANG:
raise # otherwise, OSError is likely due to g++ or clang++ not being in the path
if os.path.exists(output): return output
raise NativeOptimizerCreationException()
use_cmake_to_configure = WINDOWS # Currently only Windows uses CMake to drive the optimizer build, but set this to True to use on other platforms as well.
if use_cmake_to_configure:
return shared.Cache.get(name,
create_optimizer_cmake,
extension='exe')
else:
return shared.Cache.get(name,
create_optimizer,
extension='exe')
except NativeOptimizerCreationException, e:
shared.logging.debug('failed to build native optimizer')
handle_build_errors(outs, errs)
open(FAIL_MARKER, 'w').write(':(')
return None
def run_on_js(filename,
passes,
js_engine,
source_map=False,
extra_info=None,
just_split=False,
just_concat=False):
with ToolchainProfiler.profile_block('js_optimizer.split_markers'):
if type(passes) == str:
passes = [passes]
js = open(filename).read()
if os.linesep != '\n':
js = js.replace(os.linesep,
'\n') # we assume \n in the splitting code
# Find suffix
suffix_marker = '// EMSCRIPTEN_GENERATED_FUNCTIONS'
suffix_start = js.find(suffix_marker)
suffix = ''
if suffix_start >= 0:
suffix_end = js.find('\n', suffix_start)
suffix = js[suffix_start:suffix_end] + '\n'
# if there is metadata, we will run only on the generated functions. If there isn't, we will run on everything.
# Find markers
start_funcs = js.find(start_funcs_marker)
end_funcs = js.rfind(end_funcs_marker)
if start_funcs < 0 or end_funcs < start_funcs or not suffix:
logging.critical(
'Invalid input file. Did not contain appropriate markers. (start_funcs: %s, end_funcs: %s, suffix_start: %s'
% (start_funcs, end_funcs, suffix_start))
sys.exit(1)
minify_globals = 'minifyNames' in passes and 'asm' in passes
if minify_globals:
passes = map(lambda p: p
if p != 'minifyNames' else 'minifyLocals', passes)
start_asm = js.find(start_asm_marker)
end_asm = js.rfind(end_asm_marker)
assert (start_asm >= 0) == (end_asm >= 0)
closure = 'closure' in passes
if closure:
passes = filter(lambda p: p != 'closure',
passes) # we will do it manually
cleanup = 'cleanup' in passes
if cleanup:
passes = filter(lambda p: p != 'cleanup',
passes) # we will do it manually
split_memory = 'splitMemory' in passes
if not minify_globals:
with ToolchainProfiler.profile_block('js_optimizer.no_minify_globals'):
pre = js[:start_funcs + len(start_funcs_marker)]
post = js[end_funcs + len(end_funcs_marker):]
js = js[start_funcs + len(start_funcs_marker):end_funcs]
if 'asm' not in passes: # can have Module[..] and inlining prevention code, push those to post
class Finals:
buf = []
def process(line):
if len(line) > 0 and (line.startswith(
('Module[', 'if (globalScope)'))
or line.endswith('["X"]=1;')):
Finals.buf.append(line)
return False
return True
js = '\n'.join(filter(process, js.split('\n')))
post = '\n'.join(Finals.buf) + '\n' + post
post = end_funcs_marker + post
else:
with ToolchainProfiler.profile_block('js_optimizer.minify_globals'):
# We need to split out the asm shell as well, for minification
pre = js[:start_asm + len(start_asm_marker)]
post = js[end_asm:]
asm_shell = js[start_asm + len(start_asm_marker):start_funcs +
len(start_funcs_marker)] + '''
EMSCRIPTEN_FUNCS();
''' + js[end_funcs + len(end_funcs_marker):end_asm + len(end_asm_marker)]
js = js[start_funcs + len(start_funcs_marker):end_funcs]
# we assume there is a maximum of one new name per line
minifier = Minifier(js, js_engine)
def check_symbol_mapping(p):
if p.startswith('symbolMap='):
minifier.symbols_file = p.split('=')[1]
return False
if p == 'profilingFuncs':
minifier.profiling_funcs = True
return False
return True
passes = filter(check_symbol_mapping, passes)
asm_shell_pre, asm_shell_post = minifier.minify_shell(
asm_shell, 'minifyWhitespace' in passes,
source_map).split('EMSCRIPTEN_FUNCS();')
asm_shell_post = asm_shell_post.replace('});', '})')
pre += asm_shell_pre + '\n' + start_funcs_marker
post = end_funcs_marker + asm_shell_post + post
minify_info = minifier.serialize()
if extra_info:
for key, value in extra_info.iteritems():
assert key not in minify_info or value == minify_info[
key], [key, value, minify_info[key]]
minify_info[key] = value
#if DEBUG: print >> sys.stderr, 'minify info:', minify_info
with ToolchainProfiler.profile_block(
'js_optimizer.remove_suffix_and_split'):
# remove suffix if no longer needed
if suffix and 'last' in passes:
suffix_start = post.find(suffix_marker)
suffix_end = post.find('\n', suffix_start)
post = post[:suffix_start] + post[suffix_end:]
total_size = len(js)
funcs = split_funcs(js, just_split)
js = None
with ToolchainProfiler.profile_block('js_optimizer.split_to_chunks'):
# if we are making source maps, we want our debug numbering to start from the
# top of the file, so avoid breaking the JS into chunks
cores = 1 if source_map else int(
os.environ.get('EMCC_CORES') or multiprocessing.cpu_count())
if not just_split:
intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE))
chunk_size = min(
MAX_CHUNK_SIZE,
max(MIN_CHUNK_SIZE, total_size / intended_num_chunks))
chunks = shared.chunkify(funcs, chunk_size)
else:
# keep same chunks as before
chunks = map(lambda f: f[1], funcs)
chunks = filter(lambda chunk: len(chunk) > 0, chunks)
if DEBUG and len(chunks) > 0:
print >> sys.stderr, 'chunkification: num funcs:', len(
funcs), 'actual num chunks:', len(
chunks), 'chunk size range:', max(map(len,
chunks)), '-', min(
map(len, chunks))
funcs = None
if len(chunks) > 0:
serialized_extra_info = suffix_marker + '\n'
if minify_globals:
serialized_extra_info += '// EXTRA_INFO:' + json.dumps(
minify_info)
elif extra_info:
serialized_extra_info += '// EXTRA_INFO:' + json.dumps(
extra_info)
with ToolchainProfiler.profile_block('js_optimizer.write_chunks'):
def write_chunk(chunk, i):
temp_file = temp_files.get('.jsfunc_%d.js' % i).name
f = open(temp_file, 'w')
f.write(chunk)
f.write(serialized_extra_info)
f.close()
return temp_file
filenames = [
write_chunk(chunks[i], i) for i in range(len(chunks))
]
else:
filenames = []
with ToolchainProfiler.profile_block('run_optimizer'):
if len(filenames) > 0:
if not use_native(passes,
source_map) or not get_native_optimizer():
commands = map(
lambda filename: js_engine +
[JS_OPTIMIZER, filename, 'noPrintMetadata'] +
(['--debug'] if source_map else []) + passes, filenames)
else:
# use the native optimizer
shared.logging.debug('js optimizer using native')
assert not source_map # XXX need to use js optimizer
commands = map(
lambda filename: [get_native_optimizer(), filename] +
passes, filenames)
#print [' '.join(command) for command in commands]
cores = min(cores, len(filenames))
if len(chunks) > 1 and cores >= 2:
# We can parallelize
if DEBUG:
print >> sys.stderr, 'splitting up js optimization into %d chunks, using %d cores (total: %.2f MB)' % (
len(chunks), cores, total_size / (1024 * 1024.))
with ToolchainProfiler.profile_block('optimizer_pool'):
pool = shared.Building.get_multiprocessing_pool()
filenames = pool.map(run_on_chunk, commands, chunksize=1)
else:
# We can't parallize, but still break into chunks to avoid uglify/node memory issues
if len(chunks) > 1 and DEBUG:
print >> sys.stderr, 'splitting up js optimization into %d chunks' % (
len(chunks))
filenames = [run_on_chunk(command) for command in commands]
else:
filenames = []
for filename in filenames:
temp_files.note(filename)
with ToolchainProfiler.profile_block('split_closure_cleanup'):
if closure or cleanup or split_memory:
# run on the shell code, everything but what we js-optimize
start_asm = '// EMSCRIPTEN_START_ASM\n'
end_asm = '// EMSCRIPTEN_END_ASM\n'
cl_sep = 'wakaUnknownBefore(); var asm=wakaUnknownAfter(global,env,buffer)\n'
with temp_files.get_file('.cl.js') as cle:
c = open(cle, 'w')
pre_1, pre_2 = pre.split(start_asm)
post_1, post_2 = post.split(end_asm)
c.write(pre_1)
c.write(cl_sep)
c.write(post_2)
c.close()
cld = cle
if split_memory:
if DEBUG:
print >> sys.stderr, 'running splitMemory on shell code'
cld = run_on_chunk(js_engine +
[JS_OPTIMIZER, cld, 'splitMemoryShell'])
f = open(cld, 'a')
f.write(suffix_marker)
f.close()
if closure:
if DEBUG:
print >> sys.stderr, 'running closure on shell code'
cld = shared.Building.closure_compiler(
cld, pretty='minifyWhitespace' not in passes)
temp_files.note(cld)
elif cleanup:
if DEBUG:
print >> sys.stderr, 'running cleanup on shell code'
next = cld + '.cl.js'
temp_files.note(next)
proc = subprocess.Popen(
js_engine +
[JS_OPTIMIZER, cld, 'noPrintMetadata', 'JSDCE'] +
(['minifyWhitespace']
if 'minifyWhitespace' in passes else []),
stdout=open(next, 'w'))
proc.communicate()
assert proc.returncode == 0
cld = next
coutput = open(cld).read()
coutput = coutput.replace('wakaUnknownBefore();', start_asm)
after = 'wakaUnknownAfter'
start = coutput.find(after)
end = coutput.find(')', start)
pre = coutput[:start] + '(function(global,env,buffer) {\n' + pre_2[
pre_2.find('{') + 1:]
post = post_1 + end_asm + coutput[end + 1:]
with ToolchainProfiler.profile_block('write_pre'):
filename += '.jo.js'
f = open(filename, 'w')
f.write(pre)
pre = None
with ToolchainProfiler.profile_block('sort_or_concat'):
if not just_concat:
# sort functions by size, to make diffing easier and to improve aot times
funcses = []
for out_file in filenames:
funcses.append(split_funcs(open(out_file).read(), False))
funcs = [item for sublist in funcses for item in sublist]
funcses = None
def sorter(x, y):
diff = len(y[1]) - len(x[1])
if diff != 0: return diff
if x[0] < y[0]: return 1
elif x[0] > y[0]: return -1
return 0
if not os.environ.get('EMCC_NO_OPT_SORT'):
funcs.sort(sorter)
if 'last' in passes and len(funcs) > 0:
count = funcs[0][1].count('\n')
if count > 3000:
print >> sys.stderr, 'info: Output contains some very large functions (%s lines in %s), consider building source files with -Os or -Oz, and/or trying OUTLINING_LIMIT to break them up (see settings.js; note that the parameter there affects AST nodes, while we measure lines here, so the two may not match up)' % (
count, funcs[0][0])
for func in funcs:
f.write(func[1])
funcs = None
else:
# just concat the outputs
for out_file in filenames:
f.write(open(out_file).read())
with ToolchainProfiler.profile_block('write_post'):
f.write('\n')
f.write(post)
# No need to write suffix: if there was one, it is inside post which exists when suffix is there
f.write('\n')
f.close()
return filename
print "State", state, "from client @", client_address, "name: '", name, "'"
GPIO.setmode(GPIO.BCM) ## Use BCM pin numbering
GPIO.setup(int(sys.argv[3]), GPIO.OUT) ## Setup GPIO Pin to OUTPUT
GPIO.output(int(sys.argv[3]), state) ## State is true/false
return True
if __name__ == "__main__":
# Startup the fauxmo server
fauxmo.DEBUG = True
p = fauxmo.poller()
u = fauxmo.upnp_broadcast_responder()
u.init_socket()
p.add(u)
# Register the device callback as a fauxmo handler
d = device_handler()
for trig, port in d.TRIGGERS.items():
fauxmo.fauxmo(trig, u, p, None, port, d)
# Loop and poll for incoming Echo requests
logging.debug("Entering fauxmo polling loop")
while True:
try:
# Allow time for a ctrl-c to stop the process
p.poll(100)
time.sleep(0.1)
except Exception, e:
logging.critical("Critical exception: " + str(e))
break
def critical(msg, *args, **kwargs):
if not EnodebdLogger._LOGGER.propagate:
logging.critical(msg, *args, **kwargs)
EnodebdLogger._LOGGER.critical(msg, *args, **kwargs)
import logging
logging.basicConfig(filename='logs/logfile.log',
filemode='a',
level=logging.DEBUG,
format=f'%(levelname)s → %(asctime)s → %(message)s',
datefmt="%Y/%m/%d %H:%M:%S")
logging.info('Info')
logging.debug('Debug')
logging.warning('Warning')
logging.error('Error')
logging.critical('Critical')
logging.warning('WARNING')
def dzlog(msg, lvl='info', force=False, reset=False, arcmsg=False):
lfmat = '%(asctime)-15s: %(message)s '
callerframerecord = inspect.stack()[1]
frame = callerframerecord[0]
info = inspect.getframeinfo(frame)
msg_log = msg + ": " + info.function + ": " + str(info.lineno)
##........................................................................##
if arcmsg:
arcpy.AddMessage(msg)
##........................................................................##
if reset:
while len(logging.root.handlers) > 0:
logging.root.removeHandler(logging.root.handlers[-1])
##........................................................................##
if len(logging.root.handlers) == 0 or reset:
logging.basicConfig(filename=g_pn + os.sep + 'toolbox.log',
format=lfmat,
level=g_logging)
##........................................................................##
if force:
if lvl == 'debug':
level = logging.DEBUG
elif lvl == 'info':
level = logging.INFO
elif lvl == 'warning':
level = logging.WARNING
elif lvl == 'error':
level = logging.ERROR
elif lvl == 'critical':
level = logging.CRITICAL
if not logging.getLogger().isEnabledFor(level):
while len(logging.root.handlers) > 0:
logging.root.removeHandler(logging.root.handlers[-1])
logging.basicConfig(filename=g_pn + os.sep + 'toolbox.log',
format=lfmat,
level=level)
##........................................................................##
if lvl == 'debug':
logging.debug(msg_log)
elif lvl == 'info':
logging.info(msg_log)
elif lvl == 'warning':
logging.warning(msg_log)
elif lvl == 'error':
logging.error(msg_log)
elif lvl == 'critical':
logging.critical(msg_log)
def _ParseTestOutput(self, p):
"""Process the test output.
Args:
p: An instance of pexpect spawn class.
Returns:
A TestRunResults object.
"""
results = base_test_result.TestRunResults()
# Test case statuses.
re_run = re.compile('\[ RUN \] ?(.*)\r\n')
re_fail = re.compile('\[ FAILED \] ?(.*)\r\n')
re_ok = re.compile('\[ OK \] ?(.*?) .*\r\n')
# Test run statuses.
re_passed = re.compile('\[ PASSED \] ?(.*)\r\n')
re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n')
# Signal handlers are installed before starting tests
# to output the CRASHED marker when a crash happens.
re_crash = re.compile('\[ CRASHED \](.*)\r\n')
log = ''
try:
while True:
full_test_name = None
found = p.expect([re_run, re_passed, re_runner_fail],
timeout=self._timeout)
if found == 1: # re_passed
break
elif found == 2: # re_runner_fail
break
else: # re_run
full_test_name = p.match.group(1).replace('\r', '')
found = p.expect([re_ok, re_fail, re_crash],
timeout=self._timeout)
log = p.before.replace('\r', '')
if found == 0: # re_ok
if full_test_name == p.match.group(1).replace(
'\r', ''):
results.AddResult(
base_test_result.BaseTestResult(
full_test_name,
base_test_result.ResultType.PASS,
log=log))
elif found == 2: # re_crash
results.AddResult(
base_test_result.BaseTestResult(
full_test_name,
base_test_result.ResultType.CRASH,
log=log))
break
else: # re_fail
results.AddResult(
base_test_result.BaseTestResult(
full_test_name,
base_test_result.ResultType.FAIL,
log=log))
except pexpect.EOF:
logging.error('Test terminated - EOF')
# We're here because either the device went offline, or the test harness
# crashed without outputting the CRASHED marker (crbug.com/175538).
if not self.adb.IsOnline():
raise android_commands.errors.DeviceUnresponsiveError(
'Device %s went offline.' % self.device)
if full_test_name:
results.AddResult(
base_test_result.BaseTestResult(
full_test_name,
base_test_result.ResultType.CRASH,
log=p.before.replace('\r', '')))
except pexpect.TIMEOUT:
logging.error('Test terminated after %d second timeout.',
self._timeout)
if full_test_name:
results.AddResult(
base_test_result.BaseTestResult(
full_test_name,
base_test_result.ResultType.TIMEOUT,
log=p.before.replace('\r', '')))
finally:
p.close()
ret_code = self.test_package.GetGTestReturnCode(self.adb)
if ret_code:
logging.critical(
'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s',
ret_code, p.before, p.after)
return results
import logging
# 通过下面的方式进行简单配置输出方式与日志级别
logging.basicConfig(filename='logger.log', level=logging.DEBUG)
logging.debug('debug message')
logging.info('info message')
logging.warn('warn message')
logging.error('error message')
logging.critical('critical message')
def throttler(once=False, sleep_time=600):
"""
Main loop to check rse transfer limits.
"""
logging.info('Throttler starting')
executable = 'conveyor-throttler'
hostname = socket.getfqdn()
pid = os.getpid()
hb_thread = threading.current_thread()
heartbeat.sanity_check(executable=executable, hostname=hostname)
# Make an initial heartbeat so that all throttlers have the correct worker number on the next try
heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'],
heart_beat['nr_threads'])
logging.info(prepend_str + 'Throttler started - timeout (%s)' %
(sleep_time))
current_time = time.time()
graceful_stop.wait(10)
while not graceful_stop.is_set():
try:
heart_beat = heartbeat.live(executable,
hostname,
pid,
hb_thread,
older_than=3600)
prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'],
heart_beat['nr_threads'])
if heart_beat['assign_thread'] != 0:
logging.info(
prepend_str +
'Throttler thread id is not 0, will sleep. Only thread 0 will work'
)
if once:
break
if time.time() < current_time + sleep_time:
graceful_stop.wait(
int((current_time + sleep_time) - time.time()))
current_time = time.time()
continue
logging.info(prepend_str + "Throttler - schedule requests")
__schedule_requests()
if once:
break
if time.time() < current_time + sleep_time:
graceful_stop.wait(
int((current_time + sleep_time) - time.time()))
current_time = time.time()
except Exception:
logging.critical(prepend_str + 'Throtter crashed %s' %
(traceback.format_exc()))
if once:
break
logging.info(prepend_str + 'Throtter - graceful stop requested')
heartbeat.die(executable, hostname, pid, hb_thread)
logging.info(prepend_str + 'Throtter - graceful stop done')
from datetime import date
#logging.basicConfig(filename="checker.log", level=logging.INFO)
FORMAT = "%(asctime)-15s %(message)s"
logging.basicConfig(filename="checker.log",
format=FORMAT,
level=logging.INFO,
datefmt='%Y-%m-%d %H:%M:%S')
# GET Target Version - from file "checker.conf".
try:
result = open("checker.conf", 'r')
target_version = result.readline().strip()
except Exception as e:
#print("EXCEPTION READING: version.conf. EXCEPTION IS: %s" % (e))
logging.critical("EXCEPTION READING: checker.conf")
sys.exit(1)
# GET APP DNS NAME
state = 0
with open('terraform.tfstate', 'r') as searchfile:
for line in searchfile:
if re.search(r'instance_state', line, re.M | re.I):
state = "running"
if state == "running":
result = subprocess.run(['terraform', 'output', 'aws_instance_public_dns'],
stdout=subprocess.PIPE)
name = result.stdout.decode('utf-8').strip()
else:
logging.critical("EXCEPTION: terraform.tfstate file shows not running")
def __import_algo(self, algo: str):
try:
return __import__(algo)
except ImportError:
logging.critical("Could not import algorithm " + algo)
exit -1
with extension_loader.DlopenGuard():
try:
from caffe2.python.caffe2_pybind11_state_gpu import * # noqa
if num_cuda_devices(): # noqa
has_gpu_support = True
else:
has_gpu_support = False
except ImportError as e:
logging.warning('This caffe2 python run does not have GPU support. '
'Will run in CPU only mode.')
logging.warning('Debug message: {0}'.format(str(e)))
has_gpu_support = False
try:
from caffe2.python.caffe2_pybind11_state import * # noqa
except ImportError as e:
logging.critical('Cannot load caffe2.python. Error: {0}'.format(
str(e)))
sys.exit(1)
# libcaffe2_python contains a global Workspace that we need to properly delete
# when exiting. Otherwise, cudart will cause segfaults sometimes.
atexit.register(on_module_exit) # noqa
# Add functionalities for the TensorCPU interface.
def _TensorCPU_shape(self):
return tuple(self._shape)
def _TensorCPU_reshape(self, shape):
return self._reshape(list(shape))
def log_exception():
return logging.critical(
('Exception while Running ' if __name__ ==
'__main__' else 'Exception while Importing/using ') +
str(Path(__file__).resolve()) +
str(": {} ({})".format(whoami(), whosparent())))
def main(argv):
# ANDROID_SDK_ROOT needs to be set to the location of the SDK used to launch
# the emulator to find the system images upon launch.
emulator_sdk = constants.ANDROID_SDK_ROOT
os.environ['ANDROID_SDK_ROOT'] = emulator_sdk
arg_parser = argparse.ArgumentParser(description='AVD script.')
sub_parsers = arg_parser.add_subparsers(title='subparser', dest='command')
sub_parsers.add_parser(
'kill', help='Shutdown all existing emulators')
sub_parsers.add_parser(
'delete', help='Deleting all the avd files')
run_parser = sub_parsers.add_parser('run', help='Run emulators')
run_parser.add_argument('--name', help='Optinaly, name of existing AVD to '
'launch. If not specified, AVD\'s will be created')
run_parser.add_argument('-n', '--num', dest='emulator_count',
help='Number of emulators to launch (default is 1).',
type=int, default='1')
run_parser.add_argument('--abi', default='x86',
help='Platform of emulators to launch (x86 default)')
run_parser.add_argument('--api-level', dest='api_level',
help='API level for the image',
type=int, default=constants.ANDROID_SDK_VERSION)
run_parser.add_argument('--sdcard-size', dest='sdcard_size',
default=emulator.DEFAULT_SDCARD_SIZE,
help='Set sdcard size of the emulators'
' e.g. --sdcard-size=512M')
run_parser.add_argument('--partition-size', dest='partition_size',
default=emulator.DEFAULT_STORAGE_SIZE,
help='Default internal storage size'
' e.g. --partition-size=1024M')
run_parser.add_argument('--launch-without-kill', action='store_false',
dest='kill_and_launch', default=True,
help='Kill all emulators at launch')
run_parser.add_argument('--enable-kvm', action='store_true',
dest='enable_kvm', default=False,
help='Enable kvm for faster x86 emulator run')
arguments = arg_parser.parse_args(argv[1:])
logging.root.setLevel(logging.INFO)
devil_chromium.Initialize()
if arguments.command == 'kill':
logging.info('Killing all existing emulator and existing the program')
emulator.KillAllEmulators()
return
if arguments.command == 'delete':
emulator.DeleteAllTempAVDs()
return
# Check if SDK exist in ANDROID_SDK_ROOT
if not install_emulator_deps.CheckSDK():
raise Exception('Emulator SDK not installed in %s'
% constants.ANDROID_SDK_ROOT)
# Check if KVM is enabled for x86 AVD
if arguments.abi == 'x86':
if not install_emulator_deps.CheckKVM():
logging.warning('KVM is not installed or enabled')
arguments.enable_kvm = False
# Check if targeted system image exist
if not install_emulator_deps.CheckSystemImage(arguments.abi,
arguments.api_level):
logging.critical('ERROR: System image for %s AVD not installed. Run '
'install_emulator_deps.py', arguments.abi)
return 1
# If AVD is specified, check that the SDK has the required target. If not,
# check that the SDK has the desired target for the temporary AVD's.
api_level = arguments.api_level
if arguments.name:
android = os.path.join(constants.ANDROID_SDK_ROOT, 'tools',
'android')
avds_output = cmd_helper.GetCmdOutput([android, 'list', 'avd'])
names = re.findall(r'Name: (\w+)', avds_output)
api_levels = re.findall(r'API level (\d+)', avds_output)
try:
avd_index = names.index(arguments.name)
except ValueError:
logging.critical('ERROR: Specified AVD %s does not exist.',
arguments.name)
return 1
api_level = int(api_levels[avd_index])
if not install_emulator_deps.CheckSDKPlatform(api_level):
logging.critical('ERROR: Emulator SDK missing required target for API %d. '
'Run install_emulator_deps.py.')
return 1
if arguments.name:
emulator.LaunchEmulator(
arguments.name,
arguments.abi,
enable_kvm=arguments.enable_kvm,
kill_and_launch=arguments.reset_and_launch,
sdcard_size=arguments.sdcard_size,
storage_size=arguments.partition_size
)
else:
emulator.LaunchTempEmulators(
arguments.emulator_count,
arguments.abi,
arguments.api_level,
enable_kvm=arguments.enable_kvm,
kill_and_launch=arguments.kill_and_launch,
sdcard_size=arguments.sdcard_size,
storage_size=arguments.partition_size,
wait_for_boot=True
)
def data2bids(args):
# prepare parameter and path
orig_dir = pjoin(args.projectdir, 'data', 'bold', 'orig')
dicom_dir = pjoin(args.projectdir, 'data', 'bold', 'dicom')
nifti_dir = pjoin(args.projectdir, 'data', 'bold', 'nifti')
info_dir = pjoin(args.projectdir, 'data', 'bold', 'info')
# prepare logging
log_config(pjoin(nifti_dir, 'data2bids.log'))
# step 2 Information Reorganization
print(bcolors.BOLD_NODE + "[Node] Re-organizing..." + bcolors.ENDC)
# filter at first
# traverse the scaninfo
# generate session:input - determine
# generate session:task
# generate task:feature
scaninfo_raw = pd.read_excel(pjoin(info_dir, args.scaninfo))
# pandas:https://www.cnblogs.com/ech2o/p/11831488.html
if args.subject or args.session:
scaninfo = scaninfo_raw
if args.subject:
logging.info('DATA2BIDS: Selected subject is {}'.format(args.subject))
scaninfo = scaninfo[scaninfo['sub'].isin(args.subject)]
scaninfo.reset_index(drop=True)
if args.session:
logging.info('DATA2BIDS: selected session is {}'.format(args.session))
scaninfo = scaninfo[scaninfo['ses'].isin(args.session)]
scaninfo.reset_index(drop=True)
if args.quality_filter != 'all':
logging.info('DATA2BIDS: quality filter is {}'.format(args.quality_filter))
logging.info('DATA2BIDS: filtered scaninfo is in {}'.format(pjoin(nifti_dir, 'scaninfo_filtered.xlsx')))
scaninfo = scaninfo[scaninfo['quality'] == args.quality_filter]
scaninfo.reset_index(drop=True)
scaninfo.to_excel(pjoin(nifti_dir, 'scaninfo_filtered.xlsx'))
else:
if args.quality_filter != 'all':
logging.info('DATA2BIDS: quality filter is {}'.format(args.quality_filter))
logging.info('DATA2BIDS: filtered scaninfo is stored in {}'.format(pjoin(nifti_dir, 'scaninfo_filtered.xlsx')))
scaninfo = scaninfo_raw[scaninfo_raw['quality'] == args.quality_filter]
scaninfo.reset_index(drop=True)
scaninfo.to_excel(pjoin(nifti_dir, 'scaninfo_filtered.xlsx'))
else:
logging.info('DATA2BIDS: process all parts in {}'.format(args.scaninfo))
scaninfo = scaninfo_raw
# determine input of each session -- {sub-*/ses-* : *.tar.gz}
session_input = session_input_dict(scaninfo)
print("[news] Find {:d} session(s) waiting for processing..".format(len(session_input)))
for key, value in session_input.items():
print(bcolors.BOLD + " {:s} ".format(key) + bcolors.ENDC + "from: {:s}".format(value))
logging.info('DATA2BIDS: session-input mapping is stored in {}'.format(pjoin(nifti_dir, 'session-input.json')))
# detemine session-contained tasks --
session_task = session_task_dict(scaninfo)
print("[news] Tasks in each sub-session collected")
heu_session_task = {}
for key, value in session_task.items():
print(bcolors.BOLD + " {:s} ".format(key) + bcolors.ENDC + "contains: {}".format(value))
s_key = (key.split('-')[-1]).strip(string.digits)
if s_key not in heu_session_task.keys():
heu_session_task[s_key] = value
else:
if set(value) - set(heu_session_task[s_key]):
heu_session_task[s_key].extend(list(set(value) - set(heu_session_task[s_key])))
print("[news] Found {} kinds of session:".format(len(heu_session_task)))
for key, value in heu_session_task.items():
print(bcolors.BOLD + " {:s} ".format(key) + bcolors.ENDC + "contains: {}".format(value))
logging.info('DATA2BIDS: session-task mapping is stored in {}'.format(pjoin(nifti_dir, 'session-task.json')))
# determine task feature -- task : [protocolname dim]
task_feature = task_feature_dict(scaninfo)
print("[news] Task feature information collected..")
for key, value in task_feature.items():
print(bcolors.BOLD + " {:s} : ".format(key) + bcolors.ENDC + "protocolname = " + \
bcolors.BOLD + "{0[0]},".format(value) + bcolors.ENDC + " dim = " + \
bcolors.BOLD + "{0[1]} ".format(value) + bcolors.ENDC)
logging.info('DATA2BIDS: task-feature mapping is stored in {}'.format(pjoin(nifti_dir, 'task-feature.json')))
# step 3 Unpack
if not args.skip_unpack:
print(bcolors.BOLD_NODE + "[Node] Unpacking..." + bcolors.ENDC)
for _value in tqdm([__ for __ in session_input.values()]):
# upack
if not glob.glob(pjoin(dicom_dir, _value.replace('.tar.gz', ''))):
cmd = "tar -xzvf {:s} -C {:s}".format(pjoin(orig_dir, _value), dicom_dir)
logging.info('Unpack command: {:s}'.format(cmd))
print("[news] Running command: {:s}".format(cmd))
if not args.preview:
runcmd(cmd)
# step 4 Heuristic.py generation
print(bcolors.BOLD_NODE + "[Node] Heuristic.py Generating..." + bcolors.ENDC)
# task_feature & heu_session_task will be used
for key, value in heu_session_task.items():
check_path(pjoin(nifti_dir, 'code', key))
file = pjoin(nifti_dir, 'code', key, 'heuristic.py')
heu_creation = heucreation(file)
heu_creation.create_heuristic(value, task_feature)
print("[news] Heuristic.py completion!")
# step 5 heudiconv
print(bcolors.BOLD_NODE + "[Node] BIDS converting..." + bcolors.ENDC)
# session_input will be used
for _key, _value in tqdm(session_input.items()):
dicom_files = pjoin(dicom_dir, _value).replace('.tar.gz', '/*.IMA')
subID, sesID = _key.split('/')[0].replace('sub-', ''), _key.split('/')[1].replace('ses-', '')
if not args.skip_feature_validation:
# feature validation
if args.overwrite:
cmd = "heudiconv --files {:s} -o {:s} -f convertall -s {:s} -ss {:s} -c none --overwrite" \
.format(dicom_files, nifti_dir, subID, sesID)
else:
cmd = "heudiconv --files {:s} -o {:s} -f convertall -s {:s} -ss {:s} -c none" \
.format(dicom_files, nifti_dir, subID, sesID)
print("[news] inspecting task feature in dicominfo.tsv")
logging.info('Heudiconv command: {:s}'.format(cmd))
print("[news] command:" + bcolors.BOLD + " {}".format(cmd) + bcolors.ENDC)
if not args.preview:
runcmd(cmd)
dicominfo = pd.read_csv("{:s}/.heudiconv/{:s}/info/dicominfo_ses-{:s}.tsv" \
.format(nifti_dir, subID, sesID), sep='\t')
dicominfo_scan_feature = list(
set([(dicominfo.iloc[_run, :]['protocol_name'], dicominfo.iloc[_run, :]['dim4']) \
if dicominfo.iloc[_run, :]['dim4'] != 1 else (
dicominfo.iloc[_run, :]['protocol_name'], dicominfo.iloc[_run, :]['dim3']) \
for _run in range(len(dicominfo))]))
_check = []
for _task in session_task[_key]:
_feature = (task_feature[_task][0], task_feature[_task][1])
if 'anat' in _task:
if not any([_feature[0] == __[0] for __ in dicominfo_scan_feature]):
_check.append(any([_feature[0] == __[0] for __ in dicominfo_scan_feature]))
logging.critical("'{:s}' protocol name mismtach! Found no {:s} in {:s}/.heudiconv/{:s}/info/dicominfo_ses-{:s}.tsv" \
.format(_task, _feature[0], nifti_dir, subID, sesID))
print(bcolors.FAIL + \
"[ERROR] '{:s}' protocol name mismtach! Found no {:s} in {:s}/.heudiconv/{:s}/info/dicominfo_ses-{:s}.tsv" \
.format(_task, _feature[0], nifti_dir, subID, sesID) + bcolors.ENDC)
else:
if not _feature in dicominfo_scan_feature:
_check.append(_feature in dicominfo_scan_feature)
logging.critical('"'+_task+'" protocol name mismtach! Found no '+str(_feature)+' in '+nifti_dir+'/.heudiconv/'+subID+'/info/dicominfo_ses-'+sesID+'.tsv')
print('[ERROR] "'+_task+'" protocol name mismtach! Found no '+str(_feature)+' in '+nifti_dir+'/.heudiconv/'+subID+'/info/dicominfo_ses-'+sesID+'.tsv')
if not all(_check):
logging.critical('Feature validation failure!')
raise AssertionError(
'[ERROR] Feature validation failure! Please read [ERROR] message above or log for more details!')
print(bcolors.BOLD + "[news] Feature validation seuccess!" + bcolors.ENDC)
del _task, _feature
heuristicpy = pjoin(nifti_dir, 'code', sesID.strip(string.digits), 'heuristic.py')
if args.overwrite:
cmd = "heudiconv --files {:s} -o {:s} -f {:s} -s {:s} -ss {:s} -c dcm2niix -b --overwrite" \
.format(dicom_files, nifti_dir, heuristicpy, subID, sesID)
else:
cmd = "heudiconv --files {:s} -o {:s} -f {:s} -s {:s} -ss {:s} -c dcm2niix -b" \
.format(dicom_files, nifti_dir, heuristicpy, subID, sesID)
print("[news] Processing sub-{:s}/ses-{:s}".format(subID, sesID))
logging.info("Heudiconv command (overwrite): {:s}".format(cmd))
print("command: " + bcolors.BOLD + "{:s}".format(cmd) + bcolors.ENDC)
if not args.preview:
runcmd(cmd, timeout=3600)
# fill fmap json files
print(bcolors.BOLD_NODE + "[Node] .json Filling up..." + bcolors.ENDC)
if args.subject:
subjects = ["{:02d}".format(int(_)) for _ in args.subject]
else:
subjects = [_.replace("sub-", "") for _ in os.listdir(nifti_dir) if "sub-" in _]
sessions = {name: [] for name in (subjects)} # {sub : [session]}
jsonfiles = {name: {sesname: [] for sesname in sessions[name]} for name in
(subjects)} # {sub:{session:[files]}}
intendedfornii = {name: {sesname: [] for sesname in sessions[name]} for name in
(subjects)} # {sub:{session:[files]}}
# collect .json files waiting to fill & .nii.gz filenames
if not args.preview:
print("[news] collect .json files waiting to fill & .nii.gz filenames")
for subname in sessions.keys():
# get all the sessions under a subject
subpth = pjoin(nifti_dir, 'sub-%s' % (subname))
sessions[subname] = os.listdir(subpth)
# collect jsonfiles & values
for fold in sessions[subname]:
# path preparation
sesspth = pjoin(nifti_dir, subpth, fold)
fmappth = pjoin(sesspth, 'fmap')
funcpth = pjoin(sesspth, 'func')
# if fmap exist then clollect
if os.path.exists(fmappth):
jsonfiles[subname][fold] = [file for file in os.listdir(fmappth) if '.json' in file]
# the file path must be the relative path to sub- folder
intendedfornii[subname][fold] = ['%s/func/%s' % (fold, file) for file in os.listdir(funcpth) if
'.nii.gz' in file]
# write key:value for each json
print("[news] write key:value for each json")
for sub, ses_fold in jsonfiles.items():
for ses, files in ses_fold.items():
for file in files:
# file path
file_path = os.path.join(nifti_dir, 'sub-%s/%s' % (sub, ses), 'fmap', file)
# change mode
chmod_cmd = ' '.join(['chmod', '755', file_path])
runcmd(chmod_cmd, timeout=3600)
# load in & add IntendedFor
with open(file_path, 'r') as datafile:
data = json.load(datafile)
data['IntendedFor'] = intendedfornii[sub][ses]
# save out
with open(file_path, 'w') as datafile:
json.dump(data, datafile)
print('[news] fill up json for %s ... done!' % sub)
print('Log is saved in {}.'.format(pjoin(nifti_dir, 'data2bids.log')))
def _run_cmake_build(cmake_build_dir, args):
"""
Run the Cmake build
:param cmake_build_dir: Base directory for Cmake build
:param args: Command-line arguments
"""
cmake_cmd = ["cmake", "../", "-GUnix Makefiles"]
if args.suite == "IPC":
cmake_cmd.append("-DTARGET=" + PSA_API_TARGETS.get(args.mcu)[2])
cmake_cmd.append("-DPLATFORM_PSA_ISOLATION_LEVEL=1")
cmake_cmd.append("-DSP_HEAP_MEM_SUPP=0")
else:
cmake_cmd.append("-DTARGET=" + PSA_API_TARGETS.get(args.mcu)[1])
cmake_cmd.append("-DCPU_ARCH=" + PSA_API_TARGETS.get(args.mcu)[0])
cmake_cmd.append("-DSUITE=" + args.suite)
if args.verbose:
cmake_cmd.append("-DVERBOSE=" + str(args.verbose))
if args.include:
cmake_cmd.append("-DPSA_INCLUDE_PATHS=" + args.include)
else:
# Take in defaults
suite_include = join(
TF_M_BUILD_DIR, "trusted-firmware-m", "interface", "include"
)
if args.suite == "CRYPTO":
crypto_include = join(
TF_M_BUILD_DIR, "psa-arch-tests", "mbed-crypto", "include"
)
cmake_cmd.append("-DPSA_INCLUDE_PATHS=" + crypto_include)
elif args.suite == "IPC":
manifest_include = join(
TF_M_BUILD_DIR,
"psa-arch-tests",
"api-tests",
"platform",
"manifests",
)
cmake_cmd.append(
"-DPSA_INCLUDE_PATHS=" + manifest_include + ";" + suite_include
)
else:
# Applicable for INITIAL_ATTESTATION, INTERNAL_TRUSTED_STORAGE
# PROTECTED_STORAGE and STORAGE suites.
cmake_cmd.append("-DPSA_INCLUDE_PATHS=" + suite_include)
if args.range:
cmake_cmd.append("-DSUITE_TEST_RANGE=" + args.range)
retcode = run_cmd_output_realtime(cmake_cmd, cmake_build_dir)
if retcode:
msg = "Cmake configure failed for target %s using toolchain %s" % (
args.mcu,
args.toolchain,
)
logging.critical(msg)
sys.exit(1)
cmake_cmd = ["cmake", "--build", "."]
retcode = run_cmd_output_realtime(cmake_cmd, cmake_build_dir)
if retcode:
msg = "Cmake build failed for target %s using toolchain %s" % (
args.mcu,
args.toolchain,
)
logging.critical(msg)
sys.exit(1)
def main(args=None):
args = parser.parse_args(args)
config.read(args.config)
try:
logging.basicConfig(
format="%(levelname)s:%(message)s", level=config["logging"]["log-level"]
)
except ValueError:
logging.warning(
f"Incorrect log-level specified: {config['logging']['log-level']}. "
"Falling back to INFO."
)
logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO)
try:
realm = Realm[config["api"]["realm"]]
except KeyError:
logging.critical(
f"Configured realm \"{config['api']['realm']}\" is unknown. "
f"Choose one of: {', '.join(Realm.__members__.keys())}"
)
sys.exit(1)
if not len(config["accounts"]):
logging.warning(
"There are no configured accounts, nothing to do. "
'Check the "accounts" section in the config file.'
)
try:
account_data = account_info(
realm,
config["api"]["application-id"],
list(config["accounts"].keys()),
).values()
except ValueError as e:
logging.critical(e)
sys.exit(1)
flat_account_data = [
timestamps_to_datetime(flatten(data, strip=True), keys=TIME_FIELDS)
for data in account_data
]
rows = [
{column.name: data[column.name] for column in statistics.columns}
for data in flat_account_data
]
try:
with sa.create_engine(config["db"]["url"]).connect() as conn:
changed = False
for row in rows:
logging.info(
f"Attempting insert {row['nickname']} @ {row['updated_at']}"
)
try:
conn.execute(
statistics.insert()
.values(row)
.compile(dialect=postgresql.dialect())
)
changed = True
logging.info("Insert successful")
except IntegrityError as e:
if not isinstance(e.orig, psycopg2.errors.UniqueViolation):
raise e from e
logging.info(f"Skipping, record exists")
if config["plots"] and changed:
logging.info("Change detected, updating plots")
df = pd.read_sql(
"SELECT * from statistics ORDER BY updated_at",
conn,
index_col=["account_id", "updated_at"],
)
plt.style.use("Solarize_Light2")
for path, interval_str in config["plots"].items():
if interval_str:
figure = create_plot(
df[
df.index.get_level_values(1)
> (
pd.Timestamp.now(tz="UTC")
- pd.Timedelta(interval_str)
)
]
)
else:
figure = create_plot(df)
logging.info(f"Saving {path}")
figure.savefig(path)
except sa.exc.OperationalError as e:
logging.critical(f"Invalid database URL: {config['db']['url']} ({e})")
sys.exit(1)
except sa.exc.NoSuchModuleError:
logging.critical(f"Invalid protocol in database URL: {config['db']['url']}")
sys.exit(1)
######################################################
# In-File Config, prework
######################################################
#Read Config
config = ConfigParser()
configpath = os.path.join(os.path.dirname(__file__), '..', 'config')
try:
config.read(
os.path.join(os.path.dirname(__file__), '..', 'config',
'config_main.ini'))
except Exception:
try:
shutil.copyfile(os.path.join(configpath, 'config_main.ini.back'),
os.path.join(configpath, 'config_main.ini'))
except Exception:
logging.critical("""config_main.ini not found,
backup couldn't be restored, aborting""")
else:
logging.warn("""copied config_main.ini.bak to config_main.ini
because python couldn't find a valid config""")
else:
logging.debug("read config_main.ini successfully")
# format color dictionary
colors_raw = config._sections['colors']
colors = {}
for color in colors_raw:
colors[color] = chr(0x1b) + colors_raw[color]
######################################################
# Functions, Methods (Code)
######################################################
def main():
argparser = argparse.ArgumentParser(description='Print APK size metrics.')
argparser.add_argument(
'--min-pak-resource-size',
type=int,
default=20 * 1024,
help='Minimum byte size of displayed pak resources.')
argparser.add_argument('--chromium-output-directory',
dest='out_dir',
help='Location of the build artifacts.')
argparser.add_argument('--chartjson',
action='store_true',
help='Sets output mode to chartjson.')
argparser.add_argument('--output-dir',
default='.',
help='Directory to save chartjson to.')
argparser.add_argument(
'--dump-static-initializers',
action='store_true',
dest='dump_sis',
help='Run dump-static-initializers.py to get the list'
'of static initializers (slow).')
argparser.add_argument('--loadable_module',
action='append',
help='Use for libraries added via loadable_modules')
argparser.add_argument(
'--estimate-patch-size',
action='store_true',
help='Include patch size estimates. Useful for perf '
'builders where a reference APK is available but adds '
'~3 mins to run time.')
argparser.add_argument(
'--reference-apk-builder',
default=apk_downloader.DEFAULT_BUILDER,
help='Builder name to use for reference APK for patch '
'size estimates.')
argparser.add_argument('--reference-apk-bucket',
default=apk_downloader.DEFAULT_BUCKET,
help='Storage bucket holding reference APKs.')
argparser.add_argument('apk', help='APK file path.')
args = argparser.parse_args()
chartjson = _BASE_CHART.copy() if args.chartjson else None
out_dir, tool_prefix = _ConfigOutDirAndToolsPrefix(args.out_dir)
if args.dump_sis and not out_dir:
argparser.error(
'--dump-static-initializers requires --chromium-output-directory')
# Do not add any new metrics without also documenting them in:
# //docs/speed/binary_size/metrics.md.
PrintApkAnalysis(args.apk, tool_prefix, out_dir, chartjson=chartjson)
_PrintDexAnalysis(args.apk, chartjson=chartjson)
ignored_libs = args.loadable_module if args.loadable_module else []
si_count = AnalyzeStaticInitializers(args.apk, tool_prefix, args.dump_sis,
out_dir, ignored_libs)
perf_tests_results_helper.ReportPerfResult(chartjson,
'StaticInitializersCount',
'count', si_count, 'count')
if args.estimate_patch_size:
_PrintPatchSizeEstimate(args.apk,
args.reference_apk_builder,
args.reference_apk_bucket,
chartjson=chartjson)
if chartjson:
results_path = os.path.join(args.output_dir, 'results-chart.json')
logging.critical('Dumping json to %s', results_path)
with open(results_path, 'w') as json_file:
json.dump(chartjson, json_file)
def main(self, cfg=None):
if cfg == None:
cfg = self.cfg
else:
self.__init__(cfg)
try:
# Added to file read to catch multiple columns with same name which causes second column to overwrite first. - GDB
file_handle = open(cfg["input"], 'rU')
csv_reader = csv.reader(file_handle)
l = csv_reader.next()
if len(l) > len(set(l)):
logging.error(l)
raise KeyError(
"Input file has multiple columns of the same name. Please create unique columns and rerun."
)
# exit(1)
else:
file_handle.seek(0)
infile = csv.DictReader(file_handle)
# infile = csv.DictReader(open(args.filename,'rU')) # Old File Read - gdb
except IOError:
logging.critical("ERROR: Input file not found.")
raise
# exit(1)
infile.fieldnames = [
f.decode('unicode_escape').encode('ascii', 'ignore')
for f in infile.fieldnames
] # remove unicode - gdb 170130
for f in infile.fieldnames:
if f not in self.sfields:
if f != "repeat":
logging.warning(
"column will not be used: %s. May be inaccurate for 'plus' columns.",
f)
if 'plus.analyst' not in infile.fieldnames:
logging.warning(
"the optional plus.analyst field is not found in the source document"
)
if 'source_id' not in infile.fieldnames:
logging.warning(
"the optional source_id field is not found in the source document"
)
row = 0
for incident in infile:
row += 1
# have to look for white-space only and remove it
try:
incident = {x: incident[x].strip() for x in incident}
except AttributeError as e:
logging.error(
"Error removing white space on row {1}.".format(row))
raise e
if 'incident_id' in incident:
iid = incident['incident_id']
else:
iid = "srcrow_" + str(row)
logging.debug("Starting incident {0} on row {1}.".format(iid, row))
# logging.warning("This includes the row number")
repeat = 1
logging.info("-----> parsing incident %s", iid)
if incident.has_key('repeat'):
if incident['repeat'].lower(
) == "ignore" or incident['repeat'] == "0":
logging.info(
"Skipping row %s because 'repeat' is either 'ignore' or '0'.",
iid)
continue
repeat = self.isnum(incident['repeat'])
if not repeat:
repeat = 1
if incident.has_key('security_incident'):
if incident['security_incident'].lower() == "no":
logging.info(
"Skipping row %s because security_incident is 'no'.",
iid)
continue
outjson = self.convertCSV(incident)
while repeat > 0:
if 'plus' not in outjson:
outjson['plus'] = {}
outjson['plus']['master_id'] = str(uuid.uuid4()).upper()
yield iid, outjson
# outjson['incident_id'] = str(uuid.uuid4()).upper() ### HERE
# outjson['plus']['master_id'] = outjson['incident_id'] ###
repeat -= 1
if repeat > 0:
logging.info("Repeating %s more times on %s", repeat, iid)
def __call__(self, doc):
"""
Overlays entity annotations over tokens in a Doc object. Requires that tokens in the Doc have the custom
'gold_annotation_file' and 'file_name' extension.
:param doc: a spaCy Doc object.
:return: the same Doc object, but it now has 'gold_label' annotations.
"""
file_name = doc._.file_name
logging.debug(f"{file_name}: Called GoldAnnotator Component")
failed_overlay_count = 0
failed_identifying_span_count = 0
# check if gold annotation file path has been set.
if not hasattr(doc._, 'gold_annotation_file'):
logging.warning(
f"doc._.gold_annotation_file not defined for {file_name}; "
f"it will not be possible to fit a model with this Doc")
return doc
gold_annotations = Annotations(doc._.gold_annotation_file)
for ent in gold_annotations:
if ent.start > ent.end:
logging.critical(
f"{file_name}: Broken annotation - start is greater than end: {ent}"
)
continue
span = doc.char_span(ent.start, ent.end)
if span is None:
failed_overlay_count += 1
failed_identifying_span_count += 1
fixed_span = self.find_span(ent.start, ent.end, doc)
if fixed_span is not None:
if span is None:
logging.warning(
f"{file_name}: Fixed {ent} into: {fixed_span.text}")
failed_identifying_span_count -= 1
for token in fixed_span:
if ent.tag in self.labels or not self.labels:
token._.set('gold_label', ent.tag)
else:
# Annotation was not able to be fixed, it will be ignored - this is bad in evaluation.
logging.warning(
f"{file_name}: Could not fix annotation: {ent}")
logging.warning(
f"{file_name}: Number of failed annotation overlays with current tokenizer: {failed_overlay_count}"
)
if failed_overlay_count > .3 * len(gold_annotations):
logging.critical(
f"{file_name}: More than 30% of annotations failed to overlay")
return doc
def dump(self, remoteName, remoteHost):
"""Dumps the list of endpoints registered with the mapper
listening at addr. remoteName is a valid host name or IP
address in string format.
"""
logging.info('Retrieving endpoint list from %s' % remoteName)
entries = []
stringbinding = self.KNOWN_PROTOCOLS[
self.__port]['bindstr'] % remoteName
logging.debug('StringBinding %s' % stringbinding)
rpctransport = transport.DCERPCTransportFactory(stringbinding)
rpctransport.set_dport(self.__port)
if self.KNOWN_PROTOCOLS[self.__port]['set_host']:
rpctransport.setRemoteHost(remoteHost)
if hasattr(rpctransport, 'set_credentials'):
# This method exists only for selected protocol sequences.
rpctransport.set_credentials(self.__username, self.__password,
self.__domain, self.__lmhash,
self.__nthash)
try:
entries = self.__fetchList(rpctransport)
except Exception as e:
logging.critical('Protocol failed: %s' % e)
# Display results.
endpoints = {}
# Let's groups the UUIDS
for entry in entries:
binding = epm.PrintStringBinding(entry['tower']['Floors'],
rpctransport.getRemoteHost())
tmpUUID = str(entry['tower']['Floors'][0])
if (tmpUUID in endpoints) is not True:
endpoints[tmpUUID] = {}
endpoints[tmpUUID]['Bindings'] = list()
if uuid.uuidtup_to_bin(
uuid.string_to_uuidtup(tmpUUID))[:18] in epm.KNOWN_UUIDS:
endpoints[tmpUUID]['EXE'] = epm.KNOWN_UUIDS[
uuid.uuidtup_to_bin(uuid.string_to_uuidtup(tmpUUID))[:18]]
else:
endpoints[tmpUUID]['EXE'] = 'N/A'
endpoints[tmpUUID]['annotation'] = entry['annotation'][:-1].decode(
'utf-8')
endpoints[tmpUUID]['Bindings'].append(binding)
if tmpUUID[:36] in epm.KNOWN_PROTOCOLS:
endpoints[tmpUUID]['Protocol'] = epm.KNOWN_PROTOCOLS[
tmpUUID[:36]]
else:
endpoints[tmpUUID]['Protocol'] = "N/A"
#print "Transfer Syntax: %s" % entry['Tower']['Floors'][1]
for endpoint in list(endpoints.keys()):
print("Protocol: %s " % endpoints[endpoint]['Protocol'])
print("Provider: %s " % endpoints[endpoint]['EXE'])
print("UUID : %s %s" %
(endpoint, endpoints[endpoint]['annotation']))
print("Bindings: ")
for binding in endpoints[endpoint]['Bindings']:
print(" %s" % binding)
print("")
if entries:
num = len(entries)
if 1 == num:
logging.info('Received one endpoint.')
else:
logging.info('Received %d endpoints.' % num)
else:
logging.info('No endpoints found.')
tgt_index_fpath = "{}.{}.ann".format(args.tgt, args.emb)
tgt_index.load(tgt_index_fpath)
logging.info("Loaded {}".format(tgt_index_fpath))
args.src_index = src_index
args.tgt_index = tgt_index
args.w2v = w2v
# Some checks of the size
try:
if src_index is not None:
assert src_index.get_n_items() > 0
if tgt_index is not None:
assert tgt_index.get_n_items() > 0
except AssertionError:
logging.critical("Error loading Annoy indices. SRC size: {} TGT size: {}".format(
src_index.get_n_items(), tgt_index.get_n_items()))
logging.warning("Starting alignment of {} source and {} target documents...".format(
src_index.get_n_items() if src_index is not None else "LAZY", tgt_index.get_n_items() if tgt_index is not None else "LAZY"))
logging.warning("Will use the {} metric for refinement.".format(args.refine))
start_time = time.time()
batch_time = time.time()
args_dict = vars(args).copy()
for key in ["emb", "level", "annoy_metric", "vec_size", "lazy_source", "lazy_target"]:
del args_dict[key]
aligner = None
try:
assert args.level in SUPPORTED_ALIGN_LEVELS
except AssertionError as e:
def do_log_example():
logging.debug('Debug Info')
logging.info('Info')
logging.warning('Warning')
logging.error('Error')
logging.critical('Critical Error')
import os, time
import subprocess, select
import nexmo
# Initialization
log = logging.getLogger('sshalert')
log.setLevel(logging.INFO)
try:
source_phone_number = os.getenv("SOURCE_PHONE_NUMBER")
target_phone_number = os.getenv("TARGET_PHONE_NUMBER")
nexmo_key = os.getenv("NEXMO_KEY")
nexmo_secret = os.getenv("NEXMO_SECRET")
except:
logging.critical(
"ERROR: Have you exported all required environment variables? (TARGET_PHONE_NUMBER, NEXMO_KEY, NEXMO_SECRET)"
)
exit(1)
# Initialize the nexmo client
nexmo_client = nexmo.Client(key=nexmo_key, secret=nexmo_secret)
def poll_logfile(filename):
"""
Polls a logfile for sudo commands or ssh logins.
"""
f = subprocess.Popen(["tail", "-F", "-n", "0", filename],
encoding="utf8",
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
def blackHole(self, Ip):
logging.critical("{} Has Been Block By IpTables!".format(Ip))
# 屏蔽IP
os.system("iptables -I INPUT -s {} -j DROP".format(Ip))
return False
