def __init__(self, sample, output_counter=None):
# Override maxQueueLength to EventPerKey so that each flush
# will generate one aws key
if sample.awsS3EventPerKey:
sample.maxQueueLength = sample.awsS3EventPerKey
OutputPlugin.__init__(self, sample, output_counter)
if not boto_imported:
logger.error("There is no boto3 or botocore library available")
return
# disable any "requests" warnings
requests.packages.urllib3.disable_warnings()
# Bind passed in samples to the outputter.
self.awsS3compressiontype = sample.awsS3CompressionType if hasattr(
sample,
'awsS3CompressionType') and sample.awsS3CompressionType else None
self.awsS3eventtype = sample.awsS3EventType if hasattr(
sample, 'awsS3EventType') and sample.awsS3EventType else 'syslog'
self.awsS3objectprefix = sample.awsS3ObjectPrefix if hasattr(
sample, 'awsS3ObjectPrefix') and sample.awsS3ObjectPrefix else ""
self.awsS3objectsuffix = sample.awsS3ObjectSuffix if hasattr(
sample, 'awsS3ObjectSuffix') and sample.awsS3ObjectSuffix else ""
self.awsS3bucketname = sample.awsS3BucketName
logger.debug("Setting up the connection pool for %s in %s" %
(self._sample.name, self._app))
self._client = None
self._createConnections(sample)
logger.debug("Finished init of awsS3 plugin.")
def _transmitEvents(self, payloadstring):
targetServer = []
logger.debug("Transmission called with payloadstring: %s " %
payloadstring)
if self.httpeventoutputmode == "mirror":
targetServer = self.serverPool
else:
targetServer.append(random.choice(self.serverPool))
for server in targetServer:
logger.debug("Selected targetServer object: %s" % targetServer)
url = server['url']
headers = {}
headers['Authorization'] = server['header']
headers['content-type'] = 'application/json'
try:
payloadsize = len(payloadstring)
# response = requests.post(url, data=payloadstring, headers=headers, verify=False)
self.active_sessions.append(
self.session.post(url=url,
data=payloadstring,
headers=headers,
verify=False))
except Exception as e:
logger.error("Failed for exception: %s" % e)
logger.error(
"Failed sending events to url: %s sourcetype: %s size: %s"
% (url, self.lastsourcetype, payloadsize))
logger.debug(
"Failed sending events to url: %s headers: %s payload: %s"
% (url, headers, payloadstring))
raise e
def __init__(self, sample, output_counter=None):
OutputPlugin.__init__(self, sample, output_counter)
from eventgenconfig import Config
globals()['c'] = Config()
self._splunkUrl, self._splunkMethod, self._splunkHost, self._splunkPort = c.getSplunkUrl(self._sample) # noqa
self._splunkUser = self._sample.splunkUser
self._splunkPass = self._sample.splunkPass
if not self._sample.sessionKey:
try:
myhttp = httplib2.Http(disable_ssl_certificate_validation=True)
logger.debug("Getting session key from '%s' with user '%s' and pass '%s'" %
(self._splunkUrl + '/services/auth/login', self._splunkUser, self._splunkPass))
response = myhttp.request(
self._splunkUrl + '/services/auth/login', 'POST', headers={}, body=urllib.urlencode({
'username':
self._splunkUser, 'password':
self._splunkPass}))[1]
self._sample.sessionKey = minidom.parseString(response).getElementsByTagName(
'sessionKey')[0].childNodes[0].nodeValue
logger.debug("Got new session for splunkstream, sessionKey '%s'" % self._sample.sessionKey)
except:
logger.error("Error getting session key for non-SPLUNK_EMBEEDED for sample '%s'." %
self._sample.name + " Credentials are missing or wrong")
raise IOError("Error getting session key for non-SPLUNK_EMBEEDED for sample '%s'." % self._sample.name +
"Credentials are missing or wrong")
logger.debug("Retrieved session key '%s' for Splunk session for sample %s'" % (self._sample.sessionKey,
self._sample.name))
def main():
logger.info("Launching...")
logger.info("Search for dogs on " +
len(zoo.sites_to_monitor.items()).__str__() + " sites")
zoo.scan_for_dogs()
logger.info(
len(zoo.current_dogs).__str__() + " dogs have been added to cache")
zoo.new_dogs.clear()
logger.info("Monitoring...")
while True:
try:
time.sleep(600)
zoo.scan_for_dogs()
if len(zoo.new_dogs) == 0:
logger.info("No updates...")
continue
else:
notis = len(
zoo.new_dogs.values()).__str__() + " new dog(s) found"
SendMail(notis, "".join(zoo.new_dogs.values()))
logger.info(notis)
zoo.new_dogs.clear()
continue
except Exception as e:
logger.error("Error: " + e)
SendMail("Something went wrong!", e)
def __init__(self, time, sample=None, config=None, genqueue=None, outputqueue=None, loggingqueue=None):
# Logger already setup by config, just get an instance
# setup default options
self.profiler = config.profiler
self.config = config
self.sample = sample
self.end = getattr(self.sample, "end", -1)
self.endts = getattr(self.sample, "endts", None)
self.generatorQueue = genqueue
self.outputQueue = outputqueue
self.time = time
self.stopping = False
self.countdown = 0
self.executions = 0
self.interval = getattr(self.sample, "interval", config.interval)
logger.debug('Initializing timer for %s' % sample.name if sample is not None else "None")
# load plugins
if self.sample is not None:
rater_class = self.config.getPlugin('rater.' + self.sample.rater, self.sample)
self.rater = rater_class(self.sample)
self.generatorPlugin = self.config.getPlugin('generator.' + self.sample.generator, self.sample)
self.outputPlugin = self.config.getPlugin('output.' + self.sample.outputMode, self.sample)
if self.sample.timeMultiple < 0:
logger.error("Invalid setting for timeMultiple: {}, value should be positive".format(
self.sample.timeMultiple))
elif self.sample.timeMultiple != 1:
self.interval = self.sample.interval
logger.debug("Adjusting interval {} with timeMultiple {}, new interval: {}".format(
self.sample.interval, self.sample.timeMultiple, self.interval))
logger.info(
"Start '%s' generatorWorkers for sample '%s'" % (self.sample.config.generatorWorkers, self.sample.name))
def train_current_generation(self):
logger.info(f'Training generation {self.current_generation}')
for model in self.current_generation_models:
model_name = model['name']
logger.info(f'Training model {model_name}.')
try:
model_runs = [CNN(model, verbose=1) for _ in range(self.model_reruns)]
except Exception as error:
logger.error(error)
# revert Changes
prev_model = model['prev_model']
model = self.models[prev_model]
model = CNN.add_change_log(model, f'Reverted to model {prev_model} due to an exception on training.')
model_name = model['name']
model_runs = [CNN(model, verbose=1) for _ in range(self.model_reruns)]
logger.info(f'Training model {model_name} completed')
self.metrics.loc[model_name, 'test_Accuracy'] = np.min([cnn.accuracy[0] for cnn in model_runs])
self.metrics.loc[model_name, 'train_Accuracy'] = np.min([cnn.accuracy[1] for cnn in model_runs])
self.metrics.loc[model_name, 'training_time'] = np.max([cnn.Training_time for cnn in model_runs])
self.metrics.loc[model_name, 'over-fit'] = np.any([cnn.is_over_fitted for cnn in model_runs])
self.metrics.loc[model_name, 'prev_model'] = model['prev_model']
self.metrics.loc[model_name, 'generation'] = self.current_generation
model['layers_input_output_shape'] = [ f'layer.name: {layer.input_shape} --- {layer.output_shape}'
for layer in model_runs[0].model.layers]
self.save_model(model)
logger.info(f'Performance results for {model_name}:-\n{self.metrics.loc[model_name, :]}')
logger.info(f'Generation {self.current_generation} Training completed.\n------------------\n')
def maybe_build_base_devel():
if db.exists('antbs:docker-images:base-devel:built-today'):
return True
# No image was built in the past 24 hours, let's build one.
build_script = os.path.join(DOC_DIR, 'base-devel.sh')
build_it = False
try:
build_it = subprocess.check_output([build_script])
shutil.rmtree('/opt/antergos-packages')
except subprocess.CalledProcessError as err:
logger.error('@@-docker_util.py-@@ | Image build script failed with error: %s', err.output)
return False
except shutil.Error as err2:
logger(err2)
if build_it:
# Image was built successfully. Push it to docker hub.
push_to_hub('antergos/archlinux-base-devel')
mpkg = build_makepkg()
if not mpkg:
return False
db.psetex('antbs:docker-images:base-devel:built-today', 304800000, 'True')
return True
else:
return False
def maybe_build_base_devel():
"""
:return:
"""
if db.exists('antbs:docker-images:base-devel:built-today'):
return True
# No image was built in the past 24 hours, let's build one.
status.current_status = 'Docker images are stale. Building new images.'
build_script = os.path.join(DOC_DIR, 'base-devel.sh')
build_it = False
try:
build_it = subprocess.check_output([build_script])
except subprocess.CalledProcessError as err:
logger.error('@@-docker_util.py-@@ | Image build script failed with error: %s', err.output)
return False
except shutil.Error as err2:
logger(err2)
if build_it:
try:
# Image was built successfully. Push it to docker hub.
push_to_hub('antergos/archlinux-base-devel')
except Exception:
pass
mpkg = build_makepkg()
if not mpkg:
return False
db.setex('antbs:docker-images:base-devel:built-today', 84600, 'True')
return True
else:
return False
def get_player_action(self, engine_idx):
engine = self.engines[engine_idx]
try:
return self.players[engine_idx].get_action(engine, engine.shape,
engine.anchor,
engine.board)
except Exception as err:
logger.error(err)
import pdb
pdb.set_trace()
logger.error(err)
def _validateSeed(self, value):
"""Callback to set random seed"""
logger.debug("Validating random seed {}".format(value))
try:
value = int(value)
except:
logger.error("Could not parse int for seed {}".format(value))
raise ValueError("Could not parse int for seed {}".format(value))
logger.info("Using random seed {}".format(value))
random.seed(value)
def batch_sign(paths, uid=gpg_key, passphrase=password, is_iso=False):
"""
Batch sign several files with the key matching the given UID.
If no passphrase is given then the user is prompted for one.
The passphrase is returned to avoid further prompts.
:param paths:
:param uid:
:param passphrase:
"""
if not isinstance(paths, list):
logger.error('paths must be a list')
return False
for path in paths:
db.publish('build-output', 'Creating detached signature for %s' % path)
logger.info('[SIGN PKG] Creating detached signature for %s' % path)
# Verify existing signatures. This fails if the sig is invalid or
# non-existent. Either way a new one will be needed.
cmd = [GPG_BIN, '--verify', path + SIG_EXT]
with open(os.devnull, 'w') as f:
p = subprocess.Popen(cmd, stdout=f, stderr=f)
e = p.wait()
if e == 0:
continue
sigpath = path + '.sig'
try:
os.remove(sigpath)
except OSError:
pass
db.publish('build-output', 'Signing %s' % path)
logger.info('[SIGN PKG] Signing %s' % path)
if not passphrase:
return False
# passphrase = getpass.getpass("Enter passphrase for %s: " % uid).encode('utf-8')
cmd = [GPG_BIN, '-sbu', 'Antergos', '--batch', '--passphrase-fd', '0', path]
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate(passphrase)
if len(out) > 0:
db.publish('build-output', 'GPG OUTPUT is: %s' % out)
logger.info('GPG OUTPUT is: %s' % out)
if len(err) > 0:
db.publish('build-output', 'Signing FAILED for %s. Error output: %s' % (path, err))
logger.error('[SIGN PKG] Signing FAILED for %s. Error output: %s' % (path, err))
paths = [p for p in paths if not os.path.isdir(p) and not is_iso]
for p in paths:
remove(p)
remove(p + '.sig')
return False
return True
def SaveNumpyByNpy(self):
#=========================================================================
# 把cache save到npy文件
#=========================================================================
if len(self.precal_index) != 0 and len(self.precal_value) != 0:
np.save(self.index_path,
np.array(self.precal_index).astype(np.int32))
np.save(self.value_path,
np.array(self.precal_value).astype(np.float32))
else:
logger.error('DataProcess: data flow is empty!!!')
raise Exception('DataProcess: data flow is empty!!!')
def grade():
'''
Get test results of all students in src/students/
'''
# Get path of this file
dir_path = os.path.dirname(os.path.realpath(__file__))
# Save results to a dict
results = {}
student_ids = os.listdir(os.path.join(dir_path, 'students'))
student_ids = [x[:-3] for x in student_ids if x[-3:] == '.py']
for student_id in student_ids:
student_result = {}
student_module = None
try:
student_module = importlib.import_module(
f'students.{student_id}') # NOQA
except Exception as err:
logger.info(err, exc_info=True)
student_result['import'] = "Failed"
else:
student_result['import'] = "Success"
# Check each task
for task_id in range(1, TASK_NUM + 1):
logger.info(f"Testing {student_id} Task {task_id}")
try:
eval(f"student_module.task_{task_id}()")
except Exception as err:
logger.error(err, exc_info=True)
student_result[f"task_{task_id}"] = "WA"
else:
student_result[f"task_{task_id}"] = "AC"
# Check flake8
style_guide = flake8.get_style_guide()
student_file = os.path.join(dir_path, 'students', student_id + '.py')
report = style_guide.check_files([student_file])
if (report.get_statistics('E') == []
and report.get_statistics('W') == []):
logger.info(report.get_statistics('E'))
logger.info(report.get_statistics('W'))
student_result['flake8'] = "Pass"
else:
student_result['flake8'] = "Fail"
results[student_id] = student_result
return {
"results": results,
"task_num": TASK_NUM,
"student_num": len(student_ids)
}
def bulksend(self, msglist):
"""
Accepts list, msglist, and adds to the output buffer. If the buffer exceeds MAXQUEUELENGTH, then flush.
"""
try:
self._queue.extend(msglist)
if len(self._queue) >= self.MAXQUEUELENGTH:
self.flush()
except Exception as e:
# We don't want to exit if there's a single bad event
logger.error("Caught Exception {} while appending/flushing output queue. There may be a ".format(e) +
"faulty event or token replacement in your sample.")
def push_to_hub(repo=None):
if repo is None:
return
try:
doc.login(username=doc_user, password=doc_pass, email='*****@*****.**')
response = [line for line in doc.push(repo, stream=True, insecure_registry=True)]
if not response:
logger.info('Pushing to Docker hub might not have completed successfully.')
else:
logger.info(response)
except Exception as err:
logger.error('Pushing to docker hub failed with error: %s', err)
def build_makepkg():
dockerfile = os.path.join(DOC_DIR, 'makepkg')
try:
build_it = [line for line in
doc.build(dockerfile, 'antergos/makepkg', quiet=False, nocache=True, rm=True,
stream=True, forcerm=True)]
if build_it:
push_to_hub('antergos/makepkg')
except Exception as err:
logger.error('@@-docker_util.py-@@ | Building makepkg failed with error: %s', err)
return False
return True
def build_mkarchiso():
dockerfile = '/opt/archlinux-mkarchiso'
try:
build_it = [line for line in
doc.build(dockerfile, tag='antergos/mkarchiso', quiet=False, nocache=True,
rm=True,
stream=True, forcerm=True)]
if build_it:
push_to_hub('antergos/mkarchiso')
except Exception as err:
logger.error('@@-docker_util.py-@@ | Building makepkg failed with error: %s', err)
return False
return True
def send_message(self, message):
# send some messages
logger.info("[MQ] ready to send message, queue: %s, message: %s" %
(settings.ALIYUN_MNS_QUEUE_NAME, message))
try:
msg = Message(message)
re_msg = self.my_queue.send_message(msg)
logger.info(
"[MQ] send message success, queue: %s, MessageID: %s, message: %s"
% (settings.ALIYUN_MNS_QUEUE_NAME, re_msg.message_id, message))
except MNSExceptionBase, e:
if e.type == "QueueNotExist":
logger.error(
"[MQ] Send Message Fail! Queue not exist, queue name:%s" %
settings.ALIYUN_MNS_QUEUE_NAME)
logger.error("[MQ] Send Message Fail! Exception:%s" % e)
def draw_text(
img: Image,
text: str,
location: tuple = (0, 0),
text_color=(0, 0, 0)
) -> Image:
draw = ImageDraw.Draw(img)
try:
# For Linux
font = ImageFont.truetype("DejaVuSans.ttf", 20)
except Exception as err:
logger.error(err, exc_info=True)
# For others
font = ImageFont.load_default()
draw.text(location, text, font=font, fill=text_color)
return img
def main(args):
mydb = mysql.connector.connect(host="localhost",
user=args.user,
passwd=args.passwd,
database="starwar")
mycursor = mydb.cursor()
with open(args.answer_shapes_path, 'r') as fin:
answer_shapes = json.load(fin)
result_shapes = {}
shape_correct = 0
error = 0
with open(args.file_path, 'r') as fin:
lines = fin.readlines()
for i, line in enumerate(lines):
try:
mycursor.execute(line)
except Exception as err:
error += 1
logger.error(err, exc_info=True)
else:
results = mycursor.fetchall()
for x in results:
logger.info(x)
result_shapes[i + 1] = np.array(results).shape
result_shape = np.array(results).shape
answer_shape = answer_shapes[f"{i+1}"]
if answer_shape != list(result_shape):
logger.error(
f"Question {i+1} shape not match: "
f"yours: {result_shape} / ans: {answer_shape}")
else:
shape_correct += 1
logger.info(f"Question {i+1} shape correct")
logger.info("-------------------------------------------------------")
logger.info(f"Shape correct: {shape_correct} / {len(answer_shapes)}")
logger.info(f"Error: {error} / {len(answer_shapes)}")
logger.warning("Note that this checker only checks the shape."
"Your answer may still be wrong.")
logger.warning("The answer is not guaranteed to be correct as well; "
"open a issue if you think the answer shape is incorrect.")
def updateConfig(self, config):
OutputPlugin.updateConfig(self, config)
try:
if hasattr(self.config, 'httpeventServers') is False:
if hasattr(self._sample, 'httpeventServers'):
self.config.httpeventServers = self._sample.httpeventServers
else:
logger.error(
'outputMode %s but httpeventServers not specified for sample %s'
% (self.name, self._sample.name))
raise NoServers(
'outputMode %s but httpeventServers not specified for sample %s'
% (self.name, self._sample.name))
# set default output mode to round robin
if hasattr(
self.config,
'httpeventOutputMode') and self.config.httpeventOutputMode:
self.httpeventoutputmode = config.httpeventOutputMode
else:
if hasattr(self._sample, 'httpeventOutputMode'
) and self._sample.httpeventOutputMode:
self.httpeventoutputmode = self._sample.httpeventOutputMode
else:
self.httpeventoutputmode = 'roundrobin'
if hasattr(self.config, 'httpeventMaxPayloadSize'
) and self.config.httpeventMaxPayloadSize:
self.httpeventmaxsize = self.config.httpeventMaxPayloadSize
else:
if hasattr(self._sample, 'httpeventMaxPayloadSize'
) and self._sample.httpeventMaxPayloadSize:
self.httpeventmaxsize = self._sample.httpeventMaxPayloadSize
else:
self.httpeventmaxsize = 10000
logger.debug("Currentmax size: %s " % self.httpeventmaxsize)
if isinstance(config.httpeventServers, str):
self.httpeventServers = json.loads(config.httpeventServers)
else:
self.httpeventServers = config.httpeventServers
logger.debug("Setting up the connection pool for %s in %s" %
(self._sample.name, self._app))
self.createConnections()
logger.debug("Pool created.")
logger.debug("Finished init of %s plugin." % self.name)
except Exception as e:
logger.exception(str(e))
def _validateTimezone(self, value):
"""Callback for complexSetting timezone which will parse and validate the timezone"""
logger.debug("Parsing timezone {}".format(value))
if value.find('local') >= 0:
value = datetime.timedelta(days=1)
else:
try:
# Separate the hours and minutes (note: minutes = the int value - the hour portion)
if int(value) > 0:
mod = 100
else:
mod = -100
value = datetime.timedelta(hours=int(int(value) / 100.0), minutes=int(value) % mod)
except:
logger.error("Could not parse timezone {}".format(value))
raise ValueError("Could not parse timezone {}".format(value))
logger.debug("Parsed timezone {}".format(value))
return value
def get_backfill_time(self, current_time):
if not current_time:
current_time = self.now()
if not self.backfill:
return current_time
else:
if self.backfill[0] == '-':
backfill_time = self.backfill[1:-1]
time_unit = self.backfill[-1]
if self.backfill[-2:] == 'ms':
time_unit = 'ms'
backfill_time = self.backfill[1:-2]
return self.get_time_difference(current_time=current_time,
different_time=backfill_time,
sign='-',
time_unit=time_unit)
else:
logger.error("Backfill time is not in the past.")
return current_time
def __init__(self, sample, output_counter=None):
OutputPlugin.__init__(self, sample, output_counter)
if sample.fileName is None:
logger.error(
'outputMode file but file not specified for sample %s' %
self._sample.name)
raise ValueError(
'outputMode file but file not specified for sample %s' %
self._sample.name)
self._file = sample.pathParser(sample.fileName)
self._fileMaxBytes = sample.fileMaxBytes
self._fileBackupFiles = sample.fileBackupFiles
self._fileHandle = open(self._file, 'a')
self._fileLength = os.stat(self._file).st_size
logger.debug(
"Configured to log to '%s' with maxBytes '%s' with backupCount '%s'"
% (self._file, self._fileMaxBytes, self._fileBackupFiles))
def autograde(student_id, tasks):
''' Grade tasks specified in args.'''
# Get path of this file
dir_path = os.path.dirname(os.path.realpath(__file__))
# Check if student's file exists
student_file = os.path.join(dir_path, 'students', student_id + '.py')
assert os.path.exists(student_file), f"{student_file} not exists"
# Import student's file as module
student_module = importlib.import_module(f'students.{student_id}') # NOQA
# Run each task
for task_id in tasks:
logger.info(f"Testing Task {task_id}")
# Use try-except to catch erros in order to run througth all tasks
try:
eval(f"student_module.task_{task_id}()")
except Exception as err:
logger.error(err, exc_info=True)
def createConnections(self):
self.serverPool = []
if self.httpeventServers:
for server in self.httpeventServers.get('servers'):
if not server.get('address'):
logger.error(
'requested a connection to a httpevent server, but no address specified for sample %s'
% self._sample.name)
raise ValueError(
'requested a connection to a httpevent server, but no address specified for sample %s'
% self._sample.name)
if not server.get('port'):
logger.error(
'requested a connection to a httpevent server, but no port specified for server %s'
% server)
raise ValueError(
'requested a connection to a httpevent server, but no port specified for server %s'
% server)
if not server.get('key'):
logger.error(
'requested a connection to a httpevent server, but no key specified for server %s'
% server)
raise ValueError(
'requested a connection to a httpevent server, but no key specified for server %s'
% server)
if not ((server.get('protocol') == 'http') or
(server.get('protocol') == 'https')):
logger.error(
'requested a connection to a httpevent server, but no protocol specified for server %s'
% server)
raise ValueError(
'requested a connection to a httpevent server, but no protocol specified for server %s'
% server)
logger.debug(
"Validation Passed, Creating a requests object for server: %s"
% server.get('address'))
setserver = {}
setserver['url'] = "%s://%s:%s/services/collector" % (
server.get('protocol'), server.get('address'),
server.get('port'))
setserver['header'] = "Splunk %s" % server.get('key')
logger.debug("Adding server set to pool, server: %s" %
setserver)
self.serverPool.append(setserver)
else:
raise NoServers(
'outputMode %s but httpeventServers not specified for sample %s'
% (self.name, self._sample.name))
def remove(src):
"""
:param src:
:return:
"""
if src != str(src):
return True
if os.path.isdir(src):
try:
shutil.rmtree(src)
except Exception as err:
logger.error(err)
return True
elif os.path.isfile(src):
try:
os.remove(src)
except Exception as err:
logger.error(err)
return True
else:
return True
def _DataMerge(index_list, value_list, save_list):
# =========================================================================
# 归并各个进程处理得到的数据块,保存为npz压缩格式
# =========================================================================
index, value = [], []
try:
for index_path, value_path in zip(index_list, value_list):
with open(index_path, 'rb') as r_index:
index.append(np.load(r_index))
with open(value_path, 'rb') as r_value:
value.append(np.load(r_value))
# 获取文件数据之后自动删除文件
os.system('rm {}'.format(index_path))
os.system('rm {}'.format(value_path))
index = np.concatenate(index, 0)
value = np.concatenate(value, 0)
np.savez_compressed(save_list[0], index=index)
np.savez_compressed(save_list[1], value=value)
except IOError:
logger.error('_DataMerge: file path not exists!!!')
raise Exception('_DataMerge: fail to open file for merging!!!')
def _DataSplit():
# =========================================================================
# 从thin_all_sort分离出所需要的数据，并保存为tuple形式(x,y,z)
# =========================================================================
count = 0
head_list = []
try:
with open(setting.sorted_coord_path, 'rb') as reader:
for line in reader:
x, y, z = map(int, line.split("\t")[0].split("_"))
head_list.append((x, y, z))
count += 1
if count >= setting.num_hot_points:
break
except IOError:
logger.error('_DataSplit: file {} not exists'.format(
setting.sorted_coord_path))
raise Exception('_DataSplit: fail to open {}'.format(
setting.sorted_coord_path))
# 对(x,y,z)按照坐标元组排序(升序)
head_list.sort()
return head_list
def _transmitEvents(self, payloadstring):
logger.debug(
"Transmission called with payloadstring event number: %d " %
len(payloadstring))
records = "".join(payloadstring)
# Different key prefix for different log type
if self.awsS3eventtype == 'elbaccesslog':
s3keyname = self.awsS3objectprefix + datetime.datetime.utcnow(
).strftime("%Y%m%dT%H%MZ") + '_' + str(
uuid.uuid1()) + self.awsS3objectsuffix
elif self.awsS3eventtype == 's3accesslog':
s3keyname = self.awsS3objectprefix + datetime.datetime.utcnow(
).strftime("%Y-%m-%d-%H-%M-%S") + '-' + str(uuid.uuid1()).replace(
'-', '').upper()[0:15] + self.awsS3objectsuffix
else:
s3keyname = self.awsS3objectprefix + datetime.datetime.utcnow(
).isoformat() + str(uuid.uuid1()) + self.awsS3objectsuffix
logger.debug("Uploading %d events into s3 key: %s " %
(len(records), s3keyname))
if self.awsS3compressiontype == 'gz':
import StringIO
import gzip
out = StringIO.StringIO()
with gzip.GzipFile(fileobj=out, mode="w") as f:
f.write(records)
records = out.getvalue()
try:
response = self._client.put_object(Bucket=self.awsS3bucketname,
Key=s3keyname,
Body=records)
logger.debug("response = %s" % response)
except Exception as e:
logger.error("Failed for exception: %s" % e)
logger.debug("Failed sending events to payload: %s" %
(payloadstring))
raise e
def getSplunkUrl(self, s):
"""
If we're embedded in Splunk, get it from Splunk's Python libraries, otherwise get it from config.
Returns a tuple of ( splunkUrl, splunkMethod, splunkHost, splunkPort )
"""
if self.splunkEmbedded:
try:
import splunk.auth
splunkUrl = splunk.auth.splunk.getLocalServerInfo()
results = re.match('(http|https)://([^:/]+):(\d+).*', splunkUrl)
splunkMethod = results.groups()[0]
splunkHost = results.groups()[1]
splunkPort = results.groups()[2]
except:
import traceback
trace = traceback.format_exc()
logger.error(
'Error parsing host from splunk.auth.splunk.getLocalServerInfo() for sample %s. Stacktrace: %s' %
(s.name, trace))
raise ValueError(
'Error parsing host from splunk.auth.splunk.getLocalServerInfo() for sample %s' % s.name)
else:
# splunkMethod and splunkPort are defaulted so only check for splunkHost
if s.splunkHost is None:
logger.error("Splunk URL Requested but splunkHost not set for sample '%s'" % s.name)
raise ValueError("Splunk URL Requested but splunkHost not set for sample '%s'" % s.name)
splunkUrl = '%s://%s:%s' % (s.splunkMethod, s.splunkHost, s.splunkPort)
splunkMethod = s.splunkMethod
splunkHost = s.splunkHost
splunkPort = s.splunkPort
logger.debug(
"Getting Splunk URL: %s Method: %s Host: %s Port: %s" % (splunkUrl, splunkMethod, splunkHost, splunkPort))
return (splunkUrl, splunkMethod, splunkHost, splunkPort)
def flush(self, q):
logger.debug("Flush called on awsS3 plugin with length %d" % len(q))
if len(q) > 0:
try:
payload = []
logger.debug("Currently being called with %d events" % len(q))
for event in q:
if event.get('_raw') is None:
logger.error(
'failure outputting event, does not contain _raw')
else:
payload.append(event['_raw'])
logger.debug(
"Finished processing events, sending all to AWS S3")
self._sendPayloads(payload)
except Exception as e:
import traceback
logger.error(traceback.print_exc())
logger.error('failed sending events, reason: %s ' % e)
def _createConnections(self, sample):
try:
if hasattr(sample, 'awsKeyId') and hasattr(sample, 'awsSecretKey'):
self._client = boto3.client(
"s3",
region_name=sample.awsRegion,
aws_access_key_id=sample.awsKeyId,
aws_secret_access_key=sample.awsSecretKey)
if self._client is None:
msg = '''
[your_eventgen_stanza]
awsKeyId = YOUR_ACCESS_KEY
awsSecretKey = YOUR_SECRET_KEY
'''
logger.error(
"Failed for init boto3 client: %s, you should define correct 'awsKeyId'\
and 'awsSecretKey' in eventgen conf %s" % msg)
raise Exception(msg)
else:
self._client = boto3.client('s3', region_name=sample.awsRegion)
except Exception as e:
logger.error("Failed for init boto3 client: exception = %s" % e)
raise e
# Try list bucket method to validate if the connection works
try:
self._client.list_buckets()
except botocore.exceptions.NoCredentialsError:
msg = '''
[default]
aws_access_key_id = YOUR_ACCESS_KEY
aws_secret_access_key = YOUR_SECRET_KEY
'''
logger.error("Failed for init boto3 client, you should create "
"'~/.aws/credentials' with credential info %s" % msg)
raise
logger.debug("Init conn done, conn = %s" % self._client)
def autograde(student_id, tasks, test_data_filename, test_answers_filename):
''' Grade tasks specified in args.'''
# Get path of this file
# Check if student's file exists
student_file = os.path.join(DIR_PATH, 'students', student_id + '.py')
assert os.path.exists(student_file), f"{student_file} not exists"
# Import student's file as module
student_module = importlib.import_module(f'students.{student_id}') # NOQA
# Load testing data
test_data = parse_yaml(test_data_filename)
# Load testing answers
test_answers = parse_yaml(test_answers_filename)
# Run each task
points = {}
for task_id in tasks:
logger.info(f"Testing Task {task_id}")
# Use try-except to catch erros in order to run througth all tasks
try:
# This part is a bit dirty. If you have a better way, send a PR to
# improve!
if task_id == 6:
answer = test_answers[task_id]['answer']
result = eval(
f"student_module.task_{task_id}(**{test_data[task_id]})")
if utils.floating_judge(result, answer):
points[task_id] = test_answers[task_id]['points']
else:
logger.error(f"Your result {result}")
logger.error(f"is different from ")
logger.error(f"{test_answers[task_id]['answer']}")
points[task_id] = 0
elif task_id == 7:
time = strftime("%Y-%m-%d %H:%M:%S", gmtime())
student = student_module.task_7(student_id, time)
assert student.student_id == student_id
assert student.time == time
assert student.words_to_say != "initial value"
points[task_id] = test_answers[task_id]['points']
elif task_id == 8:
image = student_module.task_8()
assert str(type(image))[8:11] == 'PIL', type(image)
points[task_id] = test_answers[task_id]['points']
else:
result = eval(
f"student_module.task_{task_id}(**{test_data[task_id]})")
# No check
if test_answers[task_id]['check'] == 0:
points[task_id] = test_answers[task_id]['points']
# Check equality
elif test_answers[task_id]['check'] == 1:
if result == test_answers[task_id]['answer']:
points[task_id] = test_answers[task_id]['points']
else:
logger.error(f"Your result {result}")
logger.error("is different from ")
logger.error(f"{test_answers[task_id]['answer']}")
points[task_id] = 0
# Check list equality
elif test_answers[task_id]['check'] == 2:
if set(result) == set(test_answers[task_id]['answer']):
points[task_id] = test_answers[task_id]['points']
else:
logger.error(f"Your result {result}")
logger.error("is different from ")
logger.error(f"{test_answers[task_id]['answer']}")
points[task_id] = 0
# Other checks (should not be processed here)
else:
points[task_id] = None
except Exception as err:
points[task_id] = 0
logger.error(err, exc_info=True)
logger.info(f"TaskID/Points {points}")
return points
def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, pivot_timestamp=None):
if self.replacementType == 'static':
return self.replacement
# This logic is done in replay.py
elif self.replacementType == 'replaytimestamp':
pass
elif self.replacementType == 'timestamp':
if s.earliest and s.latest:
if earliestTime and latestTime:
if latestTime >= earliestTime:
if pivot_timestamp:
replacementTime = pivot_timestamp
elif s.timestamp is None:
minDelta = 0
# Compute timeDelta as total_seconds
td = latestTime - earliestTime
if not type(td) == float:
maxDelta = timeDelta2secs(td)
else:
maxDelta = td
# Get random timeDelta
randomDelta = datetime.timedelta(
seconds=random.randint(minDelta, maxDelta), microseconds=random.randint(
0, latestTime.microsecond if latestTime.microsecond > 0 else 999999))
# Compute replacmentTime
replacementTime = latestTime - randomDelta
s.timestamp = replacementTime
else:
replacementTime = s.timestamp
replacement = self.replacement.replace(
'%s',
str(round(time.mktime(replacementTime.timetuple()))).rstrip('0').rstrip('.'))
replacementTime = replacementTime.strftime(replacement)
# replacementTime == replacement for invalid strptime specifiers
if replacementTime != self.replacement.replace('%', ''):
return replacementTime
else:
logger.error(
"Invalid strptime specifier '%s' detected; will not replace" % (self.replacement))
return old
# earliestTime/latestTime not proper
else:
logger.error(("Earliest specifier '%s', value '%s' is greater than latest specifier '%s'" +
"value '%s' for sample '%s'; will not replace") %
(s.earliest, earliestTime, s.latest, latestTime, s.name))
return old
# earliest/latest not proper
else:
logger.error('Earliest or latest specifier were not set; will not replace')
return old
elif self.replacementType in ('random', 'rated'):
# Validations:
if self._integerMatch is not None:
integerMatch = self._integerMatch
else:
integerRE = re.compile('integer\[([-]?\d+):([-]?\d+)\]', re.I)
integerMatch = integerRE.match(self.replacement)
self._integerMatch = integerMatch
if self._floatMatch is not None:
floatMatch = self._floatMatch
else:
floatRE = re.compile('float\[(-?\d+|-?\d+\.(\d+)):(-?\d+|-?\d+\.(\d+))\]', re.I)
floatMatch = floatRE.match(self.replacement)
self._floatMatch = floatMatch
if self._stringMatch is not None:
stringMatch = self._stringMatch
else:
stringRE = re.compile('string\((\d+)\)', re.I)
stringMatch = stringRE.match(self.replacement)
self._stringMatch = stringMatch
if self._hexMatch is not None:
hexMatch = self._hexMatch
else:
hexRE = re.compile('hex\((\d+)\)', re.I)
hexMatch = hexRE.match(self.replacement)
self._hexMatch = hexMatch
if self._listMatch is not None:
listMatch = self._listMatch
else:
listRE = re.compile('list(\[[^\]]+\])', re.I)
listMatch = listRE.match(self.replacement)
self._listMatch = listMatch
# Valid replacements: ipv4 | ipv6 | integer[<start>:<end>] | string(<i>)
if self.replacement.lower() == 'ipv4':
x = 0
replacement = ''
while x < 4:
replacement += str(random.randint(0, 255)) + '.'
x += 1
replacement = replacement.strip('.')
return replacement
elif self.replacement.lower() == 'ipv6':
x = 0
replacement = ''
while x < 8:
replacement += hex(random.randint(0, 65535))[2:] + ':'
x += 1
replacement = replacement.strip(':')
return replacement
elif self.replacement.lower() == 'mac':
x = 0
replacement = ''
# Give me 6 blocks of 2 hex
while x < 6:
y = 0
while y < 2:
replacement += hex(random.randint(0, 15))[2:]
y += 1
replacement += ':'
x += 1
replacement = replacement.strip(':')
return replacement
elif self.replacement.lower() == 'guid':
return str(uuid.uuid4())
elif integerMatch:
startInt = int(integerMatch.group(1))
endInt = int(integerMatch.group(2))
if endInt >= startInt:
replacementInt = random.randint(startInt, endInt)
if self.replacementType == 'rated':
rateFactor = 1.0
if type(s.hourOfDayRate) == dict:
try:
rateFactor *= s.hourOfDayRate[str(s.now())]
except KeyError:
import traceback
stack = traceback.format_exc()
logger.error("Hour of day rate failed for token %s. Stacktrace %s" % stack)
if type(s.dayOfWeekRate) == dict:
try:
weekday = datetime.date.weekday(s.now())
if weekday == 6:
weekday = 0
else:
weekday += 1
rateFactor *= s.dayOfWeekRate[str(weekday)]
except KeyError:
import traceback
stack = traceback.format_exc()
logger.error("Day of week rate failed. Stacktrace %s" % stack)
replacementInt = int(round(replacementInt * rateFactor, 0))
replacement = str(replacementInt)
return replacement
else:
logger.error(
"Start integer %s greater than end integer %s; will not replace" % (startInt, endInt))
return old
elif floatMatch:
try:
startFloat = float(floatMatch.group(1))
endFloat = float(floatMatch.group(3))
significance = 0
if floatMatch.group(2) is not None:
significance = len(floatMatch.group(2))
if endFloat >= startFloat:
floatret = round(random.uniform(startFloat, endFloat), significance)
if self.replacementType == 'rated':
rateFactor = 1.0
now = s.now()
if type(s.hourOfDayRate) == dict:
try:
rateFactor *= s.hourOfDayRate[str(now.hour)]
except KeyError:
import traceback
stack = traceback.format_exc()
logger.error("Hour of day rate failed for token %s. Stacktrace %s" % stack)
if type(s.dayOfWeekRate) == dict:
try:
weekday = datetime.date.weekday(now)
if weekday == 6:
weekday = 0
else:
weekday += 1
rateFactor *= s.dayOfWeekRate[str(weekday)]
except KeyError:
import traceback
stack = traceback.format_exc()
logger.error("Day of week rate failed. Stacktrace %s" % stack)
floatret = round(floatret * rateFactor, significance)
floatret = str(floatret)
return floatret
else:
logger.error(
"Start float %s greater than end float %s; will not replace" % (startFloat, endFloat))
return old
except ValueError:
logger.error("Could not parse float[%s:%s]" % (floatMatch.group(1), floatMatch.group(4)))
return old
elif stringMatch:
strLength = int(stringMatch.group(1))
if strLength == 0:
return ''
elif strLength > 0:
replacement = ''
while len(replacement) < strLength:
# Generate a random ASCII between dec 33->126
replacement += chr(random.randint(33, 126))
# Practice safe strings
replacement = re.sub('%[0-9a-fA-F]+', '', urllib.quote(replacement))
return replacement
else:
logger.error(
"Length specifier %s for string replacement must be greater than 0; will not replace" %
(strLength))
return old
elif hexMatch:
strLength = int(hexMatch.group(1))
replacement = ''
hexList = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F']
while len(replacement) < strLength:
replacement += hexList[random.randint(0, 15)]
return replacement
elif listMatch:
try:
value = json.loads(listMatch.group(1))
except:
logger.error("Could not parse json for '%s' in sample '%s'" % (listMatch.group(1), s.name))
return old
return random.choice(value)
else:
logger.error("Unknown replacement value '%s' for replacementType '%s'; will not replace" %
(self.replacement, self.replacementType))
return old
elif self.replacementType in ('file', 'mvfile', 'seqfile'):
if self._replacementFile is not None:
replacementFile = self._replacementFile
replacementColumn = self._replacementColumn
else:
try:
paths = self.replacement.split(':')
if (len(paths) == 1):
replacementColumn = 0
else:
try: # When it's not a mvfile, there's no number on the end:
replacementColumn = int(paths[-1])
except (ValueError):
replacementColumn = 0
if (replacementColumn > 0):
# This supports having a drive-letter colon
replacementFile = s.pathParser(":".join(paths[0:-1]))
else:
replacementFile = s.pathParser(self.replacement)
except ValueError:
logger.error(
"Replacement string '%s' improperly formatted. Should be /path/to/file or /path/to/file:column"
% self.replacement)
return old
self._replacementFile = replacementFile
self._replacementColumn = replacementColumn
# If we've seen this file before, simply return already read results
# This applies only if we're looking at a multivalue file and we want to
# return the same random pick on every iteration
if replacementColumn > 0 and replacementFile in self.mvhash:
if replacementColumn > len(self.mvhash[replacementFile]):
logger.error("Index for column '%s' in replacement file '%s' is out of bounds" %
(replacementColumn, replacementFile))
return old
else:
# logger.debug("Returning mvhash: %s" % self.mvhash[replacementFile][replacementColumn-1])
return self.mvhash[replacementFile][replacementColumn - 1]
else:
# Adding caching of the token file to avoid reading it every iteration
if self._tokenfile is not None:
replacementLines = self._tokenfile
# Otherwise, lets read the file and build our cached results, pick a result and return it
else:
# logger.debug("replacementFile: %s replacementColumn: %s" %
# (replacementFile, replacementColumn))
replacementFile = os.path.abspath(replacementFile)
logger.debug("Normalized replacement file %s" % replacementFile)
if os.path.exists(replacementFile) and os.path.isfile(replacementFile):
replacementFH = open(replacementFile, 'rU')
replacementLines = replacementFH.readlines()
replacementFH.close()
if len(replacementLines) == 0:
logger.error("Replacement file '%s' is empty; will not replace" % (replacementFile))
return old
else:
self._tokenfile = replacementLines
else:
logger.error("File '%s' does not exist" % (replacementFile))
return old
if self.replacementType == 'seqfile':
# pick value one by one from replacement file
replacement = replacementLines[self._tokenfilecounter % len(replacementLines)].strip()
self._tokenfilecounter += 1
else:
# pick value randomly from replacement file
replacement = replacementLines[random.randint(0, len(replacementLines) - 1)].strip()
if replacementColumn > 0:
self.mvhash[replacementFile] = replacement.split(',')
if replacementColumn > len(self.mvhash[replacementFile]):
logger.error("Index for column '%s' in replacement file '%s' is out of bounds" %
(replacementColumn, replacementFile))
return old
else:
return self.mvhash[replacementFile][replacementColumn - 1]
else:
return replacement
elif self.replacementType == 'integerid':
temp = self.replacement
self.replacement = str(int(self.replacement) + 1)
return temp
else:
logger.error("Unknown replacementType '%s'; will not replace" % self.replacementType)
return old
doc_user = status.docker_user
doc_pass = status.docker_password
SRC_DIR = os.path.dirname(__file__) or '.'
DOC_DIR = os.path.abspath(os.path.join(SRC_DIR, '..', 'build/docker'))
BUILD_DIR = os.path.abspath(os.path.join(DOC_DIR, '..'))
# logger.debug([('SRC_DIR', SRC_DIR), ('DOC_DIR', DOC_DIR), ('BUILD_DIR', BUILD_DIR)])
# Initiate communication with build daemon
try:
doc = docker.Client(base_url='unix://var/run/docker.sock', version='auto')
# doc.build(path=DOC_DIR, tag="arch-devel", quiet=False, timeout=None)
except Exception as err:
logger.error("Cant connect to Docker daemon. Error msg: %s", err)
def create_pkgs_host_config(cache, pkgbuild_dir, result):
"""
:param cache:
:param pkgbuild_dir:
:param result:
:return:
"""
pkgs_hconfig = create_host_config(
binds={
cache:
{
'bind': '/var/cache/pacman',
def make_celery(app):
celery = Celery(app.import_name, broker=app.config['CELERY_BROKER_URL'],
backend=app.config['CELERY_RESULT_BACKEND'])
celery.conf.update(app.config)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
celery = make_celery(app)
'''
import routes
try:
db.create_all()
except Exception as e:
logger.error("init db error", e)
pass
if __name__ == '__main__':
app.run(port=5000, host='0.0.0.0', debug=True)
def autograde(student_id, tasks, test_data_filename, test_answers_filename):
''' Grade tasks specified in args.'''
# Get path of this file
# Check if student's file exists
student_file = os.path.join(DIR_PATH, 'students', student_id + '.py')
assert os.path.exists(student_file), f"{student_file} not exists"
# Import student's file as module
student_module = importlib.import_module(f'students.{student_id}') # NOQA
# Load testing data
test_data = parse_yaml(test_data_filename)
# Load testing answers
test_answers = parse_yaml(test_answers_filename)
# Run each task
points = {}
for task_id in tasks:
logger.info(f"Testing Task {task_id}")
# Use try-except to catch erros in order to run througth all tasks
try:
# This part is a bit dirty. If you have a better way, send a PR to
# improve!
if task_id == 7:
time = strftime("%Y-%m-%d %H:%M:%S", gmtime())
student = student_module.task_7(student_id, time)
assert student.student_id == student_id
assert student.time == time
assert student.words_to_say != "initial value"
points[task_id] = test_answers[task_id]['points']
elif task_id == 8:
image = student_module.task_8()
assert str(type(image))[8:11] == 'PIL', type(image)
points[task_id] = test_answers[task_id]['points']
else:
result = eval(
f"student_module.task_{task_id}(**{test_data[task_id]})")
# No check
if test_answers[task_id]['check'] == 0:
points[task_id] = test_answers[task_id]['points']
# Check equality
elif test_answers[task_id]['check'] == 1:
if result == test_answers[task_id]['answer']:
points[task_id] = test_answers[task_id]['points']
else:
logger.error(f"Your result {result}")
logger.error("is different from ")
logger.error(f"{test_answers[task_id]['answer']}")
points[task_id] = 0
# Check list equality
elif test_answers[task_id]['check'] == 2:
if set(result) == set(test_answers[task_id]['answer']):
points[task_id] = test_answers[task_id]['points']
else:
logger.error(f"Your result {result}")
logger.error("is different from ")
logger.error(f"{test_answers[task_id]['answer']}")
points[task_id] = 0
# Other checks (should not be processed here)
else:
points[task_id] = None
except Exception as err:
points[task_id] = 0
logger.error(err, exc_info=True)
logger.info(f"TaskID/Points {points}")
return points
def rate(self):
self._sample.count = int(self._sample.count)
# Let generators handle infinite count for themselves
if self._sample.count == -1 and self._sample.generator == 'default':
if not self._sample.sampleDict:
logger.error(
'No sample found for default generator, cannot generate events'
)
self._sample.count = len(self._sample.sampleDict)
self._generatorWorkers = int(self._generatorWorkers)
count = self._sample.count / self._generatorWorkers
# 5/8/12 CS We've requested not the whole file, so we should adjust count based on
# hourOfDay, dayOfWeek and randomizeCount configs
rateFactor = 1.0
if self._sample.randomizeCount:
try:
logger.debug(
"randomizeCount for sample '%s' in app '%s' is %s" %
(self._sample.name, self._sample.app,
self._sample.randomizeCount))
# If we say we're going to be 20% variable, then that means we
# can be .1% high or .1% low. Math below does that.
randBound = round(self._sample.randomizeCount * 1000, 0)
rand = random.randint(0, randBound)
randFactor = 1 + ((-((randBound / 2) - rand)) / 1000)
logger.debug("randFactor for sample '%s' in app '%s' is %s" %
(self._sample.name, self._sample.app, randFactor))
rateFactor *= randFactor
except:
import traceback
stack = traceback.format_exc()
logger.error(
"Randomize count failed for sample '%s'. Stacktrace %s" %
(self._sample.name, stack))
if type(self._sample.hourOfDayRate) == dict:
try:
rate = self._sample.hourOfDayRate[str(self._sample.now().hour)]
logger.debug(
"hourOfDayRate for sample '%s' in app '%s' is %s" %
(self._sample.name, self._sample.app, rate))
rateFactor *= rate
except KeyError:
import traceback
stack = traceback.format_exc()
logger.error(
"Hour of day rate failed for sample '%s'. Stacktrace %s" %
(self._sample.name, stack))
if type(self._sample.dayOfWeekRate) == dict:
try:
weekday = datetime.date.weekday(self._sample.now())
if weekday == 6:
weekday = 0
else:
weekday += 1
rate = self._sample.dayOfWeekRate[str(weekday)]
logger.debug(
"dayOfWeekRate for sample '%s' in app '%s' is %s" %
(self._sample.name, self._sample.app, rate))
rateFactor *= rate
except KeyError:
import traceback
stack = traceback.format_exc()
logger.error(
"Hour of day rate failed for sample '%s'. Stacktrace %s" %
(self._sample.name, stack))
if type(self._sample.minuteOfHourRate) == dict:
try:
rate = self._sample.minuteOfHourRate[str(
self._sample.now().minute)]
logger.debug(
"minuteOfHourRate for sample '%s' in app '%s' is %s" %
(self._sample.name, self._sample.app, rate))
rateFactor *= rate
except KeyError:
import traceback
stack = traceback.format_exc()
logger.error(
"Minute of hour rate failed for sample '%s'. Stacktrace %s"
% (self._sample.name, stack))
if type(self._sample.dayOfMonthRate) == dict:
try:
rate = self._sample.dayOfMonthRate[str(self._sample.now().day)]
logger.debug(
"dayOfMonthRate for sample '%s' in app '%s' is %s" %
(self._sample.name, self._sample.app, rate))
rateFactor *= rate
except KeyError:
import traceback
stack = traceback.format_exc()
logger.error(
"Day of Month rate for sample '%s' failed. Stacktrace %s"
% (self._sample.name, stack))
if type(self._sample.monthOfYearRate) == dict:
try:
rate = self._sample.monthOfYearRate[str(
self._sample.now().month)]
logger.debug(
"monthOfYearRate for sample '%s' in app '%s' is %s" %
(self._sample.name, self._sample.app, rate))
rateFactor *= rate
except KeyError:
import traceback
stack = traceback.format_exc()
logger.error(
"Month Of Year rate failed for sample '%s'. Stacktrace %s"
% (self._sample.name, stack))
ret = int(round(count * rateFactor, 0))
if rateFactor != 1.0:
logger.debug("Original count: %s Rated count: %s Rate factor: %s" %
(count, ret, rateFactor))
return ret
