def write_to_file(file_handle, lines):
"""
Write the provided lines to the provided file_handle. If an error occurs an error message will be logged to stdout.
:type lines: list
:type file_handle: file
:param file_handle: the file that the provided lines should be written
:param lines: the lines that should be written to the provided file.
:return: void
"""
if not file_handle or not isinstance(file_handle, file):
logging.error('Received an invalid file reference.')
return
if lines:
try:
if args.verbose:
logging.info("Writing {0} lines to {1}", len(lines), file_handle.name)
file_handle.writelines(lines)
if args.verbose:
logging.info("Successfully wrote {0} lines to {1}", len(lines), file_handle.name)
except IOError, e:
logging.error('There was an error while writing to {0}. {1}: {2}', file_handle.name, e.args[0], e.args[1])
def read_from_file(file_handle):
"""
Read the contents from the provided file_handle and return them. If there is an error then a message detailing the
problem will be written to stdout and None will be returned.
:type file_handle file
:param file_handle: the file object to read from.
:return: the contents of the file or, if an error is encountered, None.
"""
if file_handle and isinstance(file_handle, file):
# ADDING: verbose logging before the operation.
if args.verbose:
logging.info('Attempting to read in contents of {0}', file_handle.name)
try:
# ATTEMPT: to read the contents of the file.
contents = file_handle.read()
# PROVIDE: the user some verbose success logging.
if args.verbose:
logging.info('Successfully read the contents of {0}', file_handle.name)
# RETURN: the contents of the file.
return contents
except IOError, e:
logging.error('There was a problem reading from {0}. {1}: {2}', file_handle.name, e.args[0], e.args[1])
return None
def dataCleaning():
logging.info('===Data Cleaning Processing===')
input_file = DATA_TRAINING_SAMPLE
adClickCntList = generateTopAdsUsersByClick(input_file)
dumpList2File(adClickCntList, TMP_DATA_DIR_PATH + 'topAdClickCnt.dict')
adSet = set()
for line in file(TMP_DATA_DIR_PATH + 'topAdClickCnt.dict'):
cnt, adid = line.strip().split()
adSet.add(adid)
logging.debug(len(adSet))
ad2Users = generateAd2UsersGivenAdSet(input_file, adSet)
dumpDict2File(ad2Users, TMP_DATA_DIR_PATH + 'ad2UsersGivenAdSet.dict')
userDict = generateUser2AdGivenAd2User(TMP_DATA_DIR_PATH +
'ad2UsersGivenAdSet.dict',
adViewThreshold=10)
dumpDict2File(userDict, TMP_DATA_DIR_PATH + 'user2AdGivenAd2User.dict')
userSet = set()
logging.debug(len(userSet))
for line in file(TMP_DATA_DIR_PATH + 'user2AdGivenAd2User.dict'):
user, ads = line.strip().split('\x01')
userSet.add(user)
dumpUserRawFeatureGivenUserSet(input_file, userSet,
TMP_DATA_DIR_PATH + 'userRawFeature.dict')
def featureEngineering():
logging.info('===Feature Engineering Processing===')
query_set, desc_set, title_set = getUserFeatureSet()
aggregateUserfile = TMP_DATA_DIR_PATH + 'userRawFeature.dict'
expandId2TokensResultFile = TMP_DATA_DIR_PATH + 'userRawExpandTokens.dict'
expandFeatureId2Tokens(aggregateUserfile, expandId2TokensResultFile,
query_set, desc_set, title_set)
tmp_file = file(TMP_DATA_DIR_PATH + 'tmp', 'w')
fn_userRawExpandTokens = expandId2TokensResultFile
for line in file(fn_userRawExpandTokens):
userid, query, title, desc = line.strip().split('\x01')
tmp_file.write(query)
tmp_file.write('\n')
tmp_file.close()
lda = topicLDA.LDA(TMP_DATA_DIR_PATH + 'tmp')
lda.run(num_topics=200,
fn_bow=TMP_DATA_DIR_PATH + 'corpus.svmlight',
fn_out_topic=TMP_DATA_DIR_PATH + 'LDA_corpus.svmlight')
os.system('rm ' + TMP_DATA_DIR_PATH + 'tmp')
joinResult4SVMRanking(
fn_trainFeature=TMP_DATA_DIR_PATH + 'LDA_corpus.svmlight',
fn_ad2userStatus=TMP_DATA_DIR_PATH + 'ad2userStatus.dict',
fn_out_SVMRanking=TMP_DATA_DIR_PATH + 'finalData4SVMRanking.dat')
def run() :
Adset, UserSet = userStatusWorkflow.getPreSet(DATA_TRAINING)
for adid in Adset :
logging.info('Now handling AD:' + adid)
featureExtracter.workflow(adid, False, False)
featureExtracter.workflow(adid, True, False)
featureExtracter.workflow(adid, True, True)
def monitor_publish_request_count(p, t_pstart, append_if_p_killed):
while True:
if append_if_p_killed:
logging.warn(
'(monitor_publish_request_count) the process is to be restarted. breaking out of the publish request monitoring loop'
)
break
logging.info(
'(monitor_publish_request_count) request_count: {c}'.format(
c=request_count))
t = datetime.datetime.now()
dt = t - t_pstart
if dt.seconds < 60 * 10 and request_count == 0:
logging.info(
'(monitor_publish_request_count) request count is zero but still in the warm up period: {s} seconds'
.format(s=dt.seconds))
elif dt.seconds > 60 * 10 and request_count == 0:
logging.warn(
'(monitor_publish_request_count) request count is zero')
try:
logging.warn(
'(monitor_publish_request_count) restarting as publish req count is zero after warnup'
)
append_if_p_killed.append("dummy")
os.killpg(os.getpgid(p.pid), signal.SIGKILL
) # Send the signal to all the process groups
except Exception as ex:
print(ex)
break
time.sleep(10)
def rankingFeatureSelection () :
adset = set([line.strip().split()[1] for line in file(TMP_DATA_DIR_PATH+'topAdClickCnt.dict.final')])
#adset = set(list(adset)[:2])
blacklist = set(['20174985','3834142','3373964','4344041','8350700','2878230','3803920','20174982','4341158','6434934', '3219148','20035409'])
adset = adset - blacklist
feature = TMP_DATA_DIR_PATH + 'feature/%s.bm25.feature'
featureTransfer = TMP_DATA_DIR_PATH + 'feature/%s.bm25.feature.transfer'
featureTransferRelevance = TMP_DATA_DIR_PATH + 'feature/%s.bm25.feature.transfer.relevance'
fn_status_tpl = TMP_DATA_DIR_PATH + 'status/%s.ad2userStatus.dat'
selectionFeature = file(TMP_DATA_DIR_PATH + 'ranking/bm25.ranking', 'w')
selectionFeatureTransfer = file(TMP_DATA_DIR_PATH + 'ranking/bm25.ranking.transfer', 'w')
selectionFeatureTransferRelevance = file(TMP_DATA_DIR_PATH + 'ranking/bm25.ranking.transfer.relevance', 'w')
logging.info('Dumping adid2idx')
seeduserDict = {}
writer = file(TMP_DATA_DIR_PATH + 'ranking/adid2idx.txt', 'w')
for i, adid in enumerate(adset) :
writer.write('%s\t%d\n' % (adid, i+1))
fn_status = fn_status_tpl % adid
rset, nrset = getSeedUserSet(fn_status, adid)
seeduserDict[adid] = rset.union(nrset)
writer.close()
dumpSelectedFeature(adset, seeduserDict, feature, selectionFeature)
dumpSelectedFeature(adset, seeduserDict, featureTransfer, selectionFeatureTransfer)
dumpSelectedFeature(adset, seeduserDict, featureTransferRelevance, selectionFeatureTransferRelevance)
def generate_default_templates(resources):
"""
Create the default templates per resource that was retrieved from the `modw`.`resourcefact` table.
:param resources: a tuple of (string,int) [name, id] retrieved from the `modw`.`resourcefact` table.
:return: void
"""
if not resources:
logging.warning("No resources found. No files to create.")
else:
slurm_template_contents = retrieve_queue_template(os.path.join(akrr.curdir, 'templates','template.{0}.inp.py'), 'slurm')
pbs_template_contents = retrieve_queue_template(os.path.join(akrr.curdir, 'templates', 'template.{0}.inp.py'), 'pbs')
queues = {'slurm': slurm_template_contents, 'pbs': pbs_template_contents}
for resource in resources:
if args.verbose:
logging.info("Creating Resource Template: {0} ", resource[0] + "")
if not args.test:
for queue, contents in queues.iteritems():
file_path = os.path.join(resources_dir, resource[0] + 'resource.inp.py')
create_resource_template(file_path, queue, contents)
logging.info("Resource Template Generation Complete!")
def capture_scenario_params(scenario, ip_port):
def launch_slaves():
p = subprocess.Popen([
'./dev/venv/bin/python', 'run_param_slave.py', '-scenario',
scenario, '-stop_frame', '0', '-no_logging', '-headless',
protocol.k_arg_capture
])
p.wait()
def receive_param_loop(param_engine):
try:
while True:
objects = receiver.receive_objects()
handle_received_params(objects, param_engine)
except KeyboardInterrupt:
receiver.stop()
def handle_received_params(objects, param_engine):
for obj in objects:
logging.info(" Captured '{}' '{}' '{}'".format(
obj['name'], obj['space'], obj['dflt']))
param_engine.add_sampler(obj['name'], obj['space'], obj['dflt'])
logging.info(
f" Total Combination count {param_engine.product_size}")
logging.info(f"====== Capturing '{scenario}' scenario parameters ======")
receiver = Receiver(ip_port)
launch_slaves()
param_engine = CartesianEngine()
handle_received_params(receiver.receive_objects(), param_engine)
receiver.stop()
return param_engine
def off_parsed(args):
"""
Handles the appropriate execution of an 'Off' mode request given
the provided command line arguments.
"""
data = {
'application': args.application if args.application else ''
}
try:
result = akrrrestclient.put(
'/resources/{0}/off'.format(args.resource),
data=data)
if result.status_code == 200:
message = 'Successfully disabled {0} -> {1}.\n{2}' if args.application and args.resource \
else 'Successfully disabled all applications on {0}.\n{1}'
parameters = (args.application, args.resource, result.text) if args.application and args.resource \
else (args.resource, result.text)
log.info(message, *parameters)
else:
log.error(
'something went wrong. {0}:{1}',
result.status_code,
result.text)
except StandardError, e:
log.error('''
An error occured while communicating
with the REST API.
{0}: {1}
''',
e.args[0] if len(e.args) > 0 else '',
e.args[1] if len(e.args) > 1 else '')
def open_file(file_path, privs):
"""
Open and return the file handle to the file identified by the provided 'file_path' with the provided 'privs'.
If an error occurs during the opening of the file then it will be logged and 'None' will be returned.
:type file_path str
:type privs str
:param file_path: the path to be opened.
:param privs: the privs with which to open the provided file path.
:return: a file handle ( object ) if the open operation is successful else None.
"""
if file_path and isinstance(file_path, basestring) and privs and isinstance(privs, basestring):
# ADD: Some verbosity
if args.verbose:
logging.info("Opening with privs [{0}]: {1}", privs, file_path)
try:
# ATTEMPT: to open the file identified by the provided file_path
file_handle = open(file_path, privs)
# LET: the verbose users know we succeeded
if args.verbose:
logging.info('Successfully preformed open [{0}] on {1}', privs, file_path)
# RETURN: the file_handle
return file_handle
except IOError, e:
logging.error('Unable to open file: {0} due to {1}: {2}', file_path, e.args[0], e.args[1])
return None
def query_parsed(args):
"""
Handles the appropriate execution of a 'Query' mode request given
the provided command line arguments.
"""
dry_run = args.dryrun
verbose = args.verbose
resource = args.resource
exact = args.exact
def handle_results(results, dry_run):
if dry_run:
log.info("Would have inserted the following:")
[log.info("Id: {0:<9}Name: {1}", r['id'], r['name']) for r in results]
else:
log.info("Inserting the following:")
[log.info("Id: {0:<9}Name: {1}", r['id'], r['name']) for r in results]
insert_resources(results)
if verbose:
log.info("Retrieving requested resource(s)...")
if not resource:
results = retrieve_resources()
handle_results(results, dry_run)
else:
results = retrieve_resource(resource, exact)
handle_results(results, dry_run)
if verbose:
log.info("Requested Operation Completed")
def run():
Adset, UserSet = userStatusWorkflow.getPreSet(DATA_TRAINING)
for adid in Adset:
logging.info('Now handling AD:' + adid)
featureExtracter.workflow(adid, False, False)
featureExtracter.workflow(adid, True, False)
featureExtracter.workflow(adid, True, True)
def handle_received_params(objects, param_engine):
for obj in objects:
logging.info(" Captured '{}' '{}' '{}'".format(
obj['name'], obj['space'], obj['dflt']))
param_engine.add_sampler(obj['name'], obj['space'], obj['dflt'])
logging.info(
f" Total Combination count {param_engine.product_size}")
def create_resource_template(file_path, queue, contents):
"""
:type file_path str
:type queue str
:param file_path:
:param queue:
:param contents:
:return:
"""
privs = 'w'
output_path = file_path.format(queue)
def update_template(s,variable,inQuotes=True):
rexp='^'+variable+'\s*=\s*.*$'
replace=variable+' = '
value=globals()[variable]
if value==None:
replace+='None'
else:
if inQuotes:replace+='"'
replace+=str(value)
if inQuotes:replace+='"'
out=[]
lines=s.splitlines()
for line in lines:
out.append(re.sub(rexp,replace,line))
#s=re.sub(rexp,replace,s,flags=re.M)
return "\n".join(out)
contents=update_template(contents,'ppn',inQuotes=False)
for v in ['remoteAccessNode','remoteAccessMethod','remoteCopyMethod',
'sshUserName','sshPassword','sshPrivateKeyFile','sshPrivateKeyPassword',
'networkScratch','localScratch','akrrData','appKerDir','batchScheduler']:
contents=update_template(contents,v)
contents+="\n\n"
#contents=re.sub(r'^ppn\s*=\s*.*$','ppn = %d'%ppn,contents,flags=re.M)
#contents=re.sub(r'^remoteAccessNode\s*=\s*.*$','remoteAccessNode = "%s"'%remoteAccessNode,contents,flags=re.M)
#contents=re.sub(r'^remoteAccessMethod\s*=\s*.*$','remoteAccessMethod = "%s"'%remoteAccessNode,contents,flags=re.M)
# if
if not args.test:
output_file = open_file(output_path, privs)
write_to_file(output_file, contents)
close_file(output_file)
else:
logging.info('Test Mode: Would have written to: {0}', output_path)
logging.info('It content would be:')
print contents
def generate_self_signed_certificate(self):
log.info("Generating self-signed certificate for REST-API")
try:
output=subprocess.check_output("which openssl", shell=True)
except Exception,e:
log.error("""openssl program is not available. Install it!
For example by running on Ubuntu:
sudo apt-get install openssl""")
exit(1)
def run(self):
while self._stop_signalled == False:
try:
sample = self._queue.get_nowait()
signal_sample_taken(self._lock, self._sample_holder)
logging.info(f" #{self._index}: '{sample}'")
self.launch_and_wait(sample)
except queue.Empty:
time.sleep(0)
def getUserFeatureSet():
logging.info('=========start getUserFeatureSet processing=========')
query_set_fn = TMP_DATA_DIR_PATH + 'queryID.set'
desc_set_fn = TMP_DATA_DIR_PATH + 'descID.set'
title_set_fn = TMP_DATA_DIR_PATH + 'titleID.set'
query_set = set([query.strip() for query in file(query_set_fn)])
desc_set = set([desc.strip() for desc in file(desc_set_fn)])
title_set = set([title.strip() for title in file(title_set_fn)])
return query_set, desc_set, title_set
def getUserFeatureSet() :
logging.info('=========start getUserFeatureSet processing=========')
query_set_fn = TMP_DATA_DIR_PATH + 'queryID.set'
desc_set_fn = TMP_DATA_DIR_PATH + 'descID.set'
title_set_fn = TMP_DATA_DIR_PATH + 'titleID.set'
query_set = set([query.strip() for query in file(query_set_fn)])
desc_set = set([desc.strip() for desc in file(desc_set_fn)])
title_set = set([title.strip() for title in file(title_set_fn)])
return query_set, desc_set, title_set
def load(file: str) -> bool:
global settings
filepath = file
try:
with open(file, "r") as sett_file:
settings = json.load(sett_file)
logging.info(f"Trying {filepath}.....Success!")
return True
except Exception as e:
logging.info(f"Trying {filepath}.....Failed!")
logging.error(str(e))
return False
def new_task_parsed(args):
"""
Handles the appropriate execution of a 'New Task' mode request
given the provided command line arguments.
"""
if not (args.resource and
args.appkernel and
args.nodes):
parser.error(
'Please provide a resource, application and node count.')
exit(1)
resource = args.resource
app = args.appkernel
time_to_start=args.start_time
time_start = args.time_start# if args.time_start else '01:00'
time_end = args.time_end# if args.time_end else '05:00'
repeat_in = args.periodicity
nodes = args.nodes
node_list = [node.strip() for node in nodes.split(',')] if ',' in nodes else list(nodes)
for node in node_list:
if time_start!=None and time_end!=None:
time_to_start = calculate_random_start_time(
args.start_time,
repeat_in,
time_start,
time_end)
data = {
'resource': resource,
'app': app,
'time_to_start': time_to_start,
'repeat_in': repeat_in,
'resource_param': "{'nnodes':%s}" % (node,)
}
try:
result = akrrrestclient.post(
'/scheduled_tasks',
data=data)
if result.status_code == 200:
log.info('Successfully submitted new task')
else:
log.error(
'something went wrong. {0}:{1}',
result.status_code,
result.text)
except StandardError, e:
log.error('''
An error occured while communicating
with the REST API.
{0}: {1}
''',
e.args[0] if len(e.args) > 0 else '',
e.args[1] if len(e.args) > 1 else '')
def wall_time_parsed(args):
if not args.list and not (args.resource and
args.appkernel and
args.nodes and
args.walltime):
parser.error(
'Please provide a resource, app, node count and wall time.')
exit(1)
listing = args.list
resource = args.resource
app = args.appkernel
nodes = args.nodes
walltime = args.walltime
comments = args.comments
node_list = [node.strip() for node in nodes.split(',')] if ',' in nodes else list(nodes)
for nodes in node_list:
data = {
'resource_params': "{'nnodes':%d}" % (int(nodes),) if nodes else "{}",
'app_param':'{}',
'walltime': walltime,
'comments':comments
}
try:
result = akrrrestclient.post(
'/walltime/%s/%s'%(resource,app),
data=data) if not listing else \
akrrrestclient.get(
'/walltime/%s/%s'%(resource,app),
data=data)
if result.status_code == 200:
if not listing:
log.info('Successfully updated wall time (resource %s: application kernel: %s nodes: %d).'%(resource,app,nodes))
else:
log.info(
'Successfully queried walltime records. \n{0}',
result.text)
else:
log.error('something went wrong. {0}:{1}',
result.status_code,
result.text)
except StandardError, e:
import traceback
log.error('''
An error occured while communicating
with the REST API.
{0}: {1}
'''.strip(),
e.args[0] if len(e.args) > 0 else '',
e.args[1] if len(e.args) > 1 else '')
print traceback.print_exc()
def Evaluation() :
logging.info('===Evaluation Processing===')
fn_rankingResult = TMP_DATA_DIR_PATH + 'SVMRanking.prediction'
fn_userID4SVMRanking = TMP_DATA_DIR_PATH + 'userid4SVMRanking.dat'
fn_adId2Idx = TMP_DATA_DIR_PATH + 'adid2Idx.dict'
fn_ad2userStatus = TMP_DATA_DIR_PATH + 'ad2userStatus.dict'
fn_out_ad2userCTR = TMP_DATA_DIR_PATH + "ad2userCTR.dict"
fn_SVMRanking=TMP_DATA_DIR_PATH+'finalData4SVMRanking.dat'
#calculate ctr distribution
ctrDistribution(fn_SVMRanking, fn_rankingResult, fn_userID4SVMRanking, fn_adId2Idx, fn_ad2userStatus, fn_out_ad2userCTR)
def batch_job_parsed(args):
if not (args.resource and
args.appkernel and
args.nodes):
parser.error(
'Please provide a resource, application kernel and node count.')
exit(1)
resource = akrr.FindResourceByName(args.resource)
app = akrr.FindAppByName(args.appkernel)
nodes = args.nodes
node_list = [node.strip() for node in nodes.split(',')] if ',' in nodes else [int(nodes)]
print_only=args.print_only
verbose=args.verbose
str_io=cStringIO.StringIO()
if not verbose:
sys.stdout = sys.stderr = str_io
from akrrtaskappker import akrrTaskHandlerAppKer
# taskHandler=akrrTaskHandlerAppKer(1,resource['name'],app['name'],"{'nnodes':%s}" % (node_list[0],),"{}","{}")
# test arbitrary resourceParam like WLMheader stuff
taskHandler=akrrTaskHandlerAppKer(1,resource['name'],app['name'],"{'SlUrM':'--foo','slurm':'#SBATCH --mail=maya','nnodes':%s}" % (node_list[0],),"{}","{}")
if print_only:
taskHandler.GenerateBatchJobScript()
else:
taskHandler.CreateBatchJobScriptAndSubmitIt(doNotSubmitToQueue=True)
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
if taskHandler.status.count("ERROR")>0:
log.error('Batch job script was not generated see log below!')
print str_io.getvalue()
log.error('Batch job script was not generated see log above!')
jobScriptFullPath=os.path.join(taskHandler.taskDir,"jobfiles",taskHandler.JobScriptName)
if os.path.isfile(jobScriptFullPath):
fin=open(jobScriptFullPath,"r")
jobScriptContent=fin.read()
fin.close()
if print_only:
log.info('Below is content of generated batch job script:')
print jobScriptContent
else:
log.info("Local copy of batch job script is "+jobScriptFullPath)
print
log.info("Application kernel working directory on "+resource['name']+" is "+taskHandler.remoteTaskDir)
log.info("Batch job script location on "+resource['name']+" is "+os.path.join(taskHandler.remoteTaskDir,taskHandler.JobScriptName))
else:
log.error('Batch job script was not generated see messages above!')
if print_only:
log.info('Removing generated files from file-system as only batch job script printing was requested')
taskHandler.DeleteLocalFolder()
def load_road():
self.road = RoadBuilder().load_and_build(
self._get_map(), self._get_map_parameters())
logging.info(f"road: {self.road.source}")
if len(self.road.description):
logging.info(" " + self.road.description)
if len(self.road.asciiart):
logging.info(" " + self.road.asciiart)
if len(self.road.param_descriptions):
logging.info(" " + str(self.road.param_descriptions))
if len(self.road.params):
logging.info(" " + str(self.road.params))
def Evaluation():
logging.info('===Evaluation Processing===')
fn_rankingResult = TMP_DATA_DIR_PATH + 'SVMRanking.prediction'
fn_userID4SVMRanking = TMP_DATA_DIR_PATH + 'userid4SVMRanking.dat'
fn_adId2Idx = TMP_DATA_DIR_PATH + 'adid2Idx.dict'
fn_ad2userStatus = TMP_DATA_DIR_PATH + 'ad2userStatus.dict'
fn_out_ad2userCTR = TMP_DATA_DIR_PATH + "ad2userCTR.dict"
fn_SVMRanking = TMP_DATA_DIR_PATH + 'finalData4SVMRanking.dat'
#calculate ctr distribution
ctrDistribution(fn_SVMRanking, fn_rankingResult, fn_userID4SVMRanking,
fn_adId2Idx, fn_ad2userStatus, fn_out_ad2userCTR)
def update_publish_request_count():
global request_count
read_cnt = 0
while True:
read_cnt += 1
request_count = publish.cryptowatch.request_count.get_request_count(
minutes=5)
if read_cnt % 10 == 0:
logging.info('publish_request_count: {request_count}'.format(
request_count=request_count))
print('publish_request_count: {request_count}'.format(
request_count=request_count))
time.sleep(6)
def run(cfg, forcerun):
tz = config.get_tz(cfg)
while True:
dt_str = str(util.time.get_utcnow().astimezone(tz).date())
logging.info(cfg, 'checking if run for {dt_str} should be done'.format(dt_str=dt_str))
if not forcerun and upload.daily.history.did_upload_today():
logging.info(cfg, 'run for {dt_str} is already done'.format(dt_str=dt_str))
time.sleep(10 * 60)
continue
t_run_after = config.get_daily_ingestion_start_t(cfg)
while True:
t_cur = util.time.get_utcnow().astimezone(tz).time()
logging.info(cfg, 'checking if the schedule time for {dt_str} has reached'.format(dt_str=dt_str))
if forcerun or t_cur > t_run_after:
run_download(cfg)
run_ingests_append_combine()
run_upload(cfg)
upload.daily.history.on_upload()
break
logging.info(cfg, 'schedule time {t_run_after} not yet reached at {t_cur}'.format(t_run_after=t_run_after, t_cur=t_cur))
time.sleep(10 * 60)
if forcerun:
# forcerun runs only once
break
def init_mysql_dbs(self):
try:
log.info("Creating AKRR databases and granting permissions for AKRR user.")
db_root,cur_root=self.get_db(user=self.sql_root_name,password=self.sql_root_password)
cur_root.execute("SHOW DATABASES")
dbsNames=[v['Database'] for v in cur_root.fetchall()]
cur_root.execute("SELECT @@hostname")
results=cur_root.fetchall()
hostname=results[0]['@@hostname']
#create user if needed
#cur_root.execute("SELECT * FROM mysql.user WHERE User=%s",(self.akrr_user_name,))
#results=cur_root.fetchall()
# ENSURE: That the `mod_akrr` database is created.
if 'mod_akrr' not in dbsNames:
cur_root.execute("CREATE DATABASE IF NOT EXISTS mod_akrr")
while cur_root.nextset() is not None: pass
# ENSURE: That the `mod_appkernel` database is created.
if 'mod_appkernel' not in dbsNames:
cur_root.execute("CREATE DATABASE IF NOT EXISTS mod_appkernel")
while cur_root.nextset() is not None: pass
# ENSURE: That the user that will be used by AKRR is created with the correct privileges.
cur_root.execute("GRANT ALL ON mod_akrr.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, '%', self.akrr_user_password))
cur_root.execute("GRANT ALL ON mod_akrr.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, 'localhost', self.akrr_user_password))
cur_root.execute("GRANT ALL ON mod_akrr.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, hostname, self.akrr_user_password))
while cur_root.nextset() is not None: pass
# ENSURE: That the AKRR user has the correct privileges to the `mod_appkernel` database.
cur_root.execute("GRANT ALL ON mod_appkernel.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, '%', self.akrr_user_password))
cur_root.execute("GRANT ALL ON mod_appkernel.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, 'localhost', self.akrr_user_password))
cur_root.execute("GRANT ALL ON mod_appkernel.* TO %s@%s IDENTIFIED BY %s",(self.akrr_user_name, hostname, self.akrr_user_password))
while cur_root.nextset() is not None: pass
# ENSURE: That the AKRR modw user is created w/ the correct privileges
cur_root.execute("GRANT SELECT ON modw.resourcefact TO %s@%s IDENTIFIED BY %s",(self.xd_user_name, '%', self.xd_user_password))
cur_root.execute("GRANT SELECT ON modw.resourcefact TO %s@%s IDENTIFIED BY %s",(self.xd_user_name, 'localhost', self.xd_user_password))
cur_root.execute("GRANT SELECT ON modw.resourcefact TO %s@%s IDENTIFIED BY %s",(self.xd_user_name, hostname, self.xd_user_password))
while cur_root.nextset() is not None: pass
# ENSURE: That the newly granted privileges are flushed into active service.
cur_root.execute("FLUSH PRIVILEGES")
while cur_root.nextset() is not None: pass
db_root.commit()
except Exception,e:
log.error("Can not execute the sql install script: "+str(e))
exit(1)
def dec(filename):
KEY = int(input("Input the key shared to you\n"))
KEY = '{0:016b}'.format(int(hex(KEY),16))
logging.info("Key recieved")
logging.info("Decryption has started...")
INV_BITSBOX = sbox_dec(filename)
BITSHIFT = SHIFTRIGHT(INV_BITSBOX)
BITXNOR = decXOR(BITSHIFT,KEY)
BITS = BITXNOR
#Using
for x in tqdm(range(16)):
BITS = sbox_dec_rounds(BITS)
BITS = SHIFTRIGHT(INV_BITSBOX)
BITS = decXOR(BITSHIFT,KEY)
file_converter_dec(BITS)
def enc(filename):
KEY = create_key()
print('Share this Key %s with the message reciever'%(KEY))
KEY = '{0:016b}'.format(int(hex(KEY),16))
logging.info("Key created")
logging.info("Encryption has started...")
BITXOR = encXOR(filename,KEY)
BITSHIFT = SHIFTLEFT(BITXOR)
BITSBOX = sbox_enc(BITSHIFT)
BITS = BITSBOX
for x in tqdm(range(16)):
BITS = encXORrounds(BITS,KEY)
BITS = SHIFTLEFT(BITXOR)
BITS = sbox_enc(BITSHIFT)
file_converter_enc(BITS)
def read_akrr_creds(self):
log.info("Before Installation continues we need to setup the database.")
log.input("Please specify a database user for AKRR (This user will be created if it does not already exist):")
self.akrr_user_name=raw_input('[{0}] '.format(self.default_akrr_user))
if self.akrr_user_name=='':self.akrr_user_name=self.default_akrr_user
while True:
log.input("Please specify a password for the AKRR database user:"******"Please reenter password:"******"Entered passwords do not match. Please try again.")
def read_modw_creds(self):
log.input("Please specify the user that will be connecting to the XDMoD database (modw):")
self.xd_user_name=raw_input('[{0}] '.format(self.default_akrr_user))
if self.xd_user_name=='':self.xd_user_name=self.default_akrr_user
if self.xd_user_name==self.akrr_user_name:
log.info("Same user as for AKRR database user, will set same password")
self.xd_user_password=self.akrr_user_password
else:
while True:
log.input("Please specify the password:"******"Please reenter password:"******"Entered passwords do not match. Please try again.")
def list_parsed(args):
"""
Handles the appropriate execution of a 'List' mode request given
the provided command line arguments.
"""
verbose = args.verbose
resource = args.resource
application = args.application
status = args.status
if verbose:
log.info("Attempting to complete the requested Operation: ")
if status:
results = {}
pass
else:
results = retrieve_tasks(resource, application)
if results:
log.info('Retrieved the following: ')
[log.info("[{:<8}] Resource: {:<15} App:{:<24}",r['task_id'], r['resource'], r['app']) for r in results]
else:
log.warning('No records returned.')
if verbose:
log.info("Requested Operation Completed")
def retrieve_tasks(resource, application):
"""
Retrieve the list of currently scheduled tasks ( resource / application pairings ) from
mod_akrr.
:type resource str
:type application str
:param resource: filter the results by the provided resource
:param application: filter the results by the provided application
:return: a dict representation of the mod_akrr.SCHEDULEDTASKS table
"""
data = {
'application': application,
'resource': resource
}
try:
akrrrestclient.get_token()
except StandardError:
log.error('''
An error occured while attempting to retrieve a token
from the REST API.
''')
try:
result = akrrrestclient.get(
'/scheduled_tasks',
data=data)
if result.status_code == 200:
log.info('Successfully Completed Task Retrieval.\n{0}', result.text)
else:
log.error(
'something went wrong. {0}:{1}',
result.status_code,
result.text)
return result
except StandardError, e:
log.error('''
An error occured while communicating
with the REST API.
{0}: {1}
''',
e.args[0] if len(e.args) > 0 else '',
e.args[1] if len(e.args) > 1 else '')
def sbox_dec(filename):
logging.info("Reading file contents...")
CONTENT = []
with open(filename, 'r', encoding='utf-8') as FILE:
for x in FILE:
for y in x:
CONTENT.append(y)
BITS = []
for x in CONTENT:
BITS.append('{0:016b}'.format(int(ord(x))))
RTRBITS = []
for x in BITS:
HEX0, HEX1, HEX2, HEX3 = string_split(x)
F8B = '{0:08b}'.format(int(InvSbox[int(HEX0, 2)][int(HEX1, 2)]))
L8B = '{0:08b}'.format(int(InvSbox[int(HEX2, 2)][int(HEX3, 2)]))
joinBITS = F8B + L8B
RTRBITS.append(joinBITS)
return RTRBITS
def generate_settings_file(self):
log.info("Generating Settings File...")
with open(os.path.join(akrr_home,'setup','scripts','akrr.src.inp.py'),'r') as f:
akrr_inp_template=f.read()
restapi_rw_password=self.get_random_password()
restapi_ro_password=self.get_random_password()
var={
'akrr_user_name':self.akrr_user_name,
'akrr_user_password':self.akrr_user_password,
'xd_user_name':self.xd_user_name,
'xd_user_password':self.xd_user_password,
'restapi_rw_password':restapi_rw_password,
'restapi_ro_password':restapi_ro_password
}
akrr_inp=akrr_inp_template.format(**var)
with open(os.path.join(akrr_home,'cfg','akrr.inp.py'),'w') as f:
akrr_inp_template=f.write(akrr_inp)
log.info("Settings written to: {0}".format(os.path.join(akrr_home,'cfg','akrr.inp.py')))
def socket_entry_handler(syscall_id, syscall_object, pid):
"""Replay Always
Checks:
0: The domain of the socket
Sets:
return value: file descriptor of the new socket -1 (error)
(added as replay file descriptor)
errno
Not Implemented:
* Determine what is not implemented
"""
logging.debug('Entering socket subcall entry handler')
ecx = cint.peek_register(pid, cint.ECX)
params = extract_socketcall_parameters(pid, ecx, 3)
# Only PF_INET and PF_LOCAL socket calls are handled
execution_is_PF_INET = (params[0] == cint.PF_INET)
trace_is_PF_INET = (str(syscall_object.args[0]) == '[\'PF_INET\']')
execution_is_PF_LOCAL = (params[0] == 1) # define PF_LOCAL 1
trace_is_PF_LOCAL = (str(syscall_object.args[0]) == '[\'PF_LOCAL\']')
logging.debug('Execution is PF_INET: %s', execution_is_PF_INET)
logging.debug('Trace is PF_INET: %s', trace_is_PF_INET)
logging.debug('Exeuction is PF_LOCAL: %s', execution_is_PF_LOCAL)
logging.debug('Trace is PF_LOCAL: %s', trace_is_PF_LOCAL)
if execution_is_PF_INET != trace_is_PF_INET:
raise ReplayDeltaError(
'Encountered socket subcall with mismatch between '
'execution protocol family and trace protocol family')
if execution_is_PF_LOCAL != trace_is_PF_LOCAL:
raise ReplayDeltaError(
'Encountered socket subcall with mismatch between '
'execution protocol family and trace protocol family')
# Decide if we want to deal with this socket call or not
if trace_is_PF_INET or \
execution_is_PF_INET or \
trace_is_PF_LOCAL or \
execution_is_PF_LOCAL:
noop_current_syscall(pid)
fd = int(syscall_object.ret[0])
logging.debug('File Descriptor from trace: %s', fd)
apply_return_conditions(pid, syscall_object)
else:
logging.info('Ignoring non-PF_INET call to socket')
def generate_resource_config(resource_id, resource_name, queuing_system):
logging.info("Initiating %s at AKRR"%(resource_name,))
slurm_template_contents = retrieve_queue_template(os.path.join(akrr.curdir, 'templates', 'template.{0}.inp.py'), 'slurm')
pbs_template_contents = retrieve_queue_template(os.path.join(akrr.curdir, 'templates', 'template.{0}.inp.py'), 'pbs')
queues = {'slurm': slurm_template_contents, 'pbs': pbs_template_contents}
if not args.test:
os.mkdir(os.path.join(resources_dir, resource_name),0700)
file_path = os.path.abspath(os.path.join(resources_dir, resource_name, 'resource.inp.py'))
global resource_cfg_filename
resource_cfg_filename=file_path
create_resource_template(file_path, queues[queuing_system], queues[queuing_system])
if not args.test:
#add entry to mod_appkernel.resource
dbAK,curAK=akrr.getAKDB(True)
curAK.execute('''SELECT * FROM resource WHERE nickname=%s''', (resource_name,))
resource_in_AKDB = curAK.fetchall()
if len(resource_in_AKDB)==0:
curAK.execute('''INSERT INTO resource (resource,nickname,description,enabled,visible,xdmod_resource_id)
VALUES(%s,%s,%s,0,0,%s);''',
(resource_name,resource_name,resource_name,resource_id))
dbAK.commit()
curAK.execute('''SELECT * FROM resource WHERE nickname=%s''', (resource_name,))
resource_in_AKDB = curAK.fetchall()
resource_id_in_AKDB=resource_in_AKDB[0]['resource_id']
#add entry to mod_akrr.resource
db,cur=akrr.getDB(True)
cur.execute('''SELECT * FROM resources WHERE name=%s''', (resource_name,))
resource_in_DB = cur.fetchall()
if len(resource_in_DB)==0:
cur.execute('''INSERT INTO resources (id,xdmod_resource_id,name,enabled)
VALUES(%s,%s,%s,%s);''',
(resource_id_in_AKDB,resource_id,resource_name,0))
db.commit()
logging.info("Resource configuration is in "+file_path)
def close_file(file_handle):
"""
Close the provided file_handle. If an error is encountered than a message will be logged to stdout.
:type file_handle file
:param file_handle: the file to be closed.
:return: void
"""
if file_handle and isinstance(file_handle, file):
try:
if args.verbose:
logging.info('Attempting to close the file {0}', file_handle.name)
file_handle.close()
if args.verbose:
logging.info('Successfully closed the file {0}', file_handle.name)
except IOError, e:
logging.error('There was an error encountered while closing {0}. {1}: {2}', file_handle.name, e.args[0], e.args[1])
def featureEngineering() :
logging.info('===Feature Engineering Processing===')
query_set, desc_set, title_set = getUserFeatureSet()
aggregateUserfile = TMP_DATA_DIR_PATH + 'userRawFeature_test.dict'
expandId2TokensResultFile = TMP_DATA_DIR_PATH + 'userRawExpandTokens_test.dict'
expandFeatureId2Tokens(aggregateUserfile, expandId2TokensResultFile, query_set, desc_set, title_set)
tmp_file = file(TMP_DATA_DIR_PATH + 'tmp', 'w')
fn_userRawExpandTokens = expandId2TokensResultFile
for line in file(fn_userRawExpandTokens) :
userid, query, title, desc = line.strip().split('\x01')
tmp_file.write(query)
tmp_file.write('\n')
tmp_file.close()
lda = topicLDA.LDA(TMP_DATA_DIR_PATH + 'tmp')
LDA.run(lda, num_topics=200, raw_corpus=TMP_DATA_DIR_PATH + 'tmp', fn_bow=TMP_DATA_DIR_PATH+'corpus.svmlight_test', fn_out_topic=TMP_DATA_DIR_PATH+'LDA_corpus.svmlight_test')
os.system('rm ' + TMP_DATA_DIR_PATH + 'tmp')
joinResult4SVMRanking(fn_trainFeature=TMP_DATA_DIR_PATH+'corpus.svmlight_test',
fn_ad2userStatus=TMP_DATA_DIR_PATH+'ad2userStatus_test.dict', fn_out_SVMRanking=TMP_DATA_DIR_PATH+'finalData4SVMRanking_test.dat')
def check_rw_db(connection_func, pre_msg, post_msg):
"""
Check that the user has the correct privileges to the database
at the end of the connection provided by 'connection_func'. Specifically, checking
for read / write permissions ( and create table ).
:type connection_func function
:type pre_msg str
:type post_msg str
:param connection_func: the function that will provide a (connection, cursor) tuple.
:param pre_msg: a message to be provided to the user before the checks begin.
:param post_msg: a message to be provided to the user after the checks are successful
:return: true if the database is available / the provided user has the correct privileges.
"""
success = False
log.info(pre_msg)
try:
connection, cursor = connection_func()
try:
with connection:
result = cursor.execute("CREATE TABLE CREATE_ME(`id` INT NOT NULL PRIMARY KEY, `name` VARCHAR(48));")
success = True if result == 0 else False
if success:
log.info(post_msg, success)
else:
log.error(post_msg, success)
except MySQLdb.MySQLError, e:
log.error("Unable to create a table w/ the provided username. {0}: {1}", e.args[0], e.args[1])
connection, cursor = connection_func()
try:
with connection:
cursor.execute("DROP TABLE CREATE_ME;")
except MySQLdb.MySQLError, e:
log.error("Unable to drop the table created to check permissions. {0}: {1}", e.args[0], e.args[1])
def dataCleaning() :
logging.info('===Data Cleaning Processing===')
input_file = DATA_TRAINING_SAMPLE
adClickCntList = generateTopAdsUsersByClick(input_file)
dumpList2File(adClickCntList, TMP_DATA_DIR_PATH + 'topAdClickCnt.dict')
adSet = set()
for line in file(TMP_DATA_DIR_PATH + 'topAdClickCnt.dict') :
cnt, adid = line.strip().split()
adSet.add(adid)
logging.debug(len(adSet))
ad2Users = generateAd2UsersGivenAdSet(input_file, adSet)
dumpDict2File(ad2Users, TMP_DATA_DIR_PATH + 'ad2UsersGivenAdSet.dict')
userDict = generateUser2AdGivenAd2User(TMP_DATA_DIR_PATH + 'ad2UsersGivenAdSet.dict', adViewThreshold = 10)
dumpDict2File(userDict, TMP_DATA_DIR_PATH + 'user2AdGivenAd2User.dict')
userSet = set()
logging.debug(len(userSet))
for line in file(TMP_DATA_DIR_PATH + 'user2AdGivenAd2User.dict') :
user, ads = line.strip().split('\x01')
userSet.add(user)
dumpUserRawFeatureGivenUserSet(input_file, userSet, TMP_DATA_DIR_PATH + 'userRawFeature.dict')
def expandFeatureId2Tokens(aggregateUserfile, expandId2TokensResultFile, query_set, desc_set, title_set) :
logging.info('=========start expandFeatureId2Tokens processing=========')
description_map = dict([(line.strip().split('\t')) for line in file(DATA_DESCRIPTION) if line.split('\t',1)[0] in desc_set])
logging.debug('Read %s Done.' % DATA_DESCRIPTION)
query_map = dict([(line.strip().split('\t')) for line in file(DATA_QUERY) if line.split('\t',1)[0] in query_set])
logging.debug('Read %s Done.' % DATA_QUERY)
title_map = dict([(line.strip().split('\t')) for line in file(DATA_TITLE) if line.split('\t',1)[0] in title_set])
logging.debug('Read %s Done.' % DATA_TITLE)
#profile_map = dict([(line.strip().split('\t', 1)) for line in file(DATA_PROFILE) if line.split('\t')])
dump_format = '%s\x01%s\x01%s\x01%s\n'
expandId2TokensResult = file(expandId2TokensResultFile, 'w')
logging.debug('start joining tokens')
for line in file(aggregateUserfile) :
userID, tmp_str = line.strip().split('\x01')
queryIDlist, titleIDlist, descIDList = tmp_str.split('\x02')
queryExpandTokensStr = '|'.join([query_map[queryId] for queryId in queryIDlist.split('\t') if queryId != ''])
titleExpandTokensStr = '|'.join([title_map[titleId] for titleId in titleIDlist.split('\t') if titleId != ''])
descExpandTokensStr = '|'.join([description_map[descId] for descId in descIDList.split('\t') if descId != ''])
expandId2TokensResult.write( dump_format % \
(userID, queryExpandTokensStr, titleExpandTokensStr, descExpandTokensStr))
expandId2TokensResult.close()
def handle_results(results, dry_run):
if dry_run:
log.info("Would have inserted the following:")
[log.info("Id: {0:<9}Name: {1}", r['id'], r['name']) for r in results]
else:
log.info("Inserting the following:")
[log.info("Id: {0:<9}Name: {1}", r['id'], r['name']) for r in results]
insert_resources(results)
def dumpSelectedFeature(adset, seeduserDict, feature_tpl, writer) :
fn_status_tpl = TMP_DATA_DIR_PATH + 'status/%s.ad2userStatus.dat'
for i, adid in enumerate(adset) :
fn_status = fn_status_tpl % adid
fn_feature = feature_tpl % adid
logging.info('Handling %s \n %s' % (fn_status, fn_feature))
allset = seeduserDict[adid]
status = {}
for line in file(fn_status) :
adid, userid, click, pv = line.strip().split()
if userid not in allset : continue
if int(click) >= 1 :
status[userid] = 1
else :
status[userid] = 0
for featureLine in file(fn_feature) :
userid, rest = featureLine.strip().split('\t',1)
if userid not in status : continue
features = rest.split()
writer.write('%d qid:%d ' % (status[userid], i+1))
writer.write(' '.join(['%d:%s' % (j+1,item) for j, item in enumerate(features)]))
writer.write('\n')
writer.close()
def rankingFeatureSelection():
adset = set([
line.strip().split()[1]
for line in file(TMP_DATA_DIR_PATH + 'topAdClickCnt.dict.final')
])
#adset = set(list(adset)[:2])
blacklist = set([
'20174985', '3834142', '3373964', '4344041', '8350700', '2878230',
'3803920', '20174982', '4341158', '6434934', '3219148', '20035409'
])
adset = adset - blacklist
feature = TMP_DATA_DIR_PATH + 'feature/%s.bm25.feature'
featureTransfer = TMP_DATA_DIR_PATH + 'feature/%s.bm25.feature.transfer'
featureTransferRelevance = TMP_DATA_DIR_PATH + 'feature/%s.bm25.feature.transfer.relevance'
fn_status_tpl = TMP_DATA_DIR_PATH + 'status/%s.ad2userStatus.dat'
selectionFeature = file(TMP_DATA_DIR_PATH + 'ranking/bm25.ranking', 'w')
selectionFeatureTransfer = file(
TMP_DATA_DIR_PATH + 'ranking/bm25.ranking.transfer', 'w')
selectionFeatureTransferRelevance = file(
TMP_DATA_DIR_PATH + 'ranking/bm25.ranking.transfer.relevance', 'w')
logging.info('Dumping adid2idx')
seeduserDict = {}
writer = file(TMP_DATA_DIR_PATH + 'ranking/adid2idx.txt', 'w')
for i, adid in enumerate(adset):
writer.write('%s\t%d\n' % (adid, i + 1))
fn_status = fn_status_tpl % adid
rset, nrset = getSeedUserSet(fn_status, adid)
seeduserDict[adid] = rset.union(nrset)
writer.close()
dumpSelectedFeature(adset, seeduserDict, feature, selectionFeature)
dumpSelectedFeature(adset, seeduserDict, featureTransfer,
selectionFeatureTransfer)
dumpSelectedFeature(adset, seeduserDict, featureTransferRelevance,
selectionFeatureTransferRelevance)
def dumpSelectedFeature(adset, seeduserDict, feature_tpl, writer):
fn_status_tpl = TMP_DATA_DIR_PATH + 'status/%s.ad2userStatus.dat'
for i, adid in enumerate(adset):
fn_status = fn_status_tpl % adid
fn_feature = feature_tpl % adid
logging.info('Handling %s \n %s' % (fn_status, fn_feature))
allset = seeduserDict[adid]
status = {}
for line in file(fn_status):
adid, userid, click, pv = line.strip().split()
if userid not in allset: continue
if int(click) >= 1:
status[userid] = 1
else:
status[userid] = 0
for featureLine in file(fn_feature):
userid, rest = featureLine.strip().split('\t', 1)
if userid not in status: continue
features = rest.split()
writer.write('%d qid:%d ' % (status[userid], i + 1))
writer.write(' '.join(
['%d:%s' % (j + 1, item) for j, item in enumerate(features)]))
writer.write('\n')
writer.close()
def check_r_db(connection_func, pre_msg, post_msg):
"""
Check that the user has the correct privileges to the database
at the end of the connection provided by 'connection_func'.
Specifically checking for read permissions.
:type connection_func function
:type pre_msg str
:type post_msg str
:param connection_func: the function that will provide a (connection, cursor) tuple.
:param pre_msg: a message to be provided to the user before the checks begin.
:param post_msg: a message to be provided to the user after the checks are successful
:return: true if the database is available / the provided user has the correct privileges.
"""
success = False
log.info(pre_msg)
try:
connection, cursor = connection_func()
try:
with connection:
result = cursor.execute("SELECT COUNT(*) FROM `modw`.`resourcefact`;")
success = True if result >= 0 else False
if success:
log.info(post_msg, success)
else:
log.error(post_msg, success)
except MySQLdb.MySQLError, e:
log.error("Unable to select from `modw`.`resourcefact`. {0}: {1}", e.args[0], e.args[1])
except MySQLdb.MySQLError, e:
log.error("Unable to connect to Database. {0}: {1}", e.args[0], e.args[1])
def expandFeatureId2Tokens(aggregateUserfile, expandId2TokensResultFile,
query_set, desc_set, title_set):
logging.info('=========start expandFeatureId2Tokens processing=========')
description_map = dict([(line.strip().split('\t'))
for line in file(DATA_DESCRIPTION)
if line.split('\t', 1)[0] in desc_set])
logging.debug('Read %s Done.' % DATA_DESCRIPTION)
query_map = dict([(line.strip().split('\t')) for line in file(DATA_QUERY)
if line.split('\t', 1)[0] in query_set])
logging.debug('Read %s Done.' % DATA_QUERY)
title_map = dict([(line.strip().split('\t')) for line in file(DATA_TITLE)
if line.split('\t', 1)[0] in title_set])
logging.debug('Read %s Done.' % DATA_TITLE)
#profile_map = dict([(line.strip().split('\t', 1)) for line in file(DATA_PROFILE) if line.split('\t')])
dump_format = '%s\x01%s\x01%s\x01%s\n'
expandId2TokensResult = file(expandId2TokensResultFile, 'w')
logging.debug('start joining tokens')
for line in file(aggregateUserfile):
userID, tmp_str = line.strip().split('\x01')
queryIDlist, titleIDlist, descIDList = tmp_str.split('\x02')
queryExpandTokensStr = '|'.join([
query_map[queryId] for queryId in queryIDlist.split('\t')
if queryId != ''
])
titleExpandTokensStr = '|'.join([
title_map[titleId] for titleId in titleIDlist.split('\t')
if titleId != ''
])
descExpandTokensStr = '|'.join([
description_map[descId] for descId in descIDList.split('\t')
if descId != ''
])
expandId2TokensResult.write( dump_format % \
(userID, queryExpandTokensStr, titleExpandTokensStr, descExpandTokensStr))
expandId2TokensResult.close()
def run():
global request_count
while True:
logging.info('starting run_polygon.py')
p = subprocess.Popen(['go', 'run', 'run_ohlc.go', '4'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
preexec_fn=os.setsid)
t_pstart = datetime.datetime.now()
append_if_p_killed = []
th1 = threading.Thread(target=monitor_restart_trigger,
args=(
p,
append_if_p_killed,
))
th2 = threading.Thread(target=monitor_publish_request_count,
args=(p, t_pstart, append_if_p_killed))
th1.start()
th2.start()
while True:
th1.join(10)
logging.info('end of the join thread monitor_restart_trigger')
if append_if_p_killed:
logging.warn(
'observed that the process was killed in monitor_restart_trigger thread.'
)
# TODO: finish
# th2.cancel()
break
th2.join(10)
logging.info(
'end of the join thread monitor_publish_request_count')
if append_if_p_killed:
logging.warn(
'observed that the process was killed in monitor_publish_request_count thread.'
)
# th1.cancel()
break
logging.info(
'end of the join loop, process was not killed in the loop')
def scene_pyscenario_update(sctx):
scene = sctx['scene']
if (sctx['frame'] == 0):
scenario_name_pattern = get_scenario_arg()
scenario_name, scenario_inst = make_scenario_instance(scenario_name_pattern)
road__module_or_file = scenario_inst.get_map()
road_params = scenario_inst.get_map_parameters()
scene_road_update_file(sctx, road__module_or_file, road_params)
scenario_inst._init(scene['coord'])
logging.info(f"scenario: {scenario_name}")
logging.info(" " + scenario_inst.get_description())
monitor_names = arg_get('-monitors', '').split(',')
monitor_names = [x for x in monitor_names if len(x.strip())]
monitor_insts = []
for monitor_name_pattern in monitor_names:
monitor_name, monitor_inst = make_monitor_instance(monitor_name_pattern)
monitor_inst._init(scenario_inst)
monitor_insts.append(monitor_inst)
logging.info(f"monitor: {monitor_name}")
logging.info(" " + monitor_inst.get_description())
scene['scenario'] = scenario_inst
scene['monitors'] = monitor_insts
scene['dt'] = 1.0/60.0
scene['t'] = 0.0
scene['frame'] = 0
if scene['scenario'] is not None:
t = scene['frame'] * scene['dt']
scene['scenario']._step(scene['frame'], t, scene['dt'])
for monitor in scene['monitors']:
monitor.step(scene['frame'], t, scene['dt'])
scene['frame'] = scene['frame'] + 1
return True
def run(forcerun):
cfg = config.load('config.us.polygon.yaml')
tz = config.get_tz(cfg)
while True:
dt_str = str(util.time.get_utcnow().astimezone(tz).date())
logging.info(
cfg, 'checking if run for {dt_str} should be done'.format(
dt_str=dt_str))
if not forcerun and upload.daily.history.did_upload_today():
logging.info(
cfg, 'run for {dt_str} is already done'.format(dt_str=dt_str))
time.sleep(30 * 60)
continue
t_run_after = config.get_daily_last_record_ingestion_start_t(cfg)
while True:
t_cur = util.time.get_now_time_tz()
logging.info(
cfg, 'checking if the schedule time for {dt_str} has reached'.
format(dt_str=dt_str))
dt = util.time.time_diff_seconds(t_run_after, t_cur)
if forcerun or dt <= 0:
run_ingests()
run_upload()
upload.daily.history.on_upload()
break
logging.info(
cfg, 'schedule time {t_run_after} not yet reached at {t_cur}'.
format(t_run_after=t_run_after, t_cur=t_cur))
time.sleep(max(dt + 1, 60 * 60))
if forcerun:
# forcerun runs only once
break
def run_download(cfg):
logging.info(cfg, 'downloading us daily')
download.download.download(download.download.DEST_DIR_DAILY)
logging.info(cfg, 'download complete')
def Prediction():
logging.info('===Prediction Processing===')
features = TMP_DATA_DIR_PATH + 'finalData4SVMRanking.dat'
model = TMP_DATA_DIR_PATH + 'SVMRanking.model'
predictions = TMP_DATA_DIR_PATH + 'SVMRanking.prediction'
SVM_RANK.svm_rank_classify(features, model, predictions)
def Training():
logging.info('===Trainng Processing===')
features = TMP_DATA_DIR_PATH + 'finalData4SVMRanking.dat'
model = TMP_DATA_DIR_PATH + 'SVMRanking.model'
SVM_RANK.svm_rank_learn(features, model, args=' -c 10 ')
def genTitleDesc(inputFile, adSet, userset=None):
logging.info('Generating Click and Unclick Title Description For Per User')
ad2Profile = {}
titleSet = set()
descSet = set()
for line in file(inputFile):
fields = line.strip().split()
adid = fields[3]
userid = fields[-1]
if userid == '0' or adid not in adSet: continue
if userid not in userset: continue
if adid not in ad2Profile:
ad2Profile[adid] = {
'profile': {},
'nonClick_profile': {},
'click_profile': {}
}
profile = ad2Profile[adid]['profile']
nonClick_profile = ad2Profile[adid]['nonClick_profile']
click_profile = ad2Profile[adid]['click_profile']
title = fields[-3]
desc = fields[-2]
titleSet.add(title)
descSet.add(desc)
if userid not in profile:
profile[userid] = {'desc': set(), 'title': set()}
nonClick_profile[userid] = {'desc': set(), 'title': set()}
click_profile[userid] = {'desc': set(), 'title': set()}
if title not in profile[userid]['title']:
profile[userid]['title'].add(title)
if int(fields[0]) > 0:
click_profile[userid]['title'].add(title)
else:
nonClick_profile[userid]['title'].add(title)
if desc not in profile[userid]['desc']:
profile[userid]['desc'].add(desc)
if int(fields[0]) > 0:
click_profile[userid]['desc'].add(desc)
else:
nonClick_profile[userid]['desc'].add(desc)
fn_DESC = DATA_DESCRIPTION
fn_TITLE = DATA_TITLE
expandDesc = dict()
expandTitle = dict()
for line in file(fn_DESC):
tid, rest = line.strip().split()
if tid not in descSet: continue
expandDesc[tid] = rest
for line in file(fn_TITLE):
tid, rest = line.strip().split()
if tid not in titleSet: continue
expandTitle[tid] = rest
writers = dict((
ad,
file(TMP_DATA_DIR_PATH +
'user_title_desc/%s.user_title_desc.dat' % ad, 'w'))
for ad in adSet)
for adid in ad2Profile:
profile = ad2Profile[adid]['profile']
nonClick_profile = ad2Profile[adid]['nonClick_profile']
click_profile = ad2Profile[adid]['click_profile']
for userid in profile:
click_title = '|'.join(expandTitle[key]
for key in click_profile[userid]['title'])
title = '|'.join(expandTitle[key]
for key in profile[userid]['title'])
nonclick_title = '|'.join(
expandTitle[key] for key in nonClick_profile[userid]['title'])
click_desc = '|'.join(expandDesc[key]
for key in click_profile[userid]['desc'])
desc = '|'.join(expandDesc[key] for key in profile[userid]['desc'])
nonclick_desc = '|'.join(
expandDesc[key] for key in nonClick_profile[userid]['desc'])
writers[adid].write('\x01'.join([
adid, userid, title, click_title, nonclick_title, desc,
click_desc, nonclick_desc
]))
writers[adid].write('\n')
for ad in writers:
writers[ad].close()
import util.logging as log
from database.connection import Database
import manager
log.line("#####################")
log.line("# #")
log.line("# Scuti Server #")
log.line("# #")
log.line("#####################")
scuti.clients = {}
try:
scuti.db = Database("localhost", "root", "", "scuti")
log.info("Connected to database!")
# Room
manager.room.load_rooms()
log.info("Room manager loaded!")
# Users
manager.user.load_users()
log.info("User manager loaded!")
except Exception as e:
log.error(str(e))
exit()
log.line()
log.info("Server online!")
server = SimpleWebSocketServer("", 3000, Server)
server.serveforever()
def log_heartbeat(cfg):
while True:
logging.info(cfg, "us_finance_daily_miner: heartbeat message.")
time.sleep(30 * 60)
def run_upload(cfg):
logging.info(cfg, 'uploading us daily')
upload.daily.upload.upload(cfg)
logging.info(cfg, 'uplaod complete')
