def case_level():
sentiment_dir = '/home/' + username + '/VADER_DATA_STORE/'
similarity_dir = '/home/' + username + '/SIMILARITY_DATA_STORE/'
if (demo_local):
sentiment_dir = '../VADER_DATA_STORE'
similarity_dir = '../SIMILARITY_DATA_STORE'
outDir = '../Aggregate'
list_similarity_dir = util.getDirectoryList(similarity_dir)
outDirectory = outDir + '/CaseLevel'
for directory in list_similarity_dir:
if not directory.endswith('zip'):
util.createDirectory(outDirectory)
util.createDirectory(outDirectory + "/" + directory)
files = util.getFilesListFromDir(directory, False)
for file in files:
sentiment_list = util.getDataFromPickle(
file, sentiment_dir + "/" + directory + '/')
similarity_list = util.getDataFromPickle(
file, similarity_dir + "/" + directory + '/')
for similarity in similarity_list:
similarity[:] = [
util.normalize_similarity(score)
for score in similarity
]
if len(similarity_list) == len(sentiment_list):
ss = np.dot(sentiment_list, similarity_list)
if len(sentiment_list) == 0:
util.writeToPickle(0, outDirectory, directory, file)
else:
util.writeToPickle(ss / len(sentiment_list),
outDirectory, directory, file)
def circuityear_level():
circuityear_case_dict = util.getDataFromPickle('circuit_year_level', '../')
case_to_path_dict, case_set = pjd.get_relative_path_of_cases()
if demo_local:
outDir = '../Aggregate'
outDirectory = outDir + "/CircuitYearLevel"
util.createDirectory(outDirectory)
err_count = 0
for circuityear, case_list in circuityear_case_dict.items():
circuit = circuityear[0]
year = circuityear[1]
if year > 1963:
current_circuityear_score = np.zeros(40)
case_count = 0
for case_id in case_list:
if case_id in case_set:
err_count += 1
else:
pass
if case_id in case_to_path_dict:
case_count += 1
path = case_to_path_dict[case_id]
current_score = pkl.load(open(path, 'rb'))
current_circuityear_score += current_score
if not case_count == 0:
score = current_circuityear_score / case_count
file = '{0}_{1}.p'.format(circuit, year)
util.writeToPickle(score, outDirectory, '', file)
if err_count == 0:
print("No change needed")
else:
print("Change case to path method")
'''
def createDailyFolder():
global today
today = datetime.now()
today = today.strftime("%Y-%m-%d")
print(today)
createDirectory(f"output/{today}")
def search_active_AH(driver):
'''
1. get all references of selected auction houses as auctionHouses
2. sort auctionHouses, depending on the parameter (if normal sort or reverse)
3. uncheck all auction houses
4. traverse all auction houses, starting with the first element
5. upon selecting the first element, click the auction house (select the checkbox)
6. click search
7. after search finish, proceed with the next auction house in the traverse list
'''
createDirectory()
open_dropdownbox(driver)
not_first_run = False
sorted, webElement = auctionHouse_webElement(driver)
print(f"{sorted.items()}")
# print(f"{*sorted.items()}")
for ah in sorted:
sleep(5)
if not_first_run:
open_dropdownbox(driver)
unselect_AH(driver)
print(f"Now searching for: {ah}...")
webElement[ah].click()
one_AH_search(driver)
print(f"Finished searching for: {ah}")
not_first_run = True
def judge_level_usable():
data_frame = pjd.get_case_level_data_frame()
judge_to_case_dict = pjd.create_dict_of_judges_cases(data_frame)
case_to_path_dict, case_set = pjd.get_relative_path_of_cases()
if demo_local:
outDir = '../Aggregate'
outDirectory = outDir + "/JudgeLevelUsable"
util.createDirectory(outDirectory)
err_count = 0
for judge, case_list in judge_to_case_dict.items():
current_judge_score = np.zeros(40)
case_count = 0
for case_id in case_list:
if case_id in case_set:
err_count += 1
else:
pass
if case_id in case_to_path_dict:
case_count += 1
path = case_to_path_dict[case_id]
current_score = pkl.load(open(path, 'rb'))
current_judge_score += current_score
if not case_count == 0:
score = current_judge_score / case_count
file = judge + '.p'
util.writeToPickle(score, outDirectory, '', file)
if err_count == 0:
print("No change needed")
else:
print("Change case to path method")
def __init__(self):
self.config = load_config(CONFIG_FILE)
self.config['gpx'] = self.config['dataDir'] + '/gpx'
self.config['json'] = self.config['dataDir'] + '/json'
createDirectory(self, self.config['gpx'])
createDirectory(self, self.config['json'])
def install_notifications(self):
src = u"%scontents/%s.notifyrc" % (self.applet.package().path(), self.applet.metadata.pluginName())
dst = self.kdehome+"share/apps/%s/%s.notifyrc" % (self.applet.metadata.pluginName(), self.applet.metadata.pluginName())
print u"Installing %s to %s" % (src, dst)
if os.path.exists(self.kdehome+"share/apps"):
createDirectory(self.kdehome+"share/apps/%s" % self.applet.metadata.pluginName())
try:
shutil.copy(src, dst)
except IOError as (errno, strerror):
print "I/O error({0}): {1}".format(errno, strerror)
print "Problem writing to file: %s" % dst
except:
def check_case_exist():
data_frame = pjd.get_case_level_data_frame()
judge_to_case_dict = pjd.create_dict_of_judges_cases(data_frame)
case_to_path_dict = pjd.get_relative_path_of_cases()
if demo_local:
outDir = '../Aggregate'
outDirectory = outDir + "/JudgeLevelErrors"
util.createDirectory(outDirectory)
for judge, case_list in judge_to_case_dict.items():
case_count = 0
for case_id in case_list:
if case_id in case_to_path_dict:
case_count += 1
if not case_count == len(case_list):
file = judge + ".p"
util.writeToPickle([len(case_list), case_count], outDirectory, '',
file)
def getAttachment(self, mode):
workingDir = os.path.abspath(os.curdir)
count = 0
if mode.lower() == "unseen":
mails = self.getUnseenMails()
else:
mails = self.getAllMails()
# print(len(mails))
for mail in mails:
count += 1
mail_address = mail.from_
mail_date = mail.date.date()
# print(len(mail.attachments))
download_folder = workingDir + f"/data/{mail_date}"
print(download_folder)
createDirectory(download_folder)
for attachment in mail.attachments:
print(attachment)
filename, filecontent = (attachment.filename,
attachment.payload)
if ".XLSX" in filename:
try:
arr = filename.split(".XLSX")
filename = arr[0] + " (" + str(count) + ").XLSX"
except:
pass
print(filename)
if mail_address in MANAGER_MAIL_LIST:
folder_cate = "thanhpham"
elif mail_address in SAlE_MAIL_LIST:
folder_cate = "donhang"
elif mail_address in OPERATOR_MAIL_LIST:
folder_cate = "hopdong"
else:
continue
category_folder = f"{download_folder}/{folder_cate}"
download_path = f"{download_folder}/{folder_cate}/{filename}"
createDirectory(category_folder)
print(download_path)
print(filename)
if ".XLSX" in filename or ".xlsx" in filename:
with open(download_path, "wb") as fp:
fp.write(filecontent)
def newFolder(mw, path):
dlg = DlgNewObject(None, path, "folder", "newfolder", mw)
result = dlg.exec()
if result == QDialog.Accepted:
mw.showMessage("Create folder %s" % dlg.rname)
if utils.createDirectory(os.path.dirname(dlg.rname),
os.path.basename(dlg.rname)):
mw.showMessage("New folder %s created" % dlg.rname)
else:
mw.showMessage("Can't create %s" % dlg.rname)
def main():
if (not demo):
root_Directory = 'data/clean_Mar_20'
if (demo_local):
global outDir
outDir = "SNLP_sentiment"
root_Directory = '../../Data/clean_Mar_20'
list_of_dirs = util.getDirectoryList(root_Directory)
for directory in list_of_dirs:
if not directory.endswith('zip'):
year = int(directory)
if year >= 1964:
print(directory)
util.createDirectory(outDir)
util.createDirectory(outDir + "/" + directory)
files = util.getFilesListFromDir(directory)
for file_name in files:
para_list = util.getParaListFromFile(
file_name, directory)
avgParaSentimentList, paraSentimentList = getParaSentimentList(
para_list)
util.writeToPickle(paraSentimentList,
outDir,
directory,
file_name,
avg=False)
util.writeToPickle(avgParaSentimentList,
outDir,
directory,
file_name,
avg=True)
else:
para_list = [
"DRUMMOND, C. J. The schooner American was at Oswego in the fall of 1872, and took in a cargo of coal for Chicago, leaving Oswego on the tenth of November. A general bill of lading was given, and a high price charged for the transportation of the coal from Oswego to Chicago, being $2.75 per ton. The schooner met with adverse winds and did not arrive at Port Huron until November 29th. The weather, according to the testimony of the witnesses, was very inclement that fall, and the captain concluded that the safest course was to strip the vessel and lay up at Port Huron. The schooner accordingly remained there with her cargo during the winter, and the coal was not delivered in Chicago or received by the consignees until May 8, 1873, when the spring freight was paid by the consignees on the coal, being much less than that charged in the bill of lading. After the coal had been thus delivered by the schooner to the consignees, a libel was filed claiming the amount of freight stated in the bill of lading, the consignees having refused to pay any more than the spring price of freight. The case went to proof before the district court, where the libel was dismissed; but a cross-libel having been filed claiming that the captain of the American was negligent in wintering at Port Hur on, and that the vessel should have come on in the fall of 1872, the district court gave a decree on the cross-libel for damages against the libelants in consequence of the supposed negligence of the captain. From t hese decrees the libelants have appealed to this court, and the question is whether the decrees of the district court are right.",
"Several cities, New York City in particular for this paper, have a 311 24-hour hot line and online service, which allows anyone, residents and tourists, to report a non-emergency problem. Reported 311 problems are passed along to government services, who address and solve the problem. The records of 311 calls are publicly open and updated daily.",
"Analysis of 311 calls can clearly be of great use for a wide variety of purposes, ranging from a rich understanding of the status of a city to the effectiveness of the government services in addressing such calls. Ideally, the analysis can also support a prediction of future 311 calls, which would enable the assignment of service resources by the city government.",
"We have been extensively analyzing 311 calls in NYC. In this paper, we profile the data set and highlight a few interesting facts. We provide statistics along complaint types, geolocation, and temporal patterns and show the diversity of the big 311 data along those dimensions. We then discuss the prediction problem of number of calls, where we experiment with different sets of semantic features. We show that the prediction error for different complaint types can significantly vary if some features are not considered."
]
avgParaSentimentList, paraSentimentList = getParaSentimentList(
para_list)
print(avgParaSentimentList)
print(paraSentimentList)
def newFolder(self):
dlg = dialog.DlgNewObject(self.name, self.path, "folder", "newfolder",
self.parent)
result = dlg.exec()
if result == QDialog.Accepted:
self.parent.showMessage("Create folder %s" % dlg.rname)
if utils.createDirectory(os.path.dirname(dlg.rname),
os.path.basename(dlg.rname)):
self.parent.showMessage("New folder %s created" % dlg.rname)
else:
self.parent.showMessage("Can't create %s" % dlg.rname)
def sorted_AH_search(driver):
# open dropdownbox
# retrieve all auction house info
# sort all auction houses ascending order
#
createDirectory()
open_dropdownbox(driver)
not_first_run = False
sorted_ah_list, web_element = sorted_auction_search(driver)
for auction_house in sorted_ah_list:
sleep(5)
if not_first_run:
open_dropdownbox(driver)
unselect_AH(driver)
print(f"Now searching for: {auction_house}...")
web_element[auction_house].click()
one_AH_search(driver)
print(f"Finished searching for: {auction_house}...")
not_first_run = True
def judge_level():
data_frame = pjd.get_case_level_data_frame()
judge_to_case_dict = pjd.create_dict_of_judges_cases(data_frame)
case_to_path_dict = pjd.get_relative_path_of_cases()
if demo_local:
outDir = '../Aggregate'
outDirectory = outDir + "/JudgeLevel"
util.createDirectory(outDirectory)
for judge, case_list in judge_to_case_dict.items():
current_judge_score = np.zeros(40)
for case_id in case_list:
if case_id in case_to_path_dict:
path = case_to_path_dict[case_id]
current_score = pkl.load(open(path, 'rb'))
current_judge_score += current_score
if len(case_list) == 0:
score = np.zeros(40)
else:
score = current_judge_score / len(case_list)
file = judge + '.p'
util.writeToPickle(score, outDirectory, '', file)
def split_bam_task_list():
"""populates task inputs and outputs"""
(sentinel_path, results_path, haplotype_path, cancer_dir_path,
tmpbams_path, finalbams_path) = taskHelpers.GetProjectNamePathRunID()
inputs = []
outputs = []
prev_sentinels = []
prev_sentinels.append(taskHelpers.CreateFileList('None', -1,
sentinel_path))
split_path = "/".join([results_path, "splitbams"])
params.SetSplitBamsPath(split_path)
utils.createDirectory(split_path)
sentinels = taskHelpers.CreateFileList('{0}_split.sentinel', 1,
sentinel_path)
inputs.append(taskHelpers.CreateFileList('bam', 1, split_path + "/"))
outputs.append(
taskHelpers.CreateFileList('chr{1}.bam', 22, split_path + "/"))
sample_ids = taskHelpers.CreateFileList('{0}', 1, '')
job_parameters = taskHelpers.CreateTaskList(inputs, sentinels, outputs,
sample_ids, prev_sentinels)
for job in job_parameters:
yield job
def run(self):
try:
self.login()
self.openTradebook()
for period in self.report:
month, year = (period.get("month"), period.get("year"))
self.setTradebookDate(month, year)
outputFolder = os.path.join(
os.getcwd(), 'output', '{}_{}'.format(year, month))
if (not isDirectoryExists(outputFolder)):
createDirectory(outputFolder)
for k, v in SEGMENTS.items():
self.setTradebookSegment(v)
sleep(5)
isSaved = self.getSnapshot(getOutputPath(
outputFolder, '{}.png'.format(k)))
print("took snapshot for {} and saved={}".format(k, isSaved))
createPdf(outputFolder, "tradebook.pdf")
finally:
if(self.loggedIn):
self.logout()
self.closeDriver()
def createDirectory(meta):
hash = utils.getHash(metadata.getArtist(meta) + metadata.getTitle(meta))
directory = u"{}/{}".format(OUTPUT, hash)
utils.createDirectory(directory)
return directory
def nextResults(webdriver):
'''
#1 Get current page number
#2 Get Next page number reference
'''
getActiveLink = "//ul[@class='pagination margin-top-bottom-none']//li[@class='active']"
getNextLink = "/following-sibling::li[1]//a"
infoList = []
startDC = time()
isEnd = False
auctionHouseName = ""
while not isEnd:
print("Incur python to sleep..")
sleep(SLEEP_TIME)
expandVehicleInfoIdirect(webdriver)
auctionHouseName = getAuctionHouse(webdriver)
# print(retrieveInfoTest(webdriver))
start = time()
infoList.extend(retrieveInfoUpd(webdriver))
end = time()
activePage = WebDriverWait(webdriver, WAIT_TIME).until(
EC.presence_of_element_located((By.XPATH, getActiveLink)))
# print(f"Active Page: {activePage.text}")
nextPage = WebDriverWait(webdriver, WAIT_TIME).until(
EC.presence_of_element_located(
(By.XPATH, f"{getActiveLink}/{getNextLink}")))
print(f"Checking [Page {activePage.text}]..")
# activePage.get_attribute('class') == 'disabled' or
if nextPage.text == "»":
isEnd = True
print("Traverse reached last page..")
print()
break
else:
print(
f"Finished checking [{auctionHouseName}, Page {activePage.text}] in {(end-start):.1f} seconds."
)
print()
print(f"Next is [Page {nextPage.text}]..")
nextPage.click()
else:
print("Traverse has reached the end.")
print()
endDC = time()
timeDC = endDC - startDC
print(f"Finished collecting data in {timeDC} seconds.")
print(f"Now, getting to check for errors..")
print()
start = time()
checkErrorList = errorCheckUpd(infoList)
populate_errors = dictErrors(checkErrorList[1])
# print(populate_errors)
print("----------------------------------------------------------")
printErrors(populate_errors)
printToFile(timeDC, createDirectory(), auctionHouseName, populate_errors)
end = time()
print(f"Error checking in {end-start} seconds.")
#!/usr/bin/python
#-*- coding:utf-8 -*-
import utils
import os
import string
if __name__ == '__main__':
utils.createDirectory('figure')
options = utils.getOptions()
dataFileName = utils.getDataFileName(options)
vpFileName = utils.getVPFileName(options)
datas = utils.getDataInFile(dataFileName)
vps = utils.readDataFromFile(vpFileName)
curDatas = datas
for i in xrange(len(vps)):
print i
n = len(curDatas)
x = [[] for j in xrange(len(vps[i]) + 1)]
for j in xrange(n):
nextPosition = utils.hammingDistance(vps[i], curDatas[j])
x[nextPosition].append(j)
mx, position = (0, 0)
xp = []
yp = []
for j in xrange(len(vps[i]) + 1):
xp.append(j)
yp.append(len(x[j]))
if mx < len(x[j]):
mx, position = (len(x[j]), j)
def initialize_output(self):
createDirectory( self.outputDir )
os.chdir( self.outputDir )
print(
"Example: extract_keys.py ./unpacked/MBOOT.img ./keys 0x169e00 0x450")
quit()
mboot = sys.argv[1]
outFolder = sys.argv[2] if len(sys.argv) >= 3 else defOutFolder
offestStr = sys.argv[3] if len(sys.argv) >= 4 else defOffet
sizeStr = sys.argv[4] if len(sys.argv) >= 5 else defSize
#hwKey = sys.argv[5] if len(sys.argv) >= 6 else defKey
offset = int(offestStr, 16)
size = int(sizeStr, 16)
# Create out directory
print("[i] Create output directory")
utils.createDirectory(outFolder)
# Get the key bank section and store it
outEncKeyBankFile = os.path.join(outFolder, 'key_bank.bin')
print("[i] Save mstar key bank to {}".format(outEncKeyBankFile))
utils.copyPart(mboot, outEncKeyBankFile, offset, size)
# Unpack the key bank to key bank structure
print("[i] Unpack key bank structure")
keyBankBytes = utils.loadPart(outEncKeyBankFile, 0, size)
keyBank = utils.unpackStructure(CUSTOMER_KEY_BANK, keyBankBytes)
if (DEBUG):
# Print all
print("[i] u8SecIdentify:\n{}".format(
utils.hexString(keyBank.customer.u8SecIdentify)))
sqlCommon = sqlCommon % (sqlConf["db.user"], sqlConf["db.pass"],
sqlConf["db.host"])
sqlExecuteFile = sqlCommon + " -i \"%s\""
sqlExecuteQuery = sqlCommon + " -Q \"%s\""
if operation == "executefile":
sqlFile = sys.argv[2]
executeSqlFile(sqlFile)
elif operation == "executequery":
sqlQuery = sys.argv[2]
executeSqlQuery(sqlExecuteQuery % sqlQuery)
elif operation == "backup":
currentTime = utils.getCurrentTime()
newDbBackUpPath = os.path.join(dbBackUpPath, currentTime)
utils.show("Creating %s" % newDbBackUpPath)
utils.createDirectory(newDbBackUpPath)
backUpQuery = sqlExecuteQuery % sqlBackUpQuery
executeSqlQuery(
backUpQuery %
("OMG_GameWEB", os.path.join(newDbBackUpPath, "OMG_GameWeb.bak")))
executeSqlQuery(
backUpQuery %
("PS_Billing", os.path.join(newDbBackUpPath, "PS_Billing.bak")))
executeSqlQuery(
backUpQuery %
("PS_ChatLog", os.path.join(newDbBackUpPath, "PS_ChatLog.bak")))
executeSqlQuery(
backUpQuery %
("PS_GameData", os.path.join(newDbBackUpPath, "PS_GameData.bak")))
executeSqlQuery(
datas = utils.getDataInFile(dataFileName)
querys = utils.readDataFromFile(queryFileName)
vps = utils.readDataFromFile(vpFileName)
print len(datas), len(querys), len(vps)
cdsDatas = []
for i in xrange(len(datas)):
t = []
for j in xrange(len(vps)):
t.append(utils.hammingDistance(datas[i], vps[j]))
cdsDatas.append(t)
utils.writeDataToFile(cdsDataFileName, cdsDatas)
cdsQuerys = []
for i in xrange(len(querys)):
t = []
for j in xrange(len(vps)):
t.append(utils.hammingDistance(querys[i], vps[j]))
cdsQuerys.append(t)
utils.writeDataToFile(cdsQueryFileName, cdsQuerys)
print cdsDataFileName, cdsQueryFileName
if __name__ == '__main__':
utils.createDirectory('cds_data')
utils.createDirectory('cds_query')
options = utils.getOptions()
convertNDDSToCDS(options)
vps = []
vps.append(majorPattern[0])
while len(vps) < numberOfVP:
print len(vps)
ans, ansDataIndex = -1, -1
for i in xrange(len(datas)):
ok = False
for j in xrange(len(vps)):
if datas[i] == vps[j]:
ok = True
if ok:
continue
vps.append(datas[i])
cur = calculateMany(vps)
if cur > ans:
ans, ansDataIndex = cur, i
vps = vps[:-1]
vps.append(datas[ansDataIndex])
utils.writeDataToFile(
'vp/vp_%d_%d_%d_%s.txt' % (dim, numberOfVP, cardinality, typeOfVP),
vps)
#utils.writeDataToFile('vp/vp_%d_%d_%d_%s.txt'%(dim,numberOfVP,cardinality,typeOfVP),vps)
if __name__ == '__main__':
options = utils.getOptions()
utils.createDirectory('vp')
generateVantagePointsWithManyAlgorithm(options)
#!/usr/bin/python
#-*- coding:utf-8 -*-
import utils
if __name__ == '__main__':
utils.createDirectory('rq_result')
options = utils.getOptions()
queryRange = options['queryRange']
dataFileName = utils.getDataFileName(options)
queryFileName = utils.getQueryFileName(options)
rqResultFileName= utils.getRQResultFileName(options)
datas = utils.getDataInFile(dataFileName)
querys = utils.readDataFromFile(queryFileName)
with open(rqResultFileName,'w') as fp:
for i in xrange(len(querys)):
print '#%d'%i
fp.write(('#%d'%(i))+'\n')
ans = []
for j in xrange(len(datas)):
dist = utils.hammingDistance(querys[i],datas[j])
if dist <= queryRange:
ans.append(datas[j])
for data in ans:
for j in xrange(len(data)):
fp.write('%c '%data[j])
fp.write('\n')
if len(sys.argv) == 1:
print("Usage: unpack.py <firmware> <output folder [default: ./unpacked/]>")
quit()
inputFile = sys.argv[1]
if not os.path.exists(inputFile):
print("No such file: {}".format(inputFile))
quit()
if len(sys.argv) == 3:
outputDirectory = sys.argv[2]
else:
outputDirectory = 'unpacked'
# Create output directory
utils.createDirectory(outputDirectory)
# Find header script
# Header size is 16KB
# Non used part is filled by 0xFF
print("[i] Analizing header ...")
header = utils.loadPart(inputFile, 0, HEADER_SIZE)
utils.copyPart(inputFile, os.path.join(outputDirectory, "~header"), 0,
HEADER_SIZE)
offset = header.find('\xff'.encode(encoding='iso-8859-1'))
if offset != -1:
headerScript = header[:offset].decode()
headerScriptFound = True
if not headerScriptFound:
#!/usr/bin/python
#-*- coding:utf-8 -*-
import os
import glob
import sys
import utils
import string
alphabet = list(string.ascii_uppercase)
def makeDictionaryKeyIsAlphabet():
ret = {}
for i in xrange(len(alphabet)):
ret[alphabet[i]] = i
return ret
if __name__ == '__main__':
utils.createDirectory('ndt_data')
utils.createDirectory('ndt_query')
dictionary = makeDictionaryKeyIsAlphabet()
dataFileNames = glob.glob('data/*.txt')
for dataFileName in dataFileNames:
print dataFileName
onlyFileName = dataFileName.split('.')[0].split('/')[1]
size = onlyFileName.split('_')[1]
dim = onlyFileName.split('_')[2]
vptype = onlyFileName.split('_')[3]
cardinality = onlyFileName.split('_')[4]
queryFileName = 'query/query_%s_%s_%s_%s.txt'%(size,dim,vptype,cardinality)
datas = utils.getDataInFile(dataFileName)
querys = utils.readDataFromFile(queryFileName)
def nextResults(webdriver):
'''
#1 Get current page number
#2 Get Next page number reference
'''
import os
getActiveLink = "//ul[@class='pagination margin-top-bottom-none']//li[@class='active']"
getNextLink = "/following-sibling::li[1]//a"
infoList = []
# addInfoList = []
startDC = time()
isEnd = False
createDirectory()
auctionHouseName = ""
while not isEnd:
print("Incur python to sleep..")
sleep(SLEEP_TIME)
expandVehicleInfoIdirect(webdriver)
results = hasNoResults(webdriver)
if not auctionHouseName:
auctionHouseName = getAuctionHouse(webdriver)
if results:
print("No results displayed..")
print(f"Data collection for [{auctionHouseName}]: incomplete")
back_to_search(webdriver)
isEnd = True
break
if time() - startDC >= 240:
print("DC reached 4 minute limit")
back_to_search(webdriver)
break
startRetrieve = time()
# infoList.extend(retrieveInfoUpd(webdriver))
retrievedInfo = retrieveAllInfo(webdriver)
infoList.extend(retrievedInfo)
print(retrievedInfo)
# addInfoList.extend(retrieveInfoDetail(webdriver))
endRetrieve = time()
activePage = WebDriverWait(webdriver, WAIT_TIME).until(
EC.presence_of_element_located((By.XPATH, getActiveLink)))
nextPage = WebDriverWait(webdriver, WAIT_TIME).until(
EC.presence_of_element_located(
(By.XPATH, f"{getActiveLink}/{getNextLink}")))
print(f"Checking [Page {activePage.text}]..")
if nextPage.text == "»":
isEnd = True
print("Traverse reached last page..")
print(f"Data collection of [{auctionHouseName}]: completed")
print()
else:
print(
f"Finished checking [{auctionHouseName}, Page {activePage.text}] in {(endRetrieve-startRetrieve):.1f} seconds."
)
print()
print(f"Next is [Page {nextPage.text}]..")
try:
nextPage.click()
except Exception as e:
print(f"Error: {e}")
print(f"No Next page..")
back_to_search(webdriver)
break
else:
print("Traverse has been stopped..")
back_to_search(webdriver)
print()
endDC = time()
timeDC = endDC - startDC
print(f"Finished collecting data in {convert_time(timeDC)} seconds.")
print(f"Will now check for errors..")
print()
startEC = time()
# checkErrorList = errorCheckUpd(infoList)
# checkErrorList = errorCheck_ibc_shuppin(infoList)
# checkErrorList = errorCheckMoreInfo(infoList, addInfoList)
checkErrorList = dataVerification(infoList)
# populate_errors = dictErrors(checkErrorList[1])
populate_errors = dictErrors_shuppin(checkErrorList)
# print(populate_errors)
print("----------------------------------------------------------")
printErrors(populate_errors)
endEC = time()
timeEC = endEC - startEC
# printToFile((timeDC, timeEC),
# auctionHouseName, populate_errors)
printToFile_shuppin((timeDC, timeEC), auctionHouseName, populate_errors)
print(f"Error checking done in {convert_time(timeEC)} seconds.")
#!/usr/bin/python
#-*- coding:utf-8 -*-
import random
import os
import sys
import glob
import utils
if __name__ == '__main__':
utils.createDirectory('query')
dataFileNames = glob.glob('data/*.txt')
for dataFileName in dataFileNames:
onlyFileName = dataFileName.split('.')[0].split('/')[1]
size = onlyFileName.split('_')[1]
dim = onlyFileName.split('_')[2]
vptype = onlyFileName.split('_')[3]
cardinality = onlyFileName.split('_')[4]
queryFileName = 'query/query_%s_%s_%s_%s.txt'%(size,dim,vptype,cardinality)
if os.path.exists(queryFileName):
print '%s is exists'%(queryFileName)
continue
print queryFileName
datas = utils.getDataInFile(dataFileName)
queryDatas = []
for i in xrange(100):
queryDatas.append(datas[random.randrange(0,int(size))])
utils.writeDataToFile(queryFileName,queryDatas)
fp.write(','+str(c[j]))
fp.write('\n')
print dataFileName
def generateClusteredData(options):
alphabet = list(string.ascii_uppercase)
dim = options['numberOfDimension']
size = options['numberOfData']
cardinality = options['numberOfAlphabet']
cluster = int(options['distribution'][1:])
dataFileName = 'data/data_%d_%d_%d_%d.txt'%(size,dim,cluster,cardinality)
command = 'perl data/gdp.pl %d %d %d %d %s'%(size,cluster,dim,cardinality,dataFileName)
print command
utils.executeCommand(command)
print dataFileName
def generateData(options):
if options['distribution'] == 'u':
generateUniformData(options)
elif options['distribution'][0] == 'c':
generateClusteredData(options)
if __name__ == '__main__':
options = utils.getOptions()
utils.createDirectory('data')
generateData(options)
datas = utils.getDataInFile(dataFileName)
querys = utils.readDataFromFile(queryFileName)
vps = utils.readDataFromFile(vpFileName)
print len(datas),len(querys),len(vps)
cdsDatas = []
for i in xrange(len(datas)):
t = []
for j in xrange(len(vps)):
t.append(utils.hammingDistance(datas[i],vps[j]))
cdsDatas.append(t)
utils.writeDataToFile(cdsDataFileName,cdsDatas)
cdsQuerys = []
for i in xrange(len(querys)):
t = []
for j in xrange(len(vps)):
t.append(utils.hammingDistance(querys[i],vps[j]))
cdsQuerys.append(t)
utils.writeDataToFile(cdsQueryFileName,cdsQuerys)
print cdsDataFileName, cdsQueryFileName
if __name__ == '__main__':
utils.createDirectory('cds_data')
utils.createDirectory('cds_query')
options = utils.getOptions()
convertNDDSToCDS(options)
#!/usr/bin/python
#-*- coding:utf-8 -*-
import utils
import os
import string
import numpy as np
if __name__ == '__main__':
utils.createDirectory('figure_pair')
options = utils.getOptions()
dim = options['numberOfDimension']
dataFileName = utils.getDataFileName(options)
vpFileName = utils.getVPFileName(options)
datas = utils.getDataInFile(dataFileName)
vps = utils.readDataFromFile(vpFileName)
for i in xrange(len(vps)):
for j in xrange(i + 1, len(vps)):
cc = utils.calculateCorrelationCoefficient(vps[i], vps[j], datas)
cc = abs(cc)
imageFileName = utils.getFigurePairName(options, i, j, cc)
print imageFileName
if os.path.exists(imageFileName):
print '%s is exists' % imageFileName
continue
xp = []
yp = []
zp = []
zcnt = [[0 for ii in xrange(dim + 1)] for jj in xrange(dim + 1)]
for k in xrange(len(datas)):
root_Directory = 'data/clean_Mar_20'
list_of_dirs = utils.getDirectoryList(root_Directory)
# Training the phraser
phrase2id, id2phrase = train_phraser()
# Phrase vector of the thermometers
thermometer_vector = [
phrase_similarity.PhraseVector(thermometer) for thermometer in thermometers
]
# Getting data from the phraser
for directory in list_of_dirs:
if not directory.endswith('zip'):
print(directory)
utils.createDirectory("similarities")
utils.createDirectory("similarities/" + directory)
files = utils.getFilesListFromDir(directory)
for file_name in files:
para_list = utils.getParaListFromFile(file_name, directory)
caseLevelParaSimilarityVectorsCombined = []
for para in para_list:
set_vector = set()
sentences = sent_tokenize(para)
for sentence in sentences:
# split into words and get POS tags
words = [w.lower() for w in sentence.split()]
phraseids = apply_phraser(words, phrase2id)
for phraseId in phraseids:
def initialize():
try:
utils.createDirectory(results_path)
utils.createDirectory(cancer_dir_path)
utils.createDirectory(haplotype_path)
utils.createDirectory(tmpbams_path)
utils.createDirectory(finalbams_path)
event_list=['gain','loss']
gaincnv = params.GetGainCNV()
losscnv = params.GetLossCNV()
pipelineHelpers.Logging("INFO", log, " --- Initializing input files --- ")
vcf_path = bamhelp.GetVCF()
exons_path = bamhelp.GetExons()
reference_path = bamhelp.GetRef()
vpath, vcf = os.path.split(vcf_path)
phasedvcf = "/".join([results_path, sub('.vcf$', '_phased.vcf.gz', vcf)])
vcftobed = "/".join([results_path, sub('.vcf$', '.bed', vcf)])
hap1vcf = "/".join([results_path,"hap1_het.vcf"])
hap2vcf = "/".join([results_path, "hap2_het.vcf"])
hap1vcffiltered = "/".join([results_path, "hap1_het_filtered"])
hap2vcffiltered = "/".join([results_path, "hap2_het_filtered"])
hap1vcffilteredtobed = "/".join([results_path, "hap1_het_filtered.bed"])
hap2vcffilteredtobed = "/".join([results_path, "hap2_het_filtered.bed"])
phased_bed = "/".join([results_path, "PHASED.BED"])
utils.phaseVCF(vcf_path, phasedvcf)
utils.getVCFHaplotypes(phasedvcf, hap1vcf, hap2vcf)
utils.thinVCF(hap1vcf, hap1vcffiltered)
utils.thinVCF(hap2vcf, hap2vcffiltered)
utils.convertvcftobed(hap1vcffiltered+".recode.vcf", hap1vcffilteredtobed)
utils.convertvcftobed(hap2vcffiltered+".recode.vcf", hap2vcffilteredtobed)
cmd1 = """sed -i 's/$/\thap1/' """+ hap1vcffilteredtobed
cmd2 = """sed -i 's/$/\thap2/' """+ hap2vcffilteredtobed
cmd3 = "cat " + hap1vcffilteredtobed + " " + hap2vcffilteredtobed + " > " + 'tmp.bed'
cmd4 = "sort -V -k1,1 -k2,2 tmp.bed > " + phased_bed
utils.runCommand(cmd1)
utils.runCommand(cmd2)
utils.runCommand(cmd3)
utils.runCommand(cmd4)
os.remove('tmp.bed')
for event in event_list:
roibed = "/".join([haplotype_path, event + "_roi.bed"])
exonsinroibed = "/".join([haplotype_path, event + "_exons_in_roi.bed"])
nonhetbed = "/".join([haplotype_path, event + "_non_het.bed"])
hetbed = "/".join([haplotype_path, event + "_het.bed"])
hetsnpbed = "/".join([haplotype_path, event + "_het_snp.bed"])
if (locals()[event + 'cnv']):
utils.intersectBed( exons_path, locals()[event + 'cnv'], exonsinroibed, wa=True)
utils.intersectBed(phased_bed, exonsinroibed, hetsnpbed, wa=True)
utils.splitBed(exonsinroibed, event+'_exons_in_roi_')
utils.splitBed(hetsnpbed, event+'_het_snp_')
except:
pipelineHelpers.Logging("INFO", log, "Initialization error !")
raise
pipelineHelpers.Logging("ERROR",log, "--- initialization complete ---")
return
def train_phraser(max_phrase_length=3,
stemmer=None,
vocab=None,
min_doc_freq=None,
min_gmean=None):
# take documents and get POS-gram dictionary
numdocs = 0
docfreqs = Counter()
termfreqs = Counter()
root_Directory = 'data/clean_Mar_20'
list_of_dirs = utils.getDirectoryList(root_Directory)
for directory in list_of_dirs:
if not directory.endswith('zip'):
print(directory)
utils.createDirectory("similarities")
utils.createDirectory("similarities/" + directory)
files = utils.getFilesListFromDir(directory)
for file_name in files:
para_list = utils.getParaListFromFile(file_name, directory)
for para in para_list:
numdocs += 1
docgrams = set()
# split into sentences
sentences = sent_tokenize(para)
for sentence in sentences:
# split into words and get POS tags
words = sentence.split()
tagwords = tagsentence(words, stemmer, vocab)
for n in range(1, max_phrase_length + 1):
rawgrams = ngrams(tagwords, n)
for rawgram in rawgrams:
# skip grams that have words not in vocab
if None in rawgram:
continue
gramtags = ''.join([x[1][0] for x in rawgram])
if gramtags in tagpatterns:
# if tag sequence is allowed, add to counter
gram = '_'.join([x[0] for x in rawgram])
termfreqs[gram] += 1
docgrams.add(gram)
docfreqs.update(docgrams)
# filter vocabulary based on document frequency and make gram ids
gram2id = {}
id2gram = {}
if min_doc_freq is None:
min_doc_freq = round(numdocs / 200) + 1
i = 0
for (phrase, v) in docfreqs.most_common():
if v < min_doc_freq:
break
if min_gmean is not None:
# check geometric mean association
n = v.count('_') + 1
if len(n) >= 2:
gscore = gmean(phrase, termfreqs)
if gscore[n] < min_gmean[n]:
continue
gram2id[phrase] = i
id2gram[i] = phrase
i += 1
return gram2id, id2gram
getCornerPoints(0,[],dim,alphabet,cardinality)
vps = []
vps.append([ 'A' for i in xrange(dim) ])
#vps.append([ 'B' for i in xrange(dim) ])
#vps.append([ 'C' for i in xrange(dim) ])
#vps.append([ 'D' for i in xrange(dim) ])
for i in xrange(numberOfVP - 1):
print i
#mx, mx_idx = (0,0)
mn, mn_idx = (987654321.0,0)
for j in xrange(len(cornerPoints)):
variation = getTotalCostFunction(vps,cornerPoints[j],base)
if mn > variation:
mn, mn_idx = (variation,j)
#if mx < dist:
# mx, mx_idx = (dist,j)
print mn,mn_idx,cornerPoints[mn_idx]
vps.append(cornerPoints[mn_idx])
utils.writeDataToFile('vp/vp_%d_%d_%d_greedy.txt'%(dim,numberOfVP,cardinality),vps)
if __name__ == '__main__':
options = utils.getOptions()
utils.createDirectory('vp')
#generateVantagePoints(options)
#generateGreedyVantagePoints(options)
#generateAllRandomVantagePoints(options)
generateHeuristicVantagePoints(options)
#!/usr/bin/python
#-*- coding:utf-8 -*-
import utils
import os
import string
import numpy as np
if __name__ == '__main__':
utils.createDirectory('figure_pair')
options = utils.getOptions()
dim = options['numberOfDimension']
dataFileName = utils.getDataFileName(options)
vpFileName = utils.getVPFileName(options)
datas = utils.getDataInFile(dataFileName)
vps = utils.readDataFromFile(vpFileName)
for i in xrange(len(vps)):
for j in xrange(i+1,len(vps)):
cc = utils.calculateCorrelationCoefficient(vps[i],vps[j],datas)
cc = abs(cc)
imageFileName = utils.getFigurePairName(options,i,j,cc)
print imageFileName
if os.path.exists(imageFileName):
print '%s is exists'%imageFileName
continue
xp = []
yp = []
zp = []
zcnt = [ [ 0 for ii in xrange(dim+1) ] for jj in xrange(dim+1) ]
for k in xrange(len(datas)):
# Parts
parts = list(filter(lambda s: s.startswith('part/'), config.sections()))
print("\n")
print("[i] Date: {}".format(time.strftime("%d/%m/%Y %H:%M:%S")))
print("[i] Firmware file name: {}".format(firmwareFileName))
print("[i] Project folder: {}".format(projectFolder))
print("[i] Use hex values: {}".format(useHexValuesPrefix))
print("[i] Script firmware filename: {}".format(SCRIPT_FIRMWARE_FILE_NAME))
print("[i] DRAM_BUF_ADDR: {}".format(DRAM_BUF_ADDR))
print("[i] MAGIC_FOOTER: {}".format(MAGIC_FOOTER))
print("[i] HEADER_SIZE: {}".format(HEADER_SIZE))
# Create working directory
print('[i] Create working directory ...')
utils.createDirectory(tmpDir)
print('[i] Generating header and bin ...')
# Initial empty bin to store merged parts
open(binPart, 'w').close()
with open(headerPart, 'wb') as header:
header.write('#\n'.encode())
header.write('# Generated by mstar-bin-tools\n'.encode())
header.write('# https://github.com/dipcore/mstar-bin-tool\n'.encode())
header.write('# [email protected]\n'.encode())
header.write('#\n\n'.encode())
# Directive tool
directive = utils.directive(header, DRAM_BUF_ADDR, useHexValuesPrefix)
#!/usr/bin/python
#-*- coding:utf-8 -*-
import utils
import os
import string
if __name__ == '__main__':
utils.createDirectory('figure')
options = utils.getOptions()
dataFileName = utils.getDataFileName(options)
vpFileName = utils.getVPFileName(options)
datas = utils.getDataInFile(dataFileName)
vps = utils.readDataFromFile(vpFileName)
curDatas = datas
for i in xrange(len(vps)):
print i
n = len(curDatas)
x = [ [] for j in xrange(len(vps[i])+1) ]
for j in xrange(n):
nextPosition = utils.hammingDistance(vps[i],curDatas[j])
x[nextPosition].append(j)
mx, position = (0,0)
xp = []
yp = []
for j in xrange(len(vps[i])+1):
xp.append(j)
yp.append(len(x[j]))
if mx < len(x[j]):
mx, position = (len(x[j]),j)
