def main():
datasetDirPath = args['dataset_dir']
separate()
print("Number of images must equals to Number of xmls")
image_paths = [
f.replace('\\', '/') for f in glob.glob(datasetDirPath + '**/*.jpg')
]
xml_paths = [
f.replace('\\', '/') for f in glob.glob(datasetDirPath + '**/*.xml')
]
print(
f"\tnumber of images = {len(image_paths)}, number of xmls = {len(xml_paths)}"
)
separate()
print("Validate labels")
char_label_set = set()
for xml_path in xml_paths:
tree = ET.parse(xml_path)
root = tree.getroot()
for obj in root.findall('object'):
label = obj.find('name').text
char_label_set.add(label)
print(sorted(list(char_label_set)))
print(f"Number of all label : {len(char_label_set)}")
separate()
print("List Duplicate images and xmls")
image_names = [getFileName(image_path) for image_path in image_paths]
duplicated_image = [
count for item, count in Counter(image_names).items() if count > 1
]
xml_names = [getFileName(xml_path) for xml_path in xml_paths]
duplicated_xml = [
count for item, count in Counter(xml_names).items() if count > 1
]
print(f"\tDuplicated Images = ", duplicated_image)
print(f"\tDuplicated Xmls = ", duplicated_xml)
separate()
print("\tImage which missing xml ", set(image_names) - set(xml_names))
print("\tXml which missing image ", set(xml_names) - set(image_names))
separate()
print("Plotting Bounding Box")
font_dir = './validate/fonts/angsa.ttf'
output_dir = './validate/tmp/'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
for image_path in image_paths:
image_name = getFileName(image_path)
xml_path = os.path.join(datasetDirPath, image_name + '.xml')
drawedImage = drawBoundingBox(image_path, xml_path, font_dir=font_dir)
drawedImage.save(
os.path.join(output_dir,
getFileName(image_path) + '.jpg'))
print("Done")
separate()
def getFileList(self):
""" Use self.df to generate the list of files that need to be
uploaded to Dropbox. In addition to this the corresponding
file size, file on local Dropbox, file on Dropbox website, and
the new directories on local Dropbox that need to be made are
created.
Usage:
-----
self.getFileList()
Returns:
-------
NULL
Creates:
-------
self.fileNameList : array of str
List of all the files uploading to Dropbox.
self.fileSizeList : array of float
List of corresponding file size in bytes.
self.dropboxFileList : array of str
List of corresponding files with local Dropbox path.
self.dropboxWebFileList : array of str
List of corresponding files with cloud Dropbox path.
self.dropboxDirList : array of str
List of local Dropbox directories that need to be created.
"""
fileNameList,fileSizeList,dropboxFileList,dropboxWebFileList,dropboxDirList = [],[],[],[],[]
for inputFile, outputDir in self.df.values:
if (os.path.isfile(inputFile)):
fileSize = os.path.getsize(inputFile)
fileNameList.append(inputFile)
fileSizeList.append(fileSize)
dropboxFileList.append(outputDir + '/' +
utils.getFileName(inputFile))
dropboxWebFileList.append(
utils.getDropboxWebFileName(
outputDir + '/' + utils.getFileName(inputFile),
self.dropboxDir))
dropboxDirList.append(outputDir)
elif (os.path.isdir(inputFile)):
inputDir = inputFile
tempFileNameList, tempFileSizeList, tempDropboxFileList, tempDropboxWebFileList, tempDropboxDirList = self.getFilesInDir(
inputDir, outputDir)
for a, b, c, d, e in zip(tempFileNameList, tempFileSizeList,
tempDropboxFileList,
tempDropboxWebFileList,
tempDropboxDirList):
fileNameList.append(a)
fileSizeList.append(b)
dropboxFileList.append(c)
dropboxWebFileList.append(d)
dropboxDirList.append(e)
self.fileNameList = numpy.asarray(fileNameList)
self.fileSizeList = numpy.asarray(fileSizeList)
self.dropboxFileList = numpy.asarray(dropboxFileList)
self.dropboxWebFileList = numpy.asarray(dropboxWebFileList)
self.dropboxDirList = numpy.unique(dropboxDirList)
def createPlateXml(plateImage, newCharObjects, imageFilePath):
imageFilePath.replace('\\', '/')
folder = imageFilePath.split('/')[-2]
fileName = utils.getFileName(imageFilePath)
xml = "<annotation>\n"
#xml = xml + "<folder>" + folder + "</folder>\n"
xml = xml + "<filename>" + fileName + "</filename>\n"
#xml = xml + "<path>" + imageFilePath + "</path>\n"
xml = xml + "<size>\n"
xml = xml + "\t<width>" + str(plateImage.shape[1]) + "</width>\n"
xml = xml + "\t<height>" + str(plateImage.shape[0]) + "</height>\n"
xml = xml + "\t<depth>" + "Unspecified" + "</depth>\n"
xml = xml + "</size>\n"
xml = xml + "<segmented>" + "Unspecified" + "</segmented>\n"
for charObject in newCharObjects:
xml = xml + "<object>\n"
xml = xml + "\t<name>" + charObject['name'] + "</name>\n"
xml = xml + "\t<pose>" + "Unspecified" + "</pose>\n"
xml = xml + "\t<truncated>" + "Unspecified" + "</truncated>\n"
xml = xml + "\t<difficult>" + "Unspecified" + "</difficult>\n"
xml = xml + "\t<occluded>" + "Unspecified" + "</occluded>\n"
xml = xml + "\t<bndbox>\n"
xml = xml + "\t\t<xmin>" + str(charObject['xmin']) + "</xmin>\n"
xml = xml + "\t\t<xmax>" + str(charObject['xmax']) + "</xmax>\n"
xml = xml + "\t\t<ymin>" + str(charObject['ymin']) + "</ymin>\n"
xml = xml + "\t\t<ymax>" + str(charObject['ymax']) + "</ymax>\n"
xml = xml + "\t</bndbox>\n"
xml = xml + "</object>\n"
xml = xml + "</annotation>"
return xml
def processProductElement(function, isInit, tgNode, productElement, kwKeys):
try:
origin = function.getOrigin()
except AttributeError:
origin = None
params, inst = linkCall(function, isInit, kwKeys, *productElement)
if isinstance(origin, ti.tgnode.UsualFunctionDefinitionTGNode):
tree = origin.ast
if len(tree) == 0:
return {typeNone}
first_node = tree[0]
filename = utils.getFileName(first_node)
if not origin.mir:
origin.mir = ti.mir.JoinMirNode()
ast_visitor = ti.visitor.Visitor(filename, origin.mir)
for stmt in tree:
ast_visitor.visit(stmt)
template = ti.tgnode.FunctionTemplateTGNode(params, origin,
inst , tgNode)
scope = config.data.currentScope
config.data.currentScope = ti.sema.ScopeSema(template.getScope())
file_scope = utils.getFileScope(first_node)
ti.mir.walkChain(origin.mir, file_scope)
config.data.currentScope = scope
elif isinstance(origin, ti.tgnode.ExternalFunctionDefinitionTGNode):
types = getParamsTypes(params)
return origin.quasi(types, TGNODE=tgNode)
if inst:
return {inst}
else:
return set()
def __init__(self):
channel_names = ['L', 'K_ir', 'K_m35']
# reduced trees
full_tree, red_tree, self.full_locs, self.red_locs = data.reduceMorphology(
)
sim_tree = red_tree.__copy__(new_tree=NeuronSimTree())
# measured data
self.v_dat = data.DataContainer(with_zd=True)
# get file name
file_name = utils.getFileName(channel_names, True, suffix='_predef')
print(file_name)
# load hall of fame
with open(file_name, 'rb') as file:
hall_of_fame = pickle.load(file)
# get the original simtree with the potassium params
self.model_evaluator = optimizer.ModelEvaluator(
sim_tree,
self.v_dat,
self.red_locs[0],
self.red_locs[1],
channel_names=channel_names,
mode='evaluate')
self.model_evaluator.setParameterValues(hall_of_fame[0])
self._createHcnTees()
self._createGreensTrees()
def main():
width = args['width']
height = args['height']
datasetName = args['dataset_name']
pascalVOCDirPath = os.path.join('./dataturks/pascalVOC/', datasetName)
croppedLpDirPath = os.path.join('./cropped_lps/', datasetName)
if not os.path.exists(croppedLpDirPath):
os.makedirs(croppedLpDirPath)
for carImageFilePath in utils.listFilePaths(pascalVOCDirPath, 'jpg'):
imageName = utils.getFileName(carImageFilePath)
xmlFilePath = os.path.join(pascalVOCDirPath, imageName + '.xml')
try:
plateImage, xml = cropPlate(carImageFilePath,
xmlFilePath,
newWidth=width,
newHeight=height)
except Exception:
print("Plate Object Not Found")
continue
saveImageAndXml(imageName, plateImage, xml, croppedLpDirPath)
def optimizeModel(channel_names=None, zd=False, suffix=''):
"""
Optimizes the morphology equipped with channels in `channel_names` to
recordings with or without ZD
Parameters
----------
channel_names: list of str
Choose channel names from from {'L', 'K_m', 'K_m35', 'K_ir', 'h_HAY', 'h_u'}
zd: bool
True for data with ZD, false for data without ZD
"""
global MAX_ITER, N_OFFSPRING
if channel_names is None:
channel_names = ['L', 'K_ir', 'K_m35', 'h_u']
file_name = utils.getFileName(channel_names, zd, suffix=suffix)
full_tree, red_tree, full_locs, red_locs = data.reduceMorphology()
sim_tree = red_tree.__copy__(new_tree=NeuronSimTree())
# measured data
v_dat = data.DataContainer(with_zd=zd)
model_evaluator = ModelEvaluator(sim_tree,
v_dat,
red_locs[0],
red_locs[1],
channel_names=channel_names)
final_pop, hall_of_fame, logs, hist = optimize(model_evaluator)
# save hall of fame
file = open(file_name, 'wb')
pickle.dump(hall_of_fame, file)
file.close()
def getFileNames():
if len(sys.argv) > 2:
inName = sys.argv[1]
outName = sys.argv[2]
else:
inName = utils.getFileName('Enter the name of the log file : ')
outName = utils.getNewFileName('Enter name of the new file for output : ')
return inName, outName
def workWrapper(work, filePath, delete_after_work=config.delete_after_work):
filename = getFileName(filePath)
log.info('%s\t\tStart processing', filename)
result = work(filePath)
log.info('%s\t\tProcessing finished with result %s', filename, result)
if delete_after_work and result:
os.remove(filePath)
def __init__(self, settings):
self.logger = logging.getLogger('f2dot.forsydeparser')
self.logger.debug('Initializing the ForSyDe parser...')
self.set = settings
if settings['DIRECTION'] == "TB":
self.vertical = True
else:
self.vertical = False
self.rootProcess = utils.getFileName(settings.inFile)
def on_created(self, event, after_modification=False):
super(EventHandler_WithLog, self).on_created(event)
filename = getFileName(event.src_path)
event_label = 'Detected'
# if after_modification:
# event_label = 'Modificated'
log.info("%s\t\t%s", filename, event_label)
if self.work:
workWrapper(self.work, event.src_path, self.delete_after_work)
def processZip(file):
# res = send_zip(file)
zip_data = toBase64(file)
res = post(zip_data)
xml = formatXML(res[1])
filename = getFileBaseName(getFileName(file))
filename = os.path.splitext(filename)[0]
writeXml(filename, xml)
xml_msg = getXmlValues(res[1])
return f'\n\t\tHTTP STATUS {res[0]}\n{xml_msg}'
def webhook(name):
if request.method == "POST":
file = request.files['audio']
print(file)
fname = utils.getFileName()
file.save(secure_filename(fname))
tempaudio = utils.moveToTemp(fname)
if tempaudio != "":
voice_json = brain.getEmotionFromVoice(tempaudio)
temptext = utils.speechToText(tempaudio)
print(temptext)
text_json = brain.getEmotionFromText(temptext)
brain.getPriorityScore(voice_json, text_json, name)
return jsonify({"response": "true"})
def __init__(self, parentWidget, uploadTransaction):
QTreeWidgetItem.__init__(self, None,QTreeWidgetItem.UserType)
View.__init__(self)
self.setTreeWidget(parentWidget)
#we bootstrap the inner components and put the "LEGO" together
self._uploadTransactionModel = uploadTransaction
self._uploadTransactionProgressBar = QProgressBar()
self._uploadTransactionProgressBar.setTextVisible(True)
self._uploadTransactionProgressBar.setAutoFillBackground(True)
self.setText(UploadTransactionView.COLUMN_FILENAME,utils.getFileName(uploadTransaction.getFilePath()))
self.setText(UploadTransactionView.COLUMN_STATUS,uploadTransaction.getHumanUploadStatus())
self.getTreeWidget().setItemWidget(self,UploadTransactionView.COLUMN_PERCENTAGE,self._uploadTransactionProgressBar)
def getStringFromCondition(conj):
res = ''
first = True
value = sorted(conj, key = sortIfKey)
for if_node, flag in value:
filename = utils.getFileName(if_node.node)
lineno = utils.getLine (if_node.node)
if first:
first = False
prefix = ''
else:
prefix = ' and '
added = '%s:%d == %s' % (filename, lineno, flag)
res += prefix + added
return res
def processTemplate(templateIp):
from template import Template
utils.outputFileName = ''
if utils.outputFileKeyword != '':
utils.outputFileName = utils.getFileName(
templateIp[utils.outputFileKeyword])
if utils.outputFile != '':
utils.outputFileName = utils.outputFileName + utils.outputFile
utils.outputFileName = utils.outputFileName + utils.outputFileExt
tt = Template({'EVAL_PYTHON': 1, 'AUTO_RESET': 1})
op = tt.process(utils.ttFileName, {'TemplateInputVar': templateIp})
utils.WriteFile(utils.outputDir, utils.outputFileName, op, utils.mode)
def quasiImport(params, **kwargs):
filename = getattr(params[0], 'value', None)
tgNode = kwargs['TGNODE']
origin = utils.getFileName(tgNode.node)
if (filename is not None and
filename not in config.data.skipped_imports and
origin is not None):
try:
importer = config.data.importer
quasiAlias = QuasiAlias(filename)
module = importer.importFile(origin, quasiAlias)
res = module.getScope()
return {res}
except:
pass
return set()
def readcountrycont(ccfile = ''):
"""
Reads in the pickle file containing country->region tables, and returns the dictionary with
that mapping.
Reads in the file whose name is passed in; if no name is passed or if the file name passed
in does not exist, this will interactively query the user for the file. This assumes that
the file passed in is a pickled version of a dictionary from country to region, and simply
reconstructs the dictionary and returns that dictionary.
:param ccfile: the name of the file containing the pickle for the dictionary
:return: a dictionary mapping from country names to more general regions
"""
if (ccfile == '') or not(os.path.exists(ccfile)):
ccfile = utils.getFileName('the country/continent file')
pf = open(ccfile, 'rU')
ccdict = pickle.load(pf)
pf.close()
return ccdict
import Procedure
import register
import utils
import world
from register import dataset
# ==============================
utils.set_seed(world.seed)
print(">>SEED:", world.seed)
# ==============================
torch.autograd.set_detect_anomaly(True)
Recmodel = register.MODELS[world.model_name](world.config, dataset)
Recmodel = Recmodel.to(world.device)
bpr = utils.BPRLoss(Recmodel, world.config)
weight_file = utils.getFileName()
print(f"load and save to {weight_file}")
if world.LOAD:
try:
Recmodel.load_state_dict(torch.load(weight_file, map_location=torch.device('cpu')))
print(f"loaded model weights from {weight_file}")
except FileNotFoundError:
print(f"{weight_file} not exists, start from beginning")
best_ndcg, best_recall, best_pre = 0, 0, 0
best_ndcg_cold, best_recall_cold, best_pre_cold = 0, 0, 0
low_count, low_count_cold = 0, 0
try:
for epoch in range(world.TRAIN_epochs + 1):
print('======================')
print(f'EPOCH[{epoch}/{world.TRAIN_epochs}]')
Take an open .csv file with format country, username and create a dictionary
indexed by username with value country
"""
retDict = {}
for [country, username] in cfile:
retDict[username] = country
return retDict
if __name__ == '__main__':
if len(sys.argv) > 3:
cFileName = sys.argv[1]
userFileName = sys.argv[2]
lName = sys.argv[3]
else:
cFileName = utils.getFileName('Name of user name to country file : ')
userFileName = utils.getFileName('Name of user file : ')
clName = utils.getNewFileName('Name of class for the output file : ')
cfile = csv.reader(open(cFileName, 'r'))
nameDict = buildNameCountry(cfile)
ufile = csv.reader(open(userFileName, 'r'))
userDict = user.builddict(ufile)
clfName = clName + '_id_country.csv'
outfile = csv.writer(open(clfName, 'w'))
users = userDict.keys()
outfile.writerow(['User id', 'Country'])
for u in users:
def __init__(self, args):
self.logger = logging.getLogger('f2dot.settings')
self.logger.debug('Configuring the runtime execution...')
self.runPath = os.path.dirname(os.path.abspath(__file__))
self.configFileName = args.mode + '.conf'
# if -g option chosen
if args.generate_config:
path = args.output
if not path:
path = os.getcwd()
self.createConfFile(path, force=True)
self.logger.info('Generated config file in ' + path)
os._exit(1)
# set paths & names
self.inPathAndFile = os.path.abspath(args.input)
self.inPath, self.inFile = os.path.split(self.inPathAndFile)
if args.output:
self.outPath = os.path.abspath(args.output)
else:
self.outPath = self.inPath
# resolve config file
if args.config:
self.confFile = os.path.abspath(args.config)
else:
self.confFile = self.createConfFile(self.inPath, force=False)
self.logger.info("Using the configuration in %s", self.confFile)
for line in open(self.confFile):
if line.strip().startswith("# works with : f2dot"):
confVer = line.strip().split("# works with : f2dot-", 1)[1]
if not confVer == __init__.__version__:
self.logger.warn(
'The config file was created by another version ' +
'of the tool. Errors may occur.')
self.settingDict = {}
self.constraintDict = {}
# loading default settings & constraints
for line in utils.getConfigInSection(
os.path.join(self.runPath, 'config', 'general.conf'),
'[default settings]'):
tag, value = utils.strBeforeAfter(line, "=")
self.settingDict[tag] = value
for line in utils.getConfigInSection(
os.path.join(self.runPath, 'config', self.configFileName),
'[default settings]'):
tag, value = utils.strBeforeAfter(line, "=")
self.settingDict[tag] = value
for line in utils.getConfigInSection(
os.path.join(self.runPath, 'config', 'general.conf'),
'[setting constraints]'):
tag, value = utils.strBeforeAfter(line, "=")
self.constraintDict[tag] = value
for line in utils.getConfigInSection(
os.path.join(self.runPath, 'config', self.configFileName),
'[setting constraints]'):
tag, value = utils.strBeforeAfter(line, "=")
self.constraintDict[tag] = value
# loading custom settings and comparing them against the constraints
for line in utils.getConfigInSection(self.confFile):
tag, value = utils.strBeforeAfter(line, "=")
if tag in self.constraintDict:
if self.constraintDict[tag]:
pattern = re.compile(self.constraintDict[tag])
if not pattern.match(value):
self.logger.warn(
"The value for %s (%s) does not match pattern %s. Choosing the default value: %s",
tag, value, self.constraintDict[tag],
self.settingDict[tag])
continue
self.settingDict[tag] = value
if args.format:
self.settingDict['FORMAT'] = args.format
if args.prog:
self.settingDict['PROG'] = args.prog
self.outPathAndFile = os.path.join(
self.outPath,
utils.getFileName(self.inFile) + '.' + self.settingDict['FORMAT'])
self.logger.debug('Runtime configuration successful')
ages = {}
edu = {'p_se':0,
'p_oth':0,
'm':0,
'b':0,
'hs':0,
'jhs':0,
'el':0,
'none':0,
'other':0,
'unk':0
}
csv.field_size_limit(1000000000)
clName = sys.argv[1]
ufile = utils.getFileName('user file')
ufin = csv.reader(open(ufile, 'r'))
profile_file = utils.getFileName('student profiles')
infile = csv.reader(open(profile_file, 'r'))
profiles = prof.builddict(infile)
countryFile = utils.getFileName('username and country file')
countryDict = makeCountryDict(ufin, countryFile)
outName = raw_input("enter file name for output; nothing for stdout : ")
if outName == '':
outp = sys.stdout.write
else:
outFile = open(outName, 'w')
outp = outFile.write
outfile = csv.writer(open('anon'+sys.argv[1], 'w'))
students = profiles.keys()
def __getitem__(self, index):
try:
lr_file = self.lr_list[index]
hr_file = self.hr_list[index]
# get the HR image (as the center frame)
hr_data = cv2.imread(hr_file)
hr_data = cv2.cvtColor(hr_data, cv2.COLOR_BGR2RGB)
hr_data = np.array(hr_data)
hr_data = hr_data.astype(np.float32)
height, width, channel = hr_data.shape
hr_size = self.patch_size * self.scale
# randomly crop
lr_height = height // self.scale
lr_width = width // self.scale
# get LR image
lr_data = cv2.imread(lr_file)
lr_data = cv2.cvtColor(lr_data, cv2.COLOR_BGR2RGB)
lr_data = np.array(lr_data)
lr_data = lr_data.astype(np.float32)
b_file, f_file = getFileName(lr_file)
#b frame
lr_b_data = cv2.imread(b_file)
lr_b_data = cv2.cvtColor(lr_b_data, cv2.COLOR_BGR2RGB)
lr_b_data = np.array(lr_b_data)
lr_b_data = lr_b_data.astype(np.float32)
#f frame
lr_f_data = cv2.imread(f_file)
lr_f_data = cv2.cvtColor(lr_f_data, cv2.COLOR_BGR2RGB)
lr_f_data = np.array(lr_f_data)
lr_f_data = lr_f_data.astype(np.float32)
rnd_h = random.randint(0, max(0, lr_height - self.patch_size))
rnd_w = random.randint(0, max(0, lr_width - self.patch_size))
img_lr = lr_data[rnd_h:rnd_h + self.patch_size,
rnd_w:rnd_w + self.patch_size, :]
img_lr_b = lr_b_data[rnd_h:rnd_h + self.patch_size,
rnd_w:rnd_w + self.patch_size, :]
img_lr_f = lr_f_data[rnd_h:rnd_h + self.patch_size,
rnd_w:rnd_w + self.patch_size, :]
rnd_h_hr, rnd_w_hr = int(rnd_h * self.scale), int(rnd_w *
self.scale)
img_hr = hr_data[rnd_h_hr:rnd_h_hr + hr_size,
rnd_w_hr:rnd_w_hr + hr_size, :]
# augmentation - flip, rotate
axis1 = np.random.randint(low=-1, high=3)
img_lr = horizontal_flip(img_lr, axis=axis1)
img_lr_b = horizontal_flip(img_lr_b, axis=axis1)
img_lr_f = horizontal_flip(img_lr_f, axis=axis1)
img_hr = horizontal_flip(img_hr, axis=axis1)
# HWC to CHW, numpy to tensor
img_lr = torch.from_numpy(
np.ascontiguousarray(np.transpose(img_lr, (2, 0, 1)))).float()
img_lr_b = torch.from_numpy(
np.ascontiguousarray(np.transpose(img_lr_b,
(2, 0, 1)))).float()
img_lr_f = torch.from_numpy(
np.ascontiguousarray(np.transpose(img_lr_f,
(2, 0, 1)))).float()
img_hr = torch.from_numpy(
np.ascontiguousarray(np.transpose(img_hr, (2, 0, 1)))).float()
#可以这样认为，ascontiguousarray函数将一个内存不连续存储的数组转换为内存连续存储的数组，使得运行速度更快
except Exception as e:
random_sum = random.randrange(0, self.__len__())
return self.__getitem__(random_sum)
return img_lr_b, img_lr, img_lr_f, img_hr
def on_deleted(self, event):
super(EventHandler_WithLog, self).on_deleted(event)
filename = getFileName(event.src_path)
log.info("%s\t\tDone", filename)
# np.loadtxt(f"stats/popularity-{n}-user.txt")
# for n in names
# ]
# for i in range(len(dims)):
# pop1 = pop_item[i]
# pop1_user = pop_user[i]
# n = names[i]
# # print(n)
# pprint(utils.popularity_ratio(pop1, pop1_user, dataset))
# print(',')
else:
procedure = Procedure.Popularity_Bias
Recmodel = register.MODELS[world.model_name](world.config, dataset)
file = utils.getFileName(world.model_name,
world.dataset,
world.config['latent_dim_rec'],
layers=world.config['lightGCN_n_layers'])
file = 'teacher-' + file
weight_file = os.path.join(world.FILE_PATH, file)
print(f"Load {weight_file}")
utils.load(Recmodel, weight_file)
# ----------------------------------------------------------------------------
Recmodel = Recmodel.to(world.DEVICE)
test_results = Procedure.Test(dataset, Recmodel, 0, valid=False)
pprint(test_results)
pop, user_pop = procedure(dataset, Recmodel)
np.savetxt(f"popularity-{world.dataset}-{world.comment}.txt", pop)
np.savetxt(f"popularity-{world.dataset}-{world.comment}-user.txt",
user_pop)
import glob
import csv
import sys
import utils
import buildClassRoster as bcr
import demographics.userprofile as prof
import ipGeoloc as geo
import user
if __name__ == '__main__':
if len(sys.argv) > 1:
locname = sys.argv[1]
else:
locname = utils.getFileName('Enter name of the id=>location file :')
csv.field_size_limit(sys.maxsize)
locD = geo.readIdToLoc(locname)
flist = glob.glob('20*')
fname = flist.pop()
fin = csv.reader(open(fname + '/profiles.csv', 'r'))
pDict = prof.builddict(fin)
uin = csv.reader(open(fname +'/users.csv', 'r'))
uDict = user.builddict(uin)
fullR = bcr.buildRosterDict(pDict, uDict, locD)
for f in flist:
fin = csv.reader(open(f + '/profiles.csv', 'r'))
addDict = prof.builddict(fin)
uin = csv.reader(open(f + '/users.csv', 'r'))
#!/usr/bin/env python
"""
A simple interactive program to compare the ids in a file with those in a
certificates file
This program will prompt the user for the name of a csv file containing
only user ids, and a csv of a certificates file, and see if there are any
ids in the first file that correspond to entries in the certificates file.
"""
import csv
import sys
from classData import certificates
import utils
if __name__ == '__main__':
if len(sys.argv) > 2:
f1name = sys.argv[1]
f2name = sys.argv[2]
else:
f1name = utils.getFileName('Enter csv file with ids : ')
f2name = utils.getFileName('Enter certificates csv file name : ')
f1 = csv.reader(open(f1name, 'r'))
f2 = csv.reader(open(f2name, 'r'))
certdict = certificates.builddict(f2)
f1.readrow()
for [ident] in f1:
if ident in certdict:
print 'found new identifier ' + ident + ' in certificates file'
A simple interactive program to compare the ids in a file with those in a
certificates file
This program will prompt the user for the name of a csv file containing
only user ids, and a csv of a certificates file, and see if there are any
ids in the first file that correspond to entries in the certificates file.
"""
import csv
import sys
import certificates
import utils
if len(sys.argv) > 2:
f1name = sys.argv[1]
f2name = sys.argv[2]
else:
f1name = utils.getFileName("Enter csv file with ids : ")
f2name = utils.getFileName("Enter certificates csv file name : ")
f1 = csv.reader(open(f1name, "r"))
f2 = csv.reader(open(f2name, "r"))
certdict = certificates.builddict(f2)
f1.readrow()
for [ident] in f1:
if ident in certdict:
print "found new identifier " + ident + " in certificates file"
if world.EMBEDDING:
comment = comment + "-embed"
world.comment = comment
# ----------------------------------------------------------------------------
# set seed
utils.set_seed(world.SEED)
print(f"[SEED:{world.SEED}]")
# ----------------------------------------------------------------------------
# init model
import register
from register import dataset
# ----------------------------------------------------------------------------
# loading teacher
teacher_file = utils.getFileName(world.model_name,
world.dataset,
world.config['teacher_dim'],
layers=world.config['teacher_layer'])
teacher_file = "teacher-" + teacher_file
teacher_weight_file = os.path.join(world.FILE_PATH, teacher_file)
print('-------------------------')
world.cprint("loaded teacher weights from")
print(teacher_weight_file)
print('-------------------------')
teacher_config = utils.getTeacherConfig(world.config)
world.cprint('teacher')
teacher_model = register.MODELS[world.model_name](teacher_config,
dataset,
fix=True)
teacher_model.eval()
utils.load(teacher_model, teacher_weight_file)
# ----------------------------------------------------------------------------
import utils
import sys
def buildNameCountry(cfile):
retDict = {}
for [country, username] in cfile:
retDict[username] = country
return retDict
if len(sys.argv) > 3:
cFileName = sys.argv[1]
userFileName = sys.argv[2]
clName = sys.argv[3]
else:
cFileName = utils.getFileName('user name to country file')
userFileName = utils.getFileName('user file')
clName = raw_input("Please enter the name of the class : ")
cfile = csv.reader(open(cFileName, 'r'))
nameDict = buildNameCountry(cfile)
ufile = csv.reader(open(userFileName, 'r'))
userDict = user.builddict(ufile)
clfName = clName + '_id_country.csv'
outfile = csv.writer(open(clfName, 'w'))
users = userDict.keys()
outfile.writerow(['User id', 'Country'])
def plotTrace(channel_names,
zd,
axes=None,
plot_xlabel=True,
plot_ylabel=True,
plot_legend=True,
units='uS/cm2',
suffix='_predef'):
ts = np.array([data.T0, data.T1, data.T2, data.T3]) - 100.
pcolor = 'Orange' if zd else 'Blue'
if units == 'uS/cm2':
unit_conv = 1
elif units == 'pS/um2':
unit_conv = 0.01
else:
print('Unit type not defined, reverting to uS/cm2')
unit_conv = 1
# reduced trees
full_tree, red_tree, full_locs, red_locs = data.reduceMorphology()
sim_tree = red_tree.__copy__(new_tree=NeuronSimTree())
# measured data
v_dat = data.DataContainer(with_zd=zd)
# get file name
file_name = utils.getFileName(channel_names, zd, suffix=suffix)
# model
model_evaluator = optimizer.ModelEvaluator(sim_tree,
v_dat,
red_locs[0],
red_locs[1],
channel_names=channel_names)
# load hall of fame
file = open(file_name, 'rb')
hall_of_fame = pickle.load(file)
file.close()
# run the best fit model
model_evaluator.setParameterValues(hall_of_fame[0])
print("\n--> running model for parameter values: \n%s" %
model_evaluator.toStrParameterValues())
res = model_evaluator.runSim()
if axes is None:
pl.figure('v opt', figsize=(16, 4))
axes = [
pl.subplot(161),
pl.subplot(162),
pl.subplot(163),
pl.subplot(164),
pl.subplot(165),
pl.subplot(166)
]
pshow = True
else:
pshow = False
i0s = [int(tt / data.DT) for tt in ts]
i1s = [int((tt + 1000. - data.DT) / data.DT) for tt in ts]
t_plot = np.arange(0., 1000. - 3 * data.DT / 2., data.DT)
pchanstr = r'' + ', '.join([PSTRINGS[c_name] for c_name in channel_names])
ax = axes[0]
# ax.set_title(pchanstr)
ax.axison = False
ax.text(0.,
0.,
pchanstr,
horizontalalignment='center',
verticalalignment='center',
rotation=45.,
fontsize=labelsize)
ax.set_xlim((0., 1.))
ax.set_ylim((-1., 1.))
for ii in range(4):
ax = noFrameAx(axes[ii + 1])
i0, i1 = i0s[ii], i1s[ii]
# plot traces
ax.plot(t_plot,
v_dat.v_s[i0:i1],
c='k',
lw=lwidth,
label=r'$Exp_{soma}$')
ax.plot(t_plot,
v_dat.v_d[i0:i1],
c=pcolor,
lw=lwidth,
label=r'$Exp_{dend}$')
ax.plot(t_plot,
res['v_m'][0][i0:i1],
c='k',
lw=2 * lwidth,
alpha=.4,
label=r'$Sim_{soma}$')
ax.plot(t_plot,
res['v_m'][1][i0:i1],
c=pcolor,
lw=2 * lwidth,
alpha=.4,
label=r'$Sim_{soma}$')
if zd:
ax.set_ylim((-110., -50.))
else:
ax.set_ylim((-95., -50.))
if ii == 0:
if not zd:
ax.set_yticks([-65., -70.])
ax.set_yticklabels(['-65 mV', '-70 mV'])
else:
ax.set_yticks([-75., -80.])
ax.set_yticklabels(['-75 mV', '-80 mV'])
if ii == 3 and plot_legend:
myLegend(ax)
# draw x scalebar
ypos = ax.get_ylim()[0]
xpos = ax.get_xlim()[-1] - 40.
sblen = 400.
ax.plot([xpos - sblen, xpos], [ypos, ypos], 'k-', lw=2. * lwidth)
ax.annotate(r'%.0f ms' % sblen,
xy=(xpos - sblen / 2., ypos),
xytext=(xpos - sblen / 2., ypos - 4.),
size=labelsize,
rotation=0,
ha='center',
va='center')
# draw y scalebar
ypos = ax.get_ylim()[0] + 5.
xpos = ax.get_xlim()[-1]
sblen = 10.
ax.plot([xpos, xpos], [ypos, ypos + sblen], 'k-', lw=2. * lwidth)
ax.annotate(r'%.0f mV' % sblen,
xy=(xpos, ypos + sblen / 2.),
xytext=(xpos + 100., ypos + sblen / 2.),
size=labelsize,
rotation=90,
ha='center',
va='center')
ax = myAx(axes[5])
print('>>> parameters')
for jj, pvals in enumerate(hall_of_fame):
model_evaluator.setParameterValues(pvals)
ps = model_evaluator.getParameterValuesAsDict()
print(model_evaluator.toStrParameterValues())
d2s = np.linspace(0., 400., int(1e3))
for ii, c_name in enumerate(model_evaluator.channel_names):
func = utils.expDistr(ps['d_' + c_name],
ps['g0_' + c_name],
g_max=model_evaluator.g_max[c_name])
if jj == 0:
ax.plot(d2s,
func(d2s) * unit_conv,
c=PCOLORS[c_name],
lw=lwidth,
label=PSTRINGS[c_name])
else:
ax.plot(d2s,
func(d2s) * unit_conv,
c=PCOLORS[c_name],
lw=3. * lwidth,
alpha=.07)
if plot_legend:
myLegend(ax)
if plot_xlabel:
ax.set_xlabel(r'Distance ($\mu$m)', fontsize=labelsize)
else:
ax.set_xticklabels([])
if plot_ylabel:
if units == 'pS/um2':
ax.set_ylabel(r'Conductance (pS/$\mu$m$^2$)', fontsize=labelsize)
else:
ax.set_ylabel(r'Conductance ($\mu$S/cm$^2$)', fontsize=labelsize)
if zd:
ax.set_ylim((0., 2000. * unit_conv))
ax.set_yticks([0., 1000. * unit_conv])
else:
ax.set_ylim((0., 6000. * unit_conv))
ax.set_yticks([0., 3000. * unit_conv])
if pshow:
pl.tight_layout()
pl.show()
words, distance = vq(des_list[i][1],voc)
for w in words:
test_features[i][w] += 1
# Perform Tf-Idf vectorization
nbr_occurences = np.sum( (test_features > 0) * 1, axis = 0)
idf = np.array(np.log((1.0*len(image_paths)+1) / (1.0*nbr_occurences + 1)), 'float32')
# Scale the features
test_features = stdSlr.transform(test_features)
# Perform the predictions
predictions = []
for i in clf.predict(test_features):
predictions.append(i)
label_file = args["label"]
dict=utils.readDict(label_file)
count=0
total=len(zip(image_paths, predictions))
for image_path, prediction in zip(image_paths, predictions):
print image_path,prediction,dict[utils.getFileName(image_path)]
if(prediction==dict[utils.getFileName(image_path)]):
count+=1
print "total: "+str(total)
print "accuracy: "+str((1.0*count/total*100))+"%"
return retDict
if __name__ == '__main__':
"""
When run stand-alone, this script will query for a filename and a level of anonymity
to check for the externally-connected data fields in the .csv file. The user will also
be prompted for either a summary of the anonymity level (in which case only the number
of records that fail to be at least anonymous to the level indicated) will be printed, or
a full report, in which case the concatenation of fields that allow identification finer
than the level entered will be printed. Note that the indexes of the fields that can be
linked to external properties is hard-coded at the moment; it would be good to have a more
flexible mechanism for this but finding one that is not error prone is difficult.
"""
fname = utils.getFileName('data file to test')
if fname[-3:] == 'csv':
anonDict = makeDictFromCSV(idFields, fname)
elif fname[-2: ] == 'db':
anonDict = makeDictFromDB(idFields, fname)
else:
print 'Unknown file type; program exiting'
exit(1)
idFields = [0, 10, 11, 12, 19]
qiequivDict = makeEquivDict(anonDict)
sortedDict = sorted(qiequivDict.iterkeys())
for k in sortedDict:
print str(k), qiequivDict[k]
def __init__(self, args):
self.logger = logging.getLogger("f2dot.settings")
self.logger.debug("Configuring the runtime execution...")
self.runPath = os.path.dirname(os.path.abspath(__file__))
self.configFileName = args.mode + ".conf"
# if -g option chosen
if args.generate_config:
path = args.output
if not path:
path = os.getcwd()
self.createConfFile(path, force=True)
self.logger.info("Generated config file in " + path)
os._exit(1)
# set paths & names
self.inPathAndFile = os.path.abspath(args.input)
self.inPath, self.inFile = os.path.split(self.inPathAndFile)
if args.output:
self.outPath = os.path.abspath(args.output)
else:
self.outPath = self.inPath
# resolve config file
if args.config:
self.confFile = os.path.abspath(args.config)
else:
self.confFile = self.createConfFile(self.inPath, force=False)
self.logger.info("Using the configuration in %s", self.confFile)
for line in open(self.confFile):
if line.strip().startswith("# works with : f2dot"):
confVer = line.strip().split("# works with : f2dot-", 1)[1]
if not confVer == __init__.__version__:
self.logger.warn(
"The config file was created by another version " + "of the tool. Errors may occur."
)
self.settingDict = {}
self.constraintDict = {}
# loading default settings & constraints
for line in utils.getConfigInSection(
os.path.join(self.runPath, "config", "general.conf"), "[default settings]"
):
tag, value = utils.strBeforeAfter(line, "=")
self.settingDict[tag] = value
for line in utils.getConfigInSection(
os.path.join(self.runPath, "config", self.configFileName), "[default settings]"
):
tag, value = utils.strBeforeAfter(line, "=")
self.settingDict[tag] = value
for line in utils.getConfigInSection(
os.path.join(self.runPath, "config", "general.conf"), "[setting constraints]"
):
tag, value = utils.strBeforeAfter(line, "=")
self.constraintDict[tag] = value
for line in utils.getConfigInSection(
os.path.join(self.runPath, "config", self.configFileName), "[setting constraints]"
):
tag, value = utils.strBeforeAfter(line, "=")
self.constraintDict[tag] = value
# loading custom settings and comparing them against the constraints
for line in utils.getConfigInSection(self.confFile):
tag, value = utils.strBeforeAfter(line, "=")
if tag in self.constraintDict:
if self.constraintDict[tag]:
pattern = re.compile(self.constraintDict[tag])
if not pattern.match(value):
self.logger.warn(
"The value for %s (%s) does not match pattern %s. Choosing the default value: %s",
tag,
value,
self.constraintDict[tag],
self.settingDict[tag],
)
continue
self.settingDict[tag] = value
if args.format:
self.settingDict["FORMAT"] = args.format
if args.prog:
self.settingDict["PROG"] = args.prog
self.outPathAndFile = os.path.join(
self.outPath, utils.getFileName(self.inFile) + "." + self.settingDict["FORMAT"]
)
self.logger.debug("Runtime configuration successful")
"""
Take an open .csv file with format country, username and create a dictionary
indexed by username with value country
"""
retDict = {}
for [country, username] in cfile:
retDict[username] = country
return retDict
if __name__ == '__main__':
if len(sys.argv) > 3:
cFileName = sys.argv[1]
userFileName = sys.argv[2]
lName = sys.argv[3]
else:
cFileName = utils.getFileName('Name of user name to country file : ')
userFileName = utils.getFileName('Name of user file : ')
clName = utils.getNewFileName('Name of class for the output file : ')
cfile = csv.reader(open(cFileName, 'r'))
nameDict = buildNameCountry(cfile)
ufile = csv.reader(open(userFileName, 'r'))
userDict = user.builddict(ufile)
clfName = clName + '_id_country.csv'
outfile = csv.writer(open(clfName, 'w'))
users = userDict.keys()
outfile.writerow(['User id', 'Country'])
if __name__ == '__main__':
"""
When run stand-alone, this script will query for a filename and a level of anonymity
to check for the externally-connected data fields in the .csv file. The user will also
be prompted for either a summary of the anonymity level (in which case only the number
of records that fail to be at least anonymous to the level indicated) will be printed, or
a full report, in which case the concatenation of fields that allow identification finer
than the level entered will be printed. Note that the indexes of the fields that can be
linked to external properties is hard-coded at the moment; it would be good to have a more
flexible mechanism for this but finding one that is not error prone is difficult.
"""
idFields = [0, 6, 7, 8, 9, 17]
fname = utils.getFileName('data file to test')
kanon = utils.getIntVal('Enter value of k to test : ')
full = utils.getStringVal('Enter s for summary, f for full report : ',
['s', 'f'])
fin = open(fname, 'rU')
fread = csv.reader(fin)
totals = []
for i in range(0, kanon):
totals.append(0)
fread.next()
anonDict = makeDict(idFields, fread)
sortedDict = sorted(anonDict.iteritems(), key=operator.itemgetter(1))
for k, v in sortedDict:
if v < kanon:
from tensorboardX import SummaryWriter
import time
import Procedure
from os.path import join
# ==============================
utils.set_seed(world.seed)
print(">>SEED:", world.seed)
# ==============================
import register
from register import dataset
Recmodel = register.MODELS[world.model_name](world.config, dataset) # 初始化模型
Recmodel = Recmodel.to(world.device) # GPU or CPU
bpr = utils.BPRLoss(Recmodel, world.config) # Loss function
weight_file = utils.getFileName() # weight_file save path
print(f"load and save to {weight_file}")
if world.LOAD: # 是否加载已存在的模型权重
try:
Recmodel.load_state_dict(
torch.load(weight_file, map_location=torch.device('cpu')))
world.cprint(f"loaded model weights from {weight_file}")
except FileNotFoundError:
print(f"{weight_file} not exists, start from beginning")
Neg_k = 1 # ?
# init tensorboard 日志记录(tensorboard)
if world.tensorboard:
w: SummaryWriter = SummaryWriter(
join(world.BOARD_PATH,
time.strftime("%m-%d-%Hh%Mm%Ss-") + "-" + world.comment))
import sys
def buildNameCountry(cfile):
retDict = {}
for [country, username] in cfile:
retDict[username] = country
return retDict
if len(sys.argv) > 3:
cFileName = sys.argv[1]
userFileName = sys.argv[2]
clName = sys.argv[3]
else:
cFileName = utils.getFileName('user name to country file')
userFileName = utils.getFileName('user file')
clName = raw_input("Please enter the name of the class : ")
cfile = csv.reader(open(cFileName, 'r'))
nameDict = buildNameCountry(cfile)
ufile = csv.reader(open(userFileName, 'r'))
userDict = user.builddict(ufile)
clfName = clName + '_id_country.csv'
outfile = csv.writer(open(clfName, 'w'))
users = userDict.keys()
outfile.writerow(['User id', 'Country'])
for u in users:
def unpack_apk(sourceFilePath):
if not os.path.isfile(sourceFilePath):
return
zipFile = zipfile.ZipFile(sourceFilePath)
# 文件所在路径
fileDir = os.path.dirname(zipFile.filename)
# 文件名
fileNameWithoutExt = utils.getFileNameWithoutExtention(zipFile.filename)
# 目标路径
destDir = os.path.join(fileDir, fileNameWithoutExt)
try:
utils.removeDirRecursive(destDir)
os.mkdir(destDir)
except Exception as ex:
print ex.message
return
for file in zipFile.namelist():
if not file.startswith('assets'):
continue
print(file)
# 文件路径
itemFilePath = os.path.normpath(os.path.join(destDir, file))
# 文件所在路径
itemDir = os.path.dirname(itemFilePath)
# 创建文件所在路径的文件夹
if not os.path.isdir(itemDir):
try:
os.makedirs(itemDir)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(itemDir):
pass
else:
raise
# 读取数据
data = zipFile.read(file)
# 文件名
itemFileName = utils.getFileName(file)
# 不需要压缩的文件
if itemFileName == 'settings.xml' or os.path.dirname(file) == 'assets':
(lambda f, d:
(f.write(d), f.close()))(open(itemFilePath, 'wb'),
data) #一行语句就完成了写文件操作。仔细琢磨哦~_~
else:
# 压缩文件
itemFilePath += '.obb'
itemZipFile = zipfile.ZipFile(itemFilePath, 'w')
itemFileName = utils.getFileName(file)
itemZipFile.writestr(itemFileName, data, zipfile.ZIP_DEFLATED)
itemZipFile.close()
# 第一个拆分包压缩到 globalgamemanagers.obb 中
if file.endswith('.split0') and file.startswith('assets/bin/Data'):
globalFilePath = os.path.join(itemDir, 'globalgamemanagers.obb')
globalZipFile = zipfile.ZipFile(globalFilePath, 'a')
globalZipFile.writestr(file, ' ', zipfile.ZIP_DEFLATED)
globalZipFile.close()
zipFile.close()
def plotErrors(channel_names_list,
zd_list,
ax=None,
qs=[.01, .5, .99],
suffix='_predef'):
global PSTRINGS
full_tree, red_tree, full_locs, red_locs = data.reduceMorphology()
sim_tree = red_tree.__copy__(new_tree=NeuronSimTree())
v_qmins, v_meds, v_qmaxs = [], [], []
ticklabels = []
for channel_names, zd in zip(channel_names_list, zd_list):
# load hall of fame
file_name = utils.getFileName(channel_names, zd, suffix=suffix)
file = open(file_name, 'rb')
hall_of_fame = pickle.load(file)
file.close()
# measured data
v_dat = data.DataContainer(with_zd=zd)
# model
model_evaluator = optimizer.ModelEvaluator(sim_tree,
v_dat,
red_locs[0],
red_locs[1],
channel_names=channel_names)
# compute the errors
v_errs = calcError(model_evaluator, hall_of_fame, v_dat)
# compute quantiles
v_qmin, v_med, v_qmax = np.quantile(v_errs, qs)
v_qmins.append(v_qmin)
v_meds.append(v_med)
v_qmaxs.append(v_qmax)
# tick label for this configuration
tl = r'' + ', '.join([PSTRINGS[c_name] for c_name in channel_names])
ticklabels.append(tl)
v_meds = np.array(v_meds)
v_errs = np.abs(np.array([v_qmins, v_qmaxs]) - v_meds[None, :])
inds_h = np.where(np.logical_not(np.array(zd_list)))[0]
inds_zd = np.where(np.array(zd_list))[0]
n_h = len(inds_h)
n_zd = len(inds_zd)
ticklabels = [ticklabels[ii] for ii in inds_h]
print(v_errs)
if ax is None:
pl.figure('v errors')
ax = pl.gca()
pshow = True
else:
pshow = False
if n_h > 0:
ax.bar(np.arange(n_h),
v_meds[inds_h],
width=.3,
align='edge',
yerr=v_errs[:, inds_h],
color='Blue')
if n_zd > 0:
ax.bar(np.arange(n_zd),
v_meds[inds_zd],
width=-.3,
align='edge',
yerr=v_errs[:, inds_zd],
color='Orange')
ax.set_xticks(np.arange(np.max([n_zd, n_h])))
ax.set_xticklabels(ticklabels, rotation=60., fontsize=labelsize)
ax.set_ylabel(r'$V_{error}$ (mV)')
if pshow:
pl.show()
locD = geo.readIdToLoc(locname)
flist = glob.glob(relLoc + '20*')
fname = flist.pop()
fin = csv.reader(open(fname + '/profiles.csv', 'r'))
pDict = prof.builddict(fin)
uin = csv.reader(open(fname + '/users.csv', 'r'))
uDict = user.builddict(uin)
fullR = bcr.buildRosterDict(pDict, uDict, locD)
for f in flist:
fin = csv.reader(open(f + '/profiles.csv', 'r'))
addDict = prof.builddict(fin)
uin = csv.reader(open(f + '/users.csv', 'r'))
uDict = user.builddict(uin)
addR = bcr.buildRosterDict(addDict, uDict, locD)
for i in iter(addR):
if i not in fullR:
fullR[i] = addR[i]
outname = relLoc + 'FullRoster.csv'
bcr.writeRoster(fullR, outname)
if __name__ == '__main__':
if len(sys.argv) > 1:
locname = sys.argv[1]
else:
locname = utils.getFileName('Enter name of the id=>location file :')
main(locname)
