def CleanStepFinal():
Base_Dir=Path(__file__).resolve().parent
myfile=Base_Dir.joinpath('output\FullCrimeStats.csv')
mychecker =Base_Dir.joinpath('Resources\CrimeListing.csv')
df= pd.read_csv(myfile)
df2 = pd.read_csv(mychecker)
df2.insert(1,column='tmp',value='Crime Cleared Count ' + df2['Crime'])
df2.insert(2,column='tmp2',value='Total Crime Count ' + df2['Crime'])
df2v=df2[df2['Type']=='Violent']
df2n=df2[df2['Type']=='Non-Violent']
Cleared = df.filter(regex='Cleared ')
Crime = df.filter(regex='Crime Count ')
Cleared['Total Violent Cleared']=Cleared[df2v['tmp']].sum(axis =1 )
Cleared['Total Non-Violent Cleared']=Cleared[df2n['tmp']].sum(axis =1 )
Crime['Total Violent Crime']=Crime[df2v['tmp2']].sum(axis =1 )
Crime['Total Non-Violent Crime']=Crime[df2n['tmp2']].sum(axis =1 )
Cleared=Cleared.filter(['Total Violent Cleared','Total Non-Violent Cleared'])
Crime=Crime.filter(['Total Violent Crime','Total Non-Violent Crime'])
Merge1 = pd.concat([Cleared, Crime],join='inner', axis=1, sort=False)
Result=pd.concat([df,Merge1],axis=1,join='inner', sort=False)
outputfile = Base_Dir.joinpath('output/FullCrimeStatsFinal.csv')
Result.to_csv(outputfile)
def maybe_download(url: str, store_path: Path, filename: str, md5: str = None):
logger = logging.getLogger(__name__)
if not (store_path / filename).is_file():
try:
logger.info("Downloading file {}...".format(url + " " + filename))
with TqdmUpTo(unit='B', unit_scale=True, miniters=1,
desc=filename) as t:
local_filename, _ = urlretrieve(url,
store_path / filename,
reporthook=t.update_to)
except AttributeError as e:
logger.error(
"An error occurred when downloading the file! Please get the dataset using a browser."
)
raise e
if md5:
md5_download = file_md5(store_path / filename)
if not md5 == md5_download:
store_path.joinpath(filename).unlink()
raise ValueError("MD5 checksum error, expected %s but got %s" %
(md5, md5_download))
return store_path / filename
def Agg_Employment( inputfile, year):
row_count=0
load_count=0
since=year
until=year
cityfile = pd.read_csv(inputfile)
#police-employment-controller : Endpoints pertaining to UCR Police Employment data
#/api/police-employment/agencies/{ori}/{since}/{until} Agency level UCR Police Employment Endpoint
#base_url = 'https://api.usa.gov/crime/fbi/sapi/api/police-employment/agencies/TX0312900/2016/2016'
#base_url_e = f'https://api.usa.gov/crime/fbi/sapi/api/police-employment/agencies/{ori}/{since}/{until}'
#summarized-data-controller : Endpoints pertaining to Agency SRS Level Crime Data
#/api/summarized/agencies/{ori}/offenses/{since}/{until} Agency level SRS Crime Data Endpoint
#base_url_c=f'https://api.usa.gov/crime/fbi/sapi/api/summarized/agencies/{ori}/offenses/{since}/{until}'
p={
'api_key':'fByj7VuEfeNrsAI7MxVeiltb3Y0kULdT0XzfTbDf'
}
df = pd.DataFrame(cityfile,columns=['STATENAME','COUNTYNAME','ADDRESS_CITY','ORI9'])
for index, row in df.iterrows():
ori = row['ORI9']
base_url = f'https://api.usa.gov/crime/fbi/sapi/api/police-employment/agencies/{ori}/{since}/{until}'
#print(requests.get(base_url, p).url)
try:
j = requests.get(base_url, p).json()
print(f'Processing Employment Record {row_count} in load count of {load_count} | State: {row.STATENAME} , County: {row.COUNTYNAME}, CITY {row.ADDRESS_CITY}, ORI: {row.ORI9} ')
#print(requests.get(base_url_e, p).url)
if row_count==1:
ded= pd.DataFrame(pd.DataFrame(j['results']))
else:
ded=ded.append(pd.DataFrame(j['results']), ignore_index=True)
except:
print(f"Error with city data. Skipping {row.ADDRESS_CITY} - {row.ORI9}......")
df.drop(labels=index, inplace=True)
row_count+=1
time.sleep(1) #60 seconds / limit request of 60
E_Join = df.merge(ded, left_on = 'ORI9', right_on = 'ori')
E_agg = E_Join.groupby(['STATENAME','COUNTYNAME','ADDRESS_CITY']).aggregate({'total_pe_ct':'sum'})
Base_Dir=Path(__file__).resolve().parent
Output1=Base_Dir.joinpath('output\E_Agg.csv')
Output2=Base_Dir.joinpath('output\E_Breakdown.csv')
E_agg.to_csv(Output1)
E_Join.to_csv(Output2)
E_Join=''
E_agg=''
cityfile=''
ded=''
df=''
j=''
ori=''
def get_data_loaders(train_ann_file, test_ann_file, batch_size, test_size, image_size, use_mask):
# first, crate PyTorch dataset objects, for the train and validation data.
dataset = CocoMask(
root=Path.joinpath(Path(train_ann_file).parent.parent, train_ann_file.split('_')[1].split('.')[0]),
annFile=train_ann_file,
transforms=get_transform(train=True, image_size=image_size),
use_mask=use_mask)
dataset_test = CocoMask(
root=Path.joinpath(Path(test_ann_file).parent.parent, test_ann_file.split('_')[1].split('.')[0]),
annFile=test_ann_file,
transforms=get_transform(train=False, image_size=image_size),
use_mask=use_mask)
labels_enumeration = dataset.coco.cats
indices_val = torch.randperm(len(dataset_test)).tolist()
dataset_val = torch.utils.data.Subset(dataset_test, indices_val[:test_size])
# set train and validation data-loaders
train_loader = DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=6,
collate_fn=safe_collate, pin_memory=True)
val_loader = DataLoader(dataset_val, batch_size=batch_size, shuffle=False, num_workers=6,
collate_fn=safe_collate, pin_memory=True)
return train_loader, val_loader, labels_enumeration
class DutSpec(object):
def __init__(self):
self.this_path = Path(__file__).parent
self.spec_path = self.this_path.joinpath('spec')
file_list = os.listdir(str(self.spec_path))[:-1]
data_list = list()
for file_name in file_list:
with open(str(self.spec_path.joinpath(file_name)),
'r',
encoding='utf-8') as f:
raw_dict = yaml.load(f)
device_name = file_name.replace(".yaml", "")
if device_name == 'default':
continue
for k, v in raw_dict['feature'].items():
main = k
for s_k, s_v in v.items():
if 'type' == s_k:
sub = 'self'
f_type = v['type']
else:
sub = s_k
f_type = s_v
data_list.append((main, sub, f_type, device_name))
self.header = ['main', 'sub', 'types', 'duts']
self.df = DataFrame(data_list, columns=self.header)
self.dict_data = dict()
mains = [
item for item, count in Counter(self.df['main']).items()
if count >= 1
]
for main in mains:
self.dict_data[main] = dict()
db_of_main = self.df.loc[self.df['main'] == main]
subs = [
item for item, count in Counter(db_of_main['sub']).items()
if count >= 1
]
for sub in subs:
self.dict_data[main][sub] = dict()
db_of_sub = db_of_main.loc[db_of_main['sub'] == sub]
f_types = [
item
for item, count in Counter(db_of_sub['types']).items()
if count >= 1
]
for f_type in f_types:
duts = list(
db_of_sub.loc[db_of_sub['types'] == f_type]['duts'])
self.dict_data[main][sub][f_type] = duts
def show_table(self):
print(tabulate(self.df, headers=self.header))
def to_json(self):
with open(str(self.this_path.joinpath('req_of_ds.json')), 'w') as f:
json.dump(self.dict_data, f, indent=4)
def build_project(project_path, project_name, tree=True):
"""
Creates a basic project structure for Data Science projects.
Parameters:
-----------
project_path: str - Absolute path where you want to create the project
root directory
project_name: str - Name of your project
tree: bool - Whether a tree visualization is required. Defaults
to True
Returns:
--------
None. Creates required directories and sub-directories for a basic Data
Science project
Example:
--------
build_project --project_path=/absolute/path --project_name=name
--project_tree=True
"""
project_path = Path(project_path)
os.chdir(str(project_path))
if Path(project_name).exists() and Path(project_name).is_dir():
print 'Project already exists'
else:
folder = project_path.joinpath(project_name)
folder.mkdir(mode=0777, parents=True)
proj_path = project_path.joinpath(project_name)
os.chdir(str(proj_path.absolute()))
fol_ls = ['models', 'data', 'preprocess', 'visualize', 'results']
fil_ls = ['README.md', 'utils.py', 'requirements.txt', '__init__.py']
for fol in fol_ls:
proj = proj_path.joinpath(fol)
proj.mkdir(mode=0777, parents=True)
for fil in fil_ls:
files = proj_path.joinpath(fil)
files.touch(mode=0777)
for fol in fol_ls:
proj = proj_path.joinpath(fol)
os.chdir(str(proj.absolute()))
readme_file = proj.joinpath('README.md')
readme_file.touch(mode=0777)
if fol == 'models' or fol == 'preprocess':
init_file = proj.joinpath('__init__.py')
init_file.touch(mode=0777)
if fol == 'visualize':
init_file = proj.joinpath('__init__.py')
init_file.touch(mode=0777)
image = proj.joinpath('Images')
image.mkdir(mode=0777, parents=True)
os.chdir(str(image.absolute()))
readme_file = image.joinpath('README.md')
readme_file.touch(mode=0777)
if tree:
project_tree(project_path.joinpath(project_name))
def remove_malicious_files(dataset_path):
with open('malicious_wiki_files', 'r') as f:
malicious_file_ids = f.read().splitlines()
test_path = Path(dataset_path).joinpath(Path('test'))
train_path = Path(dataset_path).joinpath(Path('train'))
dev_path = Path(dataset_path).joinpath(Path('dev'))
deleted_file_count = 0
for id in malicious_file_ids:
file_path_suffix = Path(wiki_processor.get_file_path(id)).joinpath(id)
if test_path.joinpath(file_path_suffix).exists():
test_path.joinpath(file_path_suffix).remove()
deleted_file_count += 1
elif train_path.joinpath(file_path_suffix).exists():
train_path.joinpath(file_path_suffix).remove()
deleted_file_count += 1
elif dev_path.joinpath(file_path_suffix).exists():
dev_path.joinpath(file_path_suffix).remove()
deleted_file_count += 1
else:
raise Exception('meliciious file is not included in dataset: ' +
str(id))
print('Deleted ' + str(deleted_file_count) +
' files. Malicious file count: ' + str(len(malicious_file_ids)))
def configure_ssh(ssh_key_secret):
if ssh_key_secret is None:
yield
# If we get here, we are runnig in automation.
# We use a user hgrc, so that we are also get the system-wide hgrc
# settings.
hgrc = Path(user_config_dir("hg")).joinpath("hgrc")
if hgrc.exists():
raise FailedCommandError(
"Not overwriting `{}`; cannot configure ssh.".format(hgrc)
)
try:
ssh_key_dir = Path(tempfile.mkdtemp())
ssh_key = get_secret(ssh_key_secret)
ssh_key_file = ssh_key_dir.joinpath("id_rsa")
ssh_key_file.write_text(ssh_key["ssh_privkey"])
ssh_key_file.chmod(0o600)
hgrc_content = (
"[ui]\n"
"username = trybld\n"
"ssh = ssh -i {path} -l {user}\n".format(
path=ssh_key_file, user=ssh_key["user"]
)
)
hgrc.write_text(hgrc_content)
yield
finally:
shutil.rmtree(str(ssh_key_dir))
os.remove(str(hgrc))
def manufacture(model_name):
root_path = Path(__file__).parent
with open(str(root_path.joinpath(model_name)) + ".yaml", 'r') as f:
data = yaml.load(f)
device = Device(data['name'])
device.update_feature(**data['feature'])
return device
def path_object(self):
user_commands_dir_path = Path(ConfigUtil().user_commands_directory)
command_path = user_commands_dir_path.joinpath(self.file_name)
return command_path
def get_build_dir():
build_path = Path(build_config.build_dir)
if build_path.is_absolute():
return build_path.joinpath(build_config.name)
else:
return Path(working_path).joinpath(build_config.build_dir,
build_config.name)
def main(src, dest):
"""links configfiles from one folder to another
if links exists it verifies content
if files exist at the target side it errors
Args:
src: source folder
dest: target folder
"""
src = Path(src)
if not src.exists():
print("WARNING:", src, "does not exist, skipping linking")
return
dest = Path(dest)
for element in filter(_is_yaml_file, src.iterdir()):
_warn_on_unknown_encryption(element)
target = dest.joinpath(element.name)
# the following is fragile
if target.is_symlink():
_warn_on_missmatching_symlink(src=element, target=target)
elif target.is_file():
_warn_on_existing_file(target)
else:
target.symlink_to(element.resolve())
def __init__(self, DEBUG=False):
# import settings or use these as defaults
self.DEBUG = DEBUG
self.per_push_min_commits = 5
self.per_push_max_commits = 20
self.per_push_range_of_commits = random.randrange(
self.per_push_min_commits, self.per_push_max_commits)
self.per_commit_additions = 10
self.per_commit_subtractions = 2
self.per_commit_edits = self.per_commit_additions + self.per_commit_subtractions
self.current_path = Path.cwd()
self.parent_path = self.current_path.parent
# dummy repo we will use to make commits
self.repo = 'https://github.com/CROOOME/automated_bughit.git'
self.repo_name = self.repo.split("/")[-1].split(".")[
0] # default: repo_name [excluding extension]
self.repo_dir = Path.joinpath(self.parent_path, self.repo_name)
self.commands = [
'git add {}'.format('new_file'), # make_file
"git commit -m '{}'".format('commit message to be filled here')
]
self.extensions = [
'c', 'cpp', 'py', 'html', 'css', 'js', 'txt', 'json', 'xml'
]
def main(src, dest):
"""links configfiles from one folder to another
if links exists it verifies content
if files exist at the target side it errors
Args:
src: source folder
dest: target folder
"""
src = Path(src)
if not src.exists():
print("WARNING:", src, "does not exist, skipping linking")
return
dest = Path(dest)
for element in filter(_is_yaml_file, src.iterdir()):
_warn_on_unknown_encryption(element)
target = dest.joinpath(element.name)
# the following is fragile
if target.is_symlink():
_warn_on_missmatching_symlink(src=element, target=target)
elif target.is_file():
_warn_on_existing_file(target)
else:
target.symlink_to(element.resolve())
def _get_config_path():
xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
if xdg_config_home:
config_home = Path(xdg_config_home)
else:
config_home = Path.home().joinpath('.config')
return config_home.joinpath('jiraprompt')
def setUpClass(cls):
""" Set up the annotation DB and object tracker.
"""
objects_path = Path(objects.__file__).parent
json_path = objects_path.joinpath('data', 'annotations.json')
cls.db = AnnotationDB()
cls.db.load_file(str(json_path))
cls.object_tracker = ObjectTracker()
def set_here(wd=None):
"""
Creates a .here file at the passed directory.
Parameters
----------
wd : Path object or string
The directory that a .here file will be created in. If none is set,
uses Path.cwd()
"""
if wd is None:
wd = Path.cwd()
elif type(wd) is str:
wd = Path(wd)
wd.parent.mkdir(parents=True, exist_ok=True)
wd.joinpath(".here").touch()
def main(src, dest, force):
"""links configfiles from one folder to another
if links exists it verifies content
if files exist at the target side it errors
Args:
src: source folder
dest: target folder
force: override existing symlinks
"""
src = Path(src)
if not src.exists():
print("WARNING:", src, "does not exist, skipping linking")
return
dest = Path(dest)
for element in filter(_is_yaml_file, src.iterdir()):
_warn_on_unknown_encryption(element)
target = dest.joinpath(element.name)
if force:
try:
target.symlink_to(element.resolve())
except OSError as e:
if e.errno == errno.EEXIST:
backup_target = Path(dest.joinpath(element.name + "_bak"))
print("Replacing", target.name, "and saving backup as", backup_target.name)
# Would use 'backup_target.replace()' here but that's only supported in py3
if backup_target.exists():
os.remove(str(backup_target))
target.rename(backup_target)
target.symlink_to(element.resolve())
else:
raise
else:
if target.is_symlink():
# If symlink already exists and points to same src, do nothing.
_check_missmatching_symlink(src=element, target=target)
elif _check_existing_file(target):
target.symlink_to(element.resolve())
print("Symlink created for", target.name)
def __init__(self, mp4_file):
self.input_file = mp4_file
stem = Path(mp4_file).stem
self.album, self.artist, self.title = stem.split(' ')
parent = Path(mp4_file).parent
self.output_file = str(parent.joinpath(
'%s %s.m4a' % (self.artist, self.title)))
self.artwork_file = None
for ext in ('.jpg', '.png'):
imgfile = parent.joinpath(stem + ext)
if imgfile.exists():
self.artwork_file = str(imgfile)
break
if not self.artwork_file:
self.tempdir = tempfile.mkdtemp()
self.artwork_file = op.join(self.tempdir, 'artwork.jpg')
def main():
path = Path(__file__).parent.parent.joinpath('python-lgu-sct')
path = path.joinpath('sct')
path = path.joinpath('tftpboot')
d_list = listdir(str(path))
for d in d_list:
_path = path.joinpath(d).joinpath('Config')
config_list = listdir(str(_path))
for config in config_list:
config_path = str(_path.joinpath(config))
with open(config_path, 'r') as f:
t_list = f.readlines()
k = 1
for i, t in enumerate(t_list):
k = check_apply_exit(t, i, config, k)
check_receiver(t, i, config)
show_auto_reset(t, i, config)
check_hangul(t, i, config)
check_jumboframe(t, i, config)
def create_diff_folder():
"""
:return: path of diff files
:rtype: Path
"""
diff_folder = Path(os.getcwd()).joinpath('./diffs/')
diff_folder = diff_folder.joinpath(str(uuid.uuid4()).split('-')[-1])
if not diff_folder.exists():
diff_folder.mkdir(parents=True)
return diff_folder
def test_subfolder_support_disabled(self):
watch_folder = self._create_test_watchfolder()
test_path = Path(self.temporary_directory)
test_subfolder = test_path.joinpath(TEST_WATCHFOLDER_SUBFOLDER)
test_subfolder.mkdir()
shutil.copy(TEST_SMALL_DOCUMENT_PATH, force_text(test_subfolder))
watch_folder.check_source()
self.assertEqual(Document.objects.count(), 0)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-i',
'--in-path',
type=str,
default='DQM_V0001_R000000001__Global__CMSSW_X_Y_Z__RECO.root')
#stand!!!! new topology
parser.add_argument(
'-i2',
'--in-compare',
type=str,
default=
'/cms/ldap_home/seungjun/what2/ol/CMSSW_11_2_X_2020-10-05-1200/src/23434.999_TTbar_14TeV+2026D49PU_PMXS1S2PR+TTbar_14TeV_TuneCP5_GenSimHLBeamSpot14+PREMIX_PremixHLBeamSpot14PU+DigiTriggerPU+RecoGlobalPU+HARVESTGlobalPU/DQM_V0001_R000000001__Global__CMSSW_X_Y_Z__RECO.root'
)
#parser.add_argument('-i2', '--in-compare', type=str, default='DQM_V0001_R000000001__Global__CMSSW_X_Y_Z__RECO_origin.root')
#compare old topology
parser.add_argument('-o', '--out-dir', type=str, default='./DQM_1101_1110')
args = parser.parse_args()
root_file = ROOT.TFile(args.in_path)
root_file_compare = ROOT.TFile(args.in_compare)
root_dir = root_file.Get('DQMData/Run 1')
root_dir_compare = root_file_compare.Get('DQMData/Run 1')
filesys_dir = Path(args.out_dir)
if filesys_dir.exists():
raise IOError("Cannot create directory '{}': File exits!!!".format(
args.out_dir))
filesys_dir.mkdir()
draw_all(root_dir.Get('MuonGEMHitsV/Run summary/GEMHitsTask'),
filesys_dir.joinpath('MuonGEMHits'),
root_dir_compare.Get('MuonGEMHitsV/Run summary/GEMHitsTask'))
draw_all(root_dir.Get('MuonGEMDigisV/Run summary/GEMDigisTask'),
filesys_dir.joinpath('MuonGEMDigis'),
root_dir_compare.Get('MuonGEMDigisV/Run summary/GEMDigisTask'))
draw_all(
root_dir.Get('MuonGEMRecHitsV/Run summary/GEMRecHitsTask'),
filesys_dir.joinpath('MuonGEMRecHits'),
root_dir_compare.Get('MuonGEMRecHitsV/Run summary/GEMRecHitsTask'))
def get_resource_path(rel_path):
"""Returns the absolute path to a file or directory relative to the repository root.
Args:
rel_path(:obj:`str`): path relative to repository root
Returns:
:obj:`Path`
"""
root_dir = Path(__file__).parents[2]
return root_dir.joinpath(rel_path)
class MPIFIExpr(object):
def __init__(self, expr, exe, args, hosts, faults, nprocesses):
self._path = Path(expr).absolute()
if not self._path.exists():
self._path.mkdir()
self._exe = Path(exe).absolute()
assert(self._exe.exists()), "Executable (%s) not found" % self._exe
assert(self._exe.is_file()), "Executable (%s) is not a file" % self._exe
self._exe = str(self._exe)
self._args = args
self._hosts = hosts
self._faults = load_faults(faults)
self._nprocesses = nprocesses
def run(self):
libcare = Path(os.environ['CARE_ROOT']).joinpath(
'build/runtime/libCARERuntime.so').absolute()
assert(libcare.exists()), "the recovery runtime library is not setup yet!"
libmpifi = Path(os.environ['CARE_ROOT']).joinpath(
'tools/MPIFI/libmpifi.so').absolute()
assert(libmpifi.exists()), "the MPIFI library is not setup yet!"
CARE = "%s:%s" % (str(libcare), str(libmpifi))
# CARE = str(libmpifi)
os.environ['CARE_WORKER_ID'] = str(0)
for f in self._faults:
print("Performing fault: Addr -- %s, REG -- %s, Fvalue -- %d (%s)" %
(f['ADDR'], f['REG'], f['VALUE'], str(f['VALUE'])))
fid = 'mpifi-inject-%03d' % self._faults.index(f)
os.environ['CARE_EXPR_PATH'] = str(self._path)
os.environ['CARE_INJECTION_ID'] = fid
os.environ['CARE_TARGET_REG'] = f["REG"].upper()
os.environ['CARE_TARGET_ADDR'] = f["ADDR"]
os.environ['CARE_TARGET_DATA'] = str(f["VALUE"])
wd = self._path.joinpath(fid)
if not wd.exists():
wd.mkdir()
wd = str(wd)
os.chdir(wd)
app = MPIApp(self._exe, self._args,
self._nprocesses, self._hosts, CARE)
app.start()
app.wait(1200)
os.chdir(str(self._path))
def convertDir(inPath, outPath):
"""
Function to convert a directory of Tsurf files to VTK format.
"""
tsurfFiles = sorted(inPath.glob(tsurfSearch))
outPath.mkdir(parents=True, exist_ok=True)
for tsurfFile in tsurfFiles:
stem = tsurfFile.stem
outFile = Path.joinpath(outPath, stem).with_suffix(vtkSuffix)
convertFile(tsurfFile, outFile)
return
def upload_file(index):
# quick permission check
if not Index(index).exists():
return error_response(403, 'missing permission to upload files')
if 'file' not in request.files:
return error_response(400, 'no file content given')
uploaded_file = request.files['file']
given_mime_type = uploaded_file.mimetype
detected_mime_type = magic.from_buffer(uploaded_file.stream.read(),
mime=True)
uploaded_file.stream.seek(0)
allowed_mimes = _get_allowed_mimes(current_app.schemastore)
if (detected_mime_type not in allowed_mimes
or given_mime_type != detected_mime_type):
# Intentionally be vague: don't spell out the checks
return error_response(400, 'Invalid file type')
filename = sha1(uploaded_file.read()).hexdigest()
uploaded_file.seek(0)
# Keep published path independent of NFS mount point and document root
local_path = Path(current_app.config['UPLOAD_MNT']).joinpath(filename[:2])
local_path.mkdir(exist_ok=True, parents=True)
target_filename = local_path.joinpath(filename)
uploaded_file.save(str(target_filename))
relative_path = str(Path('/').joinpath(filename[:2]).joinpath(filename))
file_representation = {
'path': relative_path,
'mime': detected_mime_type,
'sha1': filename,
}
response = make_response(jsonify(file_representation), 201)
response.headers['Location'] = urljoin(current_app.config['UPLOAD_HOST'],
'/cdn/files/', relative_path)
try:
Customer.charge_cycles(
g.customer['_id'],
'_upload_{}'.format(given_mime_type.split('/')[0]),
current_app.config['CYCLES_FILE_UPLOAD'])
except auth.NotFoundError as error:
return error_response(httplib.UNAUTHORIZED, error.error)
return response
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--in-path', type=str, default='DQM_V0001_R000000001__Global__CMSSW_X_Y_Z__RECO.root')
parser.add_argument('-o', '--out-dir', type=str, default='./DQM')
args = parser.parse_args()
root_file = ROOT.TFile(args.in_path)
root_dir = root_file.Get('DQMData/Run 1')
filesys_dir = Path(args.out_dir)
if filesys_dir.exists():
raise IOError("Cannot create directory '{}': File exits!!!".format(args.out_dir))
filesys_dir.mkdir()
draw_all(root_dir.Get('MuonGEMHitsV/Run summary/GEMHitsTask'),
filesys_dir.joinpath('MuonGEMHits'))
draw_all(root_dir.Get('MuonGEMDigisV/Run summary/GEMDigisTask'),
filesys_dir.joinpath('MuonGEMDigis'))
draw_all(root_dir.Get('MuonGEMRecHitsV/Run summary/GEMRecHitsTask'),
filesys_dir.joinpath('MuonGEMRecHits'))
def setUpClass(cls):
""" Launch kernel and set search path for annotator.
"""
raise unittest.SkipTest("Custom IPython kernel broken in IPython v7.0")
tu.KM, tu.KC = tu.start_new_kernel(kernel_name=get_kernel_name())
objects_path = Path(test_objects.__file__).parent
json_path = objects_path.joinpath('data', 'annotations.json')
code = dedent("""\
shell = get_ipython()
shell.kernel.annotator.db.load_file('%s')
""" % json_path)
with kernel() as kc:
safe_execute(code, kc, silent=True)
def user_files():
profile_dir = Path(cmk.utils.paths.var_dir, "web", "admin")
profile_dir.mkdir(parents=True, exist_ok=True)
with profile_dir.joinpath("cached_profile.mk").open("w",
encoding="utf-8") as f: # pylint: disable=no-member
f.write(u"%r" % {
"alias": u"admin",
"connector": "default",
})
Path(cmk.utils.paths.htpasswd_file).parent.mkdir(parents=True,
exist_ok=True) # pylint: disable=no-member
with open(cmk.utils.paths.htpasswd_file, "w") as f:
f.write(
"automation:$5$rounds=535000$eDIHah5PgsY2widK$tiVBvDgq0Nwxy5zd/oNFRZ8faTlOPA2T.tx.lTeQoZ1\n"
"cmkadmin:Sl94oMGDJB/wQ\n")
class PasswordRule(CompoundRule):
"""Retrieve stored password."""
def __init__(self, *args, **kwargs):
self.spec = _("password <name> <passphrase>")
self.names = List(name='names')
self.data_path = Path().home().joinpath('speechpass')
self.data_path.mkdir(exist_ok=True)
self.language_path = self.data_path.joinpath(
natlinkstatus.NatlinkStatus().getLanguage())
self.language_path.mkdir(exist_ok=True)
for name in self.language_path.iterdir():
self.names.append(string.replace(name.name, '_', ' '))
self.extras = [
ListRef(name='name', list=self.names),
Dictation(name='passphrase'),
]
CompoundRule.__init__(self, *args, **kwargs)
def value(self, node):
name = node.get_child_by_name('name').value()
name = string.replace(name, ' ', '_')
password_file = self.language_path.joinpath(name)
if not password_file.exists():
print "file does not exist, could not decrypt password"
return None
passphrase = str(
node.get_child_by_name('passphrase').value()).strip().lower()
crypt_text = password_file.read_bytes()
try:
plaintext = decrypt(passphrase, crypt_text)
except DecryptionException:
print "incorrect passphrase"
return None
return text_to_keystr(plaintext.decode('utf8'))
def _process_begin(self):
for name in self.language_path.iterdir():
name = string.replace(name.name, '_', ' ')
if name not in self.names:
self.names.append(name)
def _process_recognition(self, node, extras):
value = self.value(node)
if value is None:
return
Key(value).execute()
def main(args):
sys.path.append(str(Path(__file__).parent))
logger = utils.setup_logger(__name__, 'cross_validate_choi.log')
utils.read_config_file(args.config)
utils.config.update(args.__dict__)
logger.debug('Running with config %s', utils.config)
configure(os.path.join('runs', args.expname))
if not args.test:
word2vec = gensim.models.KeyedVectors.load_word2vec_format(utils.config['word2vecfile'], binary=True)
else:
word2vec = None
dataset_path = Path(args.flat_choi)
with open(args.load_from, 'rb') as f:
model = torch.load(f)
model.eval()
model = maybe_cuda(model)
test_accuracy = accuracy.Accuracy()
for j in range(5):
validate_folder_numbers = range(5)
validate_folder_numbers.remove(j)
validate_folder_names = [dataset_path.joinpath(str(num)) for num in validate_folder_numbers]
dev_dataset = ChoiDataset(dataset_path , word2vec, folder=True, folders_paths=validate_folder_names)
test_dataset = ChoiDataset(dataset_path, word2vec, folder=True, folders_paths=[dataset_path.joinpath(str(j))])
dev_dl = DataLoader(dev_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
test_dl = DataLoader(test_dataset, batch_size=args.test_bs, collate_fn=collate_fn, shuffle=False,
num_workers=args.num_workers)
_, threshold = validate(model, args, j, dev_dl, logger)
test_pk = test(model, args, j, test_dl, logger, threshold, test_accuracy)
logger.debug(colored('Cross validation section {} with p_k {} and threshold {}'.format(j, test_pk, threshold),'green'))
cross_validation_pk, _ = test_accuracy.calc_accuracy()
print ('Final cross validaiton Pk is: ' + str(cross_validation_pk))
logger.debug(
colored('Final cross validaiton Pk is: {}'.format(cross_validation_pk), 'green'))
class GitRepo(object):
def __init__(self, path, remote_url=None, remote_name=None, branch_name='master'):
self.path = Path(path)
self.path_str = str(self.path)
self.remote_url = remote_url
self.branch_name = branch_name
if not remote_name:
remote_name = 'bedrock-dev' if settings.DEV else 'bedrock-prod'
self.remote_name = remote_name
def git(self, *args):
"""Run a git command against the current repo"""
curdir = os.getcwd()
try:
os.chdir(self.path_str)
output = check_output((GIT,) + args, stderr=STDOUT)
finally:
os.chdir(curdir)
return output.strip()
@property
def full_branch_name(self):
"""Full branch name with remote (e.g. origin/master)"""
return '{}/{}'.format(self.remote_name, self.branch_name)
@property
def current_hash(self):
"""The git revision ID (hash) of the current HEAD"""
return self.git('rev-parse', 'HEAD')
@property
def remote_names(self):
"""Return a list of the remote names in the repo"""
return self.git('remote').split()
def has_remote(self):
"""Return True if the repo has a remote by the correct name"""
return self.remote_name in self.remote_names
def add_remote(self):
"""Add the remote to the git repo from the init args"""
if not self.remote_url:
raise RuntimeError('remote_url required to add a remote')
self.git('remote', 'add', self.remote_name, self.remote_url)
def diff(self, start_hash, end_hash):
"""Return a 2 tuple: (modified files, deleted files)"""
diff_out = StringIO(self.git('diff', '--name-status', start_hash, end_hash))
modified = set()
removed = set()
for line in diff_out:
parts = line.split()
# delete
if parts[0] == 'D':
removed.add(parts[1])
# rename
elif parts[0][0] == 'R':
removed.add(parts[1])
modified.add(parts[2])
# everything else
else:
# some types (like copy) have two file entries
for part in parts[1:]:
modified.add(part)
return modified, removed
def clone(self):
"""Clone the repo specified in the initial arguments"""
if not self.remote_url:
raise RuntimeError('remote_url required to clone')
self.path.mkdir(parents=True, exist_ok=True)
self.git('clone', '--origin', self.remote_name, '--depth', '1',
'--branch', self.branch_name, self.remote_url, '.')
def pull(self):
"""Update the repo to the latest of the remote and branch
Return the previous hash and the new hash."""
if not self.has_remote():
self.add_remote()
old_hash = self.current_hash
self.git('fetch', self.remote_name)
self.git('checkout', '-f', self.full_branch_name)
return old_hash, self.current_hash
def update(self):
"""Updates a repo, cloning if necessary.
:return a tuple of lists of modified and deleted files if updated, None if cloned
"""
if self.path.is_dir():
if not self.path.joinpath('.git').is_dir():
rmtree(self.path_str, ignore_errors=True)
self.clone()
else:
return self.diff(*self.pull())
else:
self.clone()
return None, None
from django.core.management import BaseCommand, CommandError
import boto3
import requests
from apscheduler.schedulers.blocking import BlockingScheduler
from django_statsd.clients import statsd
from pathlib2 import Path
from pytz import utc
from raven.contrib.django.raven_compat.models import client as sentry_client
from basket.news.backends.sfmc import sfmc
TMP = Path('/tmp')
BUCKET_DIR = 'fxa-last-active-timestamp/data'
DATA_PATH = TMP.joinpath(BUCKET_DIR)
FXA_IDS = {}
FILE_DONE_KEY = 'fxa_activity:completed:%s'
FILES_IN_PROCESS = []
TWO_WEEKS = 60 * 60 * 24 * 14
UPDATE_COUNT = 0
schedule = BlockingScheduler(timezone=utc)
def log(message):
print('process_fxa_data: %s' % message)
def _fxa_id_key(fxa_id):
return 'fxa_activity:%s' % fxa_id
from nose.tools import assert_not_equal, eq_, ok_
from pathlib2 import Path
from product_details import product_details
from pyquery import PyQuery as pq
from bedrock.mozorg.tests import TestCase
from lib.l10n_utils import render
from lib.l10n_utils.dotlang import (_, _lazy, FORMAT_IDENTIFIER_RE, lang_file_has_tag,
lang_file_is_active, parse, translate)
from lib.l10n_utils.extract import extract_python
LANG_FILES = 'test_file'
ROOT_PATH = Path(__file__).with_name('test_files')
ROOT = str(ROOT_PATH)
TEMPLATE_DIRS = [str(ROOT_PATH.joinpath('templates'))]
jinja_env = Jinja2.get_default().env
@override_settings(DEV=False, LANGUAGE_CODE='en-US')
@patch.object(jinja_env.loader, 'searchpath', TEMPLATE_DIRS)
@patch.object(settings, 'ROOT_URLCONF', 'lib.l10n_utils.tests.test_files.urls')
@patch.object(settings, 'ROOT', ROOT)
class TestLangFilesActivation(TestCase):
def setUp(self):
clear_url_caches()
def test_lang_file_is_active(self):
"""
`lang_file_is_active` should return true if lang file has the
comment, and false otherwise.
class GitRepo(object):
def __init__(self, path, remote_url=None, branch_name='master'):
self.path = Path(path)
self.path_str = str(self.path)
self.remote_url = remote_url
self.branch_name = branch_name
db_latest_key = '%s:%s:%s' % (self.path_str, remote_url or '',
branch_name)
self.db_latest_key = sha256(db_latest_key).hexdigest()
def git(self, *args):
"""Run a git command against the current repo"""
curdir = os.getcwd()
try:
os.chdir(self.path_str)
output = check_output((GIT,) + args, stderr=STDOUT)
finally:
os.chdir(curdir)
return output.strip()
@property
def current_hash(self):
"""The git revision ID (hash) of the current HEAD or None if no repo"""
try:
return self.git('rev-parse', 'HEAD')
except OSError:
return None
def diff(self, start_hash, end_hash):
"""Return a 2 tuple: (modified files, deleted files)"""
diff_out = StringIO(self.git('diff', '--name-status', start_hash, end_hash))
modified = set()
removed = set()
for line in diff_out:
parts = line.split()
# delete
if parts[0] == 'D':
removed.add(parts[1])
# rename
elif parts[0][0] == 'R':
removed.add(parts[1])
modified.add(parts[2])
# everything else
else:
# some types (like copy) have two file entries
for part in parts[1:]:
modified.add(part)
return modified, removed
def clone(self):
"""Clone the repo specified in the initial arguments"""
if not self.remote_url:
raise RuntimeError('remote_url required to clone')
self.path.mkdir(parents=True, exist_ok=True)
self.git('clone', '--depth', '1', '--branch',
self.branch_name, self.remote_url, '.')
def pull(self):
"""Update the repo to the latest of the remote and branch
Return the previous hash and the new hash."""
old_hash = self.current_hash
self.git('fetch', '-f', self.remote_url, self.branch_name)
self.git('checkout', '-f', 'FETCH_HEAD')
return old_hash, self.current_hash
def update(self):
"""Updates a repo, cloning if necessary.
:return a tuple of lists of modified and deleted files if updated, None if cloned
"""
if self.path.is_dir():
if not self.path.joinpath('.git').is_dir():
rmtree(self.path_str, ignore_errors=True)
self.clone()
else:
return self.pull()
else:
self.clone()
return None, None
def reset(self, new_head):
self.git('reset', '--hard', new_head)
def get_db_latest(self):
try:
return GitRepoState.objects.get(repo_id=self.db_latest_key).latest_ref
except GitRepoState.DoesNotExist:
return None
def has_changes(self):
return self.current_hash != self.get_db_latest()
def set_db_latest(self, latest_ref=None):
latest_ref = latest_ref or self.current_hash
rs, created = GitRepoState.objects.get_or_create(repo_id=self.db_latest_key,
defaults={'latest_ref': latest_ref})
if not created:
rs.latest_ref = latest_ref
rs.save()
from django.test.client import RequestFactory
from django.test.utils import override_settings
from bedrock.base.urlresolvers import reverse
from mock import patch, Mock
from nose.tools import eq_, ok_
from pathlib2 import Path
from bedrock.firefox.firefox_details import FirefoxDesktop
from bedrock.mozorg.tests import TestCase
from bedrock.releasenotes import views
from bedrock.releasenotes.models import ProductRelease
TESTS_PATH = Path(__file__).parent
DATA_PATH = str(TESTS_PATH.joinpath('data'))
firefox_desktop = FirefoxDesktop(json_dir=DATA_PATH)
RELEASES_PATH = str(TESTS_PATH)
@override_settings(RELEASE_NOTES_PATH=RELEASES_PATH)
class TestReleaseViews(TestCase):
# Set DEV=False, otherwise all the releases will be erroneously made public
# through the following refresh function, leading to wrong results in
# get_release_or_404
@override_settings(DEV=False)
def setUp(self):
ProductRelease.objects.refresh()
caches['release-notes'].clear()
self.activate('en-US')
self.factory = RequestFactory()
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from django.test import override_settings
from mock import patch, DEFAULT
from pathlib2 import Path
from bedrock.mozorg.tests import TestCase
from bedrock.mozorg.management.commands import update_product_details_files
PD_REPO_TEST_PATH = Path(__file__).parent.joinpath('test_pd_repo')
@override_settings(PROD_DETAILS_STORAGE='PDDatabaseStorage',
PROD_DETAILS_TEST_DIR=str(PD_REPO_TEST_PATH.joinpath('product-details')))
class TestUpdateProductDetailsFiles(TestCase):
def setUp(self):
self.command = update_product_details_files.Command()
self.command.repo.path = PD_REPO_TEST_PATH
self.command.repo.path_str = str(PD_REPO_TEST_PATH)
def test_handle_diff_loads_all(self):
with patch.multiple(self.command, update_file_data=DEFAULT, validate_data=DEFAULT,
file_storage=DEFAULT, load_changes=DEFAULT, repo=DEFAULT):
options = dict(quiet=False, database='default')
self.command.update_file_data.return_value = True
self.command.handle(**options)
assert self.command.file_storage.all_json_files.called
self.command.load_changes. \
assert_called_with(options, self.command.file_storage.all_json_files())
