def parse_areas_of_study_and_write_to_database(input_arguments):
db = connect_to_database(input_arguments.db_host, input_arguments.db)
if input_arguments.erase:
db.areas.drop()
db.courses.drop()
html_univ_files = get_univ_files(input_arguments.path)
areas = dict()
for univ_file in html_univ_files:
with open(univ_file, 'rb') as file:
q = file.read()
local_areas = get_area_course_info(q)
for key, value in local_areas.items():
if key not in areas:
areas[key] = list()
for course in value:
if course not in areas[key]:
areas[key].append(course)
print(areas)
areas_dao = [{'area_id_old': generate_id(title), 'area_title': title}
for title in areas.keys()]
courses_dao = list()
for title, course_list in areas.items():
curr_courses = [{'area_id_old': generate_id(title), 'course_title': c}
for c in course_list]
courses_dao.extend(curr_courses)
db.areas.insert_many(areas_dao)
db.courses.insert_many(courses_dao)
def _format_filter_data_to_mongo_request(self, data):
result_query = {}
for key in FILTER_KEYS:
if isinstance(key, dict) or key == 'univ_location':
logger.debug(
f'Omitting filtering by univ_location or dict: {key}')
continue
if key in data:
if key == 'univ_title':
if 'univ_location' in data:
univ_ids = self._get_univ_ids(
data[key], data['univ_location'])
else:
univ_ids = [generate_id(title) for title in data[key]]
if len(univ_ids) > 0:
result_query['univ_id'] = {"$in": univ_ids}
elif key == 'area_title':
result_query['area_id_old'] = {
"$in": [generate_id(title) for title in data[key]]}
else:
if key == 'is_enrolled':
result_query[key] = data[key] == 'true'
else:
result_query[key] = {"$in": data[key]}
return result_query
def reset(self):
self.uuid = utils.generate_id()
self.request_id = utils.generate_id()
self.num_turns = 0
self.is_ongoing_turn = False
self.cur_hypothesis = ''
self.phrase = ''
self.received_messages = []
self.metrics = []
def open(self):
self.client_id = utils.generate_id()
while ClientWebSocket.client_handler.get(self.client_id,
None) is not None:
self.client_id = utils.generate_id()
ClientWebSocket.client_handler[self.client_id] = self
self.permission = dict(public_key=False, private_key=False)
logger.debug("A Client Websocket established")
print("Openned connection")
def __init__(self, my_id):
self.my_id = my_id
self.path_map = {}
self.path_index = [generate_id()]
self.creation_time = datetime.datetime.now()
self.my_titles = ActiveClient.LINK_TITLES[:]
random.shuffle(self.my_titles)
def new_client(self, easy=False):
'''Create a new client and add it to the managed list'''
self._clean_list()
cl_id = generate_id()
nc = ActiveClient(cl_id, easy=easy)
self.clients[cl_id] = nc
return nc
def show_versions(self, key):
meta = os.path.join(self.__root, utils.generate_id(key), 'meta')
if not os.path.isfile(meta):
return []
versions = []
for rec in self._read(meta).split("\n"):
if not '---' in rec:
continue
parts = rec.split("---")
ts = parts[0]
ver = parts[1]
if len(parts) > 2:
attrs = parts[2].strip()
else:
attrs = ''
v_info = {
'timestamp':
ts.strip(),
'timestamp_utc':
time.strftime('%Y-%m-%d %H:%M:%S UTC', time.gmtime(float(ts))),
'version':
ver.strip(),
'attributes':
self._deserialize_attrs(attrs),
}
versions.append(v_info)
return versions[::-1]
def process_upload_multiple(files, callback_url=None):
id = utils.generate_id()
d = utils.storage_dir_for_id(id)
os.makedirs(d)
file_id = 0
session = database.Session()
m = database.model(id, '')
session.add(m)
for file in files:
fn = file.filename
filewriter = lambda fn: file.save(fn)
filewriter(os.path.join(d, id + "_" + str(file_id) + ".ifc"))
file_id += 1
m.files.append(database.file(id, ''))
session.commit()
session.close()
if DEVELOPMENT:
t = threading.Thread(
target=lambda: worker.process_multiple(id, callback_url))
t.start()
else:
q.enqueue(worker.process_multiple, id, callback_url)
return id
def f(x):
id = utils.generate_id()
speed_limits[i] = x
runPractice.run_practice(id, speed_limits=speed_limits, **args)
score = utils.rate_race(id)
print(f'Score: {score}, speed_limits: {speed_limits}, id: {id}')
return score
def new_client(self):
'''Create a new client and add it to the managed list'''
self._clean_list()
cl_id = generate_id()
nc = ActiveClient(cl_id)
self.clients[cl_id] = nc
return nc
def put(self, key, data):
timestamp = "%.10f" % (time.time())
subdir = os.path.join(self.__root, utils.generate_id(key))
utils.mkdir_p(subdir)
self._save(subdir + "/original_key", key)
version = self._compute_version(timestamp, data)
self._save(os.path.join(subdir, version), data)
self._update_meta(subdir + "/meta", timestamp, version)
def put(self, key, data, attrs=None):
timestamp = "%.10f" % (time.time())
subdir = os.path.join(self.__root, utils.generate_id(key))
utils.mkdir_p(subdir)
self._save(subdir + "/original_key", key)
version = self._compute_version(timestamp, data)
self._save(os.path.join(subdir, version), data)
self._update_meta(subdir + "/meta", timestamp, version,
self._serialize_attrs(attrs))
return {'key': key, 'version': version}
def __init__(self, my_id, easy=False):
self.my_id = my_id
self.path_map = {}
self.path_index = [generate_id()]
self.creation_time = datetime.datetime.now()
if easy:
self.my_titles = ActiveClient.EASY_LINK_TITLES[:]
else:
self.my_titles = ActiveClient.LINK_TITLES[:]
random.shuffle(self.my_titles)
def find_node(self, addr, target=None):
self.send_message({
"t": generate_id(),
"y": "q",
"q": "find_node",
"a": {
"id": generate_node_id(),
"target": target or generate_node_id()
}
}, addr)
def get(self, key, version=None):
if version is None:
versions = self.show_versions(key)
if len(versions) == 0:
return None
version = versions[0]['version']
full_path = os.path.join(self.__root, utils.generate_id(key), version)
if os.path.isfile(full_path):
return self._read(full_path)
return None
def get_peers(self, addr, info_hash, t=None):
self.send_message({
"t": t or generate_id(),
"y": "q",
"q": "get_peers",
"a": {
"id": generate_node_id(),
"info_hash": info_hash
}
}, addr)
def add_link():
link_url = request.forms.get('link_url')
autodelete = request.forms.get('autodelete') == 'on'
password = request.forms.get('password', '')
if link_url:
while True:
id = generate_id()
if manager.get_link_by_id(id) is None:
break
manager.save(Link(id, link_url, autodelete, password))
redirect('/')
def generate_keys(company, password):
company_id = str(company.id)
company_key = utils.generate_id()
company_key_encrypted = utils.encrypt(text=company_key, password=password)
public_key = company_id + "_" + utils.generate_id()
private_key = company_id + "_" + utils.generate_id()
private_key_hash = utils.hash(string=private_key)
private_key_encrypted = utils.encrypt(text=private_key,
password=company_key)
password_hash = utils.hash(string=password)
return dict(public_key=public_key,
company_key_encrypted=company_key_encrypted,
private_key_hash=private_key_hash,
private_key_encrypted=private_key_encrypted,
password_hash=password_hash)
def login(email, password):
logout()
user = models.User.objects(email=email)
if len(user) != 1:
raise messages.user_not_found()
user = user[0]
if not utils.check_hash(string=password, hashed_string=user.password_hash):
raise messages.user_not_found()
session_id = utils.generate_id()
while redisConnection.get("session:" + session_id) is not None:
session_id = utils.generate_id()
flask.session['id'] = session_id
session_content = {"user_id": str(user.id)}
session_content = json.dumps(session_content)
redisConnection.set("session:" + session_id, session_content)
return user
def add_user(name, email, password):
password = generate_password_hash(password=password,
method='pbkdf2:sha256',
salt_length=8)
new_user = User(email=email,
password=password,
name=name,
public_id=generate_id())
db.session.add(new_user)
db.session.commit()
generate_email_verification(new_user.email, new_user.public_id, name)
return redirect(url_for('main.home'))
def save_recipe(self, recipe):
""" Save recipe in Database
Args:
recipe: Dictionary
Returns:
dictionary status
"""
if 'title' in recipe:
recipe_id = generate_id(recipe['title'])
recipe['id'] = recipe_id
return self.database.put_item(recipe)
return dict(error='Missing recipe title')
def __init__(self,
name,
audio_id,
audio_version,
start_time,
end_time,
_id=None):
self.id = utils.generate_id(_id)
self.name = name
self.audio_id = audio_id
self.audio_version = audio_version
self.start_time = start_time
self.end_time = end_time
def show_versions(self, key):
meta = os.path.join(self.__root, utils.generate_id(key), 'meta')
if not os.path.isfile(meta):
return []
versions = []
for rec in self._read(meta).split("\n"):
if not '---' in rec:
continue
ts, ver = rec.split("---")
versions.append({
'timestamp': float(ts.strip()),
'version': ver.strip()
})
return versions[::-1]
def check_personal_group(user):
p_gid = utils.generate_id(user.id)
group = Group.query.get(p_gid)
# if personal group does not exist, create it
if group is None:
_name = utils.personal_group_name(user)
group = Group(_name, user, user.id)
db_session.add(group)
db_session.commit()
# if user has no current group id, use personal group
if not user.current_group_id:
user.current_group_id = group.id
# make sure user has membership of personal group
add_user_to_group(user, user, group)
def create(cls, name, description, points, answer, slug, unlocked=False):
challenge = cls()
challenge.name = name
challenge.description = description
challenge.cid = utils.generate_id()
challenge.points = points
challenge.answer_hash = pbkdf2.crypt(answer)
challenge.cat_slug = slug
challenge.unlocked = unlocked
weight = db.session.query(db.func.max(Challenge.weight)).scalar()
challenge.weight = (weight + 1) if weight else 1
challenge.prerequisite = ''
db.session.add(challenge)
return challenge
def get_common_info_and_create_base_request(
file_name: str,
file_string: str) -> Optional[AbstractAdmissionRequest]:
univ_id, list_id = get_univ_id_and_list_id_from_filename(file_name)
course_name = HtmlParser2017.get_course(file_string)
course_id = generate_id(course_name)
type_of_education = HtmlParser2017.get_type_of_education(file_string)
if type_of_education is '':
return
# if both False - education type is 'дистанційна'
is_denna = 'денна' in type_of_education or 'вечірня' in type_of_education
is_zaochna = 'заочна' in type_of_education
return AbstractAdmissionRequest(
univ_id, list_id, is_denna, is_zaochna, course_id)
def get_recipe(self, recipe_id):
""" Load recipe from Database
Args:
recipe_id: Database 'id' or title
Returns:
dictionary
"""
if len(recipe_id) != 48 or not contains_only(recipe_id, r'[^2-7A-Z.]'):
recipe_id = generate_id(recipe_id)
if recipe_id in self.recipes:
recipe = self.recipes[recipe_id]
else:
recipe = self.database.get_item('id', recipe_id)
return recipe
def create(cls, name, description, points, answer, slug, unlocked=False):
challenge = cls()
challenge.name = name
challenge.description = description
challenge.cid = utils.generate_id()
challenge.points = points
challenge.answer_hash = pbkdf2.crypt(answer)
challenge.cat_slug = slug
challenge.unlocked = unlocked
weight = db.session.query(db.func.max(Challenge.weight)).scalar()
challenge.weight = (weight + 1) if weight else 1
challenge.prerequisite = ''
db.session.add(challenge)
return challenge
def transfer_group(operator, user, group):
if not is_owner(operator, group):
msg = "User %s is not an owner of group %s" % (operator.name,
group.name)
raise ActionNotAuthorized(msg)
if group.id == utils.generate_id(operator.id):
raise ActionNotAuthorized(
"Personal group ownership cannot be transferred")
if group.owner_id != user.id:
# make sure new owner has membership
add_user_to_group(operator, user, group)
# transfer owner
group.owner_id = user.id
db_session.commit()
def create(cls, name, description, points, answer, unlocked=False,
validator='static_pbkdf2'):
challenge = cls()
challenge.name = name
challenge.description = description
challenge.cid = utils.generate_id()
challenge.points = points
challenge.answer_hash = answer
challenge.unlocked = unlocked
challenge.validator = validator
weight = db.session.query(db.func.max(Challenge.weight)).scalar()
challenge.weight = (weight + 1) if weight else 1
challenge.prerequisite = ''
db.session.add(challenge)
return challenge
def __init__(self, api_key):
self.uuid = utils.generate_id()
self.connection_id = utils.generate_id()
self.request_id = utils.generate_id()
self.auth_token = utils.obtain_auth_token(api_key)
self.endpoint_interactive = r'wss://westus.stt.speech.microsoft.com/speech/recognition/interactive/cognitiveservices/v1'
self.endpoint_conversation = r'wss://westus.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1'
self.endpoint_dictation = r'wss://westus.stt.speech.microsoft.com/speech/recognition/dictation/cognitiveservices/v1'
self.language = 'en-US'
self.response_format = 'simple'
self.recognition_mode = 'interactive'
self.chunk_size = 8192
self.num_turns = 0
self.is_ongoing_turn = False
self.cur_hypothesis = ''
self.phrase = ''
self.received_messages = []
self.metrics = []
self.ws = None
def load_recipes(self, infile):
""" Load json data for recipes
[
{ "include": "cookies.json" }
{
"title": "Korean Meatballs",
"ingredients": {
"subtitle": "Meatballs",
"1 1/2 lb": "lean ground turkey",
"1 tsp": "ground ginger",
"1/4 tsp": "fresh ground black pepper",
"2 tsp": "Sambal Oelek or Chili Garlic Sauce",
"1 tsp": "mesquite seasoning",
"1/2": "cup Panko",
"1+ tsp": "garlic salt",
"1": "egg",
"3-4": "green onions",
"subtitle":"Spicy Apricot Glaze",
"1/2 cup": "apricot jam",
"2 tsp": "soy sauce",
"2 tsp": "Srirachi"
},
"instructions": [
"preheat oven to 400 degrees, line baking sheet with parchment",
"in a large bowl mix meatball ingredients, form into 1\" balls, cook 20-25 minutes",
"in medium bowl mix glaze ingredients, add meatballs and toss until coated",
"garnish with chopped chives or green ends of onions",
"serve over noodles or rice:
]
},
...
]
Args:
file: json file to load
"""
try:
with open(infile) as json_file:
recipes = json.load(json_file)
for recipe in recipes:
if 'include' in recipe:
self.load_recipes(recipe['include'])
elif 'title' in recipe and 'ingredients' in recipe and 'instructions' in recipe:
recipe_id = generate_id(recipe['title'])
print("Loaded " + recipe['title'])
self.recipes[recipe_id] = recipe
except (IOError, ValueError) as err:
print('Load of recipe file failed:', err.message)
async def create_new_status(request):
body = await request.json()
valid = await validate_status_creation_request(body)
if valid:
# Check that an user with this userma doesn't exists already
ident = generate_id(
int(round(time.time() * 1000)) + random.randint(1, 10000))
await Status.create(caption=body['username'],
password='******',
email='*****@*****.**')
return ValidationError()
def get_next_path(self, from_path=None):
if not self.is_valid():
raise InvalidClient()
if from_path is None:
return (self.my_titles[0], self.path_index[0])
if from_path in self.path_map:
index = self.path_index.index(from_path)
title = self.my_titles[index]
return (title, self.path_map[from_path])
if len(self.path_index) < len(self.my_titles):
new_path = generate_id()
self.path_map[from_path] = new_path
self.path_index.append(new_path)
index = len(self.path_index) - 1
title = self.my_titles[index]
return (title, new_path)
# if we get here we have exhausted all the links
return None
Fun will generate short url and store in the DB
and will return short_url
url: Input URL to Short
"""
if isinstance(url, list):
url = url[0]
_us = None
try:
_us = UrlShortner.objects.get(long_url=url)
except Exception, e:
print(str(e))
pass
if not _us:
id = generate_id()
short_url = "{}{}".format(app.config['DOMAIN'], id_to_string(id))
try:
_us = UrlShortner(id=id, short_url=short_url, long_url=url)
_us.save()
except Exception, e:
raise e
# task to get url title and store back to DB
thread.start_new_thread(get_page_title, (id, url))
print "--------- Existing Handler ----------------------"
return _us.short_url
def get_long_url(short_id, **kwargs):
"""
map short url to long_url with help of DB
def _generate_id(self):
"""Returns randomly generated id"""
return utils.generate_id()
def new_client(self, easy=False):
return RedisActiveClient(generate_id(), self.redis, easy=easy)
def _generate_all_paths(self):
paths = [generate_id() for i in range(len(self.my_titles))]
path_map = dict(zip(paths, self.my_titles))
return (paths, path_map)
def test_uniq_generate_id(self):
id1 = utils.generate_id()
id2 = utils.generate_id()
self.assertNotEqual(id1, id2)
def test_uuid_generate_id(self):
id1 = utils.generate_id()
try:
uid = uuid.UUID(id1) # noqa
except ValueError:
self.fail("Invalid uuid from generate_id")
