class Database(metaclass=Singleton):
def __init__(self):
self.db = MongoClient(DATABASE, username=DB_USER,
password=DB_PWD)[DB_NAME]
self.users = self.db['users']
self.products = self.db['products']
def load_test_collection(self, collection_name: str):
self.db.create_collection(collection_name)
with open('test_collection.js') as test_db:
for i in test_db:
self.db[collection_name].insert_one(loads(i))
def count_users(self):
return self.users.find().count()
def add_new_user(self, message):
self.users.insert_one(
dict(tgid=message.from_user.id,
tgusername=message.from_user.username,
tgfirstname=message.from_user.first_name,
is_auth=False,
time=time(),
basket={}))
@property
def show_categories(self):
return {i[CATEGORY_NAME] for i in self.db.products.find()}
def show_products_in_category(self, category: str):
return self.products.find({CATEGORY_NAME: category})
def find_product_by_str_obj_id(self, object_id: str):
return self.products.find_one({'_id': ObjectId(object_id)})
def main():
db = MongoClient(host=MONGO_HOST, port=int(MONGO_PORT))[MONGO_DATABASE]
if 'medb' not in db.collection_names():
db.create_collection('medb')
db.messages.ensure_index('id')
print(
"OK: Collection 'medb' and indexes were created in '{0}' database".
format(MONGO_DATABASE))
print("OK: Now load medb into '{0}' database".format(MONGO_DATABASE))
# unzip
call(['unzip', '../data/medb.zip'])
f = open('medb.txt', 'r')
for line in f:
# convert string into tuple
t = literal_eval(line)
db.medb.insert({'id': t[0], 'm': t[1], 'e': t[2], 'a': t[3]})
f.close()
remove('medb.txt')
print('OK: MEDB loaded.')
else:
print("ERROR: Collection 'medb' already exists")
def main():
db = MongoClient(host=MONGO_HOST, port=int(MONGO_PORT))[MONGO_DATABASE]
if 'messages' not in db.collection_names():
db.create_collection('messages')
db.messages.ensure_index('h')
# TTL-collection for 31 days
db.messages.ensure_index([('d', DESCENDING)], expireAfterSeconds=60*60*24*31)
db.messages.ensure_index('f')
db.messages.ensure_index('a')
db.messages.ensure_index('p')
db.users.ensure_index('username', unique=True)
db.charts.ensure_index('name', unique=True)
db.cache.ensure_index('type', unique=True)
print("OK: Collections and indexes were created in '{0}' database".format(MONGO_DATABASE))
else:
print("ERROR: Collection 'messages' already exists")
def test_fuzzy_match(example_IRS_service_data,
example_IRS_search_object_with_spelled_out_saint,
mock_config_object):
client = MongoClient(
"mongodb+srv://" + os.environ.get('DBUSERNAME') + ":" +
os.environ.get('PW') +
"@shelter-rm3lc.azure.mongodb.net/shelter?retryWrites=true&w=majority"
)['shelter']
if 'pytest_fuzzy_test' in client.list_collection_names():
client.drop_collection('pytest_fuzzy_test')
client.create_collection('pytest_fuzzy_test')
insert_services(example_IRS_service_data, client, 'pytest_fuzzy_test')
refresh_ngrams(client, 'pytest_fuzzy_test')
name = example_IRS_search_object_with_spelled_out_saint['name']
zip_code = example_IRS_search_object_with_spelled_out_saint['zip']
dc = locate_potential_duplicate(name, zip_code, client,
'pytest_fuzzy_test')
client.drop_collection('pytest_fuzzy_test')
assert dc == 'ST FERIOLE ISLAND PARK'
def main():
db = MongoClient(host=MONGO_HOST, port=int(MONGO_PORT))[MONGO_DATABASE]
if 'messages' not in db.collection_names():
db.create_collection('messages')
db.messages.ensure_index('h')
# TTL-collection for 31 days
db.messages.ensure_index([('d', DESCENDING)],
expireAfterSeconds=60 * 60 * 24 * 31)
db.messages.ensure_index('f')
db.messages.ensure_index('a')
db.messages.ensure_index('p')
db.users.ensure_index('username', unique=True)
db.charts.ensure_index('name', unique=True)
db.cache.ensure_index('type', unique=True)
print("OK: Collections and indexes were created in '{0}' database".
format(MONGO_DATABASE))
else:
print("ERROR: Collection 'messages' already exists")
class BotDatabase:
"""An abstraction over the MongoDB API for doing several operations that require a non-volatile storage.
Currently it's capable of doing these operations:
- Guild specific prefix (getting/setting).
"""
def __init__(self, db_uri):
"""
:param db_uri: MongoDB URI to connect to, it connects to a MongoDB cluster,
it must contain a DB name in the URI.
:raises InvalidURI: if MongoDB URI was malformed.
:raises ConnectionFailure: if cannot connect to a cluster.
:raises ConfigurationError: if no database name was already given.
"""
self._database = MongoClient(db_uri).get_default_database()
# create the collection if they don't exist already.
try:
self._collections = {
'prefixes': self._database.create_collection('prefixes'),
}
except CollectionInvalid:
self._collections = {
'prefixes': self._database.get_collection('prefixes'),
}
def write_guild_prefix(self, guild_id: int, prefix: str):
"""Add/update a guild-specific prefix in the registry, tied to a guild ID.
:param guild_id: the Snowflake of a guild.
:param prefix: prefix for the bot respond to.
"""
self._collections['prefixes'].update_one({'guild_id': guild_id},
{'$set': {
'prefix': prefix
}},
upsert=True)
def read_guild_prefix(self, guild_id: int) -> Optional[str]:
"""Get the guild-specific prefix for guild by its ID.
:param guild_id: the Snowflake of a guild.
:return: prefix for the guild (can be nothing).
"""
return subscript(
self._collections['prefixes'].find_one({'guild_id': guild_id}),
'prefix')
def create_store(request, store_name=None, initial_data=[]):
"""Creates a store with the given store_name and initial_data; Returns a
dictionary in the form of:
{
"id": <store_name>,
"oid": <list_of_initial_data_ids>
}
Keyword arguments:
request -- Django HttpRequest object
store_name -- the name of the store (optional)
initial_data -- Array of data to be stored (optional)
Note: if the store_name is not set, the implementation should create a name
for the store.
"""
# Check that store name is unique (or create unique identifier)
if not store_name:
store_name = random.choice(string.ascii_letters) + str(
uuid.uuid4())[0:8]
while (store_name in get_stores(request)):
store_name = str(uuid.uuid4())[0:8]
elif store_name in get_stores(request):
return json_response(status="ERROR",
status_code=400,
error="Store name already exists.")
# Create new collection
db = MongoClient()['stores']
new_collection = db.create_collection(store_name)
# Load initial data into the store
oid_list = []
for oid, data in enumerate(initial_data):
new_collection.insert({"oid": str(oid), "data": data})
oid_list.append(str(oid))
# Add entry to permissions
perms = db['permissions']
perms.insert({
"name":
store_name,
"owner":
request.user.username,
"perms": [{
"name": request.user.username,
"perms": ['r', 'w'],
}],
})
# Return the name of the store/status/objects created
return {"id": new_collection.name, "oid": oid_list}
def main():
db = MongoClient(host=MONGO_HOST, port=int(MONGO_PORT))[MONGO_DATABASE]
if 'medb' not in db.collection_names():
db.create_collection('medb')
db.messages.ensure_index('id')
print("OK: Collection 'medb' and indexes were created in '{0}' database".format(MONGO_DATABASE))
print("OK: Now load medb into '{0}' database".format(MONGO_DATABASE))
# unzip
call(['unzip', '../data/medb.zip'])
f = open('medb.txt', 'r')
for line in f:
# convert string into tuple
t = literal_eval(line)
db.medb.insert({'id': t[0], 'm': t[1], 'e': t[2], 'a': t[3]})
f.close()
remove('medb.txt')
print('OK: MEDB loaded.')
else:
print("ERROR: Collection 'medb' already exists")
def create_store(request, store_name=None, initial_data=[]):
"""Creates a store with the given store_name and initial_data; Returns a
dictionary in the form of:
{
"id": <store_name>,
"oid": <list_of_initial_data_ids>
}
Keyword arguments:
request -- Django HttpRequest object
store_name -- the name of the store (optional)
initial_data -- Array of data to be stored (optional)
Note: if the store_name is not set, the implementation should create a name
for the store.
"""
# Check that store name is unique (or create unique identifier)
if not store_name:
store_name = random.choice(string.ascii_letters) + str(uuid.uuid4())[0:8]
while(store_name in get_stores(request)):
store_name = str(uuid.uuid4())[0:8]
elif store_name in get_stores(request):
return json_response(status="ERROR", status_code=400, error="Store name already exists.")
# Create new collection
db = MongoClient()['stores']
new_collection = db.create_collection(store_name)
# Load initial data into the store
oid_list = []
for oid, data in enumerate(initial_data):
new_collection.insert({"oid": str(oid), "data": data})
oid_list.append(str(oid))
# Add entry to permissions
perms = db['permissions']
perms.insert({
"name": store_name,
"owner": request.user.username,
"perms": [{
"name": request.user.username,
"perms": ['r', 'w'],
}],
})
# Return the name of the store/status/objects created
return {"id": new_collection.name, "oid": oid_list}
class Server:
def __init__(self,
host,
port,
max_clients,
database_host='localhost',
database_port=27017):
self.connected = 0
self.chars = dict()
self.host = host
self.port = port
self.address = (host, port)
self.listener = self.create_srv_socket(max_clients)
self.mongo_db = MongoClient(database_host, database_port)["mmorpg_db"]
self.characters_db = self.connect_collection()
def create_srv_socket(self, max_clients):
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind(self.address)
listener.listen(max_clients)
print(f'Server started at {self.address}')
return listener
def listening_clients(self):
while True:
sock, address = self.listener.accept()
print(f'Accepted connection from {address}')
start_new_thread(self.handle_conversation, (sock, address))
self.connected += 1
print("Players connected:", self.connected)
def handle_conversation(self, sock, address):
identifier = sock.recv(2048).decode()
state = self.get_character_state(identifier)
self.chars.update({address: identifier})
print(self.chars)
sock.send(str(state).encode())
connected = True
while connected:
try:
while True:
self.handle_request(sock, state)
except EOFError as e:
print(f'Client socket to {address} has closed')
connected = False
except Exception as e:
print(f'Client {address} error: {e}')
connected = False
self.connected -= 1
del self.chars[address]
sock.close()
def handle_request(self, sock, actual_state):
request = receive(sock)
answer = get_answer(request, actual_state)
#updating mongodb
query = {"ID": answer.get("ID")}
new_values = {"$set": answer}
self.characters_db.update_one(query, new_values)
sock.sendall(str.encode(str(answer)))
def connect_collection(self):
if 'characters' in self.mongo_db.list_collection_names():
return self.mongo_db['characters']
else:
self.mongo_db.create_collection('characters')
return self.mongo_db['characters']
def get_character_state(self, identifier):
character = self.characters_db.find_one({"ID": identifier})
if not character:
print("NEW CHARACTER!!!!", identifier)
character = {
'ID': identifier,
'name': identifier,
'position': [0, 0, 0],
'health': 100,
'mana': 100,
'stamina': 100
}
self.characters_db.insert_one(character)
return character
from pymongo import MongoClient
scheme = MongoClient().exams
new_coll = scheme.create_collection('minable_min')
coll = scheme['minable']
v = 'Valutazione Insegnamento'
for doc in coll.find():
n = 0
val = 0
std = 0
p6 = 0
i = 0
for key in list(doc.keys()):
if v in key:
if 'Val [media pesata]' in key:
val = val + doc[key]
if 'Std Dev [' in key:
std = std + doc[key]
if 'Val >= 6 [' in key:
p6 = p6 + doc[key]
if 'N [istanze]' in key:
n = n + doc[key]
i = i + 1
del doc[key]
if i != 0:
doc[v + ' - N [media, istanze]'] = int(round(n / i, 0))
doc[v + ' - Std Dev [media pesata]'] = round(std / i, 2)
from pymongo import MongoClient
from mymodules import discretize
import PREPR_PARAMS as PP
scheme = MongoClient().exams
scheme.create_collection("minable_discretized")
scheme.create_collection("minable_min_discretized")
keys_substrings = [
'Val [media pesata]', 'Std Dev [media pesata]', '[std dev]', 'N [istanze]',
'>= 6', '>= 24', 'Voto [media]', 'Ritardo [semestre, media]', '>=1sem'
]
ranges = [
PP.VAL_SCORE, PP.STD_DEV, PP.STD_DEV, PP.STUDENTS, PP.PERCENT, PP.PERCENT,
PP.MARKS, PP.YEARS, PP.PERCENT
]
discretize.discretize(scheme.minable, scheme.minable_discretized,
keys_substrings, ranges, False)
discretize.discretize(scheme.minable_min, scheme.minable_min_discretized,
keys_substrings, ranges, False)
class Provider:
provider_code = ''
provider_name = ''
provider_url = ''
connect_timeout = 7
read_timeout = 30
@property
def usage_limit_cache_duration(self):
return (12 + randint(-2, 2)) * 3600
@property
def location_cache_duration(self):
return (60 + randint(-2, 2)) * 24 * 3600
def __init__(self):
self.mongo_db = MongoClient(MONGODB_URL).get_database()
self.__stations_collection = self.mongo_db.stations
self.__stations_collection.create_index([('loc', GEOSPHERE),
('status', ASCENDING),
('pv-code', ASCENDING),
('short', ASCENDING),
('name', ASCENDING)])
self.collection_names = self.mongo_db.collection_names()
self.redis = redis.StrictRedis.from_url(url=REDIS_URL,
decode_responses=True)
self.google_api_key = GOOGLE_API_KEY
self.log = get_logger(self.provider_code)
sentry_sdk.init(SENTRY_URL, environment=ENVIRONMENT)
with sentry_sdk.configure_scope() as scope:
scope.set_tag('provider', self.provider_name)
def __to_wind_direction(self, value):
if isinstance(value, ureg.Quantity):
return to_int(value.to(ureg.degree).magnitude, mandatory=True)
else:
return to_int(value, mandatory=True)
def __to_wind_speed(self, value):
if isinstance(value, ureg.Quantity):
return to_float(value.to(ureg.kilometer / ureg.hour).magnitude,
mandatory=True)
else:
return to_float(value, mandatory=True)
def __to_temperature(self, value):
if isinstance(value, ureg.Quantity):
return to_float(value.to(ureg.degC).magnitude)
else:
return to_float(value)
def __to_pressure(self, value):
if isinstance(value, ureg.Quantity):
return to_float(value.to(ureg.hPa).magnitude, ndigits=4)
else:
return to_float(value, ndigits=4)
def __compute_pressures(self, p: Pressure, altitude, temperature,
humidity):
# Normalize pressure to HPa
qfe = self.__to_pressure(p.qfe)
qnh = self.__to_pressure(p.qnh)
qff = self.__to_pressure(p.qff)
if qfe and qnh is None:
qnh = TWxUtils.StationToAltimeter(qfe, elevationM=altitude)
if qnh and qfe is None:
qfe = TWxUtils.AltimeterToStationPressure(qnh, elevationM=altitude)
if qfe and qff is None and temperature is not None and humidity is not None:
qff = TWxUtils.StationToSeaLevelPressure(qfe,
elevationM=altitude,
currentTempC=temperature,
meanTempC=temperature,
humidity=humidity)
if qff and qfe is None and temperature is not None and humidity is not None:
qfe = TWxUtils.SeaLevelToStationPressure(qff,
elevationM=altitude,
currentTempC=temperature,
meanTempC=temperature,
humidity=humidity)
return {
'qfe': to_float(qfe),
'qnh': to_float(qnh),
'qff': to_float(qff)
}
def __to_altitude(self, value):
if isinstance(value, ureg.Quantity):
return to_int(value.to(ureg.meter).magnitude)
else:
return to_int(value)
def __to_rain(self, value):
if isinstance(value, ureg.Quantity):
return to_float(
value.to(ureg.liter / (ureg.meter**2)).magnitude, 1)
else:
return to_float(value, 1)
def stations_collection(self):
return self.__stations_collection
def measures_collection(self, station_id):
if station_id not in self.collection_names:
self.mongo_db.create_collection(
station_id, **{
'capped': True,
'size': 500000,
'max': 5000
})
self.collection_names.append(station_id)
return self.mongo_db[station_id]
def add_redis_key(self, key, values, cache_duration):
pipe = self.redis.pipeline()
pipe.hmset(key, values)
pipe.expire(key, cache_duration)
pipe.execute()
def __compute_elevation(self, lat, lon):
radius = 500
nb = 6
path = f'{lat},{lon}|'
for k in range(nb):
angle = math.pi * 2 * k / nb
dx = radius * math.cos(angle)
dy = radius * math.sin(angle)
path += '{lat},{lon}'.format(
lat=str(lat + (180 / math.pi) * (dy / 6378137)),
lon=str(lon + (180 / math.pi) *
(dx / 6378137) / math.cos(lat * math.pi / 180)))
if k < nb - 1:
path += '|'
result = requests.get(
f'https://maps.googleapis.com/maps/api/elevation/json?locations={path}&key={self.google_api_key}',
timeout=(self.connect_timeout, self.read_timeout)).json()
if result['status'] == 'OVER_QUERY_LIMIT':
raise UsageLimitException('Google Elevation API OVER_QUERY_LIMIT')
elif result['status'] == 'INVALID_REQUEST':
raise ProviderException(
f'Google Elevation API INVALID_REQUEST: {result.get("error_message", "")}'
)
elif result['status'] == 'ZERO_RESULTS':
raise ProviderException('Google Elevation API ZERO_RESULTS')
elevation = float(result['results'][0]['elevation'])
is_peak = False
for point in result['results'][1:]:
try:
glide_ratio = radius / (elevation - float(point['elevation']))
except ZeroDivisionError:
glide_ratio = float('Infinity')
if 0 < glide_ratio < 6:
is_peak = True
break
return elevation, is_peak
def __get_place_geocoding_results(self, results):
lat, lon, address_long_name = None, None, None
for result in results['results']:
if result.get('geometry', {}).get('location'):
lat = result['geometry']['location']['lat']
lon = result['geometry']['location']['lng']
for component in result['address_components']:
if 'postal_code' not in component['types']:
address_long_name = component['long_name']
break
break
return lat, lon, address_long_name
def __get_place_autocomplete(self, name):
results = requests.get(
f'https://maps.googleapis.com/maps/api/place/autocomplete/json?input={name}&key={self.google_api_key}',
timeout=(self.connect_timeout, self.read_timeout)).json()
if results['status'] == 'OVER_QUERY_LIMIT':
raise UsageLimitException('Google Places API OVER_QUERY_LIMIT')
elif results['status'] == 'INVALID_REQUEST':
raise ProviderException(
f'Google Places API INVALID_REQUEST: {results.get("error_message", "")}'
)
elif results['status'] == 'ZERO_RESULTS':
raise ProviderException(
f"Google Places API ZERO_RESULTS for '{name}'")
place_id = results['predictions'][0]['place_id']
results = requests.get(
f'https://maps.googleapis.com/maps/api/geocode/json?place_id={place_id}&key={self.google_api_key}',
timeout=(self.connect_timeout, self.read_timeout)).json()
if results['status'] == 'OVER_QUERY_LIMIT':
raise UsageLimitException('Google Geocoding API OVER_QUERY_LIMIT')
elif results['status'] == 'INVALID_REQUEST':
raise ProviderException(
f'Google Geocoding API INVALID_REQUEST: {results.get("error_message", "")}'
)
elif results['status'] == 'ZERO_RESULTS':
raise ProviderException(
f"Google Geocoding API ZERO_RESULTS for '{name}'")
return self.__get_place_geocoding_results(results)
def __get_place_geocoding(self, name):
results = requests.get(
f'https://maps.googleapis.com/maps/api/geocode/json?address={name}&key={self.google_api_key}',
timeout=(self.connect_timeout, self.read_timeout)).json()
if results['status'] == 'OVER_QUERY_LIMIT':
raise UsageLimitException('Google Geocoding API OVER_QUERY_LIMIT')
elif results['status'] == 'INVALID_REQUEST':
raise ProviderException(
f'Google Geocoding API INVALID_REQUEST: {results.get("error_message", "")}'
)
elif results['status'] == 'ZERO_RESULTS':
raise ProviderException(
f"Google Geocoding API ZERO_RESULTS for '{name}'")
return self.__get_place_geocoding_results(results)
def get_station_id(self, provider_id):
return self.provider_code + '-' + str(provider_id)
def __create_station(self, provider_id, short_name, name, latitude,
longitude, altitude, is_peak, status, tz, urls,
fixes):
if fixes is None:
fixes = {}
if any((not short_name, not name, altitude is None, latitude is None,
longitude is None, not status, not tz)):
raise ProviderException('A mandatory value is none!')
station = {
'pv-id': provider_id,
'pv-code': self.provider_code,
'pv-name': self.provider_name,
'url': urls,
'short': fixes.get('short') or short_name,
'name': fixes.get('name') or name,
'alt':
self.__to_altitude(fixes['alt'] if 'alt' in fixes else altitude),
'peak': to_bool(fixes['peak'] if 'peak' in fixes else is_peak),
'loc': {
'type':
'Point',
'coordinates': [
to_float(
fixes['longitude']
if 'longitude' in fixes else longitude, 6),
to_float(
fixes['latitude'] if 'latitude' in fixes else latitude,
6)
]
},
'status': status,
'tz': tz,
'seen': arrow.utcnow().timestamp
}
return station
def save_station(self,
provider_id,
short_name,
name,
latitude,
longitude,
status,
altitude=None,
tz=None,
url=None,
default_name=None,
lookup_name=None):
if provider_id is None:
raise ProviderException("'provider id' is none!")
station_id = self.get_station_id(provider_id)
lat = to_float(latitude, 6)
lon = to_float(longitude, 6)
address_key = f'address/{lat},{lon}'
if (not short_name or not name) and not self.redis.exists(address_key):
try:
results = requests.get(
f'https://maps.googleapis.com/maps/api/geocode/json?latlng={lat},{lon}'
f'&result_type=airport|colloquial_area|locality|natural_feature|point_of_interest|neighborhood'
f'&key={self.google_api_key}',
timeout=(self.connect_timeout, self.read_timeout)).json()
if results['status'] == 'OVER_QUERY_LIMIT':
raise UsageLimitException(
'Google Geocoding API OVER_QUERY_LIMIT')
elif results['status'] == 'INVALID_REQUEST':
raise ProviderException(
f'Google Geocoding API INVALID_REQUEST: {results.get("error_message", "")}'
)
elif results['status'] == 'ZERO_RESULTS':
raise ProviderException(
'Google Geocoding API ZERO_RESULTS')
address_short_name = None
address_long_name = None
for result in results['results']:
for component in result['address_components']:
if 'postal_code' not in component['types']:
address_short_name = component['short_name']
address_long_name = component['long_name']
break
if not address_short_name or not address_long_name:
raise ProviderException(
'Google Geocoding API: No valid address name found')
self.add_redis_key(address_key, {
'short': address_short_name,
'name': address_long_name
}, self.location_cache_duration)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(address_key, {'error': repr(e)},
self.usage_limit_cache_duration)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception('Unable to call Google Geocoding API')
self.add_redis_key(address_key, {'error': repr(e)},
self.location_cache_duration)
address = lookup_name or name or short_name
geolocation_key = f'geolocation/{address}'
if (lat is None or lon is None) or (lat == 0 and lon == 0):
if not self.redis.exists(geolocation_key):
try:
lat, lon, address_long_name = self.__get_place_geocoding(
address)
if not lat or not lon or not address_long_name:
raise ProviderException(
f'Google Geocoding API: No valid geolocation found {address}'
)
self.add_redis_key(geolocation_key, {
'lat': lat,
'lon': lon,
'name': address_long_name
}, self.location_cache_duration)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(geolocation_key, {'error': repr(e)},
self.usage_limit_cache_duration)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception(
'Unable to call Google Geocoding API')
self.add_redis_key(geolocation_key, {'error': repr(e)},
self.location_cache_duration)
if self.redis.exists(geolocation_key):
if self.redis.hexists(geolocation_key, 'error'):
raise ProviderException(
f'Unable to determine station geolocation: {self.redis.hget(geolocation_key, "error")}'
)
lat = to_float(self.redis.hget(geolocation_key, 'lat'), 6)
lon = to_float(self.redis.hget(geolocation_key, 'lon'), 6)
if not name:
name = self.redis.hget(geolocation_key, 'name')
alt_key = f'alt/{lat},{lon}'
if not self.redis.exists(alt_key):
try:
elevation, is_peak = self.__compute_elevation(lat, lon)
self.add_redis_key(alt_key, {
'alt': elevation,
'is_peak': is_peak
}, self.location_cache_duration)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(alt_key, {'error': repr(e)},
self.usage_limit_cache_duration)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception('Unable to call Google Elevation API')
self.add_redis_key(alt_key, {'error': repr(e)},
self.location_cache_duration)
tz_key = f'tz/{lat},{lon}'
if not tz and not self.redis.exists(tz_key):
try:
now = arrow.utcnow().timestamp
result = requests.get(
f'https://maps.googleapis.com/maps/api/timezone/json?location={lat},{lon}'
f'×tamp={now}&key={self.google_api_key}',
timeout=(self.connect_timeout, self.read_timeout)).json()
if result['status'] == 'OVER_QUERY_LIMIT':
raise UsageLimitException(
'Google Time Zone API OVER_QUERY_LIMIT')
elif result['status'] == 'INVALID_REQUEST':
raise ProviderException(
f'Google Time Zone API INVALID_REQUEST: {result.get("error_message", "")}'
)
elif result['status'] == 'ZERO_RESULTS':
raise ProviderException(
'Google Time Zone API ZERO_RESULTS')
tz = result['timeZoneId']
dateutil.tz.gettz(tz)
self.add_redis_key(tz_key, {'tz': tz},
self.location_cache_duration)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(tz_key, {'error': repr(e)},
self.usage_limit_cache_duration)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception('Unable to call Google Time Zone API')
self.add_redis_key(tz_key, {'error': repr(e)},
self.location_cache_duration)
if not short_name:
if self.redis.hexists(address_key, 'error'):
if default_name:
short_name = default_name
else:
raise ProviderException(
f"Unable to determine station 'short': {self.redis.hget(address_key, 'error')}"
)
else:
short_name = self.redis.hget(address_key, 'short')
if not name:
if self.redis.hexists(address_key, 'error'):
if default_name:
name = default_name
else:
raise ProviderException(
f"Unable to determine station 'name': {self.redis.hget(address_key, 'error')}"
)
else:
name = self.redis.hget(address_key, 'name')
if not altitude:
if self.redis.hexists(alt_key, 'error'):
raise ProviderException(
f"Unable to determine station 'alt': {self.redis.hget(alt_key, 'error')}"
)
altitude = self.redis.hget(alt_key, 'alt')
if self.redis.hexists(alt_key, 'error') == 'error':
raise ProviderException(
f"Unable to determine station 'peak': {self.redis.hget(alt_key, 'error')}"
)
is_peak = self.redis.hget(alt_key, 'is_peak')
if not tz:
if self.redis.hexists(tz_key, 'error'):
raise ProviderException(
f"Unable to determine station 'tz': {self.redis.hget(tz_key, 'error')}"
)
tz = self.redis.hget(tz_key, 'tz')
if not url:
urls = {'default': self.provider_url}
elif isinstance(url, str):
urls = {'default': url}
elif isinstance(url, dict):
if 'default' not in url:
raise ProviderException("No 'default' key in url")
urls = url
else:
raise ProviderException('Invalid url')
fixes = self.mongo_db.stations_fix.find_one(station_id)
station = self.__create_station(provider_id, short_name, name, lat,
lon, altitude, is_peak, status, tz,
urls, fixes)
self.stations_collection().update({'_id': station_id},
{'$set': station},
upsert=True)
station['_id'] = station_id
return station
def create_measure(self,
for_station,
_id,
wind_direction,
wind_average,
wind_maximum,
temperature=None,
humidity=None,
pressure: Pressure = None,
rain=None):
if all((wind_direction is None, wind_average is None,
wind_maximum is None)):
raise ProviderException('All mandatory values are null!')
# Mandatory keys: json 'null' if not present
measure = {
'_id': int(round(_id)),
'w-dir': self.__to_wind_direction(wind_direction),
'w-avg': self.__to_wind_speed(wind_average),
'w-max': self.__to_wind_speed(wind_maximum)
}
# Optional keys
if temperature is not None:
measure['temp'] = self.__to_temperature(temperature)
if humidity is not None:
measure['hum'] = to_float(humidity, 1)
if pressure is not None and (pressure.qfe is not None or pressure.qnh
is not None or pressure.qff is not None):
measure['pres'] = self.__compute_pressures(
pressure, for_station['alt'], measure.get('temp', None),
measure.get('hum', None))
if rain is not None:
measure['rain'] = self.__to_rain(rain)
measure['time'] = arrow.now().timestamp
fixes = self.mongo_db.stations_fix.find_one(for_station['_id'])
if fixes and 'measures' in fixes:
for key, offset in fixes['measures'].items():
try:
if key in measure:
fixed_value = measure[key] + offset
if key == 'w-dir':
fixed_value = fixed_value % 360
measure[key] = fixed_value
except Exception:
self.log.exception(
f"Unable to fix '{key}' with offset '{offset}'".format(
key=key, offset=offset))
return measure
def has_measure(self, measure_collection, key):
return measure_collection.find({'_id': key}).count() > 0
def insert_new_measures(self, measure_collection, station, new_measures):
if len(new_measures) > 0:
measure_collection.insert(
sorted(new_measures, key=lambda m: m['_id']))
end_date = arrow.Arrow.fromtimestamp(
new_measures[-1]['_id'], dateutil.tz.gettz(station['tz']))
self.log.info(
'--> {end_date} ({end_date_local}), {short}/{name} ({id}): {nb} values inserted'
.format(end_date=end_date.format('YY-MM-DD HH:mm:ssZZ'),
end_date_local=end_date.to('local').format(
'YY-MM-DD HH:mm:ssZZ'),
short=station['short'],
name=station['name'],
id=station['_id'],
nb=str(len(new_measures))))
self.__add_last_measure(measure_collection, station['_id'])
def __add_last_measure(self, measure_collection, station_id):
last_measure = measure_collection.find_one({
'$query': {},
'$orderby': {
'_id': -1
}
})
if last_measure:
self.stations_collection().update({'_id': station_id},
{'$set': {
'last': last_measure
}})
class App:
def __init__(self, settings):
dirname = os.path.dirname(os.path.realpath(__file__)) + '/'
self.logger = logging.getLogger("bot")
self.help_text = open(dirname + 'docs/help.txt').read()
self.changelog_text = open(dirname + 'docs/changelog.txt').read()
self.welcome_text = open(dirname + 'docs/welcome.txt').read()
self.about_text = open(dirname + 'docs/about.txt').read()
self.settings = settings
self.wnl = WordNetLemmatizer()
remainder.configure(settings)
logs_dir = 'logs/'
if not os.path.exists(logs_dir):
os.makedirs(logs_dir)
###LOGGING
fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
access = logging.FileHandler(logs_dir + 'access.log')
access.setLevel(logging.INFO)
access.setFormatter(logging.Formatter(fmt))
error = logging.FileHandler(logs_dir + 'error.log')
error.setLevel(logging.ERROR)
error.setFormatter(logging.Formatter(fmt))
self.logger.addHandler(access)
self.logger.addHandler(error)
logging.basicConfig(format=fmt)
if env == 'debug':
logging.basicConfig(level=logging.DEBUG)
logging.warning("Cofiguration: %s" % (env,))
self.db = MongoClient(settings.mongo['uri']).get_default_database()
if 'users' not in self.db.collection_names():
self.db.create_collection('users')
self.users = self.db.users
if 'remainders' not in self.db.collection_names():
self.db.create_collection('remainders')
remainder.recover_jobs()
self.params['offset'] = 0
logging.warning('Constructed')
def listen(self):
logging.warning('Listening')
while True:
self.get_updates()
time.sleep(0.1)
# app.run()
def correct(self, string):
baseurl_correction = 'http://service.afterthedeadline.com/checkDocument'
correction = requests.get(baseurl_correction, {'data': string}).text
correction = BeautifulSoup(correction, "lxml")
if correction.find("option") is not None:
string = correction.find("option").string
return string
def add_word(self, user, string):
baseurl = 'https://translate.yandex.net/api/v1.5/tr.json/translate'
# string = re.sub(r'[^A-Za-z\s]', '', string)
# string = re.sub(r'\Wk+', ' ', string)
string = string.lower()
if len(string) == 0:
telegram.send_message(user['chat_id'], "Wrong word")
return
if 'foreign' not in user:
user['foreign'] = 'en'
user['native'] = 'ru'
if user['foreign'] == 'en':
string = self.correct(string)
if env != 'debug':
string = self.wnl.lemmatize(string)
string = string[0].upper() + string[1:]
direction = '%s-%s' % (user['foreign'], user['native'])
transtaltion = requests.get(baseurl, {
'key': self.settings.translate_yandex['token'],
'lang': direction,
'text': string
})
out_word = transtaltion.json()['text'][0]
already_has = False
for w in user['words']:
already_has |= w["en"] == string
if not already_has:
user['words'].append({"en": string, "ru": out_word,
"stage": study_settings.min_stage,
"expiration_date": datetime.datetime.utcnow() + study_settings.stages[1],
"creation_date": datetime.datetime.utcnow()})
self.users.save(user)
telegram.send_message(user['chat_id'], "Word added\n%s - %s" % (string, out_word))
else:
telegram.send_message(user['chat_id'], "Already exist!\n%s - %s" % (string, out_word))
params = {}
def get_list_word(self, user, text):
str_out = "\n".join(["%s: (%s) %s - %s" % (i + 1, w['stage'], w['en'], w['ru']) for i, w in
zip(range(10 ** 10), user['words'])])
telegram.send_message(user['chat_id'], str_out)
def start(self, user, text):
telegram.send_message(user['chat_id'], self.welcome_text)
user['state'] = States.langs_asked
def reask_langs(self, user, text):
telegram.send_message(user['chat_id'], """
Now choose your native and foreign languages.
Example: "en-ru" (en is foreign and ru is native)
""")
user['state'] = States.langs_asked
def langs_ask(self, user, text):
ans = requests.get('https://translate.yandex.net/api/v1.5/tr.json/getLangs',
{'key': self.settings.translate_yandex['token']})
lang_list = ans.json()['dirs']
if text not in lang_list:
telegram.send_message(user['chat_id'], "Please, choose any of this:\n" + "\n".join(lang_list))
else:
telegram.send_message(user['chat_id'], "\"%s\" have successfully chosen" % (text,))
user['state'] = States.idle
user['foreign'] = text[0:2]
user['native'] = text[3:5]
def help(self, user, text):
telegram.send_message(user['chat_id'], self.help_text)
def about(self, user, text):
telegram.send_message(user['chat_id'], self.about_text)
def start_train(self, user, text):
user['train']['type'] = 0
train.do_train(user, text)
def add_remainder(self, user, text):
remainder.remove_job(user)
tokens = text.split(' ')
delta = datetime.timedelta()
if len(tokens) >= 2:
tokens = tokens[1].replace(' ', '').split(':')
hours = int(tokens[0])
minutes = int(tokens[1])
delta = datetime.timedelta(hours=hours, minutes=minutes)
remainder.add_job(user, datetime.datetime.utcnow() + delta)
telegram.send_message(user['chat_id'], "Successfully set. Nearest at %s" % (datetime.datetime.now() + delta,))
def remove_remainder(self, user, text):
remainder.remove_job(user)
telegram.send_message(user['chat_id'], "Removed")
def remove(self, user, text):
if user['train']['type'] != 0:
for w in user['words']:
if w == user['train']['word']:
user['words'].remove(w)
str_out = "%s - %s" % (w['en'], w['ru'])
telegram.send_message(user['chat_id'], "Deleted:\n%s" % (str_out,))
train.do_train(user, text)
else:
tokens = text.split(" ")
if len(tokens) > 1:
cnt = int(tokens[1])
if cnt > 0:
cnt -= 1
else:
cnt = -1
str_out = "%s - %s" % (user['words'][cnt]['en'], user['words'][cnt]['ru'])
del user['words'][cnt]
telegram.send_message(user['chat_id'], "Word with index %s removed\n%s" % (cnt, str_out))
comands = {
'list': get_list_word,
'rm': remove,
'train': start_train,
'end': train.end_train,
'start': start,
'help': help,
'setremainder': add_remainder,
'reask': reask_langs,
'about': about
}
def parse_action(self, chat_id, text):
self.logger.warning("%s - %s" % (chat_id, text))
user = self.users.find_one({'chat_id': chat_id})
if user is None:
user = {'chat_id': chat_id,
'state': States.idle,
'words': [],
'train': {
'type': 0,
'words': 0,
'correct': 0,
'cadidacies': []
}}
if 'train' not in user:
user['train'] = {
'type': 0,
'words': 0,
'correct': 0,
'cadidacies': []
}
if text[0] == '/': # Command
cmd = text[1:].lower().split(' ')[0]
if cmd in self.comands:
self.comands[cmd](self, user, text)
elif user['train']['type'] != 0:
train.do_train(user, text)
elif user['state'] == States.idle:
self.add_word(user, text)
elif user['state'] == States.langs_asked:
self.langs_ask(user, text)
self.users.save(user)
def get_updates(self):
try:
messages = telegram.get_updates(self.params['offset'])
for u in messages:
if 'message' in u and 'text' in u['message']:
if u['update_id'] < self.params['offset']:
print('Error')
else:
chat_id = u['message']['chat']['id']
text = u['message']['text']
self.params['offset'] = max(self.params['offset'], u['update_id'] + 1)
try:
self.parse_action(chat_id, text)
except:
logging.error('Error! (%s, %s)' % (chat_id, text))
logging.error(traceback.print_exc())
telegram.send_message(chat_id, 'An error occurred!')
except:
logging.error('Get updates error!')
logging.error(traceback.print_exc())
self.db.meta.save(self.params)
from pymongo import MongoClient
from mymodules import aggregs
scheme = MongoClient().exams
teval = scheme["teachEval"]
teval_aggr = scheme.create_collection('teachEval_aggr')
aggr = aggregs.ParAggregator(teval, teval_aggr)
aggr.aggregate_par()
aggr.drop()
"""Clean freshly imported teaching evaluation collections."""
from pymongo import MongoClient
from mymodules import cleanings
scheme = MongoClient().exams
teval = scheme.create_collection("teachEval")
cln = cleanings.Cleaner(teval)
cln.set_qset(cleanings.QSET_OLD)
cln.clean(scheme.rawTeachingsEv1011, 2010, True)
cln.set_qset(cleanings.QSET_GEN)
cln.clean(scheme.rawTeachingsEv1112, 2011, True)
cln.clean(scheme.rawTeachingsEv1213, 2012, True)
cln.clean(scheme.rawTeachingsEv1314, 2013, True)
cln.clean(scheme.rawTeachingsEv1415, 2014, True)
cln.clean(scheme.rawTeachingsEv1516, 2015, True)
cln.clean(scheme.rawTeachingsEv1617, 2016, True)
cln.drop()
class DBHandler:
"""
DataBase Handler Class
"""
def __init__(self, dbName, host, port):
"""
:Parameters:
- `dbName`: Database's name (Str)
"""
#Start MongoDB Client (Default Host)
self.__mongoDB = MongoClient(host, port).local
# Get the dbName Collection
if dbName in self.__mongoDB.collection_names():
self.__collection = self.__mongoDB[dbName]
else:
self.__mongoDB.create_collection(dbName)
self.__collection = self.__mongoDB[dbName]
def RemoveDuplicatesBy(self, keyName):
"""
Remove duplicated documents from a Collection
"""
pipeline = [
{"$group": {
"_id": {
keyName:"${0}".format(keyName)
},
"uniqueIds": {
"$addToSet": "$_id"
},
"count": {
"$sum": 1
}}
}]
cursor = self.__collection.aggregate(pipeline)
duplicatedIds = []
for doc in cursor:
# Keep one document
del doc["uniqueIds"][0]
# Get the duplicated document's ID
for duplicatedID in doc["uniqueIds"]:
duplicatedIds.append(duplicatedID)
# Delete all duplicated IDs
self.__collection.remove({"_id": {"$in": duplicatedIds}})
def Insert(self, data):
"""
Insert a document to the Collection
:Parameters:
- `data`: Document to be inserted (Dict | Dict List)
"""
if not isinstance(data, list):
#Item Insert
self.__collection.insert_one(data)
else:
#Bulk Insert
self.__collection.insert_many(data)
def GetDocuments(self):
"""
Get all documents of the Collection
"""
dataList = []
cursor = self.__collection.find({})
for document in cursor:
dataList.append(document)
return dataList
def Empty(self):
"""
Remove all documents of the Collection
"""
self.__collection.drop()
from pymongo import MongoClient
from mymodules import merge
scheme = MongoClient().exams
teval = scheme['teachEval_aggr']
sprod = scheme['sprod']
dest = scheme.create_collection("minable")
mrg = merge.Merger(['Anno Accademico', 'Insegnamento'], False)
mrg.set_specific_keys([
'N [istanze]', 'Voto >= 24 [perc]', 'Voto [media]', 'Voto [std dev]',
'Ritardo [semestre, media]', 'Ritardo >=1sem [percent]'
], None, None)
mrg.merge_collections(teval, sprod, 'Produttivita Studenti', dest)
from pymongo import MongoClient
from mymodules import aggregs
scheme = MongoClient().exams
sprod = scheme.create_collection("sprod")
agg = aggregs.StudAggregator(scheme.rawStudentsPr1013, sprod)
agg.aggregate_stud('2010')
agg.aggregate_stud('2011')
agg.aggregate_stud('2012')
agg.aggregate_stud('2013')
from pymongo import MongoClient
scheme = MongoClient().exams
stud_gen = scheme.create_collection('stud_gen')
prod = scheme.rawStudentsPr1013
stud = scheme.sprod
# aggr doc keys
val = 'Valutazione Test Ingresso [media]'
pl = 'Studenti Laureati [percent]'
ni = 'N [istanze]'
mark = 'Voto [media pesata]'
delay = 'Ritardo [semestre, media pesata]'
for group in prod.aggregate([{"$group": {"_id": {'coorte': '$coorte'}}}]):
aggr = {
val: 0,
pl: 0,
ni: 0,
'Coorte Immatricolazione': group['_id']['coorte']
}
for doc in prod.find(group['_id']):
aggr[ni] = aggr[ni] + 1
aggr[val] = aggr[val] + doc['test']
if doc['crediti_totali'] == 180:
aggr[pl] = aggr[pl] + 1
aggr[val] = round(aggr[val] / aggr[ni], 2)
from pymongo import MongoClient
scheme = MongoClient().exams
eval_gen = scheme.create_collection('eval_gen')
teval = scheme['teachEval_aggr']
val = 'Valutazione Complessiva [media pesata]'
sdv = 'Deviazione Standard Complessiva [media pesata]'
prc = 'Percentuale Valutazioni Sufficienti [media pesata]'
n = 'Numero Valutazioni [media]'
for group in teval.aggregate([{
"$group": {
"_id": {
'Anno Accademico': '$Anno Accademico'
}
}
}]):
aggr = {val: 0, sdv: 0, prc: 0, n: 0}
i = 0
for doc in teval.find(group['_id']):
if doc['Val [media pesata]'] != 'n.c.':
aggr[val] = aggr[
val] + doc['Val [media pesata]'] * doc['N [istanze]']
aggr[sdv] = aggr[
sdv] + doc['Std Dev [media pesata]'] * doc['N [istanze]']
aggr[prc] = aggr[
prc] + doc['Val >= 6 [percent]'] * doc['N [istanze]']
aggr[n] = aggr[n] + doc['N [istanze]']
class DB:
"""Represents connection to DB and methods to deal with it.
Properties:
cli (MongoClient): Database connection, provided by 'pymongo' library.
dbname (str): Name of database all methods will connect with.
XPOSTRAIN = 0 (const int): Marker for tables with xpos training data.
XPOSTRAIN_KEEP = 1 (const int): Marker for tables with xpos training
data which makes accuracy worse, but it's better to keep for some
time and maybe makes it better in future.
EMENDPOS = 10 (const int): Marker for tables with emendpos training
data.
EMENDPOS_KEEP = 11 (const int): Like XPOSTRAIN_KEEP marker.
SYNTEXTRACT = 20 (const int): Marker for tables with syntextract
training data.
SYNTEXTRACT_KEEP = 21 (const int): Like XPOSTRAIN_KEEP marker.
"""
# These constants will be used in order to mark collections
XPOSTRAIN = 0
XPOSTRAIN_KEEP = 1
EMENDPOS = 10
EMENDPOS_KEEP = 11
SYNTEXTRACT = 20
SYNTEXTRACT_KEEP = 21
TEMPORARY = 90
def __init__(self, host='atlas', dbname='syntextua'):
"""Return MongoClient() connected to 'localhost' or 'atlas'. You must
provide %SYNTEXTDBLOG%, %SYNTEXTDBPWD% and %SYNTEXTDBHOST% environment
variables to make it work.
Args:
host (str): If 'localhost' then connect to your local database; if
'atlas' then connect to MongoDB Atlas.
dbname (str): Name of database to use.
Returns:
MongoClient: Connection to database provided by pymongo lib.
Raises:
EnvironmentError: Some of environment variables, which contains
credentials, are missing, so that connection is impossible.
RuntimeError: No database with the given name exists.
...Errors from pymongo.errors
"""
if host == 'atlas':
login = os.getenv('SYNTEXTDBLOG')
password = os.getenv('SYNTEXTDBPWD')
hosturl = os.getenv('SYNTEXTDBHOST')
if not login or not password or not hosturl:
raise EnvironmentError(
'No credentials found. Make sure that SYNTEXTDBLOG, '
'SYNTEXTDBPWD and SYNTEXTDBHOST environment variables ',
'is provided.'
)
url = 'mongodb+srv://%s:%s@%s' % (
login,
password,
hosturl,
)
self.cli = MongoClient(url)
else:
self.cli = MongoClient()
if dbname not in self.cli.list_database_names():
raise RuntimeError(
f"No database with name '{dbname}' exists on host you're "
f"trying to connect. "
)
self.cli = self.cli.get_database(dbname)
self.dbname = dbname
def createCollection(self, marker, description=""):
"""Create new collection in DB and generate a unique name for it, using
tamestamp and marker you gave. This will also write info about your
collection to 'datainfo' collection.
Args:
marker (int): Marker of the type of the collection you want to
create.
description (str): Add some description to created collection.
Returns:
Collection: Created collection. An instance of pymongo.
"""
def unique():
""" Generates next unique name
"""
return f'temp_{marker}_{time.time()}'
# Trying to find unique name for collection. Request all existing names
names = self.cli.collection_names()
# Create new name until it'll be unique
name = None
while True:
name = unique()
if name not in names:
break
self.cli.get_collection("datainfo").insert_one({
"name": name,
"created": time.time(),
"description": description
})
coll = self.cli.get_collection(name)
# Paste empty document in order to create collection
coll.insert_one({})
return coll
def deleteTemporaries(self):
"""Delete all databases with TEMPORARY marker.
Returns:
list: Name of all deleted collections.
"""
temps = list(filter(
lambda name: name[0:7] == f"temp_{self.TEMPORARY}",
self.cli.collection_names()
))
for name in temps:
self.drop(name)
return temps
def getTemps(self):
"""Get names of all temporary collections.
Return:
list: List of found collections.
"""
# Just filter collection names by first 5 symbols
return list(filter(
lambda name: name[0:5] == 'temp_',
self.cli.collection_names()
))
def drop(self, name):
"""Drop collection with the given name and delete it from "datainfo"
collection.
Args:
name (str): Name of collection you're want to delete.
"""
self.cli.get_collection(name).drop()
self.cli.get_collection("datainfo").find_one_and_delete({
"name": name
})
def substitute(self, what, by):
"""Move one collection into place of another.
1. 'by'.name = 'what'.name;
2. Delete 'what';
Args:
what (str): Name of collection that should be deleted.
by (str): Name of collection that will be renamed.
"""
what = self.cli.get_collection(what)
name = what.name
what.drop()
self.cli.get_collection(by).rename(name)
def compareCollections(
self, first, second, field, compareFunc, findUnique2=True
):
"""Call compareFunc on each pair "document from first collection"-
"document from second collection", which field has the same value, and
add the result of execution to a list.
Returns that list, unique elements from first and second collections.
! You can also use a static function mergeDocuments(dict document1,
dict document2) as compareFunc from this class in order to merge every
property of two given documents.
Scheme of calling compareFunc():
collection3.append(
compareFunc(dict document1, dict document2)
)
Args:
first (str): Name of first collection.
second (str): Name of second collection.
field (str): Name of the field to be compared.
compareFunc (function): Comparing function that will be executed.
findUnique2(bool): Set this to False in order to prevent searching
unique documents from second collection.
Returns:
dict: Result of comparing.
Example of return:
{
"third": [
...Results of comparing function.
],
"unique1": [
...Unique elements from first collection.
],
"unique2": [ (if findUnique2 == True)
...Unique elements from second collection
]
}
"""
# Get collections instead of names
first = self.cli.get_collection(first)
second = self.cli.get_collection(second)
cursor = first.find()
# The first document in collection is always empty (see, why in
# createCollection() function).
cursor.skip(1)
# ID of documents in second collection which is equal to documents in
# first collection (regulate by comparing function) will be written to
# list in order to find unique documents in second collection in the
# future (if findUnique2 == True).
commonIDs = []
collection3 = []
unique1 = []
# Iterate over cursor
for document in cursor:
match = second.find_one({
field: document[field]
})
# If match was not found, then it's a unique element in first
# collection.
if not match:
unique1.append(document)
# Else append to common.
else:
collection3.append(
compareFunc(document, match)
)
if findUnique2:
commonIDs.append(str(match["_id"]))
if not findUnique2:
return {
"third": collection3,
"unique1": unique1
}
# This code won't be executed when findUnique2 == False.
unique2 = []
# Do the same iterating for documents in second collection
cursor = second.find()
cursor.skip(1)
for document in cursor:
# Skip non-unique elements
if str(document["_id"]) in commonIDs:
continue
unique2.append(document)
return {
"third": collection3,
"unique1": unique1,
"unique2": unique2
}
def mergeCollections(
self, first, second, uniqueField, compareFunc, newName=None,
saveUnique1=True, saveUnique2=True, deleteThem=True
):
"""Merge two collections into new one. You need to provide a name of
field which will be used as key field and compare function as in
compareCollections() (see for docs there).
! Note, that inserting elements must not contain field "_id", as it's
reserved by pymongo for unique key.
! You can also use a static function mergeDocuments(dict document1,
dict document2) as compareFunc from this class in order to merge every
property of two given documents.
Args:
first (str): Name of first collection.
second (str): Name of second collection.
uniqueField (str): Name of field by which two equal documents
(candidates for merging) will be found.
newName (str): Name of new merged collection. If this parameter is
unspecified, new name will be generated:
merged_{time}_{name1}_{name2}
saveUnique1 (bool): Set to False in order to delete all unique
documents from first collection; set to True to save them.
saveUnique2 (bool): Just like 'saveUnique1', but for the second
collection.
deleteThem (bool): Delete two collections after merging. Is True
by default.
Returns:
Collection: New merged collection.
"""
comparing = self.compareCollections(
first, second, uniqueField, compareFunc, findUnique2=True
)
if not newName:
newName = f"merged_{time.time()}:{first},{second}"
merged = self.cli.create_collection(newName)
merged.insert_many(comparing["third"])
if saveUnique1:
merged.insert_many(comparing["unique1"])
if saveUnique2:
merged.insert_many(comparing["unique2"])
if deleteThem:
self.get_collection(first).drop()
self.get_collection(second).drop()
return merged
@staticmethod
def mergeDocuments(document1, document2):
"""Merge properties of two documents. Can be used as compareFunc in
mergeCollections and compareCollections.
Unique keys will be merged without any changes, but common keys will
be merged due to its type:
int Find average
float Find average
complex Find average
bool A and B
set Unite them
dict Merge keys
str Concatenate without glue
list Concatenate, delete duplicates
Args:
document1 (dict): First document.
document2 (dict): Second document.
Returns:
dict: Merged document.
"""
keys1 = list(document1.keys())
keys2 = list(document2.keys())
merged = dict()
for key in keys1:
# Inserting document must not contain "_id" field
if key == "_id":
continue
# It's the unique key from first document
if key not in keys2:
merged[key] = document1[key]
# It's the common key
else:
# Assume that type of document1[key] == document2[key]
if type(document1[key]) in [int, float, complex]:
merged[key] = (document1[key] + document2[key]) / 2
elif type(document1[key]) is bool:
merged[key] = document1[key] and document2[key]
elif type(document1[key]) is str:
merged[key] = document1[key] + document2[key]
elif type(document1[key]) is dict:
merged[key] = {**document1[key], **document2[key]}
elif type(document1[key]) is set:
merged[key] = document1[key] | document2[key]
elif type(document1[key]) is list:
merged[key] = document1[key] + document2[key]
# Unduplicate list
merged[key] = list(set(merged[key]))
keys2.remove(key)
# Unique keys from second document
for key in keys2:
if key == "_id":
continue
merged[key] = document2[key]
return merged
def close(self):
"""Cleanup client resources and disconnect from MongoDB.
"""
self.cli.client.close()
from re import sub, findall, split
from telethon.tl.functions.messages import GetMessagesRequest, EditMessageRequest
from telethon.tl.functions.channels import GetMessagesRequest as GetMessagesRequestChannel
from pymongo import MongoClient
from telethon.sync import TelegramClient, events
from telethon.helpers import add_surrogate
from os import path
from telethon.tl.types import Message
client = TelegramClient(session, api_id, api_hash, lang_code="id")
client.plugins = list()
pluginFolder = "Plugins"
db = MongoClient(dbUri)[dbName]
#initializing collection.
if "users" not in db.list_collection_names():
db.create_collection("users")
print("Collection 'users' created")
if "echos" not in db.list_collection_names():
db.create_collection("echos")
print("Collection 'echos' created")
if "afks" not in db.list_collection_names():
db.create_collection("afks")
print("Collection 'afks' created")
usersDb = db.users
echosDb = db.echos
afks = db.afks
client.afk = afks
client.db = db
client.users = usersDb
# Price (last_price), is_high_low
# UID,WID,pair,price,hl (notifsCOllection)
# UID: SocketID (UIDmap)
from pymongo import MongoClient
from bson.objectid import ObjectId
from bson.json_util import dumps
url = "mongodb://localhost:27017"
client = MongoClient(url)['notifs']
collection_list = ['notifsCollection', 'UIDMap']
for i in collection_list:
if i not in client.collection_names():
client.create_collection(i)
notifsCollection = client['notifsCollection']
UIDMap = client['UIDMap']
def set_Notification(UID, WID, pair, price, hl, msg):
notif = {'UID':UID,'WID':WID,'pair':pair,'price':price,'hl':hl,'fulfilled':False,\
'message':msg,'price_then':0.0,'not_reached':False}
_id = notifsCollection.insert_one(notif).inserted_id
return _id
def get_Notification(_all=False, _id="", WID="", Pair="", Price=0.0):
if _all:
class Provider:
provider_code = ""
provider_name = ""
provider_url = ""
connect_timeout = 7
read_timeout = 30
@property
def usage_limit_cache_duration(self):
return (12 + randint(-2, 2)) * 3600
@property
def google_api_error_cache_duration(self):
return (60 + randint(-2, 2)) * 24 * 3600
@property
def google_api_cache_duration(self):
return (120 + randint(-2, 2)) * 24 * 3600
def __init__(self):
self.mongo_db = MongoClient(MONGODB_URL).get_database()
self.__stations_collection = self.mongo_db.stations
self.__stations_collection.create_index([
("loc", GEOSPHERE),
("status", ASCENDING),
("pv-code", ASCENDING),
("short", ASCENDING),
("name", ASCENDING),
])
self.collection_names = self.mongo_db.collection_names()
self.redis = redis.StrictRedis.from_url(url=REDIS_URL,
decode_responses=True)
self.google_api_key = GOOGLE_API_KEY
self.log = logging.getLogger(self.provider_code)
sentry_sdk.set_tag("provider", self.provider_name)
def stations_collection(self):
return self.__stations_collection
def measures_collection(self, station_id):
if station_id not in self.collection_names:
self.mongo_db.create_collection(
station_id, **{
"capped": True,
"size": 500000,
"max": 5000
})
self.collection_names.append(station_id)
return self.mongo_db[station_id]
def __to_wind_direction(self, value):
if isinstance(value, ureg.Quantity):
return to_int(value.to(ureg.degree).magnitude, mandatory=True)
else:
return to_int(value, mandatory=True)
def __to_wind_speed(self, value):
if isinstance(value, ureg.Quantity):
return to_float(value.to(ureg.kilometer / ureg.hour).magnitude,
mandatory=True)
else:
return to_float(value, mandatory=True)
def __to_temperature(self, value):
if isinstance(value, ureg.Quantity):
return to_float(value.to(ureg.degC).magnitude)
else:
return to_float(value)
def __to_pressure(self, value):
if isinstance(value, ureg.Quantity):
return to_float(value.to(ureg.hPa).magnitude, ndigits=4)
else:
return to_float(value, ndigits=4)
def __compute_pressures(self, p: Pressure, altitude, temperature,
humidity):
# Normalize pressure to HPa
qfe = self.__to_pressure(p.qfe)
qnh = self.__to_pressure(p.qnh)
qff = self.__to_pressure(p.qff)
if qfe and qnh is None:
qnh = TWxUtils.StationToAltimeter(qfe, elevationM=altitude)
if qnh and qfe is None:
qfe = TWxUtils.AltimeterToStationPressure(qnh, elevationM=altitude)
if qfe and qff is None and temperature is not None and humidity is not None:
qff = TWxUtils.StationToSeaLevelPressure(qfe,
elevationM=altitude,
currentTempC=temperature,
meanTempC=temperature,
humidity=humidity)
if qff and qfe is None and temperature is not None and humidity is not None:
qfe = TWxUtils.SeaLevelToStationPressure(qff,
elevationM=altitude,
currentTempC=temperature,
meanTempC=temperature,
humidity=humidity)
return {
"qfe": to_float(qfe),
"qnh": to_float(qnh),
"qff": to_float(qff)
}
def __to_altitude(self, value):
if isinstance(value, ureg.Quantity):
return to_int(value.to(ureg.meter).magnitude)
else:
return to_int(value)
def __to_rain(self, value):
if isinstance(value, ureg.Quantity):
return to_float(
value.to(ureg.liter / (ureg.meter**2)).magnitude, 1)
else:
return to_float(value, 1)
def add_redis_key(self, key, values, cache_duration):
pipe = self.redis.pipeline()
pipe.hmset(key, values)
pipe.expire(key, cache_duration)
pipe.execute()
def call_google_api(self, api_url, api_name):
url = furl(api_url)
url.args["key"] = self.google_api_key
result = requests.get(url.url,
timeout=(self.connect_timeout,
self.read_timeout)).json()
if result["status"] == "OVER_QUERY_LIMIT":
raise UsageLimitException(f"{api_name} OVER_QUERY_LIMIT")
elif result["status"] == "INVALID_REQUEST":
raise ProviderException(
f'{api_name} INVALID_REQUEST: {result.get("error_message", "")}'
)
elif result["status"] == "ZERO_RESULTS":
raise ProviderException(f"{api_name} ZERO_RESULTS")
return result
def __compute_elevation(self, lat, lon) -> Tuple[float, bool]:
radius = 500
nb = 6
path = f"{lat},{lon}|"
for k in range(nb):
angle = math.pi * 2 * k / nb
dx = radius * math.cos(angle)
dy = radius * math.sin(angle)
path += "{lat},{lon}".format(
lat=str(lat + (180 / math.pi) * (dy / 6378137)),
lon=str(lon + (180 / math.pi) *
(dx / 6378137) / math.cos(lat * math.pi / 180)),
)
if k < nb - 1:
path += "|"
result = self.call_google_api(
f"https://maps.googleapis.com/maps/api/elevation/json?locations={path}",
"Google Elevation API")
elevation = float(result["results"][0]["elevation"])
is_peak = False
for point in result["results"][1:]:
try:
glide_ratio = radius / (elevation - float(point["elevation"]))
except ZeroDivisionError:
glide_ratio = float("Infinity")
if 0 < glide_ratio < 6:
is_peak = True
break
return elevation, is_peak
def __get_place_geocoding_results(self, results):
lat, lon, address_long_name = None, None, None
for result in results["results"]:
if result.get("geometry", {}).get("location"):
lat = result["geometry"]["location"]["lat"]
lon = result["geometry"]["location"]["lng"]
for component in result["address_components"]:
if "postal_code" not in component["types"]:
address_long_name = component["long_name"]
break
break
return lat, lon, address_long_name
def __get_place_autocomplete(self, name):
results = self.call_google_api(
f"https://maps.googleapis.com/maps/api/place/autocomplete/json?input={name}",
"Google Places API")
place_id = results["predictions"][0]["place_id"]
results = self.call_google_api(
f"https://maps.googleapis.com/maps/api/geocode/json?place_id={place_id}",
"Google Geocoding API")
return self.__get_place_geocoding_results(results)
def __get_place_geocoding(self, name):
results = self.call_google_api(
f"https://maps.googleapis.com/maps/api/geocode/json?address={name}",
"Google Geocoding API")
return self.__get_place_geocoding_results(results)
def get_station_id(self, provider_id):
return self.provider_code + "-" + str(provider_id)
def __create_station(self, provider_id, short_name, name, latitude,
longitude, altitude, is_peak, status, tz, urls,
fixes):
if fixes is None:
fixes = {}
if any((not short_name, not name, altitude is None, latitude is None,
longitude is None, not status, not tz)):
raise ProviderException("A mandatory value is none!")
station = {
"pv-id": provider_id,
"pv-code": self.provider_code,
"pv-name": self.provider_name,
"url": urls,
"short": fixes.get("short") or short_name,
"name": fixes.get("name") or name,
"alt":
self.__to_altitude(fixes["alt"] if "alt" in fixes else altitude),
"peak": to_bool(fixes["peak"] if "peak" in fixes else is_peak),
"loc": {
"type":
"Point",
"coordinates": [
to_float(
fixes["longitude"]
if "longitude" in fixes else longitude, 6),
to_float(
fixes["latitude"] if "latitude" in fixes else latitude,
6),
],
},
"status": status,
"tz": tz,
"seen": arrow.utcnow().int_timestamp,
}
return station
def save_station(
self,
provider_id,
short_name,
name,
latitude,
longitude,
status: StationStatus,
altitude=None,
tz=None,
url=None,
default_name=None,
lookup_name=None,
):
if provider_id is None:
raise ProviderException("'provider id' is none!")
station_id = self.get_station_id(provider_id)
lat = to_float(latitude, 6)
lon = to_float(longitude, 6)
address_key = f"address/{lat},{lon}"
if (not short_name or not name) and not self.redis.exists(address_key):
try:
results = self.call_google_api(
f"https://maps.googleapis.com/maps/api/geocode/json?latlng={lat},{lon}"
f"&result_type=airport|colloquial_area|locality|natural_feature|point_of_interest|neighborhood",
"Google Geocoding API",
)
address_short_name = None
address_long_name = None
for result in results["results"]:
for component in result["address_components"]:
if "postal_code" not in component["types"]:
address_short_name = component["short_name"]
address_long_name = component["long_name"]
break
if not address_short_name or not address_long_name:
raise ProviderException(
"Google Geocoding API: No valid address name found")
self.add_redis_key(
address_key,
{
"short": address_short_name,
"name": address_long_name
},
self.google_api_cache_duration,
)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(address_key, {"error": repr(e)},
self.usage_limit_cache_duration)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception("Unable to call Google Geocoding API")
self.add_redis_key(address_key, {"error": repr(e)},
self.google_api_error_cache_duration)
address = lookup_name or name or short_name
geolocation_key = f"geolocation/{address}"
if (lat is None or lon is None) or (lat == 0 and lon == 0):
if not self.redis.exists(geolocation_key):
try:
lat, lon, address_long_name = self.__get_place_geocoding(
address)
if not lat or not lon or not address_long_name:
raise ProviderException(
f"Google Geocoding API: No valid geolocation found {address}"
)
self.add_redis_key(
geolocation_key,
{
"lat": lat,
"lon": lon,
"name": address_long_name
},
self.google_api_cache_duration,
)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(geolocation_key, {"error": repr(e)},
self.usage_limit_cache_duration)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception(
"Unable to call Google Geocoding API")
self.add_redis_key(geolocation_key, {"error": repr(e)},
self.google_api_error_cache_duration)
if self.redis.exists(geolocation_key):
if self.redis.hexists(geolocation_key, "error"):
raise ProviderException(
f'Unable to determine station geolocation: {self.redis.hget(geolocation_key, "error")}'
)
lat = to_float(self.redis.hget(geolocation_key, "lat"), 6)
lon = to_float(self.redis.hget(geolocation_key, "lon"), 6)
if not name:
name = self.redis.hget(geolocation_key, "name")
alt_key = f"alt/{lat},{lon}"
if not self.redis.exists(alt_key):
try:
elevation, is_peak = self.__compute_elevation(lat, lon)
self.add_redis_key(alt_key, {
"alt": elevation,
"is_peak": str(is_peak)
}, self.google_api_cache_duration)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(alt_key, {"error": repr(e)},
self.usage_limit_cache_duration)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception("Unable to call Google Elevation API")
self.add_redis_key(alt_key, {"error": repr(e)},
self.google_api_error_cache_duration)
tz_key = f"tz/{lat},{lon}"
if not tz and not self.redis.exists(tz_key):
try:
now = arrow.utcnow().int_timestamp
result = self.call_google_api(
f"https://maps.googleapis.com/maps/api/timezone/json?location={lat},{lon}×tamp={now}",
"Google Time Zone API",
)
tz = result["timeZoneId"]
gettz(tz)
self.add_redis_key(tz_key, {"tz": tz},
self.google_api_cache_duration)
except TimeoutError as e:
raise e
except UsageLimitException as e:
self.add_redis_key(tz_key, {"error": repr(e)},
self.usage_limit_cache_duration)
except Exception as e:
if not isinstance(e, ProviderException):
self.log.exception("Unable to call Google Time Zone API")
self.add_redis_key(tz_key, {"error": repr(e)},
self.google_api_error_cache_duration)
if not short_name:
if self.redis.hexists(address_key, "error"):
if default_name:
short_name = default_name
else:
raise ProviderException(
f"Unable to determine station 'short': {self.redis.hget(address_key, 'error')}"
)
else:
short_name = self.redis.hget(address_key, "short")
if not name:
if self.redis.hexists(address_key, "error"):
if default_name:
name = default_name
else:
raise ProviderException(
f"Unable to determine station 'name': {self.redis.hget(address_key, 'error')}"
)
else:
name = self.redis.hget(address_key, "name")
if not altitude:
if self.redis.hexists(alt_key, "error"):
raise ProviderException(
f"Unable to determine station 'alt': {self.redis.hget(alt_key, 'error')}"
)
altitude = self.redis.hget(alt_key, "alt")
if self.redis.hexists(alt_key, "error") == "error":
raise ProviderException(
f"Unable to determine station 'peak': {self.redis.hget(alt_key, 'error')}"
)
is_peak = self.redis.hget(alt_key, "is_peak") == "True"
if not tz:
if self.redis.hexists(tz_key, "error"):
raise ProviderException(
f"Unable to determine station 'tz': {self.redis.hget(tz_key, 'error')}"
)
tz = self.redis.hget(tz_key, "tz")
if not url:
urls = {"default": self.provider_url}
elif isinstance(url, str):
urls = {"default": url}
elif isinstance(url, dict):
if "default" not in url:
raise ProviderException("No 'default' key in url")
urls = url
else:
raise ProviderException("Invalid url")
fixes = self.mongo_db.stations_fix.find_one(station_id)
station = self.__create_station(provider_id, short_name, name, lat,
lon, altitude, is_peak, status.value,
tz, urls, fixes)
self.stations_collection().update({"_id": station_id},
{"$set": station},
upsert=True)
station["_id"] = station_id
return station
def create_measure(
self,
for_station,
_id,
wind_direction,
wind_average,
wind_maximum,
temperature=None,
humidity=None,
pressure: Pressure = None,
rain=None,
):
if all((wind_direction is None, wind_average is None,
wind_maximum is None)):
raise ProviderException("All mandatory values are null!")
# Mandatory keys: 0 if not present
measure = {
"_id": int(round(_id)),
"w-dir": self.__to_wind_direction(wind_direction),
"w-avg": self.__to_wind_speed(wind_average),
"w-max": self.__to_wind_speed(wind_maximum),
}
# Optional keys
if temperature is not None:
measure["temp"] = self.__to_temperature(temperature)
if humidity is not None:
measure["hum"] = to_float(humidity, 1)
if pressure is not None and (pressure.qfe is not None or pressure.qnh
is not None or pressure.qff is not None):
measure["pres"] = self.__compute_pressures(
pressure, for_station["alt"], measure.get("temp", None),
measure.get("hum", None))
if rain is not None:
measure["rain"] = self.__to_rain(rain)
measure["time"] = arrow.now().int_timestamp
fixes = self.mongo_db.stations_fix.find_one(for_station["_id"])
if fixes and "measures" in fixes:
for key, offset in fixes["measures"].items():
try:
if key in measure:
fixed_value = measure[key] + offset
if key == "w-dir":
fixed_value = fixed_value % 360
measure[key] = fixed_value
except Exception as e:
self.log.exception(
f"Unable to fix '{key}' with offset '{offset}': {e}")
return measure
def has_measure(self, measure_collection, key):
return measure_collection.find({"_id": key}).count() > 0
def __add_last_measure(self, measure_collection, station_id):
last_measure = measure_collection.find_one({
"$query": {},
"$orderby": {
"_id": -1
}
})
if last_measure:
self.stations_collection().update({"_id": station_id},
{"$set": {
"last": last_measure
}})
def insert_new_measures(self, measure_collection, station, new_measures):
if len(new_measures) > 0:
measure_collection.insert(
sorted(new_measures, key=lambda m: m["_id"]))
end_date = arrow.Arrow.fromtimestamp(new_measures[-1]["_id"],
gettz(station["tz"]))
self.log.info(
"⏱ {end_date} ({end_date_local}), {short}/{name} ({id}): {nb} values inserted"
.format(
end_date=end_date.format("YY-MM-DD HH:mm:ssZZ"),
end_date_local=end_date.to("local").format(
"YY-MM-DD HH:mm:ssZZ"),
short=station["short"],
name=station["name"],
id=station["_id"],
nb=str(len(new_measures)),
))
self.__add_last_measure(measure_collection, station["_id"])
}
}
collection = 'Userinformation'
validator = {'$jsonSchema': {'bsonType': 'object', 'properties': {}}}
required = []
for field_key in user_schema:
field = user_schema[field_key]
properties = {'bsonType': field['type']}
minimum = field.get('minlength')
if type(minimum) == int:
properties['minimum'] = minimum
if field.get('required') is True: required.append(field_key)
validator['$jsonSchema']['properties'][field_key] = properties
if len(required) > 0:
validator['$jsonSchema']['required'] = required
query = [('collMod', collection), ('validator', validator)]
try:
db.create_collection(collection)
except CollectionInvalid:
pass
command_result = db.command(OrderedDict(query))
def createNewCollection(MongoClient,dummyarg):
MongoClient.create_collection("ekcollection")
except TypeError:
# for pymongo >= 3
MONGO_DB_VERSION = MONGO_CLIENT.client.server_info()['version']
if not float('.'.join(MONGO_DB_VERSION.split('.')[:-1])) >= 2.2:
raise ImproperlyConfigured(
'''
Your mongodb service doesn't support TTL
http://docs.mongodb.org/manual/tutorial/expire-data/
'''
)
# create sessions collection if needed
if MONGO_SESSIONS_COLLECTION not in MONGO_CLIENT.collection_names():
MONGO_CLIENT.create_collection(MONGO_SESSIONS_COLLECTION)
# check existing indexes
DB_COLLECTION = MONGO_CLIENT[MONGO_SESSIONS_COLLECTION]
MONGO_SESSIONS_INDEXES = DB_COLLECTION.index_information()
if len(MONGO_SESSIONS_INDEXES) <= 1:
DB_COLLECTION.ensure_index(
'session_key',
unique=True
)
DB_COLLECTION.ensure_index(
'creation_date',
expireAfterSeconds=MONGO_SESSIONS_TTL
)
def install(args):
if sys.platform == "linux" or 1 == 1:
if step_enabled("dependencies", args):
# install MongoDB and some other things if they're not installed
try:
call(["cherrydoor-install"], shell=False) # nosec
except (PermissionError, FileNotFoundError):
print("unable to install dependencies")
if args.fail:
sys.exit(1)
# generate a configuration based on default config
if (
not os.path.exists(f"{Path.home()}/.config/cherrydoor/config.json")
or "config" in args.install_steps
):
config = {
"__comment__": "This is a default config for setuptools installation - it shouldn't be used if installed from GitHub",
"host": "127.0.0.1",
"port": 5000,
"mongo": {
"url": "localhost:27017",
"name": "cherrydoor",
"username": "******",
"password": "******",
},
"login-translation": {
"username": "******",
"password": "******",
"remember-me": "Pamiętaj mnie",
"log-in": "Zaloguj się",
"message": "Musisz się zalogować by uzyskać dostęp do tej strony",
},
"secret-key": "\\xd7w7\\x04\\r\\xfc/q\\x1a\\x9b&",
"https": {
"enabled": False,
"hsts-enabled": False,
"hsts-preload": False,
},
"interface": {
"type": "serial",
"baudrate": 115200,
"port": "/dev/serial0",
"encoding": "utf-8",
},
"manufacturer-code": "18",
}
else:
with open(
f"{Path.home()}/.config/cherrydoor/config.json", "r", encoding="utf-8"
) as f:
config = json.load(f)
if step_enabled("config", args):
# create a random secret key
config["secret-key"] = os.urandom(24).hex()
# let user choose a password for the database
if step_enabled("database", args):
config["mongo"]["password"] = getpass("Wprowadź hasło do bazy danych: ")
try:
# files configuration
if not os.path.exists(f"${Path.home()}"):
os.makedirs(f"{Path.home()}/.config/cherrydoor")
with open(
f"{Path.home()}/.config/cherrydoor/config.json",
"w",
encoding="utf-8",
) as f:
json.dump(config, f, ensure_ascii=False, indent=4)
except (IOError, PermissionError):
print(
f"Nie udało się stworzyć plików w {Path.home()}/.config/cherrydoor. Spróbuj stworzyć ten folder manualnie i nadać mu właściwe uprawnienia",
file=sys.stderr,
)
if args.fail:
sys.exit(1)
if step_enabled("service", args):
service_config = f"""\
[Unit]
Description=Cherrydoor Service
After=network.target
[Service]
ExecStart={os.path.realpath(__file__).replace("install.py", "__init__.py")} start
Environment=PYTHONUNBUFFERED=1
Restart=always
Type=simple
User=ubuntu
[Install]
WantedBy=multi-user.target
"""
try:
if not os.path.exists(f"{Path.home()}/.config/systemd/user"):
os.makedirs(f"{Path.home()}/.config/systemd/user")
with open(
f"{Path.home()}/.config/systemd/user/cherrydoor.service", "w"
) as f:
f.write(service_config)
print(
f"Plik konfiguracyjny znajduje się w folderze {Path.home()}/.config/cherrydoor"
)
except (IOError, PermissionError):
print(
f"Nie udało się stworzyć pliku usługi pod {Path.home()}/.config/systemd/user/cherrydoor.service - spróbuj uruchomić skrypt z właściwymi uprawnieniami lub stworzyć ten plik manualnie. Zawartość:",
file=sys.stderr,
)
print(service_config, file=sys.stderr)
if args.fail:
sys.exit(1)
hasher = PasswordHasher(
time_cost=4,
memory_cost=65536,
parallelism=8,
hash_len=16,
salt_len=16,
encoding="utf-8",
)
db = MongoClient(
f"mongodb://{config['mongo']['url']}/{config['mongo']['name']}"
)[config["mongo"]["name"]]
if step_enabled("database", args):
try:
db.command(
"createUser",
config["mongo"]["username"],
pwd=config["mongo"]["password"],
roles=[
{"role": "readWrite", "db": config["mongo"]["name"]},
{"role": "clusterMonitor", "db": "admin"},
],
)
db.create_collection("users")
db.create_collection(
"logs", options={"size": 1073742000, "capped": True}
)
db.create_collection("settings")
db.create_collection(
"terminal", options={"size": 1048576, "capped": True, "max": 10000}
)
except OperationFailure:
pass
user_indexes = db.users.index_information()
if "username_index" not in user_indexes.keys():
db.users.create_index("username", name="username_index", unique=True)
if "cards_index" not in user_indexes.keys():
db.users.create_index("cards", name="cards_index", sparse=True)
# nosec - it's python3, not 2, Bandit...
if step_enabled("user", args) and input(
"Czy chcesz stworzć nowego użytkownika-administratora? [y/n]"
).lower() in ["y", "yes", "tak", "t"]:
# nosec - it's python3, not 2, Bandit...
username = input("Wprowadź nazwę użytkownika: ")
password = hasher.hash(getpass("Hasło: "))
db.users.insert({"username": username, "password": password, "cards": []})
print("Instalacja skończona!")
try:
service_call_args = ["systemctl", "--user", "enable", "cherrydoor"]
call(service_call_args, shell=False) # nosec
except (IOError, PermissionError):
pass
else:
print("Ten system operacyjny nie jest obecnie obsługiwany")
sys.exit()
error.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
logger.addHandler(access)
logger.addHandler(error)
if env=='debug':
logging.basicConfig(level=logging.DEBUG)
logging.debug("Debug mode")
db = MongoClient(secret_settings.mongo['uri']).ruslang
overwrite = len(list(db.wordlist.find()))
if overwrite:
db.users.drop()
db.wordlist.drop()
cols = db.collection_names()
if 'users' not in cols:
db.create_collection('users')
if 'wordlist' not in cols:
db.create_collection('wordlist')
my_import.doImport()
users = db.users
wordlist = db.wordlist
params['offset'] = 0
logging.warning('Started')
while True:
getUpdates()
time.sleep(0.1)
#app.run()
# coding=utf-8
"""
创建一个collection
https://api.mongodb.com/python/current/examples/collations.html#assign-a-default-collation-to-a-collection
"""
# %%
from pymongo import MongoClient
from pymongo.collation import Collation
db = MongoClient().hello
collection = db.create_collection('test_collection',
collation=Collation(locale='zh'))
