def before_post(self):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
col_users = db()['users']
col_user_logins = db()['user_logins']
user_info = col_users.find_one({
'mobile':
self.params['mobile'],
'password':
create_md5(self.params['password'])
})
if user_info is None:
self.set_output('user', 'login_failed')
else:
self.user_role = user_info['role']
user_info = self.after_get_one(user_info)
self.user_id = user_info['id']
if user_info['confirmed']:
self.user_id = user_info['id']
self.set_output('public_operations', 'successful')
self.token = encode_token({
'user_id': self.user_id,
'role': self.user_role
}).decode('ascii')
self.output['token'] = self.token
last_login = col_user_logins.find({
'mobile':
self.params['mobile']
}).sort('date', -1).limit(1)
if last_login.count() > 0:
user_info['last_login'] = str(last_login[0]['date'])
else:
user_info['last_login'] = ''
user_info['first_login'] = str(datetime.now())
if 'last_update' in user_info: del user_info['last_update']
if 'password_pure' in user_info:
del user_info['password_pure']
if 'password' in user_info: del user_info['password']
if '_id' in user_info: del user_info['_id']
self.output['data']['item'] = user_info
else:
self.set_output('user', 'inactive')
except:
PrintException()
self.set_output('public_operations', 'failed')
try:
col_user_logins = db()['user_logins']
col_user_logins.insert({
'user_id': self.user_id,
'mobile': self.params.get('mobile'),
'status': self.status,
'date': datetime.now(),
'notes': self.note_id
})
except:
PrintException()
self.allow_action = False
def put(self, *args, **kwargs):
try:
self.method = 'put'
self.module = 'users'
if self.pre_put():
for item in self.params:
if item not in self.inputs['put']:
self.set_output('tasks', 'wrong_params')
return False
need_consistency_update = any(
x in self.params for x in ['pic', 'name', 'family'])
if 'password' in self.params:
self.params['password'] = create_md5(
self.params['password'])
self.params['password_pure'] = self.params['password']
col_users = db()['users']
col_users.update({'_id': ObjectId(self.user_id)},
{'$set': self.params})
if need_consistency_update:
col_people = db()['people']
col_tasks = db()['tasks']
doc = {}
if 'name' in self.params:
doc['people.name'] = self.params['name']
if 'family' in self.params:
doc['people.family'] = self.params['family']
if 'pic' in self.params:
doc['people.pic'] = self.params['pic']
col_tasks.update({'people.id': self.user_id},
{'$set': doc},
multi=True)
changes = {}
if 'name' in self.params and 'pic' not in self.params:
changes['$set'] = {'name': self.params['name']}
elif 'pic' in self.params and 'name' not in self.params:
changes['$set'] = {'pic': self.params['pic']}
elif 'pic' in self.params and 'name' in self.params:
changes['$set'] = {
'name': self.params['name'],
'pic': self.params['pic']
}
col_people.update({'user_id': self.user_id},
changes,
multi=True)
self.params['last_update'] = datetime.now()
self.set_output('public_operations', 'successful')
except:
PrintException()
self.set_output('public_operations', 'failed')
if consts.LOG_ACTIVE:
self.log_status(self.output)
self.after_put()
self.kmwrite()
def before_get(self):
try:
col_tasks = db()['tasks']
if 'id' in self.params:
task = col_tasks.find_one({'_id': ObjectId(self.params['id'])})
task['id'] = str(task['_id'])
task['from_date'] = str(task['from_date'])
task['to_date'] = str(task['to_date'])
del task['_id']
del task['create_date']
del task['last_update']
del task['user_id']
self.output['data']['item'] = task
else:
tasks = []
for user_task in col_tasks.find({
'user_id': str(self.user_id)
}).sort([('create_date', -1)]):
user_task['id'] = str(user_task['_id'])
del user_task['_id']
if 'from_date' in user_task:
user_task['from_date'] = str(user_task['from_date'])
if 'to_date' in user_task:
user_task['to_date'] = str(user_task['to_date'])
del user_task['create_date']
del user_task['last_update']
del user_task['user_id']
tasks.append(user_task)
self.output['data']['list'] = tasks
self.set_output('public_operations', 'successful')
except:
PrintException()
return False
self.allow_action = False
return True
def before_post(self):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
col_users = db()['users']
result = col_users.find_one_and_update(
{
'mobile': self.params['mobile'],
# 'pure_password': self.params['old_password'],
'activation_code': self.params['activation_code']
},
{
'$set': {
'password': create_md5(self.params['new_password']),
'password_pure': self.params['new_password']
}
})
if result is None:
self.set_output('user', 'password_reset_failed')
else:
self.set_output('public_operations', 'successful')
except:
PrintException()
self.set_output('public_operations', 'failed')
return False
self.allow_action = False
return True
def post(self, *args, **kwargs):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
self.method = 'post'
if self.pre_post():
try:
col_users = db()['users']
activation_code = random_digits()
if 'mobile' in self.params:
user_info = col_users.find_one(
{'mobile': self.params['mobile']})
if user_info is None:
self.set_output('user', 'user_not_exists')
else:
col_users.update_one(
{'mobile': self.params['mobile']},
{'$set': {
'activation_code': activation_code
}})
send_sms(
sms['users']['forgot_password'][self.locale] %
activation_code, self.params['mobile'])
self.set_output('public_operations', 'successful')
else:
self.set_output('public_operations', 'successful')
self.after_post()
except:
PrintException()
self.set_output('public_operations', 'failed')
self.kmwrite()
def before_post(self):
try:
col_users = db()['users']
user = col_users.find_one({'mobile': self.params['mobile']}, {
'confirmed': 1,
'mobile': 1,
'activation_code': 1
})
if user is not None and user['confirmed'] is False:
# user['activation_code'] = random_digits() if 'activation_code' not in user\
# or user['activation_code'] == '' or\
# user['activation_code'] is None else user['activation_code']
send_sms(
sms['users']['registration_successful'][self.locale] %
user['activation_code'].encode('utf-8'), user['mobile'])
self.set_output('user', 'send_sms')
return True
elif user is None:
self.set_output('user', 'user_not_exists')
return False
elif user is not None and user['confirmed'] is True:
self.set_output('user', 'already_active')
return False
self.allow_action = False
return True
except:
PrintException()
return False
def before_post(self):
try:
self.method = 'users'
col_users = db()['users']
if col_users.count({'mobile': self.params['mobile']}) > 0:
self.set_output('user', 'mobile_exists')
return False
elif self.params.get('email') is not None:
if col_users.count({'email': self.params['email']}) > 0:
self.set_output('user', 'email_exists')
return False
# if col_users.count({'device_info.mac_address': self.params['device_info']['mac_address']}) > 0:
# self.set_output('user', 'multiple_registration')
# return False
self.params['activation_code'] = random_digits()
self.params['confirmed'] = False
self.params['role'] = 'user'
# self.params['tasks_figure'] = 'line' if 'tasks_figure' not in self.params else self.params['tasks_figure']
self.params['password_pure'] = self.params['password']
self.params['password'] = create_md5(self.params['password'])
# .encode('utf-8')
except:
PrintException()
return False
return True
def post(self, *args, **kwargs):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
self.method = 'post'
if self.pre_post():
if self.allow_action:
# TODO: Remove this code after some time, only for null values sent by retards
validated = True
new_params = deepcopy(self.params)
for k, v in self.params.items():
if v in [None, 'null']:
self.set_output('field_error', 'null')
validated = False
# break
if k in self.multilingual:
new_params[k] = {}
new_params[k][self.locale] = self.params[k]
self.params = deepcopy(new_params)
if validated:
self.params.update(self.added_data)
col = db()[self.module]
self.params['create_date'] = datetime.now()
self.params['last_update'] = datetime.now()
id = str(col.insert(self.params))
self.id = id
self.output['data']['item']['id'] = id
self.set_output('public_operations', 'successful')
self.after_post()
if consts.LOG_ACTIVE:
self.log_status(self.output)
except:
self.PrintException()
self.set_output('public_operations', 'failed')
self.kmwrite()
def insert_users():
col_users = db()['users']
# col_users.drop()
if col_users.count({'username': '******'}) == 0:
col_users.insert_one({
"_id": ObjectId("5e25ab698c90582c5785d291"),
'name': 'shahab',
'family': 'qazavi',
'mobile': 'admin',
'password': create_md5('1'),
'role': 'admin',
})
def load_permissions(self):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
col_users = db()['users']
user_info = col_users.find_one({'_id': ObjectId(self.user_id)})
if user_info is None:
self.set_output('user', 'user_not_exists')
else:
self.user_role = user_info['role']
print('module: %s role: %s user_id: %s' %
(self.module, user_info['role'], self.user_id))
if self.module is None:
self.set_output('public_operations',
'module_not_specified')
else:
col_users_roles = db()['users_roles']
user_role_info = col_users_roles.find_one({
'name':
user_info['role'],
'module':
self.module
})
print('user_role %s module %s' %
(self.user_role, self.module))
if user_role_info is None:
self.set_output('user', 'access_denied')
else:
del user_role_info['_id']
self.permissions = user_role_info['permissions']
# TODO: Some kind of shit here
import json
temp = json.dumps(self.permissions)
temp = temp.replace('$uid', self.user_id)
temp = temp.replace('#', '.')
temp = temp.replace('@', '$')
self.permissions = json.loads(temp)
except:
self.PrintException()
self.set_output('public_operations', 'failed')
return self.status
def before_delete(self):
try:
col_tasks = db()['tasks']
if col_tasks.find_one({'_id': ObjectId(self.params['id'])}):
col_tasks.delete_one({'_id': ObjectId(self.params['id'])})
self.set_output('public_operations', 'successful')
else:
self.set_output('public_operations', 'record_not_found')
return False
except:
PrintException()
return False
self.allow_action = False
return True
def before_post(self):
try:
for item in self.params:
if item not in self.inputs:
self.set_output('tasks', 'wrong_params')
return False
col_save_task = db()['save_task_query']
if col_save_task.count_documents({'name': self.params['name']
}) > 0:
self.set_output('save_task', 'duplicate_name')
return False
except:
PrintException()
return False
return True
def before_delete(self):
try:
if type(self.params['id']) is list:
if len(self.params['id']) > 1:
col_people = db()['people']
people_ids = []
for item in self.params['id']:
people_ids.append(ObjectId(item))
col_people.delete_many({'_id': {'$in': people_ids}})
self.set_output('public_operations', 'successful')
self.allow_action = False
else:
self.params['id'] = self.params['id'][0]
except:
self.PrintException()
return False
return True
def before_delete(self):
try:
self.method = 'users'
print(self.params['mobile'])
print(self.user_id)
col_users = db()['users']
col_users.delete_one({
'_id':
ObjectId(self.params['id']),
'mobile':
self.params['mobile'],
'password':
create_md5(self.params['password'])
})
self.set_output('public_operations', 'successful')
except:
PrintException()
self.allow_action = False
def initialize(self):
self.db = db()
self.permissions = None
self.start_time = datetime.now()
self.user_id = None
self.user_role = None
self.id = None
self.params = {}
self.original_params = {}
self.status = False
self.status_code = 500
self.note = ''
self.note_group = '' # Only usable in test mode
self.note_id = '' # Only usable in test mode
self.log = []
self.module = None
self.logical_delete = False
self.allow_action = True
self.url = ''
self.token = None
self.locale = 'en'
self.method = ''
self.app_version = 0
self.document_count = 0
self.added_data = {}
self.inputs = {}
self.required = {}
self.fields = []
self.sort = {}
self.doc_limit = -1
self.auto_fields = ['create_date', 'last_update']
self.conditions = {}
self.output = deepcopy(output)
self.tokenless = False
self.multilingual = []
self.casting = {
'ints': [],
'floats': [],
'dics': [],
'dates': [],
'lists': [],
'defaults': {}
}
def log_status(self, data):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
col = db()['logs']
log = deepcopy(log_template)
# col.da
doc = deepcopy(self.params)
for item in self.casting['dates']:
if item in doc.keys():
doc[item] = str(doc[item])
log.update({
'project':
'miz',
'ip':
self.request.remote_ip,
'duration':
(datetime.now() - self.start_time).total_seconds() * 1000,
'date':
datetime.now(),
'token':
self.token,
'user_id':
self.user_id,
'doc':
doc,
'original_params':
self.original_params,
'status':
self.status,
'http_code':
self.status_code,
'output':
data,
'method':
self.method,
'url':
self.request.uri,
})
try:
col.insert(log)
except:
self.PrintException()
def post(self, *args, **kwargs):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
self.method = 'post'
if self.pre_post():
self.params.update(self.added_data)
col_users = db()['users']
self.params['create_date'] = datetime.now()
self.params['last_update'] = datetime.now()
id = str(col_users.insert(self.params))
self.id = id
self.output['data']['item']['id'] = id
# self.output['token'] = encode_token({'user_id': self.id}).decode()
self.set_output('public_operations', 'successful')
self.after_post()
if consts.LOG_ACTIVE:
self.log_status(self.output)
except:
PrintException()
self.set_output('public_operations', 'failed')
self.kmwrite()
def delete(self, *args, **kwargs):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
self.method = 'delete'
self.module = self.request.uri.split('/')[2].split('?')[0]
if self.pre_delete():
if self.allow_action:
col = db()[self.module]
if self.logical_delete:
results = col.update_one(
{
'_id': ObjectId(self.params['id'])
}, {
'$set': {
'deleted': True
}
}).raw_result
if results['nModified'] == 1:
self.set_output('public_operations', 'successful')
else:
self.set_output('public_operations',
'record_not_found')
else:
if self.conditions == {}:
self.conditions = {
'_id': ObjectId(self.params['id'])
}
results = col.remove(self.conditions)
if results['n'] == 1:
self.set_output('public_operations', 'successful')
else:
self.set_output('public_operations',
'record_not_found')
self.after_delete()
if consts.LOG_ACTIVE:
self.log_status(self.output)
except:
self.PrintException()
self.set_output('public_operations', 'failed')
self.kmwrite()
def put(self, *args, **kwargs):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
self.method = 'put'
self.module = self.request.uri.split('/')[2].split('?')[0]
if self.pre_put():
# TODO: Remove this code after some time, only for null values sent by retards
if self.allow_action:
has_null = False
for k, v in self.params.items():
if v in [None, 'null']:
self.set_output('field_error', 'null')
has_null = True
if k in self.multilingual:
self.params[k + '.' + self.locale] = v
del self.params[k]
for k, v in self.conditions.items():
if v in [None, 'null']:
self.set_output('field_error', 'null')
has_null = True
break
if not has_null:
if 'create_date' in self.params:
del self.params['create_date']
self.params['last_update'] = datetime.now()
col = db()[self.module]
if self.conditions == {}:
id = self.params['id']
del self.params['id']
try:
id = ObjectId(
id
) if self.module != 'achievements' else id
query = {'_id': id}
except:
self.set_output('field_error', 'id_format')
self.PrintException()
return
else:
query = self.conditions
if not self.tokenless:
if 'put' in self.permissions:
query.update(self.permissions['put']['query'])
results = col.update_one(query, {
'$set': self.params
}).raw_result
if self.conditions == {}:
self.params['id'] = id
if results['nModified'] > 0:
self.set_output('public_operations', 'successful')
elif results['updatedExisting']:
self.set_output('public_operations',
'update_failed')
else:
self.set_output('public_operations',
'record_not_found')
if consts.LOG_ACTIVE:
self.log_status(self.output)
self.after_put()
except:
self.PrintException()
self.set_output('public_operations', 'failed')
self.Print(self.note, Colors.RED)
self.kmwrite()
__author__ = 'Shahab Qazavi & Ehsan Shirzadi'
print('Link grabber started')
import sys
import os
sys.path.append('/home/shahab/dev/newshubmotors')
sys.path.append('/home/oem/dev/newshubmotors')
sys.path.append('/root/dev/newshubmotors')
sys.path.append('/app')
from publics import db, PrintException
import time
from datetime import datetime
from bson import ObjectId
import requests
import subprocess
from bs4 import BeautifulSoup
col_news = db()['news']
col_sources = db()['sources']
col_source_links = db()['source_links']
col_engine_instances = db()['engine_instances']
col_error_logs = db()['error_logs']
count = 0
news_count = 0
link_count = 0
count_mongo = 0
find_run_mongo = 0
find_mongo = 0
insert_mongo = 0
update_mongo = 0
from queue import Queue
import threading
q = Queue()
def get(self, id=None, *args, **kwargs):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
self.method = 'get'
if self.pre_get():
# TODO: Remove this code after some time, only for null values sent by retards
if self.allow_action:
has_null = False
for k, v in self.params.items():
if v in [None, 'null']:
self.set_output('field_error', 'null')
has_null = True
if not has_null:
col = db()[self.module]
fields_dic = {}
for item in self.fields:
fields_dic[item] = 1
if id is None:
self.get_init()
if self.doc_limit > -1 and self.params[
'page'] * self.params[
'page_size'] > self.doc_limit:
self.set_output('public_operations',
'page_limit')
else:
if self.sort == {}:
if fields_dic == {}:
results = col.find(
self.conditions).skip(
(self.params['page'] - 1) *
self.params['page_size']
).limit(self.params['page_size'])
else:
results = col.find(
self.conditions, fields_dic).skip(
(self.params['page'] - 1) *
self.params['page_size']
).limit(self.params['page_size'])
else:
sort_conditions = []
for k, v in self.sort.items():
sort_conditions.append((k, v))
if fields_dic == {}:
results = col.find(self.conditions).skip((self.params['page'] - 1) * self.params['page_size'])\
.limit(self.params['page_size']).sort(sort_conditions)
else:
results = col.find(self.conditions, fields_dic).skip((self.params['page'] - 1) * self.params['page_size'])\
.limit(self.params['page_size']).sort(sort_conditions)
self.document_count = results.count()
results = self.after_get(results)
self.output['data'][
'count'] = self.document_count
self.output['data']['list'] = results
self.set_output('public_operations',
'successful')
else:
try:
# TODO: Shit happend, I should do something for this
object_id = ObjectId(
id
) if self.module != 'achievements' else id
if fields_dic == {}:
results = col.find_one({'_id': object_id})
else:
results = col.find_one({'_id': object_id},
fields_dic)
if results is None:
self.set_output('public_operations',
'record_not_found')
else:
self.output['data'][
'item'] = self.after_get_one(results)
self.output['count'] = 1
self.set_output('public_operations',
'successful')
except:
self.PrintException()
self.set_output('field_error', 'id_format')
except:
self.PrintException()
self.set_output('public_operations', 'failed')
if consts.LOG_ACTIVE:
self.log_status(self.output)
self.kmwrite()
def insert_messages():
col_server_messages = db()['server_messages']
col_server_messages.drop()
col_server_messages.insert_many([
{
'group': 'user',
'name': 'token_not_received',
'code': 401,
'status': False,
'en': 'Token not received'
},
{
'group': 'user',
'name': 'token_validated',
'code': 200,
'status': True,
'en': 'Token validated'
},
{
'group': 'user',
'name': 'user_not_exists',
'code': 401,
'status': False,
'en': 'User not exists'
},
{
'group': 'user',
'name': 'token_expired',
'code': 401,
'status': False,
'en': 'User Token Expired'
},
{
'group': 'user',
'name': 'access_denied',
'code': 401,
'status': False,
'en': 'Access denied'
},
{
'group': 'user',
'name': 'permission_not_defined',
'code': 401,
'status': False,
'en': 'Permission not defined'
},
{
'group': 'user',
'name': 'method_not_specified',
'code': 401,
'status': False,
'en': 'Method not specified'
},
{
'group': 'user',
'name': 'access_granted',
'code': 200,
'status': True,
'en': 'Access granted'
},
{
'group': 'public_operations',
'name': 'params_loaded',
'code': 200,
'status': True,
'en': 'Params loaded'
},
{
'group': 'public_operations',
'name': 'params_not_loaded',
'code': 401,
'status': False,
'en': 'Params not loaded'
},
{
'group': 'public_operations',
'name': 'page_limit',
'code': 401,
'status': False,
'en': 'Page limit reached'
},
{
'group': 'public_operations',
'name': 'record_not_found',
'code': 404,
'status': False,
'en': 'Record not found'
},
{
'group': 'public_operations',
'name': 'failed',
'code': 401,
'status': False,
'en': 'Operation failed'
},
{
'group': 'public_operations',
'name': 'successful',
'code': 200,
'status': True,
'en': 'Operation successful'
},
{
'group': 'field_error',
'name': 'required',
'code': 401,
'status': False,
'en': 'Field %s required'
},
{
'group': 'field_error',
'name': 'null',
'code': 401,
'status': False,
'en': 'Null not allowed'
},
{
'group': 'field_error',
'name': 'id_format',
'code': 401,
'status': False,
'en': 'ID format is not correct'
},
{
'group': 'user',
'name': 'login_failed',
'code': 401,
'status': False,
'en': 'Login information is not correct'
},
{
'group': 'user',
'name': 'logged_in',
'code': 200,
'status': True,
'en': 'Logged in'
},
{
'group': 'user',
'name': 'user_exist',
'code': 200,
'status': True,
'en': 'User Exist'
},
{
'group': 'user',
'name': 'wrong_token',
'code': 401,
'status': False,
'en': 'Wrong Token'
},
{
'group': 'user',
'name': 'activate_account',
'code': 200,
'status': True,
'en': 'Account Has Been Activate'
},
{
'group': 'user',
'name': 'inactive',
'code': 401,
'status': False,
'en': 'Account Is Not Activate Yet'
},
{
'group': 'get_tasks',
'name': 'get_tasks_failed',
'code': 401,
'status': False,
'en': 'Wrong argument in condition'
},
{
'group': 'delete',
'name': 'delete_kon',
'code': 200,
'status': True,
'en': 'User Has Been Deleted'
},
{
'group': 'tasks',
'name': 'wrong_params',
'code': 401,
'status': False,
'en': 'Wrong Params'
},
{
'group': 'save_task',
'name': 'duplicate_name',
'code': 409,
'status': False,
'en': 'Duplicate Query name!'
},
{
'group': 'user',
'name': 'send_sms',
'code': 200,
'status': True,
'en': 'SMS Has Been sent'
},
{
'group': 'user',
'name': 'already_active',
'code': 200,
'status': True,
'en': 'The User is already active'
},
])
from publics import db
from datetime import datetime
col_news = db()['news']
start = datetime.now()
print(col_news.count_documents({}))
for items in col_news.find({}, {'url_hash': 1}).sort('create_date', -1):
url_hash_id = []
for item in col_news.find({'url_hash': items['url_hash']}, {'_id': 1}):
url_hash_id.append(item['_id'])
if len(url_hash_id) > 0:
if items['_id'] in url_hash_id:
url_hash_id.remove(items['_id'])
col_news.delete_many({'_id': {'$in': url_hash_id}})
print(col_news.count_documents({}))
print((datetime.now() - start).total_seconds())
from bson.json_util import loads
from base_handler import BaseHandler
from publics import PrintException, create_md5, encode_token, db
from datetime import datetime, timedelta
from bson import ObjectId
import math
col_daily = db()['daily']
col_weekly = db()['weekly']
col_monthly = db()['monthly']
time_now = datetime.now()
col_dashboard_report = db()['dashboard']
col_users = db()['users']
col_services = db()['services']
col_logs = db()['logs']
first_1398 = datetime.strptime('2019-03-21', "%Y-%m-%d")
all_user = 38526
"""
Create list of user id that have services
"""
service_users_ids = []
for item in col_services.distinct('user.id'):
if item is not None:
service_users_ids.append(ObjectId(item))
# ... fake Users and miz coins ...
user_ids = []
user_in_log = []
from datetime import datetime
from pymongo import MongoClient
from publics import db
# con = MongoClient()
# db = con['ldb']
col_services = db()['services']
col_logs = db()['logs']
col_users = db()['users']
col_search_history = db()['search_history']
col_events = db()['events']
col_chat = db()['chat']
item = ""
the_time = ""
condition = {'avg_date': {'$exists': False}}, {'_id': 1, 'create_date': 1}
# modify = {'_id': item['_id']}, {'$set': {'avg_date': the_time}}
# change date in search_history collection
for item in col_search_history.find({'avg_date': {'$exists': False}}, {'_id': 1, 'create_date': 1}):
the_time = str(item['create_date'])
the_time = the_time[:10]
the_time = datetime.strptime(the_time, "%Y-%m-%d")
col_search_history.find_and_modify({'_id': item['_id']}, {'$set': {'avg_date': the_time}})
# change date in events collection
for item in col_events.find({'avg_date': {'$exists': False}}, {'_id': 1, 'create_date': 1}):
the_time = str(item['create_date'])
the_time = the_time[:10]
def get(self, id=None, *args, **kwargs):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
self.method = 'get'
if self.pre_get():
try:
if len(self.fields) > 0:
fields = {}
for item in self.fields:
fields[item] = 1
else:
fields = {
'name': 1,
'family': 1,
'email': 1,
'pic': 1,
'tasks_figure': 1,
'mobile': 1
}
col_users = db()['users']
# print('-----------------------------')
# print(self.user_id)
# print(self.fields)
# print('-----------------------------')
user_info = col_users.find_one(
{'_id': ObjectId(self.user_id)}, fields)
user_info['id'] = str(user_info['_id'])
del user_info['_id']
col_tasks = db()['tasks']
user_tasks = []
# for item in col_tasks.find({'user_id': {'$in': [self.user_id]}}):
# del item['last_update']
# del item['create_date']
# del item['user_id']
# # item['create_date'] = str(item['create_date'])
# # item['last_update'] = str(item['last_update'])
# if 'from_date' in item:
# item['from_date'] = str(item['from_date'])
# if 'to_date' in item:
# item['to_date'] = str(item['to_date'])
# item['id'] = str(item['_id'])
# del item['_id']
# user_tasks.append(item)
user_people = []
col_people = db()['people']
for item in col_people.find(
{'user_id': {
'$in': [self.user_id]
}}):
item['id'] = str(item['_id'])
del item['_id']
del item['create_date']
del item['last_update']
del item['user_id']
user_people.append(item)
self.output['data']['item'] = user_info
# self.output['data']['item']['tasks'] = user_tasks
# self.output['data']['item']['people'] = user_people
self.set_output('public_operations', 'successful')
except:
PrintException()
self.set_output('field_error', 'id_format')
except:
PrintException()
self.set_output('public_operations', 'failed')
self.kmwrite()
from pymongo import MongoClient
import requests
import socket
from publics import PrintException, db
from bson.json_util import dumps, loads
try:
socket.create_connection(('localhost', 27017))
requests.get("http://localhost:8585/v1/")
try:
# con= MongoClient()
# db = con['ldb']
col_logs = db()['logs']
logs_list = []
ids_list = []
for item in col_logs.find({'sent': {'$exists': False}}).limit(20):
ids_list.append(item['_id'])
del item['_id']
item['date'] = str(item['date'])
logs_list.append(item)
# ADMIN_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyX2lkIjoiNWRiZmQxMjAxNzQ5ZWYxYjNiMGEwMjFkIiwicm9sZSI6ImFkbWluIiwiZGF0ZSI6IjIwMTktMTEtMDQgMTE6MTA6MDguMjg4MDk1In0.AtVoRKDTv1jvcHGpRmuLJvD9gG8sWqdR6P-Ibc6WDsU'
ADMIN_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyX2lkIjoiNWRjZDU4MDZiZWU4YTZhODFjMzE0MDhkIiwicm9sZSI6ImFkbWluIiwiZGF0ZSI6IjIwMTktMTEtMTQgMTc6MDg6NDMuNDAzNDA4In0.QUCCJHXYcSMA-pyHFCQn8zBiHS7C48MpoFAS1T9JUWk'
logs = dumps(logs_list)
params = {'params': logs, 'token': ADMIN_TOKEN}
par = loads(params['params'])
print(par[0])
requests.post("http://localhost:8585/v1/collect", json=params)
from publics import db, es, es45, check_and_raise
from datetime import datetime
print(str(datetime.now()))
from bson import ObjectId
from queue import Queue
import threading
import subprocess
from crawl_engine import crawl_engine
from pymongo import UpdateOne
from pymongo.errors import BulkWriteError
from copy import copy
# check_and_raise('elasticsearch')
# check_and_raise('mongod')
col_news = db()['news']
col_engine_instances = db()['engine_instances']
col_error_logs = db()['error_logs']
q = Queue()
thread_count = 23
error_count = 0
count = 0
origin_log_list = ''
update_one_lists = []
news_contents = 0
new_contents = 0
running = threading.Event()
news_count = 0
done = True
origin_error_count = 0
import sys
sys.path.append('/home/oem/dev/log_server')
from publics import db, set_db
from bson import ObjectId
set_db('ldb')
col_users_roles = db()['users_roles']
col_users_roles.drop()
col_users_roles.insert_many([
{
"_id": ObjectId("5e25ab698c90582c5785d291"),
'name': 'admin',
'module': 'users',
'permissions': {
'allow': ['get'],
},
},
{
'name': 'user',
'module': 'profile',
'permissions': {
'allow': ['get', 'put'],
'get': {
'user_id': '$uid'
},
'put': {
'query': {
'user_id': '$uid'
},
'set': {}
},
def before_get(self):
try:
"""
amount = number of days, 12 days for example
time = (now) or (future) or (pass)
tags = [sport, art, ...]
from = (2020-05-12) or (now)
time_point = (after) or (in) or (to)
type_date = (from_date) or (to_date)
you have too many tasks and you want get some query you want from these tasks.
these tasks have date(from_date & to_date)type and this date is with start date and end date
as called from_date(start) and to_date(end)
so maybe you want to make some query that have all tasks that started before one week later or
after one week later or exactly after week, for example you want to see all tasks stared after one week later
that have sport and art tags too.
attention in this letters = all tasks stared after one week later that have sport & art tags
it's means :
all tasks started => type_date : from_date
one week => date now + 7 days, so is => amount : 7 & from : now
one week later => date now + 7 days(date now == from:now, '+' == time:future, 7 == amount:7) ,
so is =>(amount : 7) & (from : now) & (time : future)
***one week ago => (amount : 7) & (from : now) & (time : pass('-'))***
after => time_point : after
***if from now to one week later => time_point : to***
***if exactly one week later => time_point : in***
that have sport & art tags => tags : ['sport', 'art']
in this case you have to set 'time' to the 'future' and 'time_point' to the 'after' and set 'from' to the 'now'
and 'type_date' to the 'from_date' and 'amount' equal to 7 and set 'tags' to the ['sport', 'art']
this below codes makes your query with params that you set
if you set time to the now this query show you all task for today not anything else
or all task is running in specified date (from : 2020-08-12, time : now)
"""
# date_now = datetime.strptime(str(datetime.now())[:19], "%Y-%m-%d %H:%M:%S")
queries = []
col_saved_tasks = db()['save_task_query']
for item in col_saved_tasks.find({'user_id': self.user_id}):
query = {}
query['user_id'] = self.user_id
if 'tags' in item and item['tags'] != []:
query['tags'] = {'$in': item['tags']}
if 'reminder_date' in item:
query['reminder_date'] = datetime.strptime(
item['reminder_date'], "%Y-%m-%d %H:%M:%S")
if 'from' in item and item['from'] == 'now':
date_point = datetime.strptime(
str(datetime.now())[:19], "%Y-%m-%d %H:%M:%S")
elif 'from' in item and item['from'] != 'now':
date_point = datetime.strptime(item['from'],
"%Y-%m-%d %H:%M:%S")
if 'time' in item and item['time'] != 'now':
if 'time_point' in item and item['time_point'] == 'after':
if 'type_date' in item and 'amount' in item:
if item['time'] == 'pass':
query[item['type_date']] = {
'$lte':
date_point - timedelta(days=item['amount'])
}
elif item['time'] == 'future':
query[item['type_date']] = {
'$gte':
date_point + timedelta(days=item['amount'])
}
elif 'time_point' in item and item['time_point'] == 'in':
if item['time'] == 'pass':
query[item['type_date']] = date_point - timedelta(
days=item['amount'])
elif item['time'] == 'future':
query[item['type_date']] = date_point + timedelta(
days=item['amount'])
elif 'time_point' in item and item['time_point'] == 'to':
if item['time'] == 'pass':
query[item['type_date']] =\
{'$gte': date_point, '$lte': date_point - timedelta(days=item['amount'])}
elif item['time'] == 'future':
query[item['type_date']] =\
{'$gte': date_point, '$lte': date_point + timedelta(days=item['amount'])}
elif 'time' in item and item['time'] == 'now':
query['$and'] = [{
'from_date': {
'$lte': date_point
}
}, {
'to_date': {
'$gte': date_point
}
}]
# queries['id'] = str(item['_id'])
# queries[item['name']] = query
queries.append({
'id': str(item['_id']),
'name': item['name'],
'query': query
})
print(queries)
results = []
col_tasks = db()['tasks']
for items in queries:
print(items)
result_list = []
for item in col_tasks.find(items['query']):
item['id'] = str(item['_id'])
del item['_id']
if 'create_date' in item:
item['create_date'] = str(item['create_date'])
if 'last_update' in item:
item['last_update'] = str(item['last_update'])
if 'from_date' in item:
item['from_date'] = str(item['from_date'])
if 'to_date' in item:
item['to_date'] = str(item['to_date'])
result_list.append(item)
results.append({
'id': items['id'],
'name': items['name'],
'result': result_list
})
self.set_output('public_operations', 'successful')
self.output['data']['list'] = results
except:
PrintException()
self.allow_action = False
