def before_post(self):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
col_users = db()['users']
col_user_logins = db()['user_logins']
user_info = col_users.find_one({
'mobile':
self.params['mobile'],
'password':
create_md5(self.params['password'])
})
if user_info is None:
self.set_output('user', 'login_failed')
else:
self.user_role = user_info['role']
user_info = self.after_get_one(user_info)
self.user_id = user_info['id']
if user_info['confirmed']:
self.user_id = user_info['id']
self.set_output('public_operations', 'successful')
self.token = encode_token({
'user_id': self.user_id,
'role': self.user_role
}).decode('ascii')
self.output['token'] = self.token
last_login = col_user_logins.find({
'mobile':
self.params['mobile']
}).sort('date', -1).limit(1)
if last_login.count() > 0:
user_info['last_login'] = str(last_login[0]['date'])
else:
user_info['last_login'] = ''
user_info['first_login'] = str(datetime.now())
if 'last_update' in user_info: del user_info['last_update']
if 'password_pure' in user_info:
del user_info['password_pure']
if 'password' in user_info: del user_info['password']
if '_id' in user_info: del user_info['_id']
self.output['data']['item'] = user_info
else:
self.set_output('user', 'inactive')
except:
PrintException()
self.set_output('public_operations', 'failed')
try:
col_user_logins = db()['user_logins']
col_user_logins.insert({
'user_id': self.user_id,
'mobile': self.params.get('mobile'),
'status': self.status,
'date': datetime.now(),
'notes': self.note_id
})
except:
PrintException()
self.allow_action = False
def before_post(self):
for item in self.params:
if item not in self.inputs['post']:
self.set_output('tasks', 'wrong_params')
return False
print(self.params)
try:
if 'reminder_date' in self.params:
self.params['reminder_date'] = datetime.strptime(
self.params['reminder_date'], "%Y-%m-%d %H:%M:%S")
if 'from_date' in self.params:
self.params['from_date'] = datetime.strptime(
self.params['from_date'], "%Y-%m-%d %H:%M:%S")
if 'to_date' in self.params:
self.params['to_date'] = datetime.strptime(
self.params['to_date'], "%Y-%m-%d %H:%M:%S")
self.params['is_done'] = False
self.params['is_favorite'] = False
except:
PrintException()
return False
return True
def log(type, page_url, selector, data, error, source_id, engine_instance_id):
# global error_count
global log_list
try:
# from text_reader import error_count
# error_count += 1
# col_error_logs.insert_one({
# 'engine_instance_id': str(engine_instance_id),
# 'type': type,
# 'source_id': source_id,
# 'page_url': page_url,
# 'selector': selector,
# 'data': data,
# 'date': datetime.now(),
# 'error': error,
# })
log_list.append(
InsertOne({
'engine_instance_id': str(engine_instance_id),
'type': type,
'source_id': source_id,
'page_url': page_url,
'selector': selector,
'data': data,
'date': datetime.now(),
'error': error,
}))
except:
PrintException()
def before_post(self):
try:
col_users = db()['users']
user = col_users.find_one({'mobile': self.params['mobile']}, {
'confirmed': 1,
'mobile': 1,
'activation_code': 1
})
if user is not None and user['confirmed'] is False:
# user['activation_code'] = random_digits() if 'activation_code' not in user\
# or user['activation_code'] == '' or\
# user['activation_code'] is None else user['activation_code']
send_sms(
sms['users']['registration_successful'][self.locale] %
user['activation_code'].encode('utf-8'), user['mobile'])
self.set_output('user', 'send_sms')
return True
elif user is None:
self.set_output('user', 'user_not_exists')
return False
elif user is not None and user['confirmed'] is True:
self.set_output('user', 'already_active')
return False
self.allow_action = False
return True
except:
PrintException()
return False
def before_post(self):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
col_users = self.db['users']
update_result = col_users.update(
{
'mobile': self.params['mobile'],
'activation_code': self.params['activation_code']
}, {'$set': {
'confirmed': True
}})
if update_result['nModified'] == 1:
user_info = col_users.find_one({
'mobile': self.params['mobile'],
})
# self.output['token'] = encode_token(
# {'user_id': str(user_info['_id'])}).decode()
# send_notification('welcome', str(user_info['_id']), '',
# consts.NOTIFICATIONS['users']['welcome']['title'][self.locale],
# consts.NOTIFICATIONS['users']['welcome']['description'][self.locale], user=None,
# delayed=False)
self.set_output('user', 'activate_account')
else:
self.set_output('user', 'mobile_not_activated')
self.allow_action = False
return True
except:
PrintException()
return False
def before_post(self):
try:
self.method = 'users'
col_users = db()['users']
if col_users.count({'mobile': self.params['mobile']}) > 0:
self.set_output('user', 'mobile_exists')
return False
elif self.params.get('email') is not None:
if col_users.count({'email': self.params['email']}) > 0:
self.set_output('user', 'email_exists')
return False
# if col_users.count({'device_info.mac_address': self.params['device_info']['mac_address']}) > 0:
# self.set_output('user', 'multiple_registration')
# return False
self.params['activation_code'] = random_digits()
self.params['confirmed'] = False
self.params['role'] = 'user'
# self.params['tasks_figure'] = 'line' if 'tasks_figure' not in self.params else self.params['tasks_figure']
self.params['password_pure'] = self.params['password']
self.params['password'] = create_md5(self.params['password'])
# .encode('utf-8')
except:
PrintException()
return False
return True
def get_page(url):
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
global logs_list
global error_count
try:
# print(url)
result = requests.get(url, headers=headers, verify=False)
except:
try:
result = requests.get(url, verify=False)
except:
result = ''
print('error in get page')
logs_list.append(
{'type': 'read_url', 'page_url': url, 'selector': '',
'data': {},
'error': PrintException(),
'source_link_id': '',
'engine_instance_id': engine_instance_id,
'module': 'link_grabber',
'date':datetime.now()})
error_count += 1
# log_error(type='get_url', page_url=url, selector='', data={},
# error=PrintException(), source_id='',
# source_link_id='', engine_instance_id=engine_instance_id,
# module='link_grabber')
# f = open('temp.html', 'w')
# f.write(result.text)
# f.close()
finally_result = ''
if result is not None or result != '':
finally_result = BeautifulSoup(result.text, 'html.parser')
return finally_result
def before_post(self):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
col_users = db()['users']
result = col_users.find_one_and_update(
{
'mobile': self.params['mobile'],
# 'pure_password': self.params['old_password'],
'activation_code': self.params['activation_code']
},
{
'$set': {
'password': create_md5(self.params['new_password']),
'password_pure': self.params['new_password']
}
})
if result is None:
self.set_output('user', 'password_reset_failed')
else:
self.set_output('public_operations', 'successful')
except:
PrintException()
self.set_output('public_operations', 'failed')
return False
self.allow_action = False
return True
def before_get(self):
try:
col_tasks = db()['tasks']
if 'id' in self.params:
task = col_tasks.find_one({'_id': ObjectId(self.params['id'])})
task['id'] = str(task['_id'])
task['from_date'] = str(task['from_date'])
task['to_date'] = str(task['to_date'])
del task['_id']
del task['create_date']
del task['last_update']
del task['user_id']
self.output['data']['item'] = task
else:
tasks = []
for user_task in col_tasks.find({
'user_id': str(self.user_id)
}).sort([('create_date', -1)]):
user_task['id'] = str(user_task['_id'])
del user_task['_id']
if 'from_date' in user_task:
user_task['from_date'] = str(user_task['from_date'])
if 'to_date' in user_task:
user_task['to_date'] = str(user_task['to_date'])
del user_task['create_date']
del user_task['last_update']
del user_task['user_id']
tasks.append(user_task)
self.output['data']['list'] = tasks
self.set_output('public_operations', 'successful')
except:
PrintException()
return False
self.allow_action = False
return True
def before_get(self):
try:
self.sort = {'create_date': -1}
except:
PrintException()
return False
return True
def post(self, *args, **kwargs):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
self.method = 'post'
if self.pre_post():
try:
col_users = db()['users']
activation_code = random_digits()
if 'mobile' in self.params:
user_info = col_users.find_one(
{'mobile': self.params['mobile']})
if user_info is None:
self.set_output('user', 'user_not_exists')
else:
col_users.update_one(
{'mobile': self.params['mobile']},
{'$set': {
'activation_code': activation_code
}})
send_sms(
sms['users']['forgot_password'][self.locale] %
activation_code, self.params['mobile'])
self.set_output('public_operations', 'successful')
else:
self.set_output('public_operations', 'successful')
self.after_post()
except:
PrintException()
self.set_output('public_operations', 'failed')
self.kmwrite()
def before_put(self):
try:
inputs = [
'title', 'from_date', 'to_date', 'tags', 'color',
'description', 'attachment', 'location', 'remind', 'people',
'is_done', 'is_favorite', 'id'
]
# print(self.params)
for item in self.params:
if item not in inputs:
self.set_output('tasks', 'wrong_params')
return False
if 'reminder_date' in self.params:
self.params['reminder_date'] = datetime.strptime(
self.params['reminder_date'], "%Y-%m-%d %H:%M:%S")
if 'from_date' in self.params:
self.params['from_date'] = datetime.strptime(
self.params['from_date'], "%Y-%m-%d %H:%M:%S")
if 'to_date' in self.params:
self.params['to_date'] = datetime.strptime(
self.params['to_date'], "%Y-%m-%d %H:%M:%S")
except:
PrintException()
return False
return True
def after_post(self):
try:
send_sms(
sms['users']['registration_successful'][self.locale] %
self.params['activation_code'], self.params['mobile'])
except:
PrintException()
return True
def before_get(self):
try:
print(self.params)
if 'id' in self.params:
self.conditions['_id'] = ObjectId(self.params['id'])
except:
PrintException()
return False
return True
def put(self, *args, **kwargs):
try:
self.method = 'put'
self.module = 'users'
if self.pre_put():
for item in self.params:
if item not in self.inputs['put']:
self.set_output('tasks', 'wrong_params')
return False
need_consistency_update = any(
x in self.params for x in ['pic', 'name', 'family'])
if 'password' in self.params:
self.params['password'] = create_md5(
self.params['password'])
self.params['password_pure'] = self.params['password']
col_users = db()['users']
col_users.update({'_id': ObjectId(self.user_id)},
{'$set': self.params})
if need_consistency_update:
col_people = db()['people']
col_tasks = db()['tasks']
doc = {}
if 'name' in self.params:
doc['people.name'] = self.params['name']
if 'family' in self.params:
doc['people.family'] = self.params['family']
if 'pic' in self.params:
doc['people.pic'] = self.params['pic']
col_tasks.update({'people.id': self.user_id},
{'$set': doc},
multi=True)
changes = {}
if 'name' in self.params and 'pic' not in self.params:
changes['$set'] = {'name': self.params['name']}
elif 'pic' in self.params and 'name' not in self.params:
changes['$set'] = {'pic': self.params['pic']}
elif 'pic' in self.params and 'name' in self.params:
changes['$set'] = {
'name': self.params['name'],
'pic': self.params['pic']
}
col_people.update({'user_id': self.user_id},
changes,
multi=True)
self.params['last_update'] = datetime.now()
self.set_output('public_operations', 'successful')
except:
PrintException()
self.set_output('public_operations', 'failed')
if consts.LOG_ACTIVE:
self.log_status(self.output)
self.after_put()
self.kmwrite()
def after_get(self, dataset):
temp = []
try:
print('raft tooooosh')
for item in dataset:
item['id'] = str(item['_id'])
del item['_id']
del item['create_date']
del item['last_update']
del item['user_id']
temp.append(item)
except:
PrintException()
return temp
def before_delete(self):
try:
col_tasks = db()['tasks']
if col_tasks.find_one({'_id': ObjectId(self.params['id'])}):
col_tasks.delete_one({'_id': ObjectId(self.params['id'])})
self.set_output('public_operations', 'successful')
else:
self.set_output('public_operations', 'record_not_found')
return False
except:
PrintException()
return False
self.allow_action = False
return True
def before_post(self):
try:
for item in self.params:
if item not in self.inputs:
self.set_output('tasks', 'wrong_params')
return False
col_save_task = db()['save_task_query']
if col_save_task.count_documents({'name': self.params['name']
}) > 0:
self.set_output('save_task', 'duplicate_name')
return False
except:
PrintException()
return False
return True
def before_delete(self):
try:
self.method = 'users'
print(self.params['mobile'])
print(self.user_id)
col_users = db()['users']
col_users.delete_one({
'_id':
ObjectId(self.params['id']),
'mobile':
self.params['mobile'],
'password':
create_md5(self.params['password'])
})
self.set_output('public_operations', 'successful')
except:
PrintException()
self.allow_action = False
def log_error(type, page_url, selector, data, error, source_id, source_link_id, engine_instance_id, module):
try:
global logs_list
global error_count
error_count += 1
col_error_logs.insert_one({
'engine_instance_id': str(engine_instance_id),
'type': type,
'source_id': source_id,
'source_link_id': source_link_id,
'page_url': page_url,
'selector': selector,
'data': data,
'date': datetime.now(),
'error': error,
})
except:
PrintException()
print('LOG PRODUCED LOG!')
def post(self, *args, **kwargs):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
self.method = 'post'
if self.pre_post():
self.params.update(self.added_data)
col_users = db()['users']
self.params['create_date'] = datetime.now()
self.params['last_update'] = datetime.now()
id = str(col_users.insert(self.params))
self.id = id
self.output['data']['item']['id'] = id
# self.output['token'] = encode_token({'user_id': self.id}).decode()
self.set_output('public_operations', 'successful')
self.after_post()
if consts.LOG_ACTIVE:
self.log_status(self.output)
except:
PrintException()
self.set_output('public_operations', 'failed')
self.kmwrite()
def post(self, *args, **kwargs):
data = deepcopy(output)
try:
file_contents = self.request.files['image'][0]['body']
file_name = self.request.files['image'][0]['filename']
type = self.get_argument('type', '')
file_ext = '.' + file_name.split('.')[-1]
if not os.path.exists(consts.PDP_ROOT):
os.mkdir(consts.PDP_ROOT)
if not os.path.exists(consts.PDP_IMAGES):
os.mkdir(consts.PDP_IMAGES)
if type != '':
if not os.path.exists(consts.PDP_IMAGES + type):
os.mkdir(consts.PDP_IMAGES + type)
if not os.path.exists(consts.PDP_IMAGES + type + '/' +
str(datetime.today().date())):
os.mkdir(consts.PDP_IMAGES + type + '/' +
str(datetime.today().date()))
filename = create_md5(str(datetime.now()) +
file_name) + file_ext
file = open(
'%s/%s/%s' % (consts.PDP_IMAGES + type,
datetime.today().date(), filename), 'wb')
file.write(file_contents)
file.close()
self.set_output('public_operations', 'successful')
data['data']['item'] = {
'link':
'%s/%s/%s' % (consts.ODP_IMAGES + type,
datetime.today().date(), filename)
}
print(data['data']['item'])
else:
self.set_output('field_error', 'file_type')
except Exception:
self.set_output('public_operations', 'failed')
PrintException()
self.write(data)
def do_work(item_info):
global urls_hash_list
global content_list
global logs_list
global count_mongo
global error_count
source_link = item_info['source_link']
source = item_info['source']
global count
count += 1
try:
global link_count
global new_contents
link_count += 1
print(link_count)
html = get_page(source_link['url'])
if html is not None or html != '':
# print('html')
for item in html.select(source_link['box']):
try:
if source_link['link'] == '':
href = item['href']
else:
href = item.select(source_link['link'])
href = href[0]['href'] if len(href) != 0 else ''
except:
PrintException()
href = ''
logs_list.append({'type':'extract_link', 'page_url':source_link['url'], 'selector':source_link['link'], 'data':{},
'error':PrintException(), source_id:str(source['_id']),
'source_link_id':str(source_link['_id']), 'engine_instance_id':engine_instance_id,
'module':'link_grabber',
'date':datetime.now()})
error_count += 1
# log_error(type='extract_link', page_url=source_link['url'], selector=source_link['link'], data={},
# error=PrintException(), source_id=str(source['_id']),
# source_link_id=str(source_link['_id']), engine_instance_id=engine_instance_id,
# module='link_grabber')
if href[:2] == '..': href = href.replace('..', '')
# if href != '' and href[0] != '/': href = '/'+href
# if col_news.count_documents({'url': source_link['base_url'] + item.select(source_link['link'])[0]['href']}) == 0:
# col_counter = col_news.count_documents({'url': source_link['base_url'] + href})
# count_mongo += 1
# if col_counter == 0:
try:
if source_link['base_url'] not in href:
url = source_link['base_url'] + href
else:
url = href
except:
PrintException()
url = ''
logs_list.append(
{'type': 'read_url', 'page_url': source_link['url'], 'selector': '',
'data': {},
'error': PrintException(), source_id: str(source['_id']),
'source_link_id': str(source_link['_id']), 'engine_instance_id': engine_instance_id,
'module': 'link_grabber',
'date':datetime.now()})
error_count += 1
# log_error(type='read_url', page_url=source_link['url'], selector='', data={},
# error=PrintException(), source_id=str(source['_id']),
# source_link_id=str(source_link['_id']), engine_instance_id=engine_instance_id, module='link_grabber')
if source_link['title'] != '':
try:
title = item.select(source_link['title'])
# if title is None or title != '':
# print('empty title')
title = title[0].text.strip()
# if len(title) != 0 else ''
except:
try:
# print('title')
title = item.select('div.box_economic > h3 > a')[0].text
except:
try:
title = item.select('div.box_economic > h1 > a')[0].text
except:
# print('title nashod')
PrintException()
title = ''
logs_list.append(
{'type': 'extract_title', 'page_url': source_link['url'], 'selector': source_link['title'],
'data': {},
'error': PrintException(), source_id: str(source['_id']),
'source_link_id': str(source_link['_id']),
'engine_instance_id': engine_instance_id,
'module': 'link_grabber',
'date':datetime.now()})
error_count += 1
else:
title = ''
# log_error(type='extract_title', page_url=source_link['url'], selector=source_link['title'], data={},
# error=PrintException(), source_id=str(source['_id']),
# source_link_id=str(source_link['_id']), engine_instance_id=engine_instance_id, module='link_grabber')
if source_link['summary'] != '':
try:
summary = item.select(source_link['summary'])
summary = summary[0].text.strip()
# if len(summary) != 0 else ''
except:
# print('summary error')
PrintException()
summary = ''
logs_list.append(
{'type': 'extract_summary', 'page_url': source_link['url'], 'selector': source_link['summary'],
'data': {},
'error': PrintException(), source_id: str(source['_id']),
'source_link_id': str(source_link['_id']),
'engine_instance_id': engine_instance_id,
'module': 'link_grabber',
'date':datetime.now()})
error_count += 1
else:
summary = ''
# log_error(type='extract_summary', page_url=source_link['url'], selector=source_link['summary'],
# data={}, error=PrintException(), source_id=str(source['_id']),
# source_link_id=str(source_link['_id']), engine_instance_id=engine_instance_id, module='link_grabber')
if source_link['date'] != '':
try:
date = item.select(source_link['date'])
date = date[0].text.strip()
# if len(date) != 0 else ''
except:
PrintException()
date = ''
logs_list.append(
{'type': 'extract_date', 'page_url': source_link['url'], 'selector': source_link['date'],
'data': {},
'error': PrintException(), source_id: str(source['_id']),
'source_link_id': str(source_link['_id']),
'engine_instance_id': engine_instance_id,
'module': 'link_grabber',
'date':datetime.now()})
error_count += 1
else:
# print('date')
date = ''
# log_error(type='extract_date', page_url=source_link['url'], selector=source_link['date'], data={},
# error=PrintException(), source_id=str(source['_id']),
# source_link_id=str(source_link['_id']), engine_instance_id=engine_instance_id, module='link_grabber')
if source_link['image'] != '':
try:
selected = item.select(source_link['image'])
try:
image = selected[0]['data-src']
# if len(selected) != 0 else ''
except:
image = selected[0]['src']
if 'http' not in image:
if image != '' and image[0] != '/': image = '/'+image
image = source_link['base_url']+image
except:
PrintException()
image = ''
logs_list.append(
{'type': 'extract_image', 'page_url': source_link['url'], 'selector': source_link['image'],
'data': {},
'error': PrintException(), source_id: str(source['_id']),
'source_link_id': str(source_link['_id']),
'engine_instance_id': engine_instance_id,
'module': 'link_grabber',
'date':datetime.now()})
error_count += 1
else:
image = ''
# log_error(type='extract_image', page_url=source_link['url'], selector=source_link['image'],
# data={}, error=PrintException(), source_id=str(source['_id']),
# source_link_id=str(source_link['_id']), engine_instance_id=engine_instance_id, module='link_grabber')
url_hash = create_md5(url)
urls_hash_list.append(url_hash)
new_contents += 1
# if col_news.count_documents({'url_hash': url_hash}) == 0:
content_list.append({
'source_id': str(source['_id']),
'link_grabber_id': engine_instance_id,
'source_link_id': str(source_link['_id']),
'source_name': source['name'],
'create_date': datetime.now(),
'last_update': datetime.now(),
'url': url.encode('utf-8'),
'url_hash': url_hash,
'title': title.encode('utf-8'),
'summary': summary.encode('utf-8'),
'date': date,
'source_url': source_link['url'],
'status': 'summary',
'image': image,
'text_selector': source_link['text'],
'category_id': source_link['category_id'] if 'category_id' in source_link else '',
'text': '',
'html': '',
})
else:
print(html)
logs_list.append(
{'type': 'get html', 'page_url': source_link['url'], 'selector': source_link['image'],
'data': {},
'error': PrintException(), source_id: str(source['_id']),
'source_link_id': str(source_link['_id']),
'engine_instance_id': engine_instance_id,
'module': 'link_grabber',
'date':datetime.now()})
except Exception as e:
# print(e)
logs_list.append(
{'type': 'read_url', 'page_url': source_link['url'], 'selector': '',
'data': {},
'error': PrintException(), source_id: str(source['_id']),
'source_link_id': str(source_link['_id']),
'engine_instance_id': engine_instance_id,
'module': 'link_grabber',
'date':datetime.now()})
try:
# con= MongoClient()
# db = con['ldb']
col_logs = db()['logs']
logs_list = []
ids_list = []
for item in col_logs.find({'sent': {'$exists': False}}).limit(20):
ids_list.append(item['_id'])
del item['_id']
item['date'] = str(item['date'])
logs_list.append(item)
# ADMIN_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyX2lkIjoiNWRiZmQxMjAxNzQ5ZWYxYjNiMGEwMjFkIiwicm9sZSI6ImFkbWluIiwiZGF0ZSI6IjIwMTktMTEtMDQgMTE6MTA6MDguMjg4MDk1In0.AtVoRKDTv1jvcHGpRmuLJvD9gG8sWqdR6P-Ibc6WDsU'
ADMIN_TOKEN = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyX2lkIjoiNWRjZDU4MDZiZWU4YTZhODFjMzE0MDhkIiwicm9sZSI6ImFkbWluIiwiZGF0ZSI6IjIwMTktMTEtMTQgMTc6MDg6NDMuNDAzNDA4In0.QUCCJHXYcSMA-pyHFCQn8zBiHS7C48MpoFAS1T9JUWk'
logs = dumps(logs_list)
params = {'params': logs, 'token': ADMIN_TOKEN}
par = loads(params['params'])
print(par[0])
requests.post("http://localhost:8585/v1/collect", json=params)
col_logs.update_many({'_id': {
'$in': ids_list
}}, {'$set': {
'sent': True
}})
except:
PrintException()
except:
PrintException()
def before_get(self):
try:
"""
amount = number of days, 12 days for example
time = (now) or (future) or (pass)
tags = [sport, art, ...]
from = (2020-05-12) or (now)
time_point = (after) or (in) or (to)
type_date = (from_date) or (to_date)
you have too many tasks and you want get some query you want from these tasks.
these tasks have date(from_date & to_date)type and this date is with start date and end date
as called from_date(start) and to_date(end)
so maybe you want to make some query that have all tasks that started before one week later or
after one week later or exactly after week, for example you want to see all tasks stared after one week later
that have sport and art tags too.
attention in this letters = all tasks stared after one week later that have sport & art tags
it's means :
all tasks started => type_date : from_date
one week => date now + 7 days, so is => amount : 7 & from : now
one week later => date now + 7 days(date now == from:now, '+' == time:future, 7 == amount:7) ,
so is =>(amount : 7) & (from : now) & (time : future)
***one week ago => (amount : 7) & (from : now) & (time : pass('-'))***
after => time_point : after
***if from now to one week later => time_point : to***
***if exactly one week later => time_point : in***
that have sport & art tags => tags : ['sport', 'art']
in this case you have to set 'time' to the 'future' and 'time_point' to the 'after' and set 'from' to the 'now'
and 'type_date' to the 'from_date' and 'amount' equal to 7 and set 'tags' to the ['sport', 'art']
this below codes makes your query with params that you set
if you set time to the now this query show you all task for today not anything else
or all task is running in specified date (from : 2020-08-12, time : now)
"""
# date_now = datetime.strptime(str(datetime.now())[:19], "%Y-%m-%d %H:%M:%S")
queries = []
col_saved_tasks = db()['save_task_query']
for item in col_saved_tasks.find({'user_id': self.user_id}):
query = {}
query['user_id'] = self.user_id
if 'tags' in item and item['tags'] != []:
query['tags'] = {'$in': item['tags']}
if 'reminder_date' in item:
query['reminder_date'] = datetime.strptime(
item['reminder_date'], "%Y-%m-%d %H:%M:%S")
if 'from' in item and item['from'] == 'now':
date_point = datetime.strptime(
str(datetime.now())[:19], "%Y-%m-%d %H:%M:%S")
elif 'from' in item and item['from'] != 'now':
date_point = datetime.strptime(item['from'],
"%Y-%m-%d %H:%M:%S")
if 'time' in item and item['time'] != 'now':
if 'time_point' in item and item['time_point'] == 'after':
if 'type_date' in item and 'amount' in item:
if item['time'] == 'pass':
query[item['type_date']] = {
'$lte':
date_point - timedelta(days=item['amount'])
}
elif item['time'] == 'future':
query[item['type_date']] = {
'$gte':
date_point + timedelta(days=item['amount'])
}
elif 'time_point' in item and item['time_point'] == 'in':
if item['time'] == 'pass':
query[item['type_date']] = date_point - timedelta(
days=item['amount'])
elif item['time'] == 'future':
query[item['type_date']] = date_point + timedelta(
days=item['amount'])
elif 'time_point' in item and item['time_point'] == 'to':
if item['time'] == 'pass':
query[item['type_date']] =\
{'$gte': date_point, '$lte': date_point - timedelta(days=item['amount'])}
elif item['time'] == 'future':
query[item['type_date']] =\
{'$gte': date_point, '$lte': date_point + timedelta(days=item['amount'])}
elif 'time' in item and item['time'] == 'now':
query['$and'] = [{
'from_date': {
'$lte': date_point
}
}, {
'to_date': {
'$gte': date_point
}
}]
# queries['id'] = str(item['_id'])
# queries[item['name']] = query
queries.append({
'id': str(item['_id']),
'name': item['name'],
'query': query
})
print(queries)
results = []
col_tasks = db()['tasks']
for items in queries:
print(items)
result_list = []
for item in col_tasks.find(items['query']):
item['id'] = str(item['_id'])
del item['_id']
if 'create_date' in item:
item['create_date'] = str(item['create_date'])
if 'last_update' in item:
item['last_update'] = str(item['last_update'])
if 'from_date' in item:
item['from_date'] = str(item['from_date'])
if 'to_date' in item:
item['to_date'] = str(item['to_date'])
result_list.append(item)
results.append({
'id': items['id'],
'name': items['name'],
'result': result_list
})
self.set_output('public_operations', 'successful')
self.output['data']['list'] = results
except:
PrintException()
self.allow_action = False
def get(self, id=None, *args, **kwargs):
self.Print('%s fired' % inspect.stack()[0][3], Colors.GRAY)
try:
self.method = 'get'
if self.pre_get():
try:
if len(self.fields) > 0:
fields = {}
for item in self.fields:
fields[item] = 1
else:
fields = {
'name': 1,
'family': 1,
'email': 1,
'pic': 1,
'tasks_figure': 1,
'mobile': 1
}
col_users = db()['users']
# print('-----------------------------')
# print(self.user_id)
# print(self.fields)
# print('-----------------------------')
user_info = col_users.find_one(
{'_id': ObjectId(self.user_id)}, fields)
user_info['id'] = str(user_info['_id'])
del user_info['_id']
col_tasks = db()['tasks']
user_tasks = []
# for item in col_tasks.find({'user_id': {'$in': [self.user_id]}}):
# del item['last_update']
# del item['create_date']
# del item['user_id']
# # item['create_date'] = str(item['create_date'])
# # item['last_update'] = str(item['last_update'])
# if 'from_date' in item:
# item['from_date'] = str(item['from_date'])
# if 'to_date' in item:
# item['to_date'] = str(item['to_date'])
# item['id'] = str(item['_id'])
# del item['_id']
# user_tasks.append(item)
user_people = []
col_people = db()['people']
for item in col_people.find(
{'user_id': {
'$in': [self.user_id]
}}):
item['id'] = str(item['_id'])
del item['_id']
del item['create_date']
del item['last_update']
del item['user_id']
user_people.append(item)
self.output['data']['item'] = user_info
# self.output['data']['item']['tasks'] = user_tasks
# self.output['data']['item']['people'] = user_people
self.set_output('public_operations', 'successful')
except:
PrintException()
self.set_output('field_error', 'id_format')
except:
PrintException()
self.set_output('public_operations', 'failed')
self.kmwrite()
def crawl_engine(item, engine_instance_id):
global log_list
log_list = []
news_html = None
# print('inside crawl')
# item['mongo_id'] = str(item['_id'])
# del item['_id']
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
}
# headers = {
# 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36',
# }
status = ''
news_text = ''
# print('before req')
# , headers = headers, verify = False
try:
# print('get')
# print(item['url'])
result = requests.get(url=item['url'], headers=headers,
verify=False).status_code
# print('after get')
except:
# print('inja')
# print(e)
try:
# print('get sec')
result = requests.get(item['url'], verify=False)
except:
status = 'error_get_url'
result = ''
log(type='get_url',
page_url=item['url'],
selector='',
data={},
error=PrintException(),
engine_instance_id=engine_instance_id,
source_id=item['source_id'])
# print('after req')
if (result != '') or (result is not None):
try:
# print('bs4')
html = BeautifulSoup(result.text, 'html.parser')
except Exception as e:
html = ''
status = 'error_bs4'
try:
# print('select')
news_html = html.select(item['text_selector'])
if len(news_html) > 0:
news_html = news_html[0]
try:
# print('remove')
news_html = remove_hrefs(news_html)
except:
status = 'error_remove_href'
status = 'text'
source_link_info = col_source_links.find_one(
{'_id': ObjectId(item['source_link_id'])})
if 'exclude' in source_link_info:
for exclude in source_link_info['exclude']:
for ex in news_html.select(exclude):
ex.decompose()
else:
status = 'Empty'
log(type='read_text',
page_url=item['url'],
selector=item['text_selector'],
data={},
error='Empty',
engine_instance_id=engine_instance_id,
source_id=item['source_id'])
except Exception as e:
news_html = ''
status = 'error_selector'
log(type='read_text',
page_url=item['url'],
selector=item['text_selector'],
data={},
error=str(e),
engine_instance_id=engine_instance_id,
source_id=item['source_id'])
# col_news.update_one({'_id': ObjectId(item['mongo_id'])}, {'$set': {
# 'status': 'error',
# 'text': '',
# 'html': '',
# 'error': str(e),
# }})
if news_html is not None and news_html != '' and len(news_html) > 0:
try:
# print('text')
news_text = news_html.text
except:
news_text = ''
status = 'error_text'
log(type='read_text',
page_url=item['url'],
selector=item['text_selector'],
data={},
error=PrintException(),
engine_instance_id=engine_instance_id,
source_id=item['source_id'])
return status, news_text, news_html, log_list, error_count
