def run_keywords_import(import_file, import_file_ok):
if not import_file.exists():
return
import_file_already_imported = import_file_ok.exists()
if import_file_already_imported:
return
with open(str(import_file)) as import_file_fp:
json_data = json.load(import_file_fp)
targets = []
for entry in json_data:
for keyword in entry['keywords']:
target = Target(keyword=keyword,
state=entry['state'],
state_region=entry['state_region'],
region_subregion=entry['region_subregion'],
city=entry['city'],
category=entry['category'],
is_new=True)
targets.append(target)
try:
session.add_all(targets)
session.commit()
import_file_ok.touch()
except:
logging.exception(f'Failure to import {str(import_file)}')
def song_post():
""" post a song """
headers = {
"Location": url_for("song_post"),
"Content-Type": "application/json"
}
######### get the data from the form
data = request.json
post_song = models.Song(
song_name=data["song_name"],
id=data["song_id"]) ## ask sam if we really need to post seperately.
post_file = models.File(song_id=data["song_id"],
file_name=data["file_name"],
id=data["file_id"])
if not session.query(models.Song).get(
post_song.id
): #consider adding a check here for duplicate fileID too
session.add_all([post_song, post_file])
session.commit()
else:
print "************* ELSE ***************"
session.rollback()
session.flush()
return Response(json.dumps(
{"status": "failed - that song already exists"}),
500,
mimetype="application/json")
return Response(stripUnicode(data),
200,
headers=headers,
mimetype="application/json")
def _all(cls, *instances):
try:
session.add_all(instances)
session.commit()
return True
except:
warnings.warn(f'Add fails: instances={instances}')
session.rollback()
return False
def log_individuals():
timestamp = Timestamp(timestamp=time.time())
user_data = get_user_data()
if not user_data:
return
formatted_data = [(User(name=name), custom_rep[status]) for name, status in user_data if status != 'offline']
new_users = [user for user, status in formatted_data if user.name not in user_store]
session.add_all(new_users)
session.add(timestamp)
session.commit()
session.flush()
user_store.update({user.name: user.ROWID for user in new_users})
datapoints = [Datapoint(user_id=user_store[user.name], timestamp_id=timestamp.ROWID, status=status)
for user, status in formatted_data]
session.add_all(datapoints)
session.commit()
def create_user_profiles_from_invite(args):
user_list = []
for key, value in ast.literal_eval(args['user_list']).items():
if validate_user_profile_by_email(email=value['email']):
return None
user_list.append(
UserProfile(email=value['email'],
role_id=value['role'],
password=generate_password(),
company_id=args['company_id']))
try:
session.add_all(user_list)
session.commit()
except Exception as e:
session.rollback()
else:
for user in user_list:
send_mail(user.email, user.password)
return user_list
def song_post():
""" post a song """
headers = {"Location": url_for("song_post"),"Content-Type": "application/json"}
######### get the data from the form
data = request.json
post_song = models.Song(song_name=data["song_name"],id=data["song_id"]) ## ask sam if we really need to post seperately.
post_file = models.File(song_id=data["song_id"],file_name=data["file_name"],id=data["file_id"])
if not session.query(models.Song).get(post_song.id): #consider adding a check here for duplicate fileID too
session.add_all([post_song,post_file])
session.commit()
else:
print "************* ELSE ***************"
session.rollback()
session.flush()
return Response(json.dumps({"status":"failed - that song already exists"}),500, mimetype="application/json")
return Response(stripUnicode(data), 200, headers=headers, mimetype="application/json")
Base.metadata.create_all(engine)
if __name__ == '__main__':
from database import session
from sqlalchemy import desc
beyonce = User(username='******', password='******')
christina = User(username='******', password='******')
britney = User(username='******', password='******')
microphone = Item(name="microphone",
description="Mika's gold microphone.",
owner=britney)
bids = [
Bid(price=100.0, bidder=christina, item=microphone),
Bid(price=150.0, bidder=beyonce, item=microphone),
Bid(price=200.0, bidder=christina, item=microphone),
Bid(price=201.0, bidder=beyonce, item=microphone),
]
session.add_all([beyonce, christina, britney, microphone] + bids)
session.commit()
highest_bid = (session.query(Bid)
.filter(Bid.item==microphone)
.order_by(desc(Bid.price))
.first())
print(highest_bid.bidder.username)
requirement_list.append(
Requirement(technology_id=1, vacancy_id=1, experience_id=1))
requirement_list.append(
Requirement(technology_id=4, vacancy_id=2, experience_id=4))
requirement_list.append(
Requirement(technology_id=2, vacancy_id=3, experience_id=2))
requirement_list.append(
Requirement(technology_id=3, vacancy_id=4, experience_id=3))
technology_list = list()
technology_list.append(Technology(name='html', category_id=1))
technology_list.append(Technology(name='python', category_id=1))
technology_list.append(Technology(name='js', category_id=1))
technology_list.append(Technology(name='flask', category_id=2))
technology_list.append(Technology(name='react', category_id=2))
technology_list.append(Technology(name='django', category_id=2))
technology_list.append(Technology(name='docker', category_id=3))
tables_list = [
category_list, experience_list, company_range_list, profession_list,
role_list, company_list, userprofile_list, purchases_list, employee_list,
vacancy_list, technology_list, competence_list, requirement_list
]
try:
for table in tables_list:
session.add_all(table)
session.commit()
except Exception as e:
session.rollback()
def push(self):
session.add_all(self.nodes)
session.commit()
def push(self):
session.add_all(self.nodes)
session.commit()
def scrape_contacts(group, phone_number=None):
if phone_number is None:
free_accounts = session.query(TelegramAccount).filter(
TelegramAccount.active == True,
TelegramAccount.task == None).all()
account = random.choice(free_accounts)
if not account:
active_accounts = session.query(TelegramAccount).filter(
TelegramAccount.active == True).all()
account = random.choice(active_accounts)
if not account:
for adm in config.ADMIN_IDS:
bot.send_message(
adm, f'No available accounts for scrapping. '
f'Add an account at first.')
return
else:
account = session.query(TelegramAccount).filter(
TelegramAccount.phone_number == phone_number, ).first()
try:
proxy = session.query(Proxy).first()
client = TelegramClient(os.path.join(config.TELETHON_SESSIONS_DIR,
account.phone_number),
config.TELEGRAM_API_ID,
config.TELEGRAM_API_HASH,
proxy=(socks.HTTP, proxy.ip, proxy.port, True,
proxy.username, proxy.password))
client.connect()
if isinstance(group, int):
group_link = group
else:
group_link = group.lower()
account_id = client.get_me().id
group = client.get_entity(group_link)
participants = client.get_participants(group, aggressive=True)
last_messages = client.get_messages(group, 1000)
last_active_users_ids = set([msg.from_id for msg in last_messages])
if client.get_me().id not in [i.id for i in participants]:
client(JoinChannelRequest(group))
channel_admins = client.get_participants(
group, filter=ChannelParticipantsAdmins)
admins_list = list()
for i in channel_admins:
admins_list.append(i)
admins_ids = [i.id for i in admins_list]
client.disconnect()
filtered_participants = [
p for p in list(participants) if not p.bot
and p.id not in admins_ids and p.id != account_id and p.username
]
contacts = [
Contact(tg_id=user.id,
source_group=group_link,
username=user.username,
source_group_name=group.title,
priority=Contact.PRIORITY_HIGH if user.id
in last_active_users_ids else Contact.PRIORITY_LOW)
for user in filtered_participants
]
session.add_all(contacts)
session.commit()
for adm in config.ADMIN_IDS:
bot.send_message(
adm,
f'Scrapped {len(filtered_participants)} from {group.title}.\n'
f'Skipped {abs(len(filtered_participants)-len(participants))} '
f'admins, bots and users without usernames.')
except Exception as e:
for adm in config.ADMIN_IDS:
bot.send_message(adm, str(e))
