Example #1
0
def load_profiles():
    """ Load the user profiles into the DB """

    print "Profiles"

    # Delete all rows in table, so if we need to run this a second time,
    # we won't be trying to add duplicate users
    Profile.query.delete()

    foodie = Profile(type_name="Foodie",
                     type_taste=['spicy', 'salty', 'sweet', 'umami'])
    starving_student = Profile(type_name="Starving Student", type_price=['1'])
    health_nut = Profile(type_name="Health Nut",
                         type_diet=['vegan', 'gluten_free'])
    heat_seeker = Profile(type_name="Heat Seeker",
                          type_taste=['spicy'],
                          type_temp=['hot'])
    garbage_disposal = Profile(type_name="Garbage Disposal")

    # We need to add to the session or it won't ever be stored
    db.session.add(foodie)
    db.session.add(starving_student)
    db.session.add(health_nut)
    db.session.add(heat_seeker)
    db.session.add(garbage_disposal)

    # Once we're done, we should commit our work
    db.session.commit()
Example #2
0
def main():

    start_mapper()

    # create all tables
    metadata.create_all(bind=engine)

    # initiate repository
    user_repo = SqlRepository(session)

    user = User("ed", "secret")
    profile = Profile("John", "Doe", "*****@*****.**")

    user.profile = profile

    # add user and profile to database
    user_repo.add(user)

    # save
    session.commit()

    # query database
    user = user_repo.get(username="******")

    print(user, user.profile)
Example #3
0
def profil_actions():
    act = request.args.get('act')
    p_id = request.args.get('p_id')
    try:
        if act == 'active':
            schedule_job(p_id)
            return resp_format.from_dict(resp_format.MSG_OK,
                                         msg='Profile successfully activated')
        elif act == 'pause':
            unschedule_job(p_id)
            return resp_format.from_dict(resp_format.MSG_OK,
                                         msg='Profile successfully paused')
        elif act == 'delete':
            unschedule_job(p_id)
            Profile().delete_by_pk(p_id)
            return resp_format.from_dict(resp_format.MSG_OK,
                                         msg='Profile hase been deleted')
        elif act == 'run':
            profile_execute(p_id)
            return resp_format.from_dict(resp_format.MSG_OK,
                                         msg='Job completed successfully')

    except Error.ProfileException as e:
        return resp_format.from_dict(resp_format.MSG_FAIL, msg=str(e))
    except Exception, e:
        return resp_format.from_dict(resp_format.MSG_FAIL, msg=str(e))
Example #4
0
def get_settings() -> Profile:
    """Gets the user settings profile for the application, or exits on failure.

    :return: The user settings profile.
    """

    settings_path: str = "~/lecture-hoarder-settings.yaml"
    if len(sys.argv) > 1:
        settings_path = sys.argv[
            1]  # User has specified custom settings file location

    settings: Profile = Profile()
    try:
        settings.load_from_file(settings_path)
    except IOError:
        # Could not open settings file, use default values
        print("Using default settings")
    except YAMLError as err:
        # Parser error occurred
        if hasattr(err, "context_mark") and hasattr(err, "problem"):
            print(
                f"Could not parse settings file - bad syntax at line {err.context_mark.line + 1} char "
                f"{err.context_mark.column + 1}: {err.problem}")
        else:
            print("Could not parse settings file")
        sys.exit(2)
    except TypeError as err:
        # Syntax fine, bad param type
        print(f"Could not load settings file - {err}")
        sys.exit(2)

    return settings
Example #5
0
def profile():
    if request.method == 'GET':
        return profile_list()

    try:
        s = {}
        d = {}
        o = {}
        for k, v in request.form.iteritems():
            if k.startswith('src_'):
                s[k] = v
            elif k.startswith('dst_'):
                d[k] = v
            elif k.startswith('opt_'):
                o[k] = v

        p_id = Profile().create(s, d, o)
        schedule_job(p_id)

        return resp_format.from_dict(resp_format.MSG_OK,
                                     msg='Profile successfully created')
    except Error.ProfileException as e:
        return resp_format.from_dict(resp_format.MSG_FAIL, msg=str(e))
    except Error.TestConfigException as e:
        return resp_format.from_dict(resp_format.MSG_FAIL, msg=str(e))
Example #6
0
def aps_listener(event):
    text = ''

    if event.exception:
        rval = Profile().find_by_pk(int(event.job.name.replace('wj_',
                                                               '')))['profile']
        data = {
            'type': 'error',
            'event': 'scheduled_run',
            'created_at': event.scheduled_run_time,
            'u_id': rval.get('u_id'),
            'j_id': rval.get('id')
        }
        text = '"%s" was failed due to %s' % (rval['title'], event.exception)
        data['text'] = text
    else:
        rval = event.retval
        data = {
            'type': 'success',
            'event': 'scheduled_run',
            'created_at': event.scheduled_run_time,
            'u_id': rval.get('u_id'),
            'j_id': rval.get('id')
        }
        text = '"%s" has been completed successfully' % rval['title']
        data['text'] = text

    q = EventlogModel.insert(**data)
    q.execute()
Example #7
0
def addprofile():
    """ process post request to add new profile to db"""
    name = request.form.get('name')
    birthdate = request.form.get('birthdate')
    gender = request.form.get('gender')

    gender_bool = False

    if gender == 'Male':
        gender_bool = False
    elif gender == 'Female':
        gender_bool = True

    new_profile = Profile(name=name,
                          birthdate=birthdate,
                          gender=gender_bool,
                          user_id=session.get('userid'))
    db.session.add(new_profile)
    db.session.commit()
    flash("New profile {} has been created".format(name))

    profile_obj = Profile.query.filter(Profile.name == name).filter(
        Profile.user_id == session.get('userid')).first()

    return redirect('/profile/{}/{}'.format(session['userid'],
                                            profile_obj.profile_id))
Example #8
0
def profile():
    my_form = ProfileForm()  #初始化表单
    my_data = Profile(
    )  #初始化数据,没这行,数据无法放在form中,给该表单提交的数据起名为my_data,即first_name那列数据就叫my_data.first_name
    my_data.remove_none_values()  #call the function
    if my_form.validate_on_submit():  #意思是如果表单提交成功
        my_data.first_name = request.form.get('first_name')
        my_data.last_name = request.form.get('last_name')
        #print("first_name", my_data.first_name)
        #print("last_name", my_data.last_name)
        file = request.files.get('file_photo')
        if file:
            orig_filename = secure_filename(file.filename)
            new_filename = str(uuid.uuid1())  #生成uuid
            my_data.file_photo_filename = orig_filename  #To save the orignal file name
            my_data.file_photo_code = new_filename
            #上面这行是为了存储uuid,目的是如果不同用户上传的不用文件,起了同一个名,靠uuid来区分

            # save to upload folder
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], new_filename))

            #save to database
            db.session.add(my_data)
            db.session.commit()
            print("my_data", my_data.id)

            #redirect to display page
            return redirect('/profile/' +
                            str(my_data.id))  #这个意思是每个数据的特定URL,比如profile/5...

    return render_template("profile.html", my_form=my_form, my_data=my_data)
Example #9
0
def create_plant_profile(user_id, plant_id):
    """Create and return a new plant greenhouse for user."""

    plant_profile = Profile(user_id=user_id, plant_id=plant_id)

    db.session.add(plant_profile)
    db.session.commit()

    return plant_profile
Example #10
0
def load_profile():
    Profile.query.delete()

    kellie1 = Profile(user_id=1,
                      name="Matthew",
                      birthdate="2016-08-06",
                      gender=False)

    db.session.add(kellie1)

    db.session.commit()
Example #11
0
    def create(name):
        currentProfiles = session.query(Profile).all()
        #From the Class ProfileSchema
        schema = ProfileSchema(many=True)
        profiles = schema.dump(currentProfiles).data

        for p in json.loads(json.dumps(profiles)):
            print(p.get('name'))

        profile = Profile(name)
        session.add(profile)
        session.commit()
Example #12
0
def scrape_instagram_account_by_id(upstream_id, stub=False):
    """ Scrape instagram bio data for upstream ID and update a profile. """

    db_session = worker.get_session()
    proxies = _get_proxies(db_session)

    # Instagram API request.
    api_url = 'https://api.instagram.com/v1/users/{}'.format(upstream_id)

    response = requests.get(
        api_url,
        proxies=proxies,
        verify=False
    )

    response.raise_for_status()
    data = response.json()['data']

    # Update the profile.
    data = response.json()[0]
    profile = Profile('instagram', upstream_id, data['screen_name'])
    db_session.add(profile)

    try:
        db_session.commit()
    except IntegrityError:
        # Already exists: use the existing profile.
        db_session.rollback()
        profile = db_session.query(Profile) \
                            .filter(Profile.site == 'instagram') \
                            .filter(Profile.upstream_id == upstream_id) \
                            .one()

    # Update profile
    profile.last_update = datetime.now()
    profile.description = data['bio']
    profile.follower_count = int(data['counts']['followed_by'])
    profile.friend_count = int(data['counts']['follows'])
    profile.homepage = data['website']
    profile.name = data['full_name']
    profile.post_count = int(data['counts']['media'])
    profile.is_stub = stub
    db_session.commit()

    # Schedule followup jobs.
    app.queue.schedule_index_profile(profile) # index all profiles, inc stubs
    if not stub:
        app.queue.schedule_avatar(profile, data['profile_picture'])
        app.queue.schedule_posts(profile, recent=True)
        app.queue.schedule_relations(profile)

    return profile.as_dict()
    def add_new(self, user_id, email, name, password, token):
        new_profile = Profile(
            user_id=user_id,
            email=email,
            is_logged=True,
            name=name,
            password=password,
            token=token,
        )

        db_session.add(new_profile)
        db_session.commit()

        return new_profile
Example #14
0
def addUser():
    user=request.get_json()
    pro=Profile(user['username'],user['email'],user['firstname'],user['lastname'],user['password'])
    db.session.add(pro)
    try:
        db.session.commit()      
    except Exception as e:
        db.session.rollback()
        logger.error("cannot insert object"+str(e))
        return jsonify("object not validated,"+str(e))
    finally:
        db.session.close()

    return jsonify("objects added to db success")
Example #15
0
def create_profile(about, experience, skill, project, education, contact,
                   user_id):
    """Create and return a new profile."""

    profile = Profile(about=about,
                      experience=experience,
                      skill=skill,
                      project=project,
                      education=education,
                      contact=contact,
                      user_id=user_id)

    db.session.add(profile)
    db.session.commit()

    return profile
Example #16
0
def profile_update():
    """Profile updation process"""

    first_name = request.form.get("Firstname")
    last_name = request.form.get("Lastname")
    display_name = request.form.get("Displayname")
    email = request.form.get("Email")
    phonenumber = request.form.get("Phonenumber")
    date_of_birth_str = request.form.get("Birthday")
    date_of_birth = datetime.strptime(date_of_birth_str, '%Y-%m-%d')
    address_1 = request.form.get("Address1")
    address_2 = request.form.get("Address2")
    city = request.form.get("City")
    state = request.form.get("State")
    zipcode = request.form.get("Zipcode")
    country = request.form.get("Country")
    married = request.form.get("Marital_Status")
    marriage_date_str = request.form.get("Marriage_Anniversary")
    marriage_date = datetime.strptime(marriage_date_str, '%Y-%m-%d')
    kids = int(request.form.get("Kids"))
    userid = session["user_id"]

    user_profile = Profile(display_name=display_name,
                           user_id=userid,
                           email=email,
                           phonenumber=phonenumber,
                           date_of_birth=date_of_birth,
                           address_1=address_1,
                           address_2=address_2,
                           city=city,
                           state=state,
                           zipcode=zipcode,
                           country=country,
                           married=married,
                           marriage_date=marriage_date,
                           kids=kids)
    db.session.add(user_profile)
    db.session.commit()
    text = f"Birthday for {first_name} {last_name}"
    user_event = Event(profile_id=user_profile.profile_id,
                       event_type='Birthday',
                       event_date=date_of_birth,
                       event_text=text)
    db.session.add(user_event)
    db.session.commit()

    return redirect("/homepage")
Example #17
0
def create_profile(user,
                   description,
                   instagram,
                   twitter,
                   website,
                   profile_photo="https://picsum.photos/500/350?nocache"):

    profile = Profile(user=user,
                      description=description,
                      instagram=instagram,
                      twitter=twitter,
                      website=website,
                      profile_photo=profile_photo)

    db.session.add(profile)
    db.session.commit()
    return profile
Example #18
0
def register():
    form = CreateProfileForm(request.form)

    print "###NAME{}".format(form.firstname.data)
    if request.method == 'POST':#and form.validate():
        print "###############################################################"
        profile = Profile(form.firstname.data, form.lastname.data, form.username.data, form.gender.data, form.age.data, form.bio.data, form.password.data, datetime.utcnow())
        print "LETS see{}".format(profile)
        print "###############################################################"
        print form.firstname.data
        db.session.add(profile)
        db.session.commit()
        file = request.files['file']
        filename = file.filename#form.username.data#file.filename
        file.save(os.path.join("app/static/profilepics", filename))
        flash('Thank you for registering')
        return redirect(url_for('login'))
    return render_template('register.html',form=form)
Example #19
0
 def import_from_lines(self, lines):
     """
         从多行数据录入数据库
         :param lines: 待处理数据
         :return:  读取到的记录数量,成功录入的记录数量
         """
     (total, succ) = (0, 0)
     for line in lines:
         total += 1
         items = line.decode('utf-8').strip().split('\t')  # 数据库接收的是 unicode,所以要解码
         if len(items) < 7:
             CrawlerLogger.logger.info('line format error: {0}'.format(line))
             continue
         tmp_profile = Profile(0, items[0], items[1], items[2], items[3], items[4], items[5], items[6])
         if not self.insert_one_profile(tmp_profile):
             CrawlerLogger.logger.info('insert line failed: {0}'.format(line))
         else:
             succ += 1
     return total, succ
 def create(self):
     name = self.text_entry.text
     if len(name) > 0:
         today = date.today().isoformat()
         profile = Profile(name=name,
                           created_date=today,
                           modified_date=today)
         session = context['db_session']
         session.add(profile)
         # TODO: when supporting multiple characters per profile, move these
         character = Character(profile=profile,
                               name=name,
                               created_date=today,
                               modified_date=today,
                               gold=0,
                               enemies=0)
         session.add(character)
         session.commit()
         self.manager.set_screen('list_profiles')
Example #21
0
def get_profile():
    user = users.get_current_user()

    if user:
        user_id = user.user_id()
        p_key = ndb.Key(Profile, user_id)
        profile = p_key.get()
        # create a Profile if not found
        if not profile:
            profile = Profile(
                key=p_key,
                nickName=user.nickname(),
                eMail=user.email(),
            )
            profile.put()

        return profile
    else:
        return None
Example #22
0
def import_profiles():
    for file_path in filter(lambda x: x.startswith('profile-'),
                            get_parquet_file_paths()):
        table = pq.read_table(file_path)
        df = table.to_pandas()
        for idx, row in df.iterrows():
            print(idx)
            p = Profile()
            p.id = row['id']
            p.created = row['created']
            p.updated = row['updated']
            p.city = row['city']
            p.first_name = row['first_name']
            p.last_name = row['last_name']
            p.headline = row['headline']
            p.state = row['state']
            p.summary = row['summary']
            session.add(p)
        session.commit()
Example #23
0
def profile():

    my_form = ProfileForm()

    my_data = Profile()
    my_data.remove_none_values()

    # print("my_form.validate_on_submit()", my_form.validate_on_submit())
    # print(my_form.errors)
    if my_form.validate_on_submit():

        # print("************ FORM SUBMITTED****")
        my_data.first_name = request.form.get('first_name')
        my_data.last_name = request.form.get('last_name')
        my_data.email = request.form.get('email')
        my_data.dob = request.form.get('dob')
        # print("first_name", my_data.first_name)
        # print("last_name", my_data.last_name)

        # process file
        file = request.files.get('file_photo')
        if file:
            orig_filename = secure_filename(file.filename)
            file_extension = os.path.splitext(orig_filename)
            file_extension = str(file_extension[1]).lower()

            new_filename = str(uuid.uuid1()) + file_extension

            # save to upload folder
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], new_filename))
            # print("file saved")

            my_data.file_photo_filename = orig_filename
            my_data.file_photo_code = new_filename

        # ---------------EXCEL/CSV - Load into table
        data_file = request.files.get('excel_file')
        print("data_file", data_file)
        if data_file:
            orig_filename = secure_filename(data_file.filename)
            file_extension = os.path.splitext(orig_filename)
            file_extension = str(file_extension[1]).lower()

            new_filename = str(uuid.uuid1()) + file_extension

            file_full_path = os.path.join(app.config['UPLOAD_FOLDER'],
                                          new_filename)

            # save to upload folder
            data_file.save(file_full_path)
            # print("file_full_path", file_full_path)
            my_data.file_data_filename = orig_filename
            my_data.file_data_code = new_filename

            # load the data in the table using pandas
            df = pd.read_csv(file_full_path)
            rest_list_raw = df.to_dict('records')
            rest_list = []
            for rest in rest_list_raw:
                my_rest = Restaurant()
                my_rest.bill = rest['bill']
                my_rest.tip = rest['tip']
                rest_list.append(my_rest)
            db.session.bulk_save_objects(rest_list)
            db.session.commit()

        # save to database
        db.session.add(my_data)
        db.session.commit()
        # print("my_data", my_data.id)

        # redirect to display page
        return redirect('/profile/' + str(my_data.id))  # profile/5

    return render_template('profile.html', my_form=my_form, my_data=my_data)
Example #24
0
 def createMultiples(self, name):
     profile = Profile(name)
     session.add(profile)
     session.commit()
Example #25
0
    def _create_sample_profiles(self, config):
        ''' Create some sample profiles. '''

        session = app.database.get_session(self._db)
        sample_dir = os.path.join(os.path.dirname(__file__), 'sample-data')

        # Maurice Moss
        moss_twitter = Profile(site='twitter',
                               upstream_id='12345',
                               username=ProfileUsername(
                                   'maurice.moss', start_date='2014-04-01'))

        moss_twitter.usernames.append(
            ProfileUsername('maurice',
                            start_date='2013-06-01',
                            end_date='2014-03-31'))

        moss_twitter.usernames.append(
            ProfileUsername('maurie',
                            start_date='2013-02-15',
                            end_date='2013-05-30'))

        Post(author=moss_twitter,
             content='Going to the grocery store.',
             upstream_id='1234',
             upstream_created='2015-02-04 12:34:50')

        post = Post(author=moss_twitter,
                    content='Love this band!.',
                    upstream_id='2345',
                    upstream_created='2015-03-01')

        post.attachments.append(
            File(name='helloworld.txt',
                 mime='text/plain',
                 content='Hello world!\n\n'.encode('utf8')))

        moss_twitter.posts.append(post)

        with open(os.path.join(sample_dir, 'moss.jpg'), 'rb') as moss_jpg:
            moss_twitter.avatars.append(
                Avatar(url='http://foobar.com/moss-avatar.jpg',
                       mime='image/jpeg',
                       image=moss_jpg.read()))

        moss_twitter.description = "I do IT at Reynholm Industries."
        moss_twitter.post_count = 1205
        moss_twitter.friend_count = 1
        moss_twitter.follower_count = 3
        moss_twitter.join_date = dateutil.parser.parse('2013-06-01')
        moss_twitter.join_date_is_exact = False

        session.add(moss_twitter)

        # Jen Barber
        jen_twitter = Profile(site='twitter',
                              upstream_id='23456',
                              username=ProfileUsername(
                                  'jen.barber', start_date='2013-11-12'))

        jen_twitter.usernames.append(
            ProfileUsername('jenb',
                            start_date='2013-06-14',
                            end_date='2013-11-12'))

        jen_twitter.usernames.append(
            ProfileUsername('jenny',
                            start_date='2013-03-15',
                            end_date='2013-06-14'))

        with open(os.path.join(sample_dir, 'jen.jpg'), 'rb') as jen_jpg:
            jen_twitter.avatars.append(
                Avatar(url='http://foobar.com/jen-avatar.jpg',
                       mime='image/jpeg',
                       image=jen_jpg.read()))

        jen_twitter.description = "Relationship Manager for the IT department."
        jen_twitter.post_count = 1543
        jen_twitter.friend_count = 1
        jen_twitter.follower_count = 1
        jen_twitter.join_date = dateutil.parser.parse('2013-03-15')
        jen_twitter.join_date_is_exact = True

        moss_twitter.followers.append(jen_twitter)

        session.add(jen_twitter)

        # A couple of randos.
        moss_twitter.followers.append(
            Profile(site='twitter', upstream_id='345678', username='******'))

        moss_twitter.followers.append(
            Profile(site='twitter', upstream_id='456789', username='******'))

        jen_twitter.followers.append(
            Profile(site='twitter', upstream_id='567890', username='******'))

        session.commit()
Example #26
0
def scrape_twitter_account_by_id(upstream_ids, stub=False, labels={}):
    """
    Scrape twitter bio data for upstream IDs and/or updates a profile.
    Accepts twitter ID rather than username.
    """
    if len(upstream_ids) > 100:
        raise ScrapeException('Twitter API max is 100 user IDs per request.')

    db_session = worker.get_session()
    profiles = []

    # Request from Twitter API.
    api_url = 'https://api.twitter.com/1.1/users/lookup.json'
    payload = {'user_id': ','.join(upstream_ids)}
    headers = {'ACCEPT-ENCODING': None}
    response = requests.post(
        api_url,
        data=payload,
        proxies=_get_proxies(db_session),
        verify=False,
        headers=TWITTER_HEADERS
    )
    response.raise_for_status()

    # Update the profile.
    for profile_json in response.json():
        profile = Profile(
            'twitter',
            profile_json['id_str'],
            profile_json['screen_name']
        )
        profile.is_stub = stub
        profile.private = profile_json['protected']
        db_session.add(profile)

        try:
            db_session.commit()
        except IntegrityError:
            # Already exists: use the existing profile.
            db_session.rollback()
            profile = db_session.query(Profile) \
                                .filter(Profile.site=='twitter') \
                                .filter(
                                    Profile.upstream_id==profile_json['id_str']
                                )\
                                .one()
            # Profiles already in the system are either not stubs or
            # being updated to full profiles
            profile.is_stub = False


        _twitter_populate_profile(profile_json, profile)

        if profile.upstream_id in labels:
            _label_profile(db_session, profile, labels[profile.upstream_id])

        profile.last_update = datetime.now()
        db_session.commit()
        profiles.append(profile.as_dict())

        # Schedule followup jobs.
        app.queue.schedule_index_profile(profile)
        if not stub:
            app.queue.schedule_avatar(
                profile, profile_json['profile_image_url_https']
            )
            # Only get tweets and relations for unprotected profiles
            if not profile.private:
                app.queue.schedule_posts(profile, recent=True)
                app.queue.schedule_relations(profile)

    return profiles
Example #27
0
def scrape_twitter_relations(id_):
    """
    Fetch friends and followers for the Twitter user identified by `id_`.
    The number of friends and followers to fetch is configured in Admin.
    """
    redis = worker.get_redis()
    db = worker.get_session()
    profile = db.query(Profile).filter(Profile.id==id_).first()
    proxies = _get_proxies(db)
    max_results = get_config(db, 'max_relations_twitter', required=True).value

    try:
        max_results = int(max_results)
    except:
        raise ScrapeException(
            'Value of max_relations_twitter must be an integer'
        )

    friends_results = 0
    friends_ids = []
    followers_results = 0
    followers_ids = []
    friends_cursor = -1
    followers_cursor = -1

    if profile is None:
        raise ValueError('No profile exists with id={}'.format(id_))

    params = {
        'count': 5000,
        'user_id': profile.upstream_id,
        'stringify_ids': True,
    }

    # Get friends currently stored in db for this profile.
    friends_query = \
        db.query(Profile.upstream_id) \
            .join(\
                profile_join_self, \
                (profile_join_self.c.friend_id == Profile.id)
            ) \
            .filter(profile_join_self.c.follower_id == id_)
    current_friends_ids = [friend.upstream_id for friend in friends_query]


    # Get followers currently stored in db for this profile.
    followers_query = \
        db.query(Profile.upstream_id) \
            .join(\
                profile_join_self, \
                (profile_join_self.c.follower_id == Profile.id)
            ) \
            .filter(profile_join_self.c.friend_id == id_)
    current_followers_ids = [follower.upstream_id for follower in followers_query]

    ## Get friend IDs.
    friends_url = 'https://api.twitter.com/1.1/friends/ids.json'
    params['cursor'] = friends_cursor

    while friends_results < max_results:
        friends_response = requests.get(
            friends_url,
            params=params,
            proxies=proxies,
            verify=False,
            headers=TWITTER_HEADERS
        )
        friends_response.raise_for_status()

        # Ignore friends already in the db
        for friend_id in friends_response.json()['ids']:
            if friend_id not in current_friends_ids:
                friends_ids.append(friend_id)
                friends_results += 1
                if friends_results == max_results:
                    break

        friends_cursor = friends_response.json()['next_cursor']

        if friends_cursor == 0:
            break # No more results
        else:
            params['cursor'] = friends_cursor

    # Get follower IDs.
    followers_url = 'https://api.twitter.com/1.1/followers/ids.json'
    params['cursor'] = followers_cursor

    while followers_results < max_results:
        followers_response = requests.get(
            followers_url,
            params=params,
            proxies=proxies,
            verify=False,
            headers=TWITTER_HEADERS,
        )
        followers_response.raise_for_status()

        # Ignore followers already in the db
        for follower_id in followers_response.json()['ids']:
            if follower_id not in current_followers_ids:
                followers_ids.append(follower_id)
                followers_results += 1
                if followers_results == max_results:
                    break

        followers_cursor = followers_response.json()['next_cursor']

        if followers_cursor == 0:
            break # No more results
        else:
            params['cursor'] = followers_cursor

    # Get username for each of the friend/follower IDs and create
    # a relationship in QuickPin.
    user_ids = [(uid, 'friend') for uid in friends_ids] + \
               [(uid, 'follower') for uid in followers_ids]
    worker.start_job(total=len(user_ids))
    chunk_size = 100
    for chunk_start in range(0, len(user_ids), chunk_size):
        chunk_end = chunk_start + chunk_size
        chunk = user_ids[chunk_start:chunk_end]
        chunk_lookup = {id_:relation for id_,relation in chunk}

        lookup_url = 'https://api.twitter.com/1.1/users/lookup.json'
        lookup_response = requests.post(
            lookup_url,
            proxies=_get_proxies(db),
            verify=False,
            headers=TWITTER_HEADERS,
            data={'user_id': ','.join(chunk_lookup.keys())}
        )
        lookup_response.raise_for_status()
        relations = lookup_response.json()

        for related_dict in relations:
            uid = related_dict['id_str']
            username = related_dict['screen_name']
            related_profile = Profile('twitter', uid, username, is_stub=True)
            db.add(related_profile)

            try:
                db.commit()
            except IntegrityError:
                # Already exists: use the existing profile.
                db.rollback()
                related_profile = db \
                    .query(Profile) \
                    .filter(Profile.site=='twitter') \
                    .filter(Profile.upstream_id==uid) \
                    .one()

            _twitter_populate_profile(related_dict, related_profile)
            relation = chunk_lookup[uid]

            if relation == 'friend':
                profile.friends.append(related_profile)
            else: # relation == 'follower':
                profile.followers.append(related_profile)

            db.commit()

        worker.update_job(current=chunk_end)

    db.commit()
    worker.finish_job()
    redis.publish('profile_relations', json.dumps({'id': id_}))
Example #28
0
def schedule_job(p_id):
    p = Profile().find_by_pk(p_id)
    c = p['cron']
    sched.add_cron_job(profile_execute, args=[p_id], name='wj_%s' % p_id, **c)
Example #29
0
def create_or_get_profile(username: str):
    profile = Profile.get(username=username)
    if not profile:
        color = "#%02X%02X%02X" % (color_gen(), color_gen(), color_gen())
        profile = Profile(username=username, color=color)
    return profile
Example #30
0
def profile_list():
    data = Profile().retrieve(jobs=sched.get_jobs())
    return render_template('profile.html', profiles=data)