def gen_datetime(min_date=now_add_days(-365), max_date=now_add_days(365)):
	min_date = time.mktime(min_date.timetuple())
	max_date = time.mktime(max_date.timetuple())
	for __ in itertools.count():
		random_time = min_date + random.random() * (max_date - min_date)
		if settings.USE_TZ:
			yield tz_datetime.fromtimestamp(time.mktime(time.localtime(random_time))).replace(tzinfo=utc)
		else:
			yield tz_datetime.fromtimestamp(time.mktime(time.localtime(random_time)))
Esempio n. 2
0
class FieldType:
    django_to_proto_type = {
        models.CharField: 'string',
        models.FloatField: 'double',
        models.IntegerField: 'int32',
        models.AutoField: 'int32',
        models.BigIntegerField: 'int64',
        models.PositiveIntegerField: 'uint32',
        models.PositiveBigIntegerField: 'uint64',
        models.FileField: 'string',
        models.DateTimeField: 'uint64',
        models.DateField: 'uint64',
        models.ForeignKey: 'uint16',
        models.BooleanField: 'bool',
    }

    serializers = {
        models.FileField:
        lambda file: file.url,
        models.DateTimeField:
        lambda dt: int(dt.timestamp() * 1000),
        models.DateField:
        lambda dt: int(datetime(dt.year, dt.month, dt.day).timestamp() * 1000),
    }

    deserializers = {
        models.FileField: lambda file: file.url,
        models.DateTimeField: lambda ts: datetime.fromtimestamp(ts / 1000.0),
        models.DateField:
        lambda ts: datetime.fromtimestamp(ts / 1000.0).date(),
    }

    def __init__(self, django_type):
        self.djange_type = django_type
        self.proto_type = FieldType.django_to_proto_type[django_type]

    def serialize(self, value):
        try:
            return FieldType.serializers.get(self.djange_type,
                                             lambda x: x)(value)
        except:
            print('error serializing', value)
            return None

    def deserialize(self, value):
        try:
            return FieldType.deserializers.get(self.proto_type,
                                               lambda x: x)(value)
        except:
            print('error deserializing', value)
            return None
Esempio n. 3
0
    def default(self, o):
        from time import struct_time, mktime

        # See "Date Time String Format" in the ECMA-262 specification.
        if isinstance(o, struct_time):
            return self.default(datetime.fromtimestamp(mktime(o)))
        else:
            return super(CustomJSONEncoder, self).default(o)
Esempio n. 4
0
def rss(source_url, newer_then=None):
    rss_data = _rss_parse(source_url)
    entries = [dict(
        html=entry.summary,
        published=tz_dt.fromtimestamp(mktime(entry.published_parsed))
    ) for entry in rss_data.entries]
    if newer_then:
        return [entry for entry in entries if entry['published'] > newer_then]
    return entries
Esempio n. 5
0
    def get_rank_board_datas(self):
        """
        获取滚榜数据(比赛服阉割版)
        :return:
        """
        import time
        from django.utils.timezone import datetime
        asgn = self.asgn
        # 获取题目数据
        aproblem_list = asgn.problems.order_by("index")
        # 题目索引参照表
        apindex_list = {}
        for aproblem in aproblem_list:
            apindex_list[aproblem.entity.id] = aproblem.index
        parser = ParamsParser(self._request)

        # 获取最后一次刷新的时间
        last_time = parser.get_float('time', 0)
        try:
            last_time = datetime.fromtimestamp(last_time)
        except Exception as ex:
            last_time = 0

        user_list = {}

        if type(last_time) is datetime:
            # 设置了时间,就是弹时间线
            judge_status = asgn.judge_status.filter(
                create_time__gt=last_time).order_by("id")
        else:
            # 没有,就是弹初始化数据
            judge_status = asgn.judge_status.filter(
                create_time__lte=datetime.now())

        reports = EducationModel.AsgnReport.objects.filter(asgn=asgn)
        for report in reports:
            user_list[str(report.author.id)] = report.json(items=[
                'id', 'author', 'author__id', 'author__nickname',
                'author__username', 'author__realname', 'author__headimg',
                'author__sex', 'start_time'
            ])

        judge_status_list = [{
            "id": status.id,
            "problem_id": status.problem_id,
            "user_id": status.author_id,
            "flag": status.flag,
            "timestamp": int(status.create_time.timestamp())
        } for status in judge_status]

        return {
            "problem_indexs": apindex_list,
            "problem_list": [aproblem.entity.id for aproblem in aproblem_list],
            "judge_status": judge_status_list,
            "user_list": user_list,
            "nowtime": time.time()
        }
Esempio n. 6
0
def main():
    with open('bildeliste2') as f:
        images = f.readlines()
    # matchvalue = None
    # shuffle(images)
    # images = images[:100]
    imagedict = {}
    duplicates = 0
    for img in images:
        if re.search('/(reklame|banner|elm|2014/2013|slettmeg)/', img):
            continue
        year = issue = None
        fields = img.split()
        filesize_in_bytes = int(fields[0])
        unix_timestamp = int(fields[5])
        filedate = datetime.fromtimestamp(unix_timestamp)
        filepath = ''.join(fields[6:])
        filename = filepath.split('/')[-1]
        # logger.debug("{} {} {}".format(size, unix_timestamp, path))
        year_issue_match = (
            re.match(r'\./(?P<year>\d{4})/(?P<issue>\d{1,2})/(?P=issue)\D',
                     filepath)
            or re.match(r'\./(?P<year>\d{4})/(?P<issue>\d{1,2})/', filepath)
            or re.match(r'\./(?P<year>\d{4})/(?P<issue>\d{2})[^\d/]', filepath)
            or None)
        if year_issue_match:
            # matchvalue = 1
            year = year_issue_match.group('year')
            issue = year_issue_match.group('issue')
        else:
            issue_match = re.search(r'\/(?P<issue>\d{2})[^/\d]', filepath)
            if issue_match:
                # matchvalue = 2
                issue = issue_match.group('issue')
                year = filedate.year
            else:
                # matchvalue = 3
                year = filedate.year
                issue = '00'

        # logger.debug('year: {}, issue: {}, file: {}'.format(issuematch.group('year'), issuematch.group('issue'), filename))
        if year and issue:
            image_file = ImageFile(filepath, filesize_in_bytes, filedate,
                                   issue, year)
            if filename in imagedict:
                # old_filepath = imagedict[filename]
                duplicates += 1
            else:
                imagedict[filename] = image_file
        else:
            msg = 'No match {}'.format(filepath)
            logger.debug(msg)

    msg = 'all: {0} placed: {1} duplicates: {2}'.format(
        len(images), len(imagedict), duplicates)
    logger.debug(msg)
Esempio n. 7
0
def save_report_date(request):
    user = User.objects.get(username=request.user)
    if user.is_staff:
        try:
            counteragent = Counteragent.objects.get(alias=request.POST['counteragent'])
            date_time = datetime.fromtimestamp(int(request.POST['datetime'].split('.')[0]))
            last_report_datetime = LastReportDatetime(counteragent=counteragent, date_time=date_time)
            last_report_datetime.save()
            return {'result': "Saved"}
        except Exception as ex:
            return {'result': "Error while saving" }
    else:
        return {'result': "У вас нет прав на сохранение даты" }
Esempio n. 8
0
def random_datetime(first_datetime, second_datetime):
    """
    Returns random datetime between first_datetime and second_datetime.
    """

    if second_datetime < first_datetime:
        aux_datetime = first_datetime
        first_datetime = second_datetime
        second_datetime = aux_datetime
    first_timestamp = int(first_datetime.timestamp())
    second_timestamp = int(second_datetime.timestamp())
    random_timestamp = randint(first_timestamp, second_timestamp)
    naive_date_time = datetime.fromtimestamp(random_timestamp)
    date_time = make_aware(naive_date_time)
    return date_time
Esempio n. 9
0
def get_feedly_article(request_content):
	from posts.models import FeedlyAPISettings, Post, NewsAggregator
	from django.contrib.auth import get_user_model
	from posts.utils import get_favicon
	feedly_settings = FeedlyAPISettings.get_solo()
	feedly = feedly_settings.get_client()
	article = feedly.get_entry(feedly_settings.FEEDLY_API_ACCESS_TOKEN, request_content.get('entryId'))[0]
	try:
		Post.objects.get(title=request_content.get('title', None))
	except Post.DoesNotExist:
		origin_url = urlparse(article.get('origin').get('htmlUrl'))
		if 'google' in origin_url.netloc:
			article['origin']['htmlUrl'] = article.get('alternate')[0]['href'] if article.get('alternate',
			                                                                                  None) else article.get(
				'canonicalUrl')
			article['origin']['title'] = urlparse(article['origin']['htmlUrl']).netloc
		try:
			news_aggregator = NewsAggregator.objects.get(name=article.get('origin').get('title'))
		except NewsAggregator.DoesNotExist:
			na_url = urlparse(article['origin']['htmlUrl'])
			news_aggregator = NewsAggregator.objects.create(name=article.get('origin').get('title'),
			                                                url=na_url.scheme + '://' + na_url.netloc)
			temp_image = get_favicon(na_url.geturl())
			if temp_image:
				news_aggregator.logo.save(
					origin_url.netloc + "_logo",
					File(temp_image,
					     name=origin_url.netloc + "_logo"))
				news_aggregator.save()

		if request_content.get('title', False) and news_aggregator:
			post = Post.objects.create(submitter=get_user_model().objects.filter(is_superuser=True)[0],
			                           author=request_content.get('author', None),
			                           title=request_content.get('title'),
			                           news_aggregator=news_aggregator,
			                           submit_time=datetime.fromtimestamp(
				                           request_content.get('publishedTimestamp') / 1000.0),
			                           url=article['canonicalUrl'] if article.get('canonicalUrl', None) else article.get('alternate')[0]['href'],
			                           feedly_engagement=article.get('engagement', 100))

			article_image = request_content.get('visualUrl', None)
			if article_image:
				post.image_url = article_image
				post.save()
    def update_items(self, items, item_data):
        if len(items) > 1:
            raise ValueError(
                "There should only ever be exactly one-to-one " "relationships between items in RSS feeds."
            )
        item = items[0]

        published_at = item_data.pop("published_parsed", None)
        updated_at = item_data.pop("updated_parsed", None)

        item_data = self.prepare_item_content(item_data)

        h = HTMLParser()
        item.title = h.unescape(item_data["title"])
        item.displayed_from = datetime.fromtimestamp(mktime(published_at or updated_at))
        item.source_content = json.dumps(item_data, sort_keys=True)
        item.source_url = item_data.get("link") or item_data.get("id")
        item.source_id = item_data.get("id") or item_data.get("link")

        item.last_read_at = now()
Esempio n. 11
0
    def update_items(self, items, item_data):
        if len(items) > 1:
            raise ValueError('There should only ever be exactly one-to-one '
                             'relationships between items in RSS feeds.')
        item = items[0]

        published_at = item_data.pop('published_parsed', None)
        updated_at = item_data.pop('updated_parsed', None)

        item_data = self.prepare_item_content(item_data)

        h = HTMLParser()
        item.title = h.unescape(item_data['title'])
        item.displayed_from = datetime.fromtimestamp(
            mktime(published_at or updated_at))
        item.source_content = json.dumps(item_data, sort_keys=True)
        item.source_url = item_data.get('link') or item_data.get('id')

        item.last_read_at = now()
        item.save()
Esempio n. 12
0
def build_table_row(request, obj, admin_class):
    row_html = ''
    for strcolumn in admin_class.list_display:
        field_obj = obj._meta.get_field(strcolumn)
        if field_obj.choices:
            tddata = getattr(obj, 'get_%s_display' % strcolumn)()
        else:
            tddata = getattr(obj, strcolumn)
        if type(tddata).__name__ == "datetime":
            tddata = datetime.fromtimestamp(tddata.timestamp())
            tddata = tddata.strftime("%Y-%m-%d %H:%M:%S")

        row_html += "<td>%s</td>" % tddata

    editUrl = "%s/%s/change" % (request.path, obj.id)
    row_html += "<td><a href='%s'><span class='fa fa-edit'></span></a></td>" % editUrl

    for actionButton in admin_class.action_buttons:
        actionButtonEle = getattr(obj, actionButton)(obj.id)
        row_html += "<td>%s</td>" % actionButtonEle

    return mark_safe(row_html)
Esempio n. 13
0
    def entry_to_messages(self, entry):
        content_r = entry.get('content')
        if isinstance(content_r, list):
            content_join = []
            for c in content_r:
                if isinstance(c, dict):
                    content_join.append(c.get('value'))
            content = "\r\n".join(content_join) if any(content_join) else None
        else:
            content = None

        title = entry.get('title')
        if not title:
            title = None
        contents = filter(
            lambda c: c is not None,
            (title, content, entry.get('summary'), entry.get('link')))
        message_text = "\r\n".join(contents)
        message_date = entry.get('published_parsed') or entry.get(
            'updated_parsed')
        message_date = datetime.fromtimestamp(
            mktime(message_date)).replace(tzinfo=pytz.UTC) or datetime.now()
        author = entry.get('author')
        return Message(text=message_text, date=message_date, username=author)
Esempio n. 14
0
    def collect_job(self, name):
        job = self.server.get_job_info(name=name)
        try:
            last_success_build_num = job['lastSuccessfulBuild']['number']

        except BaseException as error:
            last_success_build_num = None

        # print(last_success_build_num)

        try:
            last_build_num = job['lastBuild']['number']
        except BaseException as error:
            return False

        # print(last_build_num)

        last_build_info = self.server.get_build_info(name, last_build_num)
        last_build_console = self.server.get_build_console_output(
            name, last_build_num)
        last_build_status = last_build_info['result']
        if not last_build_status:
            last_build_status = "RUNNING"
        ts = last_build_info['timestamp']
        sp = float(str(ts)[0:-3] + '.' + str(ts)[-3:])
        last_build_time = datetime.strftime(datetime.fromtimestamp(sp),
                                            "%Y-%m-%d %H:%M:%S")

        return {
            'app_name': name,
            'build_status': last_build_status,
            'last_build_time': last_build_time,
            'build_console_output': last_build_console,
            'last_success_build_num': last_success_build_num,
            'last_build_num': last_build_num
        }
Esempio n. 15
0
 def transform_data(self, data_integer):
     data = datetime.fromtimestamp(data_integer)
     return data
Esempio n. 16
0
def process_post(row, config, posts_update_run):
    lgr.info('\n\nROW - Published: {}, Id: {}'.format(row.published, row.id))

    row_last_edited = make_aware(
        datetime.fromtimestamp(int(row.get('last_edited_time')) / 1000))
    lgr.info('row_last_edited: {}'.format(row_last_edited))

    # registering a change callback
    # row.add_callback(changed_callback)

    try:
        post = Post.objects.get(nt_block__reference=row.id)
        lgr.info('editing existing post')
        post.is_published = row.published

        post_nt_block = post.nt_block
        post_nt_block.updated_run = posts_update_run
        post_nt_block.updated_at = row_last_edited
        post_nt_block.save()

        if not row.published or (post_nt_block.updated_at == row_last_edited):
            post.save()
            # lgr.info('skipped unpublished or un-updated')
            # continue

    except Post.DoesNotExist:
        lgr.info('creating new post')
        post = Post()
        post.is_published = row.published
        post.site_id = 1
        post.creator_id = 1

        try:
            post_nt_block = Block.objects.get(reference=row.id)
        except Block.DoesNotExist:
            post_nt_block = Block()
            post_nt_block.reference = row.id
            post_nt_block.config = config

        post_nt_block.updated_at = row_last_edited
        post_nt_block.updated_run = posts_update_run

        lgr.info('saving post: {}'.format(post))
        post.save()
        lgr.info('saved post: {}'.format(post))
        lgr.info('saving post_nt_block: {}'.format(post_nt_block))
        post_nt_block.post = post
        post_nt_block.save()
        lgr.info('saved post_nt_block: {}'.format(post_nt_block))

    post.heading = row.name
    # post.sub_heading = row.
    # post.category = category

    post.save()

    for tag_name in row.tags:
        tag, created = Tag.objects.get_or_create(name=tag_name, site_id=1)
        post.tags.add(tag)

    # Retrieving Page content
    # Blocks
    children_update_run = timezone.now()
    level = 0
    for child in row.children:
        block = parse_block(child, config, children_update_run, level)
        level += 1
        post.body.add(block)

    # deleted = BlogBlock.objects.filter(nt_block__updated_run__lt=children_update_run).delete()
    deleted = post.body.filter(
        nt_block__updated_run__lt=children_update_run).delete()
    lgr.info(deleted)
Esempio n. 17
0
def parse_block(child, config, children_update_run, level):
    save = True
    new_block_nt_block = False
    child_last_edited = make_aware(
        datetime.fromtimestamp(int(child.get('last_edited_time')) / 1000))

    try:
        block = BlogBlock.objects.get(nt_block__reference=child.id)
        lgr.info('parse_block existing')
        block_nt_block = block.nt_block

        block_nt_block.updated_run = children_update_run
        if block_nt_block.updated_at == child_last_edited:
            # block.save()
            # continue
            # return None
            pass

    except BlogBlock.DoesNotExist:
        block = BlogBlock()
        new_block_nt_block = True
        lgr.info('parse_block creating new')
        try:
            block_nt_block = Block.objects.get(reference=child.id)
        except Block.DoesNotExist:
            block_nt_block = Block()
            block_nt_block.reference = child.id
            block_nt_block.config = config

        block_nt_block.updated_run = children_update_run

    block_nt_block.updated_at = child_last_edited
    block.level = level

    if isinstance(child, ColumnListBlock):
        block.type = BlogBlock.COLUMN_LIST_BLOCK
        # todo save config
        save = False
        block.save()
        column_block_level = 0
        for column_block in child.children:
            lgr.info(column_block)
            cb = parse_block(column_block, config, children_update_run,
                             column_block_level)
            block.children.add(cb)
            column_block_level += 1

            column_block_child_level = 0
            for column_block_child in column_block.children:
                lgr.info(column_block_child)
                cbc = parse_block(column_block_child, config,
                                  children_update_run,
                                  column_block_child_level)
                cb.children.add(cbc)
                column_block_child_level += 1

    elif isinstance(child, ColumnBlock):
        block.type = BlogBlock.COLUMN_BLOCK
        # todo save config
        save = False
        block.save()
    elif isinstance(child, TextBlock):
        block.type = BlogBlock.TEXT
        block.config = {}
        block.config['content'] = child.title
    elif isinstance(child, DividerBlock):
        block.config = {}
        block.type = BlogBlock.DIVIDER
    elif isinstance(child, BulletedListBlock):
        block.type = BlogBlock.BULLETED_LIST
        block.config = {}
        block.config['content'] = child.title
    elif isinstance(child, NumberedListBlock):
        block.type = BlogBlock.NUMBERED_LIST_BLOCK
        block.config = {}
        block.config['content'] = child.title
    elif isinstance(child, ImageBlock):
        block.type = BlogBlock.IMAGE
        block.config = {}
        block.config['display_source'] = child.display_source
        block.config['file_id'] = child.file_id
        block.config['caption'] = child.caption
        block.config['full_width'] = child.full_width
        block.config['height'] = child.height
        block.config['page_width'] = child.page_width
        block.config['width'] = child.width
    elif isinstance(child, QuoteBlock):
        block.type = BlogBlock.QUOTE
        block.config = {}
        block.config['content'] = child.title
    elif isinstance(child, HeaderBlock):
        block.type = BlogBlock.HEADER
        block.config = {}
        block.config['content'] = child.title
    elif isinstance(child, SubheaderBlock):
        block.type = BlogBlock.SUB_HEADER
        block.config = {}
        block.config['content'] = child.title
    elif isinstance(child, SubsubheaderBlock):
        block.type = BlogBlock.SUB_SUB_HEADER
        block.config = {}
        block.config['content'] = child.title
    elif isinstance(child, CodeBlock):
        block.type = BlogBlock.CODE
        block.config = {}
        block.config['content'] = child.title
        block.config['language'] = child.language
        block.config['wrap'] = child.wrap
    elif isinstance(child, TodoBlock):
        block.type = BlogBlock.TODO
        block.config = {}
        block.config['content'] = child.title
        block.config['checked'] = child.checked
    else:
        lgr.info(child)
        lgr.info('Unknown block type: {}'.format(type(child)))

    if save:
        block.save()

    if new_block_nt_block:
        block_nt_block.block = block
    block_nt_block.save()

    return block
Esempio n. 18
0
def get_feedly_articles():
	from posts.models import FeedlyAPISettings, Post, NewsAggregator
	from django.contrib.auth import get_user_model
	from posts.utils import get_favicon
	feedly_settings = FeedlyAPISettings.get_solo()
	tag_on_feedly = 'WHN'
	feedly = feedly_settings.get_client()
	data = feedly.get_enterprise_user_tags(feedly_settings.FEEDLY_API_ACCESS_TOKEN)
	whn_tag_id = False
	for i in data:
		label = i.get('label', False)
		if label and label == tag_on_feedly:
			whn_tag_id = i.get('id', False)
			break
	if whn_tag_id:
		articles = feedly.get_feed_content(
			feedly_settings.FEEDLY_API_ACCESS_TOKEN,
			whn_tag_id
		)
		if articles.get('items', False):
			admin = get_user_model().objects.filter(is_superuser=True)[0]
			last_entry_id = ''
			for article in articles['items']:
				try:
					post = Post.objects.get(title=article.get('title', ''))
				except Post.DoesNotExist:
					origin_url = urlparse(article.get('origin').get('htmlUrl'))
					if 'google' in origin_url.netloc:
						article['origin']['htmlUrl'] = article.get('alternate')[0]['href'] if article.get('alternate', None) else article.get('canonicalUrl')
						article['origin']['title'] = urlparse(article['origin']['htmlUrl']).netloc
					try:
						news_aggregator = NewsAggregator.objects.get(name=article.get('origin').get('title'))
					except NewsAggregator.DoesNotExist:
						na_url = urlparse(article['origin']['htmlUrl'])
						news_aggregator = NewsAggregator.objects.create(name=article.get('origin').get('title'),
						                                                url=na_url.scheme + '://' + na_url.netloc)

						temp_image = get_favicon(article.get('origin').get('htmlUrl'))
						if temp_image:
							try:
								news_aggregator.logo.save(na_url.netloc + "_logo", File(temp_image))
							except OSError:
								logging.warning("Can't save logo image for news_aggregator: "
								                + str(news_aggregator.id) + '--' + str(news_aggregator.name))
							news_aggregator.save()

					if article.get('title', False) and news_aggregator and article.get('unread', False):
						post = Post.objects.create(submitter=admin, title=article.get('title'),
						                           news_aggregator=news_aggregator,
						                           submit_time=datetime.fromtimestamp(article.get('published')/1000.0),
						                           url=article['canonicalUrl'] if article.get('canonicalUrl', None) else article.get('alternate')[0]['href'],
						                           feedly_engagement=article.get('engagement', 100))
					else:
						raise Exception("Article doesn't have title.")

				article_image = article.get('visual', False)
				if article_image:
					post.image_url = article_image.get('url', None)
					post.save()
				last_entry_id = article.get('id')

			feedly.mark_tag_read(feedly_settings.FEEDLY_API_ACCESS_TOKEN, whn_tag_id, last_entry_id)

		else:
			raise Exception("No entries are found with '%s' tag." % tag_on_feedly)
	else:
		raise Exception("Can't find '%s' tag!" % tag_on_feedly)
Esempio n. 19
0
        except IndexError:
            print(f"Failed to get rfid for {email}! Skipping")
            skipped_members.append(email)
            continue

        full_name_list = user[3].split(" ")
        if full_name_list:
            first_name = full_name_list.pop(0)
            last_name = " ".join(full_name_list)
        else:
            print(f"Skipping {email} because no name was given!")
            skipped_members.append(email)
            continue

        # Get the correct joined and expired dates, and add them to the member
        date_joined = datetime.fromtimestamp(user[6])
        date_expires = datetime.fromtimestamp(user[7])

        # Select which group the member should be placed in
        mem_type = user[4].lower()
        status = user[5].lower()
        if mem_type == "member":
            if status == "new":
                group = groups["just joined"]
            elif status == "expired":
                group = groups["expired"]
            elif status == "active" and date_expires < datetime.now():
                group = groups["expired"]
            elif status == "active" and date_expires > datetime.now():
                group = groups["member"]
        elif (mem_type == "staff"
Esempio n. 20
0
    def post(self, request, *args, **kwargs):
        if request.POST.get('is_submitted'):
            cart_items = get_cart_items(self.request)
            total_weight = get_total_weight(cart_items=cart_items)
            client_data_cookie = self.request.COOKIES.get('client_data')
            client_data = cookie_parser(client_data_cookie)
            address = client_data.get('address')

            order = Order()
            order.client_name = address.get('client_name')
            order.payment = PaymentMethod.objects.get(method__exact=client_data.get('payment_method'))
            order.phone = address.get('phone')
            order.shipping_address = address.get('shipping_address')
            order.total_weight = total_weight
            order.delivery_price = get_delivery_price(cart_items)
            try:
                order.need_porter = int(client_data.get('need_porter'))
            except ValueError:
                pass

            if request.user.is_authenticated():
                order.customer = request.user

            if order.need_porter:
                order.porter_work_price = get_delivery_price(cart_items)
            order.shipping_time = datetime.fromtimestamp(client_data.get('time'), tz=pytz.timezone(settings.TIME_ZONE))
            order.save()

            total_price = 0

            json_data = []
            for cart_item in cart_items:
                cart_item.order = order
                total_price += cart_item.total_price
                cart_item.save()

                title = "{} {}".format(cart_item.variation.product.name, cart_item.variation.name)
                if cart_item.variation.color:
                    title = "{} {}".format(title, cart_item.variation.color.name)
                json_data.append({
                    'title': title,
                    'price': float(cart_item.variation.price),
                    'count': cart_item.count,
                    'total_price': float(cart_item.total_price)
                })
            order.products = json_data
            order.products_price = Decimal(total_price)
            order.order_unique_id = uuid.uuid4()

            order.total_price = order.products_price + get_delivery_price(cart_items)
            order.save()
            if order.payment.method == PaymentMethod.CASH_ON_DELIVERY:
                return redirect(reverse_lazy('orders:order_detail', args=[order.phone, order.order_unique_id]))
            elif order.payment.method == PaymentMethod.PAYME:
                # https://checkout.paycom.uz/base64(m=587f72c72cac0d162c722ae2;ac.order_id=197;a=500)
                merchant_id = settings.PAYME_MERCHANT_ID
                ac_order_id = order.id
                ac_customer_id = order.phone
                amount = int(order.total_price.real) * 100
                redirect_url_after_operation = reverse_lazy('main:home')
                ct = 15
                cr = 860
                data = "m={};ac.order_id={};ac.customer_id={};a={};c={}".format(merchant_id, ac_order_id,
                                                                                ac_customer_id, amount,
                                                                                redirect_url_after_operation)
                base64_data = base64.b64encode(data.encode('utf-8')).decode('utf-8')
                base64_data_url = "https://checkout.paycom.uz/{}".format(base64_data)
                return redirect(to=base64_data_url)
            elif order.payment.method == PaymentMethod.UZCARD:
                return redirect(reverse('orders:payment_uzcard', args=[order.phone, order.order_unique_id]))
            else:
                return redirect(reverse('main:home'))
        return redirect(reverse('main:home'))
Esempio n. 21
0
def main():
    with open('bildeliste2') as f:
        images = f.readlines()
    # matchvalue = None
    # shuffle(images)
    # images = images[:100]
    imagedict = {}
    duplicates = 0
    for img in images:
        if re.search('/(reklame|banner|elm|2014/2013|slettmeg)/', img):
            continue
        year = issue = None
        fields = img.split()
        filesize_in_bytes = int(fields[0])
        unix_timestamp = int(fields[5])
        filedate = datetime.fromtimestamp(unix_timestamp)
        filepath = ''.join(fields[6:])
        filename = filepath.split('/')[-1]
        # logger.debug("{} {} {}".format(size, unix_timestamp, path))
        year_issue_match = (
            re.match(
                r'\./(?P<year>\d{4})/(?P<issue>\d{1,2})/(?P=issue)\D',
                filepath) or re.match(
                r'\./(?P<year>\d{4})/(?P<issue>\d{1,2})/',
                filepath) or re.match(
                r'\./(?P<year>\d{4})/(?P<issue>\d{2})[^\d/]',
                filepath) or None)
        if year_issue_match:
            # matchvalue = 1
            year = year_issue_match.group('year')
            issue = year_issue_match.group('issue')
        else:
            issue_match = re.search(r'\/(?P<issue>\d{2})[^/\d]', filepath)
            if issue_match:
                # matchvalue = 2
                issue = issue_match.group('issue')
                year = filedate.year
            else:
                # matchvalue = 3
                year = filedate.year
                issue = '00'

        # logger.debug('year: {}, issue: {}, file: {}'.format(issuematch.group('year'), issuematch.group('issue'), filename))
        if year and issue:
            image_file = ImageFile(
                filepath,
                filesize_in_bytes,
                filedate,
                issue,
                year)
            if filename in imagedict:
                # old_filepath = imagedict[filename]
                duplicates += 1
            else:
                imagedict[filename] = image_file
        else:
            msg = 'No match {}'.format(filepath)
            logger.debug(msg)

    msg = 'all: {0} placed: {1} duplicates: {2}'.format(
        len(images), len(imagedict), duplicates)
    logger.debug(msg)
Esempio n. 22
0
    def parse(self, response):
        object_id = self.object_id
        spider_id = cache.get('running_spider_id')
        if object_id != spider_id:
            logger.info('Spider is running.')
            self.crawled = 0
            return

        sel = Selector(response)
        scripts = sel.xpath('//script')

        # 判断是否需要保存抓取的数据
        statistics = self.get_dict(scripts, '#getStatisticsService')
        create_time = make_aware(
            datetime.fromtimestamp(statistics['createTime'] / 1000.0))
        modify_time = make_aware(
            datetime.fromtimestamp(statistics['modifyTime'] / 1000.0))
        qs = items.StatisticsItem.django_model.objects.all().order_by('-id')
        if qs.count() > 1 and qs[0].modifyTime == modify_time:
            logger.info('Data does not change.')
            self.crawled = 0
            return

        # 统计信息
        statistics = self.explain_statistics(statistics)
        statistics['createTime'] = create_time
        statistics['modifyTime'] = modify_time

        # 国内数据
        provinces = self.get_list(scripts, '#getAreaStat')
        for province in provinces:
            cities = province.pop('cities', [])
            province.pop('locationId')
            yield scrapy.Request(province['statisticsData'],
                                 callback=self.parse_province_statistics_data,
                                 meta={
                                     'province': province,
                                     'cities': cities
                                 })

        # 时间线事件,id=“getTimelineService2” 为英文内容
        timelines = self.get_list(scripts, '#getTimelineService1')
        result = []
        for item in timelines:
            timeline = {}
            for key in ('title', 'summary', 'infoSource', 'sourceUrl',
                        'pubDate', 'pubDateStr'):
                timeline[key] = item.get(key)
            result.append(timeline)
        statistics['timelines'] = json.dumps(result)

        # 建议,id=“#getIndexRecommendList2” 为英文内容
        recommends = self.get_list(scripts, '#getIndexRecommendListundefined')
        result = []
        for item in recommends:
            recommend = {}
            for key in ('title', 'linkUrl', 'imgUrl', 'countryType',
                        'contentType', 'recordStatus', 'sort'):
                recommend[key] = item.get(key)
            result.append(recommend)
        statistics['recommends'] = json.dumps(result)

        # WHO 文章
        item = self.get_dict(scripts, '#fetchWHOArticle')
        article = {}
        for key in ('title', 'linkUrl', 'imgUrl'):
            article[key] = item.get(key)
        statistics['WHOArticle'] = json.dumps(article)

        # wiki
        wiki_result = self.get_dict(scripts, '#getWikiList')
        wikis = wiki_result['result']
        result = []
        for item in wikis:
            wiki = {}
            for key in ('title', 'linkUrl', 'imgUrl', 'description'):
                wiki[key] = item.get(key)
            result.append(wiki)
        statistics['wikis'] = json.dumps(result)

        # 购物指南
        guides = self.get_list(scripts, '#fetchGoodsGuide')
        result = []
        for item in guides:
            guide = {}
            for key in ('categoryName', 'title', 'recordStatus',
                        'contentImgUrls'):
                guide[key] = item.get(key)
            result.append(guide)
        statistics['goodsGuides'] = json.dumps(result)

        # 辟谣与防护
        rumors = self.get_list(scripts, '#getIndexRumorList')
        result = []
        for item in rumors:
            rumor = {}
            for key in ('title', 'mainSummary', 'summary', 'body', 'sourceUrl',
                        'score', 'rumorType'):
                rumor[key] = item.get(key)
            result.append(rumor)
        statistics['rumors'] = json.dumps(result)
        yield statistics

        # 国外数据
        countries = self.get_list(scripts, '#getListByCountryTypeService2true')
        for country in countries:
            country.pop('id', None)
            country['countryName'] = country.pop('provinceName', None)
            country['provinceName'] = ''
            country.pop('countryType')
            country.pop('cityName')
            country.pop('provinceId')
            country.pop('provinceName')
            country.pop('provinceShortName')
            country.pop('modifyTime', None)
            country.pop('createTime', None)
            country['incrVo'] = json.dumps(country.get('incrVo', {}))
            statistics_data = country.get('statisticsData')
            if statistics_data:
                yield scrapy.Request(
                    statistics_data,
                    callback=self.parse_country_statistics_data,
                    meta={'country': country})
            else:
                yield items.CountryItem(dailyData=[], **country)

        self.crawled = 1  # 代表爬虫已爬取数据
def gen_date(min_date=today_add_days(-365), max_date=today_add_days(365)):
	min_date = time.mktime(min_date.timetuple())
	max_date = time.mktime(max_date.timetuple())
	for __ in itertools.count():
		random_time = min_date + random.random() * (max_date - min_date)
		yield tz_datetime.fromtimestamp(time.mktime(time.localtime(random_time))).date()
Esempio n. 24
0
def inbound_mail(request):
    """Parse an inbound Mailgun event (https://documentation.mailgun.com/
    quickstart-receiving.html) and save the data as models"""
    if not verify_mailgun_token(request.POST.get('token'),
                                request.POST.get('timestamp'),
                                request.POST.get('signature')):
        return HttpResponseForbidden('Failed Mailgun token validation.')

    required_fields = (
        'sender',
        'To',
        'Date',
    )
    missing_fields = [x for x in required_fields if x not in request.POST]

    if missing_fields:
        return HttpResponseBadRequest('Missing a requied field.')

    # Parse the out sender/recipient fields
    sender_address = parseaddr(request.POST.get('sender'))
    recipient_addresses = getaddresses(request.POST.get('To').split(','))

    if sender_address[1].split('@')[1] not in \
            settings.FOIATRACKER_ALLOWED_DOMAINS:
        msg = '"%s" is not authorized to use FOIAtracker.' % sender_address[1]
        return HttpResponseForbidden(msg)

    # Make a timezone-aware datetime from the Date field
    date_field = request.POST.get('Date')
    parsed_from_email = parsedate_tz(date_field)
    if parsed_from_email is not None:
        parsed_timestamp = mktime_tz(parsed_from_email)
        parsed_datetime = datetime.fromtimestamp(parsed_timestamp)
    else:
        parsed_datetime = datetime.now()
    sent = tz_aware_date(parsed_datetime)

    # Create a model for the email
    email = InboundEmail.objects.create(
        raw=request.POST.get('body-plain', ''),
        text=request.POST.get('stripped-text', ''),
        html=request.POST.get('body-html', ''),
        sent=sent,
        sender=get_model_by_email(Sender, sender_address[1]),
        subject=request.POST.get('subject', ''))

    # Setup M2M relationships for email recipeints
    for address in recipient_addresses:
        # For addresses that aren't e-mails, like 'undisclosed-recipients'
        if '@' not in address[1]:
            continue

        if address[1].split('@')[1] in settings.FOIATRACKER_ALLOWED_DOMAINS:
            continue

        recipient = get_model_by_email(Recipient, address[1])
        email.recipients.add(recipient)

    email.save()

    # Send a message to let the user know we're ready to classify
    tasks.email_prompt(email.pk)

    # Save file attachments to S3 and attach them to the message
    attachment_count = request.POST.get('attachment-count')

    if attachment_count is not None:
        try:
            num_attachments = int(attachment_count)

            for attachment_num in range(1, num_attachments + 1):
                attachment_key = 'attachment-%s' % attachment_num
                attached_file = request.FILES.get(attachment_key)

                if attached_file is None:
                    continue

                EmailAttachment.objects.create(
                    email=email,
                    stored_file=attached_file,
                    content_type=attached_file.content_type,
                    size=attached_file.size)
        except ValueError:
            pass

    return HttpResponse()
Esempio n. 25
0
 def get_sample(self):
     return tz_datetime.fromtimestamp(
         time.mktime(
             time.localtime(self.min_date + random.random() *
                            (self.max_date - self.min_date)))).replace(
                                tzinfo=utc)